feat: expand companion platform, device controls, and memoir workflows

Add customer and operator login, BLE provisioning, device fleet controls, call memory, memoir generation, and refreshed family and maintenance portals. Include API contracts, migrations, firmware sources, tests, assets, and deployment documentation.
2026-10-08 10:06:44 +08:00
parent 98fb2965d0
commit bf1233d5df
861 changed files with 94614 additions and 3209 deletions
@@ -0,0 +1,354 @@
+package devicefleet
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"io"
+	"log/slog"
+	"net"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/mochi-mqtt/server/v2/packets"
+)
+
+type fakeStore struct{ objects map[string][]byte }
+
+type fakeDownload struct{ *bytes.Reader }
+
+func (fakeDownload) Close() error { return nil }
+
+func (f *fakeStore) PresignUpload(_ context.Context, key string, _ time.Duration) (string, error) {
+	return "https://example.test/" + key, nil
+}
+func (f *fakeStore) PresignMediaDownload(_ context.Context, key string, _ time.Duration, _, _ string, _ bool) (string, error) {
+	return "https://example.test/" + key, nil
+}
+func (f *fakeStore) OpenDownload(_ context.Context, key string) (io.ReadSeekCloser, error) {
+	content, ok := f.objects[key]
+	if !ok {
+		return nil, errors.New("missing object")
+	}
+	return fakeDownload{bytes.NewReader(content)}, nil
+}
+func (f *fakeStore) VerifyDigest(_ context.Context, key, hash string, size int64) error {
+	b := f.objects[key]
+	sum := sha256.Sum256(b)
+	if int64(len(b)) != size || hex.EncodeToString(sum[:]) != hash {
+		return errors.New("bad object")
+	}
+	return nil
+}
+func (f *fakeStore) FreezeFirmware(ctx context.Context, source, target, hash string, size int64) error {
+	f.objects[target] = bytes.Clone(f.objects[source])
+	return f.VerifyDigest(ctx, target, hash, size)
+}
+func TestHeartbeatValidation(t *testing.T) {
+	for _, b := range []Heartbeat{{true, "", 0, true}, {true, "x", -1, true}, {true, strings.Repeat("x", 81), 1, true}} {
+		if ValidateHeartbeat(b) {
+			t.Fatal(b)
+		}
+	}
+	if !ValidateHeartbeat(Heartbeat{true, "1.0.0", 0, true}) || !ValidateHeartbeat(Heartbeat{}) {
+		t.Fatal("valid heartbeat rejected")
+	}
+}
+func TestPasswordDeterministicDeviceScoped(t *testing.T) {
+	key := bytes.Repeat([]byte{7}, 32)
+	a := Password(key, "device-a")
+	if len(a) != 43 || a != Password(key, "device-a") || a == Password(key, "device-b") {
+		t.Fatal("password derivation")
+	}
+}
+
+func TestBrokerRejectsTLSURL(t *testing.T) {
+	s := &Service{BrokerURL: "mqtts://127.0.0.1:8883"}
+	if _, err := s.StartBroker(context.Background(), "127.0.0.1:0", slog.New(slog.NewTextHandler(io.Discard, nil))); err == nil {
+		t.Fatal("TLS broker URL accepted for a plaintext listener")
+	}
+}
+
+func TestFleetMQTTAndOTALifecycle(t *testing.T) {
+	dsn := os.Getenv("TEST_DATABASE_URL")
+	if dsn == "" {
+		t.Skip("isolated integration database required")
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	pool, err := pgxpool.New(ctx, dsn)
+	must(t, err)
+	defer pool.Close()
+	const device = "10000000-0000-0000-0000-000000000502"
+	const other = "10000000-0000-0000-0000-000000000501"
+	const tenant = "10000000-0000-0000-0000-000000000001"
+	const factory = "KP01-020000001234"
+	_, err = pool.Exec(ctx, `UPDATE device SET factory_device_id=$2 WHERE id=$1`, device, factory)
+	must(t, err)
+	store := &fakeStore{objects: map[string][]byte{}}
+	s := &Service{Pool: pool, Store: store, BrokerURL: "mqtt://127.0.0.1:1883", DownloadBaseURL: "https://82.157.201.114"}
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	must(t, err)
+	address := listener.Addr().String()
+	listener.Close()
+	stop, err := s.StartBroker(ctx, address, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	must(t, err)
+	defer stop()
+	config, err := s.Config(ctx, device, factory, bytes.Repeat([]byte{7}, 32))
+	must(t, err)
+	connect := func(password string) net.Conn {
+		t.Helper()
+		c, e := net.Dial("tcp", address)
+		must(t, e)
+		pk := packets.Packet{FixedHeader: packets.FixedHeader{Type: packets.Connect}, ProtocolVersion: 4, Connect: packets.ConnectParams{ProtocolName: []byte("MQTT"), ClientIdentifier: factory, Username: []byte(factory), Password: []byte(password), UsernameFlag: true, PasswordFlag: true, Clean: true, Keepalive: 60}}
+		var b bytes.Buffer
+		must(t, pk.ConnectEncode(&b))
+		_, e = c.Write(b.Bytes())
+		must(t, e)
+		return c
+	}
+	bad := connect("wrong")
+	_, ack := readPacket(t, bad)
+	if ack[len(ack)-1] == 0 {
+		t.Fatal("bad credentials accepted")
+	}
+	bad.Close()
+	c := connect(config["password"].(string))
+	defer c.Close()
+	kind, ack := readPacket(t, c)
+	if kind>>4 != packets.Connack || ack[len(ack)-1] != 0 {
+		t.Fatal("connect", kind, ack)
+	}
+	state, err := s.State(ctx, device)
+	must(t, err)
+	if state.Online {
+		t.Fatal("connection without heartbeat is not online")
+	}
+	subscribe := func(topic string, id uint16) byte {
+		pk := packets.Packet{FixedHeader: packets.FixedHeader{Type: packets.Subscribe, Qos: 1}, ProtocolVersion: 4, PacketID: id, Filters: packets.Subscriptions{{Filter: topic, Qos: 1}}}
+		var b bytes.Buffer
+		must(t, pk.SubscribeEncode(&b))
+		_, e := c.Write(b.Bytes())
+		must(t, e)
+		_, data := readPacket(t, c)
+		return data[len(data)-1]
+	}
+	if subscribe("vl/devices/another/ota", 1) != 0x80 {
+		t.Fatal("cross-device subscription allowed")
+	}
+	if subscribe(OTATopic(factory), 2) != 1 {
+		t.Fatal("own OTA subscription failed")
+	}
+	heartbeat := func(version string, code int64) {
+		payload, _ := json.Marshal(Heartbeat{true, version, code, true})
+		pk := packets.Packet{FixedHeader: packets.FixedHeader{Type: packets.Publish, Qos: 1}, ProtocolVersion: 4, PacketID: 3, TopicName: StatusTopic(factory), Payload: payload}
+		var b bytes.Buffer
+		must(t, pk.PublishEncode(&b))
+		_, e := c.Write(b.Bytes())
+		must(t, e)
+		k, _ := readPacket(t, c)
+		if k>>4 != packets.Puback {
+			t.Fatal("heartbeat not acknowledged")
+		}
+	}
+	heartbeat("factory", 0)
+	state, err = s.State(ctx, device)
+	must(t, err)
+	if !state.Online || !state.OTASupported {
+		t.Fatal("heartbeat not visible", state)
+	}
+	authorize := func(tx pgx.Tx) (string, error) {
+		var id string
+		e := tx.QueryRow(ctx, `SELECT device_id::text FROM device_twin WHERE device_id=$1 FOR UPDATE`, device).Scan(&id)
+		return id, e
+	}
+	sum := sha256.Sum256([]byte("firmware-v1"))
+	input := ReleaseInput{Model: "simulated-terminal", Version: "1.0.0", Notes: "test", Size: 11, SHA256: hex.EncodeToString(sum[:])}
+	r, err := s.Prepare(ctx, tenant, input)
+	must(t, err)
+	if r.Code <= 0 {
+		t.Fatal("missing generated sequence")
+	}
+	if _, e := s.Finalize(ctx, tenant, r.ID); e == nil {
+		t.Fatal("missing file published")
+	}
+	store.objects[r.Object] = []byte("firmware-v1")
+	input.Size = int64(len(store.objects[r.Object])) // correct the expected size by retrying the staged upload
+	r, err = s.Prepare(ctx, tenant, input)
+	must(t, err)
+	store.objects[r.Object] = []byte("firmware-v1")
+	r, err = s.Finalize(ctx, tenant, r.ID)
+	must(t, err)
+	download, err := s.Download(ctx, tenant, r.ID)
+	must(t, err)
+	if len(download.DownloadURL) > 64 || !strings.HasPrefix(download.DownloadURL, "https://82.157.201.114/f/") {
+		t.Fatal("firmware API did not return a compact download URL", download.DownloadURL)
+	}
+	adminToken := strings.TrimPrefix(download.DownloadURL, "https://82.157.201.114/f/")
+	adminGrant, err := s.ResolveDownload(ctx, adminToken)
+	must(t, err)
+	if adminGrant.SHA256 != r.SHA256 || adminGrant.Size != r.Size {
+		t.Fatal("firmware API ticket resolved to the wrong firmware", adminGrant)
+	}
+	if _, e := s.Prepare(ctx, tenant, input); e == nil {
+		t.Fatal("published version overwritten")
+	}
+	state, err = s.State(ctx, device)
+	must(t, err)
+	if !state.UpdateAvailable || state.Latest.Code != r.Code {
+		t.Fatal("latest version missing")
+	}
+	// The older rollout path can have an ASSIGNED device after its command has
+	// expired. It must still exclude a second OTA through the MQTT fleet path.
+	keyID, legacyReleaseID, legacyRolloutID := uuid.NewString(), uuid.NewString(), uuid.NewString()
+	const creator = "10000000-0000-0000-0000-000000000101"
+	const household = "10000000-0000-0000-0000-000000000201"
+	_, err = pool.Exec(ctx, `INSERT INTO firmware_signing_key(tenant_id,key_id,public_key_base64,created_by) VALUES($1,$2,'test',$3)`, tenant, keyID, creator)
+	must(t, err)
+	_, err = pool.Exec(ctx, `INSERT INTO firmware_release(id,tenant_id,household_id,model,version,object_ref,expected_size,expected_digest,signature,signing_key_id,created_by) VALUES($1,$2,$3,'simulated-terminal',$4,$4,11,$5,'test',$6,$7)`, legacyReleaseID, tenant, household, legacyReleaseID, r.SHA256, keyID, creator)
+	must(t, err)
+	_, err = pool.Exec(ctx, `INSERT INTO firmware_rollout(id,tenant_id,household_id,release_id,name,status,target_percentage,created_by) VALUES($1,$2,$3,$4,'mutual exclusion test','RUNNING',100,$5)`, legacyRolloutID, tenant, household, legacyReleaseID, creator)
+	must(t, err)
+	_, err = pool.Exec(ctx, `INSERT INTO firmware_assignment(rollout_id,tenant_id,device_id,status) VALUES($1,$2,$3,'ASSIGNED')`, legacyRolloutID, tenant, device)
+	must(t, err)
+	_, err = s.Start(ctx, uuid.NewString(), r.Code, authorize)
+	var busyError *Error
+	if !errors.As(err, &busyError) || busyError.Code != "DEVICE_BUSY" {
+		t.Fatalf("ASSIGNED rollout allowed a parallel fleet OTA: %v", err)
+	}
+	_, err = pool.Exec(ctx, `DELETE FROM firmware_rollout WHERE id=$1`, legacyRolloutID)
+	must(t, err)
+	_, err = pool.Exec(ctx, `DELETE FROM firmware_release WHERE id=$1`, legacyReleaseID)
+	must(t, err)
+	_, err = pool.Exec(ctx, `DELETE FROM firmware_signing_key WHERE tenant_id=$1 AND key_id=$2`, tenant, keyID)
+	must(t, err)
+	request := uuid.NewString()
+	task, err := s.Start(ctx, request, r.Code, authorize)
+	must(t, err)
+	typ, payload := readPacket(t, c)
+	if typ>>4 != packets.Publish || typ&1 != 0 {
+		t.Fatal("expected non-retained OTA publish")
+	}
+	topicLength := int(payload[0])<<8 | int(payload[1])
+	var command map[string]any
+	must(t, json.Unmarshal(payload[2+topicLength+2:], &command))
+	if command["task_id"] != task.ID || command["sha256"] != r.SHA256 {
+		t.Fatal("bad OTA command")
+	}
+	downloadURL, ok := command["download_url"].(string)
+	if !ok || len(downloadURL) > 64 || !strings.HasPrefix(downloadURL, "https://82.157.201.114/f/") {
+		t.Fatal("OTA command did not contain a compact download URL", downloadURL)
+	}
+	token := strings.TrimPrefix(downloadURL, "https://82.157.201.114/f/")
+	grant, err := s.ResolveDownload(ctx, token)
+	must(t, err)
+	if grant.SHA256 != r.SHA256 || grant.Size != r.Size {
+		t.Fatal("short download ticket resolved to the wrong firmware", grant)
+	}
+	reader, err := s.OpenDownload(ctx, grant)
+	must(t, err)
+	downloaded, err := io.ReadAll(reader)
+	must(t, err)
+	must(t, reader.Close())
+	if !bytes.Equal(downloaded, []byte("firmware-v1")) {
+		t.Fatal("short download returned wrong content")
+	}
+	again, err := s.Start(ctx, request, r.Code, authorize)
+	must(t, err)
+	if again.ID != task.ID {
+		t.Fatal("retry duplicated task")
+	}
+	if _, e := s.Start(ctx, uuid.NewString(), r.Code, authorize); e == nil {
+		t.Fatal("parallel OTA allowed")
+	}
+	if e := s.Report(ctx, other, Report{TaskID: task.ID, Status: "success", FirmwareVersion: r.Version, Code: r.Code}); e == nil {
+		t.Fatal("another device completed task")
+	}
+	if e := s.Report(ctx, device, Report{TaskID: task.ID, Status: "success", FirmwareVersion: "wrong", Code: r.Code}); e == nil {
+		t.Fatal("wrong firmware accepted")
+	}
+	manifest, err := s.Manifest(ctx, device, task.ID)
+	must(t, err)
+	if manifest["sha256"] != r.SHA256 || manifest["version_code"] != r.Code || manifest["model"] != r.Model {
+		t.Fatal("manifest differs from immutable release", manifest)
+	}
+	manifestToken := strings.TrimPrefix(manifest["download_url"].(string), "https://82.157.201.114/f/")
+	manifestGrant, err := s.ResolveDownload(ctx, manifestToken)
+	must(t, err)
+	if manifestGrant.SHA256 != r.SHA256 {
+		t.Fatal("HTTPS manifest ticket returned different bytes")
+	}
+	if _, err = s.Manifest(ctx, other, task.ID); err == nil {
+		t.Fatal("foreign device obtained OTA manifest")
+	}
+	must(t, s.Report(ctx, device, Report{TaskID: task.ID, Status: "accepted"}))
+	must(t, s.Report(ctx, device, Report{TaskID: task.ID, Status: "success", FirmwareVersion: r.Version, Code: r.Code}))
+	if _, err = s.Manifest(ctx, device, task.ID); err == nil {
+		t.Fatal("completed task gained a new download grant")
+	}
+	must(t, s.Report(ctx, device, Report{TaskID: task.ID, Status: "success", FirmwareVersion: r.Version, Code: r.Code}))
+	if _, err = s.ResolveDownload(ctx, token); !errors.Is(err, ErrDownloadTicketUnavailable) {
+		t.Fatal("completed OTA download ticket remained usable", err)
+	}
+	state, err = s.State(ctx, device)
+	must(t, err)
+	if state.UpdateAvailable || state.CurrentCode != r.Code || state.Task.Status != "success" {
+		t.Fatal("success not reflected", state)
+	}
+	_, err = pool.Exec(ctx, `UPDATE fleet_presence SET last_seen_at=now()-interval '91 seconds' WHERE device_id=$1`, device)
+	must(t, err)
+	state, err = s.State(ctx, device)
+	must(t, err)
+	if state.Online {
+		t.Fatal("stale heartbeat is online")
+	}
+	if _, e := s.Start(ctx, uuid.NewString(), r.Code, authorize); e == nil {
+		t.Fatal("offline upgrade allowed")
+	}
+	_, err = pool.Exec(ctx, `INSERT INTO fleet_ota(device_id,release_id,request_id,created_at) VALUES($1,$2,$3,now()-interval '31 seconds')`, device, r.ID, uuid.NewString())
+	must(t, err)
+	must(t, s.Expire(ctx))
+	state, err = s.State(ctx, device)
+	must(t, err)
+	var pending int
+	must(t, pool.QueryRow(ctx, `SELECT count(*) FROM fleet_ota WHERE device_id=$1 AND status='pending'`, device).Scan(&pending))
+	if pending != 0 {
+		t.Fatal("pending task not expired")
+	}
+}
+func must(t *testing.T, err error) {
+	t.Helper()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+func readPacket(t *testing.T, c net.Conn) (byte, []byte) {
+	t.Helper()
+	must(t, c.SetReadDeadline(time.Now().Add(5*time.Second)))
+	var first [1]byte
+	_, e := io.ReadFull(c, first[:])
+	must(t, e)
+	size, mul := 0, 1
+	for i := 0; i < 4; i++ {
+		var b [1]byte
+		_, e = io.ReadFull(c, b[:])
+		must(t, e)
+		size += int(b[0]&127) * mul
+		if b[0]&128 == 0 {
+			break
+		}
+		mul *= 128
+	}
+	body := make([]byte, size)
+	_, e = io.ReadFull(c, body)
+	must(t, e)
+	return first[0], body
+}