feat: establish v0.70 platform milestone

internal/memorytool/service.go

@@ -0,0 +1,161 @@
+package memorytool
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"data-terminal-cloud/internal/audit"
+	"data-terminal-cloud/internal/embeddingprovider"
+	"data-terminal-cloud/internal/retrieval"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+var (
+	ErrConversationUnavailable = errors.New("memory tool conversation is unavailable")
+	ErrEngineModeUnsupported   = errors.New("memory tool is unavailable for this engine mode")
+)
+
+type Result struct {
+	MemoryID        string          `json:"memory_id"`
+	MemoryVersionID string          `json:"memory_version_id"`
+	VersionNo       int64           `json:"version_no"`
+	Content         string          `json:"content"`
+	SourceRefs      json.RawMessage `json:"source_refs"`
+	Relevance       float64         `json:"relevance"`
+}
+
+type Response struct {
+	Items    []Result `json:"items"`
+	Degraded bool     `json:"degraded"`
+}
+
+type Service struct {
+	pool     *pgxpool.Pool
+	embedder embeddingprovider.Provider
+	audit    *audit.Repository
+	now      func() time.Time
+}
+
+type conversationScope struct {
+	internalID, publicID, tenantID, subjectID, taskRef, engineMode, status string
+}
+
+func NewService(pool *pgxpool.Pool, embedder embeddingprovider.Provider, auditRepository *audit.Repository) (*Service, error) {
+	if pool == nil || embedder == nil || auditRepository == nil {
+		return nil, errors.New("memory tool database, embedding provider and audit repository are required")
+	}
+	return &Service{pool: pool, embedder: embedder, audit: auditRepository, now: time.Now}, nil
+}
+
+func (s *Service) Search(ctx context.Context, claims Claims, query string, limit int, requestID string) (Response, error) {
+	started := s.now().UTC()
+	query = strings.TrimSpace(query)
+	if query == "" || utf8.RuneCountInString(query) > 500 || limit < 1 || limit > 5 {
+		return Response{}, errors.New("memory query and limit are invalid")
+	}
+	scope, err := s.resolveScope(ctx, claims)
+	if err != nil {
+		return Response{}, err
+	}
+	digest := sha256.Sum256([]byte(query))
+	queryDigest := hex.EncodeToString(digest[:])
+	embedding, err := s.embedder.Embed(ctx, query)
+	if err != nil {
+		s.recordCall(ctx, scope, queryDigest, 0, "FAILED", "EMBEDDING_UNAVAILABLE", started)
+		return Response{Items: []Result{}, Degraded: true}, nil
+	}
+	vector, err := embeddingprovider.Literal(embedding.Vector)
+	if err != nil {
+		s.recordCall(ctx, scope, queryDigest, 0, "FAILED", "EMBEDDING_INVALID", started)
+		return Response{Items: []Result{}, Degraded: true}, nil
+	}
+	tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{AccessMode: pgx.ReadWrite})
+	if err != nil {
+		return Response{}, err
+	}
+	defer tx.Rollback(context.Background())
+	memories, err := retrieval.SelectMemoryForContextVector(ctx, tx, scope.tenantID, scope.subjectID, vector, limit)
+	if err != nil {
+		return Response{}, err
+	}
+	items := make([]Result, 0, len(memories))
+	totalRunes := 0
+	for _, memory := range memories {
+		content := trimRunes(memory.Content, 240)
+		if totalRunes+utf8.RuneCountInString(content) > 800 {
+			break
+		}
+		totalRunes += utf8.RuneCountInString(content)
+		items = append(items, Result{MemoryID: memory.MemoryID, MemoryVersionID: memory.MemoryVersionID, VersionNo: memory.VersionNo, Content: content, SourceRefs: memory.SourceRefs, Relevance: memory.Relevance})
+		if _, err := tx.Exec(ctx, `INSERT INTO conversation_memory_use(tenant_id,subject_id,conversation_id,memory_id,memory_version_id,use_type,relevance,source_refs)
+			SELECT $1::uuid,$2::uuid,$3::uuid,cm.id,mv.id,'in_call_tool',$6,$7::jsonb FROM curated_memory cm JOIN memory_version mv ON mv.memory_id=cm.id
+			WHERE cm.public_id=$4::uuid AND mv.public_id=$5::uuid
+			ON CONFLICT(conversation_id,memory_version_id,use_type) DO UPDATE SET relevance=GREATEST(conversation_memory_use.relevance,EXCLUDED.relevance),used_at=now()`,
+			scope.tenantID, scope.subjectID, scope.internalID, memory.MemoryID, memory.MemoryVersionID, memory.Relevance, memory.SourceRefs); err != nil {
+			return Response{}, err
+		}
+	}
+	status := "SUCCEEDED"
+	if len(items) == 0 {
+		status = "EMPTY"
+	}
+	latency := int(s.now().UTC().Sub(started).Milliseconds())
+	if latency < 0 {
+		latency = 0
+	}
+	if _, err := tx.Exec(ctx, `INSERT INTO memory_tool_call(tenant_id,subject_id,conversation_id,query_digest,result_count,status,latency_ms,engine_mode) VALUES($1::uuid,$2::uuid,$3::uuid,$4,$5,$6,$7,$8)`, scope.tenantID, scope.subjectID, scope.internalID, queryDigest, len(items), status, latency, scope.engineMode); err != nil {
+		return Response{}, err
+	}
+	if err := s.audit.Record(ctx, tx, audit.Event{TenantID: &scope.tenantID, ActorType: "service", Action: "memory.tool.search", ResourceType: "conversation", ResourceID: &scope.publicID, RequestID: requestID, Result: "success", Detail: map[string]any{"result_count": len(items), "engine_mode": scope.engineMode}}); err != nil {
+		return Response{}, err
+	}
+	if err := tx.Commit(ctx); err != nil {
+		return Response{}, err
+	}
+	return Response{Items: items, Degraded: false}, nil
+}
+
+func (s *Service) resolveScope(ctx context.Context, claims Claims) (conversationScope, error) {
+	var scope conversationScope
+	err := s.pool.QueryRow(ctx, `SELECT c.id::text,c.public_id::text,c.tenant_id::text,c.subject_id::text,rt.provider_task_ref,c.engine_mode,c.status
+		FROM conversation c JOIN rtc_task rt ON rt.conversation_id=c.id WHERE c.public_id=$1::uuid`, claims.ConversationID).
+		Scan(&scope.internalID, &scope.publicID, &scope.tenantID, &scope.subjectID, &scope.taskRef, &scope.engineMode, &scope.status)
+	if errors.Is(err, pgx.ErrNoRows) || scope.taskRef != claims.TaskRef {
+		return conversationScope{}, ErrConversationUnavailable
+	}
+	if err != nil {
+		return conversationScope{}, fmt.Errorf("resolve memory tool conversation: %w", err)
+	}
+	if scope.engineMode != "s2s_hybrid" {
+		return conversationScope{}, ErrEngineModeUnsupported
+	}
+	if scope.status != "ACTIVE" && scope.status != "STARTING_AGENT" {
+		return conversationScope{}, ErrConversationUnavailable
+	}
+	return scope, nil
+}
+
+func (s *Service) recordCall(ctx context.Context, scope conversationScope, queryDigest string, resultCount int, status, errorCode string, started time.Time) {
+	latency := int(s.now().UTC().Sub(started).Milliseconds())
+	if latency < 0 {
+		latency = 0
+	}
+	_, _ = s.pool.Exec(ctx, `INSERT INTO memory_tool_call(tenant_id,subject_id,conversation_id,query_digest,result_count,status,error_code,latency_ms,engine_mode) VALUES($1::uuid,$2::uuid,$3::uuid,$4,$5,$6,$7,$8,$9)`, scope.tenantID, scope.subjectID, scope.internalID, queryDigest, resultCount, status, errorCode, latency, scope.engineMode)
+}
+
+func trimRunes(value string, limit int) string {
+	runes := []rune(strings.TrimSpace(value))
+	if len(runes) <= limit {
+		return string(runes)
+	}
+	return string(runes[:limit]) + "…"
+}

internal/memorytool/service_integration_test.go

@@ -0,0 +1,84 @@
+package memorytool
+
+import (
+	"context"
+	"errors"
+	"os"
+	"strings"
+	"testing"
+
+	"data-terminal-cloud/internal/audit"
+	"data-terminal-cloud/internal/developmentembedding"
+	"data-terminal-cloud/internal/embeddingprovider"
+
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+func TestSearchDerivesScopeAndRecordsActualMemoryUse(t *testing.T) {
+	databaseURL := os.Getenv("TEST_DATABASE_URL")
+	if databaseURL == "" {
+		t.Skip("TEST_DATABASE_URL is not set")
+	}
+	ctx := context.Background()
+	pool, err := pgxpool.New(ctx, databaseURL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer pool.Close()
+	contextID, conversationID, conversationPublicID := uuid.NewString(), uuid.NewString(), uuid.NewString()
+	taskRef := "memory-tool-task-" + uuid.NewString()
+	memoryID, memoryPublicID, versionID, versionPublicID := uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString()
+	defer func() {
+		_, _ = pool.Exec(ctx, `DELETE FROM audit_event WHERE request_id LIKE 'memory-tool-integration-%'`)
+		_, _ = pool.Exec(ctx, `DELETE FROM memory_tool_call WHERE conversation_id=$1::uuid`, conversationID)
+		_, _ = pool.Exec(ctx, `DELETE FROM conversation_memory_use WHERE conversation_id=$1::uuid`, conversationID)
+		_, _ = pool.Exec(ctx, `DELETE FROM rtc_task WHERE conversation_id=$1::uuid`, conversationID)
+		_, _ = pool.Exec(ctx, `DELETE FROM conversation WHERE id=$1::uuid`, conversationID)
+		_, _ = pool.Exec(ctx, `DELETE FROM context_snapshot WHERE id=$1::uuid`, contextID)
+		_, _ = pool.Exec(ctx, `DELETE FROM memory_index_entry WHERE memory_id=$1::uuid`, memoryID)
+		_, _ = pool.Exec(ctx, `UPDATE curated_memory SET current_version_id=NULL WHERE id=$1::uuid`, memoryID)
+		_, _ = pool.Exec(ctx, `DELETE FROM memory_version WHERE memory_id=$1::uuid`, memoryID)
+		_, _ = pool.Exec(ctx, `DELETE FROM curated_memory WHERE id=$1::uuid`, memoryID)
+	}()
+	if _, err := pool.Exec(ctx, `INSERT INTO context_snapshot(id,tenant_id,subject_id,profile_version,consent_snapshot_id,memory_versions,prompt_bundle_version,engine_mode,content_digest,sanitized_summary)
+		VALUES($1::uuid,'10000000-0000-0000-0000-000000000001','10000000-0000-0000-0000-000000000401',1,'10000000-0000-0000-0000-000000000702','[]','memory-tool-test','s2s_hybrid',$2,'')`, contextID, strings.Repeat("1", 64)); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := pool.Exec(ctx, `INSERT INTO conversation(id,public_id,tenant_id,household_id,subject_id,device_id,consent_snapshot_id,context_snapshot_id,provider_task_ref,engine_mode,retention_mode,status,allocation_expires_at,started_at)
+		VALUES($1::uuid,$2::uuid,'10000000-0000-0000-0000-000000000001','10000000-0000-0000-0000-000000000201','10000000-0000-0000-0000-000000000401','10000000-0000-0000-0000-000000000501','10000000-0000-0000-0000-000000000702',$3::uuid,$4,'s2s_hybrid','transcript_only','ACTIVE',now()+interval '1 hour',now())`, conversationID, conversationPublicID, contextID, taskRef); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := pool.Exec(ctx, `INSERT INTO rtc_task(conversation_id,provider,provider_task_ref,room_id,device_rtc_user_id,agent_rtc_user_id,status) VALUES($1::uuid,'mock',$2,$3,'device-user','agent-user','ACTIVE')`, conversationID, taskRef, "memory-room-"+conversationID); err != nil {
+		t.Fatal(err)
+	}
+	content := "长者曾经在河边小学教书。"
+	if _, err := pool.Exec(ctx, `INSERT INTO curated_memory(id,public_id,tenant_id,subject_id,status,sensitivity,valid_from) VALUES($1::uuid,$2::uuid,'10000000-0000-0000-0000-000000000001','10000000-0000-0000-0000-000000000401','ACTIVE','normal',now())`, memoryID, memoryPublicID); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := pool.Exec(ctx, `INSERT INTO memory_version(id,public_id,memory_id,version_no,content,source_refs,confirmed_by,content_digest) VALUES($1::uuid,$2::uuid,$3::uuid,1,$4,'[{"type":"story","story_id":"synthetic"}]','10000000-0000-0000-0000-000000000101',$5)`, versionID, versionPublicID, memoryID, content, strings.Repeat("2", 64)); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := pool.Exec(ctx, `UPDATE curated_memory SET current_version_id=$1::uuid WHERE id=$2::uuid`, versionID, memoryID); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := pool.Exec(ctx, `INSERT INTO memory_index_entry(tenant_id,subject_id,memory_id,memory_version_id,index_type,embedding,status,provider_ref) VALUES('10000000-0000-0000-0000-000000000001','10000000-0000-0000-0000-000000000401',$1::uuid,$2::uuid,'pgvector',$3::vector,'ACTIVE','integration')`, memoryID, versionID, developmentembedding.Literal(developmentembedding.Vector(content))); err != nil {
+		t.Fatal(err)
+	}
+	service, err := NewService(pool, embeddingprovider.Local{}, audit.NewRepository())
+	if err != nil {
+		t.Fatal(err)
+	}
+	claims := Claims{ConversationID: conversationPublicID, TaskRef: taskRef}
+	result, err := service.Search(ctx, claims, "河边小学", 3, "memory-tool-integration-search")
+	if err != nil || result.Degraded || len(result.Items) != 1 || result.Items[0].MemoryID != memoryPublicID || result.Items[0].MemoryVersionID != versionPublicID {
+		t.Fatalf("result=%+v err=%v", result, err)
+	}
+	var calls, uses int
+	if err := pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM memory_tool_call WHERE conversation_id=$1::uuid AND status='SUCCEEDED'),(SELECT count(*) FROM conversation_memory_use WHERE conversation_id=$1::uuid AND use_type='in_call_tool')`, conversationID).Scan(&calls, &uses); err != nil || calls != 1 || uses != 1 {
+		t.Fatalf("calls=%d uses=%d err=%v", calls, uses, err)
+	}
+	if _, err := service.Search(ctx, Claims{ConversationID: conversationPublicID, TaskRef: "wrong-task"}, "河边小学", 3, "memory-tool-integration-wrong-task"); !errors.Is(err, ErrConversationUnavailable) {
+		t.Fatalf("wrong task error=%v", err)
+	}
+}

internal/memorytool/token.go

@@ -0,0 +1,69 @@
+package memorytool
+
+import (
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"strings"
+	"time"
+)
+
+var ErrInvalidToken = errors.New("invalid memory tool token")
+
+type Claims struct {
+	ConversationID string `json:"conversation_id"`
+	TaskRef        string `json:"task_ref"`
+	ExpiresAt      int64  `json:"expires_at"`
+}
+
+type TokenSigner struct {
+	secret []byte
+	now    func() time.Time
+}
+
+func NewTokenSigner(secret []byte) (*TokenSigner, error) {
+	if len(secret) < 32 {
+		return nil, errors.New("memory tool signing secret must contain at least 32 bytes")
+	}
+	return &TokenSigner{secret: append([]byte(nil), secret...), now: time.Now}, nil
+}
+
+func (s *TokenSigner) Issue(conversationID, taskRef string, expiresAt time.Time) (string, error) {
+	if strings.TrimSpace(conversationID) == "" || strings.TrimSpace(taskRef) == "" || !expiresAt.After(s.now()) {
+		return "", ErrInvalidToken
+	}
+	payload, err := json.Marshal(Claims{ConversationID: conversationID, TaskRef: taskRef, ExpiresAt: expiresAt.UTC().Unix()})
+	if err != nil {
+		return "", err
+	}
+	encoded := base64.RawURLEncoding.EncodeToString(payload)
+	return "v1." + encoded + "." + s.signature(encoded), nil
+}
+
+func (s *TokenSigner) Verify(token string) (Claims, error) {
+	parts := strings.Split(token, ".")
+	if len(parts) != 3 || parts[0] != "v1" {
+		return Claims{}, ErrInvalidToken
+	}
+	expected := s.signature(parts[1])
+	if !hmac.Equal([]byte(expected), []byte(parts[2])) {
+		return Claims{}, ErrInvalidToken
+	}
+	payload, err := base64.RawURLEncoding.DecodeString(parts[1])
+	if err != nil {
+		return Claims{}, ErrInvalidToken
+	}
+	var claims Claims
+	if err := json.Unmarshal(payload, &claims); err != nil || claims.ConversationID == "" || claims.TaskRef == "" || claims.ExpiresAt <= s.now().UTC().Unix() {
+		return Claims{}, ErrInvalidToken
+	}
+	return claims, nil
+}
+
+func (s *TokenSigner) signature(payload string) string {
+	mac := hmac.New(sha256.New, s.secret)
+	_, _ = mac.Write([]byte("memory-tool-token-v1." + payload))
+	return base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
+}

internal/memorytool/token_test.go

@@ -0,0 +1,31 @@
+package memorytool
+
+import (
+	"errors"
+	"testing"
+	"time"
+)
+
+func TestTokenSignerBindsConversationTaskAndExpiry(t *testing.T) {
+	signer, err := NewTokenSigner([]byte("01234567890123456789012345678901"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	now := time.Date(2026, 8, 28, 12, 0, 0, 0, time.UTC)
+	signer.now = func() time.Time { return now }
+	token, err := signer.Issue("conversation-one", "task-one", now.Add(time.Minute))
+	if err != nil {
+		t.Fatal(err)
+	}
+	claims, err := signer.Verify(token)
+	if err != nil || claims.ConversationID != "conversation-one" || claims.TaskRef != "task-one" {
+		t.Fatalf("claims=%+v err=%v", claims, err)
+	}
+	if _, err := signer.Verify(token + "changed"); !errors.Is(err, ErrInvalidToken) {
+		t.Fatalf("tampered token error=%v", err)
+	}
+	signer.now = func() time.Time { return now.Add(2 * time.Minute) }
+	if _, err := signer.Verify(token); !errors.Is(err, ErrInvalidToken) {
+		t.Fatalf("expired token error=%v", err)
+	}
+}