Codex Local
bf1233d5df
ci / verify (push) Canceled after 0s
Add customer and operator login, BLE provisioning, device fleet controls, call memory, memoir generation, and refreshed family and maintenance portals. Include API contracts, migrations, firmware sources, tests, assets, and deployment documentation.
316 lines
12 KiB
Go
316 lines
12 KiB
Go
package memoir
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"data-terminal-cloud/internal/callmemory"
|
|
"data-terminal-cloud/internal/contentpolicy"
|
|
"data-terminal-cloud/internal/extraction"
|
|
"data-terminal-cloud/internal/identity"
|
|
"data-terminal-cloud/internal/outbox"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
type JSONGenerator interface {
|
|
GenerateJSON(context.Context, string, string) (json.RawMessage, error)
|
|
}
|
|
type Illustrator interface {
|
|
Generate(context.Context, string) ([]byte, error)
|
|
}
|
|
type Service struct {
|
|
Pool *pgxpool.Pool
|
|
Generator JSONGenerator
|
|
Illustrator Illustrator
|
|
Safety contentpolicy.Classifier
|
|
Model string
|
|
Workflow BookWorkflow
|
|
}
|
|
type Source struct {
|
|
ID string `json:"id"`
|
|
Version int64 `json:"version"`
|
|
Title string `json:"title"`
|
|
Content string `json:"content"`
|
|
EventTime string `json:"event_time"`
|
|
}
|
|
type Chapter struct {
|
|
StoryID string `json:"story_id"`
|
|
Title string `json:"title"`
|
|
EventTime string `json:"event_time"`
|
|
Paragraphs []string `json:"paragraphs"`
|
|
IllustrationPrompt string `json:"illustration_prompt,omitempty"`
|
|
Caption string `json:"caption"`
|
|
ImageID string `json:"image_id,omitempty"`
|
|
}
|
|
type Document struct {
|
|
Title string `json:"title"`
|
|
Subtitle string `json:"subtitle"`
|
|
Introduction string `json:"introduction"`
|
|
Chapters []Chapter `json:"chapters"`
|
|
Sources []Source `json:"sources"`
|
|
IllustrationErrors int `json:"illustration_errors"`
|
|
WorkflowRunID string `json:"workflow_run_id,omitempty"`
|
|
}
|
|
|
|
var ErrConsent = errors.New("MEMOIR_CONSENT_REQUIRED")
|
|
var ErrNoStories = errors.New("MEMOIR_NO_STORIES")
|
|
var ErrStale = errors.New("MEMOIR_SOURCE_CHANGED")
|
|
|
|
func (s *Service) access(ctx context.Context, q interface {
|
|
QueryRow(context.Context, string, ...any) pgx.Row
|
|
}, p identity.Principal, elder string, write bool) (callmemory.Access, error) {
|
|
a, e := callmemory.ResolveAccess(ctx, q, p, elder, write)
|
|
if e != nil {
|
|
return a, e
|
|
}
|
|
var ok bool
|
|
e = q.QueryRow(ctx, `SELECT sp.status='active' AND NOT EXISTS(SELECT 1 FROM privacy_request pr WHERE pr.subject_id=sp.id AND pr.request_type='DELETE')
|
|
AND NOT EXISTS(SELECT 1 FROM (VALUES('transcript_storage'),('story_extraction')) k(kind) WHERE NOT EXISTS(
|
|
SELECT 1 FROM consent_record cr WHERE cr.subject_id=sp.id AND cr.consent_type=k.kind AND cr.withdrawn_at IS NULL AND (cr.expires_at IS NULL OR cr.expires_at>now())))
|
|
FROM subject_profile sp WHERE sp.id=$1::uuid`, a.Subject).Scan(&ok)
|
|
if e == nil && !ok {
|
|
e = ErrConsent
|
|
}
|
|
return a, e
|
|
}
|
|
func (s *Service) sources(ctx context.Context, q interface {
|
|
Query(context.Context, string, ...any) (pgx.Rows, error)
|
|
}, a callmemory.Access) ([]Source, string, error) {
|
|
rows, e := q.Query(ctx, `SELECT sd.public_id::text,sd.version,sd.title,sr.content,sd.event_time FROM story_draft sd JOIN story_revision sr ON sr.id=sd.current_revision_id
|
|
WHERE sd.tenant_id=$1::uuid AND sd.subject_id=$2::uuid AND sd.status IN ('CANDIDATE','IN_REVIEW','CONFIRMED') AND sd.sensitivity='normal' AND call_story_available(sd.id)
|
|
ORDER BY sd.event_time,sd.created_at,sd.id`, a.Tenant, a.Subject)
|
|
if e != nil {
|
|
return nil, "", e
|
|
}
|
|
defer rows.Close()
|
|
v := []Source{}
|
|
for rows.Next() {
|
|
var x Source
|
|
if e = rows.Scan(&x.ID, &x.Version, &x.Title, &x.Content, &x.EventTime); e != nil {
|
|
return nil, "", e
|
|
}
|
|
v = append(v, x)
|
|
}
|
|
sort.SliceStable(v, func(i, j int) bool { return eventOrder(v[i].EventTime) < eventOrder(v[j].EventTime) })
|
|
raw, _ := json.Marshal(v)
|
|
sum := sha256.Sum256(raw)
|
|
return v, hex.EncodeToString(sum[:]), rows.Err()
|
|
}
|
|
|
|
var eventDateParts = regexp.MustCompile(`(\d{4})年(?:\s*(\d{1,2})月)?`)
|
|
|
|
func eventOrder(value string) int {
|
|
m := eventDateParts.FindStringSubmatch(value)
|
|
if len(m) == 0 {
|
|
return 999999
|
|
}
|
|
year, _ := strconv.Atoi(m[1])
|
|
month, _ := strconv.Atoi(m[2])
|
|
if month == 0 {
|
|
for season, n := range map[string]int{"春": 3, "夏": 6, "秋": 9, "冬": 12} {
|
|
if strings.Contains(value, season) {
|
|
month = n
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return year*100 + month
|
|
}
|
|
func (s *Service) Query(ctx context.Context, p identity.Principal, elder string) (map[string]any, error) {
|
|
a, e := s.access(ctx, s.Pool, p, elder, false)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
sources, digest, e := s.sources(ctx, s.Pool, a)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
out := map[string]any{"stories": sources, "can_generate": a.Write, "document": nil}
|
|
var pending bool
|
|
if e = s.Pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM memoir_chat_import WHERE subject_id=$1::uuid AND status='PENDING')`, a.Subject).Scan(&pending); e != nil {
|
|
return nil, e
|
|
}
|
|
out["pending_import"] = pending
|
|
var raw []byte
|
|
var v int64
|
|
var created time.Time
|
|
e = s.Pool.QueryRow(ctx, `SELECT document,version,created_at FROM memoir_document WHERE subject_id=$1::uuid AND tenant_id=$2::uuid AND source_digest=$3 ORDER BY version DESC LIMIT 1`, a.Subject, a.Tenant, digest).Scan(&raw, &v, &created)
|
|
if e != nil && !errors.Is(e, pgx.ErrNoRows) {
|
|
return nil, e
|
|
}
|
|
if e == nil {
|
|
out["document"] = json.RawMessage(raw)
|
|
out["version"] = v
|
|
out["created_at"] = created
|
|
}
|
|
var state, code string
|
|
e = s.Pool.QueryRow(ctx, `SELECT state,error_code FROM memoir_generation_job WHERE subject_id=$1::uuid ORDER BY created_at DESC LIMIT 1`, a.Subject).Scan(&state, &code)
|
|
if e != nil && !errors.Is(e, pgx.ErrNoRows) {
|
|
return nil, e
|
|
}
|
|
out["generation_state"] = state
|
|
out["error_code"] = code
|
|
rows, e := s.Pool.Query(ctx, `SELECT id::text,title,content,event_date,created_at FROM chat_daily_event d WHERE subject_id=$1::uuid AND tenant_id=$2::uuid
|
|
AND (d.call_session_id IS NULL OR call_story_authorized(d.call_session_id))
|
|
AND NOT EXISTS(SELECT 1 FROM jsonb_array_elements(d.source_refs) r WHERE
|
|
(r->>'import_id'<>'' AND NOT EXISTS(SELECT 1 FROM memoir_import_turn it WHERE it.id=(r->>'turn_id')::uuid AND it.content<>'[deleted]')) OR
|
|
(r->>'call_session_id'<>'' AND NOT EXISTS(SELECT 1 FROM call_utterance u WHERE u.session_id=(r->>'call_session_id')::uuid AND u.utterance_id=r->>'turn_id' AND u.source_revision=(r->>'source_revision')::bigint AND u.text_content<>'[deleted]')) OR
|
|
(r->>'conversation_id'<>'' AND NOT EXISTS(SELECT 1 FROM conversation_turn ct WHERE ct.id=(r->>'turn_id')::uuid AND ct.final_text<>'[deleted]')) OR
|
|
EXISTS(SELECT 1 FROM memory_source_suppression ms CROSS JOIN LATERAL jsonb_array_elements(ms.source_refs) suppressed
|
|
WHERE ms.subject_id=d.subject_id AND ((suppressed->>'import_id'=NULLIF(r->>'import_id','') AND (suppressed->>'turn_id' IS NULL OR suppressed->>'turn_id'=r->>'turn_id')) OR
|
|
(suppressed->>'session_id'=NULLIF(r->>'call_session_id','') AND (suppressed->>'utterance_id' IS NULL OR suppressed->>'utterance_id'=r->>'turn_id')))))
|
|
ORDER BY event_date DESC,created_at DESC LIMIT 200`, a.Subject, a.Tenant)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
defer rows.Close()
|
|
daily := []map[string]any{}
|
|
for rows.Next() {
|
|
var id, title, content string
|
|
var date, at time.Time
|
|
if e = rows.Scan(&id, &title, &content, &date, &at); e != nil {
|
|
return nil, e
|
|
}
|
|
daily = append(daily, map[string]any{"id": id, "title": title, "content": content, "record_date": date.Format("2006-01-02"), "created_at": at})
|
|
}
|
|
out["daily_events"] = daily
|
|
return out, rows.Err()
|
|
}
|
|
func (s *Service) Enqueue(ctx context.Context, p identity.Principal, elder string) (string, error) {
|
|
if s.Generator == nil && s.Workflow == nil {
|
|
return "", errors.New("MEMOIR_MODEL_NOT_CONFIGURED")
|
|
}
|
|
tx, e := s.Pool.Begin(ctx)
|
|
if e != nil {
|
|
return "", e
|
|
}
|
|
defer tx.Rollback(context.Background())
|
|
a, e := s.access(ctx, tx, p, elder, true)
|
|
if e != nil {
|
|
return "", e
|
|
}
|
|
sources, digest, e := s.sources(ctx, tx, a)
|
|
if e != nil {
|
|
return "", e
|
|
}
|
|
if len(sources) == 0 {
|
|
return "", ErrNoStories
|
|
}
|
|
var id string
|
|
e = tx.QueryRow(ctx, `INSERT INTO memoir_generation_job(subject_id,tenant_id,actor_id,source_digest) VALUES($1::uuid,$2::uuid,$3::uuid,$4)
|
|
ON CONFLICT(subject_id) WHERE state IN ('PENDING','RUNNING') DO UPDATE SET updated_at=memoir_generation_job.updated_at RETURNING id::text`, a.Subject, a.Tenant, a.User, digest).Scan(&id)
|
|
if e != nil {
|
|
return "", e
|
|
}
|
|
return id, tx.Commit(ctx)
|
|
}
|
|
|
|
type ImportInput struct {
|
|
ID string `json:"import_id"`
|
|
ElderID string `json:"elder_id"`
|
|
Title string `json:"title"`
|
|
Transcript string `json:"transcript"`
|
|
}
|
|
|
|
func (s *Service) Import(ctx context.Context, p identity.Principal, in ImportInput) (string, error) {
|
|
if _, e := uuid.Parse(in.ID); e != nil || len(in.Transcript) > 200000 || len(in.Transcript) < 10 || len([]rune(in.Title)) > 200 {
|
|
return "", errors.New("MEMOIR_IMPORT_INVALID")
|
|
}
|
|
turns, e := ParseTranscript(in.Transcript)
|
|
if e != nil {
|
|
return "", e
|
|
}
|
|
tx, e := s.Pool.Begin(ctx)
|
|
if e != nil {
|
|
return "", e
|
|
}
|
|
defer tx.Rollback(context.Background())
|
|
a, e := s.access(ctx, tx, p, in.ElderID, true)
|
|
if e != nil {
|
|
return "", e
|
|
}
|
|
if !a.Sensitive {
|
|
return "", errors.New("TRANSCRIPT_ACCESS_DENIED")
|
|
}
|
|
if _, e = tx.Exec(ctx, `INSERT INTO memory_user_state(subject_id,tenant_id) VALUES($1::uuid,$2::uuid) ON CONFLICT DO NOTHING`, a.Subject, a.Tenant); e != nil {
|
|
return "", e
|
|
}
|
|
sum := sha256.Sum256([]byte(in.Transcript))
|
|
digest := hex.EncodeToString(sum[:])
|
|
var existingSubject, existingDigest string
|
|
e = tx.QueryRow(ctx, `SELECT subject_id::text,transcript_digest FROM memoir_chat_import WHERE id=$1::uuid`, in.ID).Scan(&existingSubject, &existingDigest)
|
|
if e == nil {
|
|
if existingSubject != a.Subject || existingDigest != digest {
|
|
return "", errors.New("MEMOIR_IMPORT_CONFLICT")
|
|
}
|
|
return in.ID, tx.Commit(ctx)
|
|
}
|
|
if !errors.Is(e, pgx.ErrNoRows) {
|
|
return "", e
|
|
}
|
|
_, e = tx.Exec(ctx, `INSERT INTO memoir_chat_import(id,tenant_id,subject_id,created_by,title,transcript_digest) VALUES($1::uuid,$2::uuid,$3::uuid,$4::uuid,$5,$6)`, in.ID, a.Tenant, a.Subject, a.User, in.Title, digest)
|
|
if e != nil {
|
|
return "", e
|
|
}
|
|
for i, t := range turns {
|
|
if _, e = tx.Exec(ctx, `INSERT INTO memoir_import_turn(import_id,sequence_no,speaker,content) VALUES($1::uuid,$2,$3,$4)`, in.ID, i+1, t.Speaker, t.Content); e != nil {
|
|
return "", e
|
|
}
|
|
}
|
|
key := "memoir-import:" + in.ID
|
|
_, e = outbox.NewStore(s.Pool).Enqueue(ctx, tx, outbox.NewEvent{TenantID: &a.Tenant, AggregateType: "memoir_import", AggregateID: in.ID, EventType: extraction.ImportStoryEvent, DeduplicationKey: &key, Payload: json.RawMessage(`{}`)})
|
|
if e != nil {
|
|
return "", e
|
|
}
|
|
return in.ID, tx.Commit(ctx)
|
|
}
|
|
|
|
type ImportedTurn struct{ Speaker, Content string }
|
|
|
|
func ParseTranscript(raw string) ([]ImportedTurn, error) {
|
|
out := []ImportedTurn{}
|
|
for _, line := range strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
continue
|
|
}
|
|
speaker, body := "", ""
|
|
for prefix, role := range map[string]string{"我：": "subject", "我:": "subject", "长者：": "subject", "用户：": "subject", "豆包：": "agent", "豆包:": "agent", "助手：": "agent"} {
|
|
if strings.HasPrefix(line, prefix) {
|
|
speaker = role
|
|
body = strings.TrimSpace(strings.TrimPrefix(line, prefix))
|
|
break
|
|
}
|
|
}
|
|
if speaker == "" {
|
|
if len(out) == 0 {
|
|
return nil, errors.New("MEMOIR_SPEAKER_REQUIRED")
|
|
}
|
|
out[len(out)-1].Content += "\n" + line
|
|
continue
|
|
}
|
|
if body != "" {
|
|
out = append(out, ImportedTurn{speaker, body})
|
|
}
|
|
}
|
|
count := 0
|
|
for _, t := range out {
|
|
if t.Speaker == "subject" {
|
|
count++
|
|
}
|
|
}
|
|
if count == 0 || len(out) > 1000 {
|
|
return nil, errors.New("MEMOIR_IMPORT_INVALID")
|
|
}
|
|
return out, nil
|
|
}
|