package memoir import ( "context" "crypto/sha256" "data-terminal-cloud/internal/callmemory" "data-terminal-cloud/internal/contentpolicy" "data-terminal-cloud/internal/extraction" "data-terminal-cloud/internal/identity" "data-terminal-cloud/internal/outbox" "encoding/hex" "encoding/json" "errors" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "regexp" "sort" "strconv" "strings" "time" ) type JSONGenerator interface { GenerateJSON(context.Context, string, string) (json.RawMessage, error) } type Illustrator interface { Generate(context.Context, string) ([]byte, error) } type Service struct { Pool *pgxpool.Pool Generator JSONGenerator Illustrator Illustrator Safety contentpolicy.Classifier Model string Workflow BookWorkflow } type Source struct { ID string `json:"id"` Version int64 `json:"version"` Title string `json:"title"` Content string `json:"content"` EventTime string `json:"event_time"` } type Chapter struct { StoryID string `json:"story_id"` Title string `json:"title"` EventTime string `json:"event_time"` Paragraphs []string `json:"paragraphs"` IllustrationPrompt string `json:"illustration_prompt,omitempty"` Caption string `json:"caption"` ImageID string `json:"image_id,omitempty"` } type Document struct { Title string `json:"title"` Subtitle string `json:"subtitle"` Introduction string `json:"introduction"` Chapters []Chapter `json:"chapters"` Sources []Source `json:"sources"` IllustrationErrors int `json:"illustration_errors"` WorkflowRunID string `json:"workflow_run_id,omitempty"` } var ErrConsent = errors.New("MEMOIR_CONSENT_REQUIRED") var ErrNoStories = errors.New("MEMOIR_NO_STORIES") var ErrStale = errors.New("MEMOIR_SOURCE_CHANGED") func (s *Service) access(ctx context.Context, q interface { QueryRow(context.Context, string, ...any) pgx.Row }, p identity.Principal, elder string, write bool) (callmemory.Access, error) { a, e := callmemory.ResolveAccess(ctx, q, p, elder, write) if e != nil { return a, e } var ok bool e = q.QueryRow(ctx, `SELECT sp.status='active' AND NOT EXISTS(SELECT 1 FROM privacy_request pr WHERE pr.subject_id=sp.id AND pr.request_type='DELETE') AND NOT EXISTS(SELECT 1 FROM (VALUES('transcript_storage'),('story_extraction')) k(kind) WHERE NOT EXISTS( SELECT 1 FROM consent_record cr WHERE cr.subject_id=sp.id AND cr.consent_type=k.kind AND cr.withdrawn_at IS NULL AND (cr.expires_at IS NULL OR cr.expires_at>now()))) FROM subject_profile sp WHERE sp.id=$1::uuid`, a.Subject).Scan(&ok) if e == nil && !ok { e = ErrConsent } return a, e } func (s *Service) sources(ctx context.Context, q interface { Query(context.Context, string, ...any) (pgx.Rows, error) }, a callmemory.Access) ([]Source, string, error) { rows, e := q.Query(ctx, `SELECT sd.public_id::text,sd.version,sd.title,sr.content,sd.event_time FROM story_draft sd JOIN story_revision sr ON sr.id=sd.current_revision_id WHERE sd.tenant_id=$1::uuid AND sd.subject_id=$2::uuid AND sd.status IN ('CANDIDATE','IN_REVIEW','CONFIRMED') AND sd.sensitivity='normal' AND call_story_available(sd.id) ORDER BY sd.event_time,sd.created_at,sd.id`, a.Tenant, a.Subject) if e != nil { return nil, "", e } defer rows.Close() v := []Source{} for rows.Next() { var x Source if e = rows.Scan(&x.ID, &x.Version, &x.Title, &x.Content, &x.EventTime); e != nil { return nil, "", e } v = append(v, x) } sort.SliceStable(v, func(i, j int) bool { return eventOrder(v[i].EventTime) < eventOrder(v[j].EventTime) }) raw, _ := json.Marshal(v) sum := sha256.Sum256(raw) return v, hex.EncodeToString(sum[:]), rows.Err() } var eventDateParts = regexp.MustCompile(`(\d{4})年(?:\s*(\d{1,2})月)?`) func eventOrder(value string) int { m := eventDateParts.FindStringSubmatch(value) if len(m) == 0 { return 999999 } year, _ := strconv.Atoi(m[1]) month, _ := strconv.Atoi(m[2]) if month == 0 { for season, n := range map[string]int{"春": 3, "夏": 6, "秋": 9, "冬": 12} { if strings.Contains(value, season) { month = n break } } } return year*100 + month } func (s *Service) Query(ctx context.Context, p identity.Principal, elder string) (map[string]any, error) { a, e := s.access(ctx, s.Pool, p, elder, false) if e != nil { return nil, e } sources, digest, e := s.sources(ctx, s.Pool, a) if e != nil { return nil, e } out := map[string]any{"stories": sources, "can_generate": a.Write, "document": nil} var pending bool if e = s.Pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM memoir_chat_import WHERE subject_id=$1::uuid AND status='PENDING')`, a.Subject).Scan(&pending); e != nil { return nil, e } out["pending_import"] = pending var raw []byte var v int64 var created time.Time e = s.Pool.QueryRow(ctx, `SELECT document,version,created_at FROM memoir_document WHERE subject_id=$1::uuid AND tenant_id=$2::uuid AND source_digest=$3 ORDER BY version DESC LIMIT 1`, a.Subject, a.Tenant, digest).Scan(&raw, &v, &created) if e != nil && !errors.Is(e, pgx.ErrNoRows) { return nil, e } if e == nil { out["document"] = json.RawMessage(raw) out["version"] = v out["created_at"] = created } var state, code string e = s.Pool.QueryRow(ctx, `SELECT state,error_code FROM memoir_generation_job WHERE subject_id=$1::uuid ORDER BY created_at DESC LIMIT 1`, a.Subject).Scan(&state, &code) if e != nil && !errors.Is(e, pgx.ErrNoRows) { return nil, e } out["generation_state"] = state out["error_code"] = code rows, e := s.Pool.Query(ctx, `SELECT id::text,title,content,event_date,created_at FROM chat_daily_event d WHERE subject_id=$1::uuid AND tenant_id=$2::uuid AND (d.call_session_id IS NULL OR call_story_authorized(d.call_session_id)) AND NOT EXISTS(SELECT 1 FROM jsonb_array_elements(d.source_refs) r WHERE (r->>'import_id'<>'' AND NOT EXISTS(SELECT 1 FROM memoir_import_turn it WHERE it.id=(r->>'turn_id')::uuid AND it.content<>'[deleted]')) OR (r->>'call_session_id'<>'' AND NOT EXISTS(SELECT 1 FROM call_utterance u WHERE u.session_id=(r->>'call_session_id')::uuid AND u.utterance_id=r->>'turn_id' AND u.source_revision=(r->>'source_revision')::bigint AND u.text_content<>'[deleted]')) OR (r->>'conversation_id'<>'' AND NOT EXISTS(SELECT 1 FROM conversation_turn ct WHERE ct.id=(r->>'turn_id')::uuid AND ct.final_text<>'[deleted]')) OR EXISTS(SELECT 1 FROM memory_source_suppression ms CROSS JOIN LATERAL jsonb_array_elements(ms.source_refs) suppressed WHERE ms.subject_id=d.subject_id AND ((suppressed->>'import_id'=NULLIF(r->>'import_id','') AND (suppressed->>'turn_id' IS NULL OR suppressed->>'turn_id'=r->>'turn_id')) OR (suppressed->>'session_id'=NULLIF(r->>'call_session_id','') AND (suppressed->>'utterance_id' IS NULL OR suppressed->>'utterance_id'=r->>'turn_id'))))) ORDER BY event_date DESC,created_at DESC LIMIT 200`, a.Subject, a.Tenant) if e != nil { return nil, e } defer rows.Close() daily := []map[string]any{} for rows.Next() { var id, title, content string var date, at time.Time if e = rows.Scan(&id, &title, &content, &date, &at); e != nil { return nil, e } daily = append(daily, map[string]any{"id": id, "title": title, "content": content, "record_date": date.Format("2006-01-02"), "created_at": at}) } out["daily_events"] = daily return out, rows.Err() } func (s *Service) Enqueue(ctx context.Context, p identity.Principal, elder string) (string, error) { if s.Generator == nil && s.Workflow == nil { return "", errors.New("MEMOIR_MODEL_NOT_CONFIGURED") } tx, e := s.Pool.Begin(ctx) if e != nil { return "", e } defer tx.Rollback(context.Background()) a, e := s.access(ctx, tx, p, elder, true) if e != nil { return "", e } sources, digest, e := s.sources(ctx, tx, a) if e != nil { return "", e } if len(sources) == 0 { return "", ErrNoStories } var id string e = tx.QueryRow(ctx, `INSERT INTO memoir_generation_job(subject_id,tenant_id,actor_id,source_digest) VALUES($1::uuid,$2::uuid,$3::uuid,$4) ON CONFLICT(subject_id) WHERE state IN ('PENDING','RUNNING') DO UPDATE SET updated_at=memoir_generation_job.updated_at RETURNING id::text`, a.Subject, a.Tenant, a.User, digest).Scan(&id) if e != nil { return "", e } return id, tx.Commit(ctx) } type ImportInput struct { ID string `json:"import_id"` ElderID string `json:"elder_id"` Title string `json:"title"` Transcript string `json:"transcript"` } func (s *Service) Import(ctx context.Context, p identity.Principal, in ImportInput) (string, error) { if _, e := uuid.Parse(in.ID); e != nil || len(in.Transcript) > 200000 || len(in.Transcript) < 10 || len([]rune(in.Title)) > 200 { return "", errors.New("MEMOIR_IMPORT_INVALID") } turns, e := ParseTranscript(in.Transcript) if e != nil { return "", e } tx, e := s.Pool.Begin(ctx) if e != nil { return "", e } defer tx.Rollback(context.Background()) a, e := s.access(ctx, tx, p, in.ElderID, true) if e != nil { return "", e } if !a.Sensitive { return "", errors.New("TRANSCRIPT_ACCESS_DENIED") } if _, e = tx.Exec(ctx, `INSERT INTO memory_user_state(subject_id,tenant_id) VALUES($1::uuid,$2::uuid) ON CONFLICT DO NOTHING`, a.Subject, a.Tenant); e != nil { return "", e } sum := sha256.Sum256([]byte(in.Transcript)) digest := hex.EncodeToString(sum[:]) var existingSubject, existingDigest string e = tx.QueryRow(ctx, `SELECT subject_id::text,transcript_digest FROM memoir_chat_import WHERE id=$1::uuid`, in.ID).Scan(&existingSubject, &existingDigest) if e == nil { if existingSubject != a.Subject || existingDigest != digest { return "", errors.New("MEMOIR_IMPORT_CONFLICT") } return in.ID, tx.Commit(ctx) } if !errors.Is(e, pgx.ErrNoRows) { return "", e } _, e = tx.Exec(ctx, `INSERT INTO memoir_chat_import(id,tenant_id,subject_id,created_by,title,transcript_digest) VALUES($1::uuid,$2::uuid,$3::uuid,$4::uuid,$5,$6)`, in.ID, a.Tenant, a.Subject, a.User, in.Title, digest) if e != nil { return "", e } for i, t := range turns { if _, e = tx.Exec(ctx, `INSERT INTO memoir_import_turn(import_id,sequence_no,speaker,content) VALUES($1::uuid,$2,$3,$4)`, in.ID, i+1, t.Speaker, t.Content); e != nil { return "", e } } key := "memoir-import:" + in.ID _, e = outbox.NewStore(s.Pool).Enqueue(ctx, tx, outbox.NewEvent{TenantID: &a.Tenant, AggregateType: "memoir_import", AggregateID: in.ID, EventType: extraction.ImportStoryEvent, DeduplicationKey: &key, Payload: json.RawMessage(`{}`)}) if e != nil { return "", e } return in.ID, tx.Commit(ctx) } type ImportedTurn struct{ Speaker, Content string } func ParseTranscript(raw string) ([]ImportedTurn, error) { out := []ImportedTurn{} for _, line := range strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") { line = strings.TrimSpace(line) if line == "" { continue } speaker, body := "", "" for prefix, role := range map[string]string{"我:": "subject", "我:": "subject", "长者:": "subject", "用户:": "subject", "豆包:": "agent", "豆包:": "agent", "助手:": "agent"} { if strings.HasPrefix(line, prefix) { speaker = role body = strings.TrimSpace(strings.TrimPrefix(line, prefix)) break } } if speaker == "" { if len(out) == 0 { return nil, errors.New("MEMOIR_SPEAKER_REQUIRED") } out[len(out)-1].Content += "\n" + line continue } if body != "" { out = append(out, ImportedTurn{speaker, body}) } } count := 0 for _, t := range out { if t.Speaker == "subject" { count++ } } if count == 0 || len(out) > 1000 { return nil, errors.New("MEMOIR_IMPORT_INVALID") } return out, nil }