story/internal/memoir/service.go

package memoir

import (
	"context"
	"crypto/sha256"
	"data-terminal-cloud/internal/callmemory"
	"data-terminal-cloud/internal/contentpolicy"
	"data-terminal-cloud/internal/extraction"
	"data-terminal-cloud/internal/identity"
	"data-terminal-cloud/internal/outbox"
	"encoding/hex"
	"encoding/json"
	"errors"
	"github.com/google/uuid"
	"github.com/jackc/pgx/v5"
	"github.com/jackc/pgx/v5/pgxpool"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

type JSONGenerator interface {
	GenerateJSON(context.Context, string, string) (json.RawMessage, error)
}
type Illustrator interface {
	Generate(context.Context, string) ([]byte, error)
}
type Service struct {
	Pool        *pgxpool.Pool
	Generator   JSONGenerator
	Illustrator Illustrator
	Safety      contentpolicy.Classifier
	Model       string
	Workflow    BookWorkflow
}
type Source struct {
	ID        string `json:"id"`
	Version   int64  `json:"version"`
	Title     string `json:"title"`
	Content   string `json:"content"`
	EventTime string `json:"event_time"`
}
type Chapter struct {
	StoryID            string   `json:"story_id"`
	Title              string   `json:"title"`
	EventTime          string   `json:"event_time"`
	Paragraphs         []string `json:"paragraphs"`
	IllustrationPrompt string   `json:"illustration_prompt,omitempty"`
	Caption            string   `json:"caption"`
	ImageID            string   `json:"image_id,omitempty"`
}
type Document struct {
	Title              string    `json:"title"`
	Subtitle           string    `json:"subtitle"`
	Introduction       string    `json:"introduction"`
	Chapters           []Chapter `json:"chapters"`
	Sources            []Source  `json:"sources"`
	IllustrationErrors int       `json:"illustration_errors"`
	WorkflowRunID      string    `json:"workflow_run_id,omitempty"`
}

var ErrConsent = errors.New("MEMOIR_CONSENT_REQUIRED")
var ErrNoStories = errors.New("MEMOIR_NO_STORIES")
var ErrStale = errors.New("MEMOIR_SOURCE_CHANGED")

func (s *Service) access(ctx context.Context, q interface {
	QueryRow(context.Context, string, ...any) pgx.Row
}, p identity.Principal, elder string, write bool) (callmemory.Access, error) {
	a, e := callmemory.ResolveAccess(ctx, q, p, elder, write)
	if e != nil {
		return a, e
	}
	var ok bool
	e = q.QueryRow(ctx, `SELECT sp.status='active' AND NOT EXISTS(SELECT 1 FROM privacy_request pr WHERE pr.subject_id=sp.id AND pr.request_type='DELETE')
 AND NOT EXISTS(SELECT 1 FROM (VALUES('transcript_storage'),('story_extraction')) k(kind) WHERE NOT EXISTS(
 SELECT 1 FROM consent_record cr WHERE cr.subject_id=sp.id AND cr.consent_type=k.kind AND cr.withdrawn_at IS NULL AND (cr.expires_at IS NULL OR cr.expires_at>now())))
 FROM subject_profile sp WHERE sp.id=$1::uuid`, a.Subject).Scan(&ok)
	if e == nil && !ok {
		e = ErrConsent
	}
	return a, e
}
func (s *Service) sources(ctx context.Context, q interface {
	Query(context.Context, string, ...any) (pgx.Rows, error)
}, a callmemory.Access) ([]Source, string, error) {
	rows, e := q.Query(ctx, `SELECT sd.public_id::text,sd.version,sd.title,sr.content,sd.event_time FROM story_draft sd JOIN story_revision sr ON sr.id=sd.current_revision_id
 WHERE sd.tenant_id=$1::uuid AND sd.subject_id=$2::uuid AND sd.status IN ('CANDIDATE','IN_REVIEW','CONFIRMED') AND sd.sensitivity='normal' AND call_story_available(sd.id)
 ORDER BY sd.event_time,sd.created_at,sd.id`, a.Tenant, a.Subject)
	if e != nil {
		return nil, "", e
	}
	defer rows.Close()
	v := []Source{}
	for rows.Next() {
		var x Source
		if e = rows.Scan(&x.ID, &x.Version, &x.Title, &x.Content, &x.EventTime); e != nil {
			return nil, "", e
		}
		v = append(v, x)
	}
	sort.SliceStable(v, func(i, j int) bool { return eventOrder(v[i].EventTime) < eventOrder(v[j].EventTime) })
	raw, _ := json.Marshal(v)
	sum := sha256.Sum256(raw)
	return v, hex.EncodeToString(sum[:]), rows.Err()
}

var eventDateParts = regexp.MustCompile(`(\d{4})年(?:\s*(\d{1,2})月)?`)

func eventOrder(value string) int {
	m := eventDateParts.FindStringSubmatch(value)
	if len(m) == 0 {
		return 999999
	}
	year, _ := strconv.Atoi(m[1])
	month, _ := strconv.Atoi(m[2])
	if month == 0 {
		for season, n := range map[string]int{"春": 3, "夏": 6, "秋": 9, "冬": 12} {
			if strings.Contains(value, season) {
				month = n
				break
			}
		}
	}
	return year*100 + month
}
func (s *Service) Query(ctx context.Context, p identity.Principal, elder string) (map[string]any, error) {
	a, e := s.access(ctx, s.Pool, p, elder, false)
	if e != nil {
		return nil, e
	}
	sources, digest, e := s.sources(ctx, s.Pool, a)
	if e != nil {
		return nil, e
	}
	out := map[string]any{"stories": sources, "can_generate": a.Write, "document": nil}
	var pending bool
	if e = s.Pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM memoir_chat_import WHERE subject_id=$1::uuid AND status='PENDING')`, a.Subject).Scan(&pending); e != nil {
		return nil, e
	}
	out["pending_import"] = pending
	var raw []byte
	var v int64
	var created time.Time
	e = s.Pool.QueryRow(ctx, `SELECT document,version,created_at FROM memoir_document WHERE subject_id=$1::uuid AND tenant_id=$2::uuid AND source_digest=$3 ORDER BY version DESC LIMIT 1`, a.Subject, a.Tenant, digest).Scan(&raw, &v, &created)
	if e != nil && !errors.Is(e, pgx.ErrNoRows) {
		return nil, e
	}
	if e == nil {
		out["document"] = json.RawMessage(raw)
		out["version"] = v
		out["created_at"] = created
	}
	var state, code string
	e = s.Pool.QueryRow(ctx, `SELECT state,error_code FROM memoir_generation_job WHERE subject_id=$1::uuid ORDER BY created_at DESC LIMIT 1`, a.Subject).Scan(&state, &code)
	if e != nil && !errors.Is(e, pgx.ErrNoRows) {
		return nil, e
	}
	out["generation_state"] = state
	out["error_code"] = code
	rows, e := s.Pool.Query(ctx, `SELECT id::text,title,content,event_date,created_at FROM chat_daily_event d WHERE subject_id=$1::uuid AND tenant_id=$2::uuid
 AND (d.call_session_id IS NULL OR call_story_authorized(d.call_session_id))
 AND NOT EXISTS(SELECT 1 FROM jsonb_array_elements(d.source_refs) r WHERE
  (r->>'import_id'<>'' AND NOT EXISTS(SELECT 1 FROM memoir_import_turn it WHERE it.id=(r->>'turn_id')::uuid AND it.content<>'[deleted]')) OR
  (r->>'call_session_id'<>'' AND NOT EXISTS(SELECT 1 FROM call_utterance u WHERE u.session_id=(r->>'call_session_id')::uuid AND u.utterance_id=r->>'turn_id' AND u.source_revision=(r->>'source_revision')::bigint AND u.text_content<>'[deleted]')) OR
  (r->>'conversation_id'<>'' AND NOT EXISTS(SELECT 1 FROM conversation_turn ct WHERE ct.id=(r->>'turn_id')::uuid AND ct.final_text<>'[deleted]')) OR
  EXISTS(SELECT 1 FROM memory_source_suppression ms CROSS JOIN LATERAL jsonb_array_elements(ms.source_refs) suppressed
   WHERE ms.subject_id=d.subject_id AND ((suppressed->>'import_id'=NULLIF(r->>'import_id','') AND (suppressed->>'turn_id' IS NULL OR suppressed->>'turn_id'=r->>'turn_id')) OR
    (suppressed->>'session_id'=NULLIF(r->>'call_session_id','') AND (suppressed->>'utterance_id' IS NULL OR suppressed->>'utterance_id'=r->>'turn_id')))))
 ORDER BY event_date DESC,created_at DESC LIMIT 200`, a.Subject, a.Tenant)
	if e != nil {
		return nil, e
	}
	defer rows.Close()
	daily := []map[string]any{}
	for rows.Next() {
		var id, title, content string
		var date, at time.Time
		if e = rows.Scan(&id, &title, &content, &date, &at); e != nil {
			return nil, e
		}
		daily = append(daily, map[string]any{"id": id, "title": title, "content": content, "record_date": date.Format("2006-01-02"), "created_at": at})
	}
	out["daily_events"] = daily
	return out, rows.Err()
}
func (s *Service) Enqueue(ctx context.Context, p identity.Principal, elder string) (string, error) {
	if s.Generator == nil && s.Workflow == nil {
		return "", errors.New("MEMOIR_MODEL_NOT_CONFIGURED")
	}
	tx, e := s.Pool.Begin(ctx)
	if e != nil {
		return "", e
	}
	defer tx.Rollback(context.Background())
	a, e := s.access(ctx, tx, p, elder, true)
	if e != nil {
		return "", e
	}
	sources, digest, e := s.sources(ctx, tx, a)
	if e != nil {
		return "", e
	}
	if len(sources) == 0 {
		return "", ErrNoStories
	}
	var id string
	e = tx.QueryRow(ctx, `INSERT INTO memoir_generation_job(subject_id,tenant_id,actor_id,source_digest) VALUES($1::uuid,$2::uuid,$3::uuid,$4)
 ON CONFLICT(subject_id) WHERE state IN ('PENDING','RUNNING') DO UPDATE SET updated_at=memoir_generation_job.updated_at RETURNING id::text`, a.Subject, a.Tenant, a.User, digest).Scan(&id)
	if e != nil {
		return "", e
	}
	return id, tx.Commit(ctx)
}

type ImportInput struct {
	ID         string `json:"import_id"`
	ElderID    string `json:"elder_id"`
	Title      string `json:"title"`
	Transcript string `json:"transcript"`
}

func (s *Service) Import(ctx context.Context, p identity.Principal, in ImportInput) (string, error) {
	if _, e := uuid.Parse(in.ID); e != nil || len(in.Transcript) > 200000 || len(in.Transcript) < 10 || len([]rune(in.Title)) > 200 {
		return "", errors.New("MEMOIR_IMPORT_INVALID")
	}
	turns, e := ParseTranscript(in.Transcript)
	if e != nil {
		return "", e
	}
	tx, e := s.Pool.Begin(ctx)
	if e != nil {
		return "", e
	}
	defer tx.Rollback(context.Background())
	a, e := s.access(ctx, tx, p, in.ElderID, true)
	if e != nil {
		return "", e
	}
	if !a.Sensitive {
		return "", errors.New("TRANSCRIPT_ACCESS_DENIED")
	}
	if _, e = tx.Exec(ctx, `INSERT INTO memory_user_state(subject_id,tenant_id) VALUES($1::uuid,$2::uuid) ON CONFLICT DO NOTHING`, a.Subject, a.Tenant); e != nil {
		return "", e
	}
	sum := sha256.Sum256([]byte(in.Transcript))
	digest := hex.EncodeToString(sum[:])
	var existingSubject, existingDigest string
	e = tx.QueryRow(ctx, `SELECT subject_id::text,transcript_digest FROM memoir_chat_import WHERE id=$1::uuid`, in.ID).Scan(&existingSubject, &existingDigest)
	if e == nil {
		if existingSubject != a.Subject || existingDigest != digest {
			return "", errors.New("MEMOIR_IMPORT_CONFLICT")
		}
		return in.ID, tx.Commit(ctx)
	}
	if !errors.Is(e, pgx.ErrNoRows) {
		return "", e
	}
	_, e = tx.Exec(ctx, `INSERT INTO memoir_chat_import(id,tenant_id,subject_id,created_by,title,transcript_digest) VALUES($1::uuid,$2::uuid,$3::uuid,$4::uuid,$5,$6)`, in.ID, a.Tenant, a.Subject, a.User, in.Title, digest)
	if e != nil {
		return "", e
	}
	for i, t := range turns {
		if _, e = tx.Exec(ctx, `INSERT INTO memoir_import_turn(import_id,sequence_no,speaker,content) VALUES($1::uuid,$2,$3,$4)`, in.ID, i+1, t.Speaker, t.Content); e != nil {
			return "", e
		}
	}
	key := "memoir-import:" + in.ID
	_, e = outbox.NewStore(s.Pool).Enqueue(ctx, tx, outbox.NewEvent{TenantID: &a.Tenant, AggregateType: "memoir_import", AggregateID: in.ID, EventType: extraction.ImportStoryEvent, DeduplicationKey: &key, Payload: json.RawMessage(`{}`)})
	if e != nil {
		return "", e
	}
	return in.ID, tx.Commit(ctx)
}

type ImportedTurn struct{ Speaker, Content string }

func ParseTranscript(raw string) ([]ImportedTurn, error) {
	out := []ImportedTurn{}
	for _, line := range strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}
		speaker, body := "", ""
		for prefix, role := range map[string]string{"我：": "subject", "我:": "subject", "长者：": "subject", "用户：": "subject", "豆包：": "agent", "豆包:": "agent", "助手：": "agent"} {
			if strings.HasPrefix(line, prefix) {
				speaker = role
				body = strings.TrimSpace(strings.TrimPrefix(line, prefix))
				break
			}
		}
		if speaker == "" {
			if len(out) == 0 {
				return nil, errors.New("MEMOIR_SPEAKER_REQUIRED")
			}
			out[len(out)-1].Content += "\n" + line
			continue
		}
		if body != "" {
			out = append(out, ImportedTurn{speaker, body})
		}
	}
	count := 0
	for _, t := range out {
		if t.Speaker == "subject" {
			count++
		}
	}
	if count == 0 || len(out) > 1000 {
		return nil, errors.New("MEMOIR_IMPORT_INVALID")
	}
	return out, nil
}