235 lines
9.6 KiB
Go
235 lines
9.6 KiB
Go
|
package extraction
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"data-terminal-cloud/internal/outbox"
|
||
|
|
"encoding/json"
|
||
|
|
"errors"
|
||
|
|
"fmt"
|
||
|
|
"github.com/jackc/pgx/v5"
|
||
|
|
"strings"
|
||
|
|
"time"
|
||
|
|
"unicode/utf8"
|
||
|
|
)
|
||
|
|
|
||
|
|
func loadPreviousStories(ctx context.Context, q queryer, tenant, subject string) ([]PreviousStory, error) {
|
||
|
|
rows, e := q.Query(ctx, `SELECT sd.public_id::text,sd.version,sd.title,sr.content,sd.event_time
|
||
|
|
FROM story_draft sd JOIN story_revision sr ON sr.id=sd.current_revision_id
|
||
|
|
WHERE sd.tenant_id=$1::uuid AND sd.subject_id=$2::uuid AND sd.status NOT IN ('REJECTED','SUPERSEDED')
|
||
|
|
AND sd.sensitivity='normal' AND call_story_available(sd.id) ORDER BY sd.created_at,sd.id LIMIT 1001`, tenant, subject)
|
||
|
|
if e != nil {
|
||
|
|
return nil, e
|
||
|
|
}
|
||
|
|
defer rows.Close()
|
||
|
|
v := []PreviousStory{}
|
||
|
|
size := 0
|
||
|
|
for rows.Next() {
|
||
|
|
var s PreviousStory
|
||
|
|
if e = rows.Scan(&s.ID, &s.Version, &s.Title, &s.Content, &s.EventTime); e != nil {
|
||
|
|
return nil, e
|
||
|
|
}
|
||
|
|
size += len(s.Content)
|
||
|
|
v = append(v, s)
|
||
|
|
}
|
||
|
|
if len(v) > 1000 || size > 1500000 {
|
||
|
|
return nil, errors.New("story comparison context capacity exceeded")
|
||
|
|
}
|
||
|
|
return v, rows.Err()
|
||
|
|
}
|
||
|
|
func (p *Processor) generateEventsSafely(ctx context.Context, snapshot extractionSnapshot, date string) (GenerationResult, []PreviousStory, error) {
|
||
|
|
previous, e := loadPreviousStories(ctx, p.pool, snapshot.tenantID, snapshot.subjectID)
|
||
|
|
if e != nil {
|
||
|
|
return GenerationResult{}, nil, e
|
||
|
|
}
|
||
|
|
content := numberedSubjectContent(snapshot.turns)
|
||
|
|
// Apply the same safety policy to both current speech and existing stories
|
||
|
|
// before sending either to the provider.
|
||
|
|
for _, s := range previous {
|
||
|
|
content += "\n" + s.Content
|
||
|
|
}
|
||
|
|
decision, e := p.safety.Classify(ctx, content)
|
||
|
|
if e != nil {
|
||
|
|
return GenerationResult{}, nil, e
|
||
|
|
}
|
||
|
|
if !decision.AllowOrdinaryStory {
|
||
|
|
return sensitiveGeneration(decision), previous, nil
|
||
|
|
}
|
||
|
|
in := GenerationInput{TenantID: snapshot.tenantID, Content: numberedSubjectContent(snapshot.turns), ChatDate: date, PreviousStories: previous}
|
||
|
|
r, e := p.generator.Generate(ctx, in)
|
||
|
|
if e != nil {
|
||
|
|
return r, nil, e
|
||
|
|
}
|
||
|
|
if e = normalizeEvents(&r, in); e != nil {
|
||
|
|
return r, nil, e
|
||
|
|
}
|
||
|
|
decision, e = p.safety.Classify(ctx, generatedSafetyContent(r))
|
||
|
|
if e != nil {
|
||
|
|
return r, nil, e
|
||
|
|
}
|
||
|
|
if !decision.AllowOrdinaryStory {
|
||
|
|
return sensitiveGeneration(decision), previous, nil
|
||
|
|
}
|
||
|
|
return r, previous, nil
|
||
|
|
}
|
||
|
|
func lockStoryContext(ctx context.Context, tx pgx.Tx, snapshot extractionSnapshot, previous []PreviousStory) error {
|
||
|
|
if _, e := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,77))`, snapshot.subjectID); e != nil {
|
||
|
|
return e
|
||
|
|
}
|
||
|
|
current, e := loadPreviousStories(ctx, tx, snapshot.tenantID, snapshot.subjectID)
|
||
|
|
if e != nil {
|
||
|
|
return e
|
||
|
|
}
|
||
|
|
// Compare the whole catalog, including new stories created concurrently.
|
||
|
|
a, _ := json.Marshal(previous)
|
||
|
|
b, _ := json.Marshal(current)
|
||
|
|
if string(a) != string(b) {
|
||
|
|
return outbox.Retryable(ErrConversationContentChanged)
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
func selectEventSources(snapshot extractionSnapshot, ids []string) ([]turn, error) {
|
||
|
|
if len(ids) == 0 {
|
||
|
|
return nil, errors.New("event has no source")
|
||
|
|
}
|
||
|
|
byID := map[string]turn{}
|
||
|
|
for _, t := range subjectTurns(snapshot.turns) {
|
||
|
|
byID[t.publicID] = t
|
||
|
|
}
|
||
|
|
v := []turn{}
|
||
|
|
seen := map[string]bool{}
|
||
|
|
for _, id := range ids {
|
||
|
|
t, ok := byID[id]
|
||
|
|
if !ok {
|
||
|
|
return nil, errors.New("event references unknown elder turn")
|
||
|
|
}
|
||
|
|
if !seen[id] {
|
||
|
|
v = append(v, t)
|
||
|
|
seen[id] = true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return v, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// All changes are revisions of the same event. A supplement never replaces
|
||
|
|
// an earlier human-written paragraph or silently changes confirmed memories.
|
||
|
|
func storeEventStories(ctx context.Context, tx pgx.Tx, conversation, call, importID string, epoch int64, snapshot extractionSnapshot, r GenerationResult, version int64, previous []PreviousStory) error {
|
||
|
|
origin := conversation
|
||
|
|
if call != "" {
|
||
|
|
origin = call
|
||
|
|
}
|
||
|
|
if importID != "" {
|
||
|
|
origin = importID
|
||
|
|
}
|
||
|
|
old := map[string]PreviousStory{}
|
||
|
|
for _, s := range previous {
|
||
|
|
old[s.ID] = s
|
||
|
|
}
|
||
|
|
for index, event := range r.Stories {
|
||
|
|
var done bool
|
||
|
|
if e := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM story_generation_event WHERE origin_id=$1::uuid AND generation_version=$2 AND event_index=$3)`, origin, version, index).Scan(&done); e != nil {
|
||
|
|
return e
|
||
|
|
}
|
||
|
|
if done {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
sources, e := selectEventSources(snapshot, event.SourceIDs)
|
||
|
|
if e != nil {
|
||
|
|
return e
|
||
|
|
}
|
||
|
|
var storyID, revisionID, baseID string
|
||
|
|
revisionNo := int64(1)
|
||
|
|
body := event.Content
|
||
|
|
if event.MatchID != "" {
|
||
|
|
existing, ok := old[event.MatchID]
|
||
|
|
if !ok {
|
||
|
|
return errors.New("unknown matched story")
|
||
|
|
}
|
||
|
|
e = tx.QueryRow(ctx, `SELECT id::text,current_revision_id::text,(SELECT revision_no FROM story_revision WHERE id=current_revision_id)+1 FROM story_draft WHERE public_id=$1::uuid AND tenant_id=$2::uuid AND subject_id=$3::uuid AND version=$4 AND call_story_available(id) FOR UPDATE`, existing.ID, snapshot.tenantID, snapshot.subjectID, existing.Version).Scan(&storyID, &baseID, &revisionNo)
|
||
|
|
if e != nil {
|
||
|
|
return outbox.Retryable(ErrConversationContentChanged)
|
||
|
|
}
|
||
|
|
if len(event.NewDetails) == 0 {
|
||
|
|
if _, e = tx.Exec(ctx, `INSERT INTO story_generation_event VALUES($1::uuid,$2,$3,$4::uuid) ON CONFLICT DO NOTHING`, origin, version, index, storyID); e != nil {
|
||
|
|
return e
|
||
|
|
}
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
body = existing.Content + "\n\n" + strings.Join(event.NewDetails, "\n")
|
||
|
|
} else {
|
||
|
|
e = tx.QueryRow(ctx, `INSERT INTO story_draft(tenant_id,subject_id,conversation_id,call_session_id,call_privacy_epoch,import_id,title,status,confidence,sensitivity,generation_version,event_index,event_time)
|
||
|
|
VALUES($1::uuid,$2::uuid,NULLIF($3,'')::uuid,NULLIF($4,'')::uuid,$5,NULLIF($6,'')::uuid,$7,'CANDIDATE',$8,'normal',$9,$10,$11) RETURNING id::text`, snapshot.tenantID, snapshot.subjectID, conversation, call, optionalEpoch(call, epoch), importID, event.Title, r.Confidence, version, index, event.EventTime).Scan(&storyID)
|
||
|
|
if e != nil {
|
||
|
|
return fmt.Errorf("create event story: %w", e)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if utf8.RuneCountInString(body) > 50000 {
|
||
|
|
return errors.New("merged story too long")
|
||
|
|
}
|
||
|
|
e = tx.QueryRow(ctx, `INSERT INTO story_revision(story_id,revision_no,content,author_type,model_ref,prompt_version,base_revision_id,edit_reason)
|
||
|
|
VALUES($1::uuid,$2,$3,'model',$4,$5,NULLIF($6,'')::uuid,$7) RETURNING id::text`, storyID, revisionNo, body, r.ModelRef, r.PromptVersion, baseID, "自动整理原话；仅补充新增信息").Scan(&revisionID)
|
||
|
|
if e != nil {
|
||
|
|
return e
|
||
|
|
}
|
||
|
|
if baseID != "" {
|
||
|
|
if _, e = tx.Exec(ctx, `INSERT INTO story_source(story_revision_id,conversation_turn_id,call_session_id,call_utterance_id,call_source_revision,import_turn_id,source_start_offset,source_end_offset,source_role)
|
||
|
|
SELECT $1::uuid,conversation_turn_id,call_session_id,call_utterance_id,call_source_revision,import_turn_id,source_start_offset,source_end_offset,source_role FROM story_source WHERE story_revision_id=$2::uuid`, revisionID, baseID); e != nil {
|
||
|
|
return e
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for _, t := range sources {
|
||
|
|
_, e = tx.Exec(ctx, `INSERT INTO story_source(story_revision_id,conversation_turn_id,call_session_id,call_utterance_id,call_source_revision,import_turn_id,source_start_offset,source_end_offset,source_role)
|
||
|
|
VALUES($1::uuid,NULLIF($2,'')::uuid,NULLIF($3,'')::uuid,NULLIF($4,''),$5,NULLIF($6,'')::uuid,0,$7,'subject') ON CONFLICT DO NOTHING`, revisionID, turnForOrigin(t, conversation), call, turnForOrigin(t, call), optionalEpoch(call, t.sourceRevision), turnForOrigin(t, importID), utf8.RuneCountInString(t.text))
|
||
|
|
if e != nil {
|
||
|
|
return e
|
||
|
|
}
|
||
|
|
}
|
||
|
|
_, e = tx.Exec(ctx, `UPDATE story_draft SET current_revision_id=$2::uuid,status=CASE WHEN status IN ('CONFIRMED','IN_REVIEW') THEN 'IN_REVIEW' ELSE 'CANDIDATE' END,version=version+CASE WHEN $3<>'' THEN 1 ELSE 0 END,updated_at=now() WHERE id=$1::uuid`, storyID, revisionID, baseID)
|
||
|
|
if e != nil {
|
||
|
|
return e
|
||
|
|
}
|
||
|
|
refs := []string{}
|
||
|
|
for _, t := range sources {
|
||
|
|
refs = append(refs, t.publicID)
|
||
|
|
}
|
||
|
|
sourceJSON, _ := json.Marshal(refs)
|
||
|
|
_, e = tx.Exec(ctx, `INSERT INTO fact_candidate(tenant_id,subject_id,story_id,fact_type,content,statement_basis,time_range,confidence,sensitivity,source_turn_ids,status,generation_version)
|
||
|
|
VALUES($1::uuid,$2::uuid,$3::uuid,'life_event',$4,'subject_statement','{}',$5,'normal',$6::jsonb,'CANDIDATE',$7)`, snapshot.tenantID, snapshot.subjectID, storyID, event.FactContent, r.Confidence, sourceJSON, version)
|
||
|
|
if e != nil {
|
||
|
|
return e
|
||
|
|
}
|
||
|
|
_, e = tx.Exec(ctx, `INSERT INTO story_generation_event VALUES($1::uuid,$2,$3,$4::uuid)`, origin, version, index, storyID)
|
||
|
|
if e != nil {
|
||
|
|
return e
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for index, event := range r.DailyEvents {
|
||
|
|
sources, e := selectEventSources(snapshot, event.SourceIDs)
|
||
|
|
if e != nil {
|
||
|
|
return e
|
||
|
|
}
|
||
|
|
refs := []map[string]any{}
|
||
|
|
for _, t := range sources {
|
||
|
|
refs = append(refs, map[string]any{"conversation_id": conversation, "call_session_id": call, "import_id": importID, "turn_id": t.id, "public_id": t.publicID, "source_revision": t.sourceRevision})
|
||
|
|
}
|
||
|
|
raw, _ := json.Marshal(refs)
|
||
|
|
_, e = tx.Exec(ctx, `INSERT INTO chat_daily_event(tenant_id,subject_id,conversation_id,call_session_id,import_id,generation_version,event_index,event_date,title,content,source_refs,model_ref)
|
||
|
|
VALUES($1::uuid,$2::uuid,NULLIF($3,'')::uuid,NULLIF($4,'')::uuid,NULLIF($5,'')::uuid,$6,$7,$8,$9,$10,$11::jsonb,$12) ON CONFLICT DO NOTHING`, snapshot.tenantID, snapshot.subjectID, conversation, call, importID, version, index, time.Now().In(time.FixedZone("CST", 8*3600)).Format("2006-01-02"), event.Title, event.Content, raw, r.ModelRef)
|
||
|
|
if e != nil {
|
||
|
|
return e
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
func optionalEpoch(origin string, n int64) any {
|
||
|
|
if origin == "" {
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
return n
|
||
|
|
}
|
||
|
|
func turnForOrigin(t turn, origin string) string {
|
||
|
|
if origin == "" {
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
return t.id
|
||
|
|
}
|