feat: establish v0.70 platform milestone

internal/extraction/generator.go

@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"strings"
 	"time"
+	"unicode"
 
 	"data-terminal-cloud/internal/contentpolicy"
 	"data-terminal-cloud/internal/outbox"
@@ -21,15 +22,16 @@ type GenerationInput struct {
 }
 
 type GenerationResult struct {
-	Summary       string  `json:"summary"`
-	OpenTopic     string  `json:"open_topic"`
-	StoryTitle    string  `json:"story_title"`
-	StoryContent  string  `json:"story_content"`
-	FactContent   string  `json:"fact_content"`
-	Confidence    float64 `json:"confidence"`
-	ModelRef      string  `json:"-"`
-	PromptVersion string  `json:"-"`
-	AllowStory    bool    `json:"-"`
+	Summary       string   `json:"summary"`
+	OpenTopic     string   `json:"open_topic"`
+	StoryTitle    string   `json:"story_title"`
+	StoryContent  string   `json:"story_content"`
+	FactContent   string   `json:"fact_content"`
+	Keywords      []string `json:"keywords"`
+	Confidence    float64  `json:"confidence"`
+	ModelRef      string   `json:"-"`
+	PromptVersion string   `json:"-"`
+	AllowStory    bool     `json:"-"`
 }
 
 type Generator interface {
@@ -49,6 +51,7 @@ func (DeterministicGenerator) Generate(_ context.Context, input GenerationInput)
 		StoryTitle:    "关于“" + truncateRunes(content, 24) + "”的回忆",
 		StoryContent:  content,
 		FactContent:   content,
+		Keywords:      deterministicKeywords(content),
 		Confidence:    0.9,
 		ModelRef:      "local-deterministic-extractor",
 		PromptVersion: "story-v1",
@@ -117,7 +120,7 @@ func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (Gen
 	requestBody := map[string]any{
 		"model": g.model,
 		"messages": []map[string]string{
-			{"role": "system", "content": "你是长者口述内容整理助手。只依据输入字幕生成可人工核对的候选，不补充未出现的事实，不把推断写成事实。必须返回JSON对象，字段为summary、open_topic、story_title、story_content、fact_content、confidence。summary概括本次内容，open_topic给出一个温和的后续话题，story和fact保留原意，confidence取0到1。"},
+			{"role": "system", "content": "你是长者口述内容整理助手。只依据输入字幕生成可人工核对的候选，不补充未出现的事实，不把推断写成事实。必须返回JSON对象，字段为summary、keywords、open_topic、story_title、story_content、fact_content、confidence。summary概括本次内容，keywords是1至8个简短关键词字符串组成的数组，open_topic给出一个温和的后续话题，story和fact保留原意，confidence取0到1。"},
 			{"role": "user", "content": content},
 		},
 		"temperature":     0.2,
@@ -174,6 +177,7 @@ func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (Gen
 	result.StoryTitle = strings.TrimSpace(result.StoryTitle)
 	result.StoryContent = strings.TrimSpace(result.StoryContent)
 	result.FactContent = strings.TrimSpace(result.FactContent)
+	result.Keywords = normalizedKeywords(result.Keywords)
 	if !validGeneratedContent(result) {
 		return GenerationResult{}, outbox.Retryable(errors.New("Ark structured content failed validation"))
 	}
@@ -189,5 +193,52 @@ func validGeneratedContent(result GenerationResult) bool {
 		result.StoryTitle != "" && len([]rune(result.StoryTitle)) <= 300 &&
 		result.StoryContent != "" && len([]rune(result.StoryContent)) <= 20000 &&
 		result.FactContent != "" && len([]rune(result.FactContent)) <= 5000 &&
+		len(result.Keywords) >= 1 && len(result.Keywords) <= 8 &&
 		result.Confidence >= 0 && result.Confidence <= 1
 }
+
+func normalizedKeywords(values []string) []string {
+	result := make([]string, 0, min(len(values), 8))
+	seen := make(map[string]struct{}, len(values))
+	for _, value := range values {
+		value = strings.TrimSpace(value)
+		if value == "" || len([]rune(value)) > 40 {
+			continue
+		}
+		if _, exists := seen[value]; exists {
+			continue
+		}
+		seen[value] = struct{}{}
+		result = append(result, value)
+		if len(result) == 8 {
+			break
+		}
+	}
+	return result
+}
+
+func deterministicKeywords(content string) []string {
+	parts := strings.FieldsFunc(content, func(character rune) bool {
+		return unicode.IsSpace(character) || strings.ContainsRune("，。！？；、,.!?;：:\"“”‘’（）()", character)
+	})
+	candidates := make([]string, 0, len(parts))
+	for _, part := range parts {
+		runes := []rune(strings.TrimSpace(part))
+		if len(runes) < 2 {
+			continue
+		}
+		if len(runes) > 12 {
+			runes = runes[:12]
+		}
+		candidates = append(candidates, string(runes))
+	}
+	keywords := normalizedKeywords(candidates)
+	if len(keywords) > 0 {
+		return keywords
+	}
+	runes := []rune(content)
+	if len(runes) > 12 {
+		runes = runes[:12]
+	}
+	return []string{string(runes)}
+}

internal/extraction/generator_test.go

@@ -20,7 +20,7 @@ func TestArkGeneratorProducesValidatedContent(t *testing.T) {
 		if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body["model"] != "doubao-test" {
 			t.Fatalf("request body=%v err=%v", body, err)
 		}
-		_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": `{"summary":"谈到河边小学","open_topic":"下次可以聊聊同学","story_title":"河边小学的回忆","story_content":"小时候在河边小学读书。","fact_content":"曾在河边小学读书","confidence":0.82}`}}}})
+		_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": `{"summary":"谈到河边小学","keywords":["河边小学","童年"],"open_topic":"下次可以聊聊同学","story_title":"河边小学的回忆","story_content":"小时候在河边小学读书。","fact_content":"曾在河边小学读书","confidence":0.82}`}}}})
 	}))
 	defer server.Close()
 	generator, err := NewArkGenerator(server.URL+"/api/v3", "test-key", "doubao-test", time.Second)
@@ -28,7 +28,7 @@ func TestArkGeneratorProducesValidatedContent(t *testing.T) {
 		t.Fatal(err)
 	}
 	result, err := generator.Generate(context.Background(), GenerationInput{Content: "小时候在河边小学读书。"})
-	if err != nil || result.ModelRef != "ark:doubao-test" || result.Confidence != 0.82 || !result.AllowStory {
+	if err != nil || result.ModelRef != "ark:doubao-test" || result.Confidence != 0.82 || !result.AllowStory || len(result.Keywords) != 2 {
 		t.Fatalf("result=%+v err=%v", result, err)
 	}
 }
@@ -47,3 +47,26 @@ func TestArkGeneratorRetriesTemporaryFailure(t *testing.T) {
 		t.Fatalf("429 should be retryable: %v", err)
 	}
 }
+
+func TestDeterministicGeneratorIncludesKeywords(t *testing.T) {
+	result, err := (DeterministicGenerator{}).Generate(context.Background(), GenerationInput{Content: "小时候在河边小学读书，最喜欢和同学一起踢球。"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(result.Keywords) < 1 || len(result.Keywords) > 8 {
+		t.Fatalf("keywords=%v", result.Keywords)
+	}
+	formatted := summaryWithKeywords(result.Summary, result.Keywords)
+	if formatted == result.Summary {
+		t.Fatalf("summary did not include keywords: %q", formatted)
+	}
+}
+
+func TestRequestedGenerationVersion(t *testing.T) {
+	if got := requestedGenerationVersion(json.RawMessage(`{"generation_version":7}`)); got != 7 {
+		t.Fatalf("generation version=%d", got)
+	}
+	if got := requestedGenerationVersion(json.RawMessage(`{}`)); got != 1 {
+		t.Fatalf("default generation version=%d", got)
+	}
+}

internal/extraction/processor.go

@@ -36,6 +36,7 @@ type extractionSnapshot struct {
 	tenantID, subjectID, status string
 	storyExtractionAllowed      bool
 	turns                       []turn
+	maxSequence                 int64
 	digest                      string
 }
 
@@ -84,6 +85,10 @@ func (p *Processor) Handle(ctx context.Context, event outbox.Event) error {
 	if !snapshot.storyExtractionAllowed {
 		return nil
 	}
+	generationVersion := requestedGenerationVersion(event.Payload)
+	if generationVersion < snapshot.maxSequence {
+		return nil
+	}
 	sourceTurns := subjectTurns(snapshot.turns)
 	if len(sourceTurns) == 0 {
 		return outbox.Retryable(ErrConversationContentNotReady)
@@ -94,14 +99,6 @@ func (p *Processor) Handle(ctx context.Context, event outbox.Event) error {
 		return err
 	}
 
-	generationVersion := int64(1)
-	var payload struct {
-		GenerationVersion int64 `json:"generation_version"`
-	}
-	if json.Unmarshal(event.Payload, &payload) == nil && payload.GenerationVersion > 0 {
-		generationVersion = payload.GenerationVersion
-	}
-
 	// The model call above is deliberately outside a database transaction. The
 	// conversation row is locked only for the short revalidation and write phase.
 	tx, err := p.pool.BeginTx(ctx, pgx.TxOptions{})
@@ -151,7 +148,7 @@ func (p *Processor) generateSafely(ctx context.Context, content string) (Generat
 }
 
 func generatedSafetyContent(result GenerationResult) string {
-	return strings.Join([]string{result.Summary, result.OpenTopic, result.StoryTitle, result.StoryContent, result.FactContent}, "\n")
+	return strings.Join([]string{result.Summary, strings.Join(result.Keywords, "、"), result.OpenTopic, result.StoryTitle, result.StoryContent, result.FactContent}, "\n")
 }
 
 func loadExtractionSnapshot(ctx context.Context, q queryer, conversationID string, lock bool) (extractionSnapshot, error) {
@@ -179,7 +176,7 @@ func loadExtractionSnapshot(ctx context.Context, q queryer, conversationID strin
 	}
 	snapshot.storyExtractionAllowed = consents["story_extraction"]
 	rows, err := q.Query(ctx, `
-		SELECT id::text,public_id::text,speaker,final_text,interrupted
+		SELECT id::text,public_id::text,speaker,final_text,interrupted,sequence_no
 		FROM conversation_turn WHERE conversation_id=$1::uuid ORDER BY sequence_no,id`, conversationID)
 	if err != nil {
 		return extractionSnapshot{}, fmt.Errorf("read extraction turns: %w", err)
@@ -187,9 +184,13 @@ func loadExtractionSnapshot(ctx context.Context, q queryer, conversationID strin
 	defer rows.Close()
 	for rows.Next() {
 		var item turn
-		if err = rows.Scan(&item.id, &item.publicID, &item.speaker, &item.text, &item.interrupted); err != nil {
+		var sequence int64
+		if err = rows.Scan(&item.id, &item.publicID, &item.speaker, &item.text, &item.interrupted, &sequence); err != nil {
 			return extractionSnapshot{}, err
 		}
+		if sequence > snapshot.maxSequence {
+			snapshot.maxSequence = sequence
+		}
 		snapshot.turns = append(snapshot.turns, item)
 	}
 	if err = rows.Err(); err != nil {
@@ -199,6 +200,16 @@ func loadExtractionSnapshot(ctx context.Context, q queryer, conversationID strin
 	return snapshot, nil
 }
 
+func requestedGenerationVersion(payload json.RawMessage) int64 {
+	var value struct {
+		GenerationVersion int64 `json:"generation_version"`
+	}
+	if json.Unmarshal(payload, &value) == nil && value.GenerationVersion > 0 {
+		return value.GenerationVersion
+	}
+	return 1
+}
+
 func snapshotDigest(snapshot extractionSnapshot) string {
 	type digestTurn struct {
 		ID          string `json:"id"`
@@ -250,7 +261,7 @@ func storeGeneration(ctx context.Context, tx pgx.Tx, conversationID string, snap
 	if _, err := tx.Exec(ctx, `
 		INSERT INTO conversation_summary (tenant_id,subject_id,conversation_id,summary_type,content,source_turn_ids,model_ref,prompt_version,generation_version,confidence,status)
 		VALUES ($1::uuid,$2::uuid,$3::uuid,'conversation_summary',$4,$5::jsonb,$6,$7,$8,$9,'READY')
-		ON CONFLICT (conversation_id,summary_type,generation_version) DO NOTHING`, snapshot.tenantID, snapshot.subjectID, conversationID, generated.Summary, sourceJSON, generated.ModelRef, generated.PromptVersion, generationVersion, generated.Confidence); err != nil {
+		ON CONFLICT (conversation_id,summary_type,generation_version) DO NOTHING`, snapshot.tenantID, snapshot.subjectID, conversationID, summaryWithKeywords(generated.Summary, generated.Keywords), sourceJSON, generated.ModelRef, generated.PromptVersion, generationVersion, generated.Confidence); err != nil {
 		return fmt.Errorf("store conversation summary: %w", err)
 	}
 	if _, err := tx.Exec(ctx, `
@@ -299,6 +310,14 @@ func storeGeneration(ctx context.Context, tx pgx.Tx, conversationID string, snap
 	return tx.Commit(ctx)
 }
 
+func summaryWithKeywords(summary string, keywords []string) string {
+	keywords = normalizedKeywords(keywords)
+	if len(keywords) == 0 {
+		return summary
+	}
+	return summary + "\n关键词：" + strings.Join(keywords, "、")
+}
+
 func truncateRunes(value string, limit int) string {
 	characters := []rune(value)
 	if len(characters) <= limit {

internal/extraction/processor_integration_test.go

@@ -80,7 +80,7 @@ func TestProcessorExtractionIdempotencyAndSensitiveContent(t *testing.T) {
 		conversationID, "extract-subject-"+conversationID, "我年轻时在河边小学教书。", "extract-agent-"+conversationID, "这是很珍贵的经历。"); err != nil {
 		t.Fatal(err)
 	}
-	event := outbox.Event{AggregateID: conversationID, Payload: []byte(`{"generation_version":1}`)}
+	event := outbox.Event{AggregateID: conversationID, Payload: []byte(`{"generation_version":2}`)}
 	if err := processor.Handle(ctx, event); !errors.Is(err, ErrConversationContentNotReady) || !outbox.IsRetryable(err) {
 		t.Fatalf("active conversation extraction error=%v", err)
 	}
@@ -97,7 +97,7 @@ func TestProcessorExtractionIdempotencyAndSensitiveContent(t *testing.T) {
 			http.Error(w, "database lock", http.StatusServiceUnavailable)
 			return
 		}
-		_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"{\"summary\":\"长者谈到在河边小学教书\",\"open_topic\":\"下次可以聊聊当年的学生\",\"story_title\":\"河边小学的回忆\",\"story_content\":\"我年轻时在河边小学教书。\",\"fact_content\":\"年轻时曾在河边小学教书\",\"confidence\":0.88}"}}]}`)
+		_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"{\"summary\":\"长者谈到在河边小学教书\",\"keywords\":[\"河边小学\",\"教书经历\"],\"open_topic\":\"下次可以聊聊当年的学生\",\"story_title\":\"河边小学的回忆\",\"story_content\":\"我年轻时在河边小学教书。\",\"fact_content\":\"年轻时曾在河边小学教书\",\"confidence\":0.88}"}}]}`)
 	}))
 	defer server.Close()
 	arkGenerator, err := NewArkGenerator(server.URL+"/api/v3", "integration-key", "doubao-integration", 2*time.Second)
@@ -133,7 +133,7 @@ func TestProcessorExtractionIdempotencyAndSensitiveContent(t *testing.T) {
 		conversationID, "extract-sensitive-"+conversationID, "我的银行卡号是6222021234567890123，短信验证码是482931。"); err != nil {
 		t.Fatal(err)
 	}
-	event.Payload = []byte(`{"generation_version":2}`)
+	event.Payload = []byte(`{"generation_version":3}`)
 	if err := modelProcessor.Handle(ctx, event); err != nil {
 		t.Fatalf("extract sensitive conversation: %v", err)
 	}
@@ -141,7 +141,7 @@ func TestProcessorExtractionIdempotencyAndSensitiveContent(t *testing.T) {
 		t.Fatalf("model calls=%d want=2; sensitive content must not leave the local safety boundary", modelCalls.Load())
 	}
 	var sensitivePrompt, sensitiveSummary string
-	if err := pool.QueryRow(ctx, `SELECT prompt_version,content FROM conversation_summary WHERE conversation_id=$1::uuid AND summary_type='conversation_summary' AND generation_version=2`, conversationID).Scan(&sensitivePrompt, &sensitiveSummary); err != nil {
+	if err := pool.QueryRow(ctx, `SELECT prompt_version,content FROM conversation_summary WHERE conversation_id=$1::uuid AND summary_type='conversation_summary' AND generation_version=3`, conversationID).Scan(&sensitivePrompt, &sensitiveSummary); err != nil {
 		t.Fatal(err)
 	}
 	if !strings.HasPrefix(sensitivePrompt, "summary-sensitive-filter-v1:") || strings.Contains(sensitiveSummary, "银行卡") {