feat: establish v0.70 platform milestone

internal/extraction/generator.go

@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"strings"
 	"time"
+	"unicode"
 
 	"data-terminal-cloud/internal/contentpolicy"
 	"data-terminal-cloud/internal/outbox"
@@ -21,15 +22,16 @@ type GenerationInput struct {
 }
 
 type GenerationResult struct {
-	Summary       string  `json:"summary"`
-	OpenTopic     string  `json:"open_topic"`
-	StoryTitle    string  `json:"story_title"`
-	StoryContent  string  `json:"story_content"`
-	FactContent   string  `json:"fact_content"`
-	Confidence    float64 `json:"confidence"`
-	ModelRef      string  `json:"-"`
-	PromptVersion string  `json:"-"`
-	AllowStory    bool    `json:"-"`
+	Summary       string   `json:"summary"`
+	OpenTopic     string   `json:"open_topic"`
+	StoryTitle    string   `json:"story_title"`
+	StoryContent  string   `json:"story_content"`
+	FactContent   string   `json:"fact_content"`
+	Keywords      []string `json:"keywords"`
+	Confidence    float64  `json:"confidence"`
+	ModelRef      string   `json:"-"`
+	PromptVersion string   `json:"-"`
+	AllowStory    bool     `json:"-"`
 }
 
 type Generator interface {
@@ -49,6 +51,7 @@ func (DeterministicGenerator) Generate(_ context.Context, input GenerationInput)
 		StoryTitle:    "关于“" + truncateRunes(content, 24) + "”的回忆",
 		StoryContent:  content,
 		FactContent:   content,
+		Keywords:      deterministicKeywords(content),
 		Confidence:    0.9,
 		ModelRef:      "local-deterministic-extractor",
 		PromptVersion: "story-v1",
@@ -117,7 +120,7 @@ func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (Gen
 	requestBody := map[string]any{
 		"model": g.model,
 		"messages": []map[string]string{
-			{"role": "system", "content": "你是长者口述内容整理助手。只依据输入字幕生成可人工核对的候选，不补充未出现的事实，不把推断写成事实。必须返回JSON对象，字段为summary、open_topic、story_title、story_content、fact_content、confidence。summary概括本次内容，open_topic给出一个温和的后续话题，story和fact保留原意，confidence取0到1。"},
+			{"role": "system", "content": "你是长者口述内容整理助手。只依据输入字幕生成可人工核对的候选，不补充未出现的事实，不把推断写成事实。必须返回JSON对象，字段为summary、keywords、open_topic、story_title、story_content、fact_content、confidence。summary概括本次内容，keywords是1至8个简短关键词字符串组成的数组，open_topic给出一个温和的后续话题，story和fact保留原意，confidence取0到1。"},
 			{"role": "user", "content": content},
 		},
 		"temperature":     0.2,
@@ -174,6 +177,7 @@ func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (Gen
 	result.StoryTitle = strings.TrimSpace(result.StoryTitle)
 	result.StoryContent = strings.TrimSpace(result.StoryContent)
 	result.FactContent = strings.TrimSpace(result.FactContent)
+	result.Keywords = normalizedKeywords(result.Keywords)
 	if !validGeneratedContent(result) {
 		return GenerationResult{}, outbox.Retryable(errors.New("Ark structured content failed validation"))
 	}
@@ -189,5 +193,52 @@ func validGeneratedContent(result GenerationResult) bool {
 		result.StoryTitle != "" && len([]rune(result.StoryTitle)) <= 300 &&
 		result.StoryContent != "" && len([]rune(result.StoryContent)) <= 20000 &&
 		result.FactContent != "" && len([]rune(result.FactContent)) <= 5000 &&
+		len(result.Keywords) >= 1 && len(result.Keywords) <= 8 &&
 		result.Confidence >= 0 && result.Confidence <= 1
 }
+
+func normalizedKeywords(values []string) []string {
+	result := make([]string, 0, min(len(values), 8))
+	seen := make(map[string]struct{}, len(values))
+	for _, value := range values {
+		value = strings.TrimSpace(value)
+		if value == "" || len([]rune(value)) > 40 {
+			continue
+		}
+		if _, exists := seen[value]; exists {
+			continue
+		}
+		seen[value] = struct{}{}
+		result = append(result, value)
+		if len(result) == 8 {
+			break
+		}
+	}
+	return result
+}
+
+func deterministicKeywords(content string) []string {
+	parts := strings.FieldsFunc(content, func(character rune) bool {
+		return unicode.IsSpace(character) || strings.ContainsRune("，。！？；、,.!?;：:\"“”‘’（）()", character)
+	})
+	candidates := make([]string, 0, len(parts))
+	for _, part := range parts {
+		runes := []rune(strings.TrimSpace(part))
+		if len(runes) < 2 {
+			continue
+		}
+		if len(runes) > 12 {
+			runes = runes[:12]
+		}
+		candidates = append(candidates, string(runes))
+	}
+	keywords := normalizedKeywords(candidates)
+	if len(keywords) > 0 {
+		return keywords
+	}
+	runes := []rune(content)
+	if len(runes) > 12 {
+		runes = runes[:12]
+	}
+	return []string{string(runes)}
+}