package extraction import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "strings" "time" "unicode" "data-terminal-cloud/internal/contentpolicy" "data-terminal-cloud/internal/outbox" ) type GenerationInput struct { TenantID string Content string } type GenerationResult struct { Meaningful bool `json:"meaningful"` StoryWorthy bool `json:"story_worthy"` DecisionReason string `json:"decision_reason"` Summary string `json:"summary"` OpenTopic string `json:"open_topic"` StoryTitle string `json:"story_title"` StoryContent string `json:"story_content"` FactContent string `json:"fact_content"` Keywords []string `json:"keywords"` Confidence float64 `json:"confidence"` ModelRef string `json:"-"` PromptVersion string `json:"-"` AllowStory bool `json:"-"` } type Generator interface { Generate(context.Context, GenerationInput) (GenerationResult, error) } type DeterministicGenerator struct{} func (DeterministicGenerator) Generate(_ context.Context, input GenerationInput) (GenerationResult, error) { content := strings.TrimSpace(input.Content) if content == "" { return GenerationResult{}, errors.New("extraction content is required") } meaningful, storyWorthy, reason := deterministicContentDecision(content) if !meaningful { return GenerationResult{ Meaningful: false, StoryWorthy: false, DecisionReason: reason, Confidence: 0.9, ModelRef: "local-deterministic-extractor", PromptVersion: "content-value-v1", }, nil } storyTitle, storyContent, factContent := "", "", "" if storyWorthy { storyTitle = "关于“" + truncateRunes(content, 24) + "”的回忆" storyContent = content factContent = content } return GenerationResult{ Meaningful: true, StoryWorthy: storyWorthy, DecisionReason: reason, Summary: "长者在本次通话中谈到:" + content, OpenTopic: "下次可以继续询问长者刚才提到的经历,并让长者自行补充人物、时间和地点。", StoryTitle: storyTitle, StoryContent: storyContent, FactContent: factContent, Keywords: deterministicKeywords(content), Confidence: 0.9, ModelRef: "local-deterministic-extractor", PromptVersion: "content-value-deterministic-v2", AllowStory: storyWorthy, }, nil } func sensitiveGeneration(decision contentpolicy.Decision) GenerationResult { modelRef := strings.TrimSpace(decision.ProviderRef) if modelRef == "" { modelRef = "local-content-safety-policy" } reason := safePolicyToken(decision.Reason) return GenerationResult{ Meaningful: true, StoryWorthy: false, DecisionReason: "safety_policy_blocked_story", Summary: "本次通话包含需要人工谨慎处理的敏感信息,系统未生成普通故事卡片。", OpenTopic: "如确有必要处理本次敏感内容,应由有权限人员在受控流程中单独核对。", Confidence: 1, ModelRef: modelRef, PromptVersion: "summary-sensitive-filter-v1:" + reason, AllowStory: false, } } func safePolicyToken(value string) string { value = strings.ToLower(strings.TrimSpace(value)) if value == "" || len(value) > 64 { return "policy_block" } for _, character := range value { if (character < 'a' || character > 'z') && (character < '0' || character > '9') && character != '_' && character != '-' { return "policy_block" } } return value } type ArkGenerator struct { endpoint string apiKey string model string provider string temperature float64 maxTokens int client *http.Client } func NewArkGenerator(baseURL, apiKey, model string, timeout time.Duration) (*ArkGenerator, error) { return NewCompatibleGenerator("ark", baseURL, apiKey, model, 0.2, 1200, timeout) } func NewCompatibleGenerator(provider, baseURL, apiKey, model string, temperature float64, maxTokens int, timeout time.Duration) (*ArkGenerator, error) { provider = strings.ToLower(strings.TrimSpace(provider)) baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") apiKey = strings.TrimSpace(apiKey) model = strings.TrimSpace(model) parsed, err := url.Parse(baseURL) if err != nil || parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") { return nil, errors.New("Ark base URL must be an absolute HTTP(S) URL") } if provider != "ark" && provider != "openai_compatible" { return nil, errors.New("content model provider must be ark or openai_compatible") } if apiKey == "" || model == "" || timeout <= 0 || temperature < 0 || temperature > 2 || maxTokens < 256 || maxTokens > 8192 { return nil, errors.New("content model credentials and valid generation parameters are required") } return &ArkGenerator{ endpoint: baseURL + "/chat/completions", apiKey: apiKey, model: model, provider: provider, temperature: temperature, maxTokens: maxTokens, client: &http.Client{Timeout: timeout}, }, nil } func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (GenerationResult, error) { content := strings.TrimSpace(input.Content) if content == "" { return GenerationResult{}, errors.New("extraction content is required") } requestBody := map[string]any{ "model": g.model, "messages": []map[string]string{ {"role": "system", "content": contentValuePrompt}, {"role": "user", "content": content}, }, "temperature": g.temperature, "max_tokens": g.maxTokens, "response_format": map[string]string{"type": "json_object"}, } encoded, err := json.Marshal(requestBody) if err != nil { return GenerationResult{}, err } req, err := http.NewRequestWithContext(ctx, http.MethodPost, g.endpoint, bytes.NewReader(encoded)) if err != nil { return GenerationResult{}, fmt.Errorf("build Ark content request: %w", err) } req.Header.Set("Authorization", "Bearer "+g.apiKey) req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", "voice-platform-content-extractor/1.0") resp, err := g.client.Do(req) if err != nil { return GenerationResult{}, outbox.Retryable(fmt.Errorf("call Ark content model: %w", err)) } defer resp.Body.Close() responseBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) if readErr != nil { return GenerationResult{}, outbox.Retryable(fmt.Errorf("read Ark content response: %w", readErr)) } if resp.StatusCode < 200 || resp.StatusCode >= 300 { requestErr := fmt.Errorf("Ark content model returned HTTP %d", resp.StatusCode) if resp.StatusCode == http.StatusRequestTimeout || resp.StatusCode == http.StatusTooEarly || resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 { return GenerationResult{}, outbox.Retryable(requestErr) } return GenerationResult{}, requestErr } var completion struct { Choices []struct { Message struct { Content string `json:"content"` } `json:"message"` } `json:"choices"` } if err = json.Unmarshal(responseBody, &completion); err != nil || len(completion.Choices) == 0 { return GenerationResult{}, outbox.Retryable(errors.New("Ark content response has no usable choice")) } jsonContent := strings.TrimSpace(completion.Choices[0].Message.Content) jsonContent = strings.TrimPrefix(jsonContent, "```json") jsonContent = strings.TrimPrefix(jsonContent, "```") jsonContent = strings.TrimSuffix(jsonContent, "```") var result GenerationResult if err = json.Unmarshal([]byte(strings.TrimSpace(jsonContent)), &result); err != nil { return GenerationResult{}, outbox.Retryable(fmt.Errorf("decode Ark structured content: %w", err)) } result.Summary = strings.TrimSpace(result.Summary) result.DecisionReason = strings.TrimSpace(result.DecisionReason) result.OpenTopic = strings.TrimSpace(result.OpenTopic) result.StoryTitle = strings.TrimSpace(result.StoryTitle) result.StoryContent = strings.TrimSpace(result.StoryContent) result.FactContent = strings.TrimSpace(result.FactContent) result.Keywords = normalizedKeywords(result.Keywords) normalizeDecisionContent(&result) if !validGeneratedContent(result) { return GenerationResult{}, outbox.Retryable(errors.New("Ark structured content failed validation")) } result.ModelRef = g.provider + ":" + g.model result.PromptVersion = "content-value-" + g.provider + "-json-v3" result.AllowStory = result.Meaningful && result.StoryWorthy return result, nil } func normalizeDecisionContent(result *GenerationResult) { if !result.Meaningful { result.StoryWorthy = false result.Summary = "" result.OpenTopic = "" result.StoryTitle = "" result.StoryContent = "" result.FactContent = "" result.Keywords = nil return } if !result.StoryWorthy { result.StoryTitle = "" result.StoryContent = "" result.FactContent = "" } } func validGeneratedContent(result GenerationResult) bool { if result.Confidence < 0 || result.Confidence > 1 || result.DecisionReason == "" || len([]rune(result.DecisionReason)) > 500 { return false } if !result.Meaningful { return !result.StoryWorthy && result.Summary == "" && result.OpenTopic == "" && result.StoryTitle == "" && result.StoryContent == "" && result.FactContent == "" && len(result.Keywords) == 0 } if result.Summary == "" || len([]rune(result.Summary)) > 2000 || result.OpenTopic == "" || len([]rune(result.OpenTopic)) > 1000 || len(result.Keywords) < 1 || len(result.Keywords) > 8 { return false } if !result.StoryWorthy { return result.StoryTitle == "" && result.StoryContent == "" && result.FactContent == "" } return result.StoryTitle != "" && len([]rune(result.StoryTitle)) <= 300 && result.StoryContent != "" && len([]rune(result.StoryContent)) <= 20000 && result.FactContent != "" && len([]rune(result.FactContent)) <= 5000 && result.Confidence >= 0 && result.Confidence <= 1 } const contentValuePrompt = `你是长者聊天内容筛选与整理助手。只依据输入对话判断和整理,不补充未出现的事实,不把推断写成事实。 先做两级判断: 1. meaningful:对话是否包含值得保留的信息。只有问候、寒暄、嗯啊、测试设备、重复句、零散且无法理解的短句,必须判为false并舍弃,不生成摘要。 2. story_worthy:仅在meaningful为true时判断是否值得形成故事。具体的个人经历或回忆,且人物、时间、地点、事件、行动、感受中至少出现两类,或形成了清楚的事件脉络,才判为true。普通偏好、单个事实或简短近况可以总结,但story_worthy应为false。 必须只返回JSON对象,字段为meaningful、story_worthy、decision_reason、summary、keywords、open_topic、story_title、story_content、fact_content、confidence。 - decision_reason用一句简短中文说明判断依据。 - meaningful=false时,除decision_reason和confidence外,其余文本必须为空字符串,keywords必须为空数组,story_worthy必须为false。 - meaningful=true时,summary概括有意义内容,keywords为1至8个简短关键词,open_topic给出一个温和且不重复追问的后续话题。 - story_worthy=false时,story_title、story_content、fact_content必须为空字符串。 - story_worthy=true时,story_title和story_content忠实整理原话,fact_content只写对话中可核对的事实。 - confidence取0到1。` func normalizedKeywords(values []string) []string { result := make([]string, 0, min(len(values), 8)) seen := make(map[string]struct{}, len(values)) for _, value := range values { value = strings.TrimSpace(value) if value == "" || len([]rune(value)) > 40 { continue } if _, exists := seen[value]; exists { continue } seen[value] = struct{}{} result = append(result, value) if len(result) == 8 { break } } return result } func deterministicKeywords(content string) []string { parts := strings.FieldsFunc(content, func(character rune) bool { return unicode.IsSpace(character) || strings.ContainsRune(",。!?;、,.!?;::\"“”‘’()()", character) }) candidates := make([]string, 0, len(parts)) for _, part := range parts { runes := []rune(strings.TrimSpace(part)) if len(runes) < 2 { continue } if len(runes) > 12 { runes = runes[:12] } candidates = append(candidates, string(runes)) } keywords := normalizedKeywords(candidates) if len(keywords) > 0 { return keywords } runes := []rune(content) if len(runes) > 12 { runes = runes[:12] } return []string{string(runes)} } func deterministicContentDecision(content string) (meaningful, storyWorthy bool, reason string) { compact := strings.Map(func(character rune) rune { if unicode.IsSpace(character) || strings.ContainsRune(",。!?;、,.!?;::\"“”‘’()()-", character) { return -1 } return character }, content) for _, label := range []string{"长者", "豆包", "助手", "用户", "agent", "subject"} { compact = strings.ReplaceAll(compact, label, "") } for _, phrase := range []string{"你好", "您好", "在吗", "嗯", "啊", "哦", "好的", "知道了", "没事", "再见", "拜拜", "听得到吗", "能听见吗", "测试"} { compact = strings.ReplaceAll(compact, phrase, "") } if len([]rune(compact)) < 12 { return false, false, "内容仅为寒暄、确认或零散短句" } storySignals := 0 for _, phrase := range []string{"小时候", "年轻时", "那年", "当时", "后来", "以前", "记得", "有一次", "我们", "我和", "去了", "发生", "因为", "结果", "高兴", "难过", "想起"} { if strings.Contains(content, phrase) { storySignals++ } } storyWorthy = len([]rune(compact)) >= 48 && storySignals >= 2 if storyWorthy { return true, true, "包含较完整的个人经历与事件脉络" } return true, false, "包含可保留的信息,但尚不足以形成完整故事" }