395 lines
15 KiB
Go
395 lines
15 KiB
Go
package extraction
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/url"
|
||
"strings"
|
||
"time"
|
||
"unicode"
|
||
|
||
"data-terminal-cloud/internal/contentpolicy"
|
||
"data-terminal-cloud/internal/outbox"
|
||
)
|
||
|
||
type GenerationInput struct {
|
||
TenantID string
|
||
Content string
|
||
}
|
||
|
||
type GenerationResult struct {
|
||
Meaningful bool `json:"meaningful"`
|
||
StoryWorthy bool `json:"story_worthy"`
|
||
DecisionReason string `json:"decision_reason"`
|
||
Summary string `json:"summary"`
|
||
OpenTopic string `json:"open_topic"`
|
||
StoryTitle string `json:"story_title"`
|
||
StoryContent string `json:"story_content"`
|
||
FactContent string `json:"fact_content"`
|
||
Keywords []string `json:"keywords"`
|
||
Confidence float64 `json:"confidence"`
|
||
ModelRef string `json:"-"`
|
||
PromptVersion string `json:"-"`
|
||
AllowStory bool `json:"-"`
|
||
}
|
||
|
||
type Generator interface {
|
||
Generate(context.Context, GenerationInput) (GenerationResult, error)
|
||
}
|
||
|
||
type DeterministicGenerator struct{}
|
||
|
||
func (DeterministicGenerator) Generate(_ context.Context, input GenerationInput) (GenerationResult, error) {
|
||
content := strings.TrimSpace(input.Content)
|
||
if content == "" {
|
||
return GenerationResult{}, errors.New("extraction content is required")
|
||
}
|
||
meaningful, storyWorthy, reason := deterministicContentDecision(content)
|
||
if !meaningful {
|
||
return GenerationResult{
|
||
Meaningful: false, StoryWorthy: false, DecisionReason: reason,
|
||
Confidence: 0.9, ModelRef: "local-deterministic-extractor", PromptVersion: "content-value-v1",
|
||
}, nil
|
||
}
|
||
storyTitle, storyContent, factContent := "", "", ""
|
||
if storyWorthy {
|
||
storyTitle = "关于“" + truncateRunes(content, 24) + "”的回忆"
|
||
storyContent = content
|
||
factContent = content
|
||
}
|
||
return GenerationResult{
|
||
Meaningful: true,
|
||
StoryWorthy: storyWorthy,
|
||
DecisionReason: reason,
|
||
Summary: "长者在本次通话中谈到：" + content,
|
||
OpenTopic: "下次可以继续询问长者刚才提到的经历，并让长者自行补充人物、时间和地点。",
|
||
StoryTitle: storyTitle,
|
||
StoryContent: storyContent,
|
||
FactContent: factContent,
|
||
Keywords: deterministicKeywords(content),
|
||
Confidence: 0.9,
|
||
ModelRef: "local-deterministic-extractor",
|
||
PromptVersion: "content-value-deterministic-v2",
|
||
AllowStory: storyWorthy,
|
||
}, nil
|
||
}
|
||
|
||
func sensitiveGeneration(decision contentpolicy.Decision) GenerationResult {
|
||
modelRef := strings.TrimSpace(decision.ProviderRef)
|
||
if modelRef == "" {
|
||
modelRef = "local-content-safety-policy"
|
||
}
|
||
reason := safePolicyToken(decision.Reason)
|
||
return GenerationResult{
|
||
Meaningful: true,
|
||
StoryWorthy: false,
|
||
DecisionReason: "safety_policy_blocked_story",
|
||
Summary: "本次通话包含需要人工谨慎处理的敏感信息，系统未生成普通故事卡片。",
|
||
OpenTopic: "如确有必要处理本次敏感内容，应由有权限人员在受控流程中单独核对。",
|
||
Confidence: 1,
|
||
ModelRef: modelRef,
|
||
PromptVersion: "summary-sensitive-filter-v1:" + reason,
|
||
AllowStory: false,
|
||
}
|
||
}
|
||
|
||
func safePolicyToken(value string) string {
|
||
value = strings.ToLower(strings.TrimSpace(value))
|
||
if value == "" || len(value) > 64 {
|
||
return "policy_block"
|
||
}
|
||
for _, character := range value {
|
||
if (character < 'a' || character > 'z') && (character < '0' || character > '9') && character != '_' && character != '-' {
|
||
return "policy_block"
|
||
}
|
||
}
|
||
return value
|
||
}
|
||
|
||
type ArkGenerator struct {
|
||
endpoint string
|
||
apiKey string
|
||
model string
|
||
provider string
|
||
temperature float64
|
||
maxTokens int
|
||
client *http.Client
|
||
}
|
||
|
||
func NewArkGenerator(baseURL, apiKey, model string, timeout time.Duration) (*ArkGenerator, error) {
|
||
return NewCompatibleGenerator("ark", baseURL, apiKey, model, 0.2, 1200, timeout)
|
||
}
|
||
|
||
func NewCompatibleGenerator(provider, baseURL, apiKey, model string, temperature float64, maxTokens int, timeout time.Duration) (*ArkGenerator, error) {
|
||
provider = strings.ToLower(strings.TrimSpace(provider))
|
||
baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
|
||
apiKey = strings.TrimSpace(apiKey)
|
||
model = strings.TrimSpace(model)
|
||
parsed, err := url.Parse(baseURL)
|
||
if err != nil || parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") {
|
||
return nil, errors.New("content model base URL must be an absolute HTTP(S) URL")
|
||
}
|
||
if !SupportsCompatibleProvider(provider) {
|
||
return nil, errors.New("unsupported content model provider")
|
||
}
|
||
if apiKey == "" || model == "" || timeout <= 0 || temperature < 0 || temperature > 2 || maxTokens < 256 || maxTokens > 8192 {
|
||
return nil, errors.New("content model credentials and valid generation parameters are required")
|
||
}
|
||
return &ArkGenerator{
|
||
endpoint: baseURL + "/chat/completions", apiKey: apiKey, model: model,
|
||
provider: provider, temperature: temperature, maxTokens: maxTokens,
|
||
client: &http.Client{Timeout: timeout},
|
||
}, nil
|
||
}
|
||
|
||
func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (GenerationResult, error) {
|
||
content := strings.TrimSpace(input.Content)
|
||
if content == "" {
|
||
return GenerationResult{}, errors.New("extraction content is required")
|
||
}
|
||
requestBody := map[string]any{
|
||
"model": g.model,
|
||
"messages": []map[string]string{
|
||
{"role": "system", "content": contentValuePrompt},
|
||
{"role": "user", "content": content},
|
||
},
|
||
"temperature": g.temperature,
|
||
"max_tokens": g.maxTokens,
|
||
}
|
||
// JSON mode is sent only where the provider documents it or this project
|
||
// has exercised it. Other compatible providers still receive an explicit
|
||
// JSON-only prompt, followed by strict response extraction and validation.
|
||
if providerSupportsJSONMode(g.provider) {
|
||
requestBody["response_format"] = map[string]string{"type": "json_object"}
|
||
}
|
||
encoded, err := json.Marshal(requestBody)
|
||
if err != nil {
|
||
return GenerationResult{}, err
|
||
}
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, g.endpoint, bytes.NewReader(encoded))
|
||
if err != nil {
|
||
return GenerationResult{}, fmt.Errorf("build content model request: %w", err)
|
||
}
|
||
req.Header.Set("Authorization", "Bearer "+g.apiKey)
|
||
req.Header.Set("Content-Type", "application/json")
|
||
req.Header.Set("User-Agent", "voice-platform-content-extractor/1.0")
|
||
resp, err := g.client.Do(req)
|
||
if err != nil {
|
||
return GenerationResult{}, outbox.Retryable(fmt.Errorf("call content model: %w", err))
|
||
}
|
||
defer resp.Body.Close()
|
||
responseBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||
if readErr != nil {
|
||
return GenerationResult{}, outbox.Retryable(fmt.Errorf("read content model response: %w", readErr))
|
||
}
|
||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||
requestErr := fmt.Errorf("content model returned HTTP %d", resp.StatusCode)
|
||
if resp.StatusCode == http.StatusRequestTimeout || resp.StatusCode == http.StatusTooEarly || resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 {
|
||
return GenerationResult{}, outbox.Retryable(requestErr)
|
||
}
|
||
return GenerationResult{}, requestErr
|
||
}
|
||
var completion struct {
|
||
Choices []struct {
|
||
Message struct {
|
||
Content string `json:"content"`
|
||
} `json:"message"`
|
||
} `json:"choices"`
|
||
}
|
||
if err = json.Unmarshal(responseBody, &completion); err != nil || len(completion.Choices) == 0 {
|
||
return GenerationResult{}, outbox.Retryable(errors.New("content model response has no usable choice"))
|
||
}
|
||
jsonContent := extractJSONObject(completion.Choices[0].Message.Content)
|
||
jsonContent = strings.TrimPrefix(jsonContent, "```json")
|
||
jsonContent = strings.TrimPrefix(jsonContent, "```")
|
||
jsonContent = strings.TrimSuffix(jsonContent, "```")
|
||
var result GenerationResult
|
||
if err = json.Unmarshal([]byte(strings.TrimSpace(jsonContent)), &result); err != nil {
|
||
return GenerationResult{}, outbox.Retryable(fmt.Errorf("decode structured content: %w", err))
|
||
}
|
||
result.Summary = strings.TrimSpace(result.Summary)
|
||
result.DecisionReason = strings.TrimSpace(result.DecisionReason)
|
||
result.OpenTopic = strings.TrimSpace(result.OpenTopic)
|
||
result.StoryTitle = strings.TrimSpace(result.StoryTitle)
|
||
result.StoryContent = strings.TrimSpace(result.StoryContent)
|
||
result.FactContent = strings.TrimSpace(result.FactContent)
|
||
result.Keywords = normalizedKeywords(result.Keywords)
|
||
normalizeDecisionContent(&result)
|
||
if !validGeneratedContent(result) {
|
||
return GenerationResult{}, outbox.Retryable(errors.New("structured content failed validation"))
|
||
}
|
||
result.ModelRef = g.provider + ":" + g.model
|
||
result.PromptVersion = "content-value-" + g.provider + "-json-v3"
|
||
result.AllowStory = result.Meaningful && result.StoryWorthy
|
||
return result, nil
|
||
}
|
||
|
||
// SupportsCompatibleProvider is the single backend allowlist used both when a
|
||
// model configuration is saved and when a worker resolves that configuration.
|
||
func SupportsCompatibleProvider(provider string) bool {
|
||
switch provider {
|
||
case "ark", "dashscope", "deepseek", "moonshot", "zhipu", "qianfan", "hunyuan", "minimax", "baichuan", "siliconflow", "ollama", "openai_compatible":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
func providerSupportsJSONMode(provider string) bool {
|
||
switch provider {
|
||
case "ark", "dashscope", "deepseek", "siliconflow", "ollama", "openai_compatible":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
func extractJSONObject(content string) string {
|
||
content = strings.TrimSpace(content)
|
||
if end := strings.LastIndex(content, "</think>"); end >= 0 {
|
||
content = strings.TrimSpace(content[end+len("</think>"):])
|
||
}
|
||
content = strings.TrimPrefix(content, "```json")
|
||
content = strings.TrimPrefix(content, "```")
|
||
content = strings.TrimSuffix(content, "```")
|
||
content = strings.TrimSpace(content)
|
||
start, end := strings.Index(content, "{"), strings.LastIndex(content, "}")
|
||
if start >= 0 && end >= start {
|
||
return content[start : end+1]
|
||
}
|
||
return content
|
||
}
|
||
|
||
func normalizeDecisionContent(result *GenerationResult) {
|
||
if !result.Meaningful {
|
||
result.StoryWorthy = false
|
||
result.Summary = ""
|
||
result.OpenTopic = ""
|
||
result.StoryTitle = ""
|
||
result.StoryContent = ""
|
||
result.FactContent = ""
|
||
result.Keywords = nil
|
||
return
|
||
}
|
||
if !result.StoryWorthy {
|
||
result.StoryTitle = ""
|
||
result.StoryContent = ""
|
||
result.FactContent = ""
|
||
}
|
||
}
|
||
|
||
func validGeneratedContent(result GenerationResult) bool {
|
||
if result.Confidence < 0 || result.Confidence > 1 || result.DecisionReason == "" || len([]rune(result.DecisionReason)) > 500 {
|
||
return false
|
||
}
|
||
if !result.Meaningful {
|
||
return !result.StoryWorthy && result.Summary == "" && result.OpenTopic == "" && result.StoryTitle == "" && result.StoryContent == "" && result.FactContent == "" && len(result.Keywords) == 0
|
||
}
|
||
if result.Summary == "" || len([]rune(result.Summary)) > 2000 ||
|
||
result.OpenTopic == "" || len([]rune(result.OpenTopic)) > 1000 ||
|
||
len(result.Keywords) < 1 || len(result.Keywords) > 8 {
|
||
return false
|
||
}
|
||
if !result.StoryWorthy {
|
||
return result.StoryTitle == "" && result.StoryContent == "" && result.FactContent == ""
|
||
}
|
||
return result.StoryTitle != "" && len([]rune(result.StoryTitle)) <= 300 &&
|
||
result.StoryContent != "" && len([]rune(result.StoryContent)) <= 20000 &&
|
||
result.FactContent != "" && len([]rune(result.FactContent)) <= 5000 &&
|
||
result.Confidence >= 0 && result.Confidence <= 1
|
||
}
|
||
|
||
const contentValuePrompt = `你是长者聊天内容筛选与整理助手。只依据输入对话判断和整理，不补充未出现的事实，不把推断写成事实。
|
||
|
||
先做两级判断：
|
||
1. meaningful：对话是否包含值得保留的信息。只有问候、寒暄、嗯啊、测试设备、重复句、零散且无法理解的短句，必须判为false并舍弃，不生成摘要。
|
||
2. story_worthy：仅在meaningful为true时判断是否值得形成故事。具体的个人经历或回忆，且人物、时间、地点、事件、行动、感受中至少出现两类，或形成了清楚的事件脉络，才判为true。普通偏好、单个事实或简短近况可以总结，但story_worthy应为false。
|
||
|
||
必须只返回JSON对象，字段为meaningful、story_worthy、decision_reason、summary、keywords、open_topic、story_title、story_content、fact_content、confidence。
|
||
- decision_reason用一句简短中文说明判断依据。
|
||
- meaningful=false时，除decision_reason和confidence外，其余文本必须为空字符串，keywords必须为空数组，story_worthy必须为false。
|
||
- meaningful=true时，summary概括有意义内容，keywords为1至8个简短关键词，open_topic给出一个温和且不重复追问的后续话题。
|
||
- story_worthy=false时，story_title、story_content、fact_content必须为空字符串。
|
||
- story_worthy=true时，story_title和story_content忠实整理原话，fact_content只写对话中可核对的事实。
|
||
- confidence取0到1。`
|
||
|
||
func normalizedKeywords(values []string) []string {
|
||
result := make([]string, 0, min(len(values), 8))
|
||
seen := make(map[string]struct{}, len(values))
|
||
for _, value := range values {
|
||
value = strings.TrimSpace(value)
|
||
if value == "" || len([]rune(value)) > 40 {
|
||
continue
|
||
}
|
||
if _, exists := seen[value]; exists {
|
||
continue
|
||
}
|
||
seen[value] = struct{}{}
|
||
result = append(result, value)
|
||
if len(result) == 8 {
|
||
break
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
func deterministicKeywords(content string) []string {
|
||
parts := strings.FieldsFunc(content, func(character rune) bool {
|
||
return unicode.IsSpace(character) || strings.ContainsRune("，。！？；、,.!?;：:\"“”‘’（）()", character)
|
||
})
|
||
candidates := make([]string, 0, len(parts))
|
||
for _, part := range parts {
|
||
runes := []rune(strings.TrimSpace(part))
|
||
if len(runes) < 2 {
|
||
continue
|
||
}
|
||
if len(runes) > 12 {
|
||
runes = runes[:12]
|
||
}
|
||
candidates = append(candidates, string(runes))
|
||
}
|
||
keywords := normalizedKeywords(candidates)
|
||
if len(keywords) > 0 {
|
||
return keywords
|
||
}
|
||
runes := []rune(content)
|
||
if len(runes) > 12 {
|
||
runes = runes[:12]
|
||
}
|
||
return []string{string(runes)}
|
||
}
|
||
|
||
func deterministicContentDecision(content string) (meaningful, storyWorthy bool, reason string) {
|
||
compact := strings.Map(func(character rune) rune {
|
||
if unicode.IsSpace(character) || strings.ContainsRune("，。！？；、,.!?;：:\"“”‘’（）()-", character) {
|
||
return -1
|
||
}
|
||
return character
|
||
}, content)
|
||
for _, label := range []string{"长者", "豆包", "助手", "用户", "agent", "subject"} {
|
||
compact = strings.ReplaceAll(compact, label, "")
|
||
}
|
||
for _, phrase := range []string{"你好", "您好", "在吗", "嗯", "啊", "哦", "好的", "知道了", "没事", "再见", "拜拜", "听得到吗", "能听见吗", "测试"} {
|
||
compact = strings.ReplaceAll(compact, phrase, "")
|
||
}
|
||
if len([]rune(compact)) < 12 {
|
||
return false, false, "内容仅为寒暄、确认或零散短句"
|
||
}
|
||
storySignals := 0
|
||
for _, phrase := range []string{"小时候", "年轻时", "那年", "当时", "后来", "以前", "记得", "有一次", "我们", "我和", "去了", "发生", "因为", "结果", "高兴", "难过", "想起"} {
|
||
if strings.Contains(content, phrase) {
|
||
storySignals++
|
||
}
|
||
}
|
||
storyWorthy = len([]rune(compact)) >= 48 && storySignals >= 2
|
||
if storyWorthy {
|
||
return true, true, "包含较完整的个人经历与事件脉络"
|
||
}
|
||
return true, false, "包含可保留的信息，但尚不足以形成完整故事"
|
||
}
|