package extraction import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "strings" "time" "unicode" "data-terminal-cloud/internal/contentpolicy" "data-terminal-cloud/internal/outbox" ) type GenerationInput struct { Content string } type GenerationResult struct { Summary string `json:"summary"` OpenTopic string `json:"open_topic"` StoryTitle string `json:"story_title"` StoryContent string `json:"story_content"` FactContent string `json:"fact_content"` Keywords []string `json:"keywords"` Confidence float64 `json:"confidence"` ModelRef string `json:"-"` PromptVersion string `json:"-"` AllowStory bool `json:"-"` } type Generator interface { Generate(context.Context, GenerationInput) (GenerationResult, error) } type DeterministicGenerator struct{} func (DeterministicGenerator) Generate(_ context.Context, input GenerationInput) (GenerationResult, error) { content := strings.TrimSpace(input.Content) if content == "" { return GenerationResult{}, errors.New("extraction content is required") } return GenerationResult{ Summary: "长者在本次通话中谈到:" + content, OpenTopic: "下次可以继续询问长者刚才提到的经历,并让长者自行补充人物、时间和地点。", StoryTitle: "关于“" + truncateRunes(content, 24) + "”的回忆", StoryContent: content, FactContent: content, Keywords: deterministicKeywords(content), Confidence: 0.9, ModelRef: "local-deterministic-extractor", PromptVersion: "story-v1", AllowStory: true, }, nil } func sensitiveGeneration(decision contentpolicy.Decision) GenerationResult { modelRef := strings.TrimSpace(decision.ProviderRef) if modelRef == "" { modelRef = "local-content-safety-policy" } reason := safePolicyToken(decision.Reason) return GenerationResult{ Summary: "本次通话包含需要人工谨慎处理的敏感信息,系统未生成普通故事卡片。", OpenTopic: "如确有必要处理本次敏感内容,应由有权限人员在受控流程中单独核对。", Confidence: 1, ModelRef: modelRef, PromptVersion: "summary-sensitive-filter-v1:" + reason, AllowStory: false, } } func safePolicyToken(value string) string { value = strings.ToLower(strings.TrimSpace(value)) if value == "" || len(value) > 64 { return "policy_block" } for _, character := range value { if (character < 'a' || character > 'z') && (character < '0' || character > '9') && character != '_' && character != '-' { return "policy_block" } } return value } type ArkGenerator struct { endpoint string apiKey string model string client *http.Client } func NewArkGenerator(baseURL, apiKey, model string, timeout time.Duration) (*ArkGenerator, error) { baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") apiKey = strings.TrimSpace(apiKey) model = strings.TrimSpace(model) parsed, err := url.Parse(baseURL) if err != nil || parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") { return nil, errors.New("Ark base URL must be an absolute HTTP(S) URL") } if apiKey == "" || model == "" || timeout <= 0 { return nil, errors.New("Ark API key, content model and positive timeout are required") } return &ArkGenerator{ endpoint: baseURL + "/chat/completions", apiKey: apiKey, model: model, client: &http.Client{Timeout: timeout}, }, nil } func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (GenerationResult, error) { content := strings.TrimSpace(input.Content) if content == "" { return GenerationResult{}, errors.New("extraction content is required") } requestBody := map[string]any{ "model": g.model, "messages": []map[string]string{ {"role": "system", "content": "你是长者口述内容整理助手。只依据输入字幕生成可人工核对的候选,不补充未出现的事实,不把推断写成事实。必须返回JSON对象,字段为summary、keywords、open_topic、story_title、story_content、fact_content、confidence。summary概括本次内容,keywords是1至8个简短关键词字符串组成的数组,open_topic给出一个温和的后续话题,story和fact保留原意,confidence取0到1。"}, {"role": "user", "content": content}, }, "temperature": 0.2, "max_tokens": 1200, "response_format": map[string]string{"type": "json_object"}, } encoded, err := json.Marshal(requestBody) if err != nil { return GenerationResult{}, err } req, err := http.NewRequestWithContext(ctx, http.MethodPost, g.endpoint, bytes.NewReader(encoded)) if err != nil { return GenerationResult{}, fmt.Errorf("build Ark content request: %w", err) } req.Header.Set("Authorization", "Bearer "+g.apiKey) req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", "voice-platform-content-extractor/1.0") resp, err := g.client.Do(req) if err != nil { return GenerationResult{}, outbox.Retryable(fmt.Errorf("call Ark content model: %w", err)) } defer resp.Body.Close() responseBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) if readErr != nil { return GenerationResult{}, outbox.Retryable(fmt.Errorf("read Ark content response: %w", readErr)) } if resp.StatusCode < 200 || resp.StatusCode >= 300 { requestErr := fmt.Errorf("Ark content model returned HTTP %d", resp.StatusCode) if resp.StatusCode == http.StatusRequestTimeout || resp.StatusCode == http.StatusTooEarly || resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 { return GenerationResult{}, outbox.Retryable(requestErr) } return GenerationResult{}, requestErr } var completion struct { Choices []struct { Message struct { Content string `json:"content"` } `json:"message"` } `json:"choices"` } if err = json.Unmarshal(responseBody, &completion); err != nil || len(completion.Choices) == 0 { return GenerationResult{}, outbox.Retryable(errors.New("Ark content response has no usable choice")) } jsonContent := strings.TrimSpace(completion.Choices[0].Message.Content) jsonContent = strings.TrimPrefix(jsonContent, "```json") jsonContent = strings.TrimPrefix(jsonContent, "```") jsonContent = strings.TrimSuffix(jsonContent, "```") var result GenerationResult if err = json.Unmarshal([]byte(strings.TrimSpace(jsonContent)), &result); err != nil { return GenerationResult{}, outbox.Retryable(fmt.Errorf("decode Ark structured content: %w", err)) } result.Summary = strings.TrimSpace(result.Summary) result.OpenTopic = strings.TrimSpace(result.OpenTopic) result.StoryTitle = strings.TrimSpace(result.StoryTitle) result.StoryContent = strings.TrimSpace(result.StoryContent) result.FactContent = strings.TrimSpace(result.FactContent) result.Keywords = normalizedKeywords(result.Keywords) if !validGeneratedContent(result) { return GenerationResult{}, outbox.Retryable(errors.New("Ark structured content failed validation")) } result.ModelRef = "ark:" + g.model result.PromptVersion = "story-ark-json-v1" result.AllowStory = true return result, nil } func validGeneratedContent(result GenerationResult) bool { return result.Summary != "" && len([]rune(result.Summary)) <= 2000 && result.OpenTopic != "" && len([]rune(result.OpenTopic)) <= 1000 && result.StoryTitle != "" && len([]rune(result.StoryTitle)) <= 300 && result.StoryContent != "" && len([]rune(result.StoryContent)) <= 20000 && result.FactContent != "" && len([]rune(result.FactContent)) <= 5000 && len(result.Keywords) >= 1 && len(result.Keywords) <= 8 && result.Confidence >= 0 && result.Confidence <= 1 } func normalizedKeywords(values []string) []string { result := make([]string, 0, min(len(values), 8)) seen := make(map[string]struct{}, len(values)) for _, value := range values { value = strings.TrimSpace(value) if value == "" || len([]rune(value)) > 40 { continue } if _, exists := seen[value]; exists { continue } seen[value] = struct{}{} result = append(result, value) if len(result) == 8 { break } } return result } func deterministicKeywords(content string) []string { parts := strings.FieldsFunc(content, func(character rune) bool { return unicode.IsSpace(character) || strings.ContainsRune(",。!?;、,.!?;::\"“”‘’()()", character) }) candidates := make([]string, 0, len(parts)) for _, part := range parts { runes := []rune(strings.TrimSpace(part)) if len(runes) < 2 { continue } if len(runes) > 12 { runes = runes[:12] } candidates = append(candidates, string(runes)) } keywords := normalizedKeywords(candidates) if len(keywords) > 0 { return keywords } runes := []rune(content) if len(runes) > 12 { runes = runes[:12] } return []string{string(runes)} }