story/internal/extraction/generator.go

package extraction

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"
	"unicode"

	"data-terminal-cloud/internal/contentpolicy"
	"data-terminal-cloud/internal/outbox"
)

type GenerationInput struct {
	TenantID        string
	Content         string
	ChatDate        string
	PreviousStories []PreviousStory
}

type GenerationResult struct {
	Meaningful     bool         `json:"meaningful"`
	StoryWorthy    bool         `json:"story_worthy"`
	DecisionReason string       `json:"decision_reason"`
	Summary        string       `json:"summary"`
	OpenTopic      string       `json:"open_topic"`
	StoryTitle     string       `json:"story_title"`
	StoryContent   string       `json:"story_content"`
	FactContent    string       `json:"fact_content"`
	Keywords       []string     `json:"keywords"`
	Confidence     float64      `json:"confidence"`
	ModelRef       string       `json:"-"`
	PromptVersion  string       `json:"-"`
	AllowStory     bool         `json:"-"`
	DailyEvents    []DailyEvent `json:"daily_events,omitempty"`
	Stories        []EventStory `json:"stories,omitempty"`
}

type Generator interface {
	Generate(context.Context, GenerationInput) (GenerationResult, error)
}

type DeterministicGenerator struct{}

func (DeterministicGenerator) Generate(_ context.Context, input GenerationInput) (GenerationResult, error) {
	content := strings.TrimSpace(input.Content)
	if content == "" {
		return GenerationResult{}, errors.New("extraction content is required")
	}
	meaningful, storyWorthy, reason := deterministicContentDecision(content)
	if !meaningful {
		return GenerationResult{
			Meaningful: false, StoryWorthy: false, DecisionReason: reason,
			Confidence: 0.9, ModelRef: "local-deterministic-extractor", PromptVersion: "content-value-v1",
		}, nil
	}
	storyTitle, storyContent, factContent := "", "", ""
	openTopic := ""
	if storyWorthy {
		storyTitle = "关于“" + truncateRunes(content, 24) + "”的回忆"
		storyContent = content
		factContent = content
		openTopic = "如果长者愿意，下次可以从刚才经历里一个尚未讲清的具体细节自然接着聊。"
	}
	return GenerationResult{
		Meaningful:     true,
		StoryWorthy:    storyWorthy,
		DecisionReason: reason,
		Summary:        "长者在本次通话中谈到：" + content,
		OpenTopic:      openTopic,
		StoryTitle:     storyTitle,
		StoryContent:   storyContent,
		FactContent:    factContent,
		Keywords:       deterministicKeywords(content),
		Confidence:     0.9,
		ModelRef:       "local-deterministic-extractor",
		PromptVersion:  "content-value-deterministic-v2",
		AllowStory:     storyWorthy,
	}, nil
}

func sensitiveGeneration(decision contentpolicy.Decision) GenerationResult {
	modelRef := strings.TrimSpace(decision.ProviderRef)
	if modelRef == "" {
		modelRef = "local-content-safety-policy"
	}
	reason := safePolicyToken(decision.Reason)
	return GenerationResult{
		Meaningful:     true,
		StoryWorthy:    false,
		DecisionReason: "safety_policy_blocked_story",
		Summary:        "本次通话包含需要人工谨慎处理的敏感信息，系统未生成普通故事卡片。",
		OpenTopic:      "",
		Confidence:     1,
		ModelRef:       modelRef,
		PromptVersion:  "summary-sensitive-filter-v1:" + reason,
		AllowStory:     false,
	}
}

func safePolicyToken(value string) string {
	value = strings.ToLower(strings.TrimSpace(value))
	if value == "" || len(value) > 64 {
		return "policy_block"
	}
	for _, character := range value {
		if (character < 'a' || character > 'z') && (character < '0' || character > '9') && character != '_' && character != '-' {
			return "policy_block"
		}
	}
	return value
}

type ArkGenerator struct {
	endpoint      string
	apiKey        string
	model         string
	provider      string
	temperature   float64
	maxTokens     int
	jsonMaxTokens int
	client        *http.Client
}

func (*ArkGenerator) SupportsEventStories() bool { return true }

func NewArkGenerator(baseURL, apiKey, model string, timeout time.Duration) (*ArkGenerator, error) {
	return NewCompatibleGenerator("ark", baseURL, apiKey, model, 0.2, 8192, timeout)
}

func NewCompatibleGenerator(provider, baseURL, apiKey, model string, temperature float64, maxTokens int, timeout time.Duration) (*ArkGenerator, error) {
	provider = strings.ToLower(strings.TrimSpace(provider))
	baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
	apiKey = strings.TrimSpace(apiKey)
	model = strings.TrimSpace(model)
	parsed, err := url.Parse(baseURL)
	if err != nil || parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") {
		return nil, errors.New("content model base URL must be an absolute HTTP(S) URL")
	}
	if !SupportsCompatibleProvider(provider) {
		return nil, errors.New("unsupported content model provider")
	}
	if apiKey == "" || model == "" || timeout <= 0 || temperature < 0 || temperature > 2 || maxTokens < 256 || maxTokens > 8192 {
		return nil, errors.New("content model credentials and valid generation parameters are required")
	}
	return &ArkGenerator{
		endpoint: baseURL + "/chat/completions", apiKey: apiKey, model: model,
		provider: provider, temperature: temperature, maxTokens: maxTokens,
		client: &http.Client{Timeout: timeout},
	}, nil
}

func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (GenerationResult, error) {
	content := strings.TrimSpace(input.Content)
	if content == "" {
		return GenerationResult{}, errors.New("extraction content is required")
	}
	requestBody := map[string]any{
		"model": g.model,
		"messages": []map[string]string{
			{"role": "system", "content": contentValuePrompt},
			{"role": "user", "content": generationContent(input)},
		},
		"temperature": g.temperature,
		"max_tokens":  g.maxTokens,
	}
	if g.provider == "ark" {
		requestBody["thinking"] = map[string]string{"type": "disabled"}
	}
	// JSON mode is sent only where the provider documents it or this project
	// has exercised it. Other compatible providers still receive an explicit
	// JSON-only prompt, followed by strict response extraction and validation.
	if providerSupportsJSONMode(g.provider) {
		requestBody["response_format"] = map[string]string{"type": "json_object"}
	}
	encoded, err := json.Marshal(requestBody)
	if err != nil {
		return GenerationResult{}, err
	}
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, g.endpoint, bytes.NewReader(encoded))
	if err != nil {
		return GenerationResult{}, fmt.Errorf("build content model request: %w", err)
	}
	req.Header.Set("Authorization", "Bearer "+g.apiKey)
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("User-Agent", "voice-platform-content-extractor/1.0")
	resp, err := g.client.Do(req)
	if err != nil {
		return GenerationResult{}, outbox.Retryable(fmt.Errorf("call content model: %w", err))
	}
	defer resp.Body.Close()
	responseBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
	if readErr != nil {
		return GenerationResult{}, outbox.Retryable(fmt.Errorf("read content model response: %w", readErr))
	}
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		requestErr := fmt.Errorf("content model returned HTTP %d", resp.StatusCode)
		if resp.StatusCode == http.StatusRequestTimeout || resp.StatusCode == http.StatusTooEarly || resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 {
			return GenerationResult{}, outbox.Retryable(requestErr)
		}
		return GenerationResult{}, requestErr
	}
	var completion struct {
		Choices []struct {
			FinishReason string `json:"finish_reason"`
			Message      struct {
				Content string `json:"content"`
			} `json:"message"`
		} `json:"choices"`
	}
	if err = json.Unmarshal(responseBody, &completion); err != nil || len(completion.Choices) == 0 {
		return GenerationResult{}, outbox.Retryable(errors.New("content model response has no usable choice"))
	}
	if completion.Choices[0].FinishReason == "length" {
		return GenerationResult{}, outbox.Retryable(errors.New("content model result truncated"))
	}
	jsonContent := extractJSONObject(completion.Choices[0].Message.Content)
	jsonContent = strings.TrimPrefix(jsonContent, "```json")
	jsonContent = strings.TrimPrefix(jsonContent, "```")
	jsonContent = strings.TrimSuffix(jsonContent, "```")
	var result GenerationResult
	if err = json.Unmarshal([]byte(strings.TrimSpace(jsonContent)), &result); err != nil {
		return GenerationResult{}, outbox.Retryable(fmt.Errorf("decode structured content: %w", err))
	}
	result.Summary = strings.TrimSpace(result.Summary)
	result.DecisionReason = strings.TrimSpace(result.DecisionReason)
	result.OpenTopic = strings.TrimSpace(result.OpenTopic)
	result.StoryTitle = strings.TrimSpace(result.StoryTitle)
	result.StoryContent = strings.TrimSpace(result.StoryContent)
	result.FactContent = strings.TrimSpace(result.FactContent)
	result.Keywords = normalizedKeywords(result.Keywords)
	if result.DailyEvents != nil {
		// Review dates separately from story editing: a past event can be about
		// ordinary life, and today's speech can describe a future plan.
		raw, marshalErr := json.Marshal(map[string]any{"chat_date": input.ChatDate, "transcript": input.Content})
		if marshalErr != nil {
			return GenerationResult{}, marshalErr
		}
		checked, checkErr := g.GenerateJSON(ctx, string(raw), dailyExtractionPrompt)
		if checkErr != nil {
			return GenerationResult{}, checkErr
		}
		var daily struct {
			Events []DailyEvent `json:"daily_events"`
		}
		if json.Unmarshal(checked, &daily) != nil || daily.Events == nil {
			return GenerationResult{}, errors.New("daily classification failed validation")
		}
		result.DailyEvents = daily.Events
	}
	if err = normalizeEvents(&result, input); err != nil {
		return GenerationResult{}, outbox.Retryable(err)
	}
	normalizeDecisionContent(&result)
	if !validGeneratedContent(result) {
		return GenerationResult{}, outbox.Retryable(errors.New("structured content failed validation"))
	}
	result.ModelRef = g.provider + ":" + g.model
	for _, d := range result.DailyEvents {
		if d.TimeScope != "today_actual" || strings.TrimSpace(d.Evidence) == "" || !strings.Contains(input.Content, d.Evidence) {
			return GenerationResult{}, errors.New("daily event lacks evidence of an actual event today")
		}
	}
	actual := make([]DailyEvent, 0, len(result.DailyEvents))
	for _, d := range result.DailyEvents {
		future := false
		for _, phrase := range []string{"明天", "明早", "前几天", "打算", "计划", "尚未", "还没动手", "想着", "还想", "一会儿", "以后再"} {
			if strings.Contains(d.Content, phrase) {
				future = true
				break
			}
		}
		if !future {
			actual = append(actual, d)
		}
	}
	if result.DailyEvents != nil {
		result.DailyEvents = actual
	}
	result.PromptVersion = "content-events-" + g.provider + "-json-v5"
	result.AllowStory = result.Meaningful && result.StoryWorthy
	return result, nil
}

// SupportsCompatibleProvider is the single backend allowlist used both when a
// model configuration is saved and when a worker resolves that configuration.
func SupportsCompatibleProvider(provider string) bool {
	switch provider {
	case "ark", "dashscope", "deepseek", "moonshot", "zhipu", "qianfan", "hunyuan", "minimax", "baichuan", "siliconflow", "ollama", "openai_compatible":
		return true
	default:
		return false
	}
}

func providerSupportsJSONMode(provider string) bool {
	switch provider {
	case "ark", "dashscope", "deepseek", "siliconflow", "ollama", "openai_compatible":
		return true
	default:
		return false
	}
}

func extractJSONObject(content string) string {
	content = strings.TrimSpace(content)
	if end := strings.LastIndex(content, "</think>"); end >= 0 {
		content = strings.TrimSpace(content[end+len("</think>"):])
	}
	content = strings.TrimPrefix(content, "```json")
	content = strings.TrimPrefix(content, "```")
	content = strings.TrimSuffix(content, "```")
	content = strings.TrimSpace(content)
	start, end := strings.Index(content, "{"), strings.LastIndex(content, "}")
	if start >= 0 && end >= start {
		return content[start : end+1]
	}
	return content
}

func normalizeDecisionContent(result *GenerationResult) {
	if !result.Meaningful {
		result.Stories = nil
		result.DailyEvents = nil
		result.StoryWorthy = false
		result.Summary = ""
		result.OpenTopic = ""
		result.StoryTitle = ""
		result.StoryContent = ""
		result.FactContent = ""
		result.Keywords = nil
		return
	}
	if !result.StoryWorthy {
		result.Stories = nil
		result.StoryTitle = ""
		result.StoryContent = ""
		result.FactContent = ""
	}
}

func validGeneratedContent(result GenerationResult) bool {
	if result.Confidence < 0 || result.Confidence > 1 || result.DecisionReason == "" || len([]rune(result.DecisionReason)) > 500 {
		return false
	}
	if !result.Meaningful {
		return !result.StoryWorthy && result.Summary == "" && result.OpenTopic == "" && result.StoryTitle == "" && result.StoryContent == "" && result.FactContent == "" && len(result.Keywords) == 0
	}
	if result.Summary == "" || len([]rune(result.Summary)) > 2000 ||
		len([]rune(result.OpenTopic)) > 1000 ||
		len(result.Keywords) < 1 || len(result.Keywords) > 8 {
		return false
	}
	if !result.StoryWorthy {
		return result.StoryTitle == "" && result.StoryContent == "" && result.FactContent == ""
	}
	return result.StoryTitle != "" && len([]rune(result.StoryTitle)) <= 300 &&
		result.StoryContent != "" && len([]rune(result.StoryContent)) <= 20000 &&
		result.FactContent != "" && len([]rune(result.FactContent)) <= 5000 &&
		result.Confidence >= 0 && result.Confidence <= 1
}

const contentValuePrompt = eventExtractionPrompt

func normalizedKeywords(values []string) []string {
	result := make([]string, 0, min(len(values), 8))
	seen := make(map[string]struct{}, len(values))
	for _, value := range values {
		value = strings.TrimSpace(value)
		if value == "" || len([]rune(value)) > 40 {
			continue
		}
		if _, exists := seen[value]; exists {
			continue
		}
		seen[value] = struct{}{}
		result = append(result, value)
		if len(result) == 8 {
			break
		}
	}
	return result
}

func deterministicKeywords(content string) []string {
	parts := strings.FieldsFunc(content, func(character rune) bool {
		return unicode.IsSpace(character) || strings.ContainsRune("，。！？；、,.!?;：:\"“”‘’（）()", character)
	})
	candidates := make([]string, 0, len(parts))
	for _, part := range parts {
		runes := []rune(strings.TrimSpace(part))
		if len(runes) < 2 {
			continue
		}
		if len(runes) > 12 {
			runes = runes[:12]
		}
		candidates = append(candidates, string(runes))
	}
	keywords := normalizedKeywords(candidates)
	if len(keywords) > 0 {
		return keywords
	}
	runes := []rune(content)
	if len(runes) > 12 {
		runes = runes[:12]
	}
	return []string{string(runes)}
}

func deterministicContentDecision(content string) (meaningful, storyWorthy bool, reason string) {
	compact := strings.Map(func(character rune) rune {
		if unicode.IsSpace(character) || strings.ContainsRune("，。！？；、,.!?;：:\"“”‘’（）()-", character) {
			return -1
		}
		return character
	}, content)
	for _, label := range []string{"长者", "豆包", "助手", "用户", "agent", "subject"} {
		compact = strings.ReplaceAll(compact, label, "")
	}
	for _, phrase := range []string{"你好", "您好", "在吗", "嗯", "啊", "哦", "好的", "知道了", "没事", "再见", "拜拜", "听得到吗", "能听见吗", "测试"} {
		compact = strings.ReplaceAll(compact, phrase, "")
	}
	if len([]rune(compact)) < 12 {
		return false, false, "内容仅为寒暄、确认或零散短句"
	}
	storySignals := 0
	for _, phrase := range []string{"小时候", "年轻时", "那年", "当时", "后来", "以前", "记得", "有一次", "我们", "我和", "去了", "发生", "因为", "结果", "高兴", "难过", "想起"} {
		if strings.Contains(content, phrase) {
			storySignals++
		}
	}
	storyWorthy = len([]rune(compact)) >= 48 && storySignals >= 2
	if storyWorthy {
		return true, true, "包含较完整的个人经历与事件脉络"
	}
	return true, false, "包含可保留的信息，但尚不足以形成完整故事"
}