From 522785eda86751ece4b6ff7a8ba2ceaa3a8ff5aa Mon Sep 17 00:00:00 2001 From: Codex Local Date: Sun, 30 Aug 2026 15:20:08 +0800 Subject: [PATCH] feat: filter meaningful conversations with local model --- CHANGELOG.md | 4 + compose.yaml | 21 +++ deploy/local-model/README.md | 31 ++++ docs/device-api-compatibility.md | 2 +- internal/contentmodel/service.go | 17 +- .../contentmodel/service_integration_test.go | 67 ++++++++ internal/contentmodel/service_test.go | 25 +++ internal/extraction/generator.go | 162 ++++++++++++++---- .../extraction/generator_local_model_test.go | 58 +++++++ internal/extraction/generator_test.go | 47 ++++- internal/extraction/processor.go | 21 ++- .../extraction/processor_integration_test.go | 3 +- internal/extraction/processor_safety_test.go | 1 + internal/mockprovider/server.go | 2 +- internal/privacy/delete_processor.go | 1 + ...4_conversation_content_assessment.down.sql | 5 + ...044_conversation_content_assessment.up.sql | 28 +++ scripts/smoke-conversation.ps1 | 1 + 18 files changed, 451 insertions(+), 45 deletions(-) create mode 100644 deploy/local-model/README.md create mode 100644 internal/contentmodel/service_integration_test.go create mode 100644 internal/contentmodel/service_test.go create mode 100644 internal/extraction/generator_local_model_test.go create mode 100644 migrations/000044_conversation_content_assessment.down.sql create mode 100644 migrations/000044_conversation_content_assessment.up.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 219cc0c..e51fc94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ ## 未发布 +- 聊天总结改为大模型两级内容价值判断：先判断是否值得保留，再判断是否值得形成故事；纯寒暄不再生成摘要或故事。 +- 新增内容价值判断记录，保留模型、提示词版本、置信度和舍弃原因，支持任务幂等与隐私删除。 +- 本地开发新增 GPU Ollama/Qwen2.5 7B 部署方式，并通过大模型管理的测试、激活和 worker 生成链路完成端到端验证。 +- 修复大模型“测试连接”更新语句和审计结果枚举错误，测试通过的模型现在可以正常激活。 - 维护端默认人设升级为老人陪伴对话规范，覆盖倾听、单问题追问、沉默与重复处理、事实边界、敏感话题和医疗安全。 - 家庭端长者档案新增日常陪伴、回忆往事、兴趣聊天三个模板，并补充兴趣、优先话题和避免话题的创建入口。 - Agent固定评测加强单问题追问、先回应再追问、沉默不施压、尊重重复表达和情绪优先回应检查。 diff --git a/compose.yaml b/compose.yaml index ea97044..6fb5633 100644 --- a/compose.yaml +++ b/compose.yaml @@ -155,6 +155,26 @@ services: volumes: - ./deploy/keycloak:/opt/keycloak/data/import:ro + ollama: + profiles: [local-model] + image: ollama/ollama:latest + ports: + - "${OLLAMA_PORT:-11434}:11434" + volumes: + - ollama-data:/root/.ollama + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + healthcheck: + test: ["CMD", "ollama", "list"] + interval: 10s + timeout: 5s + retries: 30 + otel-collector: image: otel/opentelemetry-collector-contrib:latest command: ["--config=/etc/otelcol-contrib/config.yaml"] @@ -325,3 +345,4 @@ volumes: postgres-data: redis-data: minio-data: + ollama-data: diff --git a/deploy/local-model/README.md b/deploy/local-model/README.md new file mode 100644 index 0000000..ed937c1 --- /dev/null +++ b/deploy/local-model/README.md @@ -0,0 +1,31 @@ +# 本地聊天总结模型 + +本地开发环境使用 Ollama 提供 OpenAI 兼容接口，默认验证模型为 `qwen2.5:7b`。模型只监听本机 `11434` 端口，应用容器通过 `http://ollama:11434/v1` 访问。 + +## 启动与下载模型 + +```powershell +docker compose --profile local-model up -d ollama +docker compose --profile local-model exec ollama ollama pull qwen2.5:7b +``` + +随后在客户门户的“大模型管理”中创建并激活配置： + +- 服务类型：`OpenAI 兼容接口` +- 接口基础地址：`http://ollama:11434/v1` +- 模型 ID：`qwen2.5:7b` +- API Key：本地 Ollama 不校验密钥，可填写长度至少 8 位的本地占位值 +- 建议温度：`0.1` +- 建议最大输出：`1200` + +必须先执行“测试连接”，通过后才能“设为当前模型”。配置激活后，worker 会在每个总结任务开始时读取当前模型，不需要重启。 + +## 真实模型回归测试 + +```powershell +$env:LOCAL_MODEL_BASE_URL='http://localhost:11434/v1' +$env:LOCAL_MODEL_ID='qwen2.5:7b' +go test ./internal/extraction -run TestLocalModelContentValueDecisions -count=1 -v +``` + +测试覆盖三类输入：纯寒暄应舍弃、有用近况只生成摘要、完整个人回忆生成摘要和故事候选。 diff --git a/docs/device-api-compatibility.md b/docs/device-api-compatibility.md index ca7fc46..1c4ba68 100644 --- a/docs/device-api-compatibility.md +++ b/docs/device-api-compatibility.md @@ -1,7 +1,7 @@ # 设备 API 兼容性约定 > 当前设备接口主版本：`/device/v2` -> 当前接口契约：OpenAPI 0.70.0 +> 当前接口契约：OpenAPI 0.73.0 > 适用对象：云端 API、终端固件、模拟终端和设备联调工具 ## 1. 兼容目标 diff --git a/internal/contentmodel/service.go b/internal/contentmodel/service.go index 201d88e..f3a5222 100644 --- a/internal/contentmodel/service.go +++ b/internal/contentmodel/service.go @@ -209,15 +209,19 @@ func (s *Service) Test(ctx context.Context, principal identity.Principal, tenant testStatus, errorCode = "failed", normalizedErrorCode(err) } var result Model + auditResult := "success" + if testStatus != "passed" { + auditResult = "failure" + } updateErr := database.Transact(ctx, s.registry.pool, func(tx pgx.Tx) error { - row := tx.QueryRow(ctx, modelUpdateReturning+` WHERE cm.tenant_id=$1::uuid AND cm.public_id=$2::uuid`, current.tenantID, publicID, testStatus, errorCode, current.userID, current.tenantPublicID) + row := tx.QueryRow(ctx, modelUpdateReturning, current.tenantID, publicID, testStatus, errorCode, current.userID, current.tenantPublicID) updated, scanErr := scanModel(row) if scanErr != nil { return scanErr } result = updated.Model resourceID := result.ID - return s.audit.Record(ctx, tx, audit.Event{TenantID: ¤t.tenantID, ActorType: "user", ActorPublicID: ¤t.userPublicID, Action: "content_model.test", ResourceType: "content_model", ResourceID: &resourceID, RequestID: requestID, Purpose: "验证总结模型连通性", Result: testStatus, Detail: map[string]any{"error_code": errorCode}}) + return s.audit.Record(ctx, tx, audit.Event{TenantID: ¤t.tenantID, ActorType: "user", ActorPublicID: ¤t.userPublicID, Action: "content_model.test", ResourceType: "content_model", ResourceID: &resourceID, RequestID: requestID, Purpose: "验证总结模型连通性", Result: auditResult, Detail: map[string]any{"test_status": testStatus, "error_code": errorCode}}) }) if updateErr != nil { return Model{}, updateErr @@ -359,13 +363,18 @@ func normalizeInput(input *CreateInput) { func validateInput(input CreateInput) error { parsed, err := url.Parse(input.BaseURL) validProvider := input.Provider == "ark" || input.Provider == "openai_compatible" - validURL := err == nil && parsed.Host != "" && (parsed.Scheme == "https" || (parsed.Scheme == "http" && (parsed.Hostname() == "localhost" || parsed.Hostname() == "127.0.0.1"))) + validURL := err == nil && parsed.Host != "" && (parsed.Scheme == "https" || (parsed.Scheme == "http" && isApprovedLocalModelHost(parsed.Hostname()))) if !validUUID(input.TenantID) || len([]rune(input.DisplayName)) < 1 || len([]rune(input.DisplayName)) > 120 || !validProvider || !validURL || input.ModelID == "" || len(input.APIKey) < 8 || input.Temperature < 0 || input.Temperature > 2 || input.MaxTokens < 256 || input.MaxTokens > 8192 { return ErrInvalid } return nil } +func isApprovedLocalModelHost(host string) bool { + host = strings.ToLower(strings.TrimSpace(host)) + return host == "localhost" || host == "127.0.0.1" || host == "::1" || host == "ollama" +} + func credentialHint(key string) string { runes := []rune(strings.TrimSpace(key)) if len(runes) <= 4 { @@ -398,7 +407,7 @@ func validUUID(value string) bool { _, err := uuid.Parse(value); return err == n const modelSelect = `SELECT cm.public_id::text,t.public_id::text,cm.display_name,cm.provider,cm.base_url,cm.model_id,cm.credential_hint,cm.temperature::float8,cm.max_tokens,cm.status,cm.test_status,cm.last_tested_at,COALESCE(cm.last_error_code,''),cm.activated_at,cm.version,cm.created_at,cm.updated_at,cm.tenant_id::text,cm.encrypted_api_key,cm.nonce,cm.key_version FROM content_model_config cm JOIN tenant t ON t.id=cm.tenant_id` const modelReturning = `cm.public_id::text,$4::text,cm.display_name,cm.provider,cm.base_url,cm.model_id,cm.credential_hint,cm.temperature::float8,cm.max_tokens,cm.status,cm.test_status,cm.last_tested_at,COALESCE(cm.last_error_code,''),cm.activated_at,cm.version,cm.created_at,cm.updated_at,cm.tenant_id::text,cm.encrypted_api_key,cm.nonce,cm.key_version` -const modelUpdateReturning = `UPDATE content_model_config cm SET test_status=$3,last_tested_at=now(),last_error_code=NULLIF($4,''),updated_at=now(),version=version+1,updated_by=$5::uuid RETURNING cm.public_id::text,$6::text,cm.display_name,cm.provider,cm.base_url,cm.model_id,cm.credential_hint,cm.temperature::float8,cm.max_tokens,cm.status,cm.test_status,cm.last_tested_at,COALESCE(cm.last_error_code,''),cm.activated_at,cm.version,cm.created_at,cm.updated_at,cm.tenant_id::text,cm.encrypted_api_key,cm.nonce,cm.key_version` +const modelUpdateReturning = `UPDATE content_model_config cm SET test_status=$3,last_tested_at=now(),last_error_code=NULLIF($4,''),updated_at=now(),version=version+1,updated_by=$5::uuid WHERE cm.tenant_id=$1::uuid AND cm.public_id=$2::uuid RETURNING cm.public_id::text,$6::text,cm.display_name,cm.provider,cm.base_url,cm.model_id,cm.credential_hint,cm.temperature::float8,cm.max_tokens,cm.status,cm.test_status,cm.last_tested_at,COALESCE(cm.last_error_code,''),cm.activated_at,cm.version,cm.created_at,cm.updated_at,cm.tenant_id::text,cm.encrypted_api_key,cm.nonce,cm.key_version` type rowScanner interface{ Scan(...any) error } diff --git a/internal/contentmodel/service_integration_test.go b/internal/contentmodel/service_integration_test.go new file mode 100644 index 0000000..fee6e75 --- /dev/null +++ b/internal/contentmodel/service_integration_test.go @@ -0,0 +1,67 @@ +package contentmodel + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "testing" + + "data-terminal-cloud/internal/audit" + "data-terminal-cloud/internal/extraction" + "data-terminal-cloud/internal/identity" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestServiceTestsAndActivatesCompatibleModel(t *testing.T) { + databaseURL := os.Getenv("TEST_DATABASE_URL") + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": `{"meaningful":false,"story_worthy":false,"decision_reason":"连通性测试内容不形成故事","summary":"","keywords":[],"open_topic":"","story_title":"","story_content":"","fact_content":"","confidence":0.99}`}}}}) + })) + defer server.Close() + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + registry, err := NewRegistry(pool, "1f1e1d1c1b1a191817161514131211100f0e0d0c0b0a09080706050403020100", extraction.DeterministicGenerator{}) + if err != nil { + t.Fatal(err) + } + service, err := NewService(registry, audit.NewRepository()) + if err != nil { + t.Fatal(err) + } + principal := identity.Principal{Issuer: "http://localhost:58180/realms/voice-local", Subject: "00000000-0000-0000-0000-000000000101"} + created, err := service.Create(ctx, principal, CreateInput{ + TenantID: "10000000-0000-0000-0000-000000000010", DisplayName: "集成测试模型", Provider: "openai_compatible", + BaseURL: server.URL + "/v1", ModelID: "test-model", APIKey: "integration-key", Temperature: 0.1, MaxTokens: 512, + }, "content-model-integration-create") + if err != nil { + t.Fatal(err) + } + defer func() { + _, _ = pool.Exec(ctx, `DELETE FROM audit_event WHERE resource_id=$1::uuid`, created.ID) + _, _ = pool.Exec(ctx, `DELETE FROM content_model_config WHERE public_id=$1::uuid`, created.ID) + }() + tested, err := service.Test(ctx, principal, created.TenantID, created.ID, "content-model-integration-test") + if err != nil { + t.Fatal(err) + } + if tested.TestStatus != "passed" || tested.Version != created.Version+1 { + t.Fatalf("tested model=%+v", tested) + } + activated, err := service.Activate(ctx, principal, created.TenantID, created.ID, "content-model-integration-activate", tested.Version) + if err != nil { + t.Fatal(err) + } + if activated.Status != "active" || activated.Version != tested.Version+1 { + t.Fatalf("activated model=%+v", activated) + } +} diff --git a/internal/contentmodel/service_test.go b/internal/contentmodel/service_test.go new file mode 100644 index 0000000..503c295 --- /dev/null +++ b/internal/contentmodel/service_test.go @@ -0,0 +1,25 @@ +package contentmodel + +import "testing" + +func TestValidateInputAllowsComposeOllamaEndpoint(t *testing.T) { + input := CreateInput{ + TenantID: "10000000-0000-0000-0000-000000000001", DisplayName: "本地 Qwen", + Provider: "openai_compatible", BaseURL: "http://ollama:11434/v1", ModelID: "qwen2.5:7b", + APIKey: "ollama-local-key", Temperature: 0.1, MaxTokens: 1200, + } + if err := validateInput(input); err != nil { + t.Fatalf("compose Ollama endpoint was rejected: %v", err) + } +} + +func TestValidateInputRejectsUnapprovedPlainHTTPEndpoint(t *testing.T) { + input := CreateInput{ + TenantID: "10000000-0000-0000-0000-000000000001", DisplayName: "不安全端点", + Provider: "openai_compatible", BaseURL: "http://metadata.internal/v1", ModelID: "model", + APIKey: "test-api-key", Temperature: 0.1, MaxTokens: 1200, + } + if err := validateInput(input); err == nil { + t.Fatal("unapproved plaintext endpoint was accepted") + } +} diff --git a/internal/extraction/generator.go b/internal/extraction/generator.go index 9786c31..5cdeffc 100644 --- a/internal/extraction/generator.go +++ b/internal/extraction/generator.go @@ -23,16 +23,19 @@ type GenerationInput struct { } type GenerationResult struct { - Summary string `json:"summary"` - OpenTopic string `json:"open_topic"` - StoryTitle string `json:"story_title"` - StoryContent string `json:"story_content"` - FactContent string `json:"fact_content"` - Keywords []string `json:"keywords"` - Confidence float64 `json:"confidence"` - ModelRef string `json:"-"` - PromptVersion string `json:"-"` - AllowStory bool `json:"-"` + Meaningful bool `json:"meaningful"` + StoryWorthy bool `json:"story_worthy"` + DecisionReason string `json:"decision_reason"` + Summary string `json:"summary"` + OpenTopic string `json:"open_topic"` + StoryTitle string `json:"story_title"` + StoryContent string `json:"story_content"` + FactContent string `json:"fact_content"` + Keywords []string `json:"keywords"` + Confidence float64 `json:"confidence"` + ModelRef string `json:"-"` + PromptVersion string `json:"-"` + AllowStory bool `json:"-"` } type Generator interface { @@ -46,17 +49,33 @@ func (DeterministicGenerator) Generate(_ context.Context, input GenerationInput) if content == "" { return GenerationResult{}, errors.New("extraction content is required") } + meaningful, storyWorthy, reason := deterministicContentDecision(content) + if !meaningful { + return GenerationResult{ + Meaningful: false, StoryWorthy: false, DecisionReason: reason, + Confidence: 0.9, ModelRef: "local-deterministic-extractor", PromptVersion: "content-value-v1", + }, nil + } + storyTitle, storyContent, factContent := "", "", "" + if storyWorthy { + storyTitle = "关于“" + truncateRunes(content, 24) + "”的回忆" + storyContent = content + factContent = content + } return GenerationResult{ - Summary: "长者在本次通话中谈到：" + content, - OpenTopic: "下次可以继续询问长者刚才提到的经历，并让长者自行补充人物、时间和地点。", - StoryTitle: "关于“" + truncateRunes(content, 24) + "”的回忆", - StoryContent: content, - FactContent: content, - Keywords: deterministicKeywords(content), - Confidence: 0.9, - ModelRef: "local-deterministic-extractor", - PromptVersion: "story-v1", - AllowStory: true, + Meaningful: true, + StoryWorthy: storyWorthy, + DecisionReason: reason, + Summary: "长者在本次通话中谈到：" + content, + OpenTopic: "下次可以继续询问长者刚才提到的经历，并让长者自行补充人物、时间和地点。", + StoryTitle: storyTitle, + StoryContent: storyContent, + FactContent: factContent, + Keywords: deterministicKeywords(content), + Confidence: 0.9, + ModelRef: "local-deterministic-extractor", + PromptVersion: "content-value-deterministic-v2", + AllowStory: storyWorthy, }, nil } @@ -67,12 +86,15 @@ func sensitiveGeneration(decision contentpolicy.Decision) GenerationResult { } reason := safePolicyToken(decision.Reason) return GenerationResult{ - Summary: "本次通话包含需要人工谨慎处理的敏感信息，系统未生成普通故事卡片。", - OpenTopic: "如确有必要处理本次敏感内容，应由有权限人员在受控流程中单独核对。", - Confidence: 1, - ModelRef: modelRef, - PromptVersion: "summary-sensitive-filter-v1:" + reason, - AllowStory: false, + Meaningful: true, + StoryWorthy: false, + DecisionReason: "safety_policy_blocked_story", + Summary: "本次通话包含需要人工谨慎处理的敏感信息，系统未生成普通故事卡片。", + OpenTopic: "如确有必要处理本次敏感内容，应由有权限人员在受控流程中单独核对。", + Confidence: 1, + ModelRef: modelRef, + PromptVersion: "summary-sensitive-filter-v1:" + reason, + AllowStory: false, } } @@ -133,7 +155,7 @@ func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (Gen requestBody := map[string]any{ "model": g.model, "messages": []map[string]string{ - {"role": "system", "content": "你是长者口述内容整理助手。只依据输入字幕生成可人工核对的候选，不补充未出现的事实，不把推断写成事实。必须返回JSON对象，字段为summary、keywords、open_topic、story_title、story_content、fact_content、confidence。summary概括本次内容，keywords是1至8个简短关键词字符串组成的数组，open_topic给出一个温和的后续话题，story和fact保留原意，confidence取0到1。"}, + {"role": "system", "content": contentValuePrompt}, {"role": "user", "content": content}, }, "temperature": g.temperature, @@ -186,30 +208,75 @@ func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (Gen return GenerationResult{}, outbox.Retryable(fmt.Errorf("decode Ark structured content: %w", err)) } result.Summary = strings.TrimSpace(result.Summary) + result.DecisionReason = strings.TrimSpace(result.DecisionReason) result.OpenTopic = strings.TrimSpace(result.OpenTopic) result.StoryTitle = strings.TrimSpace(result.StoryTitle) result.StoryContent = strings.TrimSpace(result.StoryContent) result.FactContent = strings.TrimSpace(result.FactContent) result.Keywords = normalizedKeywords(result.Keywords) + normalizeDecisionContent(&result) if !validGeneratedContent(result) { return GenerationResult{}, outbox.Retryable(errors.New("Ark structured content failed validation")) } result.ModelRef = g.provider + ":" + g.model - result.PromptVersion = "story-" + g.provider + "-json-v2" - result.AllowStory = true + result.PromptVersion = "content-value-" + g.provider + "-json-v3" + result.AllowStory = result.Meaningful && result.StoryWorthy return result, nil } +func normalizeDecisionContent(result *GenerationResult) { + if !result.Meaningful { + result.StoryWorthy = false + result.Summary = "" + result.OpenTopic = "" + result.StoryTitle = "" + result.StoryContent = "" + result.FactContent = "" + result.Keywords = nil + return + } + if !result.StoryWorthy { + result.StoryTitle = "" + result.StoryContent = "" + result.FactContent = "" + } +} + func validGeneratedContent(result GenerationResult) bool { - return result.Summary != "" && len([]rune(result.Summary)) <= 2000 && - result.OpenTopic != "" && len([]rune(result.OpenTopic)) <= 1000 && - result.StoryTitle != "" && len([]rune(result.StoryTitle)) <= 300 && + if result.Confidence < 0 || result.Confidence > 1 || result.DecisionReason == "" || len([]rune(result.DecisionReason)) > 500 { + return false + } + if !result.Meaningful { + return !result.StoryWorthy && result.Summary == "" && result.OpenTopic == "" && result.StoryTitle == "" && result.StoryContent == "" && result.FactContent == "" && len(result.Keywords) == 0 + } + if result.Summary == "" || len([]rune(result.Summary)) > 2000 || + result.OpenTopic == "" || len([]rune(result.OpenTopic)) > 1000 || + len(result.Keywords) < 1 || len(result.Keywords) > 8 { + return false + } + if !result.StoryWorthy { + return result.StoryTitle == "" && result.StoryContent == "" && result.FactContent == "" + } + return result.StoryTitle != "" && len([]rune(result.StoryTitle)) <= 300 && result.StoryContent != "" && len([]rune(result.StoryContent)) <= 20000 && result.FactContent != "" && len([]rune(result.FactContent)) <= 5000 && - len(result.Keywords) >= 1 && len(result.Keywords) <= 8 && result.Confidence >= 0 && result.Confidence <= 1 } +const contentValuePrompt = `你是长者聊天内容筛选与整理助手。只依据输入对话判断和整理，不补充未出现的事实，不把推断写成事实。 + +先做两级判断： +1. meaningful：对话是否包含值得保留的信息。只有问候、寒暄、嗯啊、测试设备、重复句、零散且无法理解的短句，必须判为false并舍弃，不生成摘要。 +2. story_worthy：仅在meaningful为true时判断是否值得形成故事。具体的个人经历或回忆，且人物、时间、地点、事件、行动、感受中至少出现两类，或形成了清楚的事件脉络，才判为true。普通偏好、单个事实或简短近况可以总结，但story_worthy应为false。 + +必须只返回JSON对象，字段为meaningful、story_worthy、decision_reason、summary、keywords、open_topic、story_title、story_content、fact_content、confidence。 +- decision_reason用一句简短中文说明判断依据。 +- meaningful=false时，除decision_reason和confidence外，其余文本必须为空字符串，keywords必须为空数组，story_worthy必须为false。 +- meaningful=true时，summary概括有意义内容，keywords为1至8个简短关键词，open_topic给出一个温和且不重复追问的后续话题。 +- story_worthy=false时，story_title、story_content、fact_content必须为空字符串。 +- story_worthy=true时，story_title和story_content忠实整理原话，fact_content只写对话中可核对的事实。 +- confidence取0到1。` + func normalizedKeywords(values []string) []string { result := make([]string, 0, min(len(values), 8)) seen := make(map[string]struct{}, len(values)) @@ -255,3 +322,32 @@ func deterministicKeywords(content string) []string { } return []string{string(runes)} } + +func deterministicContentDecision(content string) (meaningful, storyWorthy bool, reason string) { + compact := strings.Map(func(character rune) rune { + if unicode.IsSpace(character) || strings.ContainsRune("，。！？；、,.!?;：:\"“”‘’（）()-", character) { + return -1 + } + return character + }, content) + for _, label := range []string{"长者", "豆包", "助手", "用户", "agent", "subject"} { + compact = strings.ReplaceAll(compact, label, "") + } + for _, phrase := range []string{"你好", "您好", "在吗", "嗯", "啊", "哦", "好的", "知道了", "没事", "再见", "拜拜", "听得到吗", "能听见吗", "测试"} { + compact = strings.ReplaceAll(compact, phrase, "") + } + if len([]rune(compact)) < 12 { + return false, false, "内容仅为寒暄、确认或零散短句" + } + storySignals := 0 + for _, phrase := range []string{"小时候", "年轻时", "那年", "当时", "后来", "以前", "记得", "有一次", "我们", "我和", "去了", "发生", "因为", "结果", "高兴", "难过", "想起"} { + if strings.Contains(content, phrase) { + storySignals++ + } + } + storyWorthy = len([]rune(compact)) >= 48 && storySignals >= 2 + if storyWorthy { + return true, true, "包含较完整的个人经历与事件脉络" + } + return true, false, "包含可保留的信息，但尚不足以形成完整故事" +} diff --git a/internal/extraction/generator_local_model_test.go b/internal/extraction/generator_local_model_test.go new file mode 100644 index 0000000..2604e07 --- /dev/null +++ b/internal/extraction/generator_local_model_test.go @@ -0,0 +1,58 @@ +package extraction + +import ( + "context" + "os" + "testing" + "time" +) + +func TestLocalModelContentValueDecisions(t *testing.T) { + baseURL := os.Getenv("LOCAL_MODEL_BASE_URL") + if baseURL == "" { + t.Skip("LOCAL_MODEL_BASE_URL is not set") + } + modelID := os.Getenv("LOCAL_MODEL_ID") + if modelID == "" { + modelID = "qwen2.5:7b" + } + generator, err := NewCompatibleGenerator("openai_compatible", baseURL, "ollama-local-key", modelID, 0.1, 1200, 2*time.Minute) + if err != nil { + t.Fatal(err) + } + tests := []struct { + name, content string + meaningful, storyWorth bool + }{ + { + name: "discard small talk", + content: "豆包：您好，今天过得怎么样？\n长者：还行。\n豆包：吃饭了吗？\n长者：嗯，吃了。\n豆包：那您好好休息。\n长者：好，再见。", + }, + { + name: "keep useful update without story", + content: "豆包：今天胃口怎么样？\n长者：今天胃口挺好，中午吃了一碗西红柿鸡蛋面，晚上想吃点清淡的。", + meaningful: true, + }, + { + name: "create story candidate from coherent memory", + content: "豆包：您还记得第一次当老师的情形吗？\n长者：记得。那是1978年秋天，我二十二岁，被分到河边小学。第一天是姐姐骑自行车送我去的，教室里有三十多个孩子。我紧张得把粉笔掉在地上，一个小姑娘帮我捡起来，全班都笑了，我也跟着笑。后来那个小姑娘一直读到大学，很多年后还回来看过我。", + meaningful: true, storyWorth: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + result, generateErr := generator.Generate(ctx, GenerationInput{Content: test.content}) + if generateErr != nil { + t.Fatal(generateErr) + } + if result.Meaningful != test.meaningful || result.StoryWorthy != test.storyWorth || result.AllowStory != test.storyWorth { + t.Fatalf("decision mismatch: meaningful=%v story_worthy=%v allow_story=%v reason=%q summary=%q", result.Meaningful, result.StoryWorthy, result.AllowStory, result.DecisionReason, result.Summary) + } + if !result.Meaningful && (result.Summary != "" || result.StoryContent != "") { + t.Fatalf("discarded conversation produced content: %+v", result) + } + }) + } +} diff --git a/internal/extraction/generator_test.go b/internal/extraction/generator_test.go index 1c5a5f3..913a5ef 100644 --- a/internal/extraction/generator_test.go +++ b/internal/extraction/generator_test.go @@ -20,7 +20,7 @@ func TestArkGeneratorProducesValidatedContent(t *testing.T) { if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body["model"] != "doubao-test" { t.Fatalf("request body=%v err=%v", body, err) } - _ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": `{"summary":"谈到河边小学","keywords":["河边小学","童年"],"open_topic":"下次可以聊聊同学","story_title":"河边小学的回忆","story_content":"小时候在河边小学读书。","fact_content":"曾在河边小学读书","confidence":0.82}`}}}}) + _ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": `{"meaningful":true,"story_worthy":true,"decision_reason":"包含具体的童年求学回忆","summary":"谈到河边小学","keywords":["河边小学","童年"],"open_topic":"下次可以聊聊同学","story_title":"河边小学的回忆","story_content":"小时候在河边小学读书。","fact_content":"曾在河边小学读书","confidence":0.82}`}}}}) })) defer server.Close() generator, err := NewArkGenerator(server.URL+"/api/v3", "test-key", "doubao-test", time.Second) @@ -33,6 +33,51 @@ func TestArkGeneratorProducesValidatedContent(t *testing.T) { } } +func TestCompatibleGeneratorDiscardsSmallTalk(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": `{"meaningful":false,"story_worthy":false,"decision_reason":"只有问候和确认","summary":"","keywords":[],"open_topic":"","story_title":"","story_content":"","fact_content":"","confidence":0.96}`}}}}) + })) + defer server.Close() + generator, err := NewCompatibleGenerator("openai_compatible", server.URL+"/v1", "local-key", "qwen-test", 0.1, 512, time.Second) + if err != nil { + t.Fatal(err) + } + result, err := generator.Generate(context.Background(), GenerationInput{Content: "豆包：您好。\n长者：嗯，你好。\n豆包：今天怎么样？\n长者：还行。"}) + if err != nil || result.Meaningful || result.StoryWorthy || result.AllowStory || result.DecisionReason == "" { + t.Fatalf("result=%+v err=%v", result, err) + } +} + +func TestCompatibleGeneratorKeepsMeaningfulNonStoryWithoutDraft(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": `{"meaningful":true,"story_worthy":false,"decision_reason":"有明确近况但没有事件脉络","summary":"长者今天胃口不错","keywords":["胃口","近况"],"open_topic":"下次可以聊聊最近喜欢的饭菜","story_title":"","story_content":"","fact_content":"","confidence":0.87}`}}}}) + })) + defer server.Close() + generator, err := NewCompatibleGenerator("openai_compatible", server.URL+"/v1", "local-key", "qwen-test", 0.1, 512, time.Second) + if err != nil { + t.Fatal(err) + } + result, err := generator.Generate(context.Background(), GenerationInput{Content: "长者：我今天胃口挺好的，中午吃了面条。"}) + if err != nil || !result.Meaningful || result.StoryWorthy || result.AllowStory || result.Summary == "" { + t.Fatalf("result=%+v err=%v", result, err) + } +} + +func TestCompatibleGeneratorClearsContradictoryStoryFields(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": `{"meaningful":true,"story_worthy":false,"decision_reason":"只是近况","summary":"今天胃口不错","keywords":["胃口"],"open_topic":"最近喜欢吃什么？","story_title":"模型误填标题","story_content":"模型误填故事","fact_content":"模型误填事实","confidence":0.8}`}}}}) + })) + defer server.Close() + generator, err := NewCompatibleGenerator("openai_compatible", server.URL+"/v1", "local-key", "qwen-test", 0.1, 512, time.Second) + if err != nil { + t.Fatal(err) + } + result, err := generator.Generate(context.Background(), GenerationInput{Content: "长者：今天胃口不错。"}) + if err != nil || result.AllowStory || result.StoryTitle != "" || result.StoryContent != "" || result.FactContent != "" { + t.Fatalf("result=%+v err=%v", result, err) + } +} + func TestArkGeneratorRetriesTemporaryFailure(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "busy", http.StatusTooManyRequests) diff --git a/internal/extraction/processor.go b/internal/extraction/processor.go index e3579dc..c9cae3f 100644 --- a/internal/extraction/processor.go +++ b/internal/extraction/processor.go @@ -93,7 +93,7 @@ func (p *Processor) Handle(ctx context.Context, event outbox.Event) error { if len(sourceTurns) == 0 { return outbox.Retryable(ErrConversationContentNotReady) } - content := joinedTurnContent(sourceTurns) + content := joinedConversationContent(snapshot.turns) generated, err := p.generateSafely(ctx, content, snapshot.tenantID) if err != nil { return err @@ -247,12 +247,16 @@ func subjectTurns(turns []turn) []turn { return items } -func joinedTurnContent(turns []turn) string { +func joinedConversationContent(turns []turn) string { parts := make([]string, 0, len(turns)) for _, item := range turns { - parts = append(parts, item.text) + label := "豆包" + if item.speaker == "subject" { + label = "长者" + } + parts = append(parts, label+"："+item.text) } - return strings.Join(parts, " ") + return strings.Join(parts, "\n") } func storeGeneration(ctx context.Context, tx pgx.Tx, conversationID string, snapshot extractionSnapshot, generated GenerationResult, generationVersion int64) error { @@ -262,6 +266,15 @@ func storeGeneration(ctx context.Context, tx pgx.Tx, conversationID string, snap sourcePublicIDs = append(sourcePublicIDs, item.publicID) } sourceJSON, _ := json.Marshal(sourcePublicIDs) + if _, err := tx.Exec(ctx, ` + INSERT INTO conversation_content_assessment (tenant_id,subject_id,conversation_id,generation_version,meaningful,story_worthy,decision_reason,model_ref,prompt_version,confidence) + VALUES ($1::uuid,$2::uuid,$3::uuid,$4,$5,$6,$7,$8,$9,$10) + ON CONFLICT (conversation_id,generation_version) DO NOTHING`, snapshot.tenantID, snapshot.subjectID, conversationID, generationVersion, generated.Meaningful, generated.StoryWorthy, generated.DecisionReason, generated.ModelRef, generated.PromptVersion, generated.Confidence); err != nil { + return fmt.Errorf("store conversation content assessment: %w", err) + } + if !generated.Meaningful { + return tx.Commit(ctx) + } if _, err := tx.Exec(ctx, ` INSERT INTO conversation_summary (tenant_id,subject_id,conversation_id,summary_type,content,source_turn_ids,model_ref,prompt_version,generation_version,confidence,status) VALUES ($1::uuid,$2::uuid,$3::uuid,'conversation_summary',$4,$5::jsonb,$6,$7,$8,$9,'READY') diff --git a/internal/extraction/processor_integration_test.go b/internal/extraction/processor_integration_test.go index 8125d8b..8d81ea4 100644 --- a/internal/extraction/processor_integration_test.go +++ b/internal/extraction/processor_integration_test.go @@ -40,6 +40,7 @@ func TestProcessorExtractionIdempotencyAndSensitiveContent(t *testing.T) { contextID, contextPublicID := uuid.NewString(), uuid.NewString() conversationID, conversationPublicID := uuid.NewString(), uuid.NewString() defer func() { + _, _ = pool.Exec(ctx, `DELETE FROM conversation_content_assessment WHERE conversation_id=$1::uuid`, conversationID) _, _ = pool.Exec(ctx, `DELETE FROM fact_candidate WHERE story_id IN (SELECT id FROM story_draft WHERE conversation_id=$1::uuid)`, conversationID) _, _ = pool.Exec(ctx, `DELETE FROM story_source WHERE story_revision_id IN (SELECT sr.id FROM story_revision sr JOIN story_draft sd ON sd.id=sr.story_id WHERE sd.conversation_id=$1::uuid)`, conversationID) _, _ = pool.Exec(ctx, `UPDATE story_draft SET current_revision_id=NULL WHERE conversation_id=$1::uuid`, conversationID) @@ -97,7 +98,7 @@ func TestProcessorExtractionIdempotencyAndSensitiveContent(t *testing.T) { http.Error(w, "database lock", http.StatusServiceUnavailable) return } - _, _ = io.WriteString(w, `{"choices":[{"message":{"content":"{\"summary\":\"长者谈到在河边小学教书\",\"keywords\":[\"河边小学\",\"教书经历\"],\"open_topic\":\"下次可以聊聊当年的学生\",\"story_title\":\"河边小学的回忆\",\"story_content\":\"我年轻时在河边小学教书。\",\"fact_content\":\"年轻时曾在河边小学教书\",\"confidence\":0.88}"}}]}`) + _, _ = io.WriteString(w, `{"choices":[{"message":{"content":"{\"meaningful\":true,\"story_worthy\":true,\"decision_reason\":\"包含具体的教书经历\",\"summary\":\"长者谈到在河边小学教书\",\"keywords\":[\"河边小学\",\"教书经历\"],\"open_topic\":\"下次可以聊聊当年的学生\",\"story_title\":\"河边小学的回忆\",\"story_content\":\"我年轻时在河边小学教书。\",\"fact_content\":\"年轻时曾在河边小学教书\",\"confidence\":0.88}"}}]}`) })) defer server.Close() arkGenerator, err := NewArkGenerator(server.URL+"/api/v3", "integration-key", "doubao-integration", 2*time.Second) diff --git a/internal/extraction/processor_safety_test.go b/internal/extraction/processor_safety_test.go index 6c23dd4..91ab6eb 100644 --- a/internal/extraction/processor_safety_test.go +++ b/internal/extraction/processor_safety_test.go @@ -12,6 +12,7 @@ type safetyTestGenerator struct{ calls int } func (g *safetyTestGenerator) Generate(context.Context, GenerationInput) (GenerationResult, error) { g.calls++ return GenerationResult{ + Meaningful: true, StoryWorthy: true, DecisionReason: "测试故事", Summary: "模型摘要", OpenTopic: "开放话题", StoryTitle: "标题", StoryContent: "模型输出风险", FactContent: "普通事实", AllowStory: true, }, nil diff --git a/internal/mockprovider/server.go b/internal/mockprovider/server.go index f5f1fb3..a02804c 100644 --- a/internal/mockprovider/server.go +++ b/internal/mockprovider/server.go @@ -391,7 +391,7 @@ func (s *Server) sendFinalTranscript(input voiceprovider.StartRequest) { TaskRef: input.TaskRef, ConversationID: input.ConversationID, EventType: "voice_chat.transcript.final", Status: "FINAL", OccurredAt: s.now().UTC(), Turns: []voiceprovider.FinalTurn{ - {TurnKey: "turn-1", SequenceNo: 1, Speaker: "subject", StartAt: base, EndAt: base.Add(6 * time.Second), FinalText: "我年轻时常在家乡河边散步，春天的柳树特别好看。", ProviderRef: "mock-turn-1"}, + {TurnKey: "turn-1", SequenceNo: 1, Speaker: "subject", StartAt: base, EndAt: base.Add(6 * time.Second), FinalText: "我年轻时常和姐姐在家乡河边散步。有一次春天突然下雨，我们躲在老柳树下，后来一路笑着跑回家，现在想起来还是很高兴。", ProviderRef: "mock-turn-1"}, {TurnKey: "turn-2", SequenceNo: 2, Speaker: "agent", StartAt: base.Add(7 * time.Second), EndAt: base.Add(13 * time.Second), FinalText: "听起来那是一段很温暖的回忆，下次我们可以继续聊聊家乡的春天。", ProviderRef: "mock-turn-2"}, }, } diff --git a/internal/privacy/delete_processor.go b/internal/privacy/delete_processor.go index b441144..84ba1cf 100644 --- a/internal/privacy/delete_processor.go +++ b/internal/privacy/delete_processor.go @@ -266,6 +266,7 @@ func (p *DeleteProcessor) redactPostgreSQL(ctx context.Context, request deletion `UPDATE subject_fact SET content='[已按隐私请求删除]',source_ref=NULL,confirmation_status='superseded',valid_to=COALESCE(valid_to,now()),version=version+1,updated_at=now() WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`, `UPDATE conversation_turn SET final_text='[已按隐私请求删除]',provider_ref=NULL WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`, `UPDATE conversation_summary SET content='[已按隐私请求删除]',source_turn_ids='["privacy-redacted"]'::jsonb,status='SUPERSEDED',last_error=NULL WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`, + `UPDATE conversation_content_assessment SET decision_reason='[privacy-redacted]',model_ref='privacy-redacted',prompt_version='privacy-redacted' WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`, `UPDATE story_revision sr SET content='[已按隐私请求删除]',edit_reason=NULL,model_ref=NULL,prompt_version=NULL FROM story_draft sd WHERE sr.story_id=sd.id AND sd.tenant_id=$1::uuid AND sd.subject_id=$2::uuid`, `UPDATE story_draft SET title='[已按隐私请求删除]',status='SUPERSEDED',review_reason=NULL,version=version+1,updated_at=now() WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`, `UPDATE fact_candidate SET content='[已按隐私请求删除]',time_range='{}'::jsonb,source_turn_ids='["privacy-redacted"]'::jsonb,status='SUPERSEDED',review_reason=NULL,version=version+1,updated_at=now() WHERE tenant_id=$1::uuid AND subject_id=$2::uuid`, diff --git a/migrations/000044_conversation_content_assessment.down.sql b/migrations/000044_conversation_content_assessment.down.sql new file mode 100644 index 0000000..39f332f --- /dev/null +++ b/migrations/000044_conversation_content_assessment.down.sql @@ -0,0 +1,5 @@ +BEGIN; + +DROP TABLE IF EXISTS conversation_content_assessment; + +COMMIT; diff --git a/migrations/000044_conversation_content_assessment.up.sql b/migrations/000044_conversation_content_assessment.up.sql new file mode 100644 index 0000000..3e4f997 --- /dev/null +++ b/migrations/000044_conversation_content_assessment.up.sql @@ -0,0 +1,28 @@ +BEGIN; + +CREATE TABLE conversation_content_assessment ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + public_id uuid NOT NULL UNIQUE DEFAULT gen_random_uuid(), + tenant_id uuid NOT NULL REFERENCES tenant(id), + subject_id uuid NOT NULL, + conversation_id uuid NOT NULL, + generation_version bigint NOT NULL CHECK (generation_version > 0), + meaningful boolean NOT NULL, + story_worthy boolean NOT NULL, + decision_reason varchar(500) NOT NULL CHECK (length(decision_reason) > 0), + model_ref varchar(160) NOT NULL, + prompt_version varchar(80) NOT NULL, + confidence numeric(5,4) NOT NULL CHECK (confidence BETWEEN 0 AND 1), + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT conversation_content_assessment_decision_check CHECK (meaningful OR NOT story_worthy), + CONSTRAINT conversation_content_assessment_conversation_tenant_fk FOREIGN KEY (conversation_id, tenant_id) + REFERENCES conversation (id, tenant_id), + CONSTRAINT conversation_content_assessment_subject_tenant_fk FOREIGN KEY (subject_id, tenant_id) + REFERENCES subject_profile (id, tenant_id), + CONSTRAINT conversation_content_assessment_generation_key UNIQUE (conversation_id, generation_version) +); + +CREATE INDEX conversation_content_assessment_conversation_idx + ON conversation_content_assessment (conversation_id, generation_version DESC); + +COMMIT; diff --git a/scripts/smoke-conversation.ps1 b/scripts/smoke-conversation.ps1 index 5c31883..4072290 100644 --- a/scripts/smoke-conversation.ps1 +++ b/scripts/smoke-conversation.ps1 @@ -98,6 +98,7 @@ DELETE FROM story_source WHERE story_revision_id IN (SELECT id FROM story_revisi UPDATE story_draft SET current_revision_id=NULL WHERE id IN (SELECT id FROM smoke_story_cleanup); DELETE FROM story_revision WHERE story_id IN (SELECT id FROM smoke_story_cleanup); DELETE FROM story_draft WHERE id IN (SELECT id FROM smoke_story_cleanup); +DELETE FROM conversation_content_assessment WHERE conversation_id IN (SELECT id FROM smoke_cleanup); DELETE FROM conversation_summary WHERE conversation_id IN (SELECT id FROM smoke_cleanup); DELETE FROM conversation_turn WHERE conversation_id IN (SELECT id FROM smoke_cleanup); DELETE FROM provider_event WHERE conversation_id IN (SELECT id FROM smoke_cleanup);