feat: filter meaningful conversations with local model

This commit is contained in:
Codex Local
2026-08-30 15:20:08 +08:00
parent 89f6641331
commit 522785eda8
18 changed files with 451 additions and 45 deletions
+129 -33
View File
@@ -23,16 +23,19 @@ type GenerationInput struct {
}
type GenerationResult struct {
Summary string `json:"summary"`
OpenTopic string `json:"open_topic"`
StoryTitle string `json:"story_title"`
StoryContent string `json:"story_content"`
FactContent string `json:"fact_content"`
Keywords []string `json:"keywords"`
Confidence float64 `json:"confidence"`
ModelRef string `json:"-"`
PromptVersion string `json:"-"`
AllowStory bool `json:"-"`
Meaningful bool `json:"meaningful"`
StoryWorthy bool `json:"story_worthy"`
DecisionReason string `json:"decision_reason"`
Summary string `json:"summary"`
OpenTopic string `json:"open_topic"`
StoryTitle string `json:"story_title"`
StoryContent string `json:"story_content"`
FactContent string `json:"fact_content"`
Keywords []string `json:"keywords"`
Confidence float64 `json:"confidence"`
ModelRef string `json:"-"`
PromptVersion string `json:"-"`
AllowStory bool `json:"-"`
}
type Generator interface {
@@ -46,17 +49,33 @@ func (DeterministicGenerator) Generate(_ context.Context, input GenerationInput)
if content == "" {
return GenerationResult{}, errors.New("extraction content is required")
}
meaningful, storyWorthy, reason := deterministicContentDecision(content)
if !meaningful {
return GenerationResult{
Meaningful: false, StoryWorthy: false, DecisionReason: reason,
Confidence: 0.9, ModelRef: "local-deterministic-extractor", PromptVersion: "content-value-v1",
}, nil
}
storyTitle, storyContent, factContent := "", "", ""
if storyWorthy {
storyTitle = "关于“" + truncateRunes(content, 24) + "”的回忆"
storyContent = content
factContent = content
}
return GenerationResult{
Summary: "长者在本次通话中谈到:" + content,
OpenTopic: "下次可以继续询问长者刚才提到的经历,并让长者自行补充人物、时间和地点。",
StoryTitle: "关于“" + truncateRunes(content, 24) + "”的回忆",
StoryContent: content,
FactContent: content,
Keywords: deterministicKeywords(content),
Confidence: 0.9,
ModelRef: "local-deterministic-extractor",
PromptVersion: "story-v1",
AllowStory: true,
Meaningful: true,
StoryWorthy: storyWorthy,
DecisionReason: reason,
Summary: "长者在本次通话中谈到:" + content,
OpenTopic: "下次可以继续询问长者刚才提到的经历,并让长者自行补充人物、时间和地点。",
StoryTitle: storyTitle,
StoryContent: storyContent,
FactContent: factContent,
Keywords: deterministicKeywords(content),
Confidence: 0.9,
ModelRef: "local-deterministic-extractor",
PromptVersion: "content-value-deterministic-v2",
AllowStory: storyWorthy,
}, nil
}
@@ -67,12 +86,15 @@ func sensitiveGeneration(decision contentpolicy.Decision) GenerationResult {
}
reason := safePolicyToken(decision.Reason)
return GenerationResult{
Summary: "本次通话包含需要人工谨慎处理的敏感信息,系统未生成普通故事卡片。",
OpenTopic: "如确有必要处理本次敏感内容,应由有权限人员在受控流程中单独核对。",
Confidence: 1,
ModelRef: modelRef,
PromptVersion: "summary-sensitive-filter-v1:" + reason,
AllowStory: false,
Meaningful: true,
StoryWorthy: false,
DecisionReason: "safety_policy_blocked_story",
Summary: "本次通话包含需要人工谨慎处理的敏感信息,系统未生成普通故事卡片。",
OpenTopic: "如确有必要处理本次敏感内容,应由有权限人员在受控流程中单独核对。",
Confidence: 1,
ModelRef: modelRef,
PromptVersion: "summary-sensitive-filter-v1:" + reason,
AllowStory: false,
}
}
@@ -133,7 +155,7 @@ func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (Gen
requestBody := map[string]any{
"model": g.model,
"messages": []map[string]string{
{"role": "system", "content": "你是长者口述内容整理助手。只依据输入字幕生成可人工核对的候选,不补充未出现的事实,不把推断写成事实。必须返回JSON对象,字段为summary、keywords、open_topic、story_title、story_content、fact_content、confidence。summary概括本次内容,keywords是1至8个简短关键词字符串组成的数组,open_topic给出一个温和的后续话题,story和fact保留原意,confidence取0到1。"},
{"role": "system", "content": contentValuePrompt},
{"role": "user", "content": content},
},
"temperature": g.temperature,
@@ -186,30 +208,75 @@ func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (Gen
return GenerationResult{}, outbox.Retryable(fmt.Errorf("decode Ark structured content: %w", err))
}
result.Summary = strings.TrimSpace(result.Summary)
result.DecisionReason = strings.TrimSpace(result.DecisionReason)
result.OpenTopic = strings.TrimSpace(result.OpenTopic)
result.StoryTitle = strings.TrimSpace(result.StoryTitle)
result.StoryContent = strings.TrimSpace(result.StoryContent)
result.FactContent = strings.TrimSpace(result.FactContent)
result.Keywords = normalizedKeywords(result.Keywords)
normalizeDecisionContent(&result)
if !validGeneratedContent(result) {
return GenerationResult{}, outbox.Retryable(errors.New("Ark structured content failed validation"))
}
result.ModelRef = g.provider + ":" + g.model
result.PromptVersion = "story-" + g.provider + "-json-v2"
result.AllowStory = true
result.PromptVersion = "content-value-" + g.provider + "-json-v3"
result.AllowStory = result.Meaningful && result.StoryWorthy
return result, nil
}
func normalizeDecisionContent(result *GenerationResult) {
if !result.Meaningful {
result.StoryWorthy = false
result.Summary = ""
result.OpenTopic = ""
result.StoryTitle = ""
result.StoryContent = ""
result.FactContent = ""
result.Keywords = nil
return
}
if !result.StoryWorthy {
result.StoryTitle = ""
result.StoryContent = ""
result.FactContent = ""
}
}
func validGeneratedContent(result GenerationResult) bool {
return result.Summary != "" && len([]rune(result.Summary)) <= 2000 &&
result.OpenTopic != "" && len([]rune(result.OpenTopic)) <= 1000 &&
result.StoryTitle != "" && len([]rune(result.StoryTitle)) <= 300 &&
if result.Confidence < 0 || result.Confidence > 1 || result.DecisionReason == "" || len([]rune(result.DecisionReason)) > 500 {
return false
}
if !result.Meaningful {
return !result.StoryWorthy && result.Summary == "" && result.OpenTopic == "" && result.StoryTitle == "" && result.StoryContent == "" && result.FactContent == "" && len(result.Keywords) == 0
}
if result.Summary == "" || len([]rune(result.Summary)) > 2000 ||
result.OpenTopic == "" || len([]rune(result.OpenTopic)) > 1000 ||
len(result.Keywords) < 1 || len(result.Keywords) > 8 {
return false
}
if !result.StoryWorthy {
return result.StoryTitle == "" && result.StoryContent == "" && result.FactContent == ""
}
return result.StoryTitle != "" && len([]rune(result.StoryTitle)) <= 300 &&
result.StoryContent != "" && len([]rune(result.StoryContent)) <= 20000 &&
result.FactContent != "" && len([]rune(result.FactContent)) <= 5000 &&
len(result.Keywords) >= 1 && len(result.Keywords) <= 8 &&
result.Confidence >= 0 && result.Confidence <= 1
}
const contentValuePrompt = `你是长者聊天内容筛选与整理助手。只依据输入对话判断和整理,不补充未出现的事实,不把推断写成事实。
先做两级判断:
1. meaningful:对话是否包含值得保留的信息。只有问候、寒暄、嗯啊、测试设备、重复句、零散且无法理解的短句,必须判为false并舍弃,不生成摘要。
2. story_worthy:仅在meaningful为true时判断是否值得形成故事。具体的个人经历或回忆,且人物、时间、地点、事件、行动、感受中至少出现两类,或形成了清楚的事件脉络,才判为true。普通偏好、单个事实或简短近况可以总结,但story_worthy应为false。
必须只返回JSON对象,字段为meaningful、story_worthy、decision_reason、summary、keywords、open_topic、story_title、story_content、fact_content、confidence。
- decision_reason用一句简短中文说明判断依据。
- meaningful=false时,除decision_reason和confidence外,其余文本必须为空字符串,keywords必须为空数组,story_worthy必须为false。
- meaningful=true时,summary概括有意义内容,keywords为1至8个简短关键词,open_topic给出一个温和且不重复追问的后续话题。
- story_worthy=false时,story_title、story_content、fact_content必须为空字符串。
- story_worthy=true时,story_title和story_content忠实整理原话,fact_content只写对话中可核对的事实。
- confidence取0到1。`
func normalizedKeywords(values []string) []string {
result := make([]string, 0, min(len(values), 8))
seen := make(map[string]struct{}, len(values))
@@ -255,3 +322,32 @@ func deterministicKeywords(content string) []string {
}
return []string{string(runes)}
}
func deterministicContentDecision(content string) (meaningful, storyWorthy bool, reason string) {
compact := strings.Map(func(character rune) rune {
if unicode.IsSpace(character) || strings.ContainsRune(",。!?;、,.!?;::\"“”‘’()()-", character) {
return -1
}
return character
}, content)
for _, label := range []string{"长者", "豆包", "助手", "用户", "agent", "subject"} {
compact = strings.ReplaceAll(compact, label, "")
}
for _, phrase := range []string{"你好", "您好", "在吗", "嗯", "啊", "哦", "好的", "知道了", "没事", "再见", "拜拜", "听得到吗", "能听见吗", "测试"} {
compact = strings.ReplaceAll(compact, phrase, "")
}
if len([]rune(compact)) < 12 {
return false, false, "内容仅为寒暄、确认或零散短句"
}
storySignals := 0
for _, phrase := range []string{"小时候", "年轻时", "那年", "当时", "后来", "以前", "记得", "有一次", "我们", "我和", "去了", "发生", "因为", "结果", "高兴", "难过", "想起"} {
if strings.Contains(content, phrase) {
storySignals++
}
}
storyWorthy = len([]rune(compact)) >= 48 && storySignals >= 2
if storyWorthy {
return true, true, "包含较完整的个人经历与事件脉络"
}
return true, false, "包含可保留的信息,但尚不足以形成完整故事"
}
@@ -0,0 +1,58 @@
package extraction
import (
"context"
"os"
"testing"
"time"
)
func TestLocalModelContentValueDecisions(t *testing.T) {
baseURL := os.Getenv("LOCAL_MODEL_BASE_URL")
if baseURL == "" {
t.Skip("LOCAL_MODEL_BASE_URL is not set")
}
modelID := os.Getenv("LOCAL_MODEL_ID")
if modelID == "" {
modelID = "qwen2.5:7b"
}
generator, err := NewCompatibleGenerator("openai_compatible", baseURL, "ollama-local-key", modelID, 0.1, 1200, 2*time.Minute)
if err != nil {
t.Fatal(err)
}
tests := []struct {
name, content string
meaningful, storyWorth bool
}{
{
name: "discard small talk",
content: "豆包:您好,今天过得怎么样?\n长者:还行。\n豆包:吃饭了吗?\n长者:嗯,吃了。\n豆包:那您好好休息。\n长者:好,再见。",
},
{
name: "keep useful update without story",
content: "豆包:今天胃口怎么样?\n长者:今天胃口挺好,中午吃了一碗西红柿鸡蛋面,晚上想吃点清淡的。",
meaningful: true,
},
{
name: "create story candidate from coherent memory",
content: "豆包:您还记得第一次当老师的情形吗?\n长者:记得。那是1978年秋天,我二十二岁,被分到河边小学。第一天是姐姐骑自行车送我去的,教室里有三十多个孩子。我紧张得把粉笔掉在地上,一个小姑娘帮我捡起来,全班都笑了,我也跟着笑。后来那个小姑娘一直读到大学,很多年后还回来看过我。",
meaningful: true, storyWorth: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
result, generateErr := generator.Generate(ctx, GenerationInput{Content: test.content})
if generateErr != nil {
t.Fatal(generateErr)
}
if result.Meaningful != test.meaningful || result.StoryWorthy != test.storyWorth || result.AllowStory != test.storyWorth {
t.Fatalf("decision mismatch: meaningful=%v story_worthy=%v allow_story=%v reason=%q summary=%q", result.Meaningful, result.StoryWorthy, result.AllowStory, result.DecisionReason, result.Summary)
}
if !result.Meaningful && (result.Summary != "" || result.StoryContent != "") {
t.Fatalf("discarded conversation produced content: %+v", result)
}
})
}
}
+46 -1
View File
@@ -20,7 +20,7 @@ func TestArkGeneratorProducesValidatedContent(t *testing.T) {
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body["model"] != "doubao-test" {
t.Fatalf("request body=%v err=%v", body, err)
}
_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": `{"summary":"谈到河边小学","keywords":["河边小学","童年"],"open_topic":"下次可以聊聊同学","story_title":"河边小学的回忆","story_content":"小时候在河边小学读书。","fact_content":"曾在河边小学读书","confidence":0.82}`}}}})
_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": `{"meaningful":true,"story_worthy":true,"decision_reason":"包含具体的童年求学回忆","summary":"谈到河边小学","keywords":["河边小学","童年"],"open_topic":"下次可以聊聊同学","story_title":"河边小学的回忆","story_content":"小时候在河边小学读书。","fact_content":"曾在河边小学读书","confidence":0.82}`}}}})
}))
defer server.Close()
generator, err := NewArkGenerator(server.URL+"/api/v3", "test-key", "doubao-test", time.Second)
@@ -33,6 +33,51 @@ func TestArkGeneratorProducesValidatedContent(t *testing.T) {
}
}
func TestCompatibleGeneratorDiscardsSmallTalk(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": `{"meaningful":false,"story_worthy":false,"decision_reason":"只有问候和确认","summary":"","keywords":[],"open_topic":"","story_title":"","story_content":"","fact_content":"","confidence":0.96}`}}}})
}))
defer server.Close()
generator, err := NewCompatibleGenerator("openai_compatible", server.URL+"/v1", "local-key", "qwen-test", 0.1, 512, time.Second)
if err != nil {
t.Fatal(err)
}
result, err := generator.Generate(context.Background(), GenerationInput{Content: "豆包:您好。\n长者:嗯,你好。\n豆包:今天怎么样?\n长者:还行。"})
if err != nil || result.Meaningful || result.StoryWorthy || result.AllowStory || result.DecisionReason == "" {
t.Fatalf("result=%+v err=%v", result, err)
}
}
func TestCompatibleGeneratorKeepsMeaningfulNonStoryWithoutDraft(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": `{"meaningful":true,"story_worthy":false,"decision_reason":"有明确近况但没有事件脉络","summary":"长者今天胃口不错","keywords":["胃口","近况"],"open_topic":"下次可以聊聊最近喜欢的饭菜","story_title":"","story_content":"","fact_content":"","confidence":0.87}`}}}})
}))
defer server.Close()
generator, err := NewCompatibleGenerator("openai_compatible", server.URL+"/v1", "local-key", "qwen-test", 0.1, 512, time.Second)
if err != nil {
t.Fatal(err)
}
result, err := generator.Generate(context.Background(), GenerationInput{Content: "长者:我今天胃口挺好的,中午吃了面条。"})
if err != nil || !result.Meaningful || result.StoryWorthy || result.AllowStory || result.Summary == "" {
t.Fatalf("result=%+v err=%v", result, err)
}
}
func TestCompatibleGeneratorClearsContradictoryStoryFields(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": `{"meaningful":true,"story_worthy":false,"decision_reason":"只是近况","summary":"今天胃口不错","keywords":["胃口"],"open_topic":"最近喜欢吃什么?","story_title":"模型误填标题","story_content":"模型误填故事","fact_content":"模型误填事实","confidence":0.8}`}}}})
}))
defer server.Close()
generator, err := NewCompatibleGenerator("openai_compatible", server.URL+"/v1", "local-key", "qwen-test", 0.1, 512, time.Second)
if err != nil {
t.Fatal(err)
}
result, err := generator.Generate(context.Background(), GenerationInput{Content: "长者:今天胃口不错。"})
if err != nil || result.AllowStory || result.StoryTitle != "" || result.StoryContent != "" || result.FactContent != "" {
t.Fatalf("result=%+v err=%v", result, err)
}
}
func TestArkGeneratorRetriesTemporaryFailure(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
http.Error(w, "busy", http.StatusTooManyRequests)
+17 -4
View File
@@ -93,7 +93,7 @@ func (p *Processor) Handle(ctx context.Context, event outbox.Event) error {
if len(sourceTurns) == 0 {
return outbox.Retryable(ErrConversationContentNotReady)
}
content := joinedTurnContent(sourceTurns)
content := joinedConversationContent(snapshot.turns)
generated, err := p.generateSafely(ctx, content, snapshot.tenantID)
if err != nil {
return err
@@ -247,12 +247,16 @@ func subjectTurns(turns []turn) []turn {
return items
}
func joinedTurnContent(turns []turn) string {
func joinedConversationContent(turns []turn) string {
parts := make([]string, 0, len(turns))
for _, item := range turns {
parts = append(parts, item.text)
label := "豆包"
if item.speaker == "subject" {
label = "长者"
}
parts = append(parts, label+":"+item.text)
}
return strings.Join(parts, " ")
return strings.Join(parts, "\n")
}
func storeGeneration(ctx context.Context, tx pgx.Tx, conversationID string, snapshot extractionSnapshot, generated GenerationResult, generationVersion int64) error {
@@ -262,6 +266,15 @@ func storeGeneration(ctx context.Context, tx pgx.Tx, conversationID string, snap
sourcePublicIDs = append(sourcePublicIDs, item.publicID)
}
sourceJSON, _ := json.Marshal(sourcePublicIDs)
if _, err := tx.Exec(ctx, `
INSERT INTO conversation_content_assessment (tenant_id,subject_id,conversation_id,generation_version,meaningful,story_worthy,decision_reason,model_ref,prompt_version,confidence)
VALUES ($1::uuid,$2::uuid,$3::uuid,$4,$5,$6,$7,$8,$9,$10)
ON CONFLICT (conversation_id,generation_version) DO NOTHING`, snapshot.tenantID, snapshot.subjectID, conversationID, generationVersion, generated.Meaningful, generated.StoryWorthy, generated.DecisionReason, generated.ModelRef, generated.PromptVersion, generated.Confidence); err != nil {
return fmt.Errorf("store conversation content assessment: %w", err)
}
if !generated.Meaningful {
return tx.Commit(ctx)
}
if _, err := tx.Exec(ctx, `
INSERT INTO conversation_summary (tenant_id,subject_id,conversation_id,summary_type,content,source_turn_ids,model_ref,prompt_version,generation_version,confidence,status)
VALUES ($1::uuid,$2::uuid,$3::uuid,'conversation_summary',$4,$5::jsonb,$6,$7,$8,$9,'READY')
@@ -40,6 +40,7 @@ func TestProcessorExtractionIdempotencyAndSensitiveContent(t *testing.T) {
contextID, contextPublicID := uuid.NewString(), uuid.NewString()
conversationID, conversationPublicID := uuid.NewString(), uuid.NewString()
defer func() {
_, _ = pool.Exec(ctx, `DELETE FROM conversation_content_assessment WHERE conversation_id=$1::uuid`, conversationID)
_, _ = pool.Exec(ctx, `DELETE FROM fact_candidate WHERE story_id IN (SELECT id FROM story_draft WHERE conversation_id=$1::uuid)`, conversationID)
_, _ = pool.Exec(ctx, `DELETE FROM story_source WHERE story_revision_id IN (SELECT sr.id FROM story_revision sr JOIN story_draft sd ON sd.id=sr.story_id WHERE sd.conversation_id=$1::uuid)`, conversationID)
_, _ = pool.Exec(ctx, `UPDATE story_draft SET current_revision_id=NULL WHERE conversation_id=$1::uuid`, conversationID)
@@ -97,7 +98,7 @@ func TestProcessorExtractionIdempotencyAndSensitiveContent(t *testing.T) {
http.Error(w, "database lock", http.StatusServiceUnavailable)
return
}
_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"{\"summary\":\"长者谈到在河边小学教书\",\"keywords\":[\"河边小学\",\"教书经历\"],\"open_topic\":\"下次可以聊聊当年的学生\",\"story_title\":\"河边小学的回忆\",\"story_content\":\"我年轻时在河边小学教书。\",\"fact_content\":\"年轻时曾在河边小学教书\",\"confidence\":0.88}"}}]}`)
_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"{\"meaningful\":true,\"story_worthy\":true,\"decision_reason\":\"包含具体的教书经历\",\"summary\":\"长者谈到在河边小学教书\",\"keywords\":[\"河边小学\",\"教书经历\"],\"open_topic\":\"下次可以聊聊当年的学生\",\"story_title\":\"河边小学的回忆\",\"story_content\":\"我年轻时在河边小学教书。\",\"fact_content\":\"年轻时曾在河边小学教书\",\"confidence\":0.88}"}}]}`)
}))
defer server.Close()
arkGenerator, err := NewArkGenerator(server.URL+"/api/v3", "integration-key", "doubao-integration", 2*time.Second)
@@ -12,6 +12,7 @@ type safetyTestGenerator struct{ calls int }
func (g *safetyTestGenerator) Generate(context.Context, GenerationInput) (GenerationResult, error) {
g.calls++
return GenerationResult{
Meaningful: true, StoryWorthy: true, DecisionReason: "测试故事",
Summary: "模型摘要", OpenTopic: "开放话题", StoryTitle: "标题",
StoryContent: "模型输出风险", FactContent: "普通事实", AllowStory: true,
}, nil