chore: establish local platform baseline
This commit is contained in:
Codex Local
@@ -0,0 +1,193 @@
|
||||
package extraction
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/contentpolicy"
|
||||
"data-terminal-cloud/internal/outbox"
|
||||
)
|
||||
|
||||
type GenerationInput struct {
|
||||
Content string
|
||||
}
|
||||
|
||||
type GenerationResult struct {
|
||||
Summary string `json:"summary"`
|
||||
OpenTopic string `json:"open_topic"`
|
||||
StoryTitle string `json:"story_title"`
|
||||
StoryContent string `json:"story_content"`
|
||||
FactContent string `json:"fact_content"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
ModelRef string `json:"-"`
|
||||
PromptVersion string `json:"-"`
|
||||
AllowStory bool `json:"-"`
|
||||
}
|
||||
|
||||
type Generator interface {
|
||||
Generate(context.Context, GenerationInput) (GenerationResult, error)
|
||||
}
|
||||
|
||||
type DeterministicGenerator struct{}
|
||||
|
||||
func (DeterministicGenerator) Generate(_ context.Context, input GenerationInput) (GenerationResult, error) {
|
||||
content := strings.TrimSpace(input.Content)
|
||||
if content == "" {
|
||||
return GenerationResult{}, errors.New("extraction content is required")
|
||||
}
|
||||
return GenerationResult{
|
||||
Summary: "长者在本次通话中谈到：" + content,
|
||||
OpenTopic: "下次可以继续询问长者刚才提到的经历，并让长者自行补充人物、时间和地点。",
|
||||
StoryTitle: "关于“" + truncateRunes(content, 24) + "”的回忆",
|
||||
StoryContent: content,
|
||||
FactContent: content,
|
||||
Confidence: 0.9,
|
||||
ModelRef: "local-deterministic-extractor",
|
||||
PromptVersion: "story-v1",
|
||||
AllowStory: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func sensitiveGeneration(decision contentpolicy.Decision) GenerationResult {
|
||||
modelRef := strings.TrimSpace(decision.ProviderRef)
|
||||
if modelRef == "" {
|
||||
modelRef = "local-content-safety-policy"
|
||||
}
|
||||
reason := safePolicyToken(decision.Reason)
|
||||
return GenerationResult{
|
||||
Summary: "本次通话包含需要人工谨慎处理的敏感信息，系统未生成普通故事卡片。",
|
||||
OpenTopic: "如确有必要处理本次敏感内容，应由有权限人员在受控流程中单独核对。",
|
||||
Confidence: 1,
|
||||
ModelRef: modelRef,
|
||||
PromptVersion: "summary-sensitive-filter-v1:" + reason,
|
||||
AllowStory: false,
|
||||
}
|
||||
}
|
||||
|
||||
func safePolicyToken(value string) string {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
if value == "" || len(value) > 64 {
|
||||
return "policy_block"
|
||||
}
|
||||
for _, character := range value {
|
||||
if (character < 'a' || character > 'z') && (character < '0' || character > '9') && character != '_' && character != '-' {
|
||||
return "policy_block"
|
||||
}
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
type ArkGenerator struct {
|
||||
endpoint string
|
||||
apiKey string
|
||||
model string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewArkGenerator(baseURL, apiKey, model string, timeout time.Duration) (*ArkGenerator, error) {
|
||||
baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
|
||||
apiKey = strings.TrimSpace(apiKey)
|
||||
model = strings.TrimSpace(model)
|
||||
parsed, err := url.Parse(baseURL)
|
||||
if err != nil || parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") {
|
||||
return nil, errors.New("Ark base URL must be an absolute HTTP(S) URL")
|
||||
}
|
||||
if apiKey == "" || model == "" || timeout <= 0 {
|
||||
return nil, errors.New("Ark API key, content model and positive timeout are required")
|
||||
}
|
||||
return &ArkGenerator{
|
||||
endpoint: baseURL + "/chat/completions", apiKey: apiKey, model: model,
|
||||
client: &http.Client{Timeout: timeout},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (GenerationResult, error) {
|
||||
content := strings.TrimSpace(input.Content)
|
||||
if content == "" {
|
||||
return GenerationResult{}, errors.New("extraction content is required")
|
||||
}
|
||||
requestBody := map[string]any{
|
||||
"model": g.model,
|
||||
"messages": []map[string]string{
|
||||
{"role": "system", "content": "你是长者口述内容整理助手。只依据输入字幕生成可人工核对的候选，不补充未出现的事实，不把推断写成事实。必须返回JSON对象，字段为summary、open_topic、story_title、story_content、fact_content、confidence。summary概括本次内容，open_topic给出一个温和的后续话题，story和fact保留原意，confidence取0到1。"},
|
||||
{"role": "user", "content": content},
|
||||
},
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1200,
|
||||
"response_format": map[string]string{"type": "json_object"},
|
||||
}
|
||||
encoded, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return GenerationResult{}, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, g.endpoint, bytes.NewReader(encoded))
|
||||
if err != nil {
|
||||
return GenerationResult{}, fmt.Errorf("build Ark content request: %w", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+g.apiKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", "voice-platform-content-extractor/1.0")
|
||||
resp, err := g.client.Do(req)
|
||||
if err != nil {
|
||||
return GenerationResult{}, outbox.Retryable(fmt.Errorf("call Ark content model: %w", err))
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
responseBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if readErr != nil {
|
||||
return GenerationResult{}, outbox.Retryable(fmt.Errorf("read Ark content response: %w", readErr))
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
requestErr := fmt.Errorf("Ark content model returned HTTP %d", resp.StatusCode)
|
||||
if resp.StatusCode == http.StatusRequestTimeout || resp.StatusCode == http.StatusTooEarly || resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 {
|
||||
return GenerationResult{}, outbox.Retryable(requestErr)
|
||||
}
|
||||
return GenerationResult{}, requestErr
|
||||
}
|
||||
var completion struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
}
|
||||
if err = json.Unmarshal(responseBody, &completion); err != nil || len(completion.Choices) == 0 {
|
||||
return GenerationResult{}, outbox.Retryable(errors.New("Ark content response has no usable choice"))
|
||||
}
|
||||
jsonContent := strings.TrimSpace(completion.Choices[0].Message.Content)
|
||||
jsonContent = strings.TrimPrefix(jsonContent, "```json")
|
||||
jsonContent = strings.TrimPrefix(jsonContent, "```")
|
||||
jsonContent = strings.TrimSuffix(jsonContent, "```")
|
||||
var result GenerationResult
|
||||
if err = json.Unmarshal([]byte(strings.TrimSpace(jsonContent)), &result); err != nil {
|
||||
return GenerationResult{}, outbox.Retryable(fmt.Errorf("decode Ark structured content: %w", err))
|
||||
}
|
||||
result.Summary = strings.TrimSpace(result.Summary)
|
||||
result.OpenTopic = strings.TrimSpace(result.OpenTopic)
|
||||
result.StoryTitle = strings.TrimSpace(result.StoryTitle)
|
||||
result.StoryContent = strings.TrimSpace(result.StoryContent)
|
||||
result.FactContent = strings.TrimSpace(result.FactContent)
|
||||
if !validGeneratedContent(result) {
|
||||
return GenerationResult{}, outbox.Retryable(errors.New("Ark structured content failed validation"))
|
||||
}
|
||||
result.ModelRef = "ark:" + g.model
|
||||
result.PromptVersion = "story-ark-json-v1"
|
||||
result.AllowStory = true
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func validGeneratedContent(result GenerationResult) bool {
|
||||
return result.Summary != "" && len([]rune(result.Summary)) <= 2000 &&
|
||||
result.OpenTopic != "" && len([]rune(result.OpenTopic)) <= 1000 &&
|
||||
result.StoryTitle != "" && len([]rune(result.StoryTitle)) <= 300 &&
|
||||
result.StoryContent != "" && len([]rune(result.StoryContent)) <= 20000 &&
|
||||
result.FactContent != "" && len([]rune(result.FactContent)) <= 5000 &&
|
||||
result.Confidence >= 0 && result.Confidence <= 1
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package extraction
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/outbox"
|
||||
)
|
||||
|
||||
func TestArkGeneratorProducesValidatedContent(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer test-key" || r.URL.Path != "/api/v3/chat/completions" {
|
||||
t.Fatalf("unexpected Ark request path or authorization")
|
||||
}
|
||||
var body map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body["model"] != "doubao-test" {
|
||||
t.Fatalf("request body=%v err=%v", body, err)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": `{"summary":"谈到河边小学","open_topic":"下次可以聊聊同学","story_title":"河边小学的回忆","story_content":"小时候在河边小学读书。","fact_content":"曾在河边小学读书","confidence":0.82}`}}}})
|
||||
}))
|
||||
defer server.Close()
|
||||
generator, err := NewArkGenerator(server.URL+"/api/v3", "test-key", "doubao-test", time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := generator.Generate(context.Background(), GenerationInput{Content: "小时候在河边小学读书。"})
|
||||
if err != nil || result.ModelRef != "ark:doubao-test" || result.Confidence != 0.82 || !result.AllowStory {
|
||||
t.Fatalf("result=%+v err=%v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArkGeneratorRetriesTemporaryFailure(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
http.Error(w, "busy", http.StatusTooManyRequests)
|
||||
}))
|
||||
defer server.Close()
|
||||
generator, err := NewArkGenerator(server.URL, "test-key", "doubao-test", time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = generator.Generate(context.Background(), GenerationInput{Content: "测试内容"})
|
||||
if !outbox.IsRetryable(err) {
|
||||
t.Fatalf("429 should be retryable: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
package extraction
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"data-terminal-cloud/internal/contentpolicy"
|
||||
"data-terminal-cloud/internal/outbox"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
var ErrConversationContentNotReady = errors.New("conversation content is not ready for extraction")
|
||||
var ErrConversationContentChanged = errors.New("conversation content changed while extraction was running")
|
||||
|
||||
type Processor struct {
|
||||
pool *pgxpool.Pool
|
||||
generator Generator
|
||||
safety contentpolicy.Classifier
|
||||
}
|
||||
|
||||
type turn struct {
|
||||
id, publicID, speaker, text string
|
||||
interrupted bool
|
||||
}
|
||||
|
||||
type extractionSnapshot struct {
|
||||
exists bool
|
||||
tenantID, subjectID, status string
|
||||
storyExtractionAllowed bool
|
||||
turns []turn
|
||||
digest string
|
||||
}
|
||||
|
||||
type queryer interface {
|
||||
QueryRow(context.Context, string, ...any) pgx.Row
|
||||
Query(context.Context, string, ...any) (pgx.Rows, error)
|
||||
}
|
||||
|
||||
func NewProcessor(pool *pgxpool.Pool, generators ...Generator) (*Processor, error) {
|
||||
if pool == nil {
|
||||
return nil, errors.New("database pool is required")
|
||||
}
|
||||
var generator Generator = DeterministicGenerator{}
|
||||
if len(generators) > 1 || (len(generators) == 1 && generators[0] == nil) {
|
||||
return nil, errors.New("one valid extraction generator is required")
|
||||
}
|
||||
if len(generators) == 1 {
|
||||
generator = generators[0]
|
||||
}
|
||||
return &Processor{pool: pool, generator: generator, safety: contentpolicy.LocalClassifier{}}, nil
|
||||
}
|
||||
|
||||
func NewProcessorWithSafety(pool *pgxpool.Pool, generator Generator, safety contentpolicy.Classifier) (*Processor, error) {
|
||||
if generator == nil || safety == nil {
|
||||
return nil, errors.New("valid extraction generator and content safety classifier are required")
|
||||
}
|
||||
processor, err := NewProcessor(pool, generator)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
processor.safety = safety
|
||||
return processor, nil
|
||||
}
|
||||
|
||||
func (p *Processor) Handle(ctx context.Context, event outbox.Event) error {
|
||||
snapshot, err := loadExtractionSnapshot(ctx, p.pool, event.AggregateID, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !snapshot.exists {
|
||||
return nil
|
||||
}
|
||||
if snapshot.status != "COMPLETED" && snapshot.status != "FAILED" {
|
||||
return outbox.Retryable(ErrConversationContentNotReady)
|
||||
}
|
||||
if !snapshot.storyExtractionAllowed {
|
||||
return nil
|
||||
}
|
||||
sourceTurns := subjectTurns(snapshot.turns)
|
||||
if len(sourceTurns) == 0 {
|
||||
return outbox.Retryable(ErrConversationContentNotReady)
|
||||
}
|
||||
content := joinedTurnContent(sourceTurns)
|
||||
generated, err := p.generateSafely(ctx, content)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
generationVersion := int64(1)
|
||||
var payload struct {
|
||||
GenerationVersion int64 `json:"generation_version"`
|
||||
}
|
||||
if json.Unmarshal(event.Payload, &payload) == nil && payload.GenerationVersion > 0 {
|
||||
generationVersion = payload.GenerationVersion
|
||||
}
|
||||
|
||||
// The model call above is deliberately outside a database transaction. The
|
||||
// conversation row is locked only for the short revalidation and write phase.
|
||||
tx, err := p.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
current, err := loadExtractionSnapshot(ctx, tx, event.AggregateID, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !current.exists {
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
if !current.storyExtractionAllowed {
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
if current.status != "COMPLETED" && current.status != "FAILED" {
|
||||
return outbox.Retryable(ErrConversationContentNotReady)
|
||||
}
|
||||
if current.digest != snapshot.digest {
|
||||
return outbox.Retryable(ErrConversationContentChanged)
|
||||
}
|
||||
return storeGeneration(ctx, tx, event.AggregateID, current, generated, generationVersion)
|
||||
}
|
||||
|
||||
func (p *Processor) generateSafely(ctx context.Context, content string) (GenerationResult, error) {
|
||||
decision, err := p.safety.Classify(ctx, content)
|
||||
if err != nil {
|
||||
return GenerationResult{}, err
|
||||
}
|
||||
if decision.AllowOrdinaryStory {
|
||||
generated, generateErr := p.generator.Generate(ctx, GenerationInput{Content: content})
|
||||
if generateErr != nil {
|
||||
return GenerationResult{}, generateErr
|
||||
}
|
||||
outputDecision, safetyErr := p.safety.Classify(ctx, generatedSafetyContent(generated))
|
||||
if safetyErr != nil {
|
||||
return GenerationResult{}, safetyErr
|
||||
}
|
||||
if !outputDecision.AllowOrdinaryStory {
|
||||
return sensitiveGeneration(outputDecision), nil
|
||||
}
|
||||
return generated, nil
|
||||
}
|
||||
return sensitiveGeneration(decision), nil
|
||||
}
|
||||
|
||||
func generatedSafetyContent(result GenerationResult) string {
|
||||
return strings.Join([]string{result.Summary, result.OpenTopic, result.StoryTitle, result.StoryContent, result.FactContent}, "\n")
|
||||
}
|
||||
|
||||
func loadExtractionSnapshot(ctx context.Context, q queryer, conversationID string, lock bool) (extractionSnapshot, error) {
|
||||
lockClause := ""
|
||||
if lock {
|
||||
lockClause = " FOR UPDATE OF c"
|
||||
}
|
||||
var snapshot extractionSnapshot
|
||||
var consentsJSON []byte
|
||||
err := q.QueryRow(ctx, `
|
||||
SELECT c.tenant_id::text,c.subject_id::text,c.status,cs.effective_consents
|
||||
FROM conversation c JOIN consent_snapshot cs ON cs.id=c.consent_snapshot_id AND cs.tenant_id=c.tenant_id
|
||||
JOIN subject_profile sp ON sp.id=c.subject_id AND sp.tenant_id=c.tenant_id AND sp.status IN ('active','suspended')
|
||||
WHERE c.id=$1::uuid`+lockClause, conversationID).Scan(&snapshot.tenantID, &snapshot.subjectID, &snapshot.status, &consentsJSON)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return extractionSnapshot{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return extractionSnapshot{}, fmt.Errorf("read extraction conversation: %w", err)
|
||||
}
|
||||
snapshot.exists = true
|
||||
consents := map[string]bool{}
|
||||
if err = json.Unmarshal(consentsJSON, &consents); err != nil {
|
||||
return extractionSnapshot{}, fmt.Errorf("decode extraction consents: %w", err)
|
||||
}
|
||||
snapshot.storyExtractionAllowed = consents["story_extraction"]
|
||||
rows, err := q.Query(ctx, `
|
||||
SELECT id::text,public_id::text,speaker,final_text,interrupted
|
||||
FROM conversation_turn WHERE conversation_id=$1::uuid ORDER BY sequence_no,id`, conversationID)
|
||||
if err != nil {
|
||||
return extractionSnapshot{}, fmt.Errorf("read extraction turns: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var item turn
|
||||
if err = rows.Scan(&item.id, &item.publicID, &item.speaker, &item.text, &item.interrupted); err != nil {
|
||||
return extractionSnapshot{}, err
|
||||
}
|
||||
snapshot.turns = append(snapshot.turns, item)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return extractionSnapshot{}, err
|
||||
}
|
||||
snapshot.digest = snapshotDigest(snapshot)
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
func snapshotDigest(snapshot extractionSnapshot) string {
|
||||
type digestTurn struct {
|
||||
ID string `json:"id"`
|
||||
PublicID string `json:"public_id"`
|
||||
Speaker string `json:"speaker"`
|
||||
Text string `json:"text"`
|
||||
Interrupted bool `json:"interrupted"`
|
||||
}
|
||||
digestTurns := make([]digestTurn, 0, len(snapshot.turns))
|
||||
for _, item := range snapshot.turns {
|
||||
digestTurns = append(digestTurns, digestTurn{item.id, item.publicID, item.speaker, item.text, item.interrupted})
|
||||
}
|
||||
digestInput := struct {
|
||||
Exists bool
|
||||
TenantID, SubjectID, Status string
|
||||
Allowed bool
|
||||
Turns []digestTurn
|
||||
}{snapshot.exists, snapshot.tenantID, snapshot.subjectID, snapshot.status, snapshot.storyExtractionAllowed, digestTurns}
|
||||
encoded, _ := json.Marshal(digestInput)
|
||||
digest := sha256.Sum256(encoded)
|
||||
return hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
func subjectTurns(turns []turn) []turn {
|
||||
items := make([]turn, 0, len(turns))
|
||||
for _, item := range turns {
|
||||
if item.speaker == "subject" {
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func joinedTurnContent(turns []turn) string {
|
||||
parts := make([]string, 0, len(turns))
|
||||
for _, item := range turns {
|
||||
parts = append(parts, item.text)
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func storeGeneration(ctx context.Context, tx pgx.Tx, conversationID string, snapshot extractionSnapshot, generated GenerationResult, generationVersion int64) error {
|
||||
sourceTurns := subjectTurns(snapshot.turns)
|
||||
sourcePublicIDs := make([]string, 0, len(sourceTurns))
|
||||
for _, item := range sourceTurns {
|
||||
sourcePublicIDs = append(sourcePublicIDs, item.publicID)
|
||||
}
|
||||
sourceJSON, _ := json.Marshal(sourcePublicIDs)
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO conversation_summary (tenant_id,subject_id,conversation_id,summary_type,content,source_turn_ids,model_ref,prompt_version,generation_version,confidence,status)
|
||||
VALUES ($1::uuid,$2::uuid,$3::uuid,'conversation_summary',$4,$5::jsonb,$6,$7,$8,$9,'READY')
|
||||
ON CONFLICT (conversation_id,summary_type,generation_version) DO NOTHING`, snapshot.tenantID, snapshot.subjectID, conversationID, generated.Summary, sourceJSON, generated.ModelRef, generated.PromptVersion, generationVersion, generated.Confidence); err != nil {
|
||||
return fmt.Errorf("store conversation summary: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO conversation_summary (tenant_id,subject_id,conversation_id,summary_type,content,source_turn_ids,model_ref,prompt_version,generation_version,confidence,status)
|
||||
VALUES ($1::uuid,$2::uuid,$3::uuid,'open_topic',$4,$5::jsonb,$6,$7,$8,$9,'READY')
|
||||
ON CONFLICT (conversation_id,summary_type,generation_version) DO NOTHING`, snapshot.tenantID, snapshot.subjectID, conversationID, generated.OpenTopic, sourceJSON, generated.ModelRef, generated.PromptVersion, generationVersion, generated.Confidence); err != nil {
|
||||
return fmt.Errorf("store open topic: %w", err)
|
||||
}
|
||||
if !generated.AllowStory {
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
var storyID string
|
||||
err := tx.QueryRow(ctx, `
|
||||
INSERT INTO story_draft (tenant_id,subject_id,conversation_id,title,status,confidence,sensitivity,generation_version)
|
||||
VALUES ($1::uuid,$2::uuid,$3::uuid,$4,'CANDIDATE',$5,'normal',$6)
|
||||
ON CONFLICT (conversation_id,generation_version) DO UPDATE SET updated_at=story_draft.updated_at
|
||||
RETURNING id::text`, snapshot.tenantID, snapshot.subjectID, conversationID, generated.StoryTitle, generated.Confidence, generationVersion).Scan(&storyID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("store story candidate: %w", err)
|
||||
}
|
||||
var revisionID string
|
||||
err = tx.QueryRow(ctx, `
|
||||
INSERT INTO story_revision (story_id,revision_no,content,author_type,model_ref,prompt_version)
|
||||
VALUES ($1::uuid,1,$2,'model',$3,$4)
|
||||
ON CONFLICT (story_id,revision_no) DO UPDATE SET content=story_revision.content
|
||||
RETURNING id::text`, storyID, generated.StoryContent, generated.ModelRef, generated.PromptVersion).Scan(&revisionID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("store story revision: %w", err)
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE story_draft SET current_revision_id=$1::uuid WHERE id=$2::uuid AND current_revision_id IS NULL`, revisionID, storyID); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range sourceTurns {
|
||||
if _, err = tx.Exec(ctx, `
|
||||
INSERT INTO story_source (story_revision_id,conversation_turn_id,source_start_offset,source_end_offset,source_role)
|
||||
VALUES ($1::uuid,$2::uuid,0,$3,'subject') ON CONFLICT DO NOTHING`, revisionID, item.id, utf8.RuneCountInString(item.text)); err != nil {
|
||||
return fmt.Errorf("store story source: %w", err)
|
||||
}
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `
|
||||
INSERT INTO fact_candidate (tenant_id,subject_id,story_id,fact_type,content,statement_basis,time_range,confidence,sensitivity,source_turn_ids,status,generation_version)
|
||||
SELECT $1::uuid,$2::uuid,$3::uuid,'life_event',$4,'subject_statement','{}'::jsonb,$5,'normal',$6::jsonb,'CANDIDATE',$7
|
||||
WHERE NOT EXISTS (SELECT 1 FROM fact_candidate WHERE story_id=$3::uuid AND generation_version=$7)`, snapshot.tenantID, snapshot.subjectID, storyID, generated.FactContent, generated.Confidence, sourceJSON, generationVersion); err != nil {
|
||||
return fmt.Errorf("store fact candidate: %w", err)
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func truncateRunes(value string, limit int) string {
|
||||
characters := []rune(value)
|
||||
if len(characters) <= limit {
|
||||
return value
|
||||
}
|
||||
return string(characters[:limit]) + "…"
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package extraction
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/outbox"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func TestProcessorExtractionIdempotencyAndSensitiveContent(t *testing.T) {
|
||||
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if _, err := NewProcessor(nil); err == nil {
|
||||
t.Fatal("nil database pool was accepted")
|
||||
}
|
||||
processor, err := NewProcessor(pool)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consentSnapshotID, consentSnapshotPublicID := uuid.NewString(), uuid.NewString()
|
||||
contextID, contextPublicID := uuid.NewString(), uuid.NewString()
|
||||
conversationID, conversationPublicID := uuid.NewString(), uuid.NewString()
|
||||
defer func() {
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM fact_candidate WHERE story_id IN (SELECT id FROM story_draft WHERE conversation_id=$1::uuid)`, conversationID)
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM story_source WHERE story_revision_id IN (SELECT sr.id FROM story_revision sr JOIN story_draft sd ON sd.id=sr.story_id WHERE sd.conversation_id=$1::uuid)`, conversationID)
|
||||
_, _ = pool.Exec(ctx, `UPDATE story_draft SET current_revision_id=NULL WHERE conversation_id=$1::uuid`, conversationID)
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM story_revision WHERE story_id IN (SELECT id FROM story_draft WHERE conversation_id=$1::uuid)`, conversationID)
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM story_draft WHERE conversation_id=$1::uuid`, conversationID)
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM conversation_summary WHERE conversation_id=$1::uuid`, conversationID)
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM conversation_turn WHERE conversation_id=$1::uuid`, conversationID)
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM conversation WHERE id=$1::uuid`, conversationID)
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM context_snapshot WHERE id=$1::uuid`, contextID)
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM consent_snapshot WHERE id=$1::uuid`, consentSnapshotID)
|
||||
}()
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO consent_snapshot(id,public_id,tenant_id,subject_id,snapshot_version,effective_consents,source_consent_ids)
|
||||
VALUES($1::uuid,$2::uuid,'10000000-0000-0000-0000-000000000001','10000000-0000-0000-0000-000000000401',1,
|
||||
'{"story_extraction":true,"transcript_storage":true}','[]')`, consentSnapshotID, consentSnapshotPublicID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO context_snapshot(id,public_id,tenant_id,subject_id,profile_version,consent_snapshot_id,memory_versions,prompt_bundle_version,engine_mode,content_digest,sanitized_summary)
|
||||
VALUES($1::uuid,$2::uuid,'10000000-0000-0000-0000-000000000001','10000000-0000-0000-0000-000000000401',1,
|
||||
$3::uuid,'[]','extraction-integration-v1','s2s_hybrid',$4,'合成提取上下文')`,
|
||||
contextID, contextPublicID, consentSnapshotID, "d"+strings.Repeat("0", 63)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO conversation(id,public_id,tenant_id,household_id,subject_id,device_id,consent_snapshot_id,context_snapshot_id,
|
||||
provider_task_ref,engine_mode,retention_mode,status,allocation_expires_at)
|
||||
VALUES($1::uuid,$2::uuid,'10000000-0000-0000-0000-000000000001','10000000-0000-0000-0000-000000000201',
|
||||
'10000000-0000-0000-0000-000000000401','10000000-0000-0000-0000-000000000501',
|
||||
$3::uuid,$4::uuid,$5,'s2s_hybrid','transcript_only','ACTIVE',now()+interval '5 minutes')`,
|
||||
conversationID, conversationPublicID, consentSnapshotID, contextID, "extract-task-"+conversationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO conversation_turn(tenant_id,subject_id,conversation_id,turn_key,sequence_no,speaker,start_at,end_at,final_text,source_type)
|
||||
VALUES('10000000-0000-0000-0000-000000000001','10000000-0000-0000-0000-000000000401',$1::uuid,$2,1,'subject',now()-interval '2 minutes',now()-interval '110 seconds',$3,'provider'),
|
||||
('10000000-0000-0000-0000-000000000001','10000000-0000-0000-0000-000000000401',$1::uuid,$4,2,'agent',now()-interval '100 seconds',now()-interval '90 seconds',$5,'provider')`,
|
||||
conversationID, "extract-subject-"+conversationID, "我年轻时在河边小学教书。", "extract-agent-"+conversationID, "这是很珍贵的经历。"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
event := outbox.Event{AggregateID: conversationID, Payload: []byte(`{"generation_version":1}`)}
|
||||
if err := processor.Handle(ctx, event); !errors.Is(err, ErrConversationContentNotReady) || !outbox.IsRetryable(err) {
|
||||
t.Fatalf("active conversation extraction error=%v", err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE conversation SET status='COMPLETED',started_at=now()-interval '3 minutes',ended_at=now(),stop_reason='user_hangup' WHERE id=$1::uuid`, conversationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var modelCalls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
modelCalls.Add(1)
|
||||
updateContext, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if _, updateErr := pool.Exec(updateContext, `UPDATE conversation SET updated_at=updated_at WHERE id=$1::uuid`, conversationID); updateErr != nil {
|
||||
t.Errorf("model call was made while the conversation row was transaction-locked: %v", updateErr)
|
||||
http.Error(w, "database lock", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"{\"summary\":\"长者谈到在河边小学教书\",\"open_topic\":\"下次可以聊聊当年的学生\",\"story_title\":\"河边小学的回忆\",\"story_content\":\"我年轻时在河边小学教书。\",\"fact_content\":\"年轻时曾在河边小学教书\",\"confidence\":0.88}"}}]}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
arkGenerator, err := NewArkGenerator(server.URL+"/api/v3", "integration-key", "doubao-integration", 2*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
modelProcessor, err := NewProcessor(pool, arkGenerator)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := modelProcessor.Handle(ctx, event); err != nil {
|
||||
t.Fatalf("extract completed conversation: %v", err)
|
||||
}
|
||||
if err := modelProcessor.Handle(ctx, event); err != nil {
|
||||
t.Fatalf("replay extraction: %v", err)
|
||||
}
|
||||
var summaries, stories, revisions, sources, facts int
|
||||
if err := pool.QueryRow(ctx, `SELECT
|
||||
(SELECT count(*) FROM conversation_summary WHERE conversation_id=$1::uuid),
|
||||
(SELECT count(*) FROM story_draft WHERE conversation_id=$1::uuid),
|
||||
(SELECT count(*) FROM story_revision WHERE story_id IN (SELECT id FROM story_draft WHERE conversation_id=$1::uuid)),
|
||||
(SELECT count(*) FROM story_source WHERE story_revision_id IN (SELECT sr.id FROM story_revision sr JOIN story_draft sd ON sd.id=sr.story_id WHERE sd.conversation_id=$1::uuid)),
|
||||
(SELECT count(*) FROM fact_candidate WHERE story_id IN (SELECT id FROM story_draft WHERE conversation_id=$1::uuid))`, conversationID).
|
||||
Scan(&summaries, &stories, &revisions, &sources, &facts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if summaries != 2 || stories != 1 || revisions != 1 || sources != 1 || facts != 1 {
|
||||
t.Fatalf("normal extraction counts summaries=%d stories=%d revisions=%d sources=%d facts=%d", summaries, stories, revisions, sources, facts)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO conversation_turn(tenant_id,subject_id,conversation_id,turn_key,sequence_no,speaker,start_at,end_at,final_text,source_type)
|
||||
VALUES('10000000-0000-0000-0000-000000000001','10000000-0000-0000-0000-000000000401',$1::uuid,$2,3,'subject',now()-interval '20 seconds',now()-interval '10 seconds',$3,'provider')`,
|
||||
conversationID, "extract-sensitive-"+conversationID, "我的银行卡号是6222021234567890123，短信验证码是482931。"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
event.Payload = []byte(`{"generation_version":2}`)
|
||||
if err := modelProcessor.Handle(ctx, event); err != nil {
|
||||
t.Fatalf("extract sensitive conversation: %v", err)
|
||||
}
|
||||
if modelCalls.Load() != 2 {
|
||||
t.Fatalf("model calls=%d want=2; sensitive content must not leave the local safety boundary", modelCalls.Load())
|
||||
}
|
||||
var sensitivePrompt, sensitiveSummary string
|
||||
if err := pool.QueryRow(ctx, `SELECT prompt_version,content FROM conversation_summary WHERE conversation_id=$1::uuid AND summary_type='conversation_summary' AND generation_version=2`, conversationID).Scan(&sensitivePrompt, &sensitiveSummary); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasPrefix(sensitivePrompt, "summary-sensitive-filter-v1:") || strings.Contains(sensitiveSummary, "银行卡") {
|
||||
t.Fatalf("sensitive summary prompt=%q content=%q", sensitivePrompt, sensitiveSummary)
|
||||
}
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*) FROM story_draft WHERE conversation_id=$1::uuid`, conversationID).Scan(&stories); err != nil || stories != 1 {
|
||||
t.Fatalf("sensitive extraction created story count=%d err=%v", stories, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTruncateRunes(t *testing.T) {
|
||||
if got := truncateRunes("一二三四", 3); got != "一二三…" {
|
||||
t.Fatalf("truncateRunes=%q", got)
|
||||
}
|
||||
if got := truncateRunes("一二", 3); got != "一二" {
|
||||
t.Fatalf("short truncateRunes=%q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package extraction
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"data-terminal-cloud/internal/contentpolicy"
|
||||
)
|
||||
|
||||
type safetyTestGenerator struct{ calls int }
|
||||
|
||||
func (g *safetyTestGenerator) Generate(context.Context, GenerationInput) (GenerationResult, error) {
|
||||
g.calls++
|
||||
return GenerationResult{
|
||||
Summary: "模型摘要", OpenTopic: "开放话题", StoryTitle: "标题",
|
||||
StoryContent: "模型输出风险", FactContent: "普通事实", AllowStory: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type safetyTestClassifier struct{ calls int }
|
||||
|
||||
func (c *safetyTestClassifier) Classify(_ context.Context, _ string) (contentpolicy.Decision, error) {
|
||||
c.calls++
|
||||
if c.calls == 1 {
|
||||
return contentpolicy.Decision{AllowOrdinaryStory: true, Sensitivity: "normal", ProviderRef: "managed-content-safety:test-v1"}, nil
|
||||
}
|
||||
return contentpolicy.Decision{AllowOrdinaryStory: false, Sensitivity: "sensitive", Reason: "model_output_safety", ProviderRef: "managed-content-safety:test-v1"}, nil
|
||||
}
|
||||
|
||||
func TestGenerateSafelyBlocksModelOutputBeforePersistence(t *testing.T) {
|
||||
generator := &safetyTestGenerator{}
|
||||
classifier := &safetyTestClassifier{}
|
||||
processor := &Processor{generator: generator, safety: classifier}
|
||||
|
||||
result, err := processor.generateSafely(context.Background(), "普通长者话轮")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if generator.calls != 1 || classifier.calls != 2 || result.AllowStory || result.ModelRef != "managed-content-safety:test-v1" || result.PromptVersion != "summary-sensitive-filter-v1:model_output_safety" {
|
||||
t.Fatalf("result=%+v generator_calls=%d classifier_calls=%d", result, generator.calls, classifier.calls)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user