chore: establish local platform baseline
This commit is contained in:
Codex Local
@@ -0,0 +1,193 @@
|
||||
package extraction
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/contentpolicy"
|
||||
"data-terminal-cloud/internal/outbox"
|
||||
)
|
||||
|
||||
type GenerationInput struct {
|
||||
Content string
|
||||
}
|
||||
|
||||
type GenerationResult struct {
|
||||
Summary string `json:"summary"`
|
||||
OpenTopic string `json:"open_topic"`
|
||||
StoryTitle string `json:"story_title"`
|
||||
StoryContent string `json:"story_content"`
|
||||
FactContent string `json:"fact_content"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
ModelRef string `json:"-"`
|
||||
PromptVersion string `json:"-"`
|
||||
AllowStory bool `json:"-"`
|
||||
}
|
||||
|
||||
type Generator interface {
|
||||
Generate(context.Context, GenerationInput) (GenerationResult, error)
|
||||
}
|
||||
|
||||
type DeterministicGenerator struct{}
|
||||
|
||||
func (DeterministicGenerator) Generate(_ context.Context, input GenerationInput) (GenerationResult, error) {
|
||||
content := strings.TrimSpace(input.Content)
|
||||
if content == "" {
|
||||
return GenerationResult{}, errors.New("extraction content is required")
|
||||
}
|
||||
return GenerationResult{
|
||||
Summary: "长者在本次通话中谈到：" + content,
|
||||
OpenTopic: "下次可以继续询问长者刚才提到的经历，并让长者自行补充人物、时间和地点。",
|
||||
StoryTitle: "关于“" + truncateRunes(content, 24) + "”的回忆",
|
||||
StoryContent: content,
|
||||
FactContent: content,
|
||||
Confidence: 0.9,
|
||||
ModelRef: "local-deterministic-extractor",
|
||||
PromptVersion: "story-v1",
|
||||
AllowStory: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func sensitiveGeneration(decision contentpolicy.Decision) GenerationResult {
|
||||
modelRef := strings.TrimSpace(decision.ProviderRef)
|
||||
if modelRef == "" {
|
||||
modelRef = "local-content-safety-policy"
|
||||
}
|
||||
reason := safePolicyToken(decision.Reason)
|
||||
return GenerationResult{
|
||||
Summary: "本次通话包含需要人工谨慎处理的敏感信息，系统未生成普通故事卡片。",
|
||||
OpenTopic: "如确有必要处理本次敏感内容，应由有权限人员在受控流程中单独核对。",
|
||||
Confidence: 1,
|
||||
ModelRef: modelRef,
|
||||
PromptVersion: "summary-sensitive-filter-v1:" + reason,
|
||||
AllowStory: false,
|
||||
}
|
||||
}
|
||||
|
||||
func safePolicyToken(value string) string {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
if value == "" || len(value) > 64 {
|
||||
return "policy_block"
|
||||
}
|
||||
for _, character := range value {
|
||||
if (character < 'a' || character > 'z') && (character < '0' || character > '9') && character != '_' && character != '-' {
|
||||
return "policy_block"
|
||||
}
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
type ArkGenerator struct {
|
||||
endpoint string
|
||||
apiKey string
|
||||
model string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewArkGenerator(baseURL, apiKey, model string, timeout time.Duration) (*ArkGenerator, error) {
|
||||
baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
|
||||
apiKey = strings.TrimSpace(apiKey)
|
||||
model = strings.TrimSpace(model)
|
||||
parsed, err := url.Parse(baseURL)
|
||||
if err != nil || parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") {
|
||||
return nil, errors.New("Ark base URL must be an absolute HTTP(S) URL")
|
||||
}
|
||||
if apiKey == "" || model == "" || timeout <= 0 {
|
||||
return nil, errors.New("Ark API key, content model and positive timeout are required")
|
||||
}
|
||||
return &ArkGenerator{
|
||||
endpoint: baseURL + "/chat/completions", apiKey: apiKey, model: model,
|
||||
client: &http.Client{Timeout: timeout},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (GenerationResult, error) {
|
||||
content := strings.TrimSpace(input.Content)
|
||||
if content == "" {
|
||||
return GenerationResult{}, errors.New("extraction content is required")
|
||||
}
|
||||
requestBody := map[string]any{
|
||||
"model": g.model,
|
||||
"messages": []map[string]string{
|
||||
{"role": "system", "content": "你是长者口述内容整理助手。只依据输入字幕生成可人工核对的候选，不补充未出现的事实，不把推断写成事实。必须返回JSON对象，字段为summary、open_topic、story_title、story_content、fact_content、confidence。summary概括本次内容，open_topic给出一个温和的后续话题，story和fact保留原意，confidence取0到1。"},
|
||||
{"role": "user", "content": content},
|
||||
},
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 1200,
|
||||
"response_format": map[string]string{"type": "json_object"},
|
||||
}
|
||||
encoded, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return GenerationResult{}, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, g.endpoint, bytes.NewReader(encoded))
|
||||
if err != nil {
|
||||
return GenerationResult{}, fmt.Errorf("build Ark content request: %w", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+g.apiKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", "voice-platform-content-extractor/1.0")
|
||||
resp, err := g.client.Do(req)
|
||||
if err != nil {
|
||||
return GenerationResult{}, outbox.Retryable(fmt.Errorf("call Ark content model: %w", err))
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
responseBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if readErr != nil {
|
||||
return GenerationResult{}, outbox.Retryable(fmt.Errorf("read Ark content response: %w", readErr))
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
requestErr := fmt.Errorf("Ark content model returned HTTP %d", resp.StatusCode)
|
||||
if resp.StatusCode == http.StatusRequestTimeout || resp.StatusCode == http.StatusTooEarly || resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 {
|
||||
return GenerationResult{}, outbox.Retryable(requestErr)
|
||||
}
|
||||
return GenerationResult{}, requestErr
|
||||
}
|
||||
var completion struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
}
|
||||
if err = json.Unmarshal(responseBody, &completion); err != nil || len(completion.Choices) == 0 {
|
||||
return GenerationResult{}, outbox.Retryable(errors.New("Ark content response has no usable choice"))
|
||||
}
|
||||
jsonContent := strings.TrimSpace(completion.Choices[0].Message.Content)
|
||||
jsonContent = strings.TrimPrefix(jsonContent, "```json")
|
||||
jsonContent = strings.TrimPrefix(jsonContent, "```")
|
||||
jsonContent = strings.TrimSuffix(jsonContent, "```")
|
||||
var result GenerationResult
|
||||
if err = json.Unmarshal([]byte(strings.TrimSpace(jsonContent)), &result); err != nil {
|
||||
return GenerationResult{}, outbox.Retryable(fmt.Errorf("decode Ark structured content: %w", err))
|
||||
}
|
||||
result.Summary = strings.TrimSpace(result.Summary)
|
||||
result.OpenTopic = strings.TrimSpace(result.OpenTopic)
|
||||
result.StoryTitle = strings.TrimSpace(result.StoryTitle)
|
||||
result.StoryContent = strings.TrimSpace(result.StoryContent)
|
||||
result.FactContent = strings.TrimSpace(result.FactContent)
|
||||
if !validGeneratedContent(result) {
|
||||
return GenerationResult{}, outbox.Retryable(errors.New("Ark structured content failed validation"))
|
||||
}
|
||||
result.ModelRef = "ark:" + g.model
|
||||
result.PromptVersion = "story-ark-json-v1"
|
||||
result.AllowStory = true
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func validGeneratedContent(result GenerationResult) bool {
|
||||
return result.Summary != "" && len([]rune(result.Summary)) <= 2000 &&
|
||||
result.OpenTopic != "" && len([]rune(result.OpenTopic)) <= 1000 &&
|
||||
result.StoryTitle != "" && len([]rune(result.StoryTitle)) <= 300 &&
|
||||
result.StoryContent != "" && len([]rune(result.StoryContent)) <= 20000 &&
|
||||
result.FactContent != "" && len([]rune(result.FactContent)) <= 5000 &&
|
||||
result.Confidence >= 0 && result.Confidence <= 1
|
||||
}
|
||||
Reference in New Issue
Block a user