feat: add domestic model provider presets

2026-08-30 19:55:17 +08:00
parent 522785eda8
commit dcf0fc0d7f
13 changed files with 314 additions and 28 deletions
@@ -362,7 +362,7 @@ func normalizeInput(input *CreateInput) {

 func validateInput(input CreateInput) error {
 	parsed, err := url.Parse(input.BaseURL)
-	validProvider := input.Provider == "ark" || input.Provider == "openai_compatible"
+	validProvider := extraction.SupportsCompatibleProvider(input.Provider)
 	validURL := err == nil && parsed.Host != "" && (parsed.Scheme == "https" || (parsed.Scheme == "http" && isApprovedLocalModelHost(parsed.Hostname())))
 	if !validUUID(input.TenantID) || len([]rune(input.DisplayName)) < 1 || len([]rune(input.DisplayName)) > 120 || !validProvider || !validURL || input.ModelID == "" || len(input.APIKey) < 8 || input.Temperature < 0 || input.Temperature > 2 || input.MaxTokens < 256 || input.MaxTokens > 8192 {
 		return ErrInvalid
@@ -2,6 +2,33 @@ package contentmodel

 import "testing"

+func TestValidateInputAcceptsSupportedDomesticProviders(t *testing.T) {
+	providers := []string{"ark", "dashscope", "deepseek", "moonshot", "zhipu", "qianfan", "hunyuan", "minimax", "baichuan", "siliconflow"}
+	for _, provider := range providers {
+		t.Run(provider, func(t *testing.T) {
+			input := CreateInput{
+				TenantID: "10000000-0000-0000-0000-000000000001", DisplayName: provider,
+				Provider: provider, BaseURL: "https://model.example.com/v1", ModelID: "summary-model",
+				APIKey: "test-api-key", Temperature: 0.2, MaxTokens: 1200,
+			}
+			if err := validateInput(input); err != nil {
+				t.Fatalf("supported provider was rejected: %v", err)
+			}
+		})
+	}
+}
+
+func TestValidateInputRejectsUnknownProvider(t *testing.T) {
+	input := CreateInput{
+		TenantID: "10000000-0000-0000-0000-000000000001", DisplayName: "unknown",
+		Provider: "unknown", BaseURL: "https://model.example.com/v1", ModelID: "summary-model",
+		APIKey: "test-api-key", Temperature: 0.2, MaxTokens: 1200,
+	}
+	if err := validateInput(input); err == nil {
+		t.Fatal("unknown provider was accepted")
+	}
+}
+
 func TestValidateInputAllowsComposeOllamaEndpoint(t *testing.T) {
 	input := CreateInput{
 		TenantID: "10000000-0000-0000-0000-000000000001", DisplayName: "本地 Qwen",
@@ -132,10 +132,10 @@ func NewCompatibleGenerator(provider, baseURL, apiKey, model string, temperature
 	model = strings.TrimSpace(model)
 	parsed, err := url.Parse(baseURL)
 	if err != nil || parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") {
-		return nil, errors.New("Ark base URL must be an absolute HTTP(S) URL")
+		return nil, errors.New("content model base URL must be an absolute HTTP(S) URL")
 	}
-	if provider != "ark" && provider != "openai_compatible" {
-		return nil, errors.New("content model provider must be ark or openai_compatible")
+	if !SupportsCompatibleProvider(provider) {
+		return nil, errors.New("unsupported content model provider")
 	}
 	if apiKey == "" || model == "" || timeout <= 0 || temperature < 0 || temperature > 2 || maxTokens < 256 || maxTokens > 8192 {
 		return nil, errors.New("content model credentials and valid generation parameters are required")
@@ -158,9 +158,14 @@ func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (Gen
 			{"role": "system", "content": contentValuePrompt},
 			{"role": "user", "content": content},
 		},
-		"temperature":     g.temperature,
-		"max_tokens":      g.maxTokens,
-		"response_format": map[string]string{"type": "json_object"},
+		"temperature": g.temperature,
+		"max_tokens":  g.maxTokens,
+	}
+	// JSON mode is sent only where the provider documents it or this project
+	// has exercised it. Other compatible providers still receive an explicit
+	// JSON-only prompt, followed by strict response extraction and validation.
+	if providerSupportsJSONMode(g.provider) {
+		requestBody["response_format"] = map[string]string{"type": "json_object"}
 	}
 	encoded, err := json.Marshal(requestBody)
 	if err != nil {
@@ -168,22 +173,22 @@ func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (Gen
 	}
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, g.endpoint, bytes.NewReader(encoded))
 	if err != nil {
-		return GenerationResult{}, fmt.Errorf("build Ark content request: %w", err)
+		return GenerationResult{}, fmt.Errorf("build content model request: %w", err)
 	}
 	req.Header.Set("Authorization", "Bearer "+g.apiKey)
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("User-Agent", "voice-platform-content-extractor/1.0")
 	resp, err := g.client.Do(req)
 	if err != nil {
-		return GenerationResult{}, outbox.Retryable(fmt.Errorf("call Ark content model: %w", err))
+		return GenerationResult{}, outbox.Retryable(fmt.Errorf("call content model: %w", err))
 	}
 	defer resp.Body.Close()
 	responseBody, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
 	if readErr != nil {
-		return GenerationResult{}, outbox.Retryable(fmt.Errorf("read Ark content response: %w", readErr))
+		return GenerationResult{}, outbox.Retryable(fmt.Errorf("read content model response: %w", readErr))
 	}
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		requestErr := fmt.Errorf("Ark content model returned HTTP %d", resp.StatusCode)
+		requestErr := fmt.Errorf("content model returned HTTP %d", resp.StatusCode)
 		if resp.StatusCode == http.StatusRequestTimeout || resp.StatusCode == http.StatusTooEarly || resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 {
 			return GenerationResult{}, outbox.Retryable(requestErr)
 		}
@@ -197,15 +202,15 @@ func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (Gen
 		} `json:"choices"`
 	}
 	if err = json.Unmarshal(responseBody, &completion); err != nil || len(completion.Choices) == 0 {
-		return GenerationResult{}, outbox.Retryable(errors.New("Ark content response has no usable choice"))
+		return GenerationResult{}, outbox.Retryable(errors.New("content model response has no usable choice"))
 	}
-	jsonContent := strings.TrimSpace(completion.Choices[0].Message.Content)
+	jsonContent := extractJSONObject(completion.Choices[0].Message.Content)
 	jsonContent = strings.TrimPrefix(jsonContent, "```json")
 	jsonContent = strings.TrimPrefix(jsonContent, "```")
 	jsonContent = strings.TrimSuffix(jsonContent, "```")
 	var result GenerationResult
 	if err = json.Unmarshal([]byte(strings.TrimSpace(jsonContent)), &result); err != nil {
-		return GenerationResult{}, outbox.Retryable(fmt.Errorf("decode Ark structured content: %w", err))
+		return GenerationResult{}, outbox.Retryable(fmt.Errorf("decode structured content: %w", err))
 	}
 	result.Summary = strings.TrimSpace(result.Summary)
 	result.DecisionReason = strings.TrimSpace(result.DecisionReason)
@@ -216,7 +221,7 @@ func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (Gen
 	result.Keywords = normalizedKeywords(result.Keywords)
 	normalizeDecisionContent(&result)
 	if !validGeneratedContent(result) {
-		return GenerationResult{}, outbox.Retryable(errors.New("Ark structured content failed validation"))
+		return GenerationResult{}, outbox.Retryable(errors.New("structured content failed validation"))
 	}
 	result.ModelRef = g.provider + ":" + g.model
 	result.PromptVersion = "content-value-" + g.provider + "-json-v3"
@@ -224,6 +229,42 @@ func (g *ArkGenerator) Generate(ctx context.Context, input GenerationInput) (Gen
 	return result, nil
 }

+// SupportsCompatibleProvider is the single backend allowlist used both when a
+// model configuration is saved and when a worker resolves that configuration.
+func SupportsCompatibleProvider(provider string) bool {
+	switch provider {
+	case "ark", "dashscope", "deepseek", "moonshot", "zhipu", "qianfan", "hunyuan", "minimax", "baichuan", "siliconflow", "ollama", "openai_compatible":
+		return true
+	default:
+		return false
+	}
+}
+
+func providerSupportsJSONMode(provider string) bool {
+	switch provider {
+	case "ark", "dashscope", "deepseek", "siliconflow", "ollama", "openai_compatible":
+		return true
+	default:
+		return false
+	}
+}
+
+func extractJSONObject(content string) string {
+	content = strings.TrimSpace(content)
+	if end := strings.LastIndex(content, "</think>"); end >= 0 {
+		content = strings.TrimSpace(content[end+len("</think>"):])
+	}
+	content = strings.TrimPrefix(content, "```json")
+	content = strings.TrimPrefix(content, "```")
+	content = strings.TrimSuffix(content, "```")
+	content = strings.TrimSpace(content)
+	start, end := strings.Index(content, "{"), strings.LastIndex(content, "}")
+	if start >= 0 && end >= start {
+		return content[start : end+1]
+	}
+	return content
+}
+
 func normalizeDecisionContent(result *GenerationResult) {
 	if !result.Meaningful {
 		result.StoryWorthy = false
@@ -78,6 +78,46 @@ func TestCompatibleGeneratorClearsContradictoryStoryFields(t *testing.T) {
 	}
 }

+func TestCompatibleGeneratorHandlesThinkingWrapper(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": "<think>先判断内容价值</think>\n```json\n{\"meaningful\":false,\"story_worthy\":false,\"decision_reason\":\"只有问候\",\"summary\":\"\",\"keywords\":[],\"open_topic\":\"\",\"story_title\":\"\",\"story_content\":\"\",\"fact_content\":\"\",\"confidence\":0.95}\n```"}}}})
+	}))
+	defer server.Close()
+	generator, err := NewCompatibleGenerator("minimax", server.URL+"/v1", "local-key", "MiniMax-test", 0.1, 512, time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	result, err := generator.Generate(context.Background(), GenerationInput{Content: "您好。"})
+	if err != nil || result.Meaningful || result.ModelRef != "minimax:MiniMax-test" {
+		t.Fatalf("result=%+v err=%v", result, err)
+	}
+}
+
+func TestCompatibleGeneratorOmitsUndocumentedJSONMode(t *testing.T) {
+	for _, provider := range []string{"moonshot", "zhipu", "qianfan", "hunyuan", "minimax", "baichuan"} {
+		t.Run(provider, func(t *testing.T) {
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				var body map[string]any
+				if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+					t.Fatal(err)
+				}
+				if _, exists := body["response_format"]; exists {
+					t.Fatalf("%s request unexpectedly enabled JSON mode", provider)
+				}
+				_ = json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": `{"meaningful":false,"story_worthy":false,"decision_reason":"测试内容","summary":"","keywords":[],"open_topic":"","story_title":"","story_content":"","fact_content":"","confidence":0.9}`}}}})
+			}))
+			defer server.Close()
+			generator, err := NewCompatibleGenerator(provider, server.URL, "test-key", "test-model", 0.2, 512, time.Second)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if _, err := generator.Generate(context.Background(), GenerationInput{Content: "模型连接测试"}); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
 func TestArkGeneratorRetriesTemporaryFailure(t *testing.T) {
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		http.Error(w, "busy", http.StatusTooManyRequests)