chore: establish local platform baseline

2026-08-26 06:41:54 +08:00
commit 29bdf4900f
575 changed files with 95644 additions and 0 deletions
@@ -0,0 +1,81 @@
+package agenteval
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestCollectorDerivesMachineMetricsAndKeepsScorecard(t *testing.T) {
+	suite, _ := validFixture()
+	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		if request.Header.Get("Authorization") != "Bearer test-token" {
+			http.Error(writer, "unauthorized", http.StatusUnauthorized)
+			return
+		}
+		parts := strings.Split(strings.Trim(request.URL.Path, "/"), "/")
+		conversationID := parts[3]
+		writer.Header().Set("Content-Type", "application/json")
+		if strings.HasSuffix(request.URL.Path, "/turns") {
+			_ = json.NewEncoder(writer).Encode(map[string]any{"data": map[string]any{"items": []map[string]any{
+				{"sequence_no": 1, "speaker": "subject", "start_at": "2026-08-24T10:00:00Z", "end_at": "2026-08-24T10:00:01Z", "final_text": "测试问题", "interrupted": false},
+				{"sequence_no": 2, "speaker": "agent", "start_at": "2026-08-24T10:00:01.750Z", "end_at": "2026-08-24T10:00:03Z", "final_text": "简短测试回答", "interrupted": false},
+			}}})
+			return
+		}
+		_ = json.NewEncoder(writer).Encode(map[string]any{"data": map[string]any{"id": conversationID, "status": "COMPLETED"}})
+	}))
+	defer server.Close()
+
+	manifest := CollectionManifest{
+		Schema: CollectionSchema, SuiteVersion: suite.SuiteVersion, AgentVersion: "agent-v1",
+		ModelConfigRef: suite.ModelConfigRef, Environment: "mock",
+		ExecutedAt: time.Date(2026, 8, 24, 10, 10, 0, 0, time.UTC),
+	}
+	for index, testCase := range suite.Cases {
+		flags := map[string]bool{}
+		for _, flag := range testCase.Expected.RequiredFlags {
+			flags[flag] = true
+		}
+		manifest.Cases = append(manifest.Cases, CollectionCase{
+			CaseID: testCase.ID, ConversationID: "conversation-" + string(rune('a'+index)), Flags: flags,
+		})
+	}
+	collector, err := NewCollector(server.URL, "test-token", server.Client())
+	if err != nil {
+		t.Fatalf("NewCollector() error = %v", err)
+	}
+	run, err := collector.Collect(t.Context(), suite, manifest)
+	if err != nil {
+		t.Fatalf("Collect() error = %v", err)
+	}
+	if len(run.Observations) != len(suite.Cases) {
+		t.Fatalf("observations = %d", len(run.Observations))
+	}
+	first := run.Observations[0]
+	if !first.ConversationSuccess || first.FirstResponseMS != 750 || first.ResponseChars != 6 || first.EvidenceRef != "conversation:conversation-a" || !first.Flags["salutation_correct"] {
+		t.Fatalf("first observation = %+v", first)
+	}
+}
+
+func TestCollectorRejectsMissingHumanOrJudgeScore(t *testing.T) {
+	suite, _ := validFixture()
+	manifest := CollectionManifest{
+		Schema: CollectionSchema, SuiteVersion: suite.SuiteVersion, AgentVersion: "agent-v1",
+		ModelConfigRef: suite.ModelConfigRef, Environment: "mock", ExecutedAt: time.Now(),
+	}
+	for _, testCase := range suite.Cases {
+		manifest.Cases = append(manifest.Cases, CollectionCase{CaseID: testCase.ID, ConversationID: "conversation-" + testCase.ID, Flags: map[string]bool{}})
+	}
+	collector, err := NewCollector("https://api.example.invalid", "test-token", nil)
+	if err != nil {
+		t.Fatalf("NewCollector() error = %v", err)
+	}
+	_, err = collector.Collect(t.Context(), suite, manifest)
+	if err == nil || !strings.Contains(err.Error(), "does not score required flag") {
+		t.Fatalf("expected scorecard validation error, got %v", err)
+	}
+}