82 lines
3.3 KiB
Go
82 lines
3.3 KiB
Go
|
package agenteval
|
||
|
|
|
||
|
|
import (
|
||
|
|
"encoding/json"
|
||
|
|
"net/http"
|
||
|
|
"net/http/httptest"
|
||
|
|
"strings"
|
||
|
|
"testing"
|
||
|
|
"time"
|
||
|
|
)
|
||
|
|
|
||
|
|
func TestCollectorDerivesMachineMetricsAndKeepsScorecard(t *testing.T) {
|
||
|
|
suite, _ := validFixture()
|
||
|
|
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||
|
|
if request.Header.Get("Authorization") != "Bearer test-token" {
|
||
|
|
http.Error(writer, "unauthorized", http.StatusUnauthorized)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
parts := strings.Split(strings.Trim(request.URL.Path, "/"), "/")
|
||
|
|
conversationID := parts[3]
|
||
|
|
writer.Header().Set("Content-Type", "application/json")
|
||
|
|
if strings.HasSuffix(request.URL.Path, "/turns") {
|
||
|
|
_ = json.NewEncoder(writer).Encode(map[string]any{"data": map[string]any{"items": []map[string]any{
|
||
|
|
{"sequence_no": 1, "speaker": "subject", "start_at": "2026-08-24T10:00:00Z", "end_at": "2026-08-24T10:00:01Z", "final_text": "测试问题", "interrupted": false},
|
||
|
|
{"sequence_no": 2, "speaker": "agent", "start_at": "2026-08-24T10:00:01.750Z", "end_at": "2026-08-24T10:00:03Z", "final_text": "简短测试回答", "interrupted": false},
|
||
|
|
}}})
|
||
|
|
return
|
||
|
|
}
|
||
|
|
_ = json.NewEncoder(writer).Encode(map[string]any{"data": map[string]any{"id": conversationID, "status": "COMPLETED"}})
|
||
|
|
}))
|
||
|
|
defer server.Close()
|
||
|
|
|
||
|
|
manifest := CollectionManifest{
|
||
|
|
Schema: CollectionSchema, SuiteVersion: suite.SuiteVersion, AgentVersion: "agent-v1",
|
||
|
|
ModelConfigRef: suite.ModelConfigRef, Environment: "mock",
|
||
|
|
ExecutedAt: time.Date(2026, 8, 24, 10, 10, 0, 0, time.UTC),
|
||
|
|
}
|
||
|
|
for index, testCase := range suite.Cases {
|
||
|
|
flags := map[string]bool{}
|
||
|
|
for _, flag := range testCase.Expected.RequiredFlags {
|
||
|
|
flags[flag] = true
|
||
|
|
}
|
||
|
|
manifest.Cases = append(manifest.Cases, CollectionCase{
|
||
|
|
CaseID: testCase.ID, ConversationID: "conversation-" + string(rune('a'+index)), Flags: flags,
|
||
|
|
})
|
||
|
|
}
|
||
|
|
collector, err := NewCollector(server.URL, "test-token", server.Client())
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("NewCollector() error = %v", err)
|
||
|
|
}
|
||
|
|
run, err := collector.Collect(t.Context(), suite, manifest)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("Collect() error = %v", err)
|
||
|
|
}
|
||
|
|
if len(run.Observations) != len(suite.Cases) {
|
||
|
|
t.Fatalf("observations = %d", len(run.Observations))
|
||
|
|
}
|
||
|
|
first := run.Observations[0]
|
||
|
|
if !first.ConversationSuccess || first.FirstResponseMS != 750 || first.ResponseChars != 6 || first.EvidenceRef != "conversation:conversation-a" || !first.Flags["salutation_correct"] {
|
||
|
|
t.Fatalf("first observation = %+v", first)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestCollectorRejectsMissingHumanOrJudgeScore(t *testing.T) {
|
||
|
|
suite, _ := validFixture()
|
||
|
|
manifest := CollectionManifest{
|
||
|
|
Schema: CollectionSchema, SuiteVersion: suite.SuiteVersion, AgentVersion: "agent-v1",
|
||
|
|
ModelConfigRef: suite.ModelConfigRef, Environment: "mock", ExecutedAt: time.Now(),
|
||
|
|
}
|
||
|
|
for _, testCase := range suite.Cases {
|
||
|
|
manifest.Cases = append(manifest.Cases, CollectionCase{CaseID: testCase.ID, ConversationID: "conversation-" + testCase.ID, Flags: map[string]bool{}})
|
||
|
|
}
|
||
|
|
collector, err := NewCollector("https://api.example.invalid", "test-token", nil)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("NewCollector() error = %v", err)
|
||
|
|
}
|
||
|
|
_, err = collector.Collect(t.Context(), suite, manifest)
|
||
|
|
if err == nil || !strings.Contains(err.Error(), "does not score required flag") {
|
||
|
|
t.Fatalf("expected scorecard validation error, got %v", err)
|
||
|
|
}
|
||
|
|
}
|