chore: establish local platform baseline

internal/agenteval/collector.go

@@ -0,0 +1,251 @@
+package agenteval
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"sort"
+	"strings"
+	"time"
+)
+
+const CollectionSchema = "agent-eval-collection-v1"
+
+type CollectionManifest struct {
+	Schema         string           `json:"schema"`
+	SuiteVersion   string           `json:"suite_version"`
+	AgentVersion   string           `json:"agent_version"`
+	ModelConfigRef string           `json:"model_config_ref"`
+	Environment    string           `json:"environment"`
+	ExecutedAt     time.Time        `json:"executed_at"`
+	Cases          []CollectionCase `json:"cases"`
+}
+
+type CollectionCase struct {
+	CaseID          string          `json:"case_id"`
+	ConversationID  string          `json:"conversation_id"`
+	FabricatedFacts int             `json:"fabricated_facts"`
+	Flags           map[string]bool `json:"flags"`
+}
+
+type Collector struct {
+	baseURL string
+	token   string
+	client  *http.Client
+}
+
+type conversationDetailEnvelope struct {
+	Data struct {
+		ID     string `json:"id"`
+		Status string `json:"status"`
+	} `json:"data"`
+}
+
+type conversationTurnsEnvelope struct {
+	Data struct {
+		Items      []conversationTurn `json:"items"`
+		NextCursor string             `json:"next_cursor"`
+	} `json:"data"`
+}
+
+type conversationTurn struct {
+	SequenceNo  int64     `json:"sequence_no"`
+	Speaker     string    `json:"speaker"`
+	StartAt     time.Time `json:"start_at"`
+	EndAt       time.Time `json:"end_at"`
+	FinalText   string    `json:"final_text"`
+	Interrupted bool      `json:"interrupted"`
+}
+
+func NewCollector(baseURL, accessToken string, client *http.Client) (*Collector, error) {
+	baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
+	accessToken = strings.TrimSpace(accessToken)
+	parsed, err := url.Parse(baseURL)
+	if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || accessToken == "" {
+		return nil, errors.New("valid API base URL and access token are required")
+	}
+	if client == nil {
+		client = &http.Client{Timeout: 15 * time.Second}
+	}
+	return &Collector{baseURL: baseURL, token: accessToken, client: client}, nil
+}
+
+func (c *Collector) Collect(ctx context.Context, suite Suite, manifest CollectionManifest) (ObservationRun, error) {
+	if err := validateSuite(suite); err != nil {
+		return ObservationRun{}, err
+	}
+	if err := validateCollectionManifest(suite, manifest); err != nil {
+		return ObservationRun{}, err
+	}
+	run := ObservationRun{
+		Schema: ObservationSchema, SuiteVersion: manifest.SuiteVersion,
+		AgentVersion: manifest.AgentVersion, ModelConfigRef: manifest.ModelConfigRef,
+		Environment: manifest.Environment, ExecutedAt: manifest.ExecutedAt.UTC(),
+		Observations: make([]Observation, 0, len(manifest.Cases)),
+	}
+	for _, item := range manifest.Cases {
+		detail, err := c.getDetail(ctx, item.ConversationID)
+		if err != nil {
+			return ObservationRun{}, fmt.Errorf("collect case %s detail: %w", item.CaseID, err)
+		}
+		turns, err := c.getTurns(ctx, item.ConversationID)
+		if err != nil {
+			return ObservationRun{}, fmt.Errorf("collect case %s turns: %w", item.CaseID, err)
+		}
+		firstResponseMS, responseChars := deriveTurnMetrics(turns)
+		success := detail.Data.Status == "COMPLETED" && firstResponseMS > 0
+		run.Observations = append(run.Observations, Observation{
+			CaseID: item.CaseID, EvidenceRef: "conversation:" + item.ConversationID,
+			ConversationSuccess: success, FirstResponseMS: firstResponseMS,
+			ResponseChars: responseChars, FabricatedFacts: item.FabricatedFacts,
+			Flags: cloneFlags(item.Flags),
+		})
+	}
+	return run, nil
+}
+
+func validateCollectionManifest(suite Suite, manifest CollectionManifest) error {
+	if manifest.Schema != CollectionSchema {
+		return fmt.Errorf("unsupported collection schema %q", manifest.Schema)
+	}
+	if manifest.SuiteVersion != suite.SuiteVersion || manifest.ModelConfigRef != suite.ModelConfigRef {
+		return errors.New("collection suite or model configuration does not match evaluation suite")
+	}
+	if strings.TrimSpace(manifest.AgentVersion) == "" || (manifest.Environment != "mock" && manifest.Environment != "real") || manifest.ExecutedAt.IsZero() {
+		return errors.New("collection requires agent_version, mock or real environment, and executed_at")
+	}
+	wanted := make(map[string]Case, len(suite.Cases))
+	for _, testCase := range suite.Cases {
+		wanted[testCase.ID] = testCase
+	}
+	seen := make(map[string]struct{}, len(manifest.Cases))
+	conversationIDs := make(map[string]struct{}, len(manifest.Cases))
+	for _, item := range manifest.Cases {
+		testCase, exists := wanted[item.CaseID]
+		if !exists {
+			return fmt.Errorf("collection references unknown case %q", item.CaseID)
+		}
+		if _, duplicate := seen[item.CaseID]; duplicate {
+			return fmt.Errorf("collection repeats case %q", item.CaseID)
+		}
+		seen[item.CaseID] = struct{}{}
+		if strings.TrimSpace(item.ConversationID) == "" || item.FabricatedFacts < 0 || item.Flags == nil {
+			return fmt.Errorf("collection case %q has invalid evidence", item.CaseID)
+		}
+		if _, duplicate := conversationIDs[item.ConversationID]; duplicate {
+			return fmt.Errorf("conversation %q is reused by multiple evaluation cases", item.ConversationID)
+		}
+		conversationIDs[item.ConversationID] = struct{}{}
+		for _, flag := range testCase.Expected.RequiredFlags {
+			if _, exists := item.Flags[flag]; !exists {
+				return fmt.Errorf("collection case %q does not score required flag %q", item.CaseID, flag)
+			}
+		}
+	}
+	if len(seen) != len(wanted) {
+		missing := make([]string, 0)
+		for caseID := range wanted {
+			if _, exists := seen[caseID]; !exists {
+				missing = append(missing, caseID)
+			}
+		}
+		sort.Strings(missing)
+		return fmt.Errorf("collection missing cases: %s", strings.Join(missing, ", "))
+	}
+	return nil
+}
+
+func (c *Collector) getDetail(ctx context.Context, conversationID string) (conversationDetailEnvelope, error) {
+	var envelope conversationDetailEnvelope
+	err := c.getJSON(ctx, "/api/v1/conversations/"+url.PathEscape(conversationID), &envelope)
+	if err == nil && envelope.Data.ID != "" && envelope.Data.ID != conversationID {
+		return conversationDetailEnvelope{}, errors.New("conversation detail returned a different id")
+	}
+	return envelope, err
+}
+
+func (c *Collector) getTurns(ctx context.Context, conversationID string) ([]conversationTurn, error) {
+	items := make([]conversationTurn, 0)
+	cursor := ""
+	for page := 0; page < 20; page++ {
+		path := "/api/v1/conversations/" + url.PathEscape(conversationID) + "/turns?limit=100"
+		if cursor != "" {
+			path += "&cursor=" + url.QueryEscape(cursor)
+		}
+		var envelope conversationTurnsEnvelope
+		if err := c.getJSON(ctx, path, &envelope); err != nil {
+			return nil, err
+		}
+		items = append(items, envelope.Data.Items...)
+		if envelope.Data.NextCursor == "" {
+			return items, nil
+		}
+		if envelope.Data.NextCursor == cursor {
+			return nil, errors.New("turn pagination returned the same cursor")
+		}
+		cursor = envelope.Data.NextCursor
+	}
+	return nil, errors.New("turn pagination exceeded 20 pages")
+}
+
+func (c *Collector) getJSON(ctx context.Context, path string, target any) error {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+path, nil)
+	if err != nil {
+		return err
+	}
+	request.Header.Set("Authorization", "Bearer "+c.token)
+	request.Header.Set("Accept", "application/json")
+	response, err := c.client.Do(request)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
+		return fmt.Errorf("API returned HTTP %d", response.StatusCode)
+	}
+	decoder := json.NewDecoder(io.LimitReader(response.Body, 2<<20))
+	if err := decoder.Decode(target); err != nil {
+		return fmt.Errorf("decode API response: %w", err)
+	}
+	return nil
+}
+
+func deriveTurnMetrics(turns []conversationTurn) (int, int) {
+	sorted := append([]conversationTurn(nil), turns...)
+	sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].SequenceNo < sorted[j].SequenceNo })
+	var subjectEnd time.Time
+	firstResponseMS, maxResponseChars := 0, 0
+	for _, turn := range sorted {
+		switch turn.Speaker {
+		case "subject":
+			if turn.EndAt.After(subjectEnd) {
+				subjectEnd = turn.EndAt
+			}
+		case "agent":
+			length := len([]rune(turn.FinalText))
+			if length > maxResponseChars {
+				maxResponseChars = length
+			}
+			if firstResponseMS == 0 && !subjectEnd.IsZero() && !turn.StartAt.Before(subjectEnd) {
+				firstResponseMS = int(turn.StartAt.Sub(subjectEnd).Milliseconds())
+				if firstResponseMS == 0 {
+					firstResponseMS = 1
+				}
+			}
+		}
+	}
+	return firstResponseMS, maxResponseChars
+}
+
+func cloneFlags(source map[string]bool) map[string]bool {
+	result := make(map[string]bool, len(source))
+	for key, value := range source {
+		result[key] = value
+	}
+	return result
+}

internal/agenteval/collector_test.go

@@ -0,0 +1,81 @@
+package agenteval
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestCollectorDerivesMachineMetricsAndKeepsScorecard(t *testing.T) {
+	suite, _ := validFixture()
+	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		if request.Header.Get("Authorization") != "Bearer test-token" {
+			http.Error(writer, "unauthorized", http.StatusUnauthorized)
+			return
+		}
+		parts := strings.Split(strings.Trim(request.URL.Path, "/"), "/")
+		conversationID := parts[3]
+		writer.Header().Set("Content-Type", "application/json")
+		if strings.HasSuffix(request.URL.Path, "/turns") {
+			_ = json.NewEncoder(writer).Encode(map[string]any{"data": map[string]any{"items": []map[string]any{
+				{"sequence_no": 1, "speaker": "subject", "start_at": "2026-08-24T10:00:00Z", "end_at": "2026-08-24T10:00:01Z", "final_text": "测试问题", "interrupted": false},
+				{"sequence_no": 2, "speaker": "agent", "start_at": "2026-08-24T10:00:01.750Z", "end_at": "2026-08-24T10:00:03Z", "final_text": "简短测试回答", "interrupted": false},
+			}}})
+			return
+		}
+		_ = json.NewEncoder(writer).Encode(map[string]any{"data": map[string]any{"id": conversationID, "status": "COMPLETED"}})
+	}))
+	defer server.Close()
+
+	manifest := CollectionManifest{
+		Schema: CollectionSchema, SuiteVersion: suite.SuiteVersion, AgentVersion: "agent-v1",
+		ModelConfigRef: suite.ModelConfigRef, Environment: "mock",
+		ExecutedAt: time.Date(2026, 8, 24, 10, 10, 0, 0, time.UTC),
+	}
+	for index, testCase := range suite.Cases {
+		flags := map[string]bool{}
+		for _, flag := range testCase.Expected.RequiredFlags {
+			flags[flag] = true
+		}
+		manifest.Cases = append(manifest.Cases, CollectionCase{
+			CaseID: testCase.ID, ConversationID: "conversation-" + string(rune('a'+index)), Flags: flags,
+		})
+	}
+	collector, err := NewCollector(server.URL, "test-token", server.Client())
+	if err != nil {
+		t.Fatalf("NewCollector() error = %v", err)
+	}
+	run, err := collector.Collect(t.Context(), suite, manifest)
+	if err != nil {
+		t.Fatalf("Collect() error = %v", err)
+	}
+	if len(run.Observations) != len(suite.Cases) {
+		t.Fatalf("observations = %d", len(run.Observations))
+	}
+	first := run.Observations[0]
+	if !first.ConversationSuccess || first.FirstResponseMS != 750 || first.ResponseChars != 6 || first.EvidenceRef != "conversation:conversation-a" || !first.Flags["salutation_correct"] {
+		t.Fatalf("first observation = %+v", first)
+	}
+}
+
+func TestCollectorRejectsMissingHumanOrJudgeScore(t *testing.T) {
+	suite, _ := validFixture()
+	manifest := CollectionManifest{
+		Schema: CollectionSchema, SuiteVersion: suite.SuiteVersion, AgentVersion: "agent-v1",
+		ModelConfigRef: suite.ModelConfigRef, Environment: "mock", ExecutedAt: time.Now(),
+	}
+	for _, testCase := range suite.Cases {
+		manifest.Cases = append(manifest.Cases, CollectionCase{CaseID: testCase.ID, ConversationID: "conversation-" + testCase.ID, Flags: map[string]bool{}})
+	}
+	collector, err := NewCollector("https://api.example.invalid", "test-token", nil)
+	if err != nil {
+		t.Fatalf("NewCollector() error = %v", err)
+	}
+	_, err = collector.Collect(t.Context(), suite, manifest)
+	if err == nil || !strings.Contains(err.Error(), "does not score required flag") {
+		t.Fatalf("expected scorecard validation error, got %v", err)
+	}
+}

internal/agenteval/evaluator.go

@@ -0,0 +1,505 @@
+package agenteval
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math"
+	"sort"
+	"strings"
+	"time"
+)
+
+const (
+	SuiteSchema       = "agent-eval-suite-v1"
+	ObservationSchema = "agent-eval-observations-v1"
+	ReportSchema      = "agent-eval-report-v1"
+)
+
+var requiredCategories = []string{
+	"normal_chat",
+	"long_pause",
+	"repeated_expression",
+	"name_date_ambiguity",
+	"conflicting_fact",
+	"emotional_topic",
+	"user_interruption",
+	"network_fluctuation",
+	"memory_recall",
+	"sensitive_information",
+}
+
+type Suite struct {
+	Schema         string     `json:"schema"`
+	SuiteVersion   string     `json:"suite_version"`
+	ModelConfigRef string     `json:"model_config_ref"`
+	Thresholds     Thresholds `json:"thresholds"`
+	Cases          []Case     `json:"cases"`
+}
+
+type Thresholds struct {
+	MinConversationSuccessRate  float64 `json:"min_conversation_success_rate"`
+	MaxFirstResponseP95MS       int     `json:"max_first_response_p95_ms"`
+	MinInterruptionRecoveryRate float64 `json:"min_interruption_recovery_rate"`
+	MaxAverageResponseChars     float64 `json:"max_average_response_chars"`
+	MinSalutationAccuracy       float64 `json:"min_salutation_accuracy"`
+	MinMemoryAccuracy           float64 `json:"min_memory_accuracy"`
+	MaxFabricatedFacts          int     `json:"max_fabricated_facts"`
+	MinStorySourceCompleteness  float64 `json:"min_story_source_completeness"`
+	MinSensitiveHandlingRate    float64 `json:"min_sensitive_handling_rate"`
+}
+
+type Case struct {
+	ID             string       `json:"id"`
+	Category       string       `json:"category"`
+	Description    string       `json:"description"`
+	Expected       Expectations `json:"expected"`
+	ScoringRuleRef string       `json:"scoring_rule_ref"`
+}
+
+type Expectations struct {
+	ConversationSuccess *bool    `json:"conversation_success"`
+	MaxFirstResponseMS  int      `json:"max_first_response_ms"`
+	MaxResponseChars    int      `json:"max_response_chars"`
+	MaxFabricatedFacts  int      `json:"max_fabricated_facts"`
+	RequiredFlags       []string `json:"required_flags"`
+}
+
+type ObservationRun struct {
+	Schema         string        `json:"schema"`
+	SuiteVersion   string        `json:"suite_version"`
+	AgentVersion   string        `json:"agent_version"`
+	ModelConfigRef string        `json:"model_config_ref"`
+	Environment    string        `json:"environment"`
+	ExecutedAt     time.Time     `json:"executed_at"`
+	Observations   []Observation `json:"observations"`
+}
+
+type Observation struct {
+	CaseID              string          `json:"case_id"`
+	EvidenceRef         string          `json:"evidence_ref,omitempty"`
+	ConversationSuccess bool            `json:"conversation_success"`
+	FirstResponseMS     int             `json:"first_response_ms"`
+	ResponseChars       int             `json:"response_chars"`
+	FabricatedFacts     int             `json:"fabricated_facts"`
+	Flags               map[string]bool `json:"flags"`
+}
+
+type Metrics struct {
+	CasesTotal                   int            `json:"cases_total"`
+	CasesPassed                  int            `json:"cases_passed"`
+	ConversationSuccessRate      float64        `json:"conversation_success_rate"`
+	FirstResponseP95MS           int            `json:"first_response_p95_ms"`
+	InterruptionRecoveryRate     float64        `json:"interruption_recovery_rate"`
+	AverageResponseChars         float64        `json:"average_response_chars"`
+	SalutationAccuracy           float64        `json:"salutation_accuracy"`
+	MemoryAccuracy               float64        `json:"memory_accuracy"`
+	FabricatedFacts              int            `json:"fabricated_facts"`
+	StorySourceCompleteness      float64        `json:"story_source_completeness"`
+	SensitiveInformationHandling float64        `json:"sensitive_information_handling_rate"`
+	CategoryCounts               map[string]int `json:"category_counts"`
+}
+
+type FailedCheck struct {
+	Code     string `json:"code"`
+	CaseID   string `json:"case_id,omitempty"`
+	Message  string `json:"message"`
+	Actual   any    `json:"actual,omitempty"`
+	Expected any    `json:"expected,omitempty"`
+}
+
+type Report struct {
+	Schema         string        `json:"schema"`
+	SuiteVersion   string        `json:"suite_version"`
+	SuiteSHA256    string        `json:"suite_sha256"`
+	AgentVersion   string        `json:"agent_version"`
+	ModelConfigRef string        `json:"model_config_ref"`
+	Environment    string        `json:"environment"`
+	ExecutedAt     time.Time     `json:"executed_at"`
+	EvaluatedAt    time.Time     `json:"evaluated_at"`
+	Result         string        `json:"result"`
+	Metrics        Metrics       `json:"metrics"`
+	FailedChecks   []FailedCheck `json:"failed_checks"`
+}
+
+type RegistrationPayload struct {
+	EvidenceType string         `json:"evidence_type"`
+	SuiteVersion string         `json:"suite_version"`
+	Result       string         `json:"result"`
+	Metrics      map[string]any `json:"metrics"`
+	ReportRef    string         `json:"report_ref"`
+}
+
+type Artifact struct {
+	Report              Report              `json:"report"`
+	RegistrationPayload RegistrationPayload `json:"registration_payload"`
+}
+
+func Evaluate(suite Suite, run ObservationRun, evaluatedAt time.Time) (Report, error) {
+	if err := validateSuite(suite); err != nil {
+		return Report{}, err
+	}
+	if err := validateRun(suite, run); err != nil {
+		return Report{}, err
+	}
+
+	suiteSHA256, err := SuiteSHA256(suite)
+	if err != nil {
+		return Report{}, err
+	}
+
+	observations := make(map[string]Observation, len(run.Observations))
+	for _, observation := range run.Observations {
+		observations[observation.CaseID] = observation
+	}
+
+	metrics := Metrics{CasesTotal: len(suite.Cases), CategoryCounts: map[string]int{}}
+	failed := make([]FailedCheck, 0)
+	responseTimes := make([]int, 0, len(suite.Cases))
+	responseChars := 0
+	successful := 0
+
+	for _, testCase := range suite.Cases {
+		observation := observations[testCase.ID]
+		metrics.CategoryCounts[testCase.Category]++
+		responseTimes = append(responseTimes, observation.FirstResponseMS)
+		responseChars += observation.ResponseChars
+		metrics.FabricatedFacts += observation.FabricatedFacts
+		if observation.ConversationSuccess {
+			successful++
+		}
+		caseFailures := evaluateCase(testCase, observation)
+		if len(caseFailures) == 0 {
+			metrics.CasesPassed++
+		} else {
+			failed = append(failed, caseFailures...)
+		}
+	}
+
+	metrics.ConversationSuccessRate = rate(successful, len(suite.Cases))
+	metrics.FirstResponseP95MS = percentile95(responseTimes)
+	metrics.AverageResponseChars = float64(responseChars) / float64(len(suite.Cases))
+	metrics.InterruptionRecoveryRate = flagRate(suite.Cases, observations, "user_interruption", "interruption_recovered")
+	metrics.SalutationAccuracy = requiredFlagRate(suite.Cases, observations, "salutation_correct")
+	metrics.MemoryAccuracy = requiredFlagRate(suite.Cases, observations, "memory_correct")
+	metrics.StorySourceCompleteness = requiredFlagRate(suite.Cases, observations, "story_source_complete")
+	metrics.SensitiveInformationHandling = flagRate(suite.Cases, observations, "sensitive_information", "sensitive_handling_pass")
+
+	failed = append(failed, evaluateThresholds(suite.Thresholds, metrics)...)
+	result := "PASSED"
+	if len(failed) > 0 {
+		result = "FAILED"
+	}
+	return Report{
+		Schema: ReportSchema, SuiteVersion: suite.SuiteVersion,
+		SuiteSHA256: suiteSHA256, AgentVersion: run.AgentVersion,
+		ModelConfigRef: run.ModelConfigRef, Environment: run.Environment,
+		ExecutedAt: run.ExecutedAt.UTC(), EvaluatedAt: evaluatedAt.UTC(),
+		Result: result, Metrics: metrics, FailedChecks: failed,
+	}, nil
+}
+
+func SuiteSHA256(suite Suite) (string, error) {
+	encodedSuite, err := json.Marshal(suite)
+	if err != nil {
+		return "", fmt.Errorf("encode suite: %w", err)
+	}
+	digest := sha256.Sum256(encodedSuite)
+	return hex.EncodeToString(digest[:]), nil
+}
+
+func NewArtifact(report Report, reportRef string) (Artifact, error) {
+	reportRef = strings.TrimSpace(reportRef)
+	if reportRef == "" || len([]rune(reportRef)) > 300 {
+		return Artifact{}, errors.New("report_ref must contain 1 to 300 characters")
+	}
+	metrics := map[string]any{
+		"report_schema": report.Schema, "suite_sha256": report.SuiteSHA256,
+		"agent_version": report.AgentVersion, "model_config_ref": report.ModelConfigRef,
+		"environment": report.Environment, "executed_at": report.ExecutedAt,
+		"cases_total": report.Metrics.CasesTotal, "cases_passed": report.Metrics.CasesPassed,
+		"conversation_success_rate":           report.Metrics.ConversationSuccessRate,
+		"first_response_p95_ms":               report.Metrics.FirstResponseP95MS,
+		"interruption_recovery_rate":          report.Metrics.InterruptionRecoveryRate,
+		"average_response_chars":              report.Metrics.AverageResponseChars,
+		"salutation_accuracy":                 report.Metrics.SalutationAccuracy,
+		"memory_accuracy":                     report.Metrics.MemoryAccuracy,
+		"fabricated_facts":                    report.Metrics.FabricatedFacts,
+		"story_source_completeness":           report.Metrics.StorySourceCompleteness,
+		"sensitive_information_handling_rate": report.Metrics.SensitiveInformationHandling,
+		"failed_check_count":                  len(report.FailedChecks),
+	}
+	return Artifact{Report: report, RegistrationPayload: RegistrationPayload{
+		EvidenceType: "AGENT_QUALITY_V1", SuiteVersion: report.SuiteVersion,
+		Result: report.Result, Metrics: metrics, ReportRef: reportRef,
+	}}, nil
+}
+
+func RequiredCategories() []string {
+	return append([]string(nil), requiredCategories...)
+}
+
+func validateSuite(suite Suite) error {
+	if suite.Schema != SuiteSchema {
+		return fmt.Errorf("unsupported suite schema %q", suite.Schema)
+	}
+	if strings.TrimSpace(suite.SuiteVersion) == "" || len([]rune(suite.SuiteVersion)) > 100 {
+		return errors.New("suite_version must contain 1 to 100 characters")
+	}
+	if strings.TrimSpace(suite.ModelConfigRef) == "" {
+		return errors.New("model_config_ref is required")
+	}
+	if len(suite.Cases) == 0 {
+		return errors.New("suite must contain cases")
+	}
+	if err := validateThresholds(suite.Thresholds); err != nil {
+		return err
+	}
+	seenIDs := map[string]struct{}{}
+	seenCategories := map[string]struct{}{}
+	requiredFlagsByCategory := map[string]map[string]struct{}{}
+	for index, testCase := range suite.Cases {
+		testCase.ID = strings.TrimSpace(testCase.ID)
+		if testCase.ID == "" || len([]rune(testCase.ID)) > 100 {
+			return fmt.Errorf("case %d has invalid id", index)
+		}
+		if _, exists := seenIDs[testCase.ID]; exists {
+			return fmt.Errorf("duplicate case id %q", testCase.ID)
+		}
+		seenIDs[testCase.ID] = struct{}{}
+		seenCategories[testCase.Category] = struct{}{}
+		if strings.TrimSpace(testCase.Description) == "" || strings.TrimSpace(testCase.ScoringRuleRef) == "" {
+			return fmt.Errorf("case %q requires description and scoring_rule_ref", testCase.ID)
+		}
+		if testCase.Expected.ConversationSuccess == nil || testCase.Expected.MaxFirstResponseMS <= 0 || testCase.Expected.MaxResponseChars <= 0 || testCase.Expected.MaxFabricatedFacts < 0 {
+			return fmt.Errorf("case %q has invalid expectations", testCase.ID)
+		}
+		if len(testCase.Expected.RequiredFlags) == 0 {
+			return fmt.Errorf("case %q must define required_flags", testCase.ID)
+		}
+		flags := map[string]struct{}{}
+		for _, flag := range testCase.Expected.RequiredFlags {
+			flag = strings.TrimSpace(flag)
+			if flag == "" {
+				return fmt.Errorf("case %q contains an empty required flag", testCase.ID)
+			}
+			if _, exists := flags[flag]; exists {
+				return fmt.Errorf("case %q contains duplicate required flag %q", testCase.ID, flag)
+			}
+			flags[flag] = struct{}{}
+			if requiredFlagsByCategory[testCase.Category] == nil {
+				requiredFlagsByCategory[testCase.Category] = map[string]struct{}{}
+			}
+			requiredFlagsByCategory[testCase.Category][flag] = struct{}{}
+		}
+	}
+	for _, category := range requiredCategories {
+		if _, exists := seenCategories[category]; !exists {
+			return fmt.Errorf("suite does not cover required category %q", category)
+		}
+	}
+	requiredScoringFlags := map[string]string{
+		"salutation_correct":      "normal_chat",
+		"interruption_recovered":  "user_interruption",
+		"memory_correct":          "memory_recall",
+		"sensitive_handling_pass": "sensitive_information",
+		"story_source_complete":   "",
+	}
+	for flag, category := range requiredScoringFlags {
+		found := false
+		if category != "" {
+			_, found = requiredFlagsByCategory[category][flag]
+		} else {
+			for _, flags := range requiredFlagsByCategory {
+				if _, exists := flags[flag]; exists {
+					found = true
+					break
+				}
+			}
+		}
+		if !found {
+			return fmt.Errorf("suite does not score required flag %q", flag)
+		}
+	}
+	return nil
+}
+
+func validateThresholds(thresholds Thresholds) error {
+	rates := map[string]float64{
+		"min_conversation_success_rate":  thresholds.MinConversationSuccessRate,
+		"min_interruption_recovery_rate": thresholds.MinInterruptionRecoveryRate,
+		"min_salutation_accuracy":        thresholds.MinSalutationAccuracy,
+		"min_memory_accuracy":            thresholds.MinMemoryAccuracy,
+		"min_story_source_completeness":  thresholds.MinStorySourceCompleteness,
+		"min_sensitive_handling_rate":    thresholds.MinSensitiveHandlingRate,
+	}
+	for name, value := range rates {
+		if value < 0 || value > 1 {
+			return fmt.Errorf("%s must be between 0 and 1", name)
+		}
+	}
+	if thresholds.MaxFirstResponseP95MS <= 0 || thresholds.MaxAverageResponseChars <= 0 || thresholds.MaxFabricatedFacts < 0 {
+		return errors.New("maximum thresholds must be positive and max_fabricated_facts cannot be negative")
+	}
+	return nil
+}
+
+func validateRun(suite Suite, run ObservationRun) error {
+	if run.Schema != ObservationSchema {
+		return fmt.Errorf("unsupported observation schema %q", run.Schema)
+	}
+	if run.SuiteVersion != suite.SuiteVersion {
+		return fmt.Errorf("observation suite_version %q does not match %q", run.SuiteVersion, suite.SuiteVersion)
+	}
+	if strings.TrimSpace(run.AgentVersion) == "" {
+		return errors.New("agent_version is required")
+	}
+	if run.ModelConfigRef != suite.ModelConfigRef {
+		return fmt.Errorf("model_config_ref %q does not match suite %q", run.ModelConfigRef, suite.ModelConfigRef)
+	}
+	if run.Environment != "mock" && run.Environment != "real" {
+		return errors.New("environment must be mock or real")
+	}
+	if run.ExecutedAt.IsZero() {
+		return errors.New("executed_at is required")
+	}
+	caseIDs := make(map[string]struct{}, len(suite.Cases))
+	for _, testCase := range suite.Cases {
+		caseIDs[testCase.ID] = struct{}{}
+	}
+	seen := make(map[string]struct{}, len(run.Observations))
+	for _, observation := range run.Observations {
+		if _, exists := caseIDs[observation.CaseID]; !exists {
+			return fmt.Errorf("observation references unknown case %q", observation.CaseID)
+		}
+		if _, exists := seen[observation.CaseID]; exists {
+			return fmt.Errorf("duplicate observation for case %q", observation.CaseID)
+		}
+		seen[observation.CaseID] = struct{}{}
+		if observation.FirstResponseMS < 0 || (observation.ConversationSuccess && observation.FirstResponseMS == 0) || observation.ResponseChars < 0 || observation.FabricatedFacts < 0 || observation.Flags == nil {
+			return fmt.Errorf("observation for case %q has invalid metric values", observation.CaseID)
+		}
+	}
+	if len(seen) != len(caseIDs) {
+		missing := make([]string, 0)
+		for caseID := range caseIDs {
+			if _, exists := seen[caseID]; !exists {
+				missing = append(missing, caseID)
+			}
+		}
+		sort.Strings(missing)
+		return fmt.Errorf("observations missing cases: %s", strings.Join(missing, ", "))
+	}
+	return nil
+}
+
+func evaluateCase(testCase Case, observation Observation) []FailedCheck {
+	failed := make([]FailedCheck, 0)
+	if observation.ConversationSuccess != *testCase.Expected.ConversationSuccess {
+		failed = append(failed, FailedCheck{Code: "conversation_success_mismatch", CaseID: testCase.ID, Message: "通话结果与用例期望不一致", Actual: observation.ConversationSuccess, Expected: *testCase.Expected.ConversationSuccess})
+	}
+	if observation.FirstResponseMS > testCase.Expected.MaxFirstResponseMS {
+		failed = append(failed, FailedCheck{Code: "first_response_too_slow", CaseID: testCase.ID, Message: "首个回答时延超过用例上限", Actual: observation.FirstResponseMS, Expected: testCase.Expected.MaxFirstResponseMS})
+	}
+	if observation.ResponseChars > testCase.Expected.MaxResponseChars {
+		failed = append(failed, FailedCheck{Code: "response_too_long", CaseID: testCase.ID, Message: "回答长度超过用例上限", Actual: observation.ResponseChars, Expected: testCase.Expected.MaxResponseChars})
+	}
+	if observation.FabricatedFacts > testCase.Expected.MaxFabricatedFacts {
+		failed = append(failed, FailedCheck{Code: "fabricated_fact", CaseID: testCase.ID, Message: "虚构事实数量超过用例上限", Actual: observation.FabricatedFacts, Expected: testCase.Expected.MaxFabricatedFacts})
+	}
+	for _, flag := range testCase.Expected.RequiredFlags {
+		if !observation.Flags[flag] {
+			failed = append(failed, FailedCheck{Code: "required_flag_failed", CaseID: testCase.ID, Message: "评分规则未通过: " + flag, Actual: false, Expected: true})
+		}
+	}
+	return failed
+}
+
+func evaluateThresholds(thresholds Thresholds, metrics Metrics) []FailedCheck {
+	checks := []struct {
+		code     string
+		message  string
+		actual   float64
+		expected float64
+		minimum  bool
+	}{
+		{"conversation_success_rate_regressed", "通话成功率低于发布门槛", metrics.ConversationSuccessRate, thresholds.MinConversationSuccessRate, true},
+		{"first_response_p95_regressed", "首个回答P95时延超过发布门槛", float64(metrics.FirstResponseP95MS), float64(thresholds.MaxFirstResponseP95MS), false},
+		{"interruption_recovery_regressed", "打断恢复率低于发布门槛", metrics.InterruptionRecoveryRate, thresholds.MinInterruptionRecoveryRate, true},
+		{"average_response_length_regressed", "平均回答长度超过发布门槛", metrics.AverageResponseChars, thresholds.MaxAverageResponseChars, false},
+		{"salutation_accuracy_regressed", "称呼准确率低于发布门槛", metrics.SalutationAccuracy, thresholds.MinSalutationAccuracy, true},
+		{"memory_accuracy_regressed", "记忆准确率低于发布门槛", metrics.MemoryAccuracy, thresholds.MinMemoryAccuracy, true},
+		{"story_source_regressed", "故事来源完整率低于发布门槛", metrics.StorySourceCompleteness, thresholds.MinStorySourceCompleteness, true},
+		{"sensitive_handling_regressed", "敏感信息处理通过率低于发布门槛", metrics.SensitiveInformationHandling, thresholds.MinSensitiveHandlingRate, true},
+	}
+	failed := make([]FailedCheck, 0)
+	for _, check := range checks {
+		violated := check.actual > check.expected
+		if check.minimum {
+			violated = check.actual < check.expected
+		}
+		if violated {
+			failed = append(failed, FailedCheck{Code: check.code, Message: check.message, Actual: check.actual, Expected: check.expected})
+		}
+	}
+	if metrics.FabricatedFacts > thresholds.MaxFabricatedFacts {
+		failed = append(failed, FailedCheck{Code: "fabricated_facts_regressed", Message: "虚构事实总数超过发布门槛", Actual: metrics.FabricatedFacts, Expected: thresholds.MaxFabricatedFacts})
+	}
+	return failed
+}
+
+func percentile95(values []int) int {
+	sorted := append([]int(nil), values...)
+	sort.Ints(sorted)
+	index := int(math.Ceil(float64(len(sorted))*0.95)) - 1
+	if index < 0 {
+		return 0
+	}
+	return sorted[index]
+}
+
+func rate(passed, total int) float64 {
+	if total == 0 {
+		return 0
+	}
+	return float64(passed) / float64(total)
+}
+
+func requiredFlagRate(cases []Case, observations map[string]Observation, flag string) float64 {
+	passed, total := 0, 0
+	for _, testCase := range cases {
+		if !contains(testCase.Expected.RequiredFlags, flag) {
+			continue
+		}
+		total++
+		if observations[testCase.ID].Flags[flag] {
+			passed++
+		}
+	}
+	return rate(passed, total)
+}
+
+func flagRate(cases []Case, observations map[string]Observation, category, flag string) float64 {
+	passed, total := 0, 0
+	for _, testCase := range cases {
+		if testCase.Category != category {
+			continue
+		}
+		total++
+		if observations[testCase.ID].Flags[flag] {
+			passed++
+		}
+	}
+	return rate(passed, total)
+}
+
+func contains(values []string, wanted string) bool {
+	for _, value := range values {
+		if value == wanted {
+			return true
+		}
+	}
+	return false
+}

internal/agenteval/evaluator_test.go

@@ -0,0 +1,110 @@
+package agenteval
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestEvaluatePassesCompleteSuite(t *testing.T) {
+	suite, run := validFixture()
+	report, err := Evaluate(suite, run, time.Date(2026, 8, 24, 10, 5, 0, 0, time.UTC))
+	if err != nil {
+		t.Fatalf("Evaluate() error = %v", err)
+	}
+	if report.Result != "PASSED" || report.Metrics.CasesPassed != len(suite.Cases) || len(report.FailedChecks) != 0 {
+		t.Fatalf("report = %+v", report)
+	}
+	if report.Metrics.InterruptionRecoveryRate != 1 || report.Metrics.MemoryAccuracy != 1 || report.Metrics.SensitiveInformationHandling != 1 {
+		t.Fatalf("required metrics = %+v", report.Metrics)
+	}
+	artifact, err := NewArtifact(report, "object://agent-evaluation/reports/agent-v1.json")
+	if err != nil {
+		t.Fatalf("NewArtifact() error = %v", err)
+	}
+	if artifact.RegistrationPayload.Result != "PASSED" || artifact.RegistrationPayload.Metrics["suite_sha256"] == "" {
+		t.Fatalf("registration payload = %+v", artifact.RegistrationPayload)
+	}
+}
+
+func TestEvaluateBlocksRegression(t *testing.T) {
+	suite, run := validFixture()
+	for index := range run.Observations {
+		if run.Observations[index].CaseID == "memory-recall" {
+			run.Observations[index].FabricatedFacts = 1
+			run.Observations[index].Flags["memory_correct"] = false
+		}
+	}
+	report, err := Evaluate(suite, run, time.Now())
+	if err != nil {
+		t.Fatalf("Evaluate() error = %v", err)
+	}
+	if report.Result != "FAILED" || len(report.FailedChecks) < 3 {
+		t.Fatalf("report should block regression: %+v", report)
+	}
+	codes := map[string]bool{}
+	for _, failure := range report.FailedChecks {
+		codes[failure.Code] = true
+	}
+	if !codes["required_flag_failed"] || !codes["fabricated_fact"] || !codes["memory_accuracy_regressed"] {
+		t.Fatalf("failure codes = %+v", codes)
+	}
+}
+
+func TestEvaluateRejectsIncompleteCoverage(t *testing.T) {
+	suite, run := validFixture()
+	suite.Cases = suite.Cases[:len(suite.Cases)-1]
+	run.Observations = run.Observations[:len(run.Observations)-1]
+	_, err := Evaluate(suite, run, time.Now())
+	if err == nil || !strings.Contains(err.Error(), "sensitive_information") {
+		t.Fatalf("expected missing category error, got %v", err)
+	}
+}
+
+func validFixture() (Suite, ObservationRun) {
+	truth := true
+	categories := RequiredCategories()
+	flags := []string{
+		"salutation_correct", "waited_for_user", "avoided_repetition_loop",
+		"asked_for_clarification", "preserved_confirmed_memory", "emotional_boundary_kept",
+		"interruption_recovered", "network_recovered", "memory_correct", "sensitive_handling_pass",
+	}
+	caseIDs := []string{
+		"normal-chat", "long-pause", "repeated-expression", "name-date-ambiguity", "conflicting-fact",
+		"emotional-topic", "user-interruption", "network-fluctuation", "memory-recall", "sensitive-information",
+	}
+	suite := Suite{
+		Schema: SuiteSchema, SuiteVersion: "agent-quality-v1", ModelConfigRef: "runtime-config://voice/agent-v1",
+		Thresholds: Thresholds{
+			MinConversationSuccessRate: 1, MaxFirstResponseP95MS: 1800,
+			MinInterruptionRecoveryRate: 1, MaxAverageResponseChars: 80,
+			MinSalutationAccuracy: 1, MinMemoryAccuracy: 1, MaxFabricatedFacts: 0,
+			MinStorySourceCompleteness: 1, MinSensitiveHandlingRate: 1,
+		},
+	}
+	run := ObservationRun{
+		Schema: ObservationSchema, SuiteVersion: suite.SuiteVersion, AgentVersion: "agent-v1",
+		ModelConfigRef: suite.ModelConfigRef, Environment: "mock",
+		ExecutedAt: time.Date(2026, 8, 24, 10, 0, 0, 0, time.UTC),
+	}
+	for index, category := range categories {
+		requiredFlags := []string{flags[index]}
+		if category == "conflicting_fact" {
+			requiredFlags = append(requiredFlags, "story_source_complete")
+		}
+		suite.Cases = append(suite.Cases, Case{
+			ID: caseIDs[index], Category: category, Description: "fixed synthetic case",
+			Expected:       Expectations{ConversationSuccess: &truth, MaxFirstResponseMS: 1800, MaxResponseChars: 80, MaxFabricatedFacts: 0, RequiredFlags: requiredFlags},
+			ScoringRuleRef: "agent-scoring-v1#" + category,
+		})
+		observationFlags := map[string]bool{flags[index]: true}
+		if category == "conflicting_fact" {
+			observationFlags["story_source_complete"] = true
+		}
+		run.Observations = append(run.Observations, Observation{
+			CaseID: caseIDs[index], ConversationSuccess: true, FirstResponseMS: 800 + index*50,
+			ResponseChars: 35 + index, Flags: observationFlags,
+		})
+	}
+	return suite, run
+}

internal/agenteval/registrar.go

@@ -0,0 +1,168 @@
+package agenteval
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+type Registrar struct {
+	baseURL string
+	token   string
+	client  *http.Client
+}
+
+type RegistrationResult struct {
+	StatusCode   int
+	Replayed     bool
+	RequestID    string
+	EvaluationID string
+}
+
+func ValidateArtifact(suite Suite, run ObservationRun, artifact Artifact, allowFailed bool) error {
+	if err := validateSuite(suite); err != nil {
+		return err
+	}
+	rebuiltReport, err := Evaluate(suite, run, artifact.Report.EvaluatedAt)
+	if err != nil {
+		return fmt.Errorf("rebuild report from observations: %w", err)
+	}
+	rebuiltJSON, _ := json.Marshal(rebuiltReport)
+	actualReportJSON, _ := json.Marshal(artifact.Report)
+	if !bytes.Equal(rebuiltJSON, actualReportJSON) {
+		return errors.New("report does not match the supplied observations")
+	}
+	digest, err := SuiteSHA256(suite)
+	if err != nil {
+		return err
+	}
+	report := artifact.Report
+	if report.Schema != ReportSchema || report.SuiteVersion != suite.SuiteVersion || report.ModelConfigRef != suite.ModelConfigRef || report.SuiteSHA256 != digest {
+		return errors.New("report does not match the supplied evaluation suite")
+	}
+	if report.Result != "PASSED" && report.Result != "FAILED" {
+		return errors.New("report result must be PASSED or FAILED")
+	}
+	if report.Result == "FAILED" && !allowFailed {
+		return errors.New("FAILED report registration requires explicit allow-failed")
+	}
+	if report.Metrics.CasesTotal != len(suite.Cases) || report.Metrics.CasesPassed < 0 || report.Metrics.CasesPassed > report.Metrics.CasesTotal {
+		return errors.New("report case counts do not match the evaluation suite")
+	}
+	if report.Result == "PASSED" && (len(report.FailedChecks) != 0 || report.Metrics.CasesPassed != report.Metrics.CasesTotal) {
+		return errors.New("PASSED report contains failed checks or incomplete cases")
+	}
+	if report.Result == "FAILED" && len(report.FailedChecks) == 0 {
+		return errors.New("FAILED report does not contain failure evidence")
+	}
+	expected, err := NewArtifact(report, artifact.RegistrationPayload.ReportRef)
+	if err != nil {
+		return err
+	}
+	expectedJSON, _ := json.Marshal(expected.RegistrationPayload)
+	actualJSON, _ := json.Marshal(artifact.RegistrationPayload)
+	if !bytes.Equal(expectedJSON, actualJSON) {
+		return errors.New("registration_payload does not match report contents")
+	}
+	return nil
+}
+
+func RegistrationIdempotencyKey(configID string, payload RegistrationPayload) (string, error) {
+	configID = strings.TrimSpace(configID)
+	if configID == "" {
+		return "", errors.New("configuration id is required")
+	}
+	payload = BindRegistrationPayload(configID, payload)
+	encoded, err := json.Marshal(payload)
+	if err != nil {
+		return "", fmt.Errorf("encode registration payload: %w", err)
+	}
+	digest := sha256.Sum256(append(append([]byte(configID), '\n'), encoded...))
+	return "agent-eval-" + hex.EncodeToString(digest[:16]), nil
+}
+
+func BindRegistrationPayload(configID string, payload RegistrationPayload) RegistrationPayload {
+	metrics := make(map[string]any, len(payload.Metrics)+1)
+	for key, value := range payload.Metrics {
+		metrics[key] = value
+	}
+	metrics["config_version_id"] = strings.TrimSpace(configID)
+	payload.Metrics = metrics
+	return payload
+}
+
+func NewRegistrar(baseURL, accessToken string, client *http.Client) (*Registrar, error) {
+	baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
+	accessToken = strings.TrimSpace(accessToken)
+	parsed, err := url.Parse(baseURL)
+	if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || accessToken == "" {
+		return nil, errors.New("valid API base URL and access token are required")
+	}
+	if client == nil {
+		client = &http.Client{Timeout: 15 * time.Second}
+	}
+	return &Registrar{baseURL: baseURL, token: accessToken, client: client}, nil
+}
+
+func (r *Registrar) Register(ctx context.Context, tenantID, configID string, payload RegistrationPayload) (RegistrationResult, error) {
+	tenantID, configID = strings.TrimSpace(tenantID), strings.TrimSpace(configID)
+	if tenantID == "" || configID == "" {
+		return RegistrationResult{}, errors.New("tenant and configuration ids are required")
+	}
+	payload = BindRegistrationPayload(configID, payload)
+	encoded, err := json.Marshal(payload)
+	if err != nil {
+		return RegistrationResult{}, fmt.Errorf("encode registration payload: %w", err)
+	}
+	key, err := RegistrationIdempotencyKey(configID, payload)
+	if err != nil {
+		return RegistrationResult{}, err
+	}
+	endpoint := r.baseURL + "/api/v1/ops/config-versions/" + url.PathEscape(configID) + "/evaluations"
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(encoded))
+	if err != nil {
+		return RegistrationResult{}, err
+	}
+	request.Header.Set("Authorization", "Bearer "+r.token)
+	request.Header.Set("Content-Type", "application/json")
+	request.Header.Set("Accept", "application/json")
+	request.Header.Set("X-Tenant-Id", tenantID)
+	request.Header.Set("Idempotency-Key", key)
+	response, err := r.client.Do(request)
+	if err != nil {
+		return RegistrationResult{}, err
+	}
+	defer response.Body.Close()
+	result := RegistrationResult{StatusCode: response.StatusCode, Replayed: response.Header.Get("Idempotency-Replayed") == "true", RequestID: response.Header.Get("X-Request-Id")}
+	if response.StatusCode != http.StatusCreated && response.StatusCode != http.StatusOK {
+		_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
+		return result, fmt.Errorf("evaluation registration returned HTTP %d", response.StatusCode)
+	}
+	var envelope struct {
+		RequestID string `json:"request_id"`
+		Data      struct {
+			ID string `json:"id"`
+		} `json:"data"`
+	}
+	decoder := json.NewDecoder(io.LimitReader(response.Body, 1<<20))
+	if err := decoder.Decode(&envelope); err != nil {
+		return result, fmt.Errorf("decode registration response: %w", err)
+	}
+	if envelope.RequestID != "" {
+		result.RequestID = envelope.RequestID
+	}
+	result.EvaluationID = envelope.Data.ID
+	if result.EvaluationID == "" {
+		return result, errors.New("registration response did not contain an evaluation id")
+	}
+	return result, nil
+}

internal/agenteval/registrar_test.go

@@ -0,0 +1,94 @@
+package agenteval
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestValidateArtifactRejectsTamperingAndImplicitFailedRegistration(t *testing.T) {
+	suite, run := validFixture()
+	report, err := Evaluate(suite, run, time.Now())
+	if err != nil {
+		t.Fatalf("Evaluate() error = %v", err)
+	}
+	artifact, err := NewArtifact(report, "artifact://agent-eval/pass.json")
+	if err != nil {
+		t.Fatalf("NewArtifact() error = %v", err)
+	}
+	if err = ValidateArtifact(suite, run, artifact, false); err != nil {
+		t.Fatalf("ValidateArtifact() error = %v", err)
+	}
+	artifact.RegistrationPayload.Metrics["cases_passed"] = 999
+	if err = ValidateArtifact(suite, run, artifact, false); err == nil || (!strings.Contains(err.Error(), "does not match report") && !strings.Contains(err.Error(), "does not match the supplied observations")) {
+		t.Fatalf("expected tampering error, got %v", err)
+	}
+
+	suite, run = validFixture()
+	run.Observations[0].Flags["salutation_correct"] = false
+	failedReport, err := Evaluate(suite, run, time.Now())
+	if err != nil {
+		t.Fatalf("Evaluate() failed report error = %v", err)
+	}
+	failedArtifact, _ := NewArtifact(failedReport, "artifact://agent-eval/failed.json")
+	if err = ValidateArtifact(suite, run, failedArtifact, false); err == nil || !strings.Contains(err.Error(), "allow-failed") {
+		t.Fatalf("expected explicit failed registration guard, got %v", err)
+	}
+	if err = ValidateArtifact(suite, run, failedArtifact, true); err != nil {
+		t.Fatalf("explicit failed registration should be valid: %v", err)
+	}
+}
+
+func TestRegistrarUsesDeterministicIdempotencyAndAcceptsReplay(t *testing.T) {
+	suite, run := validFixture()
+	report, _ := Evaluate(suite, run, time.Now())
+	artifact, _ := NewArtifact(report, "artifact://agent-eval/pass.json")
+	tenantID := "10000000-0000-0000-0000-000000000010"
+	configID := "20000000-0000-0000-0000-000000000020"
+	expectedKey, err := RegistrationIdempotencyKey(configID, artifact.RegistrationPayload)
+	if err != nil {
+		t.Fatalf("RegistrationIdempotencyKey() error = %v", err)
+	}
+	var calls atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		if request.Method != http.MethodPost || request.URL.Path != "/api/v1/ops/config-versions/"+configID+"/evaluations" {
+			http.Error(writer, "unexpected endpoint", http.StatusNotFound)
+			return
+		}
+		if request.Header.Get("Authorization") != "Bearer test-token" || request.Header.Get("X-Tenant-Id") != tenantID || request.Header.Get("Idempotency-Key") != expectedKey {
+			http.Error(writer, "invalid headers", http.StatusBadRequest)
+			return
+		}
+		var payload RegistrationPayload
+		if err := json.NewDecoder(request.Body).Decode(&payload); err != nil || payload.Result != "PASSED" || payload.Metrics["config_version_id"] != configID {
+			http.Error(writer, "invalid body", http.StatusBadRequest)
+			return
+		}
+		writer.Header().Set("Content-Type", "application/json")
+		writer.Header().Set("X-Request-Id", "registration-request")
+		if calls.Add(1) > 1 {
+			writer.Header().Set("Idempotency-Replayed", "true")
+			writer.WriteHeader(http.StatusOK)
+		} else {
+			writer.WriteHeader(http.StatusCreated)
+		}
+		_ = json.NewEncoder(writer).Encode(map[string]any{"request_id": "registration-request", "data": map[string]any{"id": "30000000-0000-0000-0000-000000000030"}})
+	}))
+	defer server.Close()
+	registrar, err := NewRegistrar(server.URL, "test-token", server.Client())
+	if err != nil {
+		t.Fatalf("NewRegistrar() error = %v", err)
+	}
+	first, err := registrar.Register(t.Context(), tenantID, configID, artifact.RegistrationPayload)
+	if err != nil || first.StatusCode != http.StatusCreated || first.Replayed {
+		t.Fatalf("first registration = %+v, err = %v", first, err)
+	}
+	second, err := registrar.Register(t.Context(), tenantID, configID, artifact.RegistrationPayload)
+	if err != nil || second.StatusCode != http.StatusOK || !second.Replayed || second.EvaluationID != first.EvaluationID {
+		t.Fatalf("replayed registration = %+v, err = %v", second, err)
+	}
+}