story/internal/agenteval/evaluator_test.go

package agenteval

import (
	"strings"
	"testing"
	"time"
)

func TestEvaluatePassesCompleteSuite(t *testing.T) {
	suite, run := validFixture()
	report, err := Evaluate(suite, run, time.Date(2026, 8, 24, 10, 5, 0, 0, time.UTC))
	if err != nil {
		t.Fatalf("Evaluate() error = %v", err)
	}
	if report.Result != "PASSED" || report.Metrics.CasesPassed != len(suite.Cases) || len(report.FailedChecks) != 0 {
		t.Fatalf("report = %+v", report)
	}
	if report.Metrics.InterruptionRecoveryRate != 1 || report.Metrics.MemoryAccuracy != 1 || report.Metrics.SensitiveInformationHandling != 1 {
		t.Fatalf("required metrics = %+v", report.Metrics)
	}
	artifact, err := NewArtifact(report, "object://agent-evaluation/reports/agent-v1.json")
	if err != nil {
		t.Fatalf("NewArtifact() error = %v", err)
	}
	if artifact.RegistrationPayload.Result != "PASSED" || artifact.RegistrationPayload.Metrics["suite_sha256"] == "" {
		t.Fatalf("registration payload = %+v", artifact.RegistrationPayload)
	}
}

func TestEvaluateBlocksRegression(t *testing.T) {
	suite, run := validFixture()
	for index := range run.Observations {
		if run.Observations[index].CaseID == "memory-recall" {
			run.Observations[index].FabricatedFacts = 1
			run.Observations[index].Flags["memory_correct"] = false
		}
	}
	report, err := Evaluate(suite, run, time.Now())
	if err != nil {
		t.Fatalf("Evaluate() error = %v", err)
	}
	if report.Result != "FAILED" || len(report.FailedChecks) < 3 {
		t.Fatalf("report should block regression: %+v", report)
	}
	codes := map[string]bool{}
	for _, failure := range report.FailedChecks {
		codes[failure.Code] = true
	}
	if !codes["required_flag_failed"] || !codes["fabricated_fact"] || !codes["memory_accuracy_regressed"] {
		t.Fatalf("failure codes = %+v", codes)
	}
}

func TestEvaluateRejectsIncompleteCoverage(t *testing.T) {
	suite, run := validFixture()
	suite.Cases = suite.Cases[:len(suite.Cases)-1]
	run.Observations = run.Observations[:len(run.Observations)-1]
	_, err := Evaluate(suite, run, time.Now())
	if err == nil || !strings.Contains(err.Error(), "sensitive_information") {
		t.Fatalf("expected missing category error, got %v", err)
	}
}

func validFixture() (Suite, ObservationRun) {
	truth := true
	categories := RequiredCategories()
	flags := []string{
		"salutation_correct", "waited_for_user", "avoided_repetition_loop",
		"asked_for_clarification", "preserved_confirmed_memory", "emotional_boundary_kept",
		"interruption_recovered", "network_recovered", "memory_correct", "sensitive_handling_pass",
	}
	caseIDs := []string{
		"normal-chat", "long-pause", "repeated-expression", "name-date-ambiguity", "conflicting-fact",
		"emotional-topic", "user-interruption", "network-fluctuation", "memory-recall", "sensitive-information",
	}
	suite := Suite{
		Schema: SuiteSchema, SuiteVersion: "agent-quality-v1", ModelConfigRef: "runtime-config://voice/agent-v1",
		Thresholds: Thresholds{
			MinConversationSuccessRate: 1, MaxFirstResponseP95MS: 1800,
			MinInterruptionRecoveryRate: 1, MaxAverageResponseChars: 80,
			MinSalutationAccuracy: 1, MinMemoryAccuracy: 1, MaxFabricatedFacts: 0,
			MinStorySourceCompleteness: 1, MinSensitiveHandlingRate: 1,
		},
	}
	run := ObservationRun{
		Schema: ObservationSchema, SuiteVersion: suite.SuiteVersion, AgentVersion: "agent-v1",
		ModelConfigRef: suite.ModelConfigRef, Environment: "mock",
		ExecutedAt: time.Date(2026, 8, 24, 10, 0, 0, 0, time.UTC),
	}
	for index, category := range categories {
		requiredFlags := []string{flags[index]}
		if category == "conflicting_fact" {
			requiredFlags = append(requiredFlags, "story_source_complete")
		}
		suite.Cases = append(suite.Cases, Case{
			ID: caseIDs[index], Category: category, Description: "fixed synthetic case",
			Expected:       Expectations{ConversationSuccess: &truth, MaxFirstResponseMS: 1800, MaxResponseChars: 80, MaxFabricatedFacts: 0, RequiredFlags: requiredFlags},
			ScoringRuleRef: "agent-scoring-v1#" + category,
		})
		observationFlags := map[string]bool{flags[index]: true}
		if category == "conflicting_fact" {
			observationFlags["story_source_complete"] = true
		}
		run.Observations = append(run.Observations, Observation{
			CaseID: caseIDs[index], ConversationSuccess: true, FirstResponseMS: 800 + index*50,
			ResponseChars: 35 + index, Flags: observationFlags,
		})
	}
	return suite, run
}