102 lines
5.1 KiB
Go
102 lines
5.1 KiB
Go
package agentmodelsconformance
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"data-terminal-cloud/internal/agenteval"
|
|
)
|
|
|
|
func TestRealAgentModelsCompleteEvidencePasses(t *testing.T) {
|
|
t.Parallel()
|
|
suite, observations, artifact := agentFixture(t, "real")
|
|
capture := modelsFixture("REAL_PROVIDERS", suite.ModelConfigRef)
|
|
report, err := Evaluate(capture, suite, observations, artifact, capture.CandidateVersion, capture.BuildDigest, time.Now())
|
|
if err != nil || report.Result != "PASS" || report.EvidenceScope != "REAL_AGENT_MODELS" || report.ExternalValidationRequired || report.GenerationCaseCount != 10 || len(report.EmbeddingCategories) != 6 || len(report.ContentSafetyCategories) != 6 {
|
|
t.Fatalf("unexpected report=%+v err=%v", report, err)
|
|
}
|
|
}
|
|
|
|
func TestRealAgentModelsMissingSafetyCategoryFails(t *testing.T) {
|
|
t.Parallel()
|
|
suite, observations, artifact := agentFixture(t, "real")
|
|
capture := modelsFixture("REAL_PROVIDERS", suite.ModelConfigRef)
|
|
capture.ContentSafety.Categories = capture.ContentSafety.Categories[:5]
|
|
report, err := Evaluate(capture, suite, observations, artifact, capture.CandidateVersion, capture.BuildDigest, time.Now())
|
|
if err != nil || report.Result != "FAIL" || !report.ExternalValidationRequired {
|
|
t.Fatalf("expected failed report=%+v err=%v", report, err)
|
|
}
|
|
}
|
|
|
|
func TestLocalAgentModelsCannotCompleteExternalValidation(t *testing.T) {
|
|
t.Parallel()
|
|
suite, observations, artifact := agentFixture(t, "mock")
|
|
capture := modelsFixture("LOCAL_MODELS", suite.ModelConfigRef)
|
|
report, err := Evaluate(capture, suite, observations, artifact, capture.CandidateVersion, capture.BuildDigest, time.Now())
|
|
if err != nil || report.Result != "PASS" || report.EvidenceScope != "LOCAL_AGENT_TOOLCHAIN" || !report.ExternalValidationRequired {
|
|
t.Fatalf("unexpected local report=%+v err=%v", report, err)
|
|
}
|
|
}
|
|
|
|
func TestParseCaptureRejectsUnknownFields(t *testing.T) {
|
|
t.Parallel()
|
|
content, err := json.Marshal(modelsFixture("LOCAL_MODELS", "runtime-config://voice/agent-baseline-v1"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
content = []byte(strings.TrimSuffix(string(content), "}") + `,"unexpected":true}`)
|
|
if _, err = ParseCapture(content); err == nil {
|
|
t.Fatal("expected unknown field rejection")
|
|
}
|
|
}
|
|
|
|
func agentFixture(t *testing.T, environment string) (agenteval.Suite, agenteval.ObservationRun, agenteval.Artifact) {
|
|
t.Helper()
|
|
var suite agenteval.Suite
|
|
var observations agenteval.ObservationRun
|
|
suiteBytes, err := os.ReadFile("../../testdata/agent-eval-suite.v1.json")
|
|
if err != nil || json.Unmarshal(suiteBytes, &suite) != nil {
|
|
t.Fatal("read suite", err)
|
|
}
|
|
observationBytes, err := os.ReadFile("../../testdata/agent-eval-observations.mock.v1.json")
|
|
if err != nil || json.Unmarshal(observationBytes, &observations) != nil {
|
|
t.Fatal("read observations", err)
|
|
}
|
|
observations.Environment = environment
|
|
report, err := agenteval.Evaluate(suite, observations, time.Date(2026, 8, 25, 12, 0, 0, 0, time.UTC))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
artifact, err := agenteval.NewArtifact(report, "evidence://agent-eval")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return suite, observations, artifact
|
|
}
|
|
|
|
func modelsFixture(source, modelConfigRef string) Capture {
|
|
observedAt := time.Date(2026, 8, 25, 10, 0, 0, 0, time.UTC)
|
|
providerClass, environment := "EXTERNAL_MANAGED", "STAGING"
|
|
if source == "LOCAL_MODELS" {
|
|
providerClass, environment = "LOCAL_IMPLEMENTATION", "LOCAL"
|
|
}
|
|
return Capture{
|
|
SchemaVersion: CaptureSchema, EvidenceSource: source, CandidateVersion: "0.63.0-rc.1", BuildDigest: "sha256:" + strings.Repeat("a", 64), Environment: environment,
|
|
ChangeTicketRef: "change-20260825", ExecutedByRef: "agent-qa", AttestedByRef: "release-qa", AttestedAt: observedAt.Add(time.Hour), CollectedAt: observedAt.Add(2 * time.Hour),
|
|
Generation: GenerationEvidence{ProviderClass: providerClass, ProviderRef: "generation-provider", ModelConfigRef: modelConfigRef, ModelResourceRef: "generation-model", AccountRef: "account-ref", RegionRef: "region-ref", DataTermsRef: "terms:generation", QuotaEvidenceRef: "quota:generation", EvaluationEvidenceRef: "evaluation:generation"},
|
|
Embedding: EmbeddingEvidence{ProviderClass: providerClass, ProviderRef: "embedding-provider", ModelRef: "embedding-model", Dimensions: 1536, DatasetRef: "dataset:embedding-v1", ThresholdsRef: "thresholds:embedding-v1", QueryCount: 12, PassedQueryCount: 12, DataTermsRef: "terms:embedding", QuotaEvidenceRef: "quota:embedding", Categories: categoryFixture(RequiredEmbeddingCategories)},
|
|
ContentSafety: ContentSafetyEvidence{ProviderClass: providerClass, ProviderRef: "safety-provider", PolicyRef: "safety-policy-v1", DatasetRef: "dataset:safety-v1", ThresholdsRef: "thresholds:safety-v1", CaseCount: 12, PassedCaseCount: 12, DataTermsRef: "terms:safety", AuditEvidenceRef: "audit:safety", Categories: categoryFixture(RequiredSafetyCategories)},
|
|
}
|
|
}
|
|
|
|
func categoryFixture(categories []string) []CategoryResult {
|
|
results := make([]CategoryResult, 0, len(categories))
|
|
for _, category := range categories {
|
|
results = append(results, CategoryResult{Category: category, Status: "PASS", EvidenceRef: "evidence:" + category})
|
|
}
|
|
return results
|
|
}
|