package agenteval import ( "encoding/json" "net/http" "net/http/httptest" "strings" "sync/atomic" "testing" "time" ) func TestValidateArtifactRejectsTamperingAndImplicitFailedRegistration(t *testing.T) { suite, run := validFixture() report, err := Evaluate(suite, run, time.Now()) if err != nil { t.Fatalf("Evaluate() error = %v", err) } artifact, err := NewArtifact(report, "artifact://agent-eval/pass.json") if err != nil { t.Fatalf("NewArtifact() error = %v", err) } if err = ValidateArtifact(suite, run, artifact, false); err != nil { t.Fatalf("ValidateArtifact() error = %v", err) } artifact.RegistrationPayload.Metrics["cases_passed"] = 999 if err = ValidateArtifact(suite, run, artifact, false); err == nil || (!strings.Contains(err.Error(), "does not match report") && !strings.Contains(err.Error(), "does not match the supplied observations")) { t.Fatalf("expected tampering error, got %v", err) } suite, run = validFixture() run.Observations[0].Flags["salutation_correct"] = false failedReport, err := Evaluate(suite, run, time.Now()) if err != nil { t.Fatalf("Evaluate() failed report error = %v", err) } failedArtifact, _ := NewArtifact(failedReport, "artifact://agent-eval/failed.json") if err = ValidateArtifact(suite, run, failedArtifact, false); err == nil || !strings.Contains(err.Error(), "allow-failed") { t.Fatalf("expected explicit failed registration guard, got %v", err) } if err = ValidateArtifact(suite, run, failedArtifact, true); err != nil { t.Fatalf("explicit failed registration should be valid: %v", err) } } func TestRegistrarUsesDeterministicIdempotencyAndAcceptsReplay(t *testing.T) { suite, run := validFixture() report, _ := Evaluate(suite, run, time.Now()) artifact, _ := NewArtifact(report, "artifact://agent-eval/pass.json") tenantID := "10000000-0000-0000-0000-000000000010" configID := "20000000-0000-0000-0000-000000000020" expectedKey, err := RegistrationIdempotencyKey(configID, artifact.RegistrationPayload) if err != nil { t.Fatalf("RegistrationIdempotencyKey() error = %v", err) } var calls atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { if request.Method != http.MethodPost || request.URL.Path != "/api/v1/ops/config-versions/"+configID+"/evaluations" { http.Error(writer, "unexpected endpoint", http.StatusNotFound) return } if request.Header.Get("Authorization") != "Bearer test-token" || request.Header.Get("X-Tenant-Id") != tenantID || request.Header.Get("Idempotency-Key") != expectedKey { http.Error(writer, "invalid headers", http.StatusBadRequest) return } var payload RegistrationPayload if err := json.NewDecoder(request.Body).Decode(&payload); err != nil || payload.Result != "PASSED" || payload.Metrics["config_version_id"] != configID { http.Error(writer, "invalid body", http.StatusBadRequest) return } writer.Header().Set("Content-Type", "application/json") writer.Header().Set("X-Request-Id", "registration-request") if calls.Add(1) > 1 { writer.Header().Set("Idempotency-Replayed", "true") writer.WriteHeader(http.StatusOK) } else { writer.WriteHeader(http.StatusCreated) } _ = json.NewEncoder(writer).Encode(map[string]any{"request_id": "registration-request", "data": map[string]any{"id": "30000000-0000-0000-0000-000000000030"}}) })) defer server.Close() registrar, err := NewRegistrar(server.URL, "test-token", server.Client()) if err != nil { t.Fatalf("NewRegistrar() error = %v", err) } first, err := registrar.Register(t.Context(), tenantID, configID, artifact.RegistrationPayload) if err != nil || first.StatusCode != http.StatusCreated || first.Replayed { t.Fatalf("first registration = %+v, err = %v", first, err) } second, err := registrar.Register(t.Context(), tenantID, configID, artifact.RegistrationPayload) if err != nil || second.StatusCode != http.StatusOK || !second.Replayed || second.EvaluationID != first.EvaluationID { t.Fatalf("replayed registration = %+v, err = %v", second, err) } }