89 lines
3.2 KiB
Go
89 lines
3.2 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"data-terminal-cloud/internal/agenteval"
|
|
"data-terminal-cloud/internal/agentmodelsconformance"
|
|
)
|
|
|
|
func main() {
|
|
capturePath := flag.String("capture", "", "real-agent-models-capture-v1 JSON")
|
|
suitePath := flag.String("suite", "", "agent evaluation suite JSON")
|
|
observationsPath := flag.String("observations", "", "agent evaluation observations JSON")
|
|
agentReportPath := flag.String("agent-report", "", "agent evaluation artifact JSON")
|
|
candidateVersion := flag.String("candidate-version", "", "release candidate version")
|
|
buildDigest := flag.String("build-digest", "", "sha256 immutable build digest")
|
|
outputPath := flag.String("output", "", "absolute output report path")
|
|
flag.Parse()
|
|
if *capturePath == "" || *suitePath == "" || *observationsPath == "" || *agentReportPath == "" || strings.TrimSpace(*candidateVersion) == "" || strings.TrimSpace(*buildDigest) == "" || !filepath.IsAbs(*outputPath) {
|
|
fmt.Fprintln(os.Stderr, "usage: agent-models-conformance -capture capture.json -suite suite.json -observations observations.json -agent-report report.json -candidate-version VERSION -build-digest sha256:... -output ABSOLUTE.json")
|
|
os.Exit(2)
|
|
}
|
|
var capture agentmodelsconformance.Capture
|
|
var suite agenteval.Suite
|
|
var observations agenteval.ObservationRun
|
|
var artifact agenteval.Artifact
|
|
captureBytes, err := decodeFile(*capturePath, &capture)
|
|
if err == nil {
|
|
_, err = decodeFile(*suitePath, &suite)
|
|
}
|
|
if err == nil {
|
|
_, err = decodeFile(*observationsPath, &observations)
|
|
}
|
|
if err == nil {
|
|
_, err = decodeFile(*agentReportPath, &artifact)
|
|
}
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "read conformance input:", err)
|
|
os.Exit(2)
|
|
}
|
|
suiteBytes, _ := os.ReadFile(*suitePath)
|
|
observationsBytes, _ := os.ReadFile(*observationsPath)
|
|
agentReportBytes, _ := os.ReadFile(*agentReportPath)
|
|
report, err := agentmodelsconformance.Evaluate(capture, suite, observations, artifact, *candidateVersion, *buildDigest, time.Now().UTC())
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, err)
|
|
os.Exit(2)
|
|
}
|
|
report.InputSHA256["models_capture"] = agentmodelsconformance.SHA256(captureBytes)
|
|
report.InputSHA256["agent_suite"] = agentmodelsconformance.SHA256(suiteBytes)
|
|
report.InputSHA256["agent_observations"] = agentmodelsconformance.SHA256(observationsBytes)
|
|
report.InputSHA256["agent_report"] = agentmodelsconformance.SHA256(agentReportBytes)
|
|
encoded, _ := json.MarshalIndent(report, "", " ")
|
|
encoded = append(encoded, '\n')
|
|
if err = os.WriteFile(*outputPath, encoded, 0o600); err != nil {
|
|
fmt.Fprintln(os.Stderr, "write Agent models report:", err)
|
|
os.Exit(2)
|
|
}
|
|
fmt.Printf("real Agent models conformance %s: %s\n", report.Result, *outputPath)
|
|
if report.Result != "PASS" {
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func decodeFile(path string, target any) ([]byte, error) {
|
|
content, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
decoder := json.NewDecoder(strings.NewReader(string(content)))
|
|
decoder.DisallowUnknownFields()
|
|
if err = decoder.Decode(target); err != nil {
|
|
return nil, err
|
|
}
|
|
var trailing any
|
|
if err = decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
|
return nil, errors.New("JSON contains trailing data")
|
|
}
|
|
return content, nil
|
|
}
|