97 lines
2.7 KiB
Go
97 lines
2.7 KiB
Go
package main
|
||
|
||
import (
|
||
"encoding/json"
|
||
"errors"
|
||
"flag"
|
||
"fmt"
|
||
"io"
|
||
"os"
|
||
"time"
|
||
|
||
"data-terminal-cloud/internal/agenteval"
|
||
)
|
||
|
||
func main() {
|
||
os.Exit(run(os.Args[1:], os.Stdout, os.Stderr, time.Now))
|
||
}
|
||
|
||
func run(args []string, stdout, stderr io.Writer, now func() time.Time) int {
|
||
flags := flag.NewFlagSet("agent-eval", flag.ContinueOnError)
|
||
flags.SetOutput(stderr)
|
||
suitePath := flags.String("suite", "", "评测集JSON路径")
|
||
observationsPath := flags.String("observations", "", "脱敏观测结果JSON路径")
|
||
outputPath := flags.String("out", "", "报告输出路径；省略时输出到stdout")
|
||
reportRef := flags.String("report-ref", "", "登记到运营配置评测证据中的报告引用")
|
||
if err := flags.Parse(args); err != nil {
|
||
return 2
|
||
}
|
||
if *suitePath == "" || *observationsPath == "" || *reportRef == "" || flags.NArg() != 0 {
|
||
fmt.Fprintln(stderr, "usage: agent-eval -suite <suite.json> -observations <observations.json> -report-ref <reference> [-out <report.json>]")
|
||
return 2
|
||
}
|
||
|
||
var suite agenteval.Suite
|
||
if err := decodeFile(*suitePath, &suite); err != nil {
|
||
fmt.Fprintf(stderr, "读取评测集失败: %v\n", err)
|
||
return 2
|
||
}
|
||
var observations agenteval.ObservationRun
|
||
if err := decodeFile(*observationsPath, &observations); err != nil {
|
||
fmt.Fprintf(stderr, "读取观测结果失败: %v\n", err)
|
||
return 2
|
||
}
|
||
report, err := agenteval.Evaluate(suite, observations, now())
|
||
if err != nil {
|
||
fmt.Fprintf(stderr, "评测输入无效: %v\n", err)
|
||
return 2
|
||
}
|
||
artifact, err := agenteval.NewArtifact(report, *reportRef)
|
||
if err != nil {
|
||
fmt.Fprintf(stderr, "报告引用无效: %v\n", err)
|
||
return 2
|
||
}
|
||
encoded, err := json.MarshalIndent(artifact, "", " ")
|
||
if err != nil {
|
||
fmt.Fprintf(stderr, "生成评测报告失败: %v\n", err)
|
||
return 2
|
||
}
|
||
encoded = append(encoded, '\n')
|
||
if *outputPath == "" {
|
||
if _, err = stdout.Write(encoded); err != nil {
|
||
fmt.Fprintf(stderr, "输出评测报告失败: %v\n", err)
|
||
return 2
|
||
}
|
||
} else if err = os.WriteFile(*outputPath, encoded, 0o600); err != nil {
|
||
fmt.Fprintf(stderr, "写入评测报告失败: %v\n", err)
|
||
return 2
|
||
} else {
|
||
fmt.Fprintf(stdout, "%s：%s（%d/%d）\n", report.Result, *outputPath, report.Metrics.CasesPassed, report.Metrics.CasesTotal)
|
||
}
|
||
if report.Result != "PASSED" {
|
||
return 1
|
||
}
|
||
return 0
|
||
}
|
||
|
||
func decodeFile(path string, target any) error {
|
||
file, err := os.Open(path)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer file.Close()
|
||
decoder := json.NewDecoder(file)
|
||
decoder.DisallowUnknownFields()
|
||
if err = decoder.Decode(target); err != nil {
|
||
return err
|
||
}
|
||
var trailing any
|
||
if err = decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||
if err == nil {
|
||
return errors.New("JSON文件包含多个顶层值")
|
||
}
|
||
return err
|
||
}
|
||
return nil
|
||
}
|