97 lines
2.7 KiB
Go
97 lines
2.7 KiB
Go
|
package main
|
|||
|
|
|
|||
|
|
import (
|
|||
|
|
"encoding/json"
|
|||
|
|
"errors"
|
|||
|
|
"flag"
|
|||
|
|
"fmt"
|
|||
|
|
"io"
|
|||
|
|
"os"
|
|||
|
|
"time"
|
|||
|
|
|
|||
|
|
"data-terminal-cloud/internal/agenteval"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
func main() {
|
|||
|
|
os.Exit(run(os.Args[1:], os.Stdout, os.Stderr, time.Now))
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func run(args []string, stdout, stderr io.Writer, now func() time.Time) int {
|
|||
|
|
flags := flag.NewFlagSet("agent-eval", flag.ContinueOnError)
|
|||
|
|
flags.SetOutput(stderr)
|
|||
|
|
suitePath := flags.String("suite", "", "评测集JSON路径")
|
|||
|
|
observationsPath := flags.String("observations", "", "脱敏观测结果JSON路径")
|
|||
|
|
outputPath := flags.String("out", "", "报告输出路径；省略时输出到stdout")
|
|||
|
|
reportRef := flags.String("report-ref", "", "登记到运营配置评测证据中的报告引用")
|
|||
|
|
if err := flags.Parse(args); err != nil {
|
|||
|
|
return 2
|
|||
|
|
}
|
|||
|
|
if *suitePath == "" || *observationsPath == "" || *reportRef == "" || flags.NArg() != 0 {
|
|||
|
|
fmt.Fprintln(stderr, "usage: agent-eval -suite <suite.json> -observations <observations.json> -report-ref <reference> [-out <report.json>]")
|
|||
|
|
return 2
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var suite agenteval.Suite
|
|||
|
|
if err := decodeFile(*suitePath, &suite); err != nil {
|
|||
|
|
fmt.Fprintf(stderr, "读取评测集失败: %v\n", err)
|
|||
|
|
return 2
|
|||
|
|
}
|
|||
|
|
var observations agenteval.ObservationRun
|
|||
|
|
if err := decodeFile(*observationsPath, &observations); err != nil {
|
|||
|
|
fmt.Fprintf(stderr, "读取观测结果失败: %v\n", err)
|
|||
|
|
return 2
|
|||
|
|
}
|
|||
|
|
report, err := agenteval.Evaluate(suite, observations, now())
|
|||
|
|
if err != nil {
|
|||
|
|
fmt.Fprintf(stderr, "评测输入无效: %v\n", err)
|
|||
|
|
return 2
|
|||
|
|
}
|
|||
|
|
artifact, err := agenteval.NewArtifact(report, *reportRef)
|
|||
|
|
if err != nil {
|
|||
|
|
fmt.Fprintf(stderr, "报告引用无效: %v\n", err)
|
|||
|
|
return 2
|
|||
|
|
}
|
|||
|
|
encoded, err := json.MarshalIndent(artifact, "", " ")
|
|||
|
|
if err != nil {
|
|||
|
|
fmt.Fprintf(stderr, "生成评测报告失败: %v\n", err)
|
|||
|
|
return 2
|
|||
|
|
}
|
|||
|
|
encoded = append(encoded, '\n')
|
|||
|
|
if *outputPath == "" {
|
|||
|
|
if _, err = stdout.Write(encoded); err != nil {
|
|||
|
|
fmt.Fprintf(stderr, "输出评测报告失败: %v\n", err)
|
|||
|
|
return 2
|
|||
|
|
}
|
|||
|
|
} else if err = os.WriteFile(*outputPath, encoded, 0o600); err != nil {
|
|||
|
|
fmt.Fprintf(stderr, "写入评测报告失败: %v\n", err)
|
|||
|
|
return 2
|
|||
|
|
} else {
|
|||
|
|
fmt.Fprintf(stdout, "%s：%s（%d/%d）\n", report.Result, *outputPath, report.Metrics.CasesPassed, report.Metrics.CasesTotal)
|
|||
|
|
}
|
|||
|
|
if report.Result != "PASSED" {
|
|||
|
|
return 1
|
|||
|
|
}
|
|||
|
|
return 0
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
func decodeFile(path string, target any) error {
|
|||
|
|
file, err := os.Open(path)
|
|||
|
|
if err != nil {
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
defer file.Close()
|
|||
|
|
decoder := json.NewDecoder(file)
|
|||
|
|
decoder.DisallowUnknownFields()
|
|||
|
|
if err = decoder.Decode(target); err != nil {
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
var trailing any
|
|||
|
|
if err = decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
|||
|
|
if err == nil {
|
|||
|
|
return errors.New("JSON文件包含多个顶层值")
|
|||
|
|
}
|
|||
|
|
return err
|
|||
|
|
}
|
|||
|
|
return nil
|
|||
|
|
}
|