252 lines
8.5 KiB
Go
252 lines
8.5 KiB
Go
package agenteval
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const CollectionSchema = "agent-eval-collection-v1"
|
|
|
|
type CollectionManifest struct {
|
|
Schema string `json:"schema"`
|
|
SuiteVersion string `json:"suite_version"`
|
|
AgentVersion string `json:"agent_version"`
|
|
ModelConfigRef string `json:"model_config_ref"`
|
|
Environment string `json:"environment"`
|
|
ExecutedAt time.Time `json:"executed_at"`
|
|
Cases []CollectionCase `json:"cases"`
|
|
}
|
|
|
|
type CollectionCase struct {
|
|
CaseID string `json:"case_id"`
|
|
ConversationID string `json:"conversation_id"`
|
|
FabricatedFacts int `json:"fabricated_facts"`
|
|
Flags map[string]bool `json:"flags"`
|
|
}
|
|
|
|
type Collector struct {
|
|
baseURL string
|
|
token string
|
|
client *http.Client
|
|
}
|
|
|
|
type conversationDetailEnvelope struct {
|
|
Data struct {
|
|
ID string `json:"id"`
|
|
Status string `json:"status"`
|
|
} `json:"data"`
|
|
}
|
|
|
|
type conversationTurnsEnvelope struct {
|
|
Data struct {
|
|
Items []conversationTurn `json:"items"`
|
|
NextCursor string `json:"next_cursor"`
|
|
} `json:"data"`
|
|
}
|
|
|
|
type conversationTurn struct {
|
|
SequenceNo int64 `json:"sequence_no"`
|
|
Speaker string `json:"speaker"`
|
|
StartAt time.Time `json:"start_at"`
|
|
EndAt time.Time `json:"end_at"`
|
|
FinalText string `json:"final_text"`
|
|
Interrupted bool `json:"interrupted"`
|
|
}
|
|
|
|
func NewCollector(baseURL, accessToken string, client *http.Client) (*Collector, error) {
|
|
baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
|
|
accessToken = strings.TrimSpace(accessToken)
|
|
parsed, err := url.Parse(baseURL)
|
|
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || accessToken == "" {
|
|
return nil, errors.New("valid API base URL and access token are required")
|
|
}
|
|
if client == nil {
|
|
client = &http.Client{Timeout: 15 * time.Second}
|
|
}
|
|
return &Collector{baseURL: baseURL, token: accessToken, client: client}, nil
|
|
}
|
|
|
|
func (c *Collector) Collect(ctx context.Context, suite Suite, manifest CollectionManifest) (ObservationRun, error) {
|
|
if err := validateSuite(suite); err != nil {
|
|
return ObservationRun{}, err
|
|
}
|
|
if err := validateCollectionManifest(suite, manifest); err != nil {
|
|
return ObservationRun{}, err
|
|
}
|
|
run := ObservationRun{
|
|
Schema: ObservationSchema, SuiteVersion: manifest.SuiteVersion,
|
|
AgentVersion: manifest.AgentVersion, ModelConfigRef: manifest.ModelConfigRef,
|
|
Environment: manifest.Environment, ExecutedAt: manifest.ExecutedAt.UTC(),
|
|
Observations: make([]Observation, 0, len(manifest.Cases)),
|
|
}
|
|
for _, item := range manifest.Cases {
|
|
detail, err := c.getDetail(ctx, item.ConversationID)
|
|
if err != nil {
|
|
return ObservationRun{}, fmt.Errorf("collect case %s detail: %w", item.CaseID, err)
|
|
}
|
|
turns, err := c.getTurns(ctx, item.ConversationID)
|
|
if err != nil {
|
|
return ObservationRun{}, fmt.Errorf("collect case %s turns: %w", item.CaseID, err)
|
|
}
|
|
firstResponseMS, responseChars := deriveTurnMetrics(turns)
|
|
success := detail.Data.Status == "COMPLETED" && firstResponseMS > 0
|
|
run.Observations = append(run.Observations, Observation{
|
|
CaseID: item.CaseID, EvidenceRef: "conversation:" + item.ConversationID,
|
|
ConversationSuccess: success, FirstResponseMS: firstResponseMS,
|
|
ResponseChars: responseChars, FabricatedFacts: item.FabricatedFacts,
|
|
Flags: cloneFlags(item.Flags),
|
|
})
|
|
}
|
|
return run, nil
|
|
}
|
|
|
|
func validateCollectionManifest(suite Suite, manifest CollectionManifest) error {
|
|
if manifest.Schema != CollectionSchema {
|
|
return fmt.Errorf("unsupported collection schema %q", manifest.Schema)
|
|
}
|
|
if manifest.SuiteVersion != suite.SuiteVersion || manifest.ModelConfigRef != suite.ModelConfigRef {
|
|
return errors.New("collection suite or model configuration does not match evaluation suite")
|
|
}
|
|
if strings.TrimSpace(manifest.AgentVersion) == "" || (manifest.Environment != "mock" && manifest.Environment != "real") || manifest.ExecutedAt.IsZero() {
|
|
return errors.New("collection requires agent_version, mock or real environment, and executed_at")
|
|
}
|
|
wanted := make(map[string]Case, len(suite.Cases))
|
|
for _, testCase := range suite.Cases {
|
|
wanted[testCase.ID] = testCase
|
|
}
|
|
seen := make(map[string]struct{}, len(manifest.Cases))
|
|
conversationIDs := make(map[string]struct{}, len(manifest.Cases))
|
|
for _, item := range manifest.Cases {
|
|
testCase, exists := wanted[item.CaseID]
|
|
if !exists {
|
|
return fmt.Errorf("collection references unknown case %q", item.CaseID)
|
|
}
|
|
if _, duplicate := seen[item.CaseID]; duplicate {
|
|
return fmt.Errorf("collection repeats case %q", item.CaseID)
|
|
}
|
|
seen[item.CaseID] = struct{}{}
|
|
if strings.TrimSpace(item.ConversationID) == "" || item.FabricatedFacts < 0 || item.Flags == nil {
|
|
return fmt.Errorf("collection case %q has invalid evidence", item.CaseID)
|
|
}
|
|
if _, duplicate := conversationIDs[item.ConversationID]; duplicate {
|
|
return fmt.Errorf("conversation %q is reused by multiple evaluation cases", item.ConversationID)
|
|
}
|
|
conversationIDs[item.ConversationID] = struct{}{}
|
|
for _, flag := range testCase.Expected.RequiredFlags {
|
|
if _, exists := item.Flags[flag]; !exists {
|
|
return fmt.Errorf("collection case %q does not score required flag %q", item.CaseID, flag)
|
|
}
|
|
}
|
|
}
|
|
if len(seen) != len(wanted) {
|
|
missing := make([]string, 0)
|
|
for caseID := range wanted {
|
|
if _, exists := seen[caseID]; !exists {
|
|
missing = append(missing, caseID)
|
|
}
|
|
}
|
|
sort.Strings(missing)
|
|
return fmt.Errorf("collection missing cases: %s", strings.Join(missing, ", "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *Collector) getDetail(ctx context.Context, conversationID string) (conversationDetailEnvelope, error) {
|
|
var envelope conversationDetailEnvelope
|
|
err := c.getJSON(ctx, "/api/v1/conversations/"+url.PathEscape(conversationID), &envelope)
|
|
if err == nil && envelope.Data.ID != "" && envelope.Data.ID != conversationID {
|
|
return conversationDetailEnvelope{}, errors.New("conversation detail returned a different id")
|
|
}
|
|
return envelope, err
|
|
}
|
|
|
|
func (c *Collector) getTurns(ctx context.Context, conversationID string) ([]conversationTurn, error) {
|
|
items := make([]conversationTurn, 0)
|
|
cursor := ""
|
|
for page := 0; page < 20; page++ {
|
|
path := "/api/v1/conversations/" + url.PathEscape(conversationID) + "/turns?limit=100"
|
|
if cursor != "" {
|
|
path += "&cursor=" + url.QueryEscape(cursor)
|
|
}
|
|
var envelope conversationTurnsEnvelope
|
|
if err := c.getJSON(ctx, path, &envelope); err != nil {
|
|
return nil, err
|
|
}
|
|
items = append(items, envelope.Data.Items...)
|
|
if envelope.Data.NextCursor == "" {
|
|
return items, nil
|
|
}
|
|
if envelope.Data.NextCursor == cursor {
|
|
return nil, errors.New("turn pagination returned the same cursor")
|
|
}
|
|
cursor = envelope.Data.NextCursor
|
|
}
|
|
return nil, errors.New("turn pagination exceeded 20 pages")
|
|
}
|
|
|
|
func (c *Collector) getJSON(ctx context.Context, path string, target any) error {
|
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+path, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
request.Header.Set("Authorization", "Bearer "+c.token)
|
|
request.Header.Set("Accept", "application/json")
|
|
response, err := c.client.Do(request)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer response.Body.Close()
|
|
if response.StatusCode != http.StatusOK {
|
|
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
|
|
return fmt.Errorf("API returned HTTP %d", response.StatusCode)
|
|
}
|
|
decoder := json.NewDecoder(io.LimitReader(response.Body, 2<<20))
|
|
if err := decoder.Decode(target); err != nil {
|
|
return fmt.Errorf("decode API response: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func deriveTurnMetrics(turns []conversationTurn) (int, int) {
|
|
sorted := append([]conversationTurn(nil), turns...)
|
|
sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].SequenceNo < sorted[j].SequenceNo })
|
|
var subjectEnd time.Time
|
|
firstResponseMS, maxResponseChars := 0, 0
|
|
for _, turn := range sorted {
|
|
switch turn.Speaker {
|
|
case "subject":
|
|
if turn.EndAt.After(subjectEnd) {
|
|
subjectEnd = turn.EndAt
|
|
}
|
|
case "agent":
|
|
length := len([]rune(turn.FinalText))
|
|
if length > maxResponseChars {
|
|
maxResponseChars = length
|
|
}
|
|
if firstResponseMS == 0 && !subjectEnd.IsZero() && !turn.StartAt.Before(subjectEnd) {
|
|
firstResponseMS = int(turn.StartAt.Sub(subjectEnd).Milliseconds())
|
|
if firstResponseMS == 0 {
|
|
firstResponseMS = 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return firstResponseMS, maxResponseChars
|
|
}
|
|
|
|
func cloneFlags(source map[string]bool) map[string]bool {
|
|
result := make(map[string]bool, len(source))
|
|
for key, value := range source {
|
|
result[key] = value
|
|
}
|
|
return result
|
|
}
|