506 lines
20 KiB
Go
506 lines
20 KiB
Go
|
package agenteval
|
||
|
|
|
||
|
|
import (
|
||
|
|
"crypto/sha256"
|
||
|
|
"encoding/hex"
|
||
|
|
"encoding/json"
|
||
|
|
"errors"
|
||
|
|
"fmt"
|
||
|
|
"math"
|
||
|
|
"sort"
|
||
|
|
"strings"
|
||
|
|
"time"
|
||
|
|
)
|
||
|
|
|
||
|
|
const (
|
||
|
|
SuiteSchema = "agent-eval-suite-v1"
|
||
|
|
ObservationSchema = "agent-eval-observations-v1"
|
||
|
|
ReportSchema = "agent-eval-report-v1"
|
||
|
|
)
|
||
|
|
|
||
|
|
var requiredCategories = []string{
|
||
|
|
"normal_chat",
|
||
|
|
"long_pause",
|
||
|
|
"repeated_expression",
|
||
|
|
"name_date_ambiguity",
|
||
|
|
"conflicting_fact",
|
||
|
|
"emotional_topic",
|
||
|
|
"user_interruption",
|
||
|
|
"network_fluctuation",
|
||
|
|
"memory_recall",
|
||
|
|
"sensitive_information",
|
||
|
|
}
|
||
|
|
|
||
|
|
type Suite struct {
|
||
|
|
Schema string `json:"schema"`
|
||
|
|
SuiteVersion string `json:"suite_version"`
|
||
|
|
ModelConfigRef string `json:"model_config_ref"`
|
||
|
|
Thresholds Thresholds `json:"thresholds"`
|
||
|
|
Cases []Case `json:"cases"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type Thresholds struct {
|
||
|
|
MinConversationSuccessRate float64 `json:"min_conversation_success_rate"`
|
||
|
|
MaxFirstResponseP95MS int `json:"max_first_response_p95_ms"`
|
||
|
|
MinInterruptionRecoveryRate float64 `json:"min_interruption_recovery_rate"`
|
||
|
|
MaxAverageResponseChars float64 `json:"max_average_response_chars"`
|
||
|
|
MinSalutationAccuracy float64 `json:"min_salutation_accuracy"`
|
||
|
|
MinMemoryAccuracy float64 `json:"min_memory_accuracy"`
|
||
|
|
MaxFabricatedFacts int `json:"max_fabricated_facts"`
|
||
|
|
MinStorySourceCompleteness float64 `json:"min_story_source_completeness"`
|
||
|
|
MinSensitiveHandlingRate float64 `json:"min_sensitive_handling_rate"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type Case struct {
|
||
|
|
ID string `json:"id"`
|
||
|
|
Category string `json:"category"`
|
||
|
|
Description string `json:"description"`
|
||
|
|
Expected Expectations `json:"expected"`
|
||
|
|
ScoringRuleRef string `json:"scoring_rule_ref"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type Expectations struct {
|
||
|
|
ConversationSuccess *bool `json:"conversation_success"`
|
||
|
|
MaxFirstResponseMS int `json:"max_first_response_ms"`
|
||
|
|
MaxResponseChars int `json:"max_response_chars"`
|
||
|
|
MaxFabricatedFacts int `json:"max_fabricated_facts"`
|
||
|
|
RequiredFlags []string `json:"required_flags"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type ObservationRun struct {
|
||
|
|
Schema string `json:"schema"`
|
||
|
|
SuiteVersion string `json:"suite_version"`
|
||
|
|
AgentVersion string `json:"agent_version"`
|
||
|
|
ModelConfigRef string `json:"model_config_ref"`
|
||
|
|
Environment string `json:"environment"`
|
||
|
|
ExecutedAt time.Time `json:"executed_at"`
|
||
|
|
Observations []Observation `json:"observations"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type Observation struct {
|
||
|
|
CaseID string `json:"case_id"`
|
||
|
|
EvidenceRef string `json:"evidence_ref,omitempty"`
|
||
|
|
ConversationSuccess bool `json:"conversation_success"`
|
||
|
|
FirstResponseMS int `json:"first_response_ms"`
|
||
|
|
ResponseChars int `json:"response_chars"`
|
||
|
|
FabricatedFacts int `json:"fabricated_facts"`
|
||
|
|
Flags map[string]bool `json:"flags"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type Metrics struct {
|
||
|
|
CasesTotal int `json:"cases_total"`
|
||
|
|
CasesPassed int `json:"cases_passed"`
|
||
|
|
ConversationSuccessRate float64 `json:"conversation_success_rate"`
|
||
|
|
FirstResponseP95MS int `json:"first_response_p95_ms"`
|
||
|
|
InterruptionRecoveryRate float64 `json:"interruption_recovery_rate"`
|
||
|
|
AverageResponseChars float64 `json:"average_response_chars"`
|
||
|
|
SalutationAccuracy float64 `json:"salutation_accuracy"`
|
||
|
|
MemoryAccuracy float64 `json:"memory_accuracy"`
|
||
|
|
FabricatedFacts int `json:"fabricated_facts"`
|
||
|
|
StorySourceCompleteness float64 `json:"story_source_completeness"`
|
||
|
|
SensitiveInformationHandling float64 `json:"sensitive_information_handling_rate"`
|
||
|
|
CategoryCounts map[string]int `json:"category_counts"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type FailedCheck struct {
|
||
|
|
Code string `json:"code"`
|
||
|
|
CaseID string `json:"case_id,omitempty"`
|
||
|
|
Message string `json:"message"`
|
||
|
|
Actual any `json:"actual,omitempty"`
|
||
|
|
Expected any `json:"expected,omitempty"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type Report struct {
|
||
|
|
Schema string `json:"schema"`
|
||
|
|
SuiteVersion string `json:"suite_version"`
|
||
|
|
SuiteSHA256 string `json:"suite_sha256"`
|
||
|
|
AgentVersion string `json:"agent_version"`
|
||
|
|
ModelConfigRef string `json:"model_config_ref"`
|
||
|
|
Environment string `json:"environment"`
|
||
|
|
ExecutedAt time.Time `json:"executed_at"`
|
||
|
|
EvaluatedAt time.Time `json:"evaluated_at"`
|
||
|
|
Result string `json:"result"`
|
||
|
|
Metrics Metrics `json:"metrics"`
|
||
|
|
FailedChecks []FailedCheck `json:"failed_checks"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type RegistrationPayload struct {
|
||
|
|
EvidenceType string `json:"evidence_type"`
|
||
|
|
SuiteVersion string `json:"suite_version"`
|
||
|
|
Result string `json:"result"`
|
||
|
|
Metrics map[string]any `json:"metrics"`
|
||
|
|
ReportRef string `json:"report_ref"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type Artifact struct {
|
||
|
|
Report Report `json:"report"`
|
||
|
|
RegistrationPayload RegistrationPayload `json:"registration_payload"`
|
||
|
|
}
|
||
|
|
|
||
|
|
func Evaluate(suite Suite, run ObservationRun, evaluatedAt time.Time) (Report, error) {
|
||
|
|
if err := validateSuite(suite); err != nil {
|
||
|
|
return Report{}, err
|
||
|
|
}
|
||
|
|
if err := validateRun(suite, run); err != nil {
|
||
|
|
return Report{}, err
|
||
|
|
}
|
||
|
|
|
||
|
|
suiteSHA256, err := SuiteSHA256(suite)
|
||
|
|
if err != nil {
|
||
|
|
return Report{}, err
|
||
|
|
}
|
||
|
|
|
||
|
|
observations := make(map[string]Observation, len(run.Observations))
|
||
|
|
for _, observation := range run.Observations {
|
||
|
|
observations[observation.CaseID] = observation
|
||
|
|
}
|
||
|
|
|
||
|
|
metrics := Metrics{CasesTotal: len(suite.Cases), CategoryCounts: map[string]int{}}
|
||
|
|
failed := make([]FailedCheck, 0)
|
||
|
|
responseTimes := make([]int, 0, len(suite.Cases))
|
||
|
|
responseChars := 0
|
||
|
|
successful := 0
|
||
|
|
|
||
|
|
for _, testCase := range suite.Cases {
|
||
|
|
observation := observations[testCase.ID]
|
||
|
|
metrics.CategoryCounts[testCase.Category]++
|
||
|
|
responseTimes = append(responseTimes, observation.FirstResponseMS)
|
||
|
|
responseChars += observation.ResponseChars
|
||
|
|
metrics.FabricatedFacts += observation.FabricatedFacts
|
||
|
|
if observation.ConversationSuccess {
|
||
|
|
successful++
|
||
|
|
}
|
||
|
|
caseFailures := evaluateCase(testCase, observation)
|
||
|
|
if len(caseFailures) == 0 {
|
||
|
|
metrics.CasesPassed++
|
||
|
|
} else {
|
||
|
|
failed = append(failed, caseFailures...)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
metrics.ConversationSuccessRate = rate(successful, len(suite.Cases))
|
||
|
|
metrics.FirstResponseP95MS = percentile95(responseTimes)
|
||
|
|
metrics.AverageResponseChars = float64(responseChars) / float64(len(suite.Cases))
|
||
|
|
metrics.InterruptionRecoveryRate = flagRate(suite.Cases, observations, "user_interruption", "interruption_recovered")
|
||
|
|
metrics.SalutationAccuracy = requiredFlagRate(suite.Cases, observations, "salutation_correct")
|
||
|
|
metrics.MemoryAccuracy = requiredFlagRate(suite.Cases, observations, "memory_correct")
|
||
|
|
metrics.StorySourceCompleteness = requiredFlagRate(suite.Cases, observations, "story_source_complete")
|
||
|
|
metrics.SensitiveInformationHandling = flagRate(suite.Cases, observations, "sensitive_information", "sensitive_handling_pass")
|
||
|
|
|
||
|
|
failed = append(failed, evaluateThresholds(suite.Thresholds, metrics)...)
|
||
|
|
result := "PASSED"
|
||
|
|
if len(failed) > 0 {
|
||
|
|
result = "FAILED"
|
||
|
|
}
|
||
|
|
return Report{
|
||
|
|
Schema: ReportSchema, SuiteVersion: suite.SuiteVersion,
|
||
|
|
SuiteSHA256: suiteSHA256, AgentVersion: run.AgentVersion,
|
||
|
|
ModelConfigRef: run.ModelConfigRef, Environment: run.Environment,
|
||
|
|
ExecutedAt: run.ExecutedAt.UTC(), EvaluatedAt: evaluatedAt.UTC(),
|
||
|
|
Result: result, Metrics: metrics, FailedChecks: failed,
|
||
|
|
}, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func SuiteSHA256(suite Suite) (string, error) {
|
||
|
|
encodedSuite, err := json.Marshal(suite)
|
||
|
|
if err != nil {
|
||
|
|
return "", fmt.Errorf("encode suite: %w", err)
|
||
|
|
}
|
||
|
|
digest := sha256.Sum256(encodedSuite)
|
||
|
|
return hex.EncodeToString(digest[:]), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func NewArtifact(report Report, reportRef string) (Artifact, error) {
|
||
|
|
reportRef = strings.TrimSpace(reportRef)
|
||
|
|
if reportRef == "" || len([]rune(reportRef)) > 300 {
|
||
|
|
return Artifact{}, errors.New("report_ref must contain 1 to 300 characters")
|
||
|
|
}
|
||
|
|
metrics := map[string]any{
|
||
|
|
"report_schema": report.Schema, "suite_sha256": report.SuiteSHA256,
|
||
|
|
"agent_version": report.AgentVersion, "model_config_ref": report.ModelConfigRef,
|
||
|
|
"environment": report.Environment, "executed_at": report.ExecutedAt,
|
||
|
|
"cases_total": report.Metrics.CasesTotal, "cases_passed": report.Metrics.CasesPassed,
|
||
|
|
"conversation_success_rate": report.Metrics.ConversationSuccessRate,
|
||
|
|
"first_response_p95_ms": report.Metrics.FirstResponseP95MS,
|
||
|
|
"interruption_recovery_rate": report.Metrics.InterruptionRecoveryRate,
|
||
|
|
"average_response_chars": report.Metrics.AverageResponseChars,
|
||
|
|
"salutation_accuracy": report.Metrics.SalutationAccuracy,
|
||
|
|
"memory_accuracy": report.Metrics.MemoryAccuracy,
|
||
|
|
"fabricated_facts": report.Metrics.FabricatedFacts,
|
||
|
|
"story_source_completeness": report.Metrics.StorySourceCompleteness,
|
||
|
|
"sensitive_information_handling_rate": report.Metrics.SensitiveInformationHandling,
|
||
|
|
"failed_check_count": len(report.FailedChecks),
|
||
|
|
}
|
||
|
|
return Artifact{Report: report, RegistrationPayload: RegistrationPayload{
|
||
|
|
EvidenceType: "AGENT_QUALITY_V1", SuiteVersion: report.SuiteVersion,
|
||
|
|
Result: report.Result, Metrics: metrics, ReportRef: reportRef,
|
||
|
|
}}, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func RequiredCategories() []string {
|
||
|
|
return append([]string(nil), requiredCategories...)
|
||
|
|
}
|
||
|
|
|
||
|
|
func validateSuite(suite Suite) error {
|
||
|
|
if suite.Schema != SuiteSchema {
|
||
|
|
return fmt.Errorf("unsupported suite schema %q", suite.Schema)
|
||
|
|
}
|
||
|
|
if strings.TrimSpace(suite.SuiteVersion) == "" || len([]rune(suite.SuiteVersion)) > 100 {
|
||
|
|
return errors.New("suite_version must contain 1 to 100 characters")
|
||
|
|
}
|
||
|
|
if strings.TrimSpace(suite.ModelConfigRef) == "" {
|
||
|
|
return errors.New("model_config_ref is required")
|
||
|
|
}
|
||
|
|
if len(suite.Cases) == 0 {
|
||
|
|
return errors.New("suite must contain cases")
|
||
|
|
}
|
||
|
|
if err := validateThresholds(suite.Thresholds); err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
seenIDs := map[string]struct{}{}
|
||
|
|
seenCategories := map[string]struct{}{}
|
||
|
|
requiredFlagsByCategory := map[string]map[string]struct{}{}
|
||
|
|
for index, testCase := range suite.Cases {
|
||
|
|
testCase.ID = strings.TrimSpace(testCase.ID)
|
||
|
|
if testCase.ID == "" || len([]rune(testCase.ID)) > 100 {
|
||
|
|
return fmt.Errorf("case %d has invalid id", index)
|
||
|
|
}
|
||
|
|
if _, exists := seenIDs[testCase.ID]; exists {
|
||
|
|
return fmt.Errorf("duplicate case id %q", testCase.ID)
|
||
|
|
}
|
||
|
|
seenIDs[testCase.ID] = struct{}{}
|
||
|
|
seenCategories[testCase.Category] = struct{}{}
|
||
|
|
if strings.TrimSpace(testCase.Description) == "" || strings.TrimSpace(testCase.ScoringRuleRef) == "" {
|
||
|
|
return fmt.Errorf("case %q requires description and scoring_rule_ref", testCase.ID)
|
||
|
|
}
|
||
|
|
if testCase.Expected.ConversationSuccess == nil || testCase.Expected.MaxFirstResponseMS <= 0 || testCase.Expected.MaxResponseChars <= 0 || testCase.Expected.MaxFabricatedFacts < 0 {
|
||
|
|
return fmt.Errorf("case %q has invalid expectations", testCase.ID)
|
||
|
|
}
|
||
|
|
if len(testCase.Expected.RequiredFlags) == 0 {
|
||
|
|
return fmt.Errorf("case %q must define required_flags", testCase.ID)
|
||
|
|
}
|
||
|
|
flags := map[string]struct{}{}
|
||
|
|
for _, flag := range testCase.Expected.RequiredFlags {
|
||
|
|
flag = strings.TrimSpace(flag)
|
||
|
|
if flag == "" {
|
||
|
|
return fmt.Errorf("case %q contains an empty required flag", testCase.ID)
|
||
|
|
}
|
||
|
|
if _, exists := flags[flag]; exists {
|
||
|
|
return fmt.Errorf("case %q contains duplicate required flag %q", testCase.ID, flag)
|
||
|
|
}
|
||
|
|
flags[flag] = struct{}{}
|
||
|
|
if requiredFlagsByCategory[testCase.Category] == nil {
|
||
|
|
requiredFlagsByCategory[testCase.Category] = map[string]struct{}{}
|
||
|
|
}
|
||
|
|
requiredFlagsByCategory[testCase.Category][flag] = struct{}{}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for _, category := range requiredCategories {
|
||
|
|
if _, exists := seenCategories[category]; !exists {
|
||
|
|
return fmt.Errorf("suite does not cover required category %q", category)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
requiredScoringFlags := map[string]string{
|
||
|
|
"salutation_correct": "normal_chat",
|
||
|
|
"interruption_recovered": "user_interruption",
|
||
|
|
"memory_correct": "memory_recall",
|
||
|
|
"sensitive_handling_pass": "sensitive_information",
|
||
|
|
"story_source_complete": "",
|
||
|
|
}
|
||
|
|
for flag, category := range requiredScoringFlags {
|
||
|
|
found := false
|
||
|
|
if category != "" {
|
||
|
|
_, found = requiredFlagsByCategory[category][flag]
|
||
|
|
} else {
|
||
|
|
for _, flags := range requiredFlagsByCategory {
|
||
|
|
if _, exists := flags[flag]; exists {
|
||
|
|
found = true
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if !found {
|
||
|
|
return fmt.Errorf("suite does not score required flag %q", flag)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func validateThresholds(thresholds Thresholds) error {
|
||
|
|
rates := map[string]float64{
|
||
|
|
"min_conversation_success_rate": thresholds.MinConversationSuccessRate,
|
||
|
|
"min_interruption_recovery_rate": thresholds.MinInterruptionRecoveryRate,
|
||
|
|
"min_salutation_accuracy": thresholds.MinSalutationAccuracy,
|
||
|
|
"min_memory_accuracy": thresholds.MinMemoryAccuracy,
|
||
|
|
"min_story_source_completeness": thresholds.MinStorySourceCompleteness,
|
||
|
|
"min_sensitive_handling_rate": thresholds.MinSensitiveHandlingRate,
|
||
|
|
}
|
||
|
|
for name, value := range rates {
|
||
|
|
if value < 0 || value > 1 {
|
||
|
|
return fmt.Errorf("%s must be between 0 and 1", name)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if thresholds.MaxFirstResponseP95MS <= 0 || thresholds.MaxAverageResponseChars <= 0 || thresholds.MaxFabricatedFacts < 0 {
|
||
|
|
return errors.New("maximum thresholds must be positive and max_fabricated_facts cannot be negative")
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func validateRun(suite Suite, run ObservationRun) error {
|
||
|
|
if run.Schema != ObservationSchema {
|
||
|
|
return fmt.Errorf("unsupported observation schema %q", run.Schema)
|
||
|
|
}
|
||
|
|
if run.SuiteVersion != suite.SuiteVersion {
|
||
|
|
return fmt.Errorf("observation suite_version %q does not match %q", run.SuiteVersion, suite.SuiteVersion)
|
||
|
|
}
|
||
|
|
if strings.TrimSpace(run.AgentVersion) == "" {
|
||
|
|
return errors.New("agent_version is required")
|
||
|
|
}
|
||
|
|
if run.ModelConfigRef != suite.ModelConfigRef {
|
||
|
|
return fmt.Errorf("model_config_ref %q does not match suite %q", run.ModelConfigRef, suite.ModelConfigRef)
|
||
|
|
}
|
||
|
|
if run.Environment != "mock" && run.Environment != "real" {
|
||
|
|
return errors.New("environment must be mock or real")
|
||
|
|
}
|
||
|
|
if run.ExecutedAt.IsZero() {
|
||
|
|
return errors.New("executed_at is required")
|
||
|
|
}
|
||
|
|
caseIDs := make(map[string]struct{}, len(suite.Cases))
|
||
|
|
for _, testCase := range suite.Cases {
|
||
|
|
caseIDs[testCase.ID] = struct{}{}
|
||
|
|
}
|
||
|
|
seen := make(map[string]struct{}, len(run.Observations))
|
||
|
|
for _, observation := range run.Observations {
|
||
|
|
if _, exists := caseIDs[observation.CaseID]; !exists {
|
||
|
|
return fmt.Errorf("observation references unknown case %q", observation.CaseID)
|
||
|
|
}
|
||
|
|
if _, exists := seen[observation.CaseID]; exists {
|
||
|
|
return fmt.Errorf("duplicate observation for case %q", observation.CaseID)
|
||
|
|
}
|
||
|
|
seen[observation.CaseID] = struct{}{}
|
||
|
|
if observation.FirstResponseMS < 0 || (observation.ConversationSuccess && observation.FirstResponseMS == 0) || observation.ResponseChars < 0 || observation.FabricatedFacts < 0 || observation.Flags == nil {
|
||
|
|
return fmt.Errorf("observation for case %q has invalid metric values", observation.CaseID)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if len(seen) != len(caseIDs) {
|
||
|
|
missing := make([]string, 0)
|
||
|
|
for caseID := range caseIDs {
|
||
|
|
if _, exists := seen[caseID]; !exists {
|
||
|
|
missing = append(missing, caseID)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
sort.Strings(missing)
|
||
|
|
return fmt.Errorf("observations missing cases: %s", strings.Join(missing, ", "))
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func evaluateCase(testCase Case, observation Observation) []FailedCheck {
|
||
|
|
failed := make([]FailedCheck, 0)
|
||
|
|
if observation.ConversationSuccess != *testCase.Expected.ConversationSuccess {
|
||
|
|
failed = append(failed, FailedCheck{Code: "conversation_success_mismatch", CaseID: testCase.ID, Message: "通话结果与用例期望不一致", Actual: observation.ConversationSuccess, Expected: *testCase.Expected.ConversationSuccess})
|
||
|
|
}
|
||
|
|
if observation.FirstResponseMS > testCase.Expected.MaxFirstResponseMS {
|
||
|
|
failed = append(failed, FailedCheck{Code: "first_response_too_slow", CaseID: testCase.ID, Message: "首个回答时延超过用例上限", Actual: observation.FirstResponseMS, Expected: testCase.Expected.MaxFirstResponseMS})
|
||
|
|
}
|
||
|
|
if observation.ResponseChars > testCase.Expected.MaxResponseChars {
|
||
|
|
failed = append(failed, FailedCheck{Code: "response_too_long", CaseID: testCase.ID, Message: "回答长度超过用例上限", Actual: observation.ResponseChars, Expected: testCase.Expected.MaxResponseChars})
|
||
|
|
}
|
||
|
|
if observation.FabricatedFacts > testCase.Expected.MaxFabricatedFacts {
|
||
|
|
failed = append(failed, FailedCheck{Code: "fabricated_fact", CaseID: testCase.ID, Message: "虚构事实数量超过用例上限", Actual: observation.FabricatedFacts, Expected: testCase.Expected.MaxFabricatedFacts})
|
||
|
|
}
|
||
|
|
for _, flag := range testCase.Expected.RequiredFlags {
|
||
|
|
if !observation.Flags[flag] {
|
||
|
|
failed = append(failed, FailedCheck{Code: "required_flag_failed", CaseID: testCase.ID, Message: "评分规则未通过: " + flag, Actual: false, Expected: true})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return failed
|
||
|
|
}
|
||
|
|
|
||
|
|
func evaluateThresholds(thresholds Thresholds, metrics Metrics) []FailedCheck {
|
||
|
|
checks := []struct {
|
||
|
|
code string
|
||
|
|
message string
|
||
|
|
actual float64
|
||
|
|
expected float64
|
||
|
|
minimum bool
|
||
|
|
}{
|
||
|
|
{"conversation_success_rate_regressed", "通话成功率低于发布门槛", metrics.ConversationSuccessRate, thresholds.MinConversationSuccessRate, true},
|
||
|
|
{"first_response_p95_regressed", "首个回答P95时延超过发布门槛", float64(metrics.FirstResponseP95MS), float64(thresholds.MaxFirstResponseP95MS), false},
|
||
|
|
{"interruption_recovery_regressed", "打断恢复率低于发布门槛", metrics.InterruptionRecoveryRate, thresholds.MinInterruptionRecoveryRate, true},
|
||
|
|
{"average_response_length_regressed", "平均回答长度超过发布门槛", metrics.AverageResponseChars, thresholds.MaxAverageResponseChars, false},
|
||
|
|
{"salutation_accuracy_regressed", "称呼准确率低于发布门槛", metrics.SalutationAccuracy, thresholds.MinSalutationAccuracy, true},
|
||
|
|
{"memory_accuracy_regressed", "记忆准确率低于发布门槛", metrics.MemoryAccuracy, thresholds.MinMemoryAccuracy, true},
|
||
|
|
{"story_source_regressed", "故事来源完整率低于发布门槛", metrics.StorySourceCompleteness, thresholds.MinStorySourceCompleteness, true},
|
||
|
|
{"sensitive_handling_regressed", "敏感信息处理通过率低于发布门槛", metrics.SensitiveInformationHandling, thresholds.MinSensitiveHandlingRate, true},
|
||
|
|
}
|
||
|
|
failed := make([]FailedCheck, 0)
|
||
|
|
for _, check := range checks {
|
||
|
|
violated := check.actual > check.expected
|
||
|
|
if check.minimum {
|
||
|
|
violated = check.actual < check.expected
|
||
|
|
}
|
||
|
|
if violated {
|
||
|
|
failed = append(failed, FailedCheck{Code: check.code, Message: check.message, Actual: check.actual, Expected: check.expected})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if metrics.FabricatedFacts > thresholds.MaxFabricatedFacts {
|
||
|
|
failed = append(failed, FailedCheck{Code: "fabricated_facts_regressed", Message: "虚构事实总数超过发布门槛", Actual: metrics.FabricatedFacts, Expected: thresholds.MaxFabricatedFacts})
|
||
|
|
}
|
||
|
|
return failed
|
||
|
|
}
|
||
|
|
|
||
|
|
func percentile95(values []int) int {
|
||
|
|
sorted := append([]int(nil), values...)
|
||
|
|
sort.Ints(sorted)
|
||
|
|
index := int(math.Ceil(float64(len(sorted))*0.95)) - 1
|
||
|
|
if index < 0 {
|
||
|
|
return 0
|
||
|
|
}
|
||
|
|
return sorted[index]
|
||
|
|
}
|
||
|
|
|
||
|
|
func rate(passed, total int) float64 {
|
||
|
|
if total == 0 {
|
||
|
|
return 0
|
||
|
|
}
|
||
|
|
return float64(passed) / float64(total)
|
||
|
|
}
|
||
|
|
|
||
|
|
func requiredFlagRate(cases []Case, observations map[string]Observation, flag string) float64 {
|
||
|
|
passed, total := 0, 0
|
||
|
|
for _, testCase := range cases {
|
||
|
|
if !contains(testCase.Expected.RequiredFlags, flag) {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
total++
|
||
|
|
if observations[testCase.ID].Flags[flag] {
|
||
|
|
passed++
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return rate(passed, total)
|
||
|
|
}
|
||
|
|
|
||
|
|
func flagRate(cases []Case, observations map[string]Observation, category, flag string) float64 {
|
||
|
|
passed, total := 0, 0
|
||
|
|
for _, testCase := range cases {
|
||
|
|
if testCase.Category != category {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
total++
|
||
|
|
if observations[testCase.ID].Flags[flag] {
|
||
|
|
passed++
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return rate(passed, total)
|
||
|
|
}
|
||
|
|
|
||
|
|
func contains(values []string, wanted string) bool {
|
||
|
|
for _, value := range values {
|
||
|
|
if value == wanted {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|