chore: establish local platform baseline
This commit is contained in:
Codex Local
@@ -0,0 +1,505 @@
|
||||
package agenteval
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
SuiteSchema = "agent-eval-suite-v1"
|
||||
ObservationSchema = "agent-eval-observations-v1"
|
||||
ReportSchema = "agent-eval-report-v1"
|
||||
)
|
||||
|
||||
var requiredCategories = []string{
|
||||
"normal_chat",
|
||||
"long_pause",
|
||||
"repeated_expression",
|
||||
"name_date_ambiguity",
|
||||
"conflicting_fact",
|
||||
"emotional_topic",
|
||||
"user_interruption",
|
||||
"network_fluctuation",
|
||||
"memory_recall",
|
||||
"sensitive_information",
|
||||
}
|
||||
|
||||
type Suite struct {
|
||||
Schema string `json:"schema"`
|
||||
SuiteVersion string `json:"suite_version"`
|
||||
ModelConfigRef string `json:"model_config_ref"`
|
||||
Thresholds Thresholds `json:"thresholds"`
|
||||
Cases []Case `json:"cases"`
|
||||
}
|
||||
|
||||
type Thresholds struct {
|
||||
MinConversationSuccessRate float64 `json:"min_conversation_success_rate"`
|
||||
MaxFirstResponseP95MS int `json:"max_first_response_p95_ms"`
|
||||
MinInterruptionRecoveryRate float64 `json:"min_interruption_recovery_rate"`
|
||||
MaxAverageResponseChars float64 `json:"max_average_response_chars"`
|
||||
MinSalutationAccuracy float64 `json:"min_salutation_accuracy"`
|
||||
MinMemoryAccuracy float64 `json:"min_memory_accuracy"`
|
||||
MaxFabricatedFacts int `json:"max_fabricated_facts"`
|
||||
MinStorySourceCompleteness float64 `json:"min_story_source_completeness"`
|
||||
MinSensitiveHandlingRate float64 `json:"min_sensitive_handling_rate"`
|
||||
}
|
||||
|
||||
type Case struct {
|
||||
ID string `json:"id"`
|
||||
Category string `json:"category"`
|
||||
Description string `json:"description"`
|
||||
Expected Expectations `json:"expected"`
|
||||
ScoringRuleRef string `json:"scoring_rule_ref"`
|
||||
}
|
||||
|
||||
type Expectations struct {
|
||||
ConversationSuccess *bool `json:"conversation_success"`
|
||||
MaxFirstResponseMS int `json:"max_first_response_ms"`
|
||||
MaxResponseChars int `json:"max_response_chars"`
|
||||
MaxFabricatedFacts int `json:"max_fabricated_facts"`
|
||||
RequiredFlags []string `json:"required_flags"`
|
||||
}
|
||||
|
||||
type ObservationRun struct {
|
||||
Schema string `json:"schema"`
|
||||
SuiteVersion string `json:"suite_version"`
|
||||
AgentVersion string `json:"agent_version"`
|
||||
ModelConfigRef string `json:"model_config_ref"`
|
||||
Environment string `json:"environment"`
|
||||
ExecutedAt time.Time `json:"executed_at"`
|
||||
Observations []Observation `json:"observations"`
|
||||
}
|
||||
|
||||
type Observation struct {
|
||||
CaseID string `json:"case_id"`
|
||||
EvidenceRef string `json:"evidence_ref,omitempty"`
|
||||
ConversationSuccess bool `json:"conversation_success"`
|
||||
FirstResponseMS int `json:"first_response_ms"`
|
||||
ResponseChars int `json:"response_chars"`
|
||||
FabricatedFacts int `json:"fabricated_facts"`
|
||||
Flags map[string]bool `json:"flags"`
|
||||
}
|
||||
|
||||
type Metrics struct {
|
||||
CasesTotal int `json:"cases_total"`
|
||||
CasesPassed int `json:"cases_passed"`
|
||||
ConversationSuccessRate float64 `json:"conversation_success_rate"`
|
||||
FirstResponseP95MS int `json:"first_response_p95_ms"`
|
||||
InterruptionRecoveryRate float64 `json:"interruption_recovery_rate"`
|
||||
AverageResponseChars float64 `json:"average_response_chars"`
|
||||
SalutationAccuracy float64 `json:"salutation_accuracy"`
|
||||
MemoryAccuracy float64 `json:"memory_accuracy"`
|
||||
FabricatedFacts int `json:"fabricated_facts"`
|
||||
StorySourceCompleteness float64 `json:"story_source_completeness"`
|
||||
SensitiveInformationHandling float64 `json:"sensitive_information_handling_rate"`
|
||||
CategoryCounts map[string]int `json:"category_counts"`
|
||||
}
|
||||
|
||||
type FailedCheck struct {
|
||||
Code string `json:"code"`
|
||||
CaseID string `json:"case_id,omitempty"`
|
||||
Message string `json:"message"`
|
||||
Actual any `json:"actual,omitempty"`
|
||||
Expected any `json:"expected,omitempty"`
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
Schema string `json:"schema"`
|
||||
SuiteVersion string `json:"suite_version"`
|
||||
SuiteSHA256 string `json:"suite_sha256"`
|
||||
AgentVersion string `json:"agent_version"`
|
||||
ModelConfigRef string `json:"model_config_ref"`
|
||||
Environment string `json:"environment"`
|
||||
ExecutedAt time.Time `json:"executed_at"`
|
||||
EvaluatedAt time.Time `json:"evaluated_at"`
|
||||
Result string `json:"result"`
|
||||
Metrics Metrics `json:"metrics"`
|
||||
FailedChecks []FailedCheck `json:"failed_checks"`
|
||||
}
|
||||
|
||||
type RegistrationPayload struct {
|
||||
EvidenceType string `json:"evidence_type"`
|
||||
SuiteVersion string `json:"suite_version"`
|
||||
Result string `json:"result"`
|
||||
Metrics map[string]any `json:"metrics"`
|
||||
ReportRef string `json:"report_ref"`
|
||||
}
|
||||
|
||||
type Artifact struct {
|
||||
Report Report `json:"report"`
|
||||
RegistrationPayload RegistrationPayload `json:"registration_payload"`
|
||||
}
|
||||
|
||||
func Evaluate(suite Suite, run ObservationRun, evaluatedAt time.Time) (Report, error) {
|
||||
if err := validateSuite(suite); err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
if err := validateRun(suite, run); err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
|
||||
suiteSHA256, err := SuiteSHA256(suite)
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
|
||||
observations := make(map[string]Observation, len(run.Observations))
|
||||
for _, observation := range run.Observations {
|
||||
observations[observation.CaseID] = observation
|
||||
}
|
||||
|
||||
metrics := Metrics{CasesTotal: len(suite.Cases), CategoryCounts: map[string]int{}}
|
||||
failed := make([]FailedCheck, 0)
|
||||
responseTimes := make([]int, 0, len(suite.Cases))
|
||||
responseChars := 0
|
||||
successful := 0
|
||||
|
||||
for _, testCase := range suite.Cases {
|
||||
observation := observations[testCase.ID]
|
||||
metrics.CategoryCounts[testCase.Category]++
|
||||
responseTimes = append(responseTimes, observation.FirstResponseMS)
|
||||
responseChars += observation.ResponseChars
|
||||
metrics.FabricatedFacts += observation.FabricatedFacts
|
||||
if observation.ConversationSuccess {
|
||||
successful++
|
||||
}
|
||||
caseFailures := evaluateCase(testCase, observation)
|
||||
if len(caseFailures) == 0 {
|
||||
metrics.CasesPassed++
|
||||
} else {
|
||||
failed = append(failed, caseFailures...)
|
||||
}
|
||||
}
|
||||
|
||||
metrics.ConversationSuccessRate = rate(successful, len(suite.Cases))
|
||||
metrics.FirstResponseP95MS = percentile95(responseTimes)
|
||||
metrics.AverageResponseChars = float64(responseChars) / float64(len(suite.Cases))
|
||||
metrics.InterruptionRecoveryRate = flagRate(suite.Cases, observations, "user_interruption", "interruption_recovered")
|
||||
metrics.SalutationAccuracy = requiredFlagRate(suite.Cases, observations, "salutation_correct")
|
||||
metrics.MemoryAccuracy = requiredFlagRate(suite.Cases, observations, "memory_correct")
|
||||
metrics.StorySourceCompleteness = requiredFlagRate(suite.Cases, observations, "story_source_complete")
|
||||
metrics.SensitiveInformationHandling = flagRate(suite.Cases, observations, "sensitive_information", "sensitive_handling_pass")
|
||||
|
||||
failed = append(failed, evaluateThresholds(suite.Thresholds, metrics)...)
|
||||
result := "PASSED"
|
||||
if len(failed) > 0 {
|
||||
result = "FAILED"
|
||||
}
|
||||
return Report{
|
||||
Schema: ReportSchema, SuiteVersion: suite.SuiteVersion,
|
||||
SuiteSHA256: suiteSHA256, AgentVersion: run.AgentVersion,
|
||||
ModelConfigRef: run.ModelConfigRef, Environment: run.Environment,
|
||||
ExecutedAt: run.ExecutedAt.UTC(), EvaluatedAt: evaluatedAt.UTC(),
|
||||
Result: result, Metrics: metrics, FailedChecks: failed,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func SuiteSHA256(suite Suite) (string, error) {
|
||||
encodedSuite, err := json.Marshal(suite)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("encode suite: %w", err)
|
||||
}
|
||||
digest := sha256.Sum256(encodedSuite)
|
||||
return hex.EncodeToString(digest[:]), nil
|
||||
}
|
||||
|
||||
func NewArtifact(report Report, reportRef string) (Artifact, error) {
|
||||
reportRef = strings.TrimSpace(reportRef)
|
||||
if reportRef == "" || len([]rune(reportRef)) > 300 {
|
||||
return Artifact{}, errors.New("report_ref must contain 1 to 300 characters")
|
||||
}
|
||||
metrics := map[string]any{
|
||||
"report_schema": report.Schema, "suite_sha256": report.SuiteSHA256,
|
||||
"agent_version": report.AgentVersion, "model_config_ref": report.ModelConfigRef,
|
||||
"environment": report.Environment, "executed_at": report.ExecutedAt,
|
||||
"cases_total": report.Metrics.CasesTotal, "cases_passed": report.Metrics.CasesPassed,
|
||||
"conversation_success_rate": report.Metrics.ConversationSuccessRate,
|
||||
"first_response_p95_ms": report.Metrics.FirstResponseP95MS,
|
||||
"interruption_recovery_rate": report.Metrics.InterruptionRecoveryRate,
|
||||
"average_response_chars": report.Metrics.AverageResponseChars,
|
||||
"salutation_accuracy": report.Metrics.SalutationAccuracy,
|
||||
"memory_accuracy": report.Metrics.MemoryAccuracy,
|
||||
"fabricated_facts": report.Metrics.FabricatedFacts,
|
||||
"story_source_completeness": report.Metrics.StorySourceCompleteness,
|
||||
"sensitive_information_handling_rate": report.Metrics.SensitiveInformationHandling,
|
||||
"failed_check_count": len(report.FailedChecks),
|
||||
}
|
||||
return Artifact{Report: report, RegistrationPayload: RegistrationPayload{
|
||||
EvidenceType: "AGENT_QUALITY_V1", SuiteVersion: report.SuiteVersion,
|
||||
Result: report.Result, Metrics: metrics, ReportRef: reportRef,
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func RequiredCategories() []string {
|
||||
return append([]string(nil), requiredCategories...)
|
||||
}
|
||||
|
||||
func validateSuite(suite Suite) error {
|
||||
if suite.Schema != SuiteSchema {
|
||||
return fmt.Errorf("unsupported suite schema %q", suite.Schema)
|
||||
}
|
||||
if strings.TrimSpace(suite.SuiteVersion) == "" || len([]rune(suite.SuiteVersion)) > 100 {
|
||||
return errors.New("suite_version must contain 1 to 100 characters")
|
||||
}
|
||||
if strings.TrimSpace(suite.ModelConfigRef) == "" {
|
||||
return errors.New("model_config_ref is required")
|
||||
}
|
||||
if len(suite.Cases) == 0 {
|
||||
return errors.New("suite must contain cases")
|
||||
}
|
||||
if err := validateThresholds(suite.Thresholds); err != nil {
|
||||
return err
|
||||
}
|
||||
seenIDs := map[string]struct{}{}
|
||||
seenCategories := map[string]struct{}{}
|
||||
requiredFlagsByCategory := map[string]map[string]struct{}{}
|
||||
for index, testCase := range suite.Cases {
|
||||
testCase.ID = strings.TrimSpace(testCase.ID)
|
||||
if testCase.ID == "" || len([]rune(testCase.ID)) > 100 {
|
||||
return fmt.Errorf("case %d has invalid id", index)
|
||||
}
|
||||
if _, exists := seenIDs[testCase.ID]; exists {
|
||||
return fmt.Errorf("duplicate case id %q", testCase.ID)
|
||||
}
|
||||
seenIDs[testCase.ID] = struct{}{}
|
||||
seenCategories[testCase.Category] = struct{}{}
|
||||
if strings.TrimSpace(testCase.Description) == "" || strings.TrimSpace(testCase.ScoringRuleRef) == "" {
|
||||
return fmt.Errorf("case %q requires description and scoring_rule_ref", testCase.ID)
|
||||
}
|
||||
if testCase.Expected.ConversationSuccess == nil || testCase.Expected.MaxFirstResponseMS <= 0 || testCase.Expected.MaxResponseChars <= 0 || testCase.Expected.MaxFabricatedFacts < 0 {
|
||||
return fmt.Errorf("case %q has invalid expectations", testCase.ID)
|
||||
}
|
||||
if len(testCase.Expected.RequiredFlags) == 0 {
|
||||
return fmt.Errorf("case %q must define required_flags", testCase.ID)
|
||||
}
|
||||
flags := map[string]struct{}{}
|
||||
for _, flag := range testCase.Expected.RequiredFlags {
|
||||
flag = strings.TrimSpace(flag)
|
||||
if flag == "" {
|
||||
return fmt.Errorf("case %q contains an empty required flag", testCase.ID)
|
||||
}
|
||||
if _, exists := flags[flag]; exists {
|
||||
return fmt.Errorf("case %q contains duplicate required flag %q", testCase.ID, flag)
|
||||
}
|
||||
flags[flag] = struct{}{}
|
||||
if requiredFlagsByCategory[testCase.Category] == nil {
|
||||
requiredFlagsByCategory[testCase.Category] = map[string]struct{}{}
|
||||
}
|
||||
requiredFlagsByCategory[testCase.Category][flag] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, category := range requiredCategories {
|
||||
if _, exists := seenCategories[category]; !exists {
|
||||
return fmt.Errorf("suite does not cover required category %q", category)
|
||||
}
|
||||
}
|
||||
requiredScoringFlags := map[string]string{
|
||||
"salutation_correct": "normal_chat",
|
||||
"interruption_recovered": "user_interruption",
|
||||
"memory_correct": "memory_recall",
|
||||
"sensitive_handling_pass": "sensitive_information",
|
||||
"story_source_complete": "",
|
||||
}
|
||||
for flag, category := range requiredScoringFlags {
|
||||
found := false
|
||||
if category != "" {
|
||||
_, found = requiredFlagsByCategory[category][flag]
|
||||
} else {
|
||||
for _, flags := range requiredFlagsByCategory {
|
||||
if _, exists := flags[flag]; exists {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("suite does not score required flag %q", flag)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateThresholds(thresholds Thresholds) error {
|
||||
rates := map[string]float64{
|
||||
"min_conversation_success_rate": thresholds.MinConversationSuccessRate,
|
||||
"min_interruption_recovery_rate": thresholds.MinInterruptionRecoveryRate,
|
||||
"min_salutation_accuracy": thresholds.MinSalutationAccuracy,
|
||||
"min_memory_accuracy": thresholds.MinMemoryAccuracy,
|
||||
"min_story_source_completeness": thresholds.MinStorySourceCompleteness,
|
||||
"min_sensitive_handling_rate": thresholds.MinSensitiveHandlingRate,
|
||||
}
|
||||
for name, value := range rates {
|
||||
if value < 0 || value > 1 {
|
||||
return fmt.Errorf("%s must be between 0 and 1", name)
|
||||
}
|
||||
}
|
||||
if thresholds.MaxFirstResponseP95MS <= 0 || thresholds.MaxAverageResponseChars <= 0 || thresholds.MaxFabricatedFacts < 0 {
|
||||
return errors.New("maximum thresholds must be positive and max_fabricated_facts cannot be negative")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateRun(suite Suite, run ObservationRun) error {
|
||||
if run.Schema != ObservationSchema {
|
||||
return fmt.Errorf("unsupported observation schema %q", run.Schema)
|
||||
}
|
||||
if run.SuiteVersion != suite.SuiteVersion {
|
||||
return fmt.Errorf("observation suite_version %q does not match %q", run.SuiteVersion, suite.SuiteVersion)
|
||||
}
|
||||
if strings.TrimSpace(run.AgentVersion) == "" {
|
||||
return errors.New("agent_version is required")
|
||||
}
|
||||
if run.ModelConfigRef != suite.ModelConfigRef {
|
||||
return fmt.Errorf("model_config_ref %q does not match suite %q", run.ModelConfigRef, suite.ModelConfigRef)
|
||||
}
|
||||
if run.Environment != "mock" && run.Environment != "real" {
|
||||
return errors.New("environment must be mock or real")
|
||||
}
|
||||
if run.ExecutedAt.IsZero() {
|
||||
return errors.New("executed_at is required")
|
||||
}
|
||||
caseIDs := make(map[string]struct{}, len(suite.Cases))
|
||||
for _, testCase := range suite.Cases {
|
||||
caseIDs[testCase.ID] = struct{}{}
|
||||
}
|
||||
seen := make(map[string]struct{}, len(run.Observations))
|
||||
for _, observation := range run.Observations {
|
||||
if _, exists := caseIDs[observation.CaseID]; !exists {
|
||||
return fmt.Errorf("observation references unknown case %q", observation.CaseID)
|
||||
}
|
||||
if _, exists := seen[observation.CaseID]; exists {
|
||||
return fmt.Errorf("duplicate observation for case %q", observation.CaseID)
|
||||
}
|
||||
seen[observation.CaseID] = struct{}{}
|
||||
if observation.FirstResponseMS < 0 || (observation.ConversationSuccess && observation.FirstResponseMS == 0) || observation.ResponseChars < 0 || observation.FabricatedFacts < 0 || observation.Flags == nil {
|
||||
return fmt.Errorf("observation for case %q has invalid metric values", observation.CaseID)
|
||||
}
|
||||
}
|
||||
if len(seen) != len(caseIDs) {
|
||||
missing := make([]string, 0)
|
||||
for caseID := range caseIDs {
|
||||
if _, exists := seen[caseID]; !exists {
|
||||
missing = append(missing, caseID)
|
||||
}
|
||||
}
|
||||
sort.Strings(missing)
|
||||
return fmt.Errorf("observations missing cases: %s", strings.Join(missing, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func evaluateCase(testCase Case, observation Observation) []FailedCheck {
|
||||
failed := make([]FailedCheck, 0)
|
||||
if observation.ConversationSuccess != *testCase.Expected.ConversationSuccess {
|
||||
failed = append(failed, FailedCheck{Code: "conversation_success_mismatch", CaseID: testCase.ID, Message: "通话结果与用例期望不一致", Actual: observation.ConversationSuccess, Expected: *testCase.Expected.ConversationSuccess})
|
||||
}
|
||||
if observation.FirstResponseMS > testCase.Expected.MaxFirstResponseMS {
|
||||
failed = append(failed, FailedCheck{Code: "first_response_too_slow", CaseID: testCase.ID, Message: "首个回答时延超过用例上限", Actual: observation.FirstResponseMS, Expected: testCase.Expected.MaxFirstResponseMS})
|
||||
}
|
||||
if observation.ResponseChars > testCase.Expected.MaxResponseChars {
|
||||
failed = append(failed, FailedCheck{Code: "response_too_long", CaseID: testCase.ID, Message: "回答长度超过用例上限", Actual: observation.ResponseChars, Expected: testCase.Expected.MaxResponseChars})
|
||||
}
|
||||
if observation.FabricatedFacts > testCase.Expected.MaxFabricatedFacts {
|
||||
failed = append(failed, FailedCheck{Code: "fabricated_fact", CaseID: testCase.ID, Message: "虚构事实数量超过用例上限", Actual: observation.FabricatedFacts, Expected: testCase.Expected.MaxFabricatedFacts})
|
||||
}
|
||||
for _, flag := range testCase.Expected.RequiredFlags {
|
||||
if !observation.Flags[flag] {
|
||||
failed = append(failed, FailedCheck{Code: "required_flag_failed", CaseID: testCase.ID, Message: "评分规则未通过: " + flag, Actual: false, Expected: true})
|
||||
}
|
||||
}
|
||||
return failed
|
||||
}
|
||||
|
||||
func evaluateThresholds(thresholds Thresholds, metrics Metrics) []FailedCheck {
|
||||
checks := []struct {
|
||||
code string
|
||||
message string
|
||||
actual float64
|
||||
expected float64
|
||||
minimum bool
|
||||
}{
|
||||
{"conversation_success_rate_regressed", "通话成功率低于发布门槛", metrics.ConversationSuccessRate, thresholds.MinConversationSuccessRate, true},
|
||||
{"first_response_p95_regressed", "首个回答P95时延超过发布门槛", float64(metrics.FirstResponseP95MS), float64(thresholds.MaxFirstResponseP95MS), false},
|
||||
{"interruption_recovery_regressed", "打断恢复率低于发布门槛", metrics.InterruptionRecoveryRate, thresholds.MinInterruptionRecoveryRate, true},
|
||||
{"average_response_length_regressed", "平均回答长度超过发布门槛", metrics.AverageResponseChars, thresholds.MaxAverageResponseChars, false},
|
||||
{"salutation_accuracy_regressed", "称呼准确率低于发布门槛", metrics.SalutationAccuracy, thresholds.MinSalutationAccuracy, true},
|
||||
{"memory_accuracy_regressed", "记忆准确率低于发布门槛", metrics.MemoryAccuracy, thresholds.MinMemoryAccuracy, true},
|
||||
{"story_source_regressed", "故事来源完整率低于发布门槛", metrics.StorySourceCompleteness, thresholds.MinStorySourceCompleteness, true},
|
||||
{"sensitive_handling_regressed", "敏感信息处理通过率低于发布门槛", metrics.SensitiveInformationHandling, thresholds.MinSensitiveHandlingRate, true},
|
||||
}
|
||||
failed := make([]FailedCheck, 0)
|
||||
for _, check := range checks {
|
||||
violated := check.actual > check.expected
|
||||
if check.minimum {
|
||||
violated = check.actual < check.expected
|
||||
}
|
||||
if violated {
|
||||
failed = append(failed, FailedCheck{Code: check.code, Message: check.message, Actual: check.actual, Expected: check.expected})
|
||||
}
|
||||
}
|
||||
if metrics.FabricatedFacts > thresholds.MaxFabricatedFacts {
|
||||
failed = append(failed, FailedCheck{Code: "fabricated_facts_regressed", Message: "虚构事实总数超过发布门槛", Actual: metrics.FabricatedFacts, Expected: thresholds.MaxFabricatedFacts})
|
||||
}
|
||||
return failed
|
||||
}
|
||||
|
||||
func percentile95(values []int) int {
|
||||
sorted := append([]int(nil), values...)
|
||||
sort.Ints(sorted)
|
||||
index := int(math.Ceil(float64(len(sorted))*0.95)) - 1
|
||||
if index < 0 {
|
||||
return 0
|
||||
}
|
||||
return sorted[index]
|
||||
}
|
||||
|
||||
func rate(passed, total int) float64 {
|
||||
if total == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(passed) / float64(total)
|
||||
}
|
||||
|
||||
func requiredFlagRate(cases []Case, observations map[string]Observation, flag string) float64 {
|
||||
passed, total := 0, 0
|
||||
for _, testCase := range cases {
|
||||
if !contains(testCase.Expected.RequiredFlags, flag) {
|
||||
continue
|
||||
}
|
||||
total++
|
||||
if observations[testCase.ID].Flags[flag] {
|
||||
passed++
|
||||
}
|
||||
}
|
||||
return rate(passed, total)
|
||||
}
|
||||
|
||||
func flagRate(cases []Case, observations map[string]Observation, category, flag string) float64 {
|
||||
passed, total := 0, 0
|
||||
for _, testCase := range cases {
|
||||
if testCase.Category != category {
|
||||
continue
|
||||
}
|
||||
total++
|
||||
if observations[testCase.ID].Flags[flag] {
|
||||
passed++
|
||||
}
|
||||
}
|
||||
return rate(passed, total)
|
||||
}
|
||||
|
||||
func contains(values []string, wanted string) bool {
|
||||
for _, value := range values {
|
||||
if value == wanted {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in New Issue
Block a user