282 lines
15 KiB
Go
282 lines
15 KiB
Go
|
package agentmodelsconformance
|
||
|
|
|
||
|
|
import (
|
||
|
|
"bytes"
|
||
|
|
"crypto/sha256"
|
||
|
|
"encoding/hex"
|
||
|
|
"encoding/json"
|
||
|
|
"errors"
|
||
|
|
"fmt"
|
||
|
|
"io"
|
||
|
|
"regexp"
|
||
|
|
"slices"
|
||
|
|
"strings"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"data-terminal-cloud/internal/agenteval"
|
||
|
|
"data-terminal-cloud/internal/embeddingprovider"
|
||
|
|
)
|
||
|
|
|
||
|
|
const (
|
||
|
|
CaptureSchema = "real-agent-models-capture-v1"
|
||
|
|
ReportSchema = "real-agent-models-conformance-report-v1"
|
||
|
|
)
|
||
|
|
|
||
|
|
var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
||
|
|
|
||
|
|
var RequiredEmbeddingCategories = []string{"EXACT_CONFIRMED_MEMORY", "PARAPHRASE_RECALL", "CROSS_SUBJECT_ISOLATION", "REVOKED_MEMORY_EXCLUSION", "SENSITIVE_MEMORY_EXCLUSION", "NO_MATCH_FALLBACK"}
|
||
|
|
var RequiredSafetyCategories = []string{"VERIFICATION_CODE", "FINANCIAL_ACCOUNT", "PROMPT_INJECTION", "SENSITIVE_MEDICAL", "NORMAL_BENIGN", "MODEL_OUTPUT_SAFETY"}
|
||
|
|
|
||
|
|
type Capture struct {
|
||
|
|
SchemaVersion string `json:"schema_version"`
|
||
|
|
EvidenceSource string `json:"evidence_source"`
|
||
|
|
CandidateVersion string `json:"candidate_version"`
|
||
|
|
BuildDigest string `json:"build_digest"`
|
||
|
|
Environment string `json:"environment"`
|
||
|
|
ChangeTicketRef string `json:"change_ticket_ref"`
|
||
|
|
ExecutedByRef string `json:"executed_by_ref"`
|
||
|
|
AttestedByRef string `json:"attested_by_ref"`
|
||
|
|
AttestedAt time.Time `json:"attested_at"`
|
||
|
|
CollectedAt time.Time `json:"collected_at"`
|
||
|
|
Generation GenerationEvidence `json:"generation"`
|
||
|
|
Embedding EmbeddingEvidence `json:"embedding"`
|
||
|
|
ContentSafety ContentSafetyEvidence `json:"content_safety"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type GenerationEvidence struct {
|
||
|
|
ProviderClass string `json:"provider_class"`
|
||
|
|
ProviderRef string `json:"provider_ref"`
|
||
|
|
ModelConfigRef string `json:"model_config_ref"`
|
||
|
|
ModelResourceRef string `json:"model_resource_ref"`
|
||
|
|
AccountRef string `json:"account_ref"`
|
||
|
|
RegionRef string `json:"region_ref"`
|
||
|
|
DataTermsRef string `json:"data_terms_ref"`
|
||
|
|
QuotaEvidenceRef string `json:"quota_evidence_ref"`
|
||
|
|
EvaluationEvidenceRef string `json:"evaluation_evidence_ref"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type EmbeddingEvidence struct {
|
||
|
|
ProviderClass string `json:"provider_class"`
|
||
|
|
ProviderRef string `json:"provider_ref"`
|
||
|
|
ModelRef string `json:"model_ref"`
|
||
|
|
Dimensions int `json:"dimensions"`
|
||
|
|
DatasetRef string `json:"dataset_ref"`
|
||
|
|
ThresholdsRef string `json:"thresholds_ref"`
|
||
|
|
QueryCount int `json:"query_count"`
|
||
|
|
PassedQueryCount int `json:"passed_query_count"`
|
||
|
|
DataTermsRef string `json:"data_terms_ref"`
|
||
|
|
QuotaEvidenceRef string `json:"quota_evidence_ref"`
|
||
|
|
Categories []CategoryResult `json:"categories"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type ContentSafetyEvidence struct {
|
||
|
|
ProviderClass string `json:"provider_class"`
|
||
|
|
ProviderRef string `json:"provider_ref"`
|
||
|
|
PolicyRef string `json:"policy_ref"`
|
||
|
|
DatasetRef string `json:"dataset_ref"`
|
||
|
|
ThresholdsRef string `json:"thresholds_ref"`
|
||
|
|
CaseCount int `json:"case_count"`
|
||
|
|
PassedCaseCount int `json:"passed_case_count"`
|
||
|
|
DataTermsRef string `json:"data_terms_ref"`
|
||
|
|
AuditEvidenceRef string `json:"audit_evidence_ref"`
|
||
|
|
Categories []CategoryResult `json:"categories"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type CategoryResult struct {
|
||
|
|
Category string `json:"category"`
|
||
|
|
Status string `json:"status"`
|
||
|
|
EvidenceRef string `json:"evidence_ref"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type Check struct {
|
||
|
|
Code string `json:"code"`
|
||
|
|
Status string `json:"status"`
|
||
|
|
Detail string `json:"detail"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type Report struct {
|
||
|
|
SchemaVersion string `json:"schema_version"`
|
||
|
|
Result string `json:"result"`
|
||
|
|
GeneratedAt time.Time `json:"generated_at"`
|
||
|
|
CandidateVersion string `json:"candidate_version"`
|
||
|
|
BuildDigest string `json:"build_digest"`
|
||
|
|
EvidenceScope string `json:"evidence_scope"`
|
||
|
|
ExternalValidationRequired bool `json:"external_validation_required"`
|
||
|
|
Environment string `json:"environment"`
|
||
|
|
GenerationProviderRef string `json:"generation_provider_ref"`
|
||
|
|
AgentVersion string `json:"agent_version"`
|
||
|
|
ModelConfigRef string `json:"model_config_ref"`
|
||
|
|
SuiteVersion string `json:"suite_version"`
|
||
|
|
GenerationCaseCount int `json:"generation_case_count"`
|
||
|
|
EmbeddingProviderRef string `json:"embedding_provider_ref"`
|
||
|
|
EmbeddingModelRef string `json:"embedding_model_ref"`
|
||
|
|
EmbeddingQueryCount int `json:"embedding_query_count"`
|
||
|
|
EmbeddingCategories []string `json:"embedding_categories"`
|
||
|
|
ContentSafetyProviderRef string `json:"content_safety_provider_ref"`
|
||
|
|
ContentSafetyPolicyRef string `json:"content_safety_policy_ref"`
|
||
|
|
ContentSafetyCaseCount int `json:"content_safety_case_count"`
|
||
|
|
ContentSafetyCategories []string `json:"content_safety_categories"`
|
||
|
|
InputSHA256 map[string]string `json:"input_sha256"`
|
||
|
|
Checks []Check `json:"checks"`
|
||
|
|
Disclaimer string `json:"disclaimer"`
|
||
|
|
}
|
||
|
|
|
||
|
|
func ParseCapture(content []byte) (Capture, error) {
|
||
|
|
var capture Capture
|
||
|
|
if err := decodeStrict(content, &capture); err != nil {
|
||
|
|
return Capture{}, fmt.Errorf("decode real agent models capture: %w", err)
|
||
|
|
}
|
||
|
|
return capture, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func ParseReport(content []byte) (Report, error) {
|
||
|
|
var report Report
|
||
|
|
if err := decodeStrict(content, &report); err != nil {
|
||
|
|
return Report{}, fmt.Errorf("decode real agent models report: %w", err)
|
||
|
|
}
|
||
|
|
return report, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func Evaluate(capture Capture, suite agenteval.Suite, observations agenteval.ObservationRun, artifact agenteval.Artifact, candidateVersion, buildDigest string, generatedAt time.Time) (Report, error) {
|
||
|
|
if err := agenteval.ValidateArtifact(suite, observations, artifact, false); err != nil {
|
||
|
|
return Report{}, fmt.Errorf("validate agent evaluation artifact: %w", err)
|
||
|
|
}
|
||
|
|
candidateVersion = strings.TrimSpace(candidateVersion)
|
||
|
|
buildDigest = strings.ToLower(strings.TrimSpace(buildDigest))
|
||
|
|
report := Report{
|
||
|
|
SchemaVersion: ReportSchema, Result: "PASS", GeneratedAt: generatedAt.UTC(), CandidateVersion: candidateVersion, BuildDigest: buildDigest,
|
||
|
|
EvidenceScope: "LOCAL_AGENT_TOOLCHAIN", ExternalValidationRequired: true, Environment: capture.Environment,
|
||
|
|
GenerationProviderRef: capture.Generation.ProviderRef, AgentVersion: artifact.Report.AgentVersion, ModelConfigRef: artifact.Report.ModelConfigRef, SuiteVersion: artifact.Report.SuiteVersion, GenerationCaseCount: artifact.Report.Metrics.CasesTotal,
|
||
|
|
EmbeddingProviderRef: capture.Embedding.ProviderRef, EmbeddingModelRef: capture.Embedding.ModelRef, EmbeddingQueryCount: capture.Embedding.QueryCount, EmbeddingCategories: []string{},
|
||
|
|
ContentSafetyProviderRef: capture.ContentSafety.ProviderRef, ContentSafetyPolicyRef: capture.ContentSafety.PolicyRef, ContentSafetyCaseCount: capture.ContentSafety.CaseCount, ContentSafetyCategories: []string{},
|
||
|
|
InputSHA256: map[string]string{}, Checks: []Check{},
|
||
|
|
Disclaimer: "该报告只核对同一候选的固定Agent评测、Embedding召回/隔离和内容安全证据；本地模型与规则不能替代真实账号、质量、额度和数据条款验收，也不等于业务发布批准。",
|
||
|
|
}
|
||
|
|
if capture.EvidenceSource == "REAL_PROVIDERS" {
|
||
|
|
report.EvidenceScope = "REAL_AGENT_MODELS"
|
||
|
|
}
|
||
|
|
add := func(code string, ok bool, detail string) {
|
||
|
|
status := "PASS"
|
||
|
|
if !ok {
|
||
|
|
status, report.Result = "FAIL", "FAIL"
|
||
|
|
}
|
||
|
|
report.Checks = append(report.Checks, Check{Code: code, Status: status, Detail: detail})
|
||
|
|
}
|
||
|
|
add("CAPTURE_SCHEMA", capture.SchemaVersion == CaptureSchema, "Agent模型证据必须使用"+CaptureSchema)
|
||
|
|
add("EVIDENCE_SOURCE", capture.EvidenceSource == "REAL_PROVIDERS" || capture.EvidenceSource == "LOCAL_MODELS", "证据来源必须明确为REAL_PROVIDERS或LOCAL_MODELS")
|
||
|
|
add("CANDIDATE_IDENTITY", candidateVersion != "" && digestPattern.MatchString(buildDigest), "候选版本和sha256构建摘要必须完整")
|
||
|
|
add("CAPTURE_CANDIDATE_BINDING", capture.CandidateVersion == candidateVersion && strings.ToLower(capture.BuildDigest) == buildDigest, "Agent模型采集必须绑定同一候选版本和构建摘要")
|
||
|
|
|
||
|
|
real := capture.EvidenceSource == "REAL_PROVIDERS"
|
||
|
|
environmentOK := capture.ExecutedByRef != "" && capture.AttestedByRef != "" && !capture.AttestedAt.IsZero() && !capture.CollectedAt.IsZero() && !capture.AttestedAt.After(capture.CollectedAt)
|
||
|
|
if real {
|
||
|
|
environmentOK = environmentOK && capture.ExecutedByRef != capture.AttestedByRef && (capture.Environment == "STAGING" || capture.Environment == "PRODUCTION") && capture.ChangeTicketRef != ""
|
||
|
|
} else {
|
||
|
|
environmentOK = environmentOK && capture.Environment == "LOCAL"
|
||
|
|
}
|
||
|
|
add("ENVIRONMENT_IDENTITY", environmentOK, "真实证据必须绑定目标环境、变更工单、执行人和独立复核人")
|
||
|
|
|
||
|
|
generationProviderOK := validProviderClass(capture.Generation.ProviderClass) && capture.Generation.ProviderRef != "" && capture.Generation.ModelConfigRef != "" && capture.Generation.ModelResourceRef != "" && capture.Generation.AccountRef != "" && capture.Generation.RegionRef != ""
|
||
|
|
if real {
|
||
|
|
generationProviderOK = generationProviderOK && capture.Generation.ProviderClass == "EXTERNAL_MANAGED"
|
||
|
|
}
|
||
|
|
add("GENERATION_PROVIDER", generationProviderOK, "内容生成必须绑定提供器、模型配置、模型资源、账号和区域引用")
|
||
|
|
generationReportOK := artifact.Report.Schema == agenteval.ReportSchema && artifact.Report.Result == "PASSED" && artifact.Report.Metrics.CasesTotal >= 10 && artifact.Report.Metrics.CasesPassed == artifact.Report.Metrics.CasesTotal && len(artifact.Report.FailedChecks) == 0
|
||
|
|
if real {
|
||
|
|
generationReportOK = generationReportOK && artifact.Report.Environment == "real" && observations.Environment == "real"
|
||
|
|
} else {
|
||
|
|
generationReportOK = generationReportOK && artifact.Report.Environment == "mock" && observations.Environment == "mock"
|
||
|
|
}
|
||
|
|
add("GENERATION_QUALITY", generationReportOK, "固定Agent评测必须重新计算通过、至少10例且无失败项；真实门禁只接受real观测")
|
||
|
|
modelBindingOK := capture.Generation.ModelConfigRef == artifact.Report.ModelConfigRef && artifact.Report.ModelConfigRef == suite.ModelConfigRef && artifact.Report.AgentVersion == observations.AgentVersion
|
||
|
|
add("MODEL_BINDING", modelBindingOK, "内容生成证据、评测集、观测和报告必须绑定同一模型配置与Agent版本")
|
||
|
|
|
||
|
|
embeddingProviderOK := validProviderClass(capture.Embedding.ProviderClass) && capture.Embedding.ProviderRef != "" && capture.Embedding.ModelRef != "" && capture.Embedding.Dimensions == embeddingprovider.Dimensions && capture.Embedding.DatasetRef != "" && capture.Embedding.ThresholdsRef != ""
|
||
|
|
if real {
|
||
|
|
embeddingProviderOK = embeddingProviderOK && capture.Embedding.ProviderClass == "EXTERNAL_MANAGED"
|
||
|
|
}
|
||
|
|
add("EMBEDDING_PROVIDER", embeddingProviderOK, "Embedding必须绑定提供器、模型、1536维度、数据集和阈值版本")
|
||
|
|
embeddingCoverageOK, embeddingRefsOK := validateCategories(capture.Embedding.Categories, RequiredEmbeddingCategories)
|
||
|
|
embeddingCoverageOK = embeddingCoverageOK && capture.Embedding.QueryCount >= len(RequiredEmbeddingCategories) && capture.Embedding.PassedQueryCount == capture.Embedding.QueryCount
|
||
|
|
for _, category := range RequiredEmbeddingCategories {
|
||
|
|
if containsCategory(capture.Embedding.Categories, category) {
|
||
|
|
report.EmbeddingCategories = append(report.EmbeddingCategories, category)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
add("EMBEDDING_COVERAGE", embeddingCoverageOK, "召回评测必须全量通过精确、改写、跨长者隔离、撤销排除、敏感排除和无匹配降级六类")
|
||
|
|
|
||
|
|
safetyProviderOK := validProviderClass(capture.ContentSafety.ProviderClass) && capture.ContentSafety.ProviderRef != "" && capture.ContentSafety.PolicyRef != "" && capture.ContentSafety.DatasetRef != "" && capture.ContentSafety.ThresholdsRef != ""
|
||
|
|
if real {
|
||
|
|
safetyProviderOK = safetyProviderOK && capture.ContentSafety.ProviderClass == "EXTERNAL_MANAGED"
|
||
|
|
}
|
||
|
|
add("CONTENT_SAFETY_PROVIDER", safetyProviderOK, "内容安全必须绑定提供器、策略、数据集和阈值版本")
|
||
|
|
safetyCoverageOK, safetyRefsOK := validateCategories(capture.ContentSafety.Categories, RequiredSafetyCategories)
|
||
|
|
safetyCoverageOK = safetyCoverageOK && capture.ContentSafety.CaseCount >= len(RequiredSafetyCategories) && capture.ContentSafety.PassedCaseCount == capture.ContentSafety.CaseCount
|
||
|
|
for _, category := range RequiredSafetyCategories {
|
||
|
|
if containsCategory(capture.ContentSafety.Categories, category) {
|
||
|
|
report.ContentSafetyCategories = append(report.ContentSafetyCategories, category)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
add("CONTENT_SAFETY_COVERAGE", safetyCoverageOK, "内容安全必须全量通过验证码、金融账户、提示注入、敏感医疗、正常内容和模型输出六类")
|
||
|
|
|
||
|
|
termsQuotaOK := capture.Generation.DataTermsRef != "" && capture.Generation.QuotaEvidenceRef != "" && capture.Embedding.DataTermsRef != "" && capture.Embedding.QuotaEvidenceRef != "" && capture.ContentSafety.DataTermsRef != ""
|
||
|
|
add("DATA_TERMS_QUOTA", termsQuotaOK, "真实模型使用必须保存数据地域/留存条款和生成、Embedding额度证据引用")
|
||
|
|
refsOK := capture.Generation.EvaluationEvidenceRef != "" && embeddingRefsOK && safetyRefsOK && capture.ContentSafety.AuditEvidenceRef != ""
|
||
|
|
add("EVIDENCE_REFERENCES", refsOK, "生成、Embedding和内容安全必须保存脱敏评测与审计引用")
|
||
|
|
if real && environmentOK && generationProviderOK && generationReportOK && modelBindingOK && embeddingProviderOK && embeddingCoverageOK && safetyProviderOK && safetyCoverageOK && termsQuotaOK && refsOK && report.Result == "PASS" {
|
||
|
|
report.ExternalValidationRequired = false
|
||
|
|
}
|
||
|
|
return report, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func validProviderClass(value string) bool {
|
||
|
|
return value == "EXTERNAL_MANAGED" || value == "LOCAL_IMPLEMENTATION"
|
||
|
|
}
|
||
|
|
|
||
|
|
func validateCategories(results []CategoryResult, required []string) (bool, bool) {
|
||
|
|
valid, refsOK := len(results) > 0, true
|
||
|
|
seen := map[string]bool{}
|
||
|
|
for _, result := range results {
|
||
|
|
if !slices.Contains(required, result.Category) || seen[result.Category] || result.Status != "PASS" {
|
||
|
|
valid = false
|
||
|
|
}
|
||
|
|
seen[result.Category] = true
|
||
|
|
if strings.TrimSpace(result.EvidenceRef) == "" {
|
||
|
|
refsOK = false
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for _, category := range required {
|
||
|
|
if !seen[category] {
|
||
|
|
valid = false
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return valid, refsOK
|
||
|
|
}
|
||
|
|
|
||
|
|
func containsCategory(results []CategoryResult, category string) bool {
|
||
|
|
for _, result := range results {
|
||
|
|
if result.Category == category {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
|
||
|
|
func SHA256(content []byte) string {
|
||
|
|
digest := sha256.Sum256(content)
|
||
|
|
return hex.EncodeToString(digest[:])
|
||
|
|
}
|
||
|
|
|
||
|
|
func decodeStrict(content []byte, target any) error {
|
||
|
|
decoder := json.NewDecoder(bytes.NewReader(content))
|
||
|
|
decoder.DisallowUnknownFields()
|
||
|
|
if err := decoder.Decode(target); err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
var trailing any
|
||
|
|
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||
|
|
return errors.New("trailing JSON data")
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}
|