chore: establish local platform baseline
This commit is contained in:
Codex Local
@@ -0,0 +1,281 @@
|
||||
package agentmodelsconformance
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/agenteval"
|
||||
"data-terminal-cloud/internal/embeddingprovider"
|
||||
)
|
||||
|
||||
const (
|
||||
CaptureSchema = "real-agent-models-capture-v1"
|
||||
ReportSchema = "real-agent-models-conformance-report-v1"
|
||||
)
|
||||
|
||||
var digestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
||||
|
||||
var RequiredEmbeddingCategories = []string{"EXACT_CONFIRMED_MEMORY", "PARAPHRASE_RECALL", "CROSS_SUBJECT_ISOLATION", "REVOKED_MEMORY_EXCLUSION", "SENSITIVE_MEMORY_EXCLUSION", "NO_MATCH_FALLBACK"}
|
||||
var RequiredSafetyCategories = []string{"VERIFICATION_CODE", "FINANCIAL_ACCOUNT", "PROMPT_INJECTION", "SENSITIVE_MEDICAL", "NORMAL_BENIGN", "MODEL_OUTPUT_SAFETY"}
|
||||
|
||||
type Capture struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
EvidenceSource string `json:"evidence_source"`
|
||||
CandidateVersion string `json:"candidate_version"`
|
||||
BuildDigest string `json:"build_digest"`
|
||||
Environment string `json:"environment"`
|
||||
ChangeTicketRef string `json:"change_ticket_ref"`
|
||||
ExecutedByRef string `json:"executed_by_ref"`
|
||||
AttestedByRef string `json:"attested_by_ref"`
|
||||
AttestedAt time.Time `json:"attested_at"`
|
||||
CollectedAt time.Time `json:"collected_at"`
|
||||
Generation GenerationEvidence `json:"generation"`
|
||||
Embedding EmbeddingEvidence `json:"embedding"`
|
||||
ContentSafety ContentSafetyEvidence `json:"content_safety"`
|
||||
}
|
||||
|
||||
type GenerationEvidence struct {
|
||||
ProviderClass string `json:"provider_class"`
|
||||
ProviderRef string `json:"provider_ref"`
|
||||
ModelConfigRef string `json:"model_config_ref"`
|
||||
ModelResourceRef string `json:"model_resource_ref"`
|
||||
AccountRef string `json:"account_ref"`
|
||||
RegionRef string `json:"region_ref"`
|
||||
DataTermsRef string `json:"data_terms_ref"`
|
||||
QuotaEvidenceRef string `json:"quota_evidence_ref"`
|
||||
EvaluationEvidenceRef string `json:"evaluation_evidence_ref"`
|
||||
}
|
||||
|
||||
type EmbeddingEvidence struct {
|
||||
ProviderClass string `json:"provider_class"`
|
||||
ProviderRef string `json:"provider_ref"`
|
||||
ModelRef string `json:"model_ref"`
|
||||
Dimensions int `json:"dimensions"`
|
||||
DatasetRef string `json:"dataset_ref"`
|
||||
ThresholdsRef string `json:"thresholds_ref"`
|
||||
QueryCount int `json:"query_count"`
|
||||
PassedQueryCount int `json:"passed_query_count"`
|
||||
DataTermsRef string `json:"data_terms_ref"`
|
||||
QuotaEvidenceRef string `json:"quota_evidence_ref"`
|
||||
Categories []CategoryResult `json:"categories"`
|
||||
}
|
||||
|
||||
type ContentSafetyEvidence struct {
|
||||
ProviderClass string `json:"provider_class"`
|
||||
ProviderRef string `json:"provider_ref"`
|
||||
PolicyRef string `json:"policy_ref"`
|
||||
DatasetRef string `json:"dataset_ref"`
|
||||
ThresholdsRef string `json:"thresholds_ref"`
|
||||
CaseCount int `json:"case_count"`
|
||||
PassedCaseCount int `json:"passed_case_count"`
|
||||
DataTermsRef string `json:"data_terms_ref"`
|
||||
AuditEvidenceRef string `json:"audit_evidence_ref"`
|
||||
Categories []CategoryResult `json:"categories"`
|
||||
}
|
||||
|
||||
type CategoryResult struct {
|
||||
Category string `json:"category"`
|
||||
Status string `json:"status"`
|
||||
EvidenceRef string `json:"evidence_ref"`
|
||||
}
|
||||
|
||||
type Check struct {
|
||||
Code string `json:"code"`
|
||||
Status string `json:"status"`
|
||||
Detail string `json:"detail"`
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
Result string `json:"result"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
CandidateVersion string `json:"candidate_version"`
|
||||
BuildDigest string `json:"build_digest"`
|
||||
EvidenceScope string `json:"evidence_scope"`
|
||||
ExternalValidationRequired bool `json:"external_validation_required"`
|
||||
Environment string `json:"environment"`
|
||||
GenerationProviderRef string `json:"generation_provider_ref"`
|
||||
AgentVersion string `json:"agent_version"`
|
||||
ModelConfigRef string `json:"model_config_ref"`
|
||||
SuiteVersion string `json:"suite_version"`
|
||||
GenerationCaseCount int `json:"generation_case_count"`
|
||||
EmbeddingProviderRef string `json:"embedding_provider_ref"`
|
||||
EmbeddingModelRef string `json:"embedding_model_ref"`
|
||||
EmbeddingQueryCount int `json:"embedding_query_count"`
|
||||
EmbeddingCategories []string `json:"embedding_categories"`
|
||||
ContentSafetyProviderRef string `json:"content_safety_provider_ref"`
|
||||
ContentSafetyPolicyRef string `json:"content_safety_policy_ref"`
|
||||
ContentSafetyCaseCount int `json:"content_safety_case_count"`
|
||||
ContentSafetyCategories []string `json:"content_safety_categories"`
|
||||
InputSHA256 map[string]string `json:"input_sha256"`
|
||||
Checks []Check `json:"checks"`
|
||||
Disclaimer string `json:"disclaimer"`
|
||||
}
|
||||
|
||||
func ParseCapture(content []byte) (Capture, error) {
|
||||
var capture Capture
|
||||
if err := decodeStrict(content, &capture); err != nil {
|
||||
return Capture{}, fmt.Errorf("decode real agent models capture: %w", err)
|
||||
}
|
||||
return capture, nil
|
||||
}
|
||||
|
||||
func ParseReport(content []byte) (Report, error) {
|
||||
var report Report
|
||||
if err := decodeStrict(content, &report); err != nil {
|
||||
return Report{}, fmt.Errorf("decode real agent models report: %w", err)
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func Evaluate(capture Capture, suite agenteval.Suite, observations agenteval.ObservationRun, artifact agenteval.Artifact, candidateVersion, buildDigest string, generatedAt time.Time) (Report, error) {
|
||||
if err := agenteval.ValidateArtifact(suite, observations, artifact, false); err != nil {
|
||||
return Report{}, fmt.Errorf("validate agent evaluation artifact: %w", err)
|
||||
}
|
||||
candidateVersion = strings.TrimSpace(candidateVersion)
|
||||
buildDigest = strings.ToLower(strings.TrimSpace(buildDigest))
|
||||
report := Report{
|
||||
SchemaVersion: ReportSchema, Result: "PASS", GeneratedAt: generatedAt.UTC(), CandidateVersion: candidateVersion, BuildDigest: buildDigest,
|
||||
EvidenceScope: "LOCAL_AGENT_TOOLCHAIN", ExternalValidationRequired: true, Environment: capture.Environment,
|
||||
GenerationProviderRef: capture.Generation.ProviderRef, AgentVersion: artifact.Report.AgentVersion, ModelConfigRef: artifact.Report.ModelConfigRef, SuiteVersion: artifact.Report.SuiteVersion, GenerationCaseCount: artifact.Report.Metrics.CasesTotal,
|
||||
EmbeddingProviderRef: capture.Embedding.ProviderRef, EmbeddingModelRef: capture.Embedding.ModelRef, EmbeddingQueryCount: capture.Embedding.QueryCount, EmbeddingCategories: []string{},
|
||||
ContentSafetyProviderRef: capture.ContentSafety.ProviderRef, ContentSafetyPolicyRef: capture.ContentSafety.PolicyRef, ContentSafetyCaseCount: capture.ContentSafety.CaseCount, ContentSafetyCategories: []string{},
|
||||
InputSHA256: map[string]string{}, Checks: []Check{},
|
||||
Disclaimer: "该报告只核对同一候选的固定Agent评测、Embedding召回/隔离和内容安全证据；本地模型与规则不能替代真实账号、质量、额度和数据条款验收，也不等于业务发布批准。",
|
||||
}
|
||||
if capture.EvidenceSource == "REAL_PROVIDERS" {
|
||||
report.EvidenceScope = "REAL_AGENT_MODELS"
|
||||
}
|
||||
add := func(code string, ok bool, detail string) {
|
||||
status := "PASS"
|
||||
if !ok {
|
||||
status, report.Result = "FAIL", "FAIL"
|
||||
}
|
||||
report.Checks = append(report.Checks, Check{Code: code, Status: status, Detail: detail})
|
||||
}
|
||||
add("CAPTURE_SCHEMA", capture.SchemaVersion == CaptureSchema, "Agent模型证据必须使用"+CaptureSchema)
|
||||
add("EVIDENCE_SOURCE", capture.EvidenceSource == "REAL_PROVIDERS" || capture.EvidenceSource == "LOCAL_MODELS", "证据来源必须明确为REAL_PROVIDERS或LOCAL_MODELS")
|
||||
add("CANDIDATE_IDENTITY", candidateVersion != "" && digestPattern.MatchString(buildDigest), "候选版本和sha256构建摘要必须完整")
|
||||
add("CAPTURE_CANDIDATE_BINDING", capture.CandidateVersion == candidateVersion && strings.ToLower(capture.BuildDigest) == buildDigest, "Agent模型采集必须绑定同一候选版本和构建摘要")
|
||||
|
||||
real := capture.EvidenceSource == "REAL_PROVIDERS"
|
||||
environmentOK := capture.ExecutedByRef != "" && capture.AttestedByRef != "" && !capture.AttestedAt.IsZero() && !capture.CollectedAt.IsZero() && !capture.AttestedAt.After(capture.CollectedAt)
|
||||
if real {
|
||||
environmentOK = environmentOK && capture.ExecutedByRef != capture.AttestedByRef && (capture.Environment == "STAGING" || capture.Environment == "PRODUCTION") && capture.ChangeTicketRef != ""
|
||||
} else {
|
||||
environmentOK = environmentOK && capture.Environment == "LOCAL"
|
||||
}
|
||||
add("ENVIRONMENT_IDENTITY", environmentOK, "真实证据必须绑定目标环境、变更工单、执行人和独立复核人")
|
||||
|
||||
generationProviderOK := validProviderClass(capture.Generation.ProviderClass) && capture.Generation.ProviderRef != "" && capture.Generation.ModelConfigRef != "" && capture.Generation.ModelResourceRef != "" && capture.Generation.AccountRef != "" && capture.Generation.RegionRef != ""
|
||||
if real {
|
||||
generationProviderOK = generationProviderOK && capture.Generation.ProviderClass == "EXTERNAL_MANAGED"
|
||||
}
|
||||
add("GENERATION_PROVIDER", generationProviderOK, "内容生成必须绑定提供器、模型配置、模型资源、账号和区域引用")
|
||||
generationReportOK := artifact.Report.Schema == agenteval.ReportSchema && artifact.Report.Result == "PASSED" && artifact.Report.Metrics.CasesTotal >= 10 && artifact.Report.Metrics.CasesPassed == artifact.Report.Metrics.CasesTotal && len(artifact.Report.FailedChecks) == 0
|
||||
if real {
|
||||
generationReportOK = generationReportOK && artifact.Report.Environment == "real" && observations.Environment == "real"
|
||||
} else {
|
||||
generationReportOK = generationReportOK && artifact.Report.Environment == "mock" && observations.Environment == "mock"
|
||||
}
|
||||
add("GENERATION_QUALITY", generationReportOK, "固定Agent评测必须重新计算通过、至少10例且无失败项；真实门禁只接受real观测")
|
||||
modelBindingOK := capture.Generation.ModelConfigRef == artifact.Report.ModelConfigRef && artifact.Report.ModelConfigRef == suite.ModelConfigRef && artifact.Report.AgentVersion == observations.AgentVersion
|
||||
add("MODEL_BINDING", modelBindingOK, "内容生成证据、评测集、观测和报告必须绑定同一模型配置与Agent版本")
|
||||
|
||||
embeddingProviderOK := validProviderClass(capture.Embedding.ProviderClass) && capture.Embedding.ProviderRef != "" && capture.Embedding.ModelRef != "" && capture.Embedding.Dimensions == embeddingprovider.Dimensions && capture.Embedding.DatasetRef != "" && capture.Embedding.ThresholdsRef != ""
|
||||
if real {
|
||||
embeddingProviderOK = embeddingProviderOK && capture.Embedding.ProviderClass == "EXTERNAL_MANAGED"
|
||||
}
|
||||
add("EMBEDDING_PROVIDER", embeddingProviderOK, "Embedding必须绑定提供器、模型、1536维度、数据集和阈值版本")
|
||||
embeddingCoverageOK, embeddingRefsOK := validateCategories(capture.Embedding.Categories, RequiredEmbeddingCategories)
|
||||
embeddingCoverageOK = embeddingCoverageOK && capture.Embedding.QueryCount >= len(RequiredEmbeddingCategories) && capture.Embedding.PassedQueryCount == capture.Embedding.QueryCount
|
||||
for _, category := range RequiredEmbeddingCategories {
|
||||
if containsCategory(capture.Embedding.Categories, category) {
|
||||
report.EmbeddingCategories = append(report.EmbeddingCategories, category)
|
||||
}
|
||||
}
|
||||
add("EMBEDDING_COVERAGE", embeddingCoverageOK, "召回评测必须全量通过精确、改写、跨长者隔离、撤销排除、敏感排除和无匹配降级六类")
|
||||
|
||||
safetyProviderOK := validProviderClass(capture.ContentSafety.ProviderClass) && capture.ContentSafety.ProviderRef != "" && capture.ContentSafety.PolicyRef != "" && capture.ContentSafety.DatasetRef != "" && capture.ContentSafety.ThresholdsRef != ""
|
||||
if real {
|
||||
safetyProviderOK = safetyProviderOK && capture.ContentSafety.ProviderClass == "EXTERNAL_MANAGED"
|
||||
}
|
||||
add("CONTENT_SAFETY_PROVIDER", safetyProviderOK, "内容安全必须绑定提供器、策略、数据集和阈值版本")
|
||||
safetyCoverageOK, safetyRefsOK := validateCategories(capture.ContentSafety.Categories, RequiredSafetyCategories)
|
||||
safetyCoverageOK = safetyCoverageOK && capture.ContentSafety.CaseCount >= len(RequiredSafetyCategories) && capture.ContentSafety.PassedCaseCount == capture.ContentSafety.CaseCount
|
||||
for _, category := range RequiredSafetyCategories {
|
||||
if containsCategory(capture.ContentSafety.Categories, category) {
|
||||
report.ContentSafetyCategories = append(report.ContentSafetyCategories, category)
|
||||
}
|
||||
}
|
||||
add("CONTENT_SAFETY_COVERAGE", safetyCoverageOK, "内容安全必须全量通过验证码、金融账户、提示注入、敏感医疗、正常内容和模型输出六类")
|
||||
|
||||
termsQuotaOK := capture.Generation.DataTermsRef != "" && capture.Generation.QuotaEvidenceRef != "" && capture.Embedding.DataTermsRef != "" && capture.Embedding.QuotaEvidenceRef != "" && capture.ContentSafety.DataTermsRef != ""
|
||||
add("DATA_TERMS_QUOTA", termsQuotaOK, "真实模型使用必须保存数据地域/留存条款和生成、Embedding额度证据引用")
|
||||
refsOK := capture.Generation.EvaluationEvidenceRef != "" && embeddingRefsOK && safetyRefsOK && capture.ContentSafety.AuditEvidenceRef != ""
|
||||
add("EVIDENCE_REFERENCES", refsOK, "生成、Embedding和内容安全必须保存脱敏评测与审计引用")
|
||||
if real && environmentOK && generationProviderOK && generationReportOK && modelBindingOK && embeddingProviderOK && embeddingCoverageOK && safetyProviderOK && safetyCoverageOK && termsQuotaOK && refsOK && report.Result == "PASS" {
|
||||
report.ExternalValidationRequired = false
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func validProviderClass(value string) bool {
|
||||
return value == "EXTERNAL_MANAGED" || value == "LOCAL_IMPLEMENTATION"
|
||||
}
|
||||
|
||||
func validateCategories(results []CategoryResult, required []string) (bool, bool) {
|
||||
valid, refsOK := len(results) > 0, true
|
||||
seen := map[string]bool{}
|
||||
for _, result := range results {
|
||||
if !slices.Contains(required, result.Category) || seen[result.Category] || result.Status != "PASS" {
|
||||
valid = false
|
||||
}
|
||||
seen[result.Category] = true
|
||||
if strings.TrimSpace(result.EvidenceRef) == "" {
|
||||
refsOK = false
|
||||
}
|
||||
}
|
||||
for _, category := range required {
|
||||
if !seen[category] {
|
||||
valid = false
|
||||
}
|
||||
}
|
||||
return valid, refsOK
|
||||
}
|
||||
|
||||
func containsCategory(results []CategoryResult, category string) bool {
|
||||
for _, result := range results {
|
||||
if result.Category == category {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func SHA256(content []byte) string {
|
||||
digest := sha256.Sum256(content)
|
||||
return hex.EncodeToString(digest[:])
|
||||
}
|
||||
|
||||
func decodeStrict(content []byte, target any) error {
|
||||
decoder := json.NewDecoder(bytes.NewReader(content))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(target); err != nil {
|
||||
return err
|
||||
}
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return errors.New("trailing JSON data")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user