chore: establish local platform baseline
This commit is contained in:
Codex Local
@@ -0,0 +1,306 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/audit"
|
||||
"data-terminal-cloud/internal/buildinfo"
|
||||
"data-terminal-cloud/internal/idempotency"
|
||||
"data-terminal-cloud/internal/identity"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
var (
|
||||
acceptanceCandidatePattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._+-]{0,127}$`)
|
||||
acceptanceDigestPattern = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`)
|
||||
acceptanceEnvironments = map[string]bool{"LAB": true, "STAGING": true, "PRODUCTION": true}
|
||||
acceptanceRunTypes = map[string]bool{"REAL_DEVICE_PROVIDER": true, "PILOT": true, "PRODUCTION_RELEASE": true}
|
||||
acceptanceDependencies = map[string]string{"oidc_jwks": "api", "ark_tls": "worker", "volc_rtc_tls": "worker", "content_safety_tls": "worker"}
|
||||
)
|
||||
|
||||
type AcceptanceSnapshotInput struct {
|
||||
CandidateVersion string `json:"candidate_version"`
|
||||
BuildDigest string `json:"build_digest"`
|
||||
Environment string `json:"environment"`
|
||||
RunType string `json:"run_type"`
|
||||
Scope string `json:"scope"`
|
||||
RequiredDependencies []string `json:"required_dependencies"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
type AcceptanceDatabaseState struct {
|
||||
MigrationVersion int64 `json:"migration_version"`
|
||||
Dirty bool `json:"dirty"`
|
||||
}
|
||||
|
||||
type AcceptanceDependencyState struct {
|
||||
ServiceName string `json:"service_name"`
|
||||
Dependency string `json:"dependency"`
|
||||
Status string `json:"status"`
|
||||
CheckedAt *time.Time `json:"checked_at,omitempty"`
|
||||
LastSuccessAt *time.Time `json:"last_success_at,omitempty"`
|
||||
LastFailureAt *time.Time `json:"last_failure_at,omitempty"`
|
||||
LastErrorCode string `json:"last_error_code,omitempty"`
|
||||
LatencyMS int64 `json:"latency_ms"`
|
||||
ProbeIntervalMS int64 `json:"probe_interval_ms"`
|
||||
ConsecutiveFailures int `json:"consecutive_failures"`
|
||||
Fresh bool `json:"fresh"`
|
||||
GateSatisfied bool `json:"gate_satisfied"`
|
||||
}
|
||||
|
||||
type AcceptanceOutboxState struct {
|
||||
Pending int64 `json:"pending"`
|
||||
Processing int64 `json:"processing"`
|
||||
Dead int64 `json:"dead"`
|
||||
}
|
||||
|
||||
type AcceptanceAlertState struct {
|
||||
CriticalOpen int64 `json:"critical_open"`
|
||||
HighOpen int64 `json:"high_open"`
|
||||
Blocking int64 `json:"blocking"`
|
||||
BlockingItems []AcceptanceBlockingAlert `json:"blocking_items"`
|
||||
BlockingItemsTruncated bool `json:"blocking_items_truncated"`
|
||||
}
|
||||
|
||||
type AcceptanceBlockingAlert struct {
|
||||
ID string `json:"id"`
|
||||
RuleCode string `json:"rule_code"`
|
||||
Severity string `json:"severity"`
|
||||
Status string `json:"status"`
|
||||
ResourceType string `json:"resource_type,omitempty"`
|
||||
ResourceID string `json:"resource_id,omitempty"`
|
||||
ResourceState string `json:"resource_state"`
|
||||
FirstDetectedAt time.Time `json:"first_detected_at"`
|
||||
LastDetectedAt time.Time `json:"last_detected_at"`
|
||||
}
|
||||
|
||||
type AcceptanceGate struct {
|
||||
Code string `json:"code"`
|
||||
Satisfied bool `json:"satisfied"`
|
||||
Summary string `json:"summary"`
|
||||
}
|
||||
|
||||
type AcceptanceSnapshot struct {
|
||||
ID string `json:"id"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
TenantID string `json:"tenant_id"`
|
||||
CandidateVersion string `json:"candidate_version"`
|
||||
BuildDigest string `json:"build_digest"`
|
||||
Environment string `json:"environment"`
|
||||
RunType string `json:"run_type"`
|
||||
Scope string `json:"scope"`
|
||||
RedactionPolicy string `json:"redaction_policy"`
|
||||
RuntimeBuild buildinfo.Info `json:"runtime_build"`
|
||||
Database AcceptanceDatabaseState `json:"database"`
|
||||
IntegrationIssues IntegrationIssueSummary `json:"integration_issues"`
|
||||
ExternalDependencies []AcceptanceDependencyState `json:"external_dependencies"`
|
||||
Outbox AcceptanceOutboxState `json:"outbox"`
|
||||
Alerts AcceptanceAlertState `json:"alerts"`
|
||||
Gates []AcceptanceGate `json:"gates"`
|
||||
OverallGateSatisfied bool `json:"overall_gate_satisfied"`
|
||||
Disclaimer string `json:"disclaimer"`
|
||||
}
|
||||
|
||||
func (s *Service) ExportAcceptanceSnapshot(ctx context.Context, principal identity.Principal, tenantPublicID, key, requestID string, input AcceptanceSnapshotInput) (AcceptanceSnapshot, bool, error) {
|
||||
normalizeAcceptanceSnapshotInput(&input)
|
||||
if !validAcceptanceSnapshotInput(input) {
|
||||
return AcceptanceSnapshot{}, false, ErrInvalid
|
||||
}
|
||||
a, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_operator", "ops_admin")
|
||||
if err != nil {
|
||||
return AcceptanceSnapshot{}, false, err
|
||||
}
|
||||
requestBody, _ := json.Marshal(input)
|
||||
result, err := s.idem.Execute(ctx, idempotency.Request{
|
||||
TenantID: &a.tenantID, ActorKey: "ops:" + a.userPublicID,
|
||||
Scope: "POST /api/v1/ops/diagnostics/acceptance-snapshots", Key: key,
|
||||
RequestHash: idempotency.Hash(requestBody), ExpiresAfter: 24 * time.Hour,
|
||||
}, func(operationContext context.Context, tx pgx.Tx) (idempotency.Result, error) {
|
||||
snapshot, snapshotErr := buildAcceptanceSnapshot(operationContext, tx, tenantPublicID, a.tenantID, input)
|
||||
if snapshotErr != nil {
|
||||
return idempotency.Result{}, snapshotErr
|
||||
}
|
||||
if auditErr := s.audit.Record(operationContext, tx, audit.Event{
|
||||
TenantID: &a.tenantID, ActorType: "user", ActorPublicID: &a.userPublicID,
|
||||
Action: "ops.diagnostics.acceptance_snapshot.export", ResourceType: "acceptance_snapshot", ResourceID: &snapshot.ID,
|
||||
RequestID: requestID, Purpose: input.Reason, Result: "success",
|
||||
Detail: map[string]any{
|
||||
"candidate_version": input.CandidateVersion, "build_digest": input.BuildDigest,
|
||||
"environment": input.Environment, "run_type": input.RunType, "scope": input.Scope,
|
||||
"required_dependencies": input.RequiredDependencies, "overall_gate_satisfied": snapshot.OverallGateSatisfied,
|
||||
"redaction_policy": snapshot.RedactionPolicy,
|
||||
},
|
||||
}); auditErr != nil {
|
||||
return idempotency.Result{}, auditErr
|
||||
}
|
||||
encoded, _ := json.Marshal(snapshot)
|
||||
return idempotency.Result{StatusCode: 201, Body: encoded, ResourceID: &snapshot.ID}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return AcceptanceSnapshot{}, false, err
|
||||
}
|
||||
var snapshot AcceptanceSnapshot
|
||||
if err = json.Unmarshal(result.Body, &snapshot); err != nil {
|
||||
return AcceptanceSnapshot{}, false, err
|
||||
}
|
||||
return snapshot, !result.Replayed, nil
|
||||
}
|
||||
|
||||
func normalizeAcceptanceSnapshotInput(input *AcceptanceSnapshotInput) {
|
||||
input.CandidateVersion = strings.TrimSpace(input.CandidateVersion)
|
||||
input.BuildDigest = strings.ToLower(strings.TrimSpace(input.BuildDigest))
|
||||
input.Environment = strings.ToUpper(strings.TrimSpace(input.Environment))
|
||||
input.RunType = strings.ToUpper(strings.TrimSpace(input.RunType))
|
||||
input.Scope = strings.TrimSpace(input.Scope)
|
||||
input.Reason = strings.TrimSpace(input.Reason)
|
||||
for index := range input.RequiredDependencies {
|
||||
input.RequiredDependencies[index] = strings.ToLower(strings.TrimSpace(input.RequiredDependencies[index]))
|
||||
}
|
||||
sort.Strings(input.RequiredDependencies)
|
||||
}
|
||||
|
||||
func validAcceptanceSnapshotInput(input AcceptanceSnapshotInput) bool {
|
||||
if !acceptanceCandidatePattern.MatchString(input.CandidateVersion) || !acceptanceDigestPattern.MatchString(input.BuildDigest) ||
|
||||
!acceptanceEnvironments[input.Environment] || !acceptanceRunTypes[input.RunType] ||
|
||||
input.Scope == "" || len([]rune(input.Scope)) > 500 || input.Reason == "" || len([]rune(input.Reason)) > 255 ||
|
||||
len(input.RequiredDependencies) == 0 || len(input.RequiredDependencies) > len(acceptanceDependencies) {
|
||||
return false
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, dependency := range input.RequiredDependencies {
|
||||
if _, allowed := acceptanceDependencies[dependency]; !allowed || seen[dependency] {
|
||||
return false
|
||||
}
|
||||
seen[dependency] = true
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func buildAcceptanceSnapshot(ctx context.Context, tx pgx.Tx, tenantPublicID, tenantInternalID string, input AcceptanceSnapshotInput) (AcceptanceSnapshot, error) {
|
||||
now := time.Now().UTC()
|
||||
snapshot := AcceptanceSnapshot{
|
||||
ID: uuid.NewString(), GeneratedAt: now, TenantID: tenantPublicID,
|
||||
CandidateVersion: input.CandidateVersion, BuildDigest: input.BuildDigest, Environment: input.Environment,
|
||||
RunType: input.RunType, Scope: input.Scope, RedactionPolicy: "ops-acceptance-minimal-v1",
|
||||
RuntimeBuild: buildinfo.Current(), Alerts: AcceptanceAlertState{BlockingItems: []AcceptanceBlockingAlert{}},
|
||||
Disclaimer: "本快照仅证明生成时刻的技术预检状态，不替代真实设备、真实供应商、灰度观察或发布审批结论。",
|
||||
}
|
||||
if err := tx.QueryRow(ctx, `SELECT version,dirty FROM schema_migrations ORDER BY version DESC LIMIT 1`).Scan(&snapshot.Database.MigrationVersion, &snapshot.Database.Dirty); err != nil {
|
||||
return AcceptanceSnapshot{}, err
|
||||
}
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT count(*),
|
||||
count(*) FILTER (WHERE status <> 'CLOSED'),
|
||||
count(*) FILTER (WHERE severity='CRITICAL' AND status <> 'CLOSED'),
|
||||
count(*) FILTER (WHERE severity='HIGH' AND status <> 'CLOSED'),
|
||||
count(*) FILTER (WHERE severity IN ('CRITICAL','HIGH') AND status <> 'CLOSED')
|
||||
FROM integration_issue WHERE tenant_id=$1::uuid`, tenantInternalID).Scan(
|
||||
&snapshot.IntegrationIssues.Total, &snapshot.IntegrationIssues.Open, &snapshot.IntegrationIssues.CriticalOpen,
|
||||
&snapshot.IntegrationIssues.HighOpen, &snapshot.IntegrationIssues.ReleaseBlocking,
|
||||
); err != nil {
|
||||
return AcceptanceSnapshot{}, err
|
||||
}
|
||||
snapshot.IntegrationIssues.ReleaseGateSatisfied = snapshot.IntegrationIssues.ReleaseBlocking == 0
|
||||
|
||||
for _, dependency := range input.RequiredDependencies {
|
||||
state := AcceptanceDependencyState{ServiceName: acceptanceDependencies[dependency], Dependency: dependency, Status: "MISSING"}
|
||||
var checkedAt time.Time
|
||||
err := tx.QueryRow(ctx, `SELECT status,checked_at,last_success_at,last_failure_at,COALESCE(last_error_code,''),latency_ms,probe_interval_ms,consecutive_failures FROM external_dependency_probe_state WHERE service_name=$1 AND dependency=$2`, state.ServiceName, dependency).Scan(
|
||||
&state.Status, &checkedAt, &state.LastSuccessAt, &state.LastFailureAt, &state.LastErrorCode,
|
||||
&state.LatencyMS, &state.ProbeIntervalMS, &state.ConsecutiveFailures,
|
||||
)
|
||||
if err != nil && err != pgx.ErrNoRows {
|
||||
return AcceptanceSnapshot{}, err
|
||||
}
|
||||
if err == nil {
|
||||
state.CheckedAt = &checkedAt
|
||||
state.Fresh = !checkedAt.After(now) && now.Sub(checkedAt) <= 3*time.Duration(state.ProbeIntervalMS)*time.Millisecond
|
||||
state.GateSatisfied = state.Status == "AVAILABLE" && state.Fresh
|
||||
}
|
||||
snapshot.ExternalDependencies = append(snapshot.ExternalDependencies, state)
|
||||
}
|
||||
if err := tx.QueryRow(ctx, `SELECT
|
||||
count(*) FILTER (WHERE status='pending'),
|
||||
count(*) FILTER (WHERE status='processing'),
|
||||
count(*) FILTER (WHERE status='dead')
|
||||
FROM outbox_event WHERE tenant_id=$1::uuid OR tenant_id IS NULL`, tenantInternalID).Scan(
|
||||
&snapshot.Outbox.Pending, &snapshot.Outbox.Processing, &snapshot.Outbox.Dead,
|
||||
); err != nil {
|
||||
return AcceptanceSnapshot{}, err
|
||||
}
|
||||
rows, err := tx.Query(ctx, `SELECT public_id::text,rule_code,severity,status,COALESCE(resource_type,''),COALESCE(resource_public_id::text,''),
|
||||
CASE
|
||||
WHEN resource_public_id IS NULL THEN 'AGGREGATE'
|
||||
WHEN resource_type='conversation' THEN CASE WHEN EXISTS(SELECT 1 FROM conversation c WHERE c.public_id=alert_incident.resource_public_id) THEN 'PRESENT' ELSE 'MISSING' END
|
||||
WHEN resource_type='privacy_request' THEN CASE WHEN EXISTS(SELECT 1 FROM privacy_request pr WHERE pr.public_id=alert_incident.resource_public_id) THEN 'PRESENT' ELSE 'MISSING' END
|
||||
WHEN resource_type='upload_batch' THEN CASE WHEN EXISTS(SELECT 1 FROM upload_batch ub WHERE ub.public_id=alert_incident.resource_public_id) THEN 'PRESENT' ELSE 'MISSING' END
|
||||
WHEN resource_type='firmware_rollout' THEN CASE WHEN EXISTS(SELECT 1 FROM firmware_rollout fr WHERE fr.public_id=alert_incident.resource_public_id) THEN 'PRESENT' ELSE 'MISSING' END
|
||||
WHEN resource_type='ops_resource_reference' THEN CASE WHEN EXISTS(SELECT 1 FROM ops_resource_reference rr WHERE rr.public_id=alert_incident.resource_public_id) THEN 'PRESENT' ELSE 'MISSING' END
|
||||
ELSE 'UNKNOWN'
|
||||
END,first_detected_at,last_detected_at
|
||||
FROM alert_incident
|
||||
WHERE tenant_id=$1::uuid AND status<>'RESOLVED' AND severity IN ('CRITICAL','HIGH')
|
||||
ORDER BY CASE severity WHEN 'CRITICAL' THEN 0 ELSE 1 END,first_detected_at,public_id
|
||||
LIMIT 100`, tenantInternalID)
|
||||
if err != nil {
|
||||
return AcceptanceSnapshot{}, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var item AcceptanceBlockingAlert
|
||||
if err = rows.Scan(&item.ID, &item.RuleCode, &item.Severity, &item.Status, &item.ResourceType, &item.ResourceID, &item.ResourceState, &item.FirstDetectedAt, &item.LastDetectedAt); err != nil {
|
||||
rows.Close()
|
||||
return AcceptanceSnapshot{}, err
|
||||
}
|
||||
snapshot.Alerts.BlockingItems = append(snapshot.Alerts.BlockingItems, item)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return AcceptanceSnapshot{}, err
|
||||
}
|
||||
rows.Close()
|
||||
snapshot.Alerts.BlockingItemsTruncated = snapshot.Alerts.Blocking > int64(len(snapshot.Alerts.BlockingItems))
|
||||
if err := tx.QueryRow(ctx, `SELECT
|
||||
count(*) FILTER (WHERE severity='CRITICAL'),
|
||||
count(*) FILTER (WHERE severity='HIGH'),
|
||||
count(*)
|
||||
FROM alert_incident WHERE tenant_id=$1::uuid AND status<>'RESOLVED' AND severity IN ('CRITICAL','HIGH')`, tenantInternalID).Scan(
|
||||
&snapshot.Alerts.CriticalOpen, &snapshot.Alerts.HighOpen, &snapshot.Alerts.Blocking,
|
||||
); err != nil {
|
||||
return AcceptanceSnapshot{}, err
|
||||
}
|
||||
|
||||
runtimeSatisfied := input.Environment == "LAB" || (snapshot.RuntimeBuild.Version == input.CandidateVersion && snapshot.RuntimeBuild.Commit != "unknown" && snapshot.RuntimeBuild.BuildTime != "unknown")
|
||||
dependenciesSatisfied := true
|
||||
for _, dependency := range snapshot.ExternalDependencies {
|
||||
dependenciesSatisfied = dependenciesSatisfied && dependency.GateSatisfied
|
||||
}
|
||||
outboxSatisfied := snapshot.Outbox.Pending == 0 && snapshot.Outbox.Processing == 0 && snapshot.Outbox.Dead == 0
|
||||
alertsSatisfied := snapshot.Alerts.Blocking == 0
|
||||
snapshot.Gates = []AcceptanceGate{
|
||||
{Code: "candidate_build_identified", Satisfied: runtimeSatisfied, Summary: acceptanceRuntimeSummary(input.Environment)},
|
||||
{Code: "database_migration_clean", Satisfied: !snapshot.Database.Dirty, Summary: "数据库迁移必须处于非脏状态"},
|
||||
{Code: "required_dependencies_available", Satisfied: dependenciesSatisfied, Summary: "所有必需外部依赖探针必须可用且未过期"},
|
||||
{Code: "integration_issues_clear", Satisfied: snapshot.IntegrationIssues.ReleaseGateSatisfied, Summary: "不得存在未关闭的高或严重级联调问题"},
|
||||
{Code: "outbox_clear", Satisfied: outboxSatisfied, Summary: "租户及平台级待处理、处理中和死信任务必须清零"},
|
||||
{Code: "critical_alerts_clear", Satisfied: alertsSatisfied, Summary: "不得存在未解决的高或严重级告警"},
|
||||
}
|
||||
snapshot.OverallGateSatisfied = true
|
||||
for _, gate := range snapshot.Gates {
|
||||
snapshot.OverallGateSatisfied = snapshot.OverallGateSatisfied && gate.Satisfied
|
||||
}
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
func acceptanceRuntimeSummary(environment string) string {
|
||||
if environment == "LAB" {
|
||||
return "实验室环境允许开发构建，但候选版本与镜像摘要必须明确"
|
||||
}
|
||||
return "预发布和生产环境的运行版本必须等于候选版本，并包含提交与构建时间"
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"data-terminal-cloud/internal/audit"
|
||||
"data-terminal-cloud/internal/database"
|
||||
"data-terminal-cloud/internal/idempotency"
|
||||
"data-terminal-cloud/internal/identity"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestAcceptanceSnapshotIsAuditedIdempotentAndTenantScoped(t *testing.T) {
|
||||
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := database.Open(ctx, databaseURL, "acceptance-snapshot-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
service, err := NewService(pool, idempotency.NewExecutor(pool), audit.NewRepository())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
admin := identity.Principal{Issuer: "http://localhost:58180/realms/voice-local", Subject: "00000000-0000-0000-0000-000000000101"}
|
||||
tenantPublicID := "10000000-0000-0000-0000-000000000010"
|
||||
key := "acceptance-snapshot-" + uuid.NewString()
|
||||
requestID := "acceptance-snapshot-test-" + uuid.NewString()
|
||||
alertID := uuid.NewString()
|
||||
missingConversationID := uuid.NewString()
|
||||
defer func() {
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM audit_event WHERE request_id=$1`, requestID)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM idempotency_record WHERE idempotency_key=$1`, key)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM alert_incident WHERE public_id=$1::uuid`, alertID)
|
||||
}()
|
||||
if _, err = pool.Exec(ctx, `INSERT INTO alert_incident(public_id,tenant_id,rule_code,category,severity,status,fingerprint,title,summary,resource_type,resource_public_id,first_detected_at,last_detected_at) VALUES($1::uuid,'10000000-0000-0000-0000-000000000001'::uuid,'ACCEPTANCE_SNAPSHOT_TEST','TECHNICAL','HIGH','OPEN',$2,'验收快照合成阻断','验证缺失资源定位','conversation',$3::uuid,now(),now())`, alertID, "acceptance-snapshot-test:"+alertID, missingConversationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
input := AcceptanceSnapshotInput{
|
||||
CandidateVersion: "0.50.0-rc.1",
|
||||
BuildDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Environment: "LAB",
|
||||
RunType: "REAL_DEVICE_PROVIDER",
|
||||
Scope: "本地真实OIDC与平台技术预检",
|
||||
RequiredDependencies: []string{
|
||||
"oidc_jwks",
|
||||
},
|
||||
Reason: "生成可归档的阶段8技术验收证据",
|
||||
}
|
||||
first, created, err := service.ExportAcceptanceSnapshot(ctx, admin, tenantPublicID, key, requestID, input)
|
||||
if err != nil || !created {
|
||||
t.Fatalf("first=%+v created=%t err=%v", first, created, err)
|
||||
}
|
||||
if first.ID == "" || first.Database.MigrationVersion < 36 || first.Database.Dirty || first.RedactionPolicy != "ops-acceptance-minimal-v1" || len(first.ExternalDependencies) != 1 || len(first.Gates) != 6 {
|
||||
t.Fatalf("unexpected snapshot=%+v", first)
|
||||
}
|
||||
if first.ExternalDependencies[0].Dependency != "oidc_jwks" || first.ExternalDependencies[0].ServiceName != "api" {
|
||||
t.Fatalf("unexpected dependency=%+v", first.ExternalDependencies[0])
|
||||
}
|
||||
var foundBlockingAlert bool
|
||||
for _, item := range first.Alerts.BlockingItems {
|
||||
if item.ID == alertID {
|
||||
foundBlockingAlert = item.RuleCode == "ACCEPTANCE_SNAPSHOT_TEST" && item.ResourceID == missingConversationID && item.ResourceState == "MISSING"
|
||||
}
|
||||
}
|
||||
if !foundBlockingAlert || first.Alerts.Blocking == 0 || first.Gates[5].Code != "critical_alerts_clear" || first.Gates[5].Satisfied {
|
||||
t.Fatalf("blocking alert evidence=%+v gates=%+v", first.Alerts, first.Gates)
|
||||
}
|
||||
replayed, createdAgain, err := service.ExportAcceptanceSnapshot(ctx, admin, tenantPublicID, key, requestID, input)
|
||||
if err != nil || createdAgain || replayed.ID != first.ID || !replayed.GeneratedAt.Equal(first.GeneratedAt) {
|
||||
t.Fatalf("replayed=%+v created=%t err=%v", replayed, createdAgain, err)
|
||||
}
|
||||
if _, _, err = service.ExportAcceptanceSnapshot(ctx, admin, "20000000-0000-0000-0000-000000000010", "cross-"+key, requestID, input); !errors.Is(err, ErrAccessDenied) {
|
||||
t.Fatalf("cross tenant error=%v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidAcceptanceSnapshotInputLimitsAuditReason(t *testing.T) {
|
||||
input := AcceptanceSnapshotInput{
|
||||
CandidateVersion: "0.55.0-rc.1",
|
||||
BuildDigest: "sha256:" + strings.Repeat("a", 64),
|
||||
Environment: "LAB",
|
||||
RunType: "PILOT",
|
||||
Scope: "本地试点技术预检",
|
||||
RequiredDependencies: []string{"oidc_jwks"},
|
||||
Reason: strings.Repeat("理", 255),
|
||||
}
|
||||
if !validAcceptanceSnapshotInput(input) {
|
||||
t.Fatal("255-character audit reason should be accepted")
|
||||
}
|
||||
input.Reason += "由"
|
||||
if validAcceptanceSnapshotInput(input) {
|
||||
t.Fatal("reason exceeding audit_event.purpose capacity should be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptanceSnapshotAllowsAllProductionExternalDependencies(t *testing.T) {
|
||||
input := AcceptanceSnapshotInput{
|
||||
CandidateVersion: "0.64.0-rc.1", BuildDigest: "sha256:" + strings.Repeat("b", 64),
|
||||
Environment: "PRODUCTION", RunType: "PRODUCTION_RELEASE", Scope: "生产候选外部依赖预检", Reason: "归档发布技术门禁",
|
||||
RequiredDependencies: []string{"oidc_jwks", "ark_tls", "volc_rtc_tls", "content_safety_tls"},
|
||||
}
|
||||
if !validAcceptanceSnapshotInput(input) {
|
||||
t.Fatal("all four supported production dependencies should be accepted")
|
||||
}
|
||||
input.RequiredDependencies = append(input.RequiredDependencies, "content_safety_tls")
|
||||
if validAcceptanceSnapshotInput(input) {
|
||||
t.Fatal("duplicated content safety dependency should be rejected")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,378 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/audit"
|
||||
"data-terminal-cloud/internal/concurrency"
|
||||
"data-terminal-cloud/internal/database"
|
||||
"data-terminal-cloud/internal/idempotency"
|
||||
"data-terminal-cloud/internal/identity"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
var integrationIssueStatuses = map[string]bool{
|
||||
"OPEN": true, "IN_PROGRESS": true, "RESOLVED": true, "VERIFIED": true, "CLOSED": true,
|
||||
}
|
||||
|
||||
var integrationIssueSeverities = map[string]bool{
|
||||
"LOW": true, "MEDIUM": true, "HIGH": true, "CRITICAL": true,
|
||||
}
|
||||
|
||||
type IntegrationIssue struct {
|
||||
ID string `json:"id"`
|
||||
TenantID string `json:"tenant_id"`
|
||||
Phenomenon string `json:"phenomenon"`
|
||||
Severity string `json:"severity"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
ConversationID string `json:"conversation_id,omitempty"`
|
||||
OccurredAt time.Time `json:"occurred_at"`
|
||||
RequestID string `json:"request_id,omitempty"`
|
||||
ResponsibleParty string `json:"responsible_party"`
|
||||
Status string `json:"status"`
|
||||
ResolutionResult string `json:"resolution_result,omitempty"`
|
||||
RootCause string `json:"root_cause,omitempty"`
|
||||
FixedVersion string `json:"fixed_version,omitempty"`
|
||||
RegressionConclusion string `json:"regression_conclusion,omitempty"`
|
||||
RegressionEvidence string `json:"regression_evidence,omitempty"`
|
||||
CreatedBy string `json:"created_by"`
|
||||
UpdatedBy string `json:"updated_by"`
|
||||
Version int64 `json:"version"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type CreateIntegrationIssueInput struct {
|
||||
Phenomenon string `json:"phenomenon"`
|
||||
Severity string `json:"severity"`
|
||||
DeviceID string `json:"device_id,omitempty"`
|
||||
ConversationID string `json:"conversation_id,omitempty"`
|
||||
OccurredAt time.Time `json:"occurred_at"`
|
||||
RequestID string `json:"request_id,omitempty"`
|
||||
ResponsibleParty string `json:"responsible_party"`
|
||||
}
|
||||
|
||||
type UpdateIntegrationIssueInput struct {
|
||||
Status *string `json:"status,omitempty"`
|
||||
Severity *string `json:"severity,omitempty"`
|
||||
ResponsibleParty *string `json:"responsible_party,omitempty"`
|
||||
ResolutionResult *string `json:"resolution_result,omitempty"`
|
||||
RootCause *string `json:"root_cause,omitempty"`
|
||||
FixedVersion *string `json:"fixed_version,omitempty"`
|
||||
RegressionConclusion *string `json:"regression_conclusion,omitempty"`
|
||||
RegressionEvidence *string `json:"regression_evidence,omitempty"`
|
||||
}
|
||||
|
||||
type IntegrationIssueSummary struct {
|
||||
Total int64 `json:"total"`
|
||||
Open int64 `json:"open"`
|
||||
CriticalOpen int64 `json:"critical_open"`
|
||||
HighOpen int64 `json:"high_open"`
|
||||
ReleaseBlocking int64 `json:"release_blocking"`
|
||||
ReleaseGateSatisfied bool `json:"release_gate_satisfied"`
|
||||
}
|
||||
|
||||
func (s *Service) CreateIntegrationIssue(ctx context.Context, principal identity.Principal, tenantPublicID, key, requestID string, input CreateIntegrationIssueInput) (IntegrationIssue, bool, error) {
|
||||
normalizeIntegrationIssueCreate(&input)
|
||||
if !validIntegrationIssueCreate(input) {
|
||||
return IntegrationIssue{}, false, ErrInvalid
|
||||
}
|
||||
currentActor, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_operator", "ops_admin")
|
||||
if err != nil {
|
||||
return IntegrationIssue{}, false, err
|
||||
}
|
||||
requestBody, _ := json.Marshal(input)
|
||||
result, err := s.idem.Execute(ctx, idempotency.Request{
|
||||
TenantID: ¤tActor.tenantID, ActorKey: "ops:" + currentActor.userPublicID,
|
||||
Scope: "POST /api/v1/ops/integration-issues", Key: key,
|
||||
RequestHash: idempotency.Hash(requestBody), ExpiresAfter: 24 * time.Hour,
|
||||
}, func(operationContext context.Context, tx pgx.Tx) (idempotency.Result, error) {
|
||||
deviceID, conversationID, resolveErr := resolveIntegrationIssueReferences(operationContext, tx, currentActor.tenantID, input.DeviceID, input.ConversationID)
|
||||
if resolveErr != nil {
|
||||
return idempotency.Result{}, resolveErr
|
||||
}
|
||||
publicID := uuid.NewString()
|
||||
var internalID string
|
||||
if queryErr := tx.QueryRow(operationContext, `
|
||||
INSERT INTO integration_issue(public_id,tenant_id,phenomenon,severity,device_id,conversation_id,occurred_at,request_id,responsible_party,created_by,updated_by)
|
||||
VALUES($1::uuid,$2::uuid,$3,$4,$5::uuid,$6::uuid,$7,NULLIF($8,''),$9,$10::uuid,$10::uuid)
|
||||
RETURNING id::text`, publicID, currentActor.tenantID, input.Phenomenon, input.Severity, deviceID, conversationID, input.OccurredAt, input.RequestID, input.ResponsibleParty, currentActor.userID).Scan(&internalID); queryErr != nil {
|
||||
return idempotency.Result{}, queryErr
|
||||
}
|
||||
resourceID := internalID
|
||||
if auditErr := s.audit.Record(operationContext, tx, audit.Event{
|
||||
TenantID: ¤tActor.tenantID, ActorType: "user", ActorPublicID: ¤tActor.userPublicID,
|
||||
Action: "ops.integration_issue.create", ResourceType: "integration_issue", ResourceID: &resourceID,
|
||||
RequestID: requestID, Purpose: "记录联调问题", Result: "success",
|
||||
Detail: map[string]any{"issue_id": publicID, "severity": input.Severity, "device_id": input.DeviceID, "conversation_id": input.ConversationID, "responsible_party": input.ResponsibleParty},
|
||||
}); auditErr != nil {
|
||||
return idempotency.Result{}, auditErr
|
||||
}
|
||||
item, queryErr := getIntegrationIssue(operationContext, tx, currentActor.tenantID, publicID, false)
|
||||
if queryErr != nil {
|
||||
return idempotency.Result{}, queryErr
|
||||
}
|
||||
body, _ := json.Marshal(item)
|
||||
return idempotency.Result{StatusCode: 201, Body: body, ResourceID: &internalID}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return IntegrationIssue{}, false, err
|
||||
}
|
||||
var item IntegrationIssue
|
||||
if err := json.Unmarshal(result.Body, &item); err != nil {
|
||||
return IntegrationIssue{}, false, err
|
||||
}
|
||||
return item, !result.Replayed, nil
|
||||
}
|
||||
|
||||
func (s *Service) ListIntegrationIssues(ctx context.Context, principal identity.Principal, tenantPublicID, status string) ([]IntegrationIssue, error) {
|
||||
currentActor, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_viewer", "ops_operator", "ops_approver", "ops_admin")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
status = strings.ToUpper(strings.TrimSpace(status))
|
||||
if status != "" && !integrationIssueStatuses[status] {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, integrationIssueSelect+` WHERE ii.tenant_id=$1::uuid AND (NULLIF($2,'') IS NULL OR ii.status=$2) ORDER BY ii.updated_at DESC,ii.id DESC LIMIT 100`, currentActor.tenantID, status)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []IntegrationIssue{}
|
||||
for rows.Next() {
|
||||
item, scanErr := scanIntegrationIssue(rows)
|
||||
if scanErr != nil {
|
||||
return nil, scanErr
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Service) SummarizeIntegrationIssues(ctx context.Context, principal identity.Principal, tenantPublicID string) (IntegrationIssueSummary, error) {
|
||||
currentActor, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_viewer", "ops_operator", "ops_approver", "ops_admin")
|
||||
if err != nil {
|
||||
return IntegrationIssueSummary{}, err
|
||||
}
|
||||
var summary IntegrationIssueSummary
|
||||
err = s.pool.QueryRow(ctx, `
|
||||
SELECT count(*),
|
||||
count(*) FILTER (WHERE status <> 'CLOSED'),
|
||||
count(*) FILTER (WHERE severity='CRITICAL' AND status <> 'CLOSED'),
|
||||
count(*) FILTER (WHERE severity='HIGH' AND status <> 'CLOSED'),
|
||||
count(*) FILTER (WHERE severity IN ('CRITICAL','HIGH') AND status <> 'CLOSED')
|
||||
FROM integration_issue
|
||||
WHERE tenant_id=$1::uuid`, currentActor.tenantID).Scan(
|
||||
&summary.Total, &summary.Open, &summary.CriticalOpen, &summary.HighOpen, &summary.ReleaseBlocking,
|
||||
)
|
||||
if err != nil {
|
||||
return IntegrationIssueSummary{}, err
|
||||
}
|
||||
summary.ReleaseGateSatisfied = summary.ReleaseBlocking == 0
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func (s *Service) UpdateIntegrationIssue(ctx context.Context, principal identity.Principal, tenantPublicID, publicID, requestID string, expectedVersion int64, input UpdateIntegrationIssueInput) (IntegrationIssue, error) {
|
||||
if _, err := uuid.Parse(publicID); err != nil {
|
||||
return IntegrationIssue{}, ErrInvalid
|
||||
}
|
||||
normalizeIntegrationIssueUpdate(&input)
|
||||
if input.Status == nil && input.Severity == nil && input.ResponsibleParty == nil && input.ResolutionResult == nil && input.RootCause == nil && input.FixedVersion == nil && input.RegressionConclusion == nil && input.RegressionEvidence == nil {
|
||||
return IntegrationIssue{}, ErrInvalid
|
||||
}
|
||||
currentActor, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_operator", "ops_admin")
|
||||
if err != nil {
|
||||
return IntegrationIssue{}, err
|
||||
}
|
||||
var updated IntegrationIssue
|
||||
err = database.Transact(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
current, lockErr := getIntegrationIssue(ctx, tx, currentActor.tenantID, publicID, true)
|
||||
if lockErr != nil {
|
||||
return lockErr
|
||||
}
|
||||
if err := concurrency.RequireVersion(expectedVersion, current.Version); err != nil {
|
||||
return err
|
||||
}
|
||||
previousStatus := current.Status
|
||||
if input.Status != nil {
|
||||
current.Status = *input.Status
|
||||
}
|
||||
if input.Severity != nil {
|
||||
current.Severity = *input.Severity
|
||||
}
|
||||
if input.ResponsibleParty != nil {
|
||||
current.ResponsibleParty = *input.ResponsibleParty
|
||||
}
|
||||
if input.ResolutionResult != nil {
|
||||
current.ResolutionResult = *input.ResolutionResult
|
||||
}
|
||||
if input.RootCause != nil {
|
||||
current.RootCause = *input.RootCause
|
||||
}
|
||||
if input.FixedVersion != nil {
|
||||
current.FixedVersion = *input.FixedVersion
|
||||
}
|
||||
if input.RegressionConclusion != nil {
|
||||
current.RegressionConclusion = *input.RegressionConclusion
|
||||
}
|
||||
if input.RegressionEvidence != nil {
|
||||
current.RegressionEvidence = *input.RegressionEvidence
|
||||
}
|
||||
if !validIntegrationIssueState(current) {
|
||||
return ErrInvalid
|
||||
}
|
||||
command, commandErr := tx.Exec(ctx, `
|
||||
UPDATE integration_issue
|
||||
SET status=$3,severity=$4,responsible_party=$5,resolution_result=NULLIF($6,''),root_cause=NULLIF($7,''),fixed_version=NULLIF($8,''),
|
||||
regression_conclusion=NULLIF($9,''),regression_evidence=NULLIF($10,''),updated_by=$11::uuid,version=version+1,updated_at=now()
|
||||
WHERE tenant_id=$1::uuid AND public_id=$2::uuid AND version=$12`, currentActor.tenantID, publicID, current.Status, current.Severity, current.ResponsibleParty, current.ResolutionResult, current.RootCause, current.FixedVersion, current.RegressionConclusion, current.RegressionEvidence, currentActor.userID, expectedVersion)
|
||||
if commandErr != nil {
|
||||
return commandErr
|
||||
}
|
||||
if command.RowsAffected() != 1 {
|
||||
return concurrency.ErrVersionConflict
|
||||
}
|
||||
var internalID string
|
||||
if queryErr := tx.QueryRow(ctx, `SELECT id::text FROM integration_issue WHERE tenant_id=$1::uuid AND public_id=$2::uuid`, currentActor.tenantID, publicID).Scan(&internalID); queryErr != nil {
|
||||
return queryErr
|
||||
}
|
||||
if auditErr := s.audit.Record(ctx, tx, audit.Event{
|
||||
TenantID: ¤tActor.tenantID, ActorType: "user", ActorPublicID: ¤tActor.userPublicID,
|
||||
Action: "ops.integration_issue.update", ResourceType: "integration_issue", ResourceID: &internalID,
|
||||
RequestID: requestID, Purpose: "维护联调问题闭环", Result: "success",
|
||||
Detail: map[string]any{"issue_id": publicID, "from_status": previousStatus, "to_status": current.Status, "severity": current.Severity, "fixed_version": current.FixedVersion, "responsible_party": current.ResponsibleParty},
|
||||
}); auditErr != nil {
|
||||
return auditErr
|
||||
}
|
||||
updated, lockErr = getIntegrationIssue(ctx, tx, currentActor.tenantID, publicID, false)
|
||||
return lockErr
|
||||
})
|
||||
return updated, err
|
||||
}
|
||||
|
||||
func normalizeIntegrationIssueCreate(input *CreateIntegrationIssueInput) {
|
||||
input.Phenomenon = strings.TrimSpace(input.Phenomenon)
|
||||
input.Severity = strings.ToUpper(strings.TrimSpace(input.Severity))
|
||||
input.DeviceID = strings.TrimSpace(input.DeviceID)
|
||||
input.ConversationID = strings.TrimSpace(input.ConversationID)
|
||||
input.RequestID = strings.TrimSpace(input.RequestID)
|
||||
input.ResponsibleParty = strings.TrimSpace(input.ResponsibleParty)
|
||||
}
|
||||
|
||||
func validIntegrationIssueCreate(input CreateIntegrationIssueInput) bool {
|
||||
if input.Phenomenon == "" || len([]rune(input.Phenomenon)) > 2000 || !integrationIssueSeverities[input.Severity] || input.ResponsibleParty == "" || len([]rune(input.ResponsibleParty)) > 120 || input.OccurredAt.IsZero() {
|
||||
return false
|
||||
}
|
||||
if input.DeviceID != "" {
|
||||
if _, err := uuid.Parse(input.DeviceID); err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if input.ConversationID != "" {
|
||||
if _, err := uuid.Parse(input.ConversationID); err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return input.RequestID == "" || requestIDPattern.MatchString(input.RequestID)
|
||||
}
|
||||
|
||||
func normalizeIntegrationIssueUpdate(input *UpdateIntegrationIssueInput) {
|
||||
if input.Status != nil {
|
||||
value := strings.ToUpper(strings.TrimSpace(*input.Status))
|
||||
input.Status = &value
|
||||
}
|
||||
if input.Severity != nil {
|
||||
value := strings.ToUpper(strings.TrimSpace(*input.Severity))
|
||||
input.Severity = &value
|
||||
}
|
||||
trimIssueField(&input.ResponsibleParty)
|
||||
trimIssueField(&input.ResolutionResult)
|
||||
trimIssueField(&input.RootCause)
|
||||
trimIssueField(&input.FixedVersion)
|
||||
trimIssueField(&input.RegressionConclusion)
|
||||
trimIssueField(&input.RegressionEvidence)
|
||||
}
|
||||
|
||||
func trimIssueField(value **string) {
|
||||
if *value != nil {
|
||||
trimmed := strings.TrimSpace(**value)
|
||||
*value = &trimmed
|
||||
}
|
||||
}
|
||||
|
||||
func validIntegrationIssueState(item IntegrationIssue) bool {
|
||||
if !integrationIssueStatuses[item.Status] || !integrationIssueSeverities[item.Severity] || item.ResponsibleParty == "" || len([]rune(item.ResponsibleParty)) > 120 || len([]rune(item.ResolutionResult)) > 4000 || len([]rune(item.RootCause)) > 4000 || len([]rune(item.FixedVersion)) > 128 || len([]rune(item.RegressionConclusion)) > 4000 || len([]rune(item.RegressionEvidence)) > 4000 {
|
||||
return false
|
||||
}
|
||||
if (item.Status == "RESOLVED" || item.Status == "VERIFIED" || item.Status == "CLOSED") && (item.ResolutionResult == "" || item.RootCause == "" || item.FixedVersion == "") {
|
||||
return false
|
||||
}
|
||||
return (item.Status != "VERIFIED" && item.Status != "CLOSED") || (item.RegressionConclusion != "" && item.RegressionEvidence != "")
|
||||
}
|
||||
|
||||
func resolveIntegrationIssueReferences(ctx context.Context, tx pgx.Tx, tenantID, devicePublicID, conversationPublicID string) (*string, *string, error) {
|
||||
var deviceID, conversationID *string
|
||||
if devicePublicID != "" {
|
||||
var value string
|
||||
if err := tx.QueryRow(ctx, `SELECT id::text FROM device WHERE tenant_id=$1::uuid AND public_id=$2::uuid`, tenantID, devicePublicID).Scan(&value); errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil, ErrNotFound
|
||||
} else if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
deviceID = &value
|
||||
}
|
||||
if conversationPublicID != "" {
|
||||
var value, conversationDeviceID string
|
||||
if err := tx.QueryRow(ctx, `SELECT id::text,device_id::text FROM conversation WHERE tenant_id=$1::uuid AND public_id=$2::uuid`, tenantID, conversationPublicID).Scan(&value, &conversationDeviceID); errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil, ErrNotFound
|
||||
} else if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if deviceID != nil && *deviceID != conversationDeviceID {
|
||||
return nil, nil, ErrInvalid
|
||||
}
|
||||
conversationID = &value
|
||||
}
|
||||
return deviceID, conversationID, nil
|
||||
}
|
||||
|
||||
const integrationIssueSelect = `
|
||||
SELECT ii.public_id::text,t.public_id::text,ii.phenomenon,ii.severity,COALESCE(d.public_id::text,''),COALESCE(c.public_id::text,''),
|
||||
ii.occurred_at,COALESCE(ii.request_id,''),ii.responsible_party,ii.status,
|
||||
COALESCE(ii.resolution_result,''),COALESCE(ii.root_cause,''),COALESCE(ii.fixed_version,''),COALESCE(ii.regression_conclusion,''),COALESCE(ii.regression_evidence,''),
|
||||
creator.public_id::text,updater.public_id::text,ii.version,ii.created_at,ii.updated_at
|
||||
FROM integration_issue ii
|
||||
JOIN tenant t ON t.id=ii.tenant_id
|
||||
JOIN user_account creator ON creator.id=ii.created_by AND creator.tenant_id=ii.tenant_id
|
||||
JOIN user_account updater ON updater.id=ii.updated_by AND updater.tenant_id=ii.tenant_id
|
||||
LEFT JOIN device d ON d.id=ii.device_id AND d.tenant_id=ii.tenant_id
|
||||
LEFT JOIN conversation c ON c.id=ii.conversation_id AND c.tenant_id=ii.tenant_id`
|
||||
|
||||
type integrationIssueScanner interface{ Scan(...any) error }
|
||||
|
||||
func scanIntegrationIssue(row integrationIssueScanner) (IntegrationIssue, error) {
|
||||
var item IntegrationIssue
|
||||
err := row.Scan(&item.ID, &item.TenantID, &item.Phenomenon, &item.Severity, &item.DeviceID, &item.ConversationID, &item.OccurredAt, &item.RequestID, &item.ResponsibleParty, &item.Status, &item.ResolutionResult, &item.RootCause, &item.FixedVersion, &item.RegressionConclusion, &item.RegressionEvidence, &item.CreatedBy, &item.UpdatedBy, &item.Version, &item.CreatedAt, &item.UpdatedAt)
|
||||
return item, err
|
||||
}
|
||||
|
||||
func getIntegrationIssue(ctx context.Context, query interface {
|
||||
QueryRow(context.Context, string, ...any) pgx.Row
|
||||
}, tenantID, publicID string, lock bool) (IntegrationIssue, error) {
|
||||
queryText := integrationIssueSelect + ` WHERE ii.tenant_id=$1::uuid AND ii.public_id=$2::uuid`
|
||||
if lock {
|
||||
queryText += ` FOR UPDATE OF ii`
|
||||
}
|
||||
item, err := scanIntegrationIssue(query.QueryRow(ctx, queryText, tenantID, publicID))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return IntegrationIssue{}, ErrNotFound
|
||||
}
|
||||
return item, err
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/audit"
|
||||
"data-terminal-cloud/internal/concurrency"
|
||||
"data-terminal-cloud/internal/database"
|
||||
"data-terminal-cloud/internal/idempotency"
|
||||
"data-terminal-cloud/internal/identity"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestIntegrationIssueLifecycleIsTenantScopedIdempotentAndVersioned(t *testing.T) {
|
||||
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := database.Open(ctx, databaseURL, "integration-issue-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
service, err := NewService(pool, idempotency.NewExecutor(pool), audit.NewRepository())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
admin := identity.Principal{Issuer: "http://localhost:58180/realms/voice-local", Subject: "00000000-0000-0000-0000-000000000101"}
|
||||
tenantPublicID := "10000000-0000-0000-0000-000000000010"
|
||||
devicePublicID := "10000000-0000-0000-0000-000000000511"
|
||||
key := "integration-issue-" + uuid.NewString()
|
||||
requestPrefix := "integration-issue-test-" + uuid.NewString()
|
||||
baselineSummary, err := service.SummarizeIntegrationIssues(ctx, admin, tenantPublicID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var issueID string
|
||||
defer func() {
|
||||
if issueID != "" {
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM integration_issue WHERE public_id=$1::uuid`, issueID)
|
||||
}
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM audit_event WHERE request_id LIKE $1`, requestPrefix+"%")
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM idempotency_record WHERE idempotency_key=$1`, key)
|
||||
}()
|
||||
|
||||
input := CreateIntegrationIssueInput{
|
||||
Phenomenon: "设备入房后供应商任务未在预期时间进入活动状态",
|
||||
Severity: "HIGH",
|
||||
DeviceID: devicePublicID, OccurredAt: time.Now().UTC().Add(-time.Minute),
|
||||
RequestID: requestPrefix + "-source", ResponsibleParty: "Agent研发组",
|
||||
}
|
||||
created, firstCreated, err := service.CreateIntegrationIssue(ctx, admin, tenantPublicID, key, requestPrefix+"-create", input)
|
||||
if err != nil || !firstCreated || created.Status != "OPEN" || created.Severity != "HIGH" || created.Version != 1 || created.DeviceID != devicePublicID {
|
||||
t.Fatalf("created=%+v first=%t err=%v", created, firstCreated, err)
|
||||
}
|
||||
issueID = created.ID
|
||||
replayed, replayCreated, err := service.CreateIntegrationIssue(ctx, admin, tenantPublicID, key, requestPrefix+"-replay", input)
|
||||
if err != nil || replayCreated || replayed.ID != created.ID {
|
||||
t.Fatalf("replayed=%+v created=%t err=%v", replayed, replayCreated, err)
|
||||
}
|
||||
items, err := service.ListIntegrationIssues(ctx, admin, tenantPublicID, "OPEN")
|
||||
if err != nil || len(items) == 0 || items[0].ID != created.ID {
|
||||
t.Fatalf("items=%+v err=%v", items, err)
|
||||
}
|
||||
blockingSummary, err := service.SummarizeIntegrationIssues(ctx, admin, tenantPublicID)
|
||||
if err != nil || blockingSummary.ReleaseBlocking != baselineSummary.ReleaseBlocking+1 || blockingSummary.ReleaseGateSatisfied {
|
||||
t.Fatalf("blocking summary=%+v baseline=%+v err=%v", blockingSummary, baselineSummary, err)
|
||||
}
|
||||
|
||||
resolvedStatus, resolution := "RESOLVED", "修正供应商任务启动回调的路由配置"
|
||||
rootCause, fixedVersion := "回调路由错误地使用了旧供应商事件类型", "0.49.0-rc.1+build.36"
|
||||
if _, err = service.UpdateIntegrationIssue(ctx, admin, tenantPublicID, created.ID, requestPrefix+"-invalid-resolve", 1, UpdateIntegrationIssueInput{Status: &resolvedStatus}); !errors.Is(err, ErrInvalid) {
|
||||
t.Fatalf("missing resolution error=%v", err)
|
||||
}
|
||||
resolved, err := service.UpdateIntegrationIssue(ctx, admin, tenantPublicID, created.ID, requestPrefix+"-resolve", 1, UpdateIntegrationIssueInput{Status: &resolvedStatus, ResolutionResult: &resolution, RootCause: &rootCause, FixedVersion: &fixedVersion})
|
||||
if err != nil || resolved.Status != "RESOLVED" || resolved.Version != 2 || resolved.ResolutionResult != resolution || resolved.RootCause != rootCause || resolved.FixedVersion != fixedVersion {
|
||||
t.Fatalf("resolved=%+v err=%v", resolved, err)
|
||||
}
|
||||
if _, err = service.UpdateIntegrationIssue(ctx, admin, tenantPublicID, created.ID, requestPrefix+"-stale", 1, UpdateIntegrationIssueInput{ResponsibleParty: &resolution}); !errors.Is(err, concurrency.ErrVersionConflict) {
|
||||
t.Fatalf("stale version error=%v", err)
|
||||
}
|
||||
verifiedStatus, conclusion := "VERIFIED", "连续三次模拟联调均进入ACTIVE且没有遗留任务"
|
||||
evidence := "请求关联号 integration-regression-36；诊断包 artifact-36；重复3次"
|
||||
if _, err = service.UpdateIntegrationIssue(ctx, admin, tenantPublicID, created.ID, requestPrefix+"-invalid-verify", 2, UpdateIntegrationIssueInput{Status: &verifiedStatus, RegressionConclusion: &conclusion}); !errors.Is(err, ErrInvalid) {
|
||||
t.Fatalf("missing regression evidence error=%v", err)
|
||||
}
|
||||
verified, err := service.UpdateIntegrationIssue(ctx, admin, tenantPublicID, created.ID, requestPrefix+"-verify", 2, UpdateIntegrationIssueInput{Status: &verifiedStatus, RegressionConclusion: &conclusion, RegressionEvidence: &evidence})
|
||||
if err != nil || verified.Status != "VERIFIED" || verified.Version != 3 || verified.RegressionConclusion != conclusion || verified.RegressionEvidence != evidence {
|
||||
t.Fatalf("verified=%+v err=%v", verified, err)
|
||||
}
|
||||
closedStatus := "CLOSED"
|
||||
closed, err := service.UpdateIntegrationIssue(ctx, admin, tenantPublicID, created.ID, requestPrefix+"-close", 3, UpdateIntegrationIssueInput{Status: &closedStatus})
|
||||
if err != nil || closed.Version != 4 || closed.Status != "CLOSED" {
|
||||
t.Fatalf("closed=%+v err=%v", closed, err)
|
||||
}
|
||||
closedSummary, err := service.SummarizeIntegrationIssues(ctx, admin, tenantPublicID)
|
||||
if err != nil || closedSummary.ReleaseBlocking != baselineSummary.ReleaseBlocking || closedSummary.ReleaseGateSatisfied != baselineSummary.ReleaseGateSatisfied {
|
||||
t.Fatalf("closed summary=%+v baseline=%+v err=%v", closedSummary, baselineSummary, err)
|
||||
}
|
||||
if _, err = service.ListIntegrationIssues(ctx, admin, "20000000-0000-0000-0000-000000000010", ""); !errors.Is(err, ErrAccessDenied) {
|
||||
t.Fatalf("cross tenant error=%v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
_ "time/tzdata"
|
||||
|
||||
"data-terminal-cloud/internal/audit"
|
||||
"data-terminal-cloud/internal/idempotency"
|
||||
"data-terminal-cloud/internal/identity"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
type PilotDailySnapshotInput struct {
|
||||
CandidateVersion string `json:"candidate_version"`
|
||||
BuildDigest string `json:"build_digest"`
|
||||
LocalDate string `json:"local_date"`
|
||||
Timezone string `json:"timezone"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
type PilotDeviceState struct {
|
||||
Bound int64 `json:"bound"`
|
||||
Eligible int64 `json:"eligible"`
|
||||
Quarantined int64 `json:"quarantined"`
|
||||
Revoked int64 `json:"revoked"`
|
||||
Online int64 `json:"online"`
|
||||
Offline int64 `json:"offline"`
|
||||
ReportedDuringDay int64 `json:"reported_during_day"`
|
||||
}
|
||||
|
||||
type PilotCountByStatus struct {
|
||||
Status string `json:"status"`
|
||||
Count int64 `json:"count"`
|
||||
}
|
||||
|
||||
type PilotConversationState struct {
|
||||
Total int64 `json:"total"`
|
||||
Statuses []PilotCountByStatus `json:"statuses"`
|
||||
}
|
||||
|
||||
type PilotProviderState struct {
|
||||
Provider string `json:"provider"`
|
||||
CurrentStatus string `json:"current_status"`
|
||||
ConsecutiveFailures int64 `json:"consecutive_failures"`
|
||||
FailedEventsDuringDay int64 `json:"failed_events_during_day"`
|
||||
FailedCallsDuringDay int64 `json:"failed_calls_during_day"`
|
||||
NotificationBacklog int64 `json:"notification_backlog"`
|
||||
}
|
||||
|
||||
type PilotUploadState struct {
|
||||
CreatedDuringDay int64 `json:"created_during_day"`
|
||||
CompletedDuringDay int64 `json:"completed_during_day"`
|
||||
FailedDuringDay int64 `json:"failed_during_day"`
|
||||
OpenBatches int64 `json:"open_batches"`
|
||||
PendingSegments int64 `json:"pending_segments"`
|
||||
RejectedSegments int64 `json:"rejected_segments"`
|
||||
}
|
||||
|
||||
type PilotAlertState struct {
|
||||
OpenedDuringDay int64 `json:"opened_during_day"`
|
||||
ResolvedDuringDay int64 `json:"resolved_during_day"`
|
||||
Open int64 `json:"open"`
|
||||
HighOpen int64 `json:"high_open"`
|
||||
CriticalOpen int64 `json:"critical_open"`
|
||||
}
|
||||
|
||||
type PilotCostState struct {
|
||||
Currency string `json:"currency"`
|
||||
CostMicros int64 `json:"cost_micros"`
|
||||
LineCount int64 `json:"line_count"`
|
||||
}
|
||||
|
||||
type PilotDailySnapshot struct {
|
||||
ID string `json:"id"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
TenantID string `json:"tenant_id"`
|
||||
CandidateVersion string `json:"candidate_version"`
|
||||
BuildDigest string `json:"build_digest"`
|
||||
LocalDate string `json:"local_date"`
|
||||
Timezone string `json:"timezone"`
|
||||
WindowStart time.Time `json:"window_start"`
|
||||
WindowEnd time.Time `json:"window_end"`
|
||||
RedactionPolicy string `json:"redaction_policy"`
|
||||
Devices PilotDeviceState `json:"devices"`
|
||||
Conversations PilotConversationState `json:"conversations"`
|
||||
Providers []PilotProviderState `json:"providers"`
|
||||
Uploads PilotUploadState `json:"uploads"`
|
||||
Alerts PilotAlertState `json:"alerts"`
|
||||
Costs []PilotCostState `json:"costs"`
|
||||
Disclaimer string `json:"disclaimer"`
|
||||
}
|
||||
|
||||
var pilotConversationStatuses = []string{"PREPARING", "ROOM_ISSUED", "STARTING_AGENT", "ACTIVE", "STOPPING", "COMPLETED", "FAILED", "EXPIRED"}
|
||||
|
||||
func (s *Service) ExportPilotDailySnapshot(ctx context.Context, principal identity.Principal, tenantPublicID, key, requestID string, input PilotDailySnapshotInput) (PilotDailySnapshot, bool, error) {
|
||||
normalizePilotDailySnapshotInput(&input)
|
||||
_, windowStart, windowEnd, err := validatePilotDailySnapshotInput(input, time.Now().UTC())
|
||||
if err != nil {
|
||||
return PilotDailySnapshot{}, false, ErrInvalid
|
||||
}
|
||||
a, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_operator", "ops_admin")
|
||||
if err != nil {
|
||||
return PilotDailySnapshot{}, false, err
|
||||
}
|
||||
requestBody, _ := json.Marshal(input)
|
||||
result, err := s.idem.Execute(ctx, idempotency.Request{
|
||||
TenantID: &a.tenantID, ActorKey: "ops:" + a.userPublicID,
|
||||
Scope: "POST /api/v1/ops/diagnostics/pilot-daily-snapshots", Key: key,
|
||||
RequestHash: idempotency.Hash(requestBody), ExpiresAfter: 24 * time.Hour,
|
||||
}, func(operationContext context.Context, tx pgx.Tx) (idempotency.Result, error) {
|
||||
snapshot, buildErr := buildPilotDailySnapshot(operationContext, tx, tenantPublicID, a.tenantID, input, windowStart, windowEnd)
|
||||
if buildErr != nil {
|
||||
return idempotency.Result{}, buildErr
|
||||
}
|
||||
if auditErr := s.audit.Record(operationContext, tx, audit.Event{
|
||||
TenantID: &a.tenantID, ActorType: "user", ActorPublicID: &a.userPublicID,
|
||||
Action: "ops.diagnostics.pilot_daily_snapshot.export", ResourceType: "pilot_daily_snapshot", ResourceID: &snapshot.ID,
|
||||
RequestID: requestID, Purpose: input.Reason, Result: "success",
|
||||
Detail: map[string]any{"candidate_version": input.CandidateVersion, "build_digest": input.BuildDigest, "local_date": input.LocalDate, "timezone": input.Timezone, "window_start": windowStart, "window_end": windowEnd, "redaction_policy": snapshot.RedactionPolicy},
|
||||
}); auditErr != nil {
|
||||
return idempotency.Result{}, auditErr
|
||||
}
|
||||
encoded, _ := json.Marshal(snapshot)
|
||||
return idempotency.Result{StatusCode: 201, Body: encoded, ResourceID: &snapshot.ID}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return PilotDailySnapshot{}, false, err
|
||||
}
|
||||
var snapshot PilotDailySnapshot
|
||||
if err = json.Unmarshal(result.Body, &snapshot); err != nil {
|
||||
return PilotDailySnapshot{}, false, err
|
||||
}
|
||||
return snapshot, !result.Replayed, nil
|
||||
}
|
||||
|
||||
func normalizePilotDailySnapshotInput(input *PilotDailySnapshotInput) {
|
||||
input.CandidateVersion = strings.TrimSpace(input.CandidateVersion)
|
||||
input.BuildDigest = strings.ToLower(strings.TrimSpace(input.BuildDigest))
|
||||
input.LocalDate = strings.TrimSpace(input.LocalDate)
|
||||
input.Timezone = strings.TrimSpace(input.Timezone)
|
||||
input.Reason = strings.TrimSpace(input.Reason)
|
||||
}
|
||||
|
||||
func validatePilotDailySnapshotInput(input PilotDailySnapshotInput, now time.Time) (*time.Location, time.Time, time.Time, error) {
|
||||
if !acceptanceCandidatePattern.MatchString(input.CandidateVersion) || !acceptanceDigestPattern.MatchString(input.BuildDigest) || input.LocalDate == "" || input.Timezone == "" || len([]rune(input.Timezone)) > 80 || input.Reason == "" || len([]rune(input.Reason)) > 255 {
|
||||
return nil, time.Time{}, time.Time{}, ErrInvalid
|
||||
}
|
||||
location, err := time.LoadLocation(input.Timezone)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, time.Time{}, ErrInvalid
|
||||
}
|
||||
windowStart, err := time.ParseInLocation("2006-01-02", input.LocalDate, location)
|
||||
if err != nil || windowStart.Format("2006-01-02") != input.LocalDate {
|
||||
return nil, time.Time{}, time.Time{}, ErrInvalid
|
||||
}
|
||||
today := now.In(location)
|
||||
todayStart := time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, location)
|
||||
if windowStart.After(todayStart) {
|
||||
return nil, time.Time{}, time.Time{}, ErrInvalid
|
||||
}
|
||||
return location, windowStart.UTC(), windowStart.AddDate(0, 0, 1).UTC(), nil
|
||||
}
|
||||
|
||||
func buildPilotDailySnapshot(ctx context.Context, tx pgx.Tx, tenantPublicID, tenantInternalID string, input PilotDailySnapshotInput, windowStart, windowEnd time.Time) (PilotDailySnapshot, error) {
|
||||
snapshot := PilotDailySnapshot{
|
||||
ID: uuid.NewString(), GeneratedAt: time.Now().UTC(), TenantID: tenantPublicID, CandidateVersion: input.CandidateVersion, BuildDigest: input.BuildDigest, LocalDate: input.LocalDate, Timezone: input.Timezone,
|
||||
WindowStart: windowStart, WindowEnd: windowEnd, RedactionPolicy: "ops-pilot-daily-minimal-v1",
|
||||
Conversations: PilotConversationState{Statuses: []PilotCountByStatus{}}, Providers: []PilotProviderState{}, Costs: []PilotCostState{},
|
||||
Disclaimer: "本快照混合所选自然日的活动统计与生成时刻的当前积压状态，只用于试点运行复盘；不替代原始监控、真实设备体验、财务账单或发布审批。",
|
||||
}
|
||||
if err := tx.QueryRow(ctx, `WITH bound AS (
|
||||
SELECT d.id,d.status,dt.last_reported_at,dps.online
|
||||
FROM device_binding db JOIN device d ON d.id=db.device_id AND d.tenant_id=db.tenant_id
|
||||
LEFT JOIN device_twin dt ON dt.device_id=d.id AND dt.tenant_id=d.tenant_id
|
||||
LEFT JOIN device_presence_state dps ON dps.device_id=d.id AND dps.tenant_id=d.tenant_id
|
||||
WHERE db.tenant_id=$1::uuid AND db.unbound_at IS NULL)
|
||||
SELECT count(*),count(*) FILTER(WHERE status='registered'),count(*) FILTER(WHERE status='quarantined'),count(*) FILTER(WHERE status='revoked'),
|
||||
count(*) FILTER(WHERE status='registered' AND COALESCE(online,false)),count(*) FILTER(WHERE status='registered' AND NOT COALESCE(online,false)),
|
||||
count(*) FILTER(WHERE status='registered' AND last_reported_at >= $2 AND last_reported_at < $3) FROM bound`, tenantInternalID, windowStart, windowEnd).Scan(
|
||||
&snapshot.Devices.Bound, &snapshot.Devices.Eligible, &snapshot.Devices.Quarantined, &snapshot.Devices.Revoked,
|
||||
&snapshot.Devices.Online, &snapshot.Devices.Offline, &snapshot.Devices.ReportedDuringDay,
|
||||
); err != nil {
|
||||
return PilotDailySnapshot{}, err
|
||||
}
|
||||
conversationCounts := map[string]int64{}
|
||||
rows, err := tx.Query(ctx, `SELECT status,count(*) FROM conversation WHERE tenant_id=$1::uuid AND created_at >= $2 AND created_at < $3 GROUP BY status`, tenantInternalID, windowStart, windowEnd)
|
||||
if err != nil {
|
||||
return PilotDailySnapshot{}, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var status string
|
||||
var count int64
|
||||
if err = rows.Scan(&status, &count); err != nil {
|
||||
rows.Close()
|
||||
return PilotDailySnapshot{}, err
|
||||
}
|
||||
conversationCounts[status] = count
|
||||
snapshot.Conversations.Total += count
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return PilotDailySnapshot{}, err
|
||||
}
|
||||
rows.Close()
|
||||
for _, status := range pilotConversationStatuses {
|
||||
snapshot.Conversations.Statuses = append(snapshot.Conversations.Statuses, PilotCountByStatus{Status: status, Count: conversationCounts[status]})
|
||||
}
|
||||
rows, err = tx.Query(ctx, `WITH providers AS (
|
||||
SELECT provider FROM provider_dependency_state WHERE tenant_id=$1::uuid
|
||||
UNION SELECT pe.provider FROM provider_event pe WHERE pe.tenant_id=$1::uuid AND pe.received_at >= $2 AND pe.received_at < $3
|
||||
UNION SELECT rt.provider FROM conversation c JOIN rtc_task rt ON rt.conversation_id=c.id WHERE c.tenant_id=$1::uuid AND c.created_at >= $2 AND c.created_at < $3)
|
||||
SELECT p.provider,COALESCE(pds.status,'UNKNOWN'),COALESCE(pds.consecutive_failures,0),
|
||||
(SELECT count(*) FROM provider_event pe WHERE pe.tenant_id=$1::uuid AND pe.provider=p.provider AND pe.processing_status='failed' AND pe.received_at >= $2 AND pe.received_at < $3),
|
||||
(SELECT count(DISTINCT c.id) FROM conversation c
|
||||
JOIN rtc_task rt ON rt.conversation_id=c.id
|
||||
JOIN conversation_state_history csh ON csh.conversation_id=c.id
|
||||
WHERE c.tenant_id=$1::uuid AND rt.provider=p.provider AND csh.to_status='FAILED' AND csh.occurred_at >= $2 AND csh.occurred_at < $3),
|
||||
(SELECT count(*) FROM provider_event pe WHERE pe.tenant_id=$1::uuid AND pe.provider=p.provider AND pe.processing_status IN ('pending','failed'))
|
||||
FROM providers p LEFT JOIN provider_dependency_state pds ON pds.tenant_id=$1::uuid AND pds.provider=p.provider ORDER BY p.provider`, tenantInternalID, windowStart, windowEnd)
|
||||
if err != nil {
|
||||
return PilotDailySnapshot{}, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var item PilotProviderState
|
||||
if err = rows.Scan(&item.Provider, &item.CurrentStatus, &item.ConsecutiveFailures, &item.FailedEventsDuringDay, &item.FailedCallsDuringDay, &item.NotificationBacklog); err != nil {
|
||||
rows.Close()
|
||||
return PilotDailySnapshot{}, err
|
||||
}
|
||||
snapshot.Providers = append(snapshot.Providers, item)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return PilotDailySnapshot{}, err
|
||||
}
|
||||
rows.Close()
|
||||
if err = tx.QueryRow(ctx, `SELECT
|
||||
count(*) FILTER(WHERE created_at >= $2 AND created_at < $3),
|
||||
count(*) FILTER(WHERE completed_at >= $2 AND completed_at < $3),
|
||||
count(*) FILTER(WHERE status='FAILED' AND updated_at >= $2 AND updated_at < $3),
|
||||
count(*) FILTER(WHERE status IN ('PENDING','UPLOADING','VERIFYING')),
|
||||
(SELECT count(*) FROM media_segment WHERE tenant_id=$1::uuid AND status IN ('DECLARED','AUTHORIZED','UPLOADED','VERIFYING')),
|
||||
(SELECT count(*) FROM media_segment WHERE tenant_id=$1::uuid AND status='REJECTED')
|
||||
FROM upload_batch WHERE tenant_id=$1::uuid`, tenantInternalID, windowStart, windowEnd).Scan(
|
||||
&snapshot.Uploads.CreatedDuringDay, &snapshot.Uploads.CompletedDuringDay, &snapshot.Uploads.FailedDuringDay,
|
||||
&snapshot.Uploads.OpenBatches, &snapshot.Uploads.PendingSegments, &snapshot.Uploads.RejectedSegments,
|
||||
); err != nil {
|
||||
return PilotDailySnapshot{}, err
|
||||
}
|
||||
if err = tx.QueryRow(ctx, `SELECT
|
||||
count(*) FILTER(WHERE first_detected_at >= $2 AND first_detected_at < $3),
|
||||
count(*) FILTER(WHERE resolved_at >= $2 AND resolved_at < $3),
|
||||
count(*) FILTER(WHERE status <> 'RESOLVED'),
|
||||
count(*) FILTER(WHERE status <> 'RESOLVED' AND severity='HIGH'),
|
||||
count(*) FILTER(WHERE status <> 'RESOLVED' AND severity='CRITICAL')
|
||||
FROM alert_incident WHERE tenant_id=$1::uuid`, tenantInternalID, windowStart, windowEnd).Scan(
|
||||
&snapshot.Alerts.OpenedDuringDay, &snapshot.Alerts.ResolvedDuringDay, &snapshot.Alerts.Open,
|
||||
&snapshot.Alerts.HighOpen, &snapshot.Alerts.CriticalOpen,
|
||||
); err != nil {
|
||||
return PilotDailySnapshot{}, err
|
||||
}
|
||||
rows, err = tx.Query(ctx, `SELECT ccl.currency,COALESCE(sum(ccl.cost_micros),0)::bigint,count(*)
|
||||
FROM conversation_cost_line ccl JOIN conversation_usage_event cue ON cue.id=ccl.usage_event_id
|
||||
WHERE ccl.tenant_id=$1::uuid AND cue.occurred_at >= $2 AND cue.occurred_at < $3
|
||||
GROUP BY ccl.currency ORDER BY ccl.currency`, tenantInternalID, windowStart, windowEnd)
|
||||
if err != nil {
|
||||
return PilotDailySnapshot{}, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var item PilotCostState
|
||||
if err = rows.Scan(&item.Currency, &item.CostMicros, &item.LineCount); err != nil {
|
||||
rows.Close()
|
||||
return PilotDailySnapshot{}, err
|
||||
}
|
||||
snapshot.Costs = append(snapshot.Costs, item)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return PilotDailySnapshot{}, err
|
||||
}
|
||||
rows.Close()
|
||||
sort.Slice(snapshot.Providers, func(i, j int) bool { return snapshot.Providers[i].Provider < snapshot.Providers[j].Provider })
|
||||
return snapshot, nil
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestValidatePilotDailySnapshotInputUsesNaturalDay(t *testing.T) {
|
||||
now := time.Date(2026, 3, 9, 12, 0, 0, 0, time.UTC)
|
||||
_, start, end, err := validatePilotDailySnapshotInput(PilotDailySnapshotInput{
|
||||
CandidateVersion: "0.55.0-rc.test",
|
||||
BuildDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
LocalDate: "2026-03-08",
|
||||
Timezone: "America/New_York",
|
||||
Reason: "试点日报归档",
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := end.Sub(start); got != 23*time.Hour {
|
||||
t.Fatalf("DST natural day duration=%s, want 23h", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatePilotDailySnapshotInputRejectsInvalidTimezoneAndFutureDate(t *testing.T) {
|
||||
now := time.Date(2026, 8, 25, 12, 0, 0, 0, time.UTC)
|
||||
for _, input := range []PilotDailySnapshotInput{
|
||||
{LocalDate: "2026-08-25", Timezone: "Asia/Shanghai", Reason: "试点日报归档"},
|
||||
{CandidateVersion: "0.55.0-rc.test", BuildDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", LocalDate: "2026-08-25", Timezone: "Mars/Base", Reason: "试点日报归档"},
|
||||
{CandidateVersion: "0.55.0-rc.test", BuildDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", LocalDate: "2026-08-26", Timezone: "Asia/Shanghai", Reason: "试点日报归档"},
|
||||
} {
|
||||
if _, _, _, err := validatePilotDailySnapshotInput(input, now); err == nil {
|
||||
t.Fatalf("expected invalid input: %+v", input)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/audit"
|
||||
"data-terminal-cloud/internal/idempotency"
|
||||
"data-terminal-cloud/internal/identity"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
type PilotWeeklySnapshotInput struct {
|
||||
CandidateVersion string `json:"candidate_version"`
|
||||
BuildDigest string `json:"build_digest"`
|
||||
WeekStart string `json:"week_start"`
|
||||
Timezone string `json:"timezone"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
type PilotWeeklyContentReview struct {
|
||||
StoriesGenerated int64 `json:"stories_generated"`
|
||||
StoriesReviewTouched int64 `json:"stories_review_touched"`
|
||||
StoriesManuallyRevised int64 `json:"stories_manually_revised"`
|
||||
ManualRevisionRateAvailable bool `json:"manual_revision_rate_available"`
|
||||
ManualRevisionRateBasisPoints int64 `json:"manual_revision_rate_basis_points"`
|
||||
StoriesConfirmed int64 `json:"stories_confirmed"`
|
||||
StoriesRejected int64 `json:"stories_rejected"`
|
||||
StoryConfirmationRateAvailable bool `json:"story_confirmation_rate_available"`
|
||||
StoryConfirmationRateBasisPoints int64 `json:"story_confirmation_rate_basis_points"`
|
||||
MemoriesCreated int64 `json:"memories_created"`
|
||||
MemoryRevisions int64 `json:"memory_revisions"`
|
||||
CurrentMemoryIndexPending int64 `json:"current_memory_index_pending"`
|
||||
CurrentMemoryIndexFailed int64 `json:"current_memory_index_failed"`
|
||||
}
|
||||
|
||||
type PilotWeeklyCountByType struct {
|
||||
Type string `json:"type"`
|
||||
Count int64 `json:"count"`
|
||||
}
|
||||
|
||||
type PilotWeeklyCountByStatus struct {
|
||||
Status string `json:"status"`
|
||||
Count int64 `json:"count"`
|
||||
}
|
||||
|
||||
type PilotWeeklyPrivacyState struct {
|
||||
RequestsCreatedDuringWeek int64 `json:"requests_created_during_week"`
|
||||
ByType []PilotWeeklyCountByType `json:"by_type"`
|
||||
ByStatus []PilotWeeklyCountByStatus `json:"by_status"`
|
||||
CurrentOpen int64 `json:"current_open"`
|
||||
CurrentPartialFailed int64 `json:"current_partial_failed"`
|
||||
}
|
||||
|
||||
type PilotWeeklyFirmwareVersion struct {
|
||||
Model string `json:"model"`
|
||||
FirmwareVersion string `json:"firmware_version"`
|
||||
DeviceCount int64 `json:"device_count"`
|
||||
}
|
||||
|
||||
type PilotWeeklyConfigState struct {
|
||||
EligibleDevices int64 `json:"eligible_devices"`
|
||||
Aligned int64 `json:"aligned"`
|
||||
Drifted int64 `json:"drifted"`
|
||||
Unreported int64 `json:"unreported"`
|
||||
ActiveRollouts int64 `json:"active_rollouts"`
|
||||
PausedRollouts int64 `json:"paused_rollouts"`
|
||||
}
|
||||
|
||||
type PilotWeeklyExternalEvidence struct {
|
||||
Code string `json:"code"`
|
||||
Status string `json:"status"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
type PilotWeeklySnapshot struct {
|
||||
ID string `json:"id"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
TenantID string `json:"tenant_id"`
|
||||
CandidateVersion string `json:"candidate_version"`
|
||||
BuildDigest string `json:"build_digest"`
|
||||
WeekStart string `json:"week_start"`
|
||||
Timezone string `json:"timezone"`
|
||||
WindowStart time.Time `json:"window_start"`
|
||||
WindowEnd time.Time `json:"window_end"`
|
||||
IsComplete bool `json:"is_complete"`
|
||||
RedactionPolicy string `json:"redaction_policy"`
|
||||
ContentReview PilotWeeklyContentReview `json:"content_review"`
|
||||
Privacy PilotWeeklyPrivacyState `json:"privacy"`
|
||||
FirmwareVersions []PilotWeeklyFirmwareVersion `json:"firmware_versions"`
|
||||
Config PilotWeeklyConfigState `json:"config"`
|
||||
ExternalEvidence []PilotWeeklyExternalEvidence `json:"external_evidence"`
|
||||
Disclaimer string `json:"disclaimer"`
|
||||
}
|
||||
|
||||
var pilotPrivacyTypes = []string{"ACCESS", "CORRECTION", "CONSENT_WITHDRAWAL", "DELETE"}
|
||||
var pilotPrivacyStatuses = []string{"SUBMITTED", "VERIFYING", "EXECUTING", "COMPLETED", "PARTIAL_FAILED"}
|
||||
|
||||
func (s *Service) ExportPilotWeeklySnapshot(ctx context.Context, principal identity.Principal, tenantPublicID, key, requestID string, input PilotWeeklySnapshotInput) (PilotWeeklySnapshot, bool, error) {
|
||||
normalizePilotWeeklySnapshotInput(&input)
|
||||
_, windowStart, windowEnd, err := validatePilotWeeklySnapshotInput(input, time.Now().UTC())
|
||||
if err != nil {
|
||||
return PilotWeeklySnapshot{}, false, ErrInvalid
|
||||
}
|
||||
a, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_operator", "ops_admin")
|
||||
if err != nil {
|
||||
return PilotWeeklySnapshot{}, false, err
|
||||
}
|
||||
requestBody, _ := json.Marshal(input)
|
||||
result, err := s.idem.Execute(ctx, idempotency.Request{
|
||||
TenantID: &a.tenantID, ActorKey: "ops:" + a.userPublicID,
|
||||
Scope: "POST /api/v1/ops/diagnostics/pilot-weekly-snapshots", Key: key,
|
||||
RequestHash: idempotency.Hash(requestBody), ExpiresAfter: 24 * time.Hour,
|
||||
}, func(operationContext context.Context, tx pgx.Tx) (idempotency.Result, error) {
|
||||
snapshot, buildErr := buildPilotWeeklySnapshot(operationContext, tx, tenantPublicID, a.tenantID, input, windowStart, windowEnd)
|
||||
if buildErr != nil {
|
||||
return idempotency.Result{}, buildErr
|
||||
}
|
||||
if auditErr := s.audit.Record(operationContext, tx, audit.Event{
|
||||
TenantID: &a.tenantID, ActorType: "user", ActorPublicID: &a.userPublicID,
|
||||
Action: "ops.diagnostics.pilot_weekly_snapshot.export", ResourceType: "pilot_weekly_snapshot", ResourceID: &snapshot.ID,
|
||||
RequestID: requestID, Purpose: input.Reason, Result: "success",
|
||||
Detail: map[string]any{"candidate_version": input.CandidateVersion, "build_digest": input.BuildDigest, "week_start": input.WeekStart, "timezone": input.Timezone, "window_start": windowStart, "window_end": windowEnd, "is_complete": snapshot.IsComplete, "redaction_policy": snapshot.RedactionPolicy},
|
||||
}); auditErr != nil {
|
||||
return idempotency.Result{}, auditErr
|
||||
}
|
||||
encoded, _ := json.Marshal(snapshot)
|
||||
return idempotency.Result{StatusCode: 201, Body: encoded, ResourceID: &snapshot.ID}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return PilotWeeklySnapshot{}, false, err
|
||||
}
|
||||
var snapshot PilotWeeklySnapshot
|
||||
if err = json.Unmarshal(result.Body, &snapshot); err != nil {
|
||||
return PilotWeeklySnapshot{}, false, err
|
||||
}
|
||||
return snapshot, !result.Replayed, nil
|
||||
}
|
||||
|
||||
func normalizePilotWeeklySnapshotInput(input *PilotWeeklySnapshotInput) {
|
||||
input.CandidateVersion = strings.TrimSpace(input.CandidateVersion)
|
||||
input.BuildDigest = strings.ToLower(strings.TrimSpace(input.BuildDigest))
|
||||
input.WeekStart = strings.TrimSpace(input.WeekStart)
|
||||
input.Timezone = strings.TrimSpace(input.Timezone)
|
||||
input.Reason = strings.TrimSpace(input.Reason)
|
||||
}
|
||||
|
||||
func validatePilotWeeklySnapshotInput(input PilotWeeklySnapshotInput, now time.Time) (*time.Location, time.Time, time.Time, error) {
|
||||
if !acceptanceCandidatePattern.MatchString(input.CandidateVersion) || !acceptanceDigestPattern.MatchString(input.BuildDigest) || input.WeekStart == "" || input.Timezone == "" || len([]rune(input.Timezone)) > 80 || input.Reason == "" || len([]rune(input.Reason)) > 255 {
|
||||
return nil, time.Time{}, time.Time{}, ErrInvalid
|
||||
}
|
||||
location, err := time.LoadLocation(input.Timezone)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, time.Time{}, ErrInvalid
|
||||
}
|
||||
weekStart, err := time.ParseInLocation("2006-01-02", input.WeekStart, location)
|
||||
if err != nil || weekStart.Format("2006-01-02") != input.WeekStart || weekStart.Weekday() != time.Monday {
|
||||
return nil, time.Time{}, time.Time{}, ErrInvalid
|
||||
}
|
||||
today := now.In(location)
|
||||
todayStart := time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, location)
|
||||
if weekStart.After(todayStart) {
|
||||
return nil, time.Time{}, time.Time{}, ErrInvalid
|
||||
}
|
||||
return location, weekStart.UTC(), weekStart.AddDate(0, 0, 7).UTC(), nil
|
||||
}
|
||||
|
||||
func buildPilotWeeklySnapshot(ctx context.Context, tx pgx.Tx, tenantPublicID, tenantInternalID string, input PilotWeeklySnapshotInput, windowStart, windowEnd time.Time) (PilotWeeklySnapshot, error) {
|
||||
generatedAt := time.Now().UTC()
|
||||
snapshot := PilotWeeklySnapshot{
|
||||
ID: uuid.NewString(), GeneratedAt: generatedAt, TenantID: tenantPublicID, CandidateVersion: input.CandidateVersion, BuildDigest: input.BuildDigest, WeekStart: input.WeekStart, Timezone: input.Timezone,
|
||||
WindowStart: windowStart, WindowEnd: windowEnd, IsComplete: !generatedAt.Before(windowEnd), RedactionPolicy: "ops-pilot-weekly-minimal-v1",
|
||||
Privacy: PilotWeeklyPrivacyState{ByType: []PilotWeeklyCountByType{}, ByStatus: []PilotWeeklyCountByStatus{}},
|
||||
FirmwareVersions: []PilotWeeklyFirmwareVersion{},
|
||||
ExternalEvidence: []PilotWeeklyExternalEvidence{{Code: "complaints", Status: "REQUIRED", Reason: "当前平台没有权威投诉台账；归档时必须附客服或CRM的脱敏周度投诉证据。"}},
|
||||
Disclaimer: "本周报只汇总平台内可证明的周度活动与生成时刻状态；投诉、真实用户体验、供应商账单和业务结论必须附外部证据，不得由本周报推断。",
|
||||
}
|
||||
|
||||
if err := tx.QueryRow(ctx, `WITH review_actions AS (
|
||||
SELECT resource_id,action FROM audit_event
|
||||
WHERE tenant_id=$1::uuid AND occurred_at >= $2 AND occurred_at < $3
|
||||
AND result='success' AND action IN ('story.revise','sharing.story.edit','story.confirm','story.reject'))
|
||||
SELECT
|
||||
(SELECT count(*) FROM story_draft WHERE tenant_id=$1::uuid AND created_at >= $2 AND created_at < $3),
|
||||
count(DISTINCT resource_id) FILTER(WHERE resource_id IS NOT NULL),
|
||||
count(DISTINCT resource_id) FILTER(WHERE resource_id IS NOT NULL AND action IN ('story.revise','sharing.story.edit')),
|
||||
count(*) FILTER(WHERE action='story.confirm'),
|
||||
count(*) FILTER(WHERE action='story.reject')
|
||||
FROM review_actions`, tenantInternalID, windowStart, windowEnd).Scan(
|
||||
&snapshot.ContentReview.StoriesGenerated, &snapshot.ContentReview.StoriesReviewTouched,
|
||||
&snapshot.ContentReview.StoriesManuallyRevised, &snapshot.ContentReview.StoriesConfirmed, &snapshot.ContentReview.StoriesRejected,
|
||||
); err != nil {
|
||||
return PilotWeeklySnapshot{}, err
|
||||
}
|
||||
if snapshot.ContentReview.StoriesReviewTouched > 0 {
|
||||
snapshot.ContentReview.ManualRevisionRateAvailable = true
|
||||
snapshot.ContentReview.ManualRevisionRateBasisPoints = rateBasisPoints(snapshot.ContentReview.StoriesManuallyRevised, snapshot.ContentReview.StoriesReviewTouched)
|
||||
}
|
||||
decisions := snapshot.ContentReview.StoriesConfirmed + snapshot.ContentReview.StoriesRejected
|
||||
if decisions > 0 {
|
||||
snapshot.ContentReview.StoryConfirmationRateAvailable = true
|
||||
snapshot.ContentReview.StoryConfirmationRateBasisPoints = rateBasisPoints(snapshot.ContentReview.StoriesConfirmed, decisions)
|
||||
}
|
||||
if err := tx.QueryRow(ctx, `SELECT
|
||||
count(*) FILTER(WHERE mv.version_no=1 AND mv.confirmed_at >= $2 AND mv.confirmed_at < $3),
|
||||
count(*) FILTER(WHERE mv.version_no>1 AND mv.confirmed_at >= $2 AND mv.confirmed_at < $3),
|
||||
count(DISTINCT mie.id) FILTER(WHERE mie.status='PENDING' AND cm.status NOT IN ('PENDING_DELETE','DELETED')),
|
||||
count(DISTINCT mie.id) FILTER(WHERE mie.status='FAILED' AND cm.status NOT IN ('PENDING_DELETE','DELETED'))
|
||||
FROM curated_memory cm
|
||||
LEFT JOIN memory_version mv ON mv.memory_id=cm.id
|
||||
LEFT JOIN memory_index_entry mie ON mie.memory_version_id=cm.current_version_id AND mie.index_type='pgvector'
|
||||
WHERE cm.tenant_id=$1::uuid`, tenantInternalID, windowStart, windowEnd).Scan(
|
||||
&snapshot.ContentReview.MemoriesCreated, &snapshot.ContentReview.MemoryRevisions,
|
||||
&snapshot.ContentReview.CurrentMemoryIndexPending, &snapshot.ContentReview.CurrentMemoryIndexFailed,
|
||||
); err != nil {
|
||||
return PilotWeeklySnapshot{}, err
|
||||
}
|
||||
|
||||
privacyTypeCounts := map[string]int64{}
|
||||
privacyStatusCounts := map[string]int64{}
|
||||
rows, err := tx.Query(ctx, `SELECT request_type,status,count(*) FROM privacy_request WHERE tenant_id=$1::uuid AND created_at >= $2 AND created_at < $3 GROUP BY request_type,status`, tenantInternalID, windowStart, windowEnd)
|
||||
if err != nil {
|
||||
return PilotWeeklySnapshot{}, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var requestType, status string
|
||||
var count int64
|
||||
if err = rows.Scan(&requestType, &status, &count); err != nil {
|
||||
rows.Close()
|
||||
return PilotWeeklySnapshot{}, err
|
||||
}
|
||||
privacyTypeCounts[requestType] += count
|
||||
privacyStatusCounts[status] += count
|
||||
snapshot.Privacy.RequestsCreatedDuringWeek += count
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return PilotWeeklySnapshot{}, err
|
||||
}
|
||||
rows.Close()
|
||||
for _, requestType := range pilotPrivacyTypes {
|
||||
snapshot.Privacy.ByType = append(snapshot.Privacy.ByType, PilotWeeklyCountByType{Type: requestType, Count: privacyTypeCounts[requestType]})
|
||||
}
|
||||
for _, status := range pilotPrivacyStatuses {
|
||||
snapshot.Privacy.ByStatus = append(snapshot.Privacy.ByStatus, PilotWeeklyCountByStatus{Status: status, Count: privacyStatusCounts[status]})
|
||||
}
|
||||
if err = tx.QueryRow(ctx, `SELECT count(*) FILTER(WHERE status<>'COMPLETED'),count(*) FILTER(WHERE status='PARTIAL_FAILED') FROM privacy_request WHERE tenant_id=$1::uuid`, tenantInternalID).Scan(&snapshot.Privacy.CurrentOpen, &snapshot.Privacy.CurrentPartialFailed); err != nil {
|
||||
return PilotWeeklySnapshot{}, err
|
||||
}
|
||||
|
||||
rows, err = tx.Query(ctx, `SELECT d.model,COALESCE(dt.firmware_version,'UNREPORTED'),count(*)
|
||||
FROM device_binding db JOIN device d ON d.id=db.device_id AND d.tenant_id=db.tenant_id
|
||||
LEFT JOIN device_twin dt ON dt.device_id=d.id AND dt.tenant_id=d.tenant_id
|
||||
WHERE db.tenant_id=$1::uuid AND db.unbound_at IS NULL AND d.status='registered'
|
||||
GROUP BY d.model,COALESCE(dt.firmware_version,'UNREPORTED') ORDER BY d.model,COALESCE(dt.firmware_version,'UNREPORTED')`, tenantInternalID)
|
||||
if err != nil {
|
||||
return PilotWeeklySnapshot{}, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var item PilotWeeklyFirmwareVersion
|
||||
if err = rows.Scan(&item.Model, &item.FirmwareVersion, &item.DeviceCount); err != nil {
|
||||
rows.Close()
|
||||
return PilotWeeklySnapshot{}, err
|
||||
}
|
||||
snapshot.FirmwareVersions = append(snapshot.FirmwareVersions, item)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return PilotWeeklySnapshot{}, err
|
||||
}
|
||||
rows.Close()
|
||||
if err = tx.QueryRow(ctx, `WITH eligible AS (
|
||||
SELECT dt.desired_revision,dt.reported_revision
|
||||
FROM device_binding db JOIN device d ON d.id=db.device_id AND d.tenant_id=db.tenant_id
|
||||
LEFT JOIN device_twin dt ON dt.device_id=d.id AND dt.tenant_id=d.tenant_id
|
||||
WHERE db.tenant_id=$1::uuid AND db.unbound_at IS NULL AND d.status='registered')
|
||||
SELECT count(*),count(*) FILTER(WHERE reported_revision IS NOT NULL AND reported_revision=desired_revision),
|
||||
count(*) FILTER(WHERE reported_revision IS NOT NULL AND reported_revision<>desired_revision),
|
||||
count(*) FILTER(WHERE reported_revision IS NULL),
|
||||
(SELECT count(*) FROM runtime_config_rollout WHERE tenant_id=$1::uuid AND status='ACTIVE'),
|
||||
(SELECT count(*) FROM runtime_config_rollout WHERE tenant_id=$1::uuid AND status='PAUSED') FROM eligible`, tenantInternalID).Scan(
|
||||
&snapshot.Config.EligibleDevices, &snapshot.Config.Aligned, &snapshot.Config.Drifted, &snapshot.Config.Unreported,
|
||||
&snapshot.Config.ActiveRollouts, &snapshot.Config.PausedRollouts,
|
||||
); err != nil {
|
||||
return PilotWeeklySnapshot{}, err
|
||||
}
|
||||
sort.Slice(snapshot.FirmwareVersions, func(i, j int) bool {
|
||||
if snapshot.FirmwareVersions[i].Model == snapshot.FirmwareVersions[j].Model {
|
||||
return snapshot.FirmwareVersions[i].FirmwareVersion < snapshot.FirmwareVersions[j].FirmwareVersion
|
||||
}
|
||||
return snapshot.FirmwareVersions[i].Model < snapshot.FirmwareVersions[j].Model
|
||||
})
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
func rateBasisPoints(numerator, denominator int64) int64 {
|
||||
if numerator <= 0 || denominator <= 0 {
|
||||
return 0
|
||||
}
|
||||
return (numerator*10000 + denominator/2) / denominator
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestValidatePilotWeeklySnapshotInputUsesSevenCalendarDays(t *testing.T) {
|
||||
now := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC)
|
||||
_, start, end, err := validatePilotWeeklySnapshotInput(PilotWeeklySnapshotInput{
|
||||
CandidateVersion: "0.55.0-rc.test",
|
||||
BuildDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
WeekStart: "2026-03-02",
|
||||
Timezone: "America/New_York",
|
||||
Reason: "Archive weekly pilot evidence",
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := end.Sub(start); got != 167*time.Hour {
|
||||
t.Fatalf("DST weekly duration=%s, want 167h", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatePilotWeeklySnapshotInputRejectsNonMondayAndFutureWeek(t *testing.T) {
|
||||
now := time.Date(2026, 8, 25, 12, 0, 0, 0, time.UTC)
|
||||
for _, input := range []PilotWeeklySnapshotInput{
|
||||
{WeekStart: "2026-08-24", Timezone: "Asia/Shanghai", Reason: "Archive weekly pilot evidence"},
|
||||
{CandidateVersion: "0.55.0-rc.test", BuildDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", WeekStart: "2026-08-25", Timezone: "Asia/Shanghai", Reason: "Archive weekly pilot evidence"},
|
||||
{CandidateVersion: "0.55.0-rc.test", BuildDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", WeekStart: "2026-08-31", Timezone: "Asia/Shanghai", Reason: "Archive weekly pilot evidence"},
|
||||
} {
|
||||
if _, _, _, err := validatePilotWeeklySnapshotInput(input, now); err == nil {
|
||||
t.Fatalf("expected invalid input: %+v", input)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateBasisPointsRoundsToNearestPoint(t *testing.T) {
|
||||
if got := rateBasisPoints(2, 3); got != 6667 {
|
||||
t.Fatalf("rate=%d, want 6667", got)
|
||||
}
|
||||
if got := rateBasisPoints(0, 0); got != 0 {
|
||||
t.Fatalf("empty rate=%d, want 0", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,642 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/audit"
|
||||
"data-terminal-cloud/internal/idempotency"
|
||||
"data-terminal-cloud/internal/identity"
|
||||
"data-terminal-cloud/internal/outbox"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalid = errors.New("diagnostic request is invalid")
|
||||
ErrAccessDenied = errors.New("diagnostic access is denied")
|
||||
ErrNotFound = errors.New("diagnostic resource was not found")
|
||||
ErrStateConflict = errors.New("diagnostic action conflicts with current state")
|
||||
requestIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:/-]{2,127}$`)
|
||||
searchFieldAllowed = map[string]bool{"device_id": true, "conversation_id": true, "room_id": true, "provider_task_ref": true, "request_id": true}
|
||||
retryableJobTypeAllowed = map[string]bool{
|
||||
"platform.noop": true,
|
||||
"conversation.room_issued": true,
|
||||
"conversation.agent_start_requested": true,
|
||||
"conversation.agent_start_reconcile_requested": true,
|
||||
"conversation.agent_stop_requested": true,
|
||||
"conversation.agent_stop_reconcile_requested": true,
|
||||
"conversation.content_extraction_requested": true,
|
||||
"memory.index_requested": true,
|
||||
"memory.delete_requested": true,
|
||||
"privacy.access_requested": true,
|
||||
"privacy.delete_requested": true,
|
||||
"export.generate_requested": true,
|
||||
"media.verify_requested": true,
|
||||
"alert.webhook_test_requested": true,
|
||||
}
|
||||
)
|
||||
|
||||
type Record struct {
|
||||
DeviceID string `json:"device_id"`
|
||||
DeviceStatus string `json:"device_status"`
|
||||
DeviceOnline bool `json:"device_online"`
|
||||
FirmwareVersion string `json:"firmware_version,omitempty"`
|
||||
DesiredConfigRevision int64 `json:"desired_config_revision"`
|
||||
ReportedConfigRevision *int64 `json:"reported_config_revision,omitempty"`
|
||||
LastDeviceErrorCode string `json:"last_device_error_code,omitempty"`
|
||||
LastDeviceErrorAt *time.Time `json:"last_device_error_at,omitempty"`
|
||||
ConversationID string `json:"conversation_id,omitempty"`
|
||||
ConversationStatus string `json:"conversation_status,omitempty"`
|
||||
ConversationFailureCode string `json:"conversation_failure_code,omitempty"`
|
||||
AllocationExpiresAt *time.Time `json:"allocation_expires_at,omitempty"`
|
||||
CredentialIssuedAt *time.Time `json:"credential_issued_at,omitempty"`
|
||||
CredentialExpiresAt *time.Time `json:"credential_expires_at,omitempty"`
|
||||
RoomID string `json:"room_id,omitempty"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
ProviderTaskRef string `json:"provider_task_ref,omitempty"`
|
||||
ProviderTaskStatus string `json:"provider_task_status,omitempty"`
|
||||
ProviderStartAttempts int `json:"provider_start_attempts"`
|
||||
ProviderStopAttempts int `json:"provider_stop_attempts"`
|
||||
ProviderNotificationCount int64 `json:"provider_notification_count"`
|
||||
LastProviderEvent string `json:"last_provider_event,omitempty"`
|
||||
LastProviderEventStatus string `json:"last_provider_event_status,omitempty"`
|
||||
LastProviderEventAt *time.Time `json:"last_provider_event_at,omitempty"`
|
||||
LastRequestID string `json:"last_request_id,omitempty"`
|
||||
Timings *Timings `json:"timings,omitempty"`
|
||||
}
|
||||
|
||||
type Timings struct {
|
||||
ConversationCreatedAt time.Time `json:"conversation_created_at"`
|
||||
RoomIssuedAt *time.Time `json:"room_issued_at,omitempty"`
|
||||
DeviceActivatedAt *time.Time `json:"device_activated_at,omitempty"`
|
||||
AgentActiveAt *time.Time `json:"agent_active_at,omitempty"`
|
||||
StopRequestedAt *time.Time `json:"stop_requested_at,omitempty"`
|
||||
ConversationEndedAt *time.Time `json:"conversation_ended_at,omitempty"`
|
||||
RoomIssueMS *int64 `json:"room_issue_ms,omitempty"`
|
||||
DeviceJoinMS *int64 `json:"device_join_ms,omitempty"`
|
||||
AgentStartMS *int64 `json:"agent_start_ms,omitempty"`
|
||||
AgentStopMS *int64 `json:"agent_stop_ms,omitempty"`
|
||||
TotalMS *int64 `json:"total_ms,omitempty"`
|
||||
}
|
||||
|
||||
type ActionResult struct {
|
||||
Action string `json:"action"`
|
||||
DeviceID string `json:"device_id"`
|
||||
ConversationID string `json:"conversation_id,omitempty"`
|
||||
ConversationStatus string `json:"conversation_status,omitempty"`
|
||||
CommandID string `json:"command_id,omitempty"`
|
||||
CommandStatus string `json:"command_status,omitempty"`
|
||||
JobID string `json:"job_id,omitempty"`
|
||||
JobType string `json:"job_type,omitempty"`
|
||||
ManualRetryCount int `json:"manual_retry_count,omitempty"`
|
||||
Queued bool `json:"queued"`
|
||||
}
|
||||
|
||||
type Job struct {
|
||||
ID string `json:"id"`
|
||||
EventType string `json:"event_type"`
|
||||
AggregateType string `json:"aggregate_type"`
|
||||
Status string `json:"status"`
|
||||
AttemptCount int `json:"attempt_count"`
|
||||
ManualRetryCount int `json:"manual_retry_count"`
|
||||
LastErrorCode string `json:"last_error_code,omitempty"`
|
||||
LastManualRetryAt *time.Time `json:"last_manual_retry_at,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type Bundle struct {
|
||||
ID string `json:"id"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
SearchField string `json:"search_field"`
|
||||
SearchValue string `json:"search_value"`
|
||||
RedactionPolicy string `json:"redaction_policy"`
|
||||
Items []Record `json:"items"`
|
||||
}
|
||||
|
||||
type ExternalDependency struct {
|
||||
ServiceName string `json:"service_name"`
|
||||
Dependency string `json:"dependency"`
|
||||
Status string `json:"status"`
|
||||
CheckedAt time.Time `json:"checked_at"`
|
||||
LastSuccessAt *time.Time `json:"last_success_at,omitempty"`
|
||||
LastFailureAt *time.Time `json:"last_failure_at,omitempty"`
|
||||
LastErrorCode string `json:"last_error_code,omitempty"`
|
||||
LatencyMS int64 `json:"latency_ms"`
|
||||
ProbeIntervalMS int64 `json:"probe_interval_ms"`
|
||||
ConsecutiveFailures int `json:"consecutive_failures"`
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
pool *pgxpool.Pool
|
||||
idem *idempotency.Executor
|
||||
audit *audit.Repository
|
||||
outbox *outbox.Store
|
||||
}
|
||||
|
||||
func NewService(pool *pgxpool.Pool, idem *idempotency.Executor, auditRepository *audit.Repository) (*Service, error) {
|
||||
if pool == nil || idem == nil || auditRepository == nil {
|
||||
return nil, errors.New("diagnostic service dependencies are required")
|
||||
}
|
||||
return &Service{pool: pool, idem: idem, audit: auditRepository, outbox: outbox.NewStore(pool)}, nil
|
||||
}
|
||||
|
||||
type actor struct{ tenantID, userID, userPublicID string }
|
||||
|
||||
func (s *Service) resolveActor(ctx context.Context, principal identity.Principal, tenantPublicID string, roles ...string) (actor, error) {
|
||||
if _, err := uuid.Parse(tenantPublicID); err != nil {
|
||||
return actor{}, ErrInvalid
|
||||
}
|
||||
var result actor
|
||||
var role string
|
||||
err := s.pool.QueryRow(ctx, `SELECT t.id::text,ua.id::text,ua.public_id::text,ua.platform_role FROM tenant t JOIN user_account ua ON ua.tenant_id=t.id AND ua.status='active' WHERE t.public_id=$1::uuid AND ua.oidc_issuer=$2 AND ua.oidc_subject=$3`, tenantPublicID, principal.Issuer, principal.Subject).Scan(&result.tenantID, &result.userID, &result.userPublicID, &role)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return actor{}, ErrAccessDenied
|
||||
}
|
||||
if err != nil {
|
||||
return actor{}, err
|
||||
}
|
||||
for _, allowed := range roles {
|
||||
if role == allowed {
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
return actor{}, ErrAccessDenied
|
||||
}
|
||||
|
||||
func (s *Service) Search(ctx context.Context, principal identity.Principal, tenantPublicID, field, value string) ([]Record, error) {
|
||||
field, value = strings.TrimSpace(field), strings.TrimSpace(value)
|
||||
if !searchFieldAllowed[field] || value == "" || len(value) > 200 {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
if field == "device_id" || field == "conversation_id" {
|
||||
if _, err := uuid.Parse(value); err != nil {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
} else if !requestIDPattern.MatchString(value) {
|
||||
return nil, ErrInvalid
|
||||
}
|
||||
currentActor, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_viewer", "ops_operator", "ops_approver", "ops_admin")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
condition := map[string]string{
|
||||
"device_id": "d.public_id=$2::uuid",
|
||||
"conversation_id": "c.public_id=$2::uuid",
|
||||
"room_id": "rt.room_id=$2",
|
||||
"provider_task_ref": "rt.provider_task_ref=$2",
|
||||
"request_id": "EXISTS(SELECT 1 FROM audit_event ae WHERE ae.tenant_id=c.tenant_id AND ae.request_id=$2 AND ae.resource_id IN(c.id,c.public_id,d.id,d.public_id))",
|
||||
}[field]
|
||||
query := diagnosticSelect + " WHERE d.tenant_id=$1::uuid AND " + condition + " ORDER BY c.created_at DESC NULLS LAST LIMIT 50"
|
||||
rows, err := s.pool.Query(ctx, query, currentActor.tenantID, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []Record{}
|
||||
for rows.Next() {
|
||||
item, scanErr := scanRecord(rows)
|
||||
if scanErr != nil {
|
||||
return nil, scanErr
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Service) ListExternalDependencies(ctx context.Context, principal identity.Principal, tenantPublicID string) ([]ExternalDependency, error) {
|
||||
if _, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_viewer", "ops_operator", "ops_approver", "ops_admin"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `SELECT service_name,dependency,status,checked_at,last_success_at,last_failure_at,COALESCE(last_error_code,''),latency_ms,probe_interval_ms,consecutive_failures FROM external_dependency_probe_state ORDER BY service_name,dependency`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []ExternalDependency{}
|
||||
for rows.Next() {
|
||||
var item ExternalDependency
|
||||
if err = rows.Scan(&item.ServiceName, &item.Dependency, &item.Status, &item.CheckedAt, &item.LastSuccessAt, &item.LastFailureAt, &item.LastErrorCode, &item.LatencyMS, &item.ProbeIntervalMS, &item.ConsecutiveFailures); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Service) StopConversation(ctx context.Context, principal identity.Principal, tenantPublicID, conversationPublicID, key, requestID, reason string) (ActionResult, bool, error) {
|
||||
reason = strings.TrimSpace(reason)
|
||||
if _, err := uuid.Parse(conversationPublicID); err != nil || reason == "" || len([]rune(reason)) > 500 {
|
||||
return ActionResult{}, false, ErrInvalid
|
||||
}
|
||||
a, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_operator", "ops_admin")
|
||||
if err != nil {
|
||||
return ActionResult{}, false, err
|
||||
}
|
||||
body, _ := json.Marshal(map[string]string{"conversation_id": conversationPublicID, "reason": reason})
|
||||
result, err := s.idem.Execute(ctx, idempotency.Request{TenantID: &a.tenantID, ActorKey: "ops:" + a.userPublicID, Scope: "POST /api/v1/ops/diagnostics/conversations/stop", Key: key, RequestHash: idempotency.Hash(body), ExpiresAfter: 24 * time.Hour}, func(operationContext context.Context, tx pgx.Tx) (idempotency.Result, error) {
|
||||
var internalID, deviceID, status, rtcStatus string
|
||||
err := tx.QueryRow(operationContext, `SELECT c.id::text,d.public_id::text,c.status,rt.status FROM conversation c JOIN device d ON d.id=c.device_id JOIN rtc_task rt ON rt.conversation_id=c.id WHERE c.tenant_id=$1::uuid AND c.public_id=$2::uuid FOR UPDATE OF c,rt`, a.tenantID, conversationPublicID).Scan(&internalID, &deviceID, &status, &rtcStatus)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return idempotency.Result{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return idempotency.Result{}, err
|
||||
}
|
||||
queued := false
|
||||
fromStatus := status
|
||||
switch status {
|
||||
case "COMPLETED", "FAILED", "EXPIRED", "STOPPING":
|
||||
case "PREPARING", "ROOM_ISSUED":
|
||||
status = "EXPIRED"
|
||||
if _, err = tx.Exec(operationContext, `UPDATE conversation SET status='EXPIRED',stop_reason='ops_stop_before_start',ended_at=now(),lease_expires_at=NULL,version=version+1,updated_at=now() WHERE id=$1::uuid`, internalID); err == nil {
|
||||
_, err = tx.Exec(operationContext, `UPDATE rtc_task SET status='COMPLETED',lease_expires_at=NULL,updated_at=now() WHERE conversation_id=$1::uuid`, internalID)
|
||||
}
|
||||
case "STARTING_AGENT", "ACTIVE":
|
||||
status, queued = "STOPPING", true
|
||||
if _, err = tx.Exec(operationContext, `UPDATE conversation SET status='STOPPING',stop_reason='ops_stop',lease_expires_at=NULL,version=version+1,updated_at=now() WHERE id=$1::uuid`, internalID); err == nil {
|
||||
_, err = tx.Exec(operationContext, `UPDATE rtc_task SET status=CASE WHEN status='ACTIVE' THEN 'STOPPING' ELSE 'UNKNOWN' END,lease_expires_at=NULL,updated_at=now() WHERE conversation_id=$1::uuid`, internalID)
|
||||
}
|
||||
if err == nil {
|
||||
dedupe := "ops-stop:" + internalID
|
||||
payload, _ := json.Marshal(map[string]string{"conversation_id": conversationPublicID, "reason": "ops_stop"})
|
||||
_, err = s.outbox.Enqueue(operationContext, tx, outbox.NewEvent{TenantID: &a.tenantID, AggregateType: "conversation", AggregateID: internalID, EventType: "conversation.agent_stop_requested", DeduplicationKey: &dedupe, Payload: payload})
|
||||
}
|
||||
default:
|
||||
return idempotency.Result{}, ErrStateConflict
|
||||
}
|
||||
if err != nil {
|
||||
return idempotency.Result{}, err
|
||||
}
|
||||
if status != fromStatus {
|
||||
if _, err = tx.Exec(operationContext, `INSERT INTO conversation_state_history(conversation_id,from_status,to_status,reason,source) VALUES($1::uuid,$2,$3,$4,'operator')`, internalID, fromStatus, status, "ops diagnostics: "+reason); err != nil {
|
||||
return idempotency.Result{}, err
|
||||
}
|
||||
}
|
||||
if err = s.audit.Record(operationContext, tx, audit.Event{TenantID: &a.tenantID, ActorType: "user", ActorPublicID: &a.userPublicID, Action: "ops.diagnostics.conversation.stop", ResourceType: "conversation", ResourceID: &conversationPublicID, RequestID: requestID, Purpose: reason, Result: "success", Detail: map[string]any{"from_rtc_status": rtcStatus, "to_status": status}}); err != nil {
|
||||
return idempotency.Result{}, err
|
||||
}
|
||||
output := ActionResult{Action: "stop_conversation", DeviceID: deviceID, ConversationID: conversationPublicID, ConversationStatus: status, Queued: queued}
|
||||
encoded, _ := json.Marshal(output)
|
||||
return idempotency.Result{StatusCode: 200, Body: encoded, ResourceID: &internalID}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return ActionResult{}, false, err
|
||||
}
|
||||
var output ActionResult
|
||||
if err = json.Unmarshal(result.Body, &output); err != nil {
|
||||
return ActionResult{}, false, err
|
||||
}
|
||||
return output, !result.Replayed, nil
|
||||
}
|
||||
|
||||
func (s *Service) ListDeadJobs(ctx context.Context, principal identity.Principal, tenantPublicID string) ([]Job, error) {
|
||||
a, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_viewer", "ops_operator", "ops_approver", "ops_admin")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `SELECT public_id::text,event_type,aggregate_type,status,attempt_count,manual_retry_count,COALESCE(last_error_code,''),last_manual_retry_at,created_at FROM outbox_event WHERE tenant_id=$1::uuid AND status='dead' ORDER BY created_at DESC LIMIT 100`, a.tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []Job{}
|
||||
for rows.Next() {
|
||||
var item Job
|
||||
if err := rows.Scan(&item.ID, &item.EventType, &item.AggregateType, &item.Status, &item.AttemptCount, &item.ManualRetryCount, &item.LastErrorCode, &item.LastManualRetryAt, &item.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Service) RetryDeadJob(ctx context.Context, principal identity.Principal, tenantPublicID, jobPublicID, key, requestID, reason string) (ActionResult, bool, error) {
|
||||
reason = strings.TrimSpace(reason)
|
||||
if _, err := uuid.Parse(jobPublicID); err != nil || reason == "" || len([]rune(reason)) > 500 {
|
||||
return ActionResult{}, false, ErrInvalid
|
||||
}
|
||||
a, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_operator", "ops_admin")
|
||||
if err != nil {
|
||||
return ActionResult{}, false, err
|
||||
}
|
||||
body, _ := json.Marshal(map[string]string{"job_id": jobPublicID, "reason": reason})
|
||||
result, err := s.idem.Execute(ctx, idempotency.Request{TenantID: &a.tenantID, ActorKey: "ops:" + a.userPublicID, Scope: "POST /api/v1/ops/diagnostics/jobs/retry", Key: key, RequestHash: idempotency.Hash(body), ExpiresAfter: 24 * time.Hour}, func(op context.Context, tx pgx.Tx) (idempotency.Result, error) {
|
||||
var internalID, eventType, status string
|
||||
var manualRetries int
|
||||
if queryErr := tx.QueryRow(op, `SELECT id::text,event_type,status,manual_retry_count FROM outbox_event WHERE tenant_id=$1::uuid AND public_id=$2::uuid FOR UPDATE`, a.tenantID, jobPublicID).Scan(&internalID, &eventType, &status, &manualRetries); errors.Is(queryErr, pgx.ErrNoRows) {
|
||||
return idempotency.Result{}, ErrNotFound
|
||||
} else if queryErr != nil {
|
||||
return idempotency.Result{}, queryErr
|
||||
}
|
||||
if status != "dead" {
|
||||
return idempotency.Result{}, ErrStateConflict
|
||||
}
|
||||
if !retryableJobTypeAllowed[eventType] {
|
||||
return idempotency.Result{}, ErrStateConflict
|
||||
}
|
||||
manualRetries++
|
||||
if _, updateErr := tx.Exec(op, `UPDATE outbox_event SET status='pending',attempt_count=0,available_at=now(),locked_at=NULL,locked_by=NULL,dead_at=NULL,last_error_code=NULL,manual_retry_count=$2,last_manual_retry_at=now(),last_manual_retry_by=$3::uuid,updated_at=now() WHERE id=$1::uuid`, internalID, manualRetries, a.userID); updateErr != nil {
|
||||
return idempotency.Result{}, updateErr
|
||||
}
|
||||
if auditErr := s.audit.Record(op, tx, audit.Event{TenantID: &a.tenantID, ActorType: "user", ActorPublicID: &a.userPublicID, Action: "ops.diagnostics.job.retry", ResourceType: "outbox_job", ResourceID: &jobPublicID, RequestID: requestID, Purpose: reason, Result: "success", Detail: map[string]any{"event_type": eventType, "from_status": status, "to_status": "pending", "manual_retry_count": manualRetries}}); auditErr != nil {
|
||||
return idempotency.Result{}, auditErr
|
||||
}
|
||||
output := ActionResult{Action: "retry_job", JobID: jobPublicID, JobType: eventType, ManualRetryCount: manualRetries, Queued: true}
|
||||
encoded, _ := json.Marshal(output)
|
||||
return idempotency.Result{StatusCode: 200, Body: encoded, ResourceID: &internalID}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return ActionResult{}, false, err
|
||||
}
|
||||
var output ActionResult
|
||||
if err = json.Unmarshal(result.Body, &output); err != nil {
|
||||
return ActionResult{}, false, err
|
||||
}
|
||||
return output, !result.Replayed, nil
|
||||
}
|
||||
|
||||
func (s *Service) RevokeDevice(ctx context.Context, principal identity.Principal, tenantPublicID, devicePublicID, key, requestID, reason string) (ActionResult, bool, error) {
|
||||
reason = strings.TrimSpace(reason)
|
||||
if _, err := uuid.Parse(devicePublicID); err != nil || reason == "" || len([]rune(reason)) > 500 {
|
||||
return ActionResult{}, false, ErrInvalid
|
||||
}
|
||||
a, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_admin")
|
||||
if err != nil {
|
||||
return ActionResult{}, false, err
|
||||
}
|
||||
body, _ := json.Marshal(map[string]string{"device_id": devicePublicID, "reason": reason})
|
||||
result, err := s.idem.Execute(ctx, idempotency.Request{TenantID: &a.tenantID, ActorKey: "ops:" + a.userPublicID, Scope: "POST /api/v1/ops/diagnostics/devices/revoke", Key: key, RequestHash: idempotency.Hash(body), ExpiresAfter: 24 * time.Hour}, func(op context.Context, tx pgx.Tx) (idempotency.Result, error) {
|
||||
var deviceID, oldStatus string
|
||||
if queryErr := tx.QueryRow(op, `SELECT id::text,status FROM device WHERE tenant_id=$1::uuid AND public_id=$2::uuid FOR UPDATE`, a.tenantID, devicePublicID).Scan(&deviceID, &oldStatus); errors.Is(queryErr, pgx.ErrNoRows) {
|
||||
return idempotency.Result{}, ErrNotFound
|
||||
} else if queryErr != nil {
|
||||
return idempotency.Result{}, queryErr
|
||||
}
|
||||
if oldStatus != "revoked" {
|
||||
if _, updateErr := tx.Exec(op, `UPDATE device SET status='revoked',revoked_at=COALESCE(revoked_at,now()),version=version+1,updated_at=now() WHERE id=$1::uuid`, deviceID); updateErr != nil {
|
||||
return idempotency.Result{}, updateErr
|
||||
}
|
||||
}
|
||||
|
||||
var conversationInternalID, conversationPublicID, conversationStatus, rtcStatus string
|
||||
queued := false
|
||||
conversationQueryErr := tx.QueryRow(op, `SELECT c.id::text,c.public_id::text,c.status,rt.status FROM conversation c JOIN rtc_task rt ON rt.conversation_id=c.id WHERE c.tenant_id=$1::uuid AND c.device_id=$2::uuid AND c.status NOT IN('COMPLETED','FAILED','EXPIRED') ORDER BY c.created_at DESC LIMIT 1 FOR UPDATE OF c,rt`, a.tenantID, deviceID).Scan(&conversationInternalID, &conversationPublicID, &conversationStatus, &rtcStatus)
|
||||
if conversationQueryErr != nil && !errors.Is(conversationQueryErr, pgx.ErrNoRows) {
|
||||
return idempotency.Result{}, conversationQueryErr
|
||||
}
|
||||
if conversationQueryErr == nil {
|
||||
fromStatus := conversationStatus
|
||||
switch conversationStatus {
|
||||
case "PREPARING", "ROOM_ISSUED":
|
||||
conversationStatus = "EXPIRED"
|
||||
if _, updateErr := tx.Exec(op, `UPDATE conversation SET status='EXPIRED',stop_reason='device_revoked_before_start',ended_at=now(),lease_expires_at=NULL,version=version+1,updated_at=now() WHERE id=$1::uuid`, conversationInternalID); updateErr != nil {
|
||||
return idempotency.Result{}, updateErr
|
||||
}
|
||||
if _, updateErr := tx.Exec(op, `UPDATE rtc_task SET status='COMPLETED',lease_expires_at=NULL,updated_at=now() WHERE conversation_id=$1::uuid`, conversationInternalID); updateErr != nil {
|
||||
return idempotency.Result{}, updateErr
|
||||
}
|
||||
case "STARTING_AGENT", "ACTIVE":
|
||||
conversationStatus, queued = "STOPPING", true
|
||||
if _, updateErr := tx.Exec(op, `UPDATE conversation SET status='STOPPING',stop_reason='device_revoked',lease_expires_at=NULL,version=version+1,updated_at=now() WHERE id=$1::uuid`, conversationInternalID); updateErr != nil {
|
||||
return idempotency.Result{}, updateErr
|
||||
}
|
||||
if _, updateErr := tx.Exec(op, `UPDATE rtc_task SET status=CASE WHEN status='ACTIVE' THEN 'STOPPING' ELSE 'UNKNOWN' END,lease_expires_at=NULL,updated_at=now() WHERE conversation_id=$1::uuid`, conversationInternalID); updateErr != nil {
|
||||
return idempotency.Result{}, updateErr
|
||||
}
|
||||
dedupe := "ops-stop:" + conversationInternalID
|
||||
payload, _ := json.Marshal(map[string]string{"conversation_id": conversationPublicID, "reason": "device_revoked"})
|
||||
if _, enqueueErr := s.outbox.Enqueue(op, tx, outbox.NewEvent{TenantID: &a.tenantID, AggregateType: "conversation", AggregateID: conversationInternalID, EventType: "conversation.agent_stop_requested", DeduplicationKey: &dedupe, Payload: payload}); enqueueErr != nil {
|
||||
return idempotency.Result{}, enqueueErr
|
||||
}
|
||||
case "STOPPING":
|
||||
default:
|
||||
return idempotency.Result{}, ErrStateConflict
|
||||
}
|
||||
if conversationStatus != fromStatus {
|
||||
if _, historyErr := tx.Exec(op, `INSERT INTO conversation_state_history(conversation_id,from_status,to_status,reason,source) VALUES($1::uuid,$2,$3,$4,'operator')`, conversationInternalID, fromStatus, conversationStatus, "device revoked: "+reason); historyErr != nil {
|
||||
return idempotency.Result{}, historyErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
detail := map[string]any{"from_status": oldStatus, "to_status": "revoked", "conversation_queued": queued}
|
||||
if conversationPublicID != "" {
|
||||
detail["conversation_id"] = conversationPublicID
|
||||
detail["conversation_status"] = conversationStatus
|
||||
detail["from_rtc_status"] = rtcStatus
|
||||
}
|
||||
if auditErr := s.audit.Record(op, tx, audit.Event{TenantID: &a.tenantID, ActorType: "user", ActorPublicID: &a.userPublicID, Action: "ops.diagnostics.device.revoke", ResourceType: "device", ResourceID: &devicePublicID, RequestID: requestID, Purpose: reason, Result: "success", Detail: detail}); auditErr != nil {
|
||||
return idempotency.Result{}, auditErr
|
||||
}
|
||||
output := ActionResult{Action: "revoke_device", DeviceID: devicePublicID, ConversationID: conversationPublicID, ConversationStatus: conversationStatus, Queued: queued}
|
||||
encoded, _ := json.Marshal(output)
|
||||
return idempotency.Result{StatusCode: 200, Body: encoded, ResourceID: &deviceID}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return ActionResult{}, false, err
|
||||
}
|
||||
var output ActionResult
|
||||
if err = json.Unmarshal(result.Body, &output); err != nil {
|
||||
return ActionResult{}, false, err
|
||||
}
|
||||
return output, !result.Replayed, nil
|
||||
}
|
||||
|
||||
func (s *Service) ExportBundle(ctx context.Context, principal identity.Principal, tenantPublicID, field, value, key, requestID, reason string) (Bundle, bool, error) {
|
||||
reason = strings.TrimSpace(reason)
|
||||
if reason == "" || len([]rune(reason)) > 500 {
|
||||
return Bundle{}, false, ErrInvalid
|
||||
}
|
||||
a, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_operator", "ops_admin")
|
||||
if err != nil {
|
||||
return Bundle{}, false, err
|
||||
}
|
||||
items, err := s.Search(ctx, principal, tenantPublicID, field, value)
|
||||
if err != nil {
|
||||
return Bundle{}, false, err
|
||||
}
|
||||
body, _ := json.Marshal(map[string]string{"field": field, "value": value, "reason": reason})
|
||||
result, err := s.idem.Execute(ctx, idempotency.Request{TenantID: &a.tenantID, ActorKey: "ops:" + a.userPublicID, Scope: "POST /api/v1/ops/diagnostics/bundles", Key: key, RequestHash: idempotency.Hash(body), ExpiresAfter: 24 * time.Hour}, func(op context.Context, tx pgx.Tx) (idempotency.Result, error) {
|
||||
bundle := Bundle{ID: uuid.NewString(), GeneratedAt: time.Now().UTC(), SearchField: field, SearchValue: value, RedactionPolicy: "ops-diagnostic-minimal-v1", Items: items}
|
||||
if auditErr := s.audit.Record(op, tx, audit.Event{TenantID: &a.tenantID, ActorType: "user", ActorPublicID: &a.userPublicID, Action: "ops.diagnostics.bundle.export", ResourceType: "diagnostic_bundle", ResourceID: &bundle.ID, RequestID: requestID, Purpose: reason, Result: "success", Detail: map[string]any{"search_field": field, "item_count": len(items), "redaction_policy": bundle.RedactionPolicy}}); auditErr != nil {
|
||||
return idempotency.Result{}, auditErr
|
||||
}
|
||||
encoded, _ := json.Marshal(bundle)
|
||||
return idempotency.Result{StatusCode: 201, Body: encoded, ResourceID: &bundle.ID}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return Bundle{}, false, err
|
||||
}
|
||||
var bundle Bundle
|
||||
if err = json.Unmarshal(result.Body, &bundle); err != nil {
|
||||
return Bundle{}, false, err
|
||||
}
|
||||
return bundle, !result.Replayed, nil
|
||||
}
|
||||
|
||||
func (s *Service) ReconcileConversation(ctx context.Context, principal identity.Principal, tenantPublicID, conversationPublicID, key, requestID, reason string) (ActionResult, bool, error) {
|
||||
reason = strings.TrimSpace(reason)
|
||||
if _, err := uuid.Parse(conversationPublicID); err != nil || reason == "" || len([]rune(reason)) > 500 {
|
||||
return ActionResult{}, false, ErrInvalid
|
||||
}
|
||||
a, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_operator", "ops_admin")
|
||||
if err != nil {
|
||||
return ActionResult{}, false, err
|
||||
}
|
||||
body, _ := json.Marshal(map[string]string{"conversation_id": conversationPublicID, "reason": reason})
|
||||
result, err := s.idem.Execute(ctx, idempotency.Request{TenantID: &a.tenantID, ActorKey: "ops:" + a.userPublicID, Scope: "POST /api/v1/ops/diagnostics/conversations/reconcile", Key: key, RequestHash: idempotency.Hash(body), ExpiresAfter: 24 * time.Hour}, func(operationContext context.Context, tx pgx.Tx) (idempotency.Result, error) {
|
||||
var internalID, deviceID, status, taskRef string
|
||||
if queryErr := tx.QueryRow(operationContext, `SELECT c.id::text,d.public_id::text,c.status,rt.provider_task_ref FROM conversation c JOIN device d ON d.id=c.device_id JOIN rtc_task rt ON rt.conversation_id=c.id WHERE c.tenant_id=$1::uuid AND c.public_id=$2::uuid FOR UPDATE OF c`, a.tenantID, conversationPublicID).Scan(&internalID, &deviceID, &status, &taskRef); errors.Is(queryErr, pgx.ErrNoRows) {
|
||||
return idempotency.Result{}, ErrNotFound
|
||||
} else if queryErr != nil {
|
||||
return idempotency.Result{}, queryErr
|
||||
}
|
||||
eventType := ""
|
||||
if status == "STARTING_AGENT" {
|
||||
eventType = "conversation.agent_start_reconcile_requested"
|
||||
} else if status == "STOPPING" {
|
||||
eventType = "conversation.agent_stop_reconcile_requested"
|
||||
}
|
||||
queued := eventType != ""
|
||||
if queued {
|
||||
dedupe := "ops-reconcile:" + taskRef + ":" + key
|
||||
payload, _ := json.Marshal(map[string]string{"conversation_id": conversationPublicID, "provider_task_ref": taskRef, "reason": "ops_reconcile"})
|
||||
if _, enqueueErr := s.outbox.Enqueue(operationContext, tx, outbox.NewEvent{TenantID: &a.tenantID, AggregateType: "conversation", AggregateID: internalID, EventType: eventType, DeduplicationKey: &dedupe, Payload: payload}); enqueueErr != nil {
|
||||
return idempotency.Result{}, enqueueErr
|
||||
}
|
||||
}
|
||||
if auditErr := s.audit.Record(operationContext, tx, audit.Event{TenantID: &a.tenantID, ActorType: "user", ActorPublicID: &a.userPublicID, Action: "ops.diagnostics.conversation.reconcile", ResourceType: "conversation", ResourceID: &conversationPublicID, RequestID: requestID, Purpose: reason, Result: "success", Detail: map[string]any{"status": status, "queued": queued}}); auditErr != nil {
|
||||
return idempotency.Result{}, auditErr
|
||||
}
|
||||
output := ActionResult{Action: "reconcile_conversation", DeviceID: deviceID, ConversationID: conversationPublicID, ConversationStatus: status, Queued: queued}
|
||||
encoded, _ := json.Marshal(output)
|
||||
return idempotency.Result{StatusCode: 200, Body: encoded, ResourceID: &internalID}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return ActionResult{}, false, err
|
||||
}
|
||||
var output ActionResult
|
||||
if err = json.Unmarshal(result.Body, &output); err != nil {
|
||||
return ActionResult{}, false, err
|
||||
}
|
||||
return output, !result.Replayed, nil
|
||||
}
|
||||
|
||||
func (s *Service) RefreshDeviceConfig(ctx context.Context, principal identity.Principal, tenantPublicID, devicePublicID, key, requestID, reason string) (ActionResult, bool, error) {
|
||||
reason = strings.TrimSpace(reason)
|
||||
if _, err := uuid.Parse(devicePublicID); err != nil || reason == "" || len([]rune(reason)) > 500 {
|
||||
return ActionResult{}, false, ErrInvalid
|
||||
}
|
||||
a, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_operator", "ops_admin")
|
||||
if err != nil {
|
||||
return ActionResult{}, false, err
|
||||
}
|
||||
body, _ := json.Marshal(map[string]string{"device_id": devicePublicID, "reason": reason})
|
||||
result, err := s.idem.Execute(ctx, idempotency.Request{TenantID: &a.tenantID, ActorKey: "ops:" + a.userPublicID, Scope: "POST /api/v1/ops/diagnostics/devices/config-refresh", Key: key, RequestHash: idempotency.Hash(body), ExpiresAfter: 24 * time.Hour}, func(operationContext context.Context, tx pgx.Tx) (idempotency.Result, error) {
|
||||
var deviceID string
|
||||
var revision int64
|
||||
if queryErr := tx.QueryRow(operationContext, `SELECT d.id::text,dt.desired_revision FROM device d JOIN device_twin dt ON dt.device_id=d.id AND dt.tenant_id=d.tenant_id WHERE d.tenant_id=$1::uuid AND d.public_id=$2::uuid AND d.status='registered' FOR SHARE OF d`, a.tenantID, devicePublicID).Scan(&deviceID, &revision); errors.Is(queryErr, pgx.ErrNoRows) {
|
||||
return idempotency.Result{}, ErrNotFound
|
||||
} else if queryErr != nil {
|
||||
return idempotency.Result{}, queryErr
|
||||
}
|
||||
commandPublicID := uuid.NewString()
|
||||
commandInternalID := uuid.NewString()
|
||||
payload, _ := json.Marshal(map[string]any{"desired_revision": revision, "reason_code": "ops_config_refresh"})
|
||||
digest := idempotency.Hash(payload)
|
||||
_, insertErr := tx.Exec(operationContext, `INSERT INTO device_command(id,public_id,tenant_id,device_id,command_type,payload,status,requested_by,idempotency_key,request_digest,expires_at) VALUES($1::uuid,$2::uuid,$3::uuid,$4::uuid,'apply_config',$5::jsonb,'QUEUED',$6::uuid,$7,$8,now()+interval '1 hour')`, commandInternalID, commandPublicID, a.tenantID, deviceID, payload, a.userID, "ops-refresh:"+key, digest)
|
||||
if insertErr != nil {
|
||||
return idempotency.Result{}, insertErr
|
||||
}
|
||||
if auditErr := s.audit.Record(operationContext, tx, audit.Event{TenantID: &a.tenantID, ActorType: "user", ActorPublicID: &a.userPublicID, Action: "ops.diagnostics.device.config_refresh", ResourceType: "device", ResourceID: &devicePublicID, RequestID: requestID, Purpose: reason, Result: "success", Detail: map[string]any{"desired_revision": revision, "command_id": commandPublicID}}); auditErr != nil {
|
||||
return idempotency.Result{}, auditErr
|
||||
}
|
||||
output := ActionResult{Action: "refresh_config", DeviceID: devicePublicID, CommandID: commandPublicID, CommandStatus: "QUEUED", Queued: true}
|
||||
encoded, _ := json.Marshal(output)
|
||||
return idempotency.Result{StatusCode: 201, Body: encoded, ResourceID: &commandInternalID}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return ActionResult{}, false, err
|
||||
}
|
||||
var output ActionResult
|
||||
if err = json.Unmarshal(result.Body, &output); err != nil {
|
||||
return ActionResult{}, false, err
|
||||
}
|
||||
return output, !result.Replayed, nil
|
||||
}
|
||||
|
||||
const diagnosticSelect = `SELECT
|
||||
d.public_id::text,d.status,COALESCE(dp.online,false),COALESCE(dt.firmware_version,''),
|
||||
COALESCE(dt.desired_revision,1),dt.reported_revision,COALESCE(dt.last_error_code,''),dt.last_error_at,
|
||||
COALESCE(c.public_id::text,''),COALESCE(c.status,''),COALESCE(c.failure_code,''),c.allocation_expires_at,
|
||||
rt.credential_issued_at,rt.credential_expires_at,
|
||||
COALESCE(rt.room_id,''),COALESCE(rt.provider,''),COALESCE(rt.provider_task_ref,''),COALESCE(rt.status,''),
|
||||
COALESCE(rt.start_attempts,0),COALESCE(rt.stop_attempts,0),COALESCE(pe.event_count,0),
|
||||
COALESCE(pe.event_type,''),COALESCE(pe.event_status,''),pe.received_at,COALESCE(ae.request_id,''),
|
||||
c.created_at,ch.room_issued_at,ch.device_activated_at,ch.agent_active_at,ch.stop_requested_at,c.ended_at
|
||||
FROM device d
|
||||
LEFT JOIN device_twin dt ON dt.device_id=d.id AND dt.tenant_id=d.tenant_id
|
||||
LEFT JOIN device_presence_state dp ON dp.device_id=d.id AND dp.tenant_id=d.tenant_id
|
||||
LEFT JOIN conversation c ON c.device_id=d.id AND c.tenant_id=d.tenant_id
|
||||
LEFT JOIN rtc_task rt ON rt.conversation_id=c.id
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT count(*) OVER() event_count,event_type,COALESCE(sanitized_payload->>'status','') event_status,received_at
|
||||
FROM provider_event WHERE conversation_id=c.id ORDER BY received_at DESC LIMIT 1
|
||||
) pe ON true
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT
|
||||
MIN(occurred_at) FILTER (WHERE to_status='ROOM_ISSUED') room_issued_at,
|
||||
MIN(occurred_at) FILTER (WHERE to_status='STARTING_AGENT') device_activated_at,
|
||||
MIN(occurred_at) FILTER (WHERE to_status='ACTIVE') agent_active_at,
|
||||
MIN(occurred_at) FILTER (WHERE to_status='STOPPING') stop_requested_at
|
||||
FROM conversation_state_history WHERE conversation_id=c.id
|
||||
) ch ON c.id IS NOT NULL
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT request_id FROM audit_event
|
||||
WHERE tenant_id=d.tenant_id AND resource_id IN(d.id,d.public_id,c.id,c.public_id) AND request_id IS NOT NULL
|
||||
ORDER BY occurred_at DESC LIMIT 1
|
||||
) ae ON true`
|
||||
|
||||
type rowScanner interface{ Scan(...any) error }
|
||||
|
||||
func scanRecord(row rowScanner) (Record, error) {
|
||||
var item Record
|
||||
var createdAt, roomIssuedAt, deviceActivatedAt, agentActiveAt, stopRequestedAt, endedAt *time.Time
|
||||
err := row.Scan(&item.DeviceID, &item.DeviceStatus, &item.DeviceOnline, &item.FirmwareVersion, &item.DesiredConfigRevision, &item.ReportedConfigRevision, &item.LastDeviceErrorCode, &item.LastDeviceErrorAt, &item.ConversationID, &item.ConversationStatus, &item.ConversationFailureCode, &item.AllocationExpiresAt, &item.CredentialIssuedAt, &item.CredentialExpiresAt, &item.RoomID, &item.Provider, &item.ProviderTaskRef, &item.ProviderTaskStatus, &item.ProviderStartAttempts, &item.ProviderStopAttempts, &item.ProviderNotificationCount, &item.LastProviderEvent, &item.LastProviderEventStatus, &item.LastProviderEventAt, &item.LastRequestID, &createdAt, &roomIssuedAt, &deviceActivatedAt, &agentActiveAt, &stopRequestedAt, &endedAt)
|
||||
if err == nil && createdAt != nil {
|
||||
item.Timings = &Timings{
|
||||
ConversationCreatedAt: *createdAt,
|
||||
RoomIssuedAt: roomIssuedAt,
|
||||
DeviceActivatedAt: deviceActivatedAt,
|
||||
AgentActiveAt: agentActiveAt,
|
||||
StopRequestedAt: stopRequestedAt,
|
||||
ConversationEndedAt: endedAt,
|
||||
RoomIssueMS: elapsedMilliseconds(createdAt, roomIssuedAt),
|
||||
DeviceJoinMS: elapsedMilliseconds(roomIssuedAt, deviceActivatedAt),
|
||||
AgentStartMS: elapsedMilliseconds(deviceActivatedAt, agentActiveAt),
|
||||
AgentStopMS: elapsedMilliseconds(stopRequestedAt, endedAt),
|
||||
TotalMS: elapsedMilliseconds(createdAt, endedAt),
|
||||
}
|
||||
}
|
||||
return item, err
|
||||
}
|
||||
|
||||
func elapsedMilliseconds(start, end *time.Time) *int64 {
|
||||
if start == nil || end == nil || end.Before(*start) {
|
||||
return nil
|
||||
}
|
||||
value := end.Sub(*start).Milliseconds()
|
||||
return &value
|
||||
}
|
||||
|
||||
func (r Record) String() string {
|
||||
return fmt.Sprintf("device=%s conversation=%s", r.DeviceID, r.ConversationID)
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
package diagnostics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"data-terminal-cloud/internal/audit"
|
||||
"data-terminal-cloud/internal/database"
|
||||
"data-terminal-cloud/internal/idempotency"
|
||||
"data-terminal-cloud/internal/identity"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestDiagnosticSearchAndConfigRefreshAreTenantScopedAndIdempotent(t *testing.T) {
|
||||
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := database.Open(ctx, databaseURL, "diagnostic-integration-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
service, err := NewService(pool, idempotency.NewExecutor(pool), audit.NewRepository())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
admin := identity.Principal{Issuer: "http://localhost:58180/realms/voice-local", Subject: "00000000-0000-0000-0000-000000000101"}
|
||||
tenantPublicID := "10000000-0000-0000-0000-000000000010"
|
||||
devicePublicID := "10000000-0000-0000-0000-000000000511"
|
||||
key := "diagnostic-refresh-integration"
|
||||
defer func() {
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM audit_event WHERE request_id='diagnostic-refresh-request'`)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM device_command WHERE tenant_id='10000000-0000-0000-0000-000000000001'::uuid AND idempotency_key=$1`, "ops-refresh:"+key)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM idempotency_record WHERE actor_key='ops:10000000-0000-0000-0000-000000000111' AND idempotency_key=$1`, key)
|
||||
}()
|
||||
items, err := service.Search(ctx, admin, tenantPublicID, "device_id", devicePublicID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(items) == 0 || items[0].DeviceID != devicePublicID || items[0].DesiredConfigRevision < 1 {
|
||||
t.Fatalf("items=%+v", items)
|
||||
}
|
||||
first, created, err := service.RefreshDeviceConfig(ctx, admin, tenantPublicID, devicePublicID, key, "diagnostic-refresh-request", "验证配置刷新")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, createdAgain, err := service.RefreshDeviceConfig(ctx, admin, tenantPublicID, devicePublicID, key, "diagnostic-refresh-request", "验证配置刷新")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !created || createdAgain || first.CommandID == "" || first.CommandID != second.CommandID || !first.Queued {
|
||||
t.Fatalf("first=%+v second=%+v created=%t/%t", first, second, created, createdAgain)
|
||||
}
|
||||
otherTenant := "20000000-0000-0000-0000-000000000010"
|
||||
if _, err = service.Search(ctx, admin, otherTenant, "device_id", devicePublicID); !errors.Is(err, ErrAccessDenied) {
|
||||
t.Fatalf("cross tenant error=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticConversationActionsCoverStableAndProviderStates(t *testing.T) {
|
||||
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := database.Open(ctx, databaseURL, "diagnostic-actions-integration-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
service, err := NewService(pool, idempotency.NewExecutor(pool), audit.NewRepository())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
admin := identity.Principal{Issuer: "http://localhost:58180/realms/voice-local", Subject: "00000000-0000-0000-0000-000000000101"}
|
||||
approver := identity.Principal{Issuer: admin.Issuer, Subject: "00000000-0000-0000-0000-000000000102"}
|
||||
tenantPublicID := "10000000-0000-0000-0000-000000000010"
|
||||
tenantID := "10000000-0000-0000-0000-000000000001"
|
||||
householdID := "10000000-0000-0000-0000-000000000201"
|
||||
subjectID := "10000000-0000-0000-0000-000000000401"
|
||||
deviceID := "10000000-0000-0000-0000-000000000501"
|
||||
consentID, contextID := uuid.NewString(), uuid.NewString()
|
||||
conversationIDs := []string{uuid.NewString(), uuid.NewString()}
|
||||
internalIDs := []string{uuid.NewString(), uuid.NewString()}
|
||||
keys := []string{"diagnostic-stop-before-start", "diagnostic-reconcile-starting", "diagnostic-stop-active", "diagnostic-reconcile-stopping"}
|
||||
defer func() {
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM conversation_state_history WHERE conversation_id=ANY($1::uuid[])`, internalIDs)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM rtc_task WHERE conversation_id=ANY($1::uuid[])`, internalIDs)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM outbox_event WHERE aggregate_id=ANY($1::uuid[])`, internalIDs)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM audit_event WHERE request_id LIKE 'diagnostic-actions-%'`)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM idempotency_record WHERE actor_key='ops:10000000-0000-0000-0000-000000000111' AND idempotency_key=ANY($1::text[])`, keys)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM web_event WHERE resource_public_id=ANY($1::uuid[])`, conversationIDs)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM conversation WHERE id=ANY($1::uuid[])`, internalIDs)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM context_snapshot WHERE id=$1::uuid`, contextID)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM consent_snapshot WHERE id=$1::uuid`, consentID)
|
||||
}()
|
||||
if _, err = pool.Exec(ctx, `INSERT INTO consent_snapshot(id,tenant_id,subject_id,snapshot_version,effective_consents) VALUES($1::uuid,$2::uuid,$3::uuid,9001,'{"voice_chat":true}'::jsonb)`, consentID, tenantID, subjectID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = pool.Exec(ctx, `INSERT INTO context_snapshot(id,tenant_id,subject_id,profile_version,consent_snapshot_id,prompt_bundle_version,engine_mode,content_digest) VALUES($1::uuid,$2::uuid,$3::uuid,1,$4::uuid,'diagnostic-test','s2s_hybrid',repeat('9',64))`, contextID, tenantID, subjectID, consentID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
insertConversation := func(index int, status, rtcStatus string) {
|
||||
t.Helper()
|
||||
taskRef, roomID := "diagnostic-task-"+conversationIDs[index], "diagnostic-room-"+conversationIDs[index]
|
||||
if _, insertErr := pool.Exec(ctx, `INSERT INTO conversation(id,public_id,tenant_id,household_id,subject_id,device_id,consent_snapshot_id,context_snapshot_id,provider_task_ref,engine_mode,retention_mode,status,allocation_expires_at) VALUES($1::uuid,$2::uuid,$3::uuid,$4::uuid,$5::uuid,$6::uuid,$7::uuid,$8::uuid,$9,'s2s_hybrid','transcript_only',$10,now()+interval '5 minutes')`, internalIDs[index], conversationIDs[index], tenantID, householdID, subjectID, deviceID, consentID, contextID, taskRef, status); insertErr != nil {
|
||||
t.Fatal(insertErr)
|
||||
}
|
||||
if _, insertErr := pool.Exec(ctx, `INSERT INTO rtc_task(conversation_id,provider,provider_task_ref,room_id,device_rtc_user_id,agent_rtc_user_id,status) VALUES($1::uuid,'mock',$2,$3,'device-test','agent-test',$4)`, internalIDs[index], taskRef, roomID, rtcStatus); insertErr != nil {
|
||||
t.Fatal(insertErr)
|
||||
}
|
||||
}
|
||||
|
||||
insertConversation(0, "ROOM_ISSUED", "ALLOCATED")
|
||||
stopped, created, err := service.StopConversation(ctx, admin, tenantPublicID, conversationIDs[0], keys[0], "diagnostic-actions-stop-before", "联调启动前停止")
|
||||
if err != nil || !created || stopped.ConversationStatus != "EXPIRED" || stopped.Queued {
|
||||
t.Fatalf("pre-start stop=%+v created=%t err=%v", stopped, created, err)
|
||||
}
|
||||
var preStartRTCStatus string
|
||||
if err = pool.QueryRow(ctx, `SELECT status FROM rtc_task WHERE conversation_id=$1::uuid`, internalIDs[0]).Scan(&preStartRTCStatus); err != nil || preStartRTCStatus != "COMPLETED" {
|
||||
t.Fatalf("pre-start rtc status=%s err=%v", preStartRTCStatus, err)
|
||||
}
|
||||
replayed, createdAgain, err := service.StopConversation(ctx, admin, tenantPublicID, conversationIDs[0], keys[0], "diagnostic-actions-stop-before", "联调启动前停止")
|
||||
if err != nil || createdAgain || replayed != stopped {
|
||||
t.Fatalf("replayed stop=%+v created=%t err=%v", replayed, createdAgain, err)
|
||||
}
|
||||
|
||||
insertConversation(1, "STARTING_AGENT", "STARTING")
|
||||
createdAt := time.Date(2026, time.August, 23, 10, 0, 0, 0, time.UTC)
|
||||
roomIssuedAt := createdAt.Add(250 * time.Millisecond)
|
||||
deviceActivatedAt := roomIssuedAt.Add(2 * time.Second)
|
||||
if _, err = pool.Exec(ctx, `UPDATE conversation SET created_at=$2 WHERE id=$1::uuid`, internalIDs[1], createdAt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = pool.Exec(ctx, `INSERT INTO conversation_state_history(conversation_id,from_status,to_status,reason,source,occurred_at) VALUES($1::uuid,'PREPARING','ROOM_ISSUED','timing fixture','platform',$2),($1::uuid,'ROOM_ISSUED','STARTING_AGENT','timing fixture','device',$3)`, internalIDs[1], roomIssuedAt, deviceActivatedAt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
timingItems, err := service.Search(ctx, admin, tenantPublicID, "conversation_id", conversationIDs[1])
|
||||
if err != nil || len(timingItems) != 1 || timingItems[0].Timings == nil {
|
||||
t.Fatalf("timing search items=%+v err=%v", timingItems, err)
|
||||
}
|
||||
if timingItems[0].Timings.RoomIssueMS == nil || *timingItems[0].Timings.RoomIssueMS != 250 || timingItems[0].Timings.DeviceJoinMS == nil || *timingItems[0].Timings.DeviceJoinMS != 2000 || timingItems[0].Timings.AgentStartMS != nil {
|
||||
t.Fatalf("unexpected handshake timings: %+v", timingItems[0].Timings)
|
||||
}
|
||||
if timingItems[0].CredentialIssuedAt == nil || timingItems[0].CredentialExpiresAt == nil || !timingItems[0].CredentialExpiresAt.After(*timingItems[0].CredentialIssuedAt) {
|
||||
t.Fatalf("credential metadata is missing or invalid: %+v", timingItems[0])
|
||||
}
|
||||
reconciled, created, err := service.ReconcileConversation(ctx, admin, tenantPublicID, conversationIDs[1], keys[1], "diagnostic-actions-reconcile-start", "核对启动结果")
|
||||
if err != nil || !created || !reconciled.Queued || reconciled.ConversationStatus != "STARTING_AGENT" {
|
||||
t.Fatalf("start reconcile=%+v created=%t err=%v", reconciled, created, err)
|
||||
}
|
||||
if _, _, err = service.StopConversation(ctx, approver, tenantPublicID, conversationIDs[1], "diagnostic-approver-denied", "diagnostic-actions-denied", "只读审批角色不应停止"); !errors.Is(err, ErrAccessDenied) {
|
||||
t.Fatalf("approver stop error=%v", err)
|
||||
}
|
||||
stopped, created, err = service.StopConversation(ctx, admin, tenantPublicID, conversationIDs[1], keys[2], "diagnostic-actions-stop-active", "停止可能已经启动的任务")
|
||||
if err != nil || !created || !stopped.Queued || stopped.ConversationStatus != "STOPPING" {
|
||||
t.Fatalf("active stop=%+v created=%t err=%v", stopped, created, err)
|
||||
}
|
||||
reconciled, created, err = service.ReconcileConversation(ctx, admin, tenantPublicID, conversationIDs[1], keys[3], "diagnostic-actions-reconcile-stop", "核对停止结果")
|
||||
if err != nil || !created || !reconciled.Queued || reconciled.ConversationStatus != "STOPPING" {
|
||||
t.Fatalf("stop reconcile=%+v created=%t err=%v", reconciled, created, err)
|
||||
}
|
||||
var stopEvents, reconcileEvents int
|
||||
if err = pool.QueryRow(ctx, `SELECT count(*) FILTER(WHERE event_type='conversation.agent_stop_requested'),count(*) FILTER(WHERE event_type LIKE 'conversation.agent_%_reconcile_requested') FROM outbox_event WHERE aggregate_id=$1::uuid`, internalIDs[1]).Scan(&stopEvents, &reconcileEvents); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stopEvents != 1 || reconcileEvents != 2 {
|
||||
t.Fatalf("stop events=%d reconcile events=%d", stopEvents, reconcileEvents)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticJobRetryBundleAndDeviceRevocation(t *testing.T) {
|
||||
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := database.Open(ctx, databaseURL, "diagnostic-recovery-integration-test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
service, err := NewService(pool, idempotency.NewExecutor(pool), audit.NewRepository())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
admin := identity.Principal{Issuer: "http://localhost:58180/realms/voice-local", Subject: "00000000-0000-0000-0000-000000000101"}
|
||||
tenantPublicID, tenantID := "10000000-0000-0000-0000-000000000010", "10000000-0000-0000-0000-000000000001"
|
||||
deviceID, devicePublicID := uuid.NewString(), uuid.NewString()
|
||||
jobInternalID, aggregateID := uuid.NewString(), uuid.NewString()
|
||||
unknownJobInternalID := uuid.NewString()
|
||||
consentID, contextID := uuid.NewString(), uuid.NewString()
|
||||
conversationInternalID, conversationPublicID := uuid.NewString(), uuid.NewString()
|
||||
keys := []string{"diagnostic-job-retry", "diagnostic-bundle-export", "diagnostic-device-revoke"}
|
||||
defer func() {
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM audit_event WHERE request_id LIKE 'diagnostic-recovery-%'`)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM idempotency_record WHERE actor_key='ops:10000000-0000-0000-0000-000000000111' AND idempotency_key=ANY($1::text[])`, keys)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM outbox_event WHERE id=ANY($1::uuid[]) OR aggregate_id=$2::uuid`, []string{jobInternalID, unknownJobInternalID}, conversationInternalID)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM conversation_state_history WHERE conversation_id=$1::uuid`, conversationInternalID)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM rtc_task WHERE conversation_id=$1::uuid`, conversationInternalID)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM web_event WHERE resource_public_id=$1::uuid`, conversationPublicID)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM conversation WHERE id=$1::uuid`, conversationInternalID)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM context_snapshot WHERE id=$1::uuid`, contextID)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM consent_snapshot WHERE id=$1::uuid`, consentID)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM device_twin WHERE device_id=$1::uuid`, deviceID)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM device WHERE id=$1::uuid`, deviceID)
|
||||
}()
|
||||
if _, err = pool.Exec(ctx, `INSERT INTO device(id,public_id,tenant_id,serial_digest,model,status,credential_ref) VALUES($1::uuid,$2::uuid,$3::uuid,$4,'diagnostic-test','registered','test://diagnostic-recovery')`, deviceID, devicePublicID, tenantID, strings.Repeat("8", 64)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = pool.Exec(ctx, `INSERT INTO device_twin(device_id,tenant_id,desired_revision) VALUES($1::uuid,$2::uuid,3)`, deviceID, tenantID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = pool.Exec(ctx, `INSERT INTO consent_snapshot(id,tenant_id,subject_id,snapshot_version,effective_consents) VALUES($1::uuid,$2::uuid,'10000000-0000-0000-0000-000000000401'::uuid,9101,'{"voice_chat":true}'::jsonb)`, consentID, tenantID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = pool.Exec(ctx, `INSERT INTO context_snapshot(id,tenant_id,subject_id,profile_version,consent_snapshot_id,prompt_bundle_version,engine_mode,content_digest) VALUES($1::uuid,$2::uuid,'10000000-0000-0000-0000-000000000401'::uuid,1,$3::uuid,'diagnostic-revoke','s2s_hybrid',repeat('7',64))`, contextID, tenantID, consentID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taskRef := "diagnostic-revoke-task-" + conversationPublicID
|
||||
if _, err = pool.Exec(ctx, `INSERT INTO conversation(id,public_id,tenant_id,household_id,subject_id,device_id,consent_snapshot_id,context_snapshot_id,provider_task_ref,engine_mode,retention_mode,status,allocation_expires_at) VALUES($1::uuid,$2::uuid,$3::uuid,'10000000-0000-0000-0000-000000000201'::uuid,'10000000-0000-0000-0000-000000000401'::uuid,$4::uuid,$5::uuid,$6::uuid,$7,'s2s_hybrid','transcript_only','ACTIVE',now()+interval '5 minutes')`, conversationInternalID, conversationPublicID, tenantID, deviceID, consentID, contextID, taskRef); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = pool.Exec(ctx, `INSERT INTO rtc_task(conversation_id,provider,provider_task_ref,room_id,device_rtc_user_id,agent_rtc_user_id,status) VALUES($1::uuid,'mock',$2,$3,'diagnostic-device','diagnostic-agent','ACTIVE')`, conversationInternalID, taskRef, "diagnostic-revoke-room-"+conversationPublicID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = pool.Exec(ctx, `INSERT INTO outbox_event(id,tenant_id,aggregate_type,aggregate_id,event_type,payload,status,attempt_count,last_error_code,dead_at,payload_hash) VALUES($1::uuid,$2::uuid,'platform',$3::uuid,'platform.noop','{}'::jsonb,'dead',8,'TEMPORARY_DEPENDENCY_FAILURE',now(),repeat('0',64))`, jobInternalID, tenantID, aggregateID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = pool.Exec(ctx, `INSERT INTO outbox_event(id,tenant_id,aggregate_type,aggregate_id,event_type,payload,status,attempt_count,last_error_code,dead_at,payload_hash) VALUES($1::uuid,$2::uuid,'platform',$3::uuid,'unsupported.diagnostic_job','{}'::jsonb,'dead',8,'HANDLER_NOT_REGISTERED',now(),repeat('1',64))`, unknownJobInternalID, tenantID, aggregateID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jobs, err := service.ListDeadJobs(ctx, admin, tenantPublicID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var job Job
|
||||
for _, candidate := range jobs {
|
||||
if candidate.EventType == "platform.noop" && candidate.AttemptCount == 8 {
|
||||
job = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
if job.ID == "" {
|
||||
t.Fatalf("dead job not listed: %+v", jobs)
|
||||
}
|
||||
retried, created, err := service.RetryDeadJob(ctx, admin, tenantPublicID, job.ID, keys[0], "diagnostic-recovery-job", "依赖恢复后受控重试")
|
||||
if err != nil || !created || !retried.Queued || retried.ManualRetryCount != 1 {
|
||||
t.Fatalf("retry=%+v created=%t err=%v", retried, created, err)
|
||||
}
|
||||
replayed, createdAgain, err := service.RetryDeadJob(ctx, admin, tenantPublicID, job.ID, keys[0], "diagnostic-recovery-job", "依赖恢复后受控重试")
|
||||
if err != nil || createdAgain || replayed != retried {
|
||||
t.Fatalf("retry replay=%+v created=%t err=%v", replayed, createdAgain, err)
|
||||
}
|
||||
var manualRetries int
|
||||
if err = pool.QueryRow(ctx, `SELECT manual_retry_count FROM outbox_event WHERE id=$1::uuid`, jobInternalID).Scan(&manualRetries); err != nil || manualRetries != 1 {
|
||||
t.Fatalf("manual retries=%d err=%v", manualRetries, err)
|
||||
}
|
||||
var unknownJobPublicID string
|
||||
if err = pool.QueryRow(ctx, `SELECT public_id::text FROM outbox_event WHERE id=$1::uuid`, unknownJobInternalID).Scan(&unknownJobPublicID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err = service.RetryDeadJob(ctx, admin, tenantPublicID, unknownJobPublicID, "diagnostic-unknown-job", "diagnostic-recovery-unknown-job", "不支持的作业类型不得重试"); !errors.Is(err, ErrStateConflict) {
|
||||
t.Fatalf("unknown job retry error=%v", err)
|
||||
}
|
||||
|
||||
bundle, created, err := service.ExportBundle(ctx, admin, tenantPublicID, "device_id", devicePublicID, keys[1], "diagnostic-recovery-bundle", "导出脱敏联调证据")
|
||||
if err != nil || !created || bundle.ID == "" || len(bundle.Items) != 1 || bundle.Items[0].DeviceID != devicePublicID || bundle.RedactionPolicy != "ops-diagnostic-minimal-v1" {
|
||||
t.Fatalf("bundle=%+v created=%t err=%v", bundle, created, err)
|
||||
}
|
||||
revoked, created, err := service.RevokeDevice(ctx, admin, tenantPublicID, devicePublicID, keys[2], "diagnostic-recovery-revoke", "设备证书疑似泄露，立即吊销")
|
||||
if err != nil || !created || revoked.Action != "revoke_device" || !revoked.Queued || revoked.ConversationID != conversationPublicID || revoked.ConversationStatus != "STOPPING" {
|
||||
t.Fatalf("revoke=%+v created=%t err=%v", revoked, created, err)
|
||||
}
|
||||
var deviceStatus string
|
||||
var revokedAtPresent bool
|
||||
if err = pool.QueryRow(ctx, `SELECT status,revoked_at IS NOT NULL FROM device WHERE id=$1::uuid`, deviceID).Scan(&deviceStatus, &revokedAtPresent); err != nil || deviceStatus != "revoked" || !revokedAtPresent {
|
||||
t.Fatalf("device status=%s revoked_at=%t err=%v", deviceStatus, revokedAtPresent, err)
|
||||
}
|
||||
var stopEvents int
|
||||
if err = pool.QueryRow(ctx, `SELECT count(*) FROM outbox_event WHERE aggregate_id=$1::uuid AND event_type='conversation.agent_stop_requested'`, conversationInternalID).Scan(&stopEvents); err != nil || stopEvents != 1 {
|
||||
t.Fatalf("revoke stop events=%d err=%v", stopEvents, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticInputValidationDoesNotReachDependencies(t *testing.T) {
|
||||
if _, err := NewService(nil, nil, nil); err == nil {
|
||||
t.Fatal("nil dependencies must be rejected")
|
||||
}
|
||||
service := &Service{}
|
||||
principal := identity.Principal{}
|
||||
if _, err := service.Search(context.Background(), principal, "bad-tenant", "unknown", "x"); !errors.Is(err, ErrInvalid) {
|
||||
t.Fatalf("invalid search error=%v", err)
|
||||
}
|
||||
if _, _, err := service.StopConversation(context.Background(), principal, "bad-tenant", "not-a-uuid", "key", "request", "reason"); !errors.Is(err, ErrInvalid) {
|
||||
t.Fatalf("invalid stop error=%v", err)
|
||||
}
|
||||
if _, _, err := service.ReconcileConversation(context.Background(), principal, "bad-tenant", uuid.NewString(), "key", "request", ""); !errors.Is(err, ErrInvalid) {
|
||||
t.Fatalf("invalid reconcile error=%v", err)
|
||||
}
|
||||
if _, _, err := service.RefreshDeviceConfig(context.Background(), principal, "bad-tenant", "not-a-uuid", "key", "request", "reason"); !errors.Is(err, ErrInvalid) {
|
||||
t.Fatalf("invalid refresh error=%v", err)
|
||||
}
|
||||
if got := (Record{DeviceID: "device", ConversationID: "conversation"}).String(); got != "device=device conversation=conversation" {
|
||||
t.Fatalf("record string=%q", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user