293 lines
14 KiB
Go
293 lines
14 KiB
Go
package diagnostics
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
_ "time/tzdata"
|
|
|
|
"data-terminal-cloud/internal/audit"
|
|
"data-terminal-cloud/internal/idempotency"
|
|
"data-terminal-cloud/internal/identity"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
type PilotDailySnapshotInput struct {
|
|
CandidateVersion string `json:"candidate_version"`
|
|
BuildDigest string `json:"build_digest"`
|
|
LocalDate string `json:"local_date"`
|
|
Timezone string `json:"timezone"`
|
|
Reason string `json:"reason"`
|
|
}
|
|
|
|
type PilotDeviceState struct {
|
|
Bound int64 `json:"bound"`
|
|
Eligible int64 `json:"eligible"`
|
|
Quarantined int64 `json:"quarantined"`
|
|
Revoked int64 `json:"revoked"`
|
|
Online int64 `json:"online"`
|
|
Offline int64 `json:"offline"`
|
|
ReportedDuringDay int64 `json:"reported_during_day"`
|
|
}
|
|
|
|
type PilotCountByStatus struct {
|
|
Status string `json:"status"`
|
|
Count int64 `json:"count"`
|
|
}
|
|
|
|
type PilotConversationState struct {
|
|
Total int64 `json:"total"`
|
|
Statuses []PilotCountByStatus `json:"statuses"`
|
|
}
|
|
|
|
type PilotProviderState struct {
|
|
Provider string `json:"provider"`
|
|
CurrentStatus string `json:"current_status"`
|
|
ConsecutiveFailures int64 `json:"consecutive_failures"`
|
|
FailedEventsDuringDay int64 `json:"failed_events_during_day"`
|
|
FailedCallsDuringDay int64 `json:"failed_calls_during_day"`
|
|
NotificationBacklog int64 `json:"notification_backlog"`
|
|
}
|
|
|
|
type PilotUploadState struct {
|
|
CreatedDuringDay int64 `json:"created_during_day"`
|
|
CompletedDuringDay int64 `json:"completed_during_day"`
|
|
FailedDuringDay int64 `json:"failed_during_day"`
|
|
OpenBatches int64 `json:"open_batches"`
|
|
PendingSegments int64 `json:"pending_segments"`
|
|
RejectedSegments int64 `json:"rejected_segments"`
|
|
}
|
|
|
|
type PilotAlertState struct {
|
|
OpenedDuringDay int64 `json:"opened_during_day"`
|
|
ResolvedDuringDay int64 `json:"resolved_during_day"`
|
|
Open int64 `json:"open"`
|
|
HighOpen int64 `json:"high_open"`
|
|
CriticalOpen int64 `json:"critical_open"`
|
|
}
|
|
|
|
type PilotCostState struct {
|
|
Currency string `json:"currency"`
|
|
CostMicros int64 `json:"cost_micros"`
|
|
LineCount int64 `json:"line_count"`
|
|
}
|
|
|
|
type PilotDailySnapshot struct {
|
|
ID string `json:"id"`
|
|
GeneratedAt time.Time `json:"generated_at"`
|
|
TenantID string `json:"tenant_id"`
|
|
CandidateVersion string `json:"candidate_version"`
|
|
BuildDigest string `json:"build_digest"`
|
|
LocalDate string `json:"local_date"`
|
|
Timezone string `json:"timezone"`
|
|
WindowStart time.Time `json:"window_start"`
|
|
WindowEnd time.Time `json:"window_end"`
|
|
RedactionPolicy string `json:"redaction_policy"`
|
|
Devices PilotDeviceState `json:"devices"`
|
|
Conversations PilotConversationState `json:"conversations"`
|
|
Providers []PilotProviderState `json:"providers"`
|
|
Uploads PilotUploadState `json:"uploads"`
|
|
Alerts PilotAlertState `json:"alerts"`
|
|
Costs []PilotCostState `json:"costs"`
|
|
Disclaimer string `json:"disclaimer"`
|
|
}
|
|
|
|
var pilotConversationStatuses = []string{"PREPARING", "ROOM_ISSUED", "STARTING_AGENT", "ACTIVE", "STOPPING", "COMPLETED", "FAILED", "EXPIRED"}
|
|
|
|
func (s *Service) ExportPilotDailySnapshot(ctx context.Context, principal identity.Principal, tenantPublicID, key, requestID string, input PilotDailySnapshotInput) (PilotDailySnapshot, bool, error) {
|
|
normalizePilotDailySnapshotInput(&input)
|
|
_, windowStart, windowEnd, err := validatePilotDailySnapshotInput(input, time.Now().UTC())
|
|
if err != nil {
|
|
return PilotDailySnapshot{}, false, ErrInvalid
|
|
}
|
|
a, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_operator", "ops_admin")
|
|
if err != nil {
|
|
return PilotDailySnapshot{}, false, err
|
|
}
|
|
requestBody, _ := json.Marshal(input)
|
|
result, err := s.idem.Execute(ctx, idempotency.Request{
|
|
TenantID: &a.tenantID, ActorKey: "ops:" + a.userPublicID,
|
|
Scope: "POST /api/v1/ops/diagnostics/pilot-daily-snapshots", Key: key,
|
|
RequestHash: idempotency.Hash(requestBody), ExpiresAfter: 24 * time.Hour,
|
|
}, func(operationContext context.Context, tx pgx.Tx) (idempotency.Result, error) {
|
|
snapshot, buildErr := buildPilotDailySnapshot(operationContext, tx, tenantPublicID, a.tenantID, input, windowStart, windowEnd)
|
|
if buildErr != nil {
|
|
return idempotency.Result{}, buildErr
|
|
}
|
|
if auditErr := s.audit.Record(operationContext, tx, audit.Event{
|
|
TenantID: &a.tenantID, ActorType: "user", ActorPublicID: &a.userPublicID,
|
|
Action: "ops.diagnostics.pilot_daily_snapshot.export", ResourceType: "pilot_daily_snapshot", ResourceID: &snapshot.ID,
|
|
RequestID: requestID, Purpose: input.Reason, Result: "success",
|
|
Detail: map[string]any{"candidate_version": input.CandidateVersion, "build_digest": input.BuildDigest, "local_date": input.LocalDate, "timezone": input.Timezone, "window_start": windowStart, "window_end": windowEnd, "redaction_policy": snapshot.RedactionPolicy},
|
|
}); auditErr != nil {
|
|
return idempotency.Result{}, auditErr
|
|
}
|
|
encoded, _ := json.Marshal(snapshot)
|
|
return idempotency.Result{StatusCode: 201, Body: encoded, ResourceID: &snapshot.ID}, nil
|
|
})
|
|
if err != nil {
|
|
return PilotDailySnapshot{}, false, err
|
|
}
|
|
var snapshot PilotDailySnapshot
|
|
if err = json.Unmarshal(result.Body, &snapshot); err != nil {
|
|
return PilotDailySnapshot{}, false, err
|
|
}
|
|
return snapshot, !result.Replayed, nil
|
|
}
|
|
|
|
func normalizePilotDailySnapshotInput(input *PilotDailySnapshotInput) {
|
|
input.CandidateVersion = strings.TrimSpace(input.CandidateVersion)
|
|
input.BuildDigest = strings.ToLower(strings.TrimSpace(input.BuildDigest))
|
|
input.LocalDate = strings.TrimSpace(input.LocalDate)
|
|
input.Timezone = strings.TrimSpace(input.Timezone)
|
|
input.Reason = strings.TrimSpace(input.Reason)
|
|
}
|
|
|
|
func validatePilotDailySnapshotInput(input PilotDailySnapshotInput, now time.Time) (*time.Location, time.Time, time.Time, error) {
|
|
if !acceptanceCandidatePattern.MatchString(input.CandidateVersion) || !acceptanceDigestPattern.MatchString(input.BuildDigest) || input.LocalDate == "" || input.Timezone == "" || len([]rune(input.Timezone)) > 80 || input.Reason == "" || len([]rune(input.Reason)) > 255 {
|
|
return nil, time.Time{}, time.Time{}, ErrInvalid
|
|
}
|
|
location, err := time.LoadLocation(input.Timezone)
|
|
if err != nil {
|
|
return nil, time.Time{}, time.Time{}, ErrInvalid
|
|
}
|
|
windowStart, err := time.ParseInLocation("2006-01-02", input.LocalDate, location)
|
|
if err != nil || windowStart.Format("2006-01-02") != input.LocalDate {
|
|
return nil, time.Time{}, time.Time{}, ErrInvalid
|
|
}
|
|
today := now.In(location)
|
|
todayStart := time.Date(today.Year(), today.Month(), today.Day(), 0, 0, 0, 0, location)
|
|
if windowStart.After(todayStart) {
|
|
return nil, time.Time{}, time.Time{}, ErrInvalid
|
|
}
|
|
return location, windowStart.UTC(), windowStart.AddDate(0, 0, 1).UTC(), nil
|
|
}
|
|
|
|
func buildPilotDailySnapshot(ctx context.Context, tx pgx.Tx, tenantPublicID, tenantInternalID string, input PilotDailySnapshotInput, windowStart, windowEnd time.Time) (PilotDailySnapshot, error) {
|
|
snapshot := PilotDailySnapshot{
|
|
ID: uuid.NewString(), GeneratedAt: time.Now().UTC(), TenantID: tenantPublicID, CandidateVersion: input.CandidateVersion, BuildDigest: input.BuildDigest, LocalDate: input.LocalDate, Timezone: input.Timezone,
|
|
WindowStart: windowStart, WindowEnd: windowEnd, RedactionPolicy: "ops-pilot-daily-minimal-v1",
|
|
Conversations: PilotConversationState{Statuses: []PilotCountByStatus{}}, Providers: []PilotProviderState{}, Costs: []PilotCostState{},
|
|
Disclaimer: "本快照混合所选自然日的活动统计与生成时刻的当前积压状态，只用于试点运行复盘；不替代原始监控、真实设备体验、财务账单或发布审批。",
|
|
}
|
|
if err := tx.QueryRow(ctx, `WITH bound AS (
|
|
SELECT d.id,d.status,dt.last_reported_at,dps.online
|
|
FROM device_binding db JOIN device d ON d.id=db.device_id AND d.tenant_id=db.tenant_id
|
|
LEFT JOIN device_twin dt ON dt.device_id=d.id AND dt.tenant_id=d.tenant_id
|
|
LEFT JOIN device_presence_state dps ON dps.device_id=d.id AND dps.tenant_id=d.tenant_id
|
|
WHERE db.tenant_id=$1::uuid AND db.unbound_at IS NULL)
|
|
SELECT count(*),count(*) FILTER(WHERE status='registered'),count(*) FILTER(WHERE status='quarantined'),count(*) FILTER(WHERE status='revoked'),
|
|
count(*) FILTER(WHERE status='registered' AND COALESCE(online,false)),count(*) FILTER(WHERE status='registered' AND NOT COALESCE(online,false)),
|
|
count(*) FILTER(WHERE status='registered' AND last_reported_at >= $2 AND last_reported_at < $3) FROM bound`, tenantInternalID, windowStart, windowEnd).Scan(
|
|
&snapshot.Devices.Bound, &snapshot.Devices.Eligible, &snapshot.Devices.Quarantined, &snapshot.Devices.Revoked,
|
|
&snapshot.Devices.Online, &snapshot.Devices.Offline, &snapshot.Devices.ReportedDuringDay,
|
|
); err != nil {
|
|
return PilotDailySnapshot{}, err
|
|
}
|
|
conversationCounts := map[string]int64{}
|
|
rows, err := tx.Query(ctx, `SELECT status,count(*) FROM conversation WHERE tenant_id=$1::uuid AND created_at >= $2 AND created_at < $3 GROUP BY status`, tenantInternalID, windowStart, windowEnd)
|
|
if err != nil {
|
|
return PilotDailySnapshot{}, err
|
|
}
|
|
for rows.Next() {
|
|
var status string
|
|
var count int64
|
|
if err = rows.Scan(&status, &count); err != nil {
|
|
rows.Close()
|
|
return PilotDailySnapshot{}, err
|
|
}
|
|
conversationCounts[status] = count
|
|
snapshot.Conversations.Total += count
|
|
}
|
|
if err = rows.Err(); err != nil {
|
|
rows.Close()
|
|
return PilotDailySnapshot{}, err
|
|
}
|
|
rows.Close()
|
|
for _, status := range pilotConversationStatuses {
|
|
snapshot.Conversations.Statuses = append(snapshot.Conversations.Statuses, PilotCountByStatus{Status: status, Count: conversationCounts[status]})
|
|
}
|
|
rows, err = tx.Query(ctx, `WITH providers AS (
|
|
SELECT provider FROM provider_dependency_state WHERE tenant_id=$1::uuid
|
|
UNION SELECT pe.provider FROM provider_event pe WHERE pe.tenant_id=$1::uuid AND pe.received_at >= $2 AND pe.received_at < $3
|
|
UNION SELECT rt.provider FROM conversation c JOIN rtc_task rt ON rt.conversation_id=c.id WHERE c.tenant_id=$1::uuid AND c.created_at >= $2 AND c.created_at < $3
|
|
UNION SELECT hs.provider FROM conversation c JOIN hardware_agent_session hs ON hs.conversation_id=c.id WHERE c.tenant_id=$1::uuid AND c.created_at >= $2 AND c.created_at < $3)
|
|
SELECT p.provider,COALESCE(pds.status,'UNKNOWN'),COALESCE(pds.consecutive_failures,0),
|
|
(SELECT count(*) FROM provider_event pe WHERE pe.tenant_id=$1::uuid AND pe.provider=p.provider AND pe.processing_status='failed' AND pe.received_at >= $2 AND pe.received_at < $3)+
|
|
(SELECT count(*) FROM hardware_agent_event he JOIN hardware_agent_session hs ON hs.conversation_id=he.conversation_id WHERE he.tenant_id=$1::uuid AND hs.provider=p.provider AND he.event_type='session_failed' AND he.occurred_at >= $2 AND he.occurred_at < $3),
|
|
(SELECT count(DISTINCT c.id) FROM conversation c
|
|
LEFT JOIN rtc_task rt ON rt.conversation_id=c.id
|
|
LEFT JOIN hardware_agent_session hs ON hs.conversation_id=c.id
|
|
JOIN conversation_state_history csh ON csh.conversation_id=c.id
|
|
WHERE c.tenant_id=$1::uuid AND COALESCE(rt.provider,hs.provider)=p.provider AND csh.to_status='FAILED' AND csh.occurred_at >= $2 AND csh.occurred_at < $3),
|
|
(SELECT count(*) FROM provider_event pe WHERE pe.tenant_id=$1::uuid AND pe.provider=p.provider AND pe.processing_status IN ('pending','failed'))
|
|
FROM providers p LEFT JOIN provider_dependency_state pds ON pds.tenant_id=$1::uuid AND pds.provider=p.provider ORDER BY p.provider`, tenantInternalID, windowStart, windowEnd)
|
|
if err != nil {
|
|
return PilotDailySnapshot{}, err
|
|
}
|
|
for rows.Next() {
|
|
var item PilotProviderState
|
|
if err = rows.Scan(&item.Provider, &item.CurrentStatus, &item.ConsecutiveFailures, &item.FailedEventsDuringDay, &item.FailedCallsDuringDay, &item.NotificationBacklog); err != nil {
|
|
rows.Close()
|
|
return PilotDailySnapshot{}, err
|
|
}
|
|
snapshot.Providers = append(snapshot.Providers, item)
|
|
}
|
|
if err = rows.Err(); err != nil {
|
|
rows.Close()
|
|
return PilotDailySnapshot{}, err
|
|
}
|
|
rows.Close()
|
|
if err = tx.QueryRow(ctx, `SELECT
|
|
count(*) FILTER(WHERE created_at >= $2 AND created_at < $3),
|
|
count(*) FILTER(WHERE completed_at >= $2 AND completed_at < $3),
|
|
count(*) FILTER(WHERE status='FAILED' AND updated_at >= $2 AND updated_at < $3),
|
|
count(*) FILTER(WHERE status IN ('PENDING','UPLOADING','VERIFYING')),
|
|
(SELECT count(*) FROM media_segment WHERE tenant_id=$1::uuid AND status IN ('DECLARED','AUTHORIZED','UPLOADED','VERIFYING')),
|
|
(SELECT count(*) FROM media_segment WHERE tenant_id=$1::uuid AND status='REJECTED')
|
|
FROM upload_batch WHERE tenant_id=$1::uuid`, tenantInternalID, windowStart, windowEnd).Scan(
|
|
&snapshot.Uploads.CreatedDuringDay, &snapshot.Uploads.CompletedDuringDay, &snapshot.Uploads.FailedDuringDay,
|
|
&snapshot.Uploads.OpenBatches, &snapshot.Uploads.PendingSegments, &snapshot.Uploads.RejectedSegments,
|
|
); err != nil {
|
|
return PilotDailySnapshot{}, err
|
|
}
|
|
if err = tx.QueryRow(ctx, `SELECT
|
|
count(*) FILTER(WHERE first_detected_at >= $2 AND first_detected_at < $3),
|
|
count(*) FILTER(WHERE resolved_at >= $2 AND resolved_at < $3),
|
|
count(*) FILTER(WHERE status <> 'RESOLVED'),
|
|
count(*) FILTER(WHERE status <> 'RESOLVED' AND severity='HIGH'),
|
|
count(*) FILTER(WHERE status <> 'RESOLVED' AND severity='CRITICAL')
|
|
FROM alert_incident WHERE tenant_id=$1::uuid`, tenantInternalID, windowStart, windowEnd).Scan(
|
|
&snapshot.Alerts.OpenedDuringDay, &snapshot.Alerts.ResolvedDuringDay, &snapshot.Alerts.Open,
|
|
&snapshot.Alerts.HighOpen, &snapshot.Alerts.CriticalOpen,
|
|
); err != nil {
|
|
return PilotDailySnapshot{}, err
|
|
}
|
|
rows, err = tx.Query(ctx, `SELECT ccl.currency,COALESCE(sum(ccl.cost_micros),0)::bigint,count(*)
|
|
FROM conversation_cost_line ccl JOIN conversation_usage_event cue ON cue.id=ccl.usage_event_id
|
|
WHERE ccl.tenant_id=$1::uuid AND cue.occurred_at >= $2 AND cue.occurred_at < $3
|
|
GROUP BY ccl.currency ORDER BY ccl.currency`, tenantInternalID, windowStart, windowEnd)
|
|
if err != nil {
|
|
return PilotDailySnapshot{}, err
|
|
}
|
|
for rows.Next() {
|
|
var item PilotCostState
|
|
if err = rows.Scan(&item.Currency, &item.CostMicros, &item.LineCount); err != nil {
|
|
rows.Close()
|
|
return PilotDailySnapshot{}, err
|
|
}
|
|
snapshot.Costs = append(snapshot.Costs, item)
|
|
}
|
|
if err = rows.Err(); err != nil {
|
|
rows.Close()
|
|
return PilotDailySnapshot{}, err
|
|
}
|
|
rows.Close()
|
|
sort.Slice(snapshot.Providers, func(i, j int) bool { return snapshot.Providers[i].Provider < snapshot.Providers[j].Provider })
|
|
return snapshot, nil
|
|
}
|