chore: establish local platform baseline

2026-08-26 06:41:54 +08:00
commit 29bdf4900f
575 changed files with 95644 additions and 0 deletions
@@ -0,0 +1,12 @@
+global:
+  scrape_interval: 10s
+  evaluation_interval: 10s
+
+rule_files:
+  - /etc/prometheus/rules/*.yml
+
+scrape_configs:
+  - job_name: voice-platform
+    metrics_path: /metrics
+    static_configs:
+      - targets: [api:8080, callbacks:8081, worker:8082]
@@ -0,0 +1,96 @@
+groups:
+  - name: data-terminal-cloud-availability
+    rules:
+      - alert: PlatformServiceDown
+        expr: up{job="voice-platform"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Platform service is unavailable"
+          description: "Prometheus has failed to scrape {{ $labels.instance }} for more than two minutes."
+
+      - alert: PlatformHighHTTP5xxRate
+        expr: |
+          (
+            sum by (service) (rate(voice_platform_http_requests_total{status=~"5.."}[5m]))
+            /
+            clamp_min(sum by (service) (rate(voice_platform_http_requests_total[5m])), 0.01)
+          ) > 0.05
+          and
+          sum by (service) (rate(voice_platform_http_requests_total[5m])) > 0.1
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "HTTP 5xx rate exceeds 5%"
+          description: "{{ $labels.service }} has sustained a 5xx rate above 5%."
+
+      - alert: PlatformHighP95Latency
+        expr: |
+          histogram_quantile(
+            0.95,
+            sum by (service, route, le) (rate(voice_platform_http_request_duration_seconds_bucket[5m]))
+          ) > 1.5
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "HTTP p95 latency exceeds 1.5 seconds"
+          description: "{{ $labels.service }} route {{ $labels.route }} has sustained high latency."
+
+      - alert: PlatformExternalDependencyUnavailable
+        expr: voice_platform_dependency_up{dependency=~"oidc_jwks|ark_tls|volc_rtc_tls|content_safety_tls"} == 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "External platform dependency is unavailable"
+          description: "{{ $labels.service }} cannot reach or validate {{ $labels.dependency }} for more than five minutes."
+
+  - name: data-terminal-cloud-workflows
+    rules:
+      - alert: OutboxDeadLetterPresent
+        expr: voice_platform_outbox_events{status="dead"} > 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Outbox dead-letter events require intervention"
+          description: "At least one reliable background task exhausted its retry budget."
+
+      - alert: OutboxPendingBacklog
+        expr: voice_platform_outbox_events{status="pending"} > 100
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Outbox pending backlog is growing"
+          description: "More than 100 events have remained pending for ten minutes."
+
+      - alert: PrivacyDeletionPartialFailure
+        expr: voice_platform_privacy_requests{status="PARTIAL_FAILED"} > 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "A privacy request has partially failed"
+          description: "A subject-level privacy workflow requires retry or manual reconciliation."
+
+      - alert: FirmwareAssignmentFailure
+        expr: voice_platform_firmware_assignments{status="FAILED"} > 0
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: "A firmware assignment failed"
+          description: "Pause the rollout, inspect the device result, and select a verified rollback release if necessary."
+
+      - alert: FirmwareRolloutStalled
+        expr: voice_platform_firmware_stalled_rollouts > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "A firmware rollout is stalled"
+          description: "A running rollout has not changed for at least 15 minutes."