groups: - name: data-terminal-cloud-availability rules: - alert: PlatformServiceDown expr: up{job="voice-platform"} == 0 for: 2m labels: severity: critical annotations: summary: "Platform service is unavailable" description: "Prometheus has failed to scrape {{ $labels.instance }} for more than two minutes." - alert: PlatformHighHTTP5xxRate expr: | ( sum by (service) (rate(voice_platform_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum by (service) (rate(voice_platform_http_requests_total[5m])), 0.01) ) > 0.05 and sum by (service) (rate(voice_platform_http_requests_total[5m])) > 0.1 for: 5m labels: severity: critical annotations: summary: "HTTP 5xx rate exceeds 5%" description: "{{ $labels.service }} has sustained a 5xx rate above 5%." - alert: PlatformHighP95Latency expr: | histogram_quantile( 0.95, sum by (service, route, le) (rate(voice_platform_http_request_duration_seconds_bucket[5m])) ) > 1.5 for: 10m labels: severity: warning annotations: summary: "HTTP p95 latency exceeds 1.5 seconds" description: "{{ $labels.service }} route {{ $labels.route }} has sustained high latency." - alert: PlatformExternalDependencyUnavailable expr: voice_platform_dependency_up{dependency=~"oidc_jwks|ark_tls|volc_rtc_tls|content_safety_tls"} == 0 for: 5m labels: severity: critical annotations: summary: "External platform dependency is unavailable" description: "{{ $labels.service }} cannot reach or validate {{ $labels.dependency }} for more than five minutes." - name: data-terminal-cloud-workflows rules: - alert: OutboxDeadLetterPresent expr: voice_platform_outbox_events{status="dead"} > 0 for: 2m labels: severity: critical annotations: summary: "Outbox dead-letter events require intervention" description: "At least one reliable background task exhausted its retry budget." - alert: OutboxPendingBacklog expr: voice_platform_outbox_events{status="pending"} > 100 for: 10m labels: severity: warning annotations: summary: "Outbox pending backlog is growing" description: "More than 100 events have remained pending for ten minutes." - alert: PrivacyDeletionPartialFailure expr: voice_platform_privacy_requests{status="PARTIAL_FAILED"} > 0 for: 2m labels: severity: critical annotations: summary: "A privacy request has partially failed" description: "A subject-level privacy workflow requires retry or manual reconciliation." - alert: FirmwareAssignmentFailure expr: voice_platform_firmware_assignments{status="FAILED"} > 0 for: 2m labels: severity: warning annotations: summary: "A firmware assignment failed" description: "Pause the rollout, inspect the device result, and select a verified rollback release if necessary." - alert: FirmwareRolloutStalled expr: voice_platform_firmware_stalled_rollouts > 0 for: 5m labels: severity: warning annotations: summary: "A firmware rollout is stalled" description: "A running rollout has not changed for at least 15 minutes."