97 lines
3.6 KiB
YAML
97 lines
3.6 KiB
YAML
|
groups:
|
||
|
|
- name: data-terminal-cloud-availability
|
||
|
|
rules:
|
||
|
|
- alert: PlatformServiceDown
|
||
|
|
expr: up{job="voice-platform"} == 0
|
||
|
|
for: 2m
|
||
|
|
labels:
|
||
|
|
severity: critical
|
||
|
|
annotations:
|
||
|
|
summary: "Platform service is unavailable"
|
||
|
|
description: "Prometheus has failed to scrape {{ $labels.instance }} for more than two minutes."
|
||
|
|
|
||
|
|
- alert: PlatformHighHTTP5xxRate
|
||
|
|
expr: |
|
||
|
|
(
|
||
|
|
sum by (service) (rate(voice_platform_http_requests_total{status=~"5.."}[5m]))
|
||
|
|
/
|
||
|
|
clamp_min(sum by (service) (rate(voice_platform_http_requests_total[5m])), 0.01)
|
||
|
|
) > 0.05
|
||
|
|
and
|
||
|
|
sum by (service) (rate(voice_platform_http_requests_total[5m])) > 0.1
|
||
|
|
for: 5m
|
||
|
|
labels:
|
||
|
|
severity: critical
|
||
|
|
annotations:
|
||
|
|
summary: "HTTP 5xx rate exceeds 5%"
|
||
|
|
description: "{{ $labels.service }} has sustained a 5xx rate above 5%."
|
||
|
|
|
||
|
|
- alert: PlatformHighP95Latency
|
||
|
|
expr: |
|
||
|
|
histogram_quantile(
|
||
|
|
0.95,
|
||
|
|
sum by (service, route, le) (rate(voice_platform_http_request_duration_seconds_bucket[5m]))
|
||
|
|
) > 1.5
|
||
|
|
for: 10m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
annotations:
|
||
|
|
summary: "HTTP p95 latency exceeds 1.5 seconds"
|
||
|
|
description: "{{ $labels.service }} route {{ $labels.route }} has sustained high latency."
|
||
|
|
|
||
|
|
- alert: PlatformExternalDependencyUnavailable
|
||
|
|
expr: voice_platform_dependency_up{dependency=~"oidc_jwks|ark_tls|volc_rtc_tls|content_safety_tls"} == 0
|
||
|
|
for: 5m
|
||
|
|
labels:
|
||
|
|
severity: critical
|
||
|
|
annotations:
|
||
|
|
summary: "External platform dependency is unavailable"
|
||
|
|
description: "{{ $labels.service }} cannot reach or validate {{ $labels.dependency }} for more than five minutes."
|
||
|
|
|
||
|
|
- name: data-terminal-cloud-workflows
|
||
|
|
rules:
|
||
|
|
- alert: OutboxDeadLetterPresent
|
||
|
|
expr: voice_platform_outbox_events{status="dead"} > 0
|
||
|
|
for: 2m
|
||
|
|
labels:
|
||
|
|
severity: critical
|
||
|
|
annotations:
|
||
|
|
summary: "Outbox dead-letter events require intervention"
|
||
|
|
description: "At least one reliable background task exhausted its retry budget."
|
||
|
|
|
||
|
|
- alert: OutboxPendingBacklog
|
||
|
|
expr: voice_platform_outbox_events{status="pending"} > 100
|
||
|
|
for: 10m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
annotations:
|
||
|
|
summary: "Outbox pending backlog is growing"
|
||
|
|
description: "More than 100 events have remained pending for ten minutes."
|
||
|
|
|
||
|
|
- alert: PrivacyDeletionPartialFailure
|
||
|
|
expr: voice_platform_privacy_requests{status="PARTIAL_FAILED"} > 0
|
||
|
|
for: 2m
|
||
|
|
labels:
|
||
|
|
severity: critical
|
||
|
|
annotations:
|
||
|
|
summary: "A privacy request has partially failed"
|
||
|
|
description: "A subject-level privacy workflow requires retry or manual reconciliation."
|
||
|
|
|
||
|
|
- alert: FirmwareAssignmentFailure
|
||
|
|
expr: voice_platform_firmware_assignments{status="FAILED"} > 0
|
||
|
|
for: 2m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
annotations:
|
||
|
|
summary: "A firmware assignment failed"
|
||
|
|
description: "Pause the rollout, inspect the device result, and select a verified rollback release if necessary."
|
||
|
|
|
||
|
|
- alert: FirmwareRolloutStalled
|
||
|
|
expr: voice_platform_firmware_stalled_rollouts > 0
|
||
|
|
for: 5m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
annotations:
|
||
|
|
summary: "A firmware rollout is stalled"
|
||
|
|
description: "A running rollout has not changed for at least 15 minutes."
|