feat: monitor cloud server health

2026-08-26 20:41:26 +08:00
parent fb4691c7b2
commit 0bf375155e
8 changed files with 429 additions and 2 deletions
@@ -0,0 +1,232 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+project_root=${PLATFORM_ROOT:-/opt/data-terminal-cloud/current}
+backup_root=${BACKUP_ROOT:-/var/backups/data-terminal-cloud}
+state_dir=${MONITOR_STATE_DIR:-/var/lib/data-terminal-cloud-monitor}
+monitor_config=${MONITOR_CONFIG_FILE:-/etc/data-terminal-cloud/monitor.env}
+backup_max_age_seconds=${BACKUP_MAX_AGE_SECONDS:-129600}
+certificate_min_seconds=${CERTIFICATE_MIN_SECONDS:-129600}
+disk_warning_percent=${DISK_WARNING_PERCENT:-80}
+disk_critical_percent=${DISK_CRITICAL_PERCENT:-90}
+
+if [[ -f "$monitor_config" ]]; then
+  # The installer creates this file as root:root 0600.
+  # shellcheck disable=SC1090
+  source "$monitor_config"
+fi
+alert_webhook_url=${ALERT_WEBHOOK_URL:-}
+alert_webhook_secret_file=${ALERT_WEBHOOK_SECRET_FILE:-/etc/data-terminal-cloud/monitor-webhook-secret}
+
+for numeric in "$backup_max_age_seconds" "$certificate_min_seconds" "$disk_warning_percent" "$disk_critical_percent"; do
+  [[ "$numeric" =~ ^[0-9]+$ ]] || { echo "Monitor thresholds must be non-negative integers" >&2; exit 2; }
+done
+if (( disk_warning_percent >= disk_critical_percent || disk_critical_percent > 100 )); then
+  echo "Disk thresholds are invalid" >&2
+  exit 2
+fi
+
+compose=(docker compose -f "$project_root/compose.yaml" -f "$project_root/deploy/server/compose.override.yaml" --env-file "$project_root/deploy/server/.env")
+mkdir -p "$state_dir"
+chmod 0750 "$state_dir"
+umask 077
+checks_file=$(mktemp "$state_dir/.checks-XXXXXX")
+status_tmp=$(mktemp "$state_dir/.status-XXXXXX")
+oidc_tmp=$(mktemp "$state_dir/.oidc-XXXXXX")
+curl_config=$(mktemp "$state_dir/.curl-XXXXXX")
+status_file="$state_dir/status.json"
+critical_count=0
+warning_count=0
+
+cleanup() {
+  rm -f -- "$checks_file" "$status_tmp" "$oidc_tmp" "$curl_config"
+}
+trap cleanup EXIT INT TERM
+
+record_check() {
+  local name=$1
+  local status=$2
+  local value=${3:-}
+  value=${value//$'\t'/ }
+  value=${value//$'\r'/ }
+  value=${value//$'\n'/ }
+  printf '%s\t%s\t%s\n' "$name" "$status" "$value" >>"$checks_file"
+  case "$status" in
+    critical) critical_count=$((critical_count + 1)) ;;
+    warning) warning_count=$((warning_count + 1)) ;;
+  esac
+}
+
+public_origin=$(sed -n 's/^PUBLIC_ORIGIN=//p' "$project_root/deploy/server/.env")
+if [[ "$public_origin" == https://* ]]; then
+  record_check public_origin healthy "$public_origin"
+else
+  record_check public_origin critical invalid_https_origin
+fi
+
+if curl --fail --silent --show-error --max-time 10 "$public_origin/api/v1/version" >/dev/null; then
+  record_check public_https healthy http_200
+else
+  record_check public_https critical request_failed
+fi
+
+expected_issuer="$public_origin/realms/voice-local"
+if curl --fail --silent --show-error --max-time 10 \
+  "$expected_issuer/.well-known/openid-configuration" -o "$oidc_tmp"; then
+  actual_issuer=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1], encoding="utf-8")).get("issuer", ""))' "$oidc_tmp" 2>/dev/null || true)
+  if [[ "$actual_issuer" == "$expected_issuer" ]]; then
+    record_check oidc_issuer healthy issuer_matches
+  else
+    record_check oidc_issuer critical issuer_mismatch
+  fi
+else
+  record_check oidc_issuer critical request_failed
+fi
+
+required_services=(postgres redis minio keycloak api callbacks worker web gateway mock-provider mock-alert-receiver)
+for service in "${required_services[@]}"; do
+  container=$("${compose[@]}" ps -q "$service")
+  if [[ -z "$container" ]]; then
+    record_check "container_$service" critical missing
+    continue
+  fi
+  state=$(docker inspect -f '{{.State.Status}}' "$container")
+  health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
+  if [[ "$state" != running ]]; then
+    record_check "container_$service" critical "$state"
+  elif [[ "$health" != none && "$health" != healthy ]]; then
+    record_check "container_$service" critical "$health"
+  else
+    record_check "container_$service" healthy "$state/$health"
+  fi
+done
+
+latest_backup=$(find "$backup_root" -maxdepth 1 -type f -name 'platform-*.tar.gz.enc' -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n1 | cut -d' ' -f2- || true)
+if [[ -z "$latest_backup" || ! -f "$latest_backup" ]]; then
+  record_check backup_freshness critical missing
+else
+  now_epoch=$(date +%s)
+  backup_epoch=$(stat -c %Y "$latest_backup")
+  backup_age=$((now_epoch - backup_epoch))
+  if (( backup_age > backup_max_age_seconds )); then
+    record_check backup_freshness critical "age_seconds=$backup_age"
+  elif [[ ! -f "$latest_backup.sha256" ]] || ! (cd "$backup_root" && sha256sum -c "$(basename "$latest_backup").sha256" >/dev/null 2>&1); then
+    record_check backup_freshness critical checksum_failed
+  else
+    record_check backup_freshness healthy "age_seconds=$backup_age"
+  fi
+fi
+
+backup_result=$(systemctl show platform-backup.service -p Result --value 2>/dev/null || true)
+backup_exit=$(systemctl show platform-backup.service -p ExecMainStatus --value 2>/dev/null || true)
+if [[ "$backup_result" == success && "$backup_exit" == 0 ]]; then
+  record_check backup_service healthy success
+else
+  record_check backup_service critical "result=${backup_result:-unknown}/exit=${backup_exit:-unknown}"
+fi
+
+for timer in platform-backup.timer certbot-ip-renew.timer; do
+  timer_enabled=$(systemctl is-enabled "$timer" 2>/dev/null || true)
+  timer_active=$(systemctl is-active "$timer" 2>/dev/null || true)
+  if [[ "$timer_enabled" == enabled && "$timer_active" == active ]]; then
+    record_check "$timer" healthy enabled/active
+  else
+    record_check "$timer" critical "${timer_enabled:-unknown}/${timer_active:-unknown}"
+  fi
+done
+
+public_hostname=$(python3 -c 'import sys,urllib.parse; print(urllib.parse.urlsplit(sys.argv[1]).hostname or "")' "$public_origin")
+certificate_file=${TLS_CERTIFICATE_FILE:-/etc/letsencrypt/live/$public_hostname/fullchain.pem}
+if [[ ! -r "$certificate_file" ]]; then
+  record_check certificate critical missing
+else
+  certificate_end=$(openssl x509 -in "$certificate_file" -noout -enddate | cut -d= -f2-)
+  certificate_end_epoch=$(date -d "$certificate_end" +%s)
+  certificate_seconds=$((certificate_end_epoch - $(date +%s)))
+  if (( certificate_seconds < certificate_min_seconds )); then
+    record_check certificate critical "remaining_seconds=$certificate_seconds"
+  else
+    record_check certificate healthy "remaining_seconds=$certificate_seconds"
+  fi
+fi
+
+disk_used=$(df -P "$backup_root" | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
+if [[ ! "$disk_used" =~ ^[0-9]+$ ]]; then
+  record_check disk_usage critical unreadable
+elif (( disk_used >= disk_critical_percent )); then
+  record_check disk_usage critical "used_percent=$disk_used"
+elif (( disk_used >= disk_warning_percent )); then
+  record_check disk_usage warning "used_percent=$disk_used"
+else
+  record_check disk_usage healthy "used_percent=$disk_used"
+fi
+
+if (( critical_count > 0 )); then
+  overall=critical
+elif (( warning_count > 0 )); then
+  overall=warning
+else
+  overall=healthy
+fi
+observed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+previous_status=$(python3 -c 'import json,sys
+try: print(json.load(open(sys.argv[1], encoding="utf-8")).get("status", "unknown"))
+except Exception: print("unknown")' "$status_file" 2>/dev/null || true)
+
+python3 - "$checks_file" "$status_tmp" "$overall" "$observed_at" <<'PY'
+import json
+import sys
+
+checks = []
+with open(sys.argv[1], encoding="utf-8") as source:
+    for line in source:
+        name, status, value = line.rstrip("\n").split("\t", 2)
+        checks.append({"name": name, "status": status, "value": value})
+payload = {
+    "schema": "platform-server-health-v1",
+    "status": sys.argv[3],
+    "observed_at": sys.argv[4],
+    "checks": checks,
+}
+with open(sys.argv[2], "w", encoding="utf-8") as target:
+    json.dump(payload, target, ensure_ascii=False, separators=(",", ":"))
+    target.write("\n")
+PY
+chmod 0644 "$status_tmp"
+mv -f "$status_tmp" "$status_file"
+
+if [[ -n "$alert_webhook_url" && "$previous_status" != "$overall" ]]; then
+  if [[ ! -r "$alert_webhook_secret_file" ]]; then
+    record_check alert_delivery critical secret_missing
+    echo "Alert webhook is configured but its HMAC secret is unreadable" >&2
+    exit 5
+  fi
+  timestamp=$(date +%s)
+  signature=$(python3 - "$alert_webhook_secret_file" "$status_file" "$timestamp" <<'PY'
+import hashlib
+import hmac
+import sys
+
+secret = open(sys.argv[1], "rb").read().rstrip(b"\r\n")
+body = open(sys.argv[2], "rb").read()
+print(hmac.new(secret, sys.argv[3].encode() + b"." + body, hashlib.sha256).hexdigest())
+PY
+)
+  python3 - "$alert_webhook_url" "$curl_config" <<'PY'
+import sys
+
+url = sys.argv[1].replace("\\", "\\\\").replace('"', '\\"')
+open(sys.argv[2], "w", encoding="utf-8").write(f'url = "{url}"\n')
+PY
+  curl --fail --silent --show-error --max-time 10 --config "$curl_config" \
+    -H 'Content-Type: application/json' \
+    -H "X-Platform-Timestamp: $timestamp" \
+    -H "X-Platform-Signature: sha256=$signature" \
+    --data-binary "@$status_file" >/dev/null
+fi
+
+printf 'platform_health status=%s critical=%s warning=%s status_file=%s\n' \
+  "$overall" "$critical_count" "$warning_count" "$status_file"
+if (( critical_count > 0 )); then
+  exit 1
+fi