#!/usr/bin/env bash set -Eeuo pipefail project_root=${PLATFORM_ROOT:-/opt/data-terminal-cloud/current} backup_root=${BACKUP_ROOT:-/var/backups/data-terminal-cloud} state_dir=${MONITOR_STATE_DIR:-/var/lib/data-terminal-cloud-monitor} monitor_config=${MONITOR_CONFIG_FILE:-/etc/data-terminal-cloud/monitor.env} offsite_config_file=${OFFSITE_CONFIG_FILE:-/etc/data-terminal-cloud/offsite-backup.env} backup_max_age_seconds=${BACKUP_MAX_AGE_SECONDS:-129600} certificate_min_seconds=${CERTIFICATE_MIN_SECONDS:-129600} disk_warning_percent=${DISK_WARNING_PERCENT:-80} disk_critical_percent=${DISK_CRITICAL_PERCENT:-90} if [[ -f "$monitor_config" ]]; then # The installer creates this file as root:root 0600. # shellcheck disable=SC1090 source "$monitor_config" fi alert_webhook_url=${ALERT_WEBHOOK_URL:-} alert_webhook_secret_file=${ALERT_WEBHOOK_SECRET_FILE:-/etc/data-terminal-cloud/monitor-webhook-secret} if [[ -f "$offsite_config_file" ]]; then # shellcheck disable=SC1090 source "$offsite_config_file" fi offsite_enabled=${OFFSITE_BACKUP_ENABLED:-false} offsite_required=${OFFSITE_BACKUP_REQUIRED:-false} offsite_state_file=${OFFSITE_STATE_FILE:-/var/lib/data-terminal-cloud-offsite/status.json} offsite_max_age_seconds=${OFFSITE_MAX_AGE_SECONDS:-129600} for numeric in "$backup_max_age_seconds" "$certificate_min_seconds" "$disk_warning_percent" "$disk_critical_percent" "$offsite_max_age_seconds"; do [[ "$numeric" =~ ^[0-9]+$ ]] || { echo "Monitor thresholds must be non-negative integers" >&2; exit 2; } done if (( disk_warning_percent >= disk_critical_percent || disk_critical_percent > 100 )); then echo "Disk thresholds are invalid" >&2 exit 2 fi compose=(docker compose -f "$project_root/compose.yaml" -f "$project_root/deploy/server/compose.override.yaml" --env-file "$project_root/deploy/server/.env") mkdir -p "$state_dir" chmod 0750 "$state_dir" umask 077 checks_file=$(mktemp "$state_dir/.checks-XXXXXX") status_tmp=$(mktemp "$state_dir/.status-XXXXXX") oidc_tmp=$(mktemp "$state_dir/.oidc-XXXXXX") curl_config=$(mktemp "$state_dir/.curl-XXXXXX") status_file="$state_dir/status.json" critical_count=0 warning_count=0 cleanup() { rm -f -- "$checks_file" "$status_tmp" "$oidc_tmp" "$curl_config" } trap cleanup EXIT INT TERM record_check() { local name=$1 local status=$2 local value=${3:-} value=${value//$'\t'/ } value=${value//$'\r'/ } value=${value//$'\n'/ } printf '%s\t%s\t%s\n' "$name" "$status" "$value" >>"$checks_file" case "$status" in critical) critical_count=$((critical_count + 1)) ;; warning) warning_count=$((warning_count + 1)) ;; esac } public_origin=$(sed -n 's/^PUBLIC_ORIGIN=//p' "$project_root/deploy/server/.env") if [[ "$public_origin" == https://* ]]; then record_check public_origin healthy "$public_origin" else record_check public_origin critical invalid_https_origin fi if curl --fail --silent --show-error --max-time 10 "$public_origin/api/v1/version" >/dev/null; then record_check public_https healthy http_200 else record_check public_https critical request_failed fi expected_issuer="$public_origin/realms/voice-local" if curl --fail --silent --show-error --max-time 10 \ "$expected_issuer/.well-known/openid-configuration" -o "$oidc_tmp"; then actual_issuer=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1], encoding="utf-8")).get("issuer", ""))' "$oidc_tmp" 2>/dev/null || true) if [[ "$actual_issuer" == "$expected_issuer" ]]; then record_check oidc_issuer healthy issuer_matches else record_check oidc_issuer critical issuer_mismatch fi else record_check oidc_issuer critical request_failed fi required_services=(postgres redis minio keycloak api callbacks worker web gateway mock-provider mock-alert-receiver) for service in "${required_services[@]}"; do container=$("${compose[@]}" ps -q "$service") if [[ -z "$container" ]]; then record_check "container_$service" critical missing continue fi state=$(docker inspect -f '{{.State.Status}}' "$container") health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container") if [[ "$state" != running ]]; then record_check "container_$service" critical "$state" elif [[ "$health" != none && "$health" != healthy ]]; then record_check "container_$service" critical "$health" else record_check "container_$service" healthy "$state/$health" fi done latest_backup=$(find "$backup_root" -maxdepth 1 -type f -name 'platform-*.tar.gz.enc' -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n1 | cut -d' ' -f2- || true) if [[ -z "$latest_backup" || ! -f "$latest_backup" ]]; then record_check backup_freshness critical missing else now_epoch=$(date +%s) backup_epoch=$(stat -c %Y "$latest_backup") backup_age=$((now_epoch - backup_epoch)) if (( backup_age > backup_max_age_seconds )); then record_check backup_freshness critical "age_seconds=$backup_age" elif [[ ! -f "$latest_backup.sha256" ]] || ! (cd "$backup_root" && sha256sum -c "$(basename "$latest_backup").sha256" >/dev/null 2>&1); then record_check backup_freshness critical checksum_failed else record_check backup_freshness healthy "age_seconds=$backup_age" fi fi if [[ "$offsite_enabled" == true || "$offsite_required" == true ]]; then if [[ ! -r "$offsite_state_file" ]]; then record_check offsite_backup critical status_missing else offsite_evidence=$(python3 -c 'import json,sys try: d=json.load(open(sys.argv[1], encoding="utf-8")); print("|".join([str(d.get("status", "")), str(d.get("verified_epoch", 0)), str(d.get("source_archive", ""))])) except Exception: print("invalid|0|")' "$offsite_state_file") IFS='|' read -r offsite_status offsite_epoch offsite_source <<<"$offsite_evidence" offsite_age=$(($(date +%s) - offsite_epoch)) if [[ "$offsite_status" != verified ]]; then record_check offsite_backup critical "status=${offsite_status:-invalid}" elif [[ "$offsite_source" != "$(basename "${latest_backup:-missing}")" ]]; then record_check offsite_backup critical source_mismatch elif (( offsite_epoch <= 0 || offsite_age > offsite_max_age_seconds )); then record_check offsite_backup critical "age_seconds=$offsite_age" else record_check offsite_backup healthy "age_seconds=$offsite_age" fi fi else record_check offsite_backup healthy not_required fi backup_result=$(systemctl show platform-backup.service -p Result --value 2>/dev/null || true) backup_exit=$(systemctl show platform-backup.service -p ExecMainStatus --value 2>/dev/null || true) if [[ "$backup_result" == success && "$backup_exit" == 0 ]]; then record_check backup_service healthy success else record_check backup_service critical "result=${backup_result:-unknown}/exit=${backup_exit:-unknown}" fi timers=(platform-backup.timer certbot-ip-renew.timer) if [[ -f "$offsite_config_file" ]]; then timers+=(platform-offsite-backup.timer) fi for timer in "${timers[@]}"; do timer_enabled=$(systemctl is-enabled "$timer" 2>/dev/null || true) timer_active=$(systemctl is-active "$timer" 2>/dev/null || true) if [[ "$timer_enabled" == enabled && "$timer_active" == active ]]; then record_check "$timer" healthy enabled/active else record_check "$timer" critical "${timer_enabled:-unknown}/${timer_active:-unknown}" fi done public_hostname=$(python3 -c 'import sys,urllib.parse; print(urllib.parse.urlsplit(sys.argv[1]).hostname or "")' "$public_origin") certificate_file=${TLS_CERTIFICATE_FILE:-/etc/letsencrypt/live/$public_hostname/fullchain.pem} if [[ ! -r "$certificate_file" ]]; then record_check certificate critical missing else certificate_end=$(openssl x509 -in "$certificate_file" -noout -enddate | cut -d= -f2-) certificate_end_epoch=$(date -d "$certificate_end" +%s) certificate_seconds=$((certificate_end_epoch - $(date +%s))) if (( certificate_seconds < certificate_min_seconds )); then record_check certificate critical "remaining_seconds=$certificate_seconds" else record_check certificate healthy "remaining_seconds=$certificate_seconds" fi fi disk_used=$(df -P "$backup_root" | awk 'NR==2 {gsub(/%/, "", $5); print $5}') if [[ ! "$disk_used" =~ ^[0-9]+$ ]]; then record_check disk_usage critical unreadable elif (( disk_used >= disk_critical_percent )); then record_check disk_usage critical "used_percent=$disk_used" elif (( disk_used >= disk_warning_percent )); then record_check disk_usage warning "used_percent=$disk_used" else record_check disk_usage healthy "used_percent=$disk_used" fi if (( critical_count > 0 )); then overall=critical elif (( warning_count > 0 )); then overall=warning else overall=healthy fi observed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) previous_status=$(python3 -c 'import json,sys try: print(json.load(open(sys.argv[1], encoding="utf-8")).get("status", "unknown")) except Exception: print("unknown")' "$status_file" 2>/dev/null || true) python3 - "$checks_file" "$status_tmp" "$overall" "$observed_at" <<'PY' import json import sys checks = [] with open(sys.argv[1], encoding="utf-8") as source: for line in source: name, status, value = line.rstrip("\n").split("\t", 2) checks.append({"name": name, "status": status, "value": value}) payload = { "schema": "platform-server-health-v1", "status": sys.argv[3], "observed_at": sys.argv[4], "checks": checks, } with open(sys.argv[2], "w", encoding="utf-8") as target: json.dump(payload, target, ensure_ascii=False, separators=(",", ":")) target.write("\n") PY chmod 0644 "$status_tmp" mv -f "$status_tmp" "$status_file" if [[ -n "$alert_webhook_url" && "$previous_status" != "$overall" ]]; then if [[ ! -r "$alert_webhook_secret_file" ]]; then record_check alert_delivery critical secret_missing echo "Alert webhook is configured but its HMAC secret is unreadable" >&2 exit 5 fi timestamp=$(date +%s) signature=$(python3 - "$alert_webhook_secret_file" "$status_file" "$timestamp" <<'PY' import hashlib import hmac import sys secret = open(sys.argv[1], "rb").read().rstrip(b"\r\n") body = open(sys.argv[2], "rb").read() print(hmac.new(secret, sys.argv[3].encode() + b"." + body, hashlib.sha256).hexdigest()) PY ) python3 - "$alert_webhook_url" "$curl_config" <<'PY' import sys url = sys.argv[1].replace("\\", "\\\\").replace('"', '\\"') open(sys.argv[2], "w", encoding="utf-8").write(f'url = "{url}"\n') PY curl --fail --silent --show-error --max-time 10 --config "$curl_config" \ -H 'Content-Type: application/json' \ -H "X-Platform-Timestamp: $timestamp" \ -H "X-Platform-Signature: sha256=$signature" \ --data-binary "@$status_file" >/dev/null fi printf 'platform_health status=%s critical=%s warning=%s status_file=%s\n' \ "$overall" "$critical_count" "$warning_count" "$status_file" if (( critical_count > 0 )); then exit 1 fi