story/deploy/server/health-monitor.sh

#!/usr/bin/env bash
set -Eeuo pipefail

project_root=${PLATFORM_ROOT:-/opt/data-terminal-cloud/current}
backup_root=${BACKUP_ROOT:-/var/backups/data-terminal-cloud}
state_dir=${MONITOR_STATE_DIR:-/var/lib/data-terminal-cloud-monitor}
monitor_config=${MONITOR_CONFIG_FILE:-/etc/data-terminal-cloud/monitor.env}
offsite_config_file=${OFFSITE_CONFIG_FILE:-/etc/data-terminal-cloud/offsite-backup.env}
backup_max_age_seconds=${BACKUP_MAX_AGE_SECONDS:-129600}
certificate_min_seconds=${CERTIFICATE_MIN_SECONDS:-129600}
disk_warning_percent=${DISK_WARNING_PERCENT:-80}
disk_critical_percent=${DISK_CRITICAL_PERCENT:-90}

if [[ -f "$monitor_config" ]]; then
  # The installer creates this file as root:root 0600.
  # shellcheck disable=SC1090
  source "$monitor_config"
fi
alert_webhook_url=${ALERT_WEBHOOK_URL:-}
alert_webhook_secret_file=${ALERT_WEBHOOK_SECRET_FILE:-/etc/data-terminal-cloud/monitor-webhook-secret}
platform_health_report_url=${PLATFORM_HEALTH_REPORT_URL:-}
platform_health_report_secret_file=${PLATFORM_HEALTH_REPORT_SECRET_FILE:-/etc/data-terminal-cloud/platform-health-callback-secret}
expected_public_tcp_ports=${EXPECTED_PUBLIC_TCP_PORTS:-22,80,443}
deployment_profile=${HOST_DEPLOYMENT_PROFILE:-demo}
if [[ -f "$offsite_config_file" ]]; then
  # shellcheck disable=SC1090
  source "$offsite_config_file"
fi
offsite_enabled=${OFFSITE_BACKUP_ENABLED:-false}
offsite_required=${OFFSITE_BACKUP_REQUIRED:-false}
offsite_state_file=${OFFSITE_STATE_FILE:-/var/lib/data-terminal-cloud-offsite/status.json}
offsite_max_age_seconds=${OFFSITE_MAX_AGE_SECONDS:-129600}

for numeric in "$backup_max_age_seconds" "$certificate_min_seconds" "$disk_warning_percent" "$disk_critical_percent" "$offsite_max_age_seconds"; do
  [[ "$numeric" =~ ^[0-9]+$ ]] || { echo "Monitor thresholds must be non-negative integers" >&2; exit 2; }
done
if (( disk_warning_percent >= disk_critical_percent || disk_critical_percent > 100 )); then
  echo "Disk thresholds are invalid" >&2
  exit 2
fi
if [[ ! "$expected_public_tcp_ports" =~ ^[0-9]+(,[0-9]+)*$ ]]; then
  echo "EXPECTED_PUBLIC_TCP_PORTS must contain comma-separated integers" >&2
  exit 2
fi
if [[ "$deployment_profile" != demo && "$deployment_profile" != production ]]; then
  echo "HOST_DEPLOYMENT_PROFILE must be demo or production" >&2
  exit 2
fi

compose=(docker compose -f "$project_root/compose.yaml" -f "$project_root/deploy/server/compose.override.yaml" --env-file "$project_root/deploy/server/.env")
mkdir -p "$state_dir"
chmod 0750 "$state_dir"
umask 077
checks_file=$(mktemp "$state_dir/.checks-XXXXXX")
status_tmp=$(mktemp "$state_dir/.status-XXXXXX")
oidc_tmp=$(mktemp "$state_dir/.oidc-XXXXXX")
curl_config=$(mktemp "$state_dir/.curl-XXXXXX")
status_file="$state_dir/status.json"
critical_count=0
warning_count=0

cleanup() {
  rm -f -- "$checks_file" "$status_tmp" "$oidc_tmp" "$curl_config"
}
trap cleanup EXIT INT TERM

record_check() {
  local name=$1
  local status=$2
  local value=${3:-}
  value=${value//$'\t'/ }
  value=${value//$'\r'/ }
  value=${value//$'\n'/ }
  printf '%s\t%s\t%s\n' "$name" "$status" "$value" >>"$checks_file"
  case "$status" in
    critical) critical_count=$((critical_count + 1)) ;;
    warning) warning_count=$((warning_count + 1)) ;;
  esac
}

record_check deployment_profile healthy "$deployment_profile"
if [[ -z "$alert_webhook_url" ]]; then
  if [[ "$deployment_profile" == production ]]; then
    record_check external_alert_receiver critical not_configured
  else
    record_check external_alert_receiver healthy not_configured
  fi
elif [[ ! -r "$alert_webhook_secret_file" ]]; then
  record_check external_alert_receiver critical secret_missing
elif (( $(tr -d '\r\n' <"$alert_webhook_secret_file" | wc -c) < 32 )); then
  record_check external_alert_receiver critical secret_invalid
else
  record_check external_alert_receiver healthy configured
fi

public_origin=$(sed -n 's/^PUBLIC_ORIGIN=//p' "$project_root/deploy/server/.env")
if [[ "$public_origin" == https://* ]]; then
  record_check public_origin healthy "$public_origin"
else
  record_check public_origin critical invalid_https_origin
fi

if curl --fail --silent --show-error --max-time 10 "$public_origin/api/v1/version" >/dev/null; then
  record_check public_https healthy http_200
else
  record_check public_https critical request_failed
fi

if ! command -v ss >/dev/null 2>&1; then
  record_check public_tcp_listeners critical probe_unavailable
else
  listener_evidence=$(ss -H -ltn | python3 "$project_root/deploy/server/public_listener_policy.py" "$expected_public_tcp_ports" 2>/dev/null || true)
  IFS='|' read -r listener_status listener_detail <<<"$listener_evidence"
  if [[ "$listener_status" == healthy ]]; then
    record_check public_tcp_listeners healthy "$listener_detail"
  elif [[ "$listener_status" == critical ]]; then
    record_check public_tcp_listeners critical "$listener_detail"
  else
    record_check public_tcp_listeners critical invalid_probe_result
  fi
fi

expected_issuer="$public_origin/realms/voice-local"
if curl --fail --silent --show-error --max-time 10 \
  "$expected_issuer/.well-known/openid-configuration" -o "$oidc_tmp"; then
  actual_issuer=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1], encoding="utf-8")).get("issuer", ""))' "$oidc_tmp" 2>/dev/null || true)
  if [[ "$actual_issuer" == "$expected_issuer" ]]; then
    record_check oidc_issuer healthy issuer_matches
  else
    record_check oidc_issuer critical issuer_mismatch
  fi
else
  record_check oidc_issuer critical request_failed
fi

required_services=(postgres redis minio keycloak api callbacks worker web gateway)
if [[ "$deployment_profile" == demo ]]; then
  required_services+=(mock-provider mock-alert-receiver)
fi
for service in "${required_services[@]}"; do
  container=$("${compose[@]}" ps -q "$service")
  if [[ -z "$container" ]]; then
    record_check "container_$service" critical missing
    continue
  fi
  state=$(docker inspect -f '{{.State.Status}}' "$container")
  health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
  if [[ "$state" != running ]]; then
    record_check "container_$service" critical "$state"
  elif [[ "$health" != none && "$health" != healthy ]]; then
    record_check "container_$service" critical "$health"
  else
    record_check "container_$service" healthy "$state/$health"
  fi
done

application_environments=()
application_environment_missing=false
for application_service in api callbacks worker; do
  application_container=$("${compose[@]}" ps -q "$application_service")
  if [[ -z "$application_container" ]]; then
    application_environment_missing=true
    continue
  fi
  application_environment=$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$application_container" | sed -n 's/^APP_ENV=//p' | head -n1)
  if [[ -z "$application_environment" ]]; then
    application_environment_missing=true
  else
    application_environments+=("$application_environment")
  fi
done
if [[ "$application_environment_missing" == true || ${#application_environments[@]} -ne 3 ]]; then
  record_check application_environment critical missing
elif [[ "${application_environments[0]}" != "${application_environments[1]}" || "${application_environments[0]}" != "${application_environments[2]}" ]]; then
  record_check application_environment critical mixed
elif [[ "$deployment_profile" == production && "${application_environments[0]}" != prod ]]; then
  record_check application_environment critical "${application_environments[0]}"
elif [[ "$deployment_profile" == demo && "${application_environments[0]}" != local ]]; then
  record_check application_environment critical "${application_environments[0]}"
else
  record_check application_environment healthy "${application_environments[0]}"
fi

keycloak_container=$("${compose[@]}" ps -q keycloak)
if [[ -z "$keycloak_container" ]]; then
  record_check identity_service_mode critical missing
elif docker inspect -f '{{json .Config.Cmd}}' "$keycloak_container" | grep -q 'start-dev'; then
  if [[ "$deployment_profile" == production ]]; then
    record_check identity_service_mode critical development
  else
    record_check identity_service_mode healthy development
  fi
else
  record_check identity_service_mode healthy hardened
fi

mock_services_running=()
for mock_service in mock-provider mock-alert-receiver; do
  mock_container=$("${compose[@]}" ps -q "$mock_service")
  if [[ -n "$mock_container" && $(docker inspect -f '{{.State.Status}}' "$mock_container") == running ]]; then
    mock_services_running+=("$mock_service")
  fi
done
if [[ "$deployment_profile" == production ]]; then
  if (( ${#mock_services_running[@]} > 0 )); then
    record_check mock_services critical unexpected_running
  else
    record_check mock_services healthy disabled
  fi
elif (( ${#mock_services_running[@]} == 2 )); then
  record_check mock_services healthy enabled_for_demo
else
  record_check mock_services critical incomplete_demo_stack
fi

latest_backup=$(find "$backup_root" -maxdepth 1 -type f -name 'platform-*.tar.gz.enc' -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n1 | cut -d' ' -f2- || true)
if [[ -z "$latest_backup" || ! -f "$latest_backup" ]]; then
  record_check backup_freshness critical missing
else
  now_epoch=$(date +%s)
  backup_epoch=$(stat -c %Y "$latest_backup")
  backup_age=$((now_epoch - backup_epoch))
  if (( backup_age > backup_max_age_seconds )); then
    record_check backup_freshness critical "age_seconds=$backup_age"
  elif [[ ! -f "$latest_backup.sha256" ]] || ! (cd "$backup_root" && sha256sum -c "$(basename "$latest_backup").sha256" >/dev/null 2>&1); then
    record_check backup_freshness critical checksum_failed
  else
    record_check backup_freshness healthy "age_seconds=$backup_age"
  fi
fi

if [[ "$offsite_enabled" == true || "$offsite_required" == true ]]; then
  if [[ ! -r "$offsite_state_file" ]]; then
    record_check offsite_backup critical status_missing
  else
    offsite_evidence=$(python3 -c 'import json,sys
try:
 d=json.load(open(sys.argv[1], encoding="utf-8")); print("|".join([str(d.get("status", "")), str(d.get("verified_epoch", 0)), str(d.get("source_archive", ""))]))
except Exception: print("invalid|0|")' "$offsite_state_file")
    IFS='|' read -r offsite_status offsite_epoch offsite_source <<<"$offsite_evidence"
    offsite_age=$(($(date +%s) - offsite_epoch))
    if [[ "$offsite_status" != verified ]]; then
      record_check offsite_backup critical "status=${offsite_status:-invalid}"
    elif [[ "$offsite_source" != "$(basename "${latest_backup:-missing}")" ]]; then
      record_check offsite_backup critical source_mismatch
    elif (( offsite_epoch <= 0 || offsite_age > offsite_max_age_seconds )); then
      record_check offsite_backup critical "age_seconds=$offsite_age"
    else
      record_check offsite_backup healthy "age_seconds=$offsite_age"
    fi
  fi
else
  if [[ "$deployment_profile" == production ]]; then
    record_check offsite_backup critical not_required
  else
    record_check offsite_backup healthy not_required
  fi
fi

backup_result=$(systemctl show platform-backup.service -p Result --value 2>/dev/null || true)
backup_exit=$(systemctl show platform-backup.service -p ExecMainStatus --value 2>/dev/null || true)
if [[ "$backup_result" == success && "$backup_exit" == 0 ]]; then
  record_check backup_service healthy success
else
  record_check backup_service critical "result=${backup_result:-unknown}/exit=${backup_exit:-unknown}"
fi

timers=(platform-backup.timer certbot-ip-renew.timer)
if [[ -f "$offsite_config_file" ]]; then
  timers+=(platform-offsite-backup.timer)
fi
for timer in "${timers[@]}"; do
  timer_enabled=$(systemctl is-enabled "$timer" 2>/dev/null || true)
  timer_active=$(systemctl is-active "$timer" 2>/dev/null || true)
  if [[ "$timer_enabled" == enabled && "$timer_active" == active ]]; then
    record_check "$timer" healthy enabled/active
  else
    record_check "$timer" critical "${timer_enabled:-unknown}/${timer_active:-unknown}"
  fi
done

public_hostname=$(python3 -c 'import sys,urllib.parse; print(urllib.parse.urlsplit(sys.argv[1]).hostname or "")' "$public_origin")
certificate_file=${TLS_CERTIFICATE_FILE:-/etc/letsencrypt/live/$public_hostname/fullchain.pem}
if [[ ! -r "$certificate_file" ]]; then
  record_check certificate critical missing
else
  certificate_end=$(openssl x509 -in "$certificate_file" -noout -enddate | cut -d= -f2-)
  certificate_end_epoch=$(date -d "$certificate_end" +%s)
  certificate_seconds=$((certificate_end_epoch - $(date +%s)))
  if (( certificate_seconds < certificate_min_seconds )); then
    record_check certificate critical "remaining_seconds=$certificate_seconds"
  else
    record_check certificate healthy "remaining_seconds=$certificate_seconds"
  fi
fi

disk_used=$(df -P "$backup_root" | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
if [[ ! "$disk_used" =~ ^[0-9]+$ ]]; then
  record_check disk_usage critical unreadable
elif (( disk_used >= disk_critical_percent )); then
  record_check disk_usage critical "used_percent=$disk_used"
elif (( disk_used >= disk_warning_percent )); then
  record_check disk_usage warning "used_percent=$disk_used"
else
  record_check disk_usage healthy "used_percent=$disk_used"
fi

if (( critical_count > 0 )); then
  overall=critical
elif (( warning_count > 0 )); then
  overall=warning
else
  overall=healthy
fi
observed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
previous_status=$(python3 -c 'import json,sys
try: print(json.load(open(sys.argv[1], encoding="utf-8")).get("status", "unknown"))
except Exception: print("unknown")' "$status_file" 2>/dev/null || true)

python3 - "$checks_file" "$status_tmp" "$overall" "$observed_at" <<'PY'
import json
import sys

checks = []
with open(sys.argv[1], encoding="utf-8") as source:
    for line in source:
        name, status, value = line.rstrip("\n").split("\t", 2)
        checks.append({"name": name, "status": status, "value": value})
payload = {
    "schema": "platform-server-health-v1",
    "status": sys.argv[3],
    "observed_at": sys.argv[4],
    "checks": checks,
}
with open(sys.argv[2], "w", encoding="utf-8") as target:
    json.dump(payload, target, ensure_ascii=False, separators=(",", ":"))
    target.write("\n")
PY
chmod 0644 "$status_tmp"
mv -f "$status_tmp" "$status_file"

post_signed_status() {
  local target_url=$1
  local secret_file=$2
  local timestamp signature
  timestamp=$(date +%s)
  signature=$(python3 - "$secret_file" "$status_file" "$timestamp" <<'PY'
import hashlib
import hmac
import sys

secret = open(sys.argv[1], "rb").read().rstrip(b"\r\n")
body = open(sys.argv[2], "rb").read()
print(hmac.new(secret, sys.argv[3].encode() + b"." + body, hashlib.sha256).hexdigest())
PY
)
  python3 - "$target_url" "$curl_config" <<'PY'
import sys

url = sys.argv[1].replace("\\", "\\\\").replace('"', '\\"')
open(sys.argv[2], "w", encoding="utf-8").write(f'url = "{url}"\n')
PY
  curl --fail --silent --show-error --max-time 10 --config "$curl_config" \
    -H 'Content-Type: application/json' \
    -H "X-Platform-Timestamp: $timestamp" \
    -H "X-Platform-Signature: sha256=$signature" \
    --data-binary "@$status_file" >/dev/null
}

if [[ -n "$platform_health_report_url" ]]; then
  if [[ ! -r "$platform_health_report_secret_file" ]]; then
    echo "Platform health reporting is configured but its HMAC secret is unreadable" >&2
    exit 6
  fi
  post_signed_status "$platform_health_report_url" "$platform_health_report_secret_file"
fi

if [[ -n "$alert_webhook_url" && "$previous_status" != "$overall" ]]; then
  if [[ ! -r "$alert_webhook_secret_file" ]]; then
    record_check alert_delivery critical secret_missing
    echo "Alert webhook is configured but its HMAC secret is unreadable" >&2
    exit 5
  fi
  post_signed_status "$alert_webhook_url" "$alert_webhook_secret_file"
fi

printf 'platform_health status=%s critical=%s warning=%s status_file=%s\n' \
  "$overall" "$critical_count" "$warning_count" "$status_file"
if (( critical_count > 0 )); then
  exit 1
fi