233 lines
8.9 KiB
Bash
Executable File
233 lines
8.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
project_root=${PLATFORM_ROOT:-/opt/data-terminal-cloud/current}
|
|
backup_root=${BACKUP_ROOT:-/var/backups/data-terminal-cloud}
|
|
state_dir=${MONITOR_STATE_DIR:-/var/lib/data-terminal-cloud-monitor}
|
|
monitor_config=${MONITOR_CONFIG_FILE:-/etc/data-terminal-cloud/monitor.env}
|
|
backup_max_age_seconds=${BACKUP_MAX_AGE_SECONDS:-129600}
|
|
certificate_min_seconds=${CERTIFICATE_MIN_SECONDS:-129600}
|
|
disk_warning_percent=${DISK_WARNING_PERCENT:-80}
|
|
disk_critical_percent=${DISK_CRITICAL_PERCENT:-90}
|
|
|
|
if [[ -f "$monitor_config" ]]; then
|
|
# The installer creates this file as root:root 0600.
|
|
# shellcheck disable=SC1090
|
|
source "$monitor_config"
|
|
fi
|
|
alert_webhook_url=${ALERT_WEBHOOK_URL:-}
|
|
alert_webhook_secret_file=${ALERT_WEBHOOK_SECRET_FILE:-/etc/data-terminal-cloud/monitor-webhook-secret}
|
|
|
|
for numeric in "$backup_max_age_seconds" "$certificate_min_seconds" "$disk_warning_percent" "$disk_critical_percent"; do
|
|
[[ "$numeric" =~ ^[0-9]+$ ]] || { echo "Monitor thresholds must be non-negative integers" >&2; exit 2; }
|
|
done
|
|
if (( disk_warning_percent >= disk_critical_percent || disk_critical_percent > 100 )); then
|
|
echo "Disk thresholds are invalid" >&2
|
|
exit 2
|
|
fi
|
|
|
|
compose=(docker compose -f "$project_root/compose.yaml" -f "$project_root/deploy/server/compose.override.yaml" --env-file "$project_root/deploy/server/.env")
|
|
mkdir -p "$state_dir"
|
|
chmod 0750 "$state_dir"
|
|
umask 077
|
|
checks_file=$(mktemp "$state_dir/.checks-XXXXXX")
|
|
status_tmp=$(mktemp "$state_dir/.status-XXXXXX")
|
|
oidc_tmp=$(mktemp "$state_dir/.oidc-XXXXXX")
|
|
curl_config=$(mktemp "$state_dir/.curl-XXXXXX")
|
|
status_file="$state_dir/status.json"
|
|
critical_count=0
|
|
warning_count=0
|
|
|
|
cleanup() {
|
|
rm -f -- "$checks_file" "$status_tmp" "$oidc_tmp" "$curl_config"
|
|
}
|
|
trap cleanup EXIT INT TERM
|
|
|
|
record_check() {
|
|
local name=$1
|
|
local status=$2
|
|
local value=${3:-}
|
|
value=${value//$'\t'/ }
|
|
value=${value//$'\r'/ }
|
|
value=${value//$'\n'/ }
|
|
printf '%s\t%s\t%s\n' "$name" "$status" "$value" >>"$checks_file"
|
|
case "$status" in
|
|
critical) critical_count=$((critical_count + 1)) ;;
|
|
warning) warning_count=$((warning_count + 1)) ;;
|
|
esac
|
|
}
|
|
|
|
public_origin=$(sed -n 's/^PUBLIC_ORIGIN=//p' "$project_root/deploy/server/.env")
|
|
if [[ "$public_origin" == https://* ]]; then
|
|
record_check public_origin healthy "$public_origin"
|
|
else
|
|
record_check public_origin critical invalid_https_origin
|
|
fi
|
|
|
|
if curl --fail --silent --show-error --max-time 10 "$public_origin/api/v1/version" >/dev/null; then
|
|
record_check public_https healthy http_200
|
|
else
|
|
record_check public_https critical request_failed
|
|
fi
|
|
|
|
expected_issuer="$public_origin/realms/voice-local"
|
|
if curl --fail --silent --show-error --max-time 10 \
|
|
"$expected_issuer/.well-known/openid-configuration" -o "$oidc_tmp"; then
|
|
actual_issuer=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1], encoding="utf-8")).get("issuer", ""))' "$oidc_tmp" 2>/dev/null || true)
|
|
if [[ "$actual_issuer" == "$expected_issuer" ]]; then
|
|
record_check oidc_issuer healthy issuer_matches
|
|
else
|
|
record_check oidc_issuer critical issuer_mismatch
|
|
fi
|
|
else
|
|
record_check oidc_issuer critical request_failed
|
|
fi
|
|
|
|
required_services=(postgres redis minio keycloak api callbacks worker web gateway mock-provider mock-alert-receiver)
|
|
for service in "${required_services[@]}"; do
|
|
container=$("${compose[@]}" ps -q "$service")
|
|
if [[ -z "$container" ]]; then
|
|
record_check "container_$service" critical missing
|
|
continue
|
|
fi
|
|
state=$(docker inspect -f '{{.State.Status}}' "$container")
|
|
health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
|
|
if [[ "$state" != running ]]; then
|
|
record_check "container_$service" critical "$state"
|
|
elif [[ "$health" != none && "$health" != healthy ]]; then
|
|
record_check "container_$service" critical "$health"
|
|
else
|
|
record_check "container_$service" healthy "$state/$health"
|
|
fi
|
|
done
|
|
|
|
latest_backup=$(find "$backup_root" -maxdepth 1 -type f -name 'platform-*.tar.gz.enc' -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n1 | cut -d' ' -f2- || true)
|
|
if [[ -z "$latest_backup" || ! -f "$latest_backup" ]]; then
|
|
record_check backup_freshness critical missing
|
|
else
|
|
now_epoch=$(date +%s)
|
|
backup_epoch=$(stat -c %Y "$latest_backup")
|
|
backup_age=$((now_epoch - backup_epoch))
|
|
if (( backup_age > backup_max_age_seconds )); then
|
|
record_check backup_freshness critical "age_seconds=$backup_age"
|
|
elif [[ ! -f "$latest_backup.sha256" ]] || ! (cd "$backup_root" && sha256sum -c "$(basename "$latest_backup").sha256" >/dev/null 2>&1); then
|
|
record_check backup_freshness critical checksum_failed
|
|
else
|
|
record_check backup_freshness healthy "age_seconds=$backup_age"
|
|
fi
|
|
fi
|
|
|
|
backup_result=$(systemctl show platform-backup.service -p Result --value 2>/dev/null || true)
|
|
backup_exit=$(systemctl show platform-backup.service -p ExecMainStatus --value 2>/dev/null || true)
|
|
if [[ "$backup_result" == success && "$backup_exit" == 0 ]]; then
|
|
record_check backup_service healthy success
|
|
else
|
|
record_check backup_service critical "result=${backup_result:-unknown}/exit=${backup_exit:-unknown}"
|
|
fi
|
|
|
|
for timer in platform-backup.timer certbot-ip-renew.timer; do
|
|
timer_enabled=$(systemctl is-enabled "$timer" 2>/dev/null || true)
|
|
timer_active=$(systemctl is-active "$timer" 2>/dev/null || true)
|
|
if [[ "$timer_enabled" == enabled && "$timer_active" == active ]]; then
|
|
record_check "$timer" healthy enabled/active
|
|
else
|
|
record_check "$timer" critical "${timer_enabled:-unknown}/${timer_active:-unknown}"
|
|
fi
|
|
done
|
|
|
|
public_hostname=$(python3 -c 'import sys,urllib.parse; print(urllib.parse.urlsplit(sys.argv[1]).hostname or "")' "$public_origin")
|
|
certificate_file=${TLS_CERTIFICATE_FILE:-/etc/letsencrypt/live/$public_hostname/fullchain.pem}
|
|
if [[ ! -r "$certificate_file" ]]; then
|
|
record_check certificate critical missing
|
|
else
|
|
certificate_end=$(openssl x509 -in "$certificate_file" -noout -enddate | cut -d= -f2-)
|
|
certificate_end_epoch=$(date -d "$certificate_end" +%s)
|
|
certificate_seconds=$((certificate_end_epoch - $(date +%s)))
|
|
if (( certificate_seconds < certificate_min_seconds )); then
|
|
record_check certificate critical "remaining_seconds=$certificate_seconds"
|
|
else
|
|
record_check certificate healthy "remaining_seconds=$certificate_seconds"
|
|
fi
|
|
fi
|
|
|
|
disk_used=$(df -P "$backup_root" | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
|
if [[ ! "$disk_used" =~ ^[0-9]+$ ]]; then
|
|
record_check disk_usage critical unreadable
|
|
elif (( disk_used >= disk_critical_percent )); then
|
|
record_check disk_usage critical "used_percent=$disk_used"
|
|
elif (( disk_used >= disk_warning_percent )); then
|
|
record_check disk_usage warning "used_percent=$disk_used"
|
|
else
|
|
record_check disk_usage healthy "used_percent=$disk_used"
|
|
fi
|
|
|
|
if (( critical_count > 0 )); then
|
|
overall=critical
|
|
elif (( warning_count > 0 )); then
|
|
overall=warning
|
|
else
|
|
overall=healthy
|
|
fi
|
|
observed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
previous_status=$(python3 -c 'import json,sys
|
|
try: print(json.load(open(sys.argv[1], encoding="utf-8")).get("status", "unknown"))
|
|
except Exception: print("unknown")' "$status_file" 2>/dev/null || true)
|
|
|
|
python3 - "$checks_file" "$status_tmp" "$overall" "$observed_at" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
checks = []
|
|
with open(sys.argv[1], encoding="utf-8") as source:
|
|
for line in source:
|
|
name, status, value = line.rstrip("\n").split("\t", 2)
|
|
checks.append({"name": name, "status": status, "value": value})
|
|
payload = {
|
|
"schema": "platform-server-health-v1",
|
|
"status": sys.argv[3],
|
|
"observed_at": sys.argv[4],
|
|
"checks": checks,
|
|
}
|
|
with open(sys.argv[2], "w", encoding="utf-8") as target:
|
|
json.dump(payload, target, ensure_ascii=False, separators=(",", ":"))
|
|
target.write("\n")
|
|
PY
|
|
chmod 0644 "$status_tmp"
|
|
mv -f "$status_tmp" "$status_file"
|
|
|
|
if [[ -n "$alert_webhook_url" && "$previous_status" != "$overall" ]]; then
|
|
if [[ ! -r "$alert_webhook_secret_file" ]]; then
|
|
record_check alert_delivery critical secret_missing
|
|
echo "Alert webhook is configured but its HMAC secret is unreadable" >&2
|
|
exit 5
|
|
fi
|
|
timestamp=$(date +%s)
|
|
signature=$(python3 - "$alert_webhook_secret_file" "$status_file" "$timestamp" <<'PY'
|
|
import hashlib
|
|
import hmac
|
|
import sys
|
|
|
|
secret = open(sys.argv[1], "rb").read().rstrip(b"\r\n")
|
|
body = open(sys.argv[2], "rb").read()
|
|
print(hmac.new(secret, sys.argv[3].encode() + b"." + body, hashlib.sha256).hexdigest())
|
|
PY
|
|
)
|
|
python3 - "$alert_webhook_url" "$curl_config" <<'PY'
|
|
import sys
|
|
|
|
url = sys.argv[1].replace("\\", "\\\\").replace('"', '\\"')
|
|
open(sys.argv[2], "w", encoding="utf-8").write(f'url = "{url}"\n')
|
|
PY
|
|
curl --fail --silent --show-error --max-time 10 --config "$curl_config" \
|
|
-H 'Content-Type: application/json' \
|
|
-H "X-Platform-Timestamp: $timestamp" \
|
|
-H "X-Platform-Signature: sha256=$signature" \
|
|
--data-binary "@$status_file" >/dev/null
|
|
fi
|
|
|
|
printf 'platform_health status=%s critical=%s warning=%s status_file=%s\n' \
|
|
"$overall" "$critical_count" "$warning_count" "$status_file"
|
|
if (( critical_count > 0 )); then
|
|
exit 1
|
|
fi
|