Codex Local
bf1233d5df
ci / verify (push) Canceled after 0s
Add customer and operator login, BLE provisioning, device fleet controls, call memory, memoir generation, and refreshed family and maintenance portals. Include API contracts, migrations, firmware sources, tests, assets, and deployment documentation.
392 lines
16 KiB
Bash
Executable File
392 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
project_root=${PLATFORM_ROOT:-/opt/data-terminal-cloud/current}
|
|
backup_root=${BACKUP_ROOT:-/var/backups/data-terminal-cloud}
|
|
state_dir=${MONITOR_STATE_DIR:-/var/lib/data-terminal-cloud-monitor}
|
|
monitor_config=${MONITOR_CONFIG_FILE:-/etc/data-terminal-cloud/monitor.env}
|
|
offsite_config_file=${OFFSITE_CONFIG_FILE:-/etc/data-terminal-cloud/offsite-backup.env}
|
|
backup_max_age_seconds=${BACKUP_MAX_AGE_SECONDS:-129600}
|
|
certificate_min_seconds=${CERTIFICATE_MIN_SECONDS:-129600}
|
|
disk_warning_percent=${DISK_WARNING_PERCENT:-80}
|
|
disk_critical_percent=${DISK_CRITICAL_PERCENT:-90}
|
|
|
|
if [[ -f "$monitor_config" ]]; then
|
|
# The installer creates this file as root:root 0600.
|
|
# shellcheck disable=SC1090
|
|
source "$monitor_config"
|
|
fi
|
|
alert_webhook_url=${ALERT_WEBHOOK_URL:-}
|
|
alert_webhook_secret_file=${ALERT_WEBHOOK_SECRET_FILE:-/etc/data-terminal-cloud/monitor-webhook-secret}
|
|
platform_health_report_url=${PLATFORM_HEALTH_REPORT_URL:-}
|
|
platform_health_report_secret_file=${PLATFORM_HEALTH_REPORT_SECRET_FILE:-/etc/data-terminal-cloud/platform-health-callback-secret}
|
|
expected_public_tcp_ports=${EXPECTED_PUBLIC_TCP_PORTS:-22,80,443,1883}
|
|
deployment_profile=${HOST_DEPLOYMENT_PROFILE:-demo}
|
|
if [[ -f "$offsite_config_file" ]]; then
|
|
# shellcheck disable=SC1090
|
|
source "$offsite_config_file"
|
|
fi
|
|
offsite_enabled=${OFFSITE_BACKUP_ENABLED:-false}
|
|
offsite_required=${OFFSITE_BACKUP_REQUIRED:-false}
|
|
offsite_state_file=${OFFSITE_STATE_FILE:-/var/lib/data-terminal-cloud-offsite/status.json}
|
|
offsite_max_age_seconds=${OFFSITE_MAX_AGE_SECONDS:-129600}
|
|
|
|
for numeric in "$backup_max_age_seconds" "$certificate_min_seconds" "$disk_warning_percent" "$disk_critical_percent" "$offsite_max_age_seconds"; do
|
|
[[ "$numeric" =~ ^[0-9]+$ ]] || { echo "Monitor thresholds must be non-negative integers" >&2; exit 2; }
|
|
done
|
|
if (( disk_warning_percent >= disk_critical_percent || disk_critical_percent > 100 )); then
|
|
echo "Disk thresholds are invalid" >&2
|
|
exit 2
|
|
fi
|
|
if [[ ! "$expected_public_tcp_ports" =~ ^[0-9]+(,[0-9]+)*$ ]]; then
|
|
echo "EXPECTED_PUBLIC_TCP_PORTS must contain comma-separated integers" >&2
|
|
exit 2
|
|
fi
|
|
if [[ "$deployment_profile" != demo && "$deployment_profile" != production ]]; then
|
|
echo "HOST_DEPLOYMENT_PROFILE must be demo or production" >&2
|
|
exit 2
|
|
fi
|
|
|
|
compose=(docker compose -f "$project_root/compose.yaml" -f "$project_root/deploy/server/compose.override.yaml" --env-file "$project_root/deploy/server/.env")
|
|
mkdir -p "$state_dir"
|
|
chmod 0750 "$state_dir"
|
|
umask 077
|
|
checks_file=$(mktemp "$state_dir/.checks-XXXXXX")
|
|
status_tmp=$(mktemp "$state_dir/.status-XXXXXX")
|
|
oidc_tmp=$(mktemp "$state_dir/.oidc-XXXXXX")
|
|
curl_config=$(mktemp "$state_dir/.curl-XXXXXX")
|
|
status_file="$state_dir/status.json"
|
|
critical_count=0
|
|
warning_count=0
|
|
|
|
cleanup() {
|
|
rm -f -- "$checks_file" "$status_tmp" "$oidc_tmp" "$curl_config"
|
|
}
|
|
trap cleanup EXIT INT TERM
|
|
|
|
record_check() {
|
|
local name=$1
|
|
local status=$2
|
|
local value=${3:-}
|
|
value=${value//$'\t'/ }
|
|
value=${value//$'\r'/ }
|
|
value=${value//$'\n'/ }
|
|
printf '%s\t%s\t%s\n' "$name" "$status" "$value" >>"$checks_file"
|
|
case "$status" in
|
|
critical) critical_count=$((critical_count + 1)) ;;
|
|
warning) warning_count=$((warning_count + 1)) ;;
|
|
esac
|
|
}
|
|
|
|
record_check deployment_profile healthy "$deployment_profile"
|
|
if [[ -z "$alert_webhook_url" ]]; then
|
|
if [[ "$deployment_profile" == production ]]; then
|
|
record_check external_alert_receiver critical not_configured
|
|
else
|
|
record_check external_alert_receiver healthy not_configured
|
|
fi
|
|
elif [[ ! -r "$alert_webhook_secret_file" ]]; then
|
|
record_check external_alert_receiver critical secret_missing
|
|
elif (( $(tr -d '\r\n' <"$alert_webhook_secret_file" | wc -c) < 32 )); then
|
|
record_check external_alert_receiver critical secret_invalid
|
|
else
|
|
record_check external_alert_receiver healthy configured
|
|
fi
|
|
|
|
public_origin=$(sed -n 's/^PUBLIC_ORIGIN=//p' "$project_root/deploy/server/.env")
|
|
if [[ "$public_origin" == https://* ]]; then
|
|
record_check public_origin healthy "$public_origin"
|
|
else
|
|
record_check public_origin critical invalid_https_origin
|
|
fi
|
|
|
|
if curl --fail --silent --show-error --max-time 10 "$public_origin/api/v1/version" >/dev/null; then
|
|
record_check public_https healthy http_200
|
|
else
|
|
record_check public_https critical request_failed
|
|
fi
|
|
|
|
if ! command -v ss >/dev/null 2>&1; then
|
|
record_check public_tcp_listeners critical probe_unavailable
|
|
else
|
|
listener_evidence=$(ss -H -ltn | python3 "$project_root/deploy/server/public_listener_policy.py" "$expected_public_tcp_ports" 2>/dev/null || true)
|
|
IFS='|' read -r listener_status listener_detail <<<"$listener_evidence"
|
|
if [[ "$listener_status" == healthy ]]; then
|
|
record_check public_tcp_listeners healthy "$listener_detail"
|
|
elif [[ "$listener_status" == critical ]]; then
|
|
record_check public_tcp_listeners critical "$listener_detail"
|
|
else
|
|
record_check public_tcp_listeners critical invalid_probe_result
|
|
fi
|
|
fi
|
|
|
|
expected_issuer="$public_origin/realms/voice-local"
|
|
if curl --fail --silent --show-error --max-time 10 \
|
|
"$expected_issuer/.well-known/openid-configuration" -o "$oidc_tmp"; then
|
|
actual_issuer=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1], encoding="utf-8")).get("issuer", ""))' "$oidc_tmp" 2>/dev/null || true)
|
|
if [[ "$actual_issuer" == "$expected_issuer" ]]; then
|
|
record_check oidc_issuer healthy issuer_matches
|
|
else
|
|
record_check oidc_issuer critical issuer_mismatch
|
|
fi
|
|
else
|
|
record_check oidc_issuer critical request_failed
|
|
fi
|
|
|
|
required_services=(postgres redis minio keycloak api callbacks worker web gateway)
|
|
if [[ "$deployment_profile" == demo ]]; then
|
|
required_services+=(mock-provider mock-alert-receiver)
|
|
fi
|
|
for service in "${required_services[@]}"; do
|
|
container=$("${compose[@]}" ps -q "$service")
|
|
if [[ -z "$container" ]]; then
|
|
record_check "container_$service" critical missing
|
|
continue
|
|
fi
|
|
state=$(docker inspect -f '{{.State.Status}}' "$container")
|
|
health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container")
|
|
if [[ "$state" != running ]]; then
|
|
record_check "container_$service" critical "$state"
|
|
elif [[ "$health" != none && "$health" != healthy ]]; then
|
|
record_check "container_$service" critical "$health"
|
|
else
|
|
record_check "container_$service" healthy "$state/$health"
|
|
fi
|
|
done
|
|
|
|
application_environments=()
|
|
application_environment_missing=false
|
|
for application_service in api callbacks worker; do
|
|
application_container=$("${compose[@]}" ps -q "$application_service")
|
|
if [[ -z "$application_container" ]]; then
|
|
application_environment_missing=true
|
|
continue
|
|
fi
|
|
application_environment=$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$application_container" | sed -n 's/^APP_ENV=//p' | head -n1)
|
|
if [[ -z "$application_environment" ]]; then
|
|
application_environment_missing=true
|
|
else
|
|
application_environments+=("$application_environment")
|
|
fi
|
|
done
|
|
if [[ "$application_environment_missing" == true || ${#application_environments[@]} -ne 3 ]]; then
|
|
record_check application_environment critical missing
|
|
elif [[ "${application_environments[0]}" != "${application_environments[1]}" || "${application_environments[0]}" != "${application_environments[2]}" ]]; then
|
|
record_check application_environment critical mixed
|
|
elif [[ "$deployment_profile" == production && "${application_environments[0]}" != prod ]]; then
|
|
record_check application_environment critical "${application_environments[0]}"
|
|
elif [[ "$deployment_profile" == demo && "${application_environments[0]}" != local ]]; then
|
|
record_check application_environment critical "${application_environments[0]}"
|
|
else
|
|
record_check application_environment healthy "${application_environments[0]}"
|
|
fi
|
|
|
|
keycloak_container=$("${compose[@]}" ps -q keycloak)
|
|
if [[ -z "$keycloak_container" ]]; then
|
|
record_check identity_service_mode critical missing
|
|
elif docker inspect -f '{{json .Config.Cmd}}' "$keycloak_container" | grep -q 'start-dev'; then
|
|
if [[ "$deployment_profile" == production ]]; then
|
|
record_check identity_service_mode critical development
|
|
else
|
|
record_check identity_service_mode healthy development
|
|
fi
|
|
else
|
|
record_check identity_service_mode healthy hardened
|
|
fi
|
|
|
|
mock_services_running=()
|
|
for mock_service in mock-provider mock-alert-receiver; do
|
|
mock_container=$("${compose[@]}" ps -q "$mock_service")
|
|
if [[ -n "$mock_container" && $(docker inspect -f '{{.State.Status}}' "$mock_container") == running ]]; then
|
|
mock_services_running+=("$mock_service")
|
|
fi
|
|
done
|
|
if [[ "$deployment_profile" == production ]]; then
|
|
if (( ${#mock_services_running[@]} > 0 )); then
|
|
record_check mock_services critical unexpected_running
|
|
else
|
|
record_check mock_services healthy disabled
|
|
fi
|
|
elif (( ${#mock_services_running[@]} == 2 )); then
|
|
record_check mock_services healthy enabled_for_demo
|
|
else
|
|
record_check mock_services critical incomplete_demo_stack
|
|
fi
|
|
|
|
latest_backup=$(find "$backup_root" -maxdepth 1 -type f -name 'platform-*.tar.gz.enc' -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n1 | cut -d' ' -f2- || true)
|
|
if [[ -z "$latest_backup" || ! -f "$latest_backup" ]]; then
|
|
record_check backup_freshness critical missing
|
|
else
|
|
now_epoch=$(date +%s)
|
|
backup_epoch=$(stat -c %Y "$latest_backup")
|
|
backup_age=$((now_epoch - backup_epoch))
|
|
if (( backup_age > backup_max_age_seconds )); then
|
|
record_check backup_freshness critical "age_seconds=$backup_age"
|
|
elif [[ ! -f "$latest_backup.sha256" ]] || ! (cd "$backup_root" && sha256sum -c "$(basename "$latest_backup").sha256" >/dev/null 2>&1); then
|
|
record_check backup_freshness critical checksum_failed
|
|
else
|
|
record_check backup_freshness healthy "age_seconds=$backup_age"
|
|
fi
|
|
fi
|
|
|
|
if [[ "$offsite_enabled" == true || "$offsite_required" == true ]]; then
|
|
if [[ ! -r "$offsite_state_file" ]]; then
|
|
record_check offsite_backup critical status_missing
|
|
else
|
|
offsite_evidence=$(python3 -c 'import json,sys
|
|
try:
|
|
d=json.load(open(sys.argv[1], encoding="utf-8")); print("|".join([str(d.get("status", "")), str(d.get("verified_epoch", 0)), str(d.get("source_archive", ""))]))
|
|
except Exception: print("invalid|0|")' "$offsite_state_file")
|
|
IFS='|' read -r offsite_status offsite_epoch offsite_source <<<"$offsite_evidence"
|
|
offsite_age=$(($(date +%s) - offsite_epoch))
|
|
if [[ "$offsite_status" != verified ]]; then
|
|
record_check offsite_backup critical "status=${offsite_status:-invalid}"
|
|
elif [[ "$offsite_source" != "$(basename "${latest_backup:-missing}")" ]]; then
|
|
record_check offsite_backup critical source_mismatch
|
|
elif (( offsite_epoch <= 0 || offsite_age > offsite_max_age_seconds )); then
|
|
record_check offsite_backup critical "age_seconds=$offsite_age"
|
|
else
|
|
record_check offsite_backup healthy "age_seconds=$offsite_age"
|
|
fi
|
|
fi
|
|
else
|
|
if [[ "$deployment_profile" == production ]]; then
|
|
record_check offsite_backup critical not_required
|
|
else
|
|
record_check offsite_backup healthy not_required
|
|
fi
|
|
fi
|
|
|
|
backup_result=$(systemctl show platform-backup.service -p Result --value 2>/dev/null || true)
|
|
backup_exit=$(systemctl show platform-backup.service -p ExecMainStatus --value 2>/dev/null || true)
|
|
if [[ "$backup_result" == success && "$backup_exit" == 0 ]]; then
|
|
record_check backup_service healthy success
|
|
else
|
|
record_check backup_service critical "result=${backup_result:-unknown}/exit=${backup_exit:-unknown}"
|
|
fi
|
|
|
|
timers=(platform-backup.timer certbot-ip-renew.timer)
|
|
if [[ -f "$offsite_config_file" ]]; then
|
|
timers+=(platform-offsite-backup.timer)
|
|
fi
|
|
for timer in "${timers[@]}"; do
|
|
timer_enabled=$(systemctl is-enabled "$timer" 2>/dev/null || true)
|
|
timer_active=$(systemctl is-active "$timer" 2>/dev/null || true)
|
|
if [[ "$timer_enabled" == enabled && "$timer_active" == active ]]; then
|
|
record_check "$timer" healthy enabled/active
|
|
else
|
|
record_check "$timer" critical "${timer_enabled:-unknown}/${timer_active:-unknown}"
|
|
fi
|
|
done
|
|
|
|
public_hostname=$(python3 -c 'import sys,urllib.parse; print(urllib.parse.urlsplit(sys.argv[1]).hostname or "")' "$public_origin")
|
|
certificate_file=${TLS_CERTIFICATE_FILE:-/etc/letsencrypt/live/$public_hostname/fullchain.pem}
|
|
if [[ ! -r "$certificate_file" ]]; then
|
|
record_check certificate critical missing
|
|
else
|
|
certificate_end=$(openssl x509 -in "$certificate_file" -noout -enddate | cut -d= -f2-)
|
|
certificate_end_epoch=$(date -d "$certificate_end" +%s)
|
|
certificate_seconds=$((certificate_end_epoch - $(date +%s)))
|
|
if (( certificate_seconds < certificate_min_seconds )); then
|
|
record_check certificate critical "remaining_seconds=$certificate_seconds"
|
|
else
|
|
record_check certificate healthy "remaining_seconds=$certificate_seconds"
|
|
fi
|
|
fi
|
|
|
|
disk_used=$(df -P "$backup_root" | awk 'NR==2 {gsub(/%/, "", $5); print $5}')
|
|
if [[ ! "$disk_used" =~ ^[0-9]+$ ]]; then
|
|
record_check disk_usage critical unreadable
|
|
elif (( disk_used >= disk_critical_percent )); then
|
|
record_check disk_usage critical "used_percent=$disk_used"
|
|
elif (( disk_used >= disk_warning_percent )); then
|
|
record_check disk_usage warning "used_percent=$disk_used"
|
|
else
|
|
record_check disk_usage healthy "used_percent=$disk_used"
|
|
fi
|
|
|
|
if (( critical_count > 0 )); then
|
|
overall=critical
|
|
elif (( warning_count > 0 )); then
|
|
overall=warning
|
|
else
|
|
overall=healthy
|
|
fi
|
|
observed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
previous_status=$(python3 -c 'import json,sys
|
|
try: print(json.load(open(sys.argv[1], encoding="utf-8")).get("status", "unknown"))
|
|
except Exception: print("unknown")' "$status_file" 2>/dev/null || true)
|
|
|
|
python3 - "$checks_file" "$status_tmp" "$overall" "$observed_at" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
checks = []
|
|
with open(sys.argv[1], encoding="utf-8") as source:
|
|
for line in source:
|
|
name, status, value = line.rstrip("\n").split("\t", 2)
|
|
checks.append({"name": name, "status": status, "value": value})
|
|
payload = {
|
|
"schema": "platform-server-health-v1",
|
|
"status": sys.argv[3],
|
|
"observed_at": sys.argv[4],
|
|
"checks": checks,
|
|
}
|
|
with open(sys.argv[2], "w", encoding="utf-8") as target:
|
|
json.dump(payload, target, ensure_ascii=False, separators=(",", ":"))
|
|
target.write("\n")
|
|
PY
|
|
chmod 0644 "$status_tmp"
|
|
mv -f "$status_tmp" "$status_file"
|
|
|
|
post_signed_status() {
|
|
local target_url=$1
|
|
local secret_file=$2
|
|
local timestamp signature
|
|
timestamp=$(date +%s)
|
|
signature=$(python3 - "$secret_file" "$status_file" "$timestamp" <<'PY'
|
|
import hashlib
|
|
import hmac
|
|
import sys
|
|
|
|
secret = open(sys.argv[1], "rb").read().rstrip(b"\r\n")
|
|
body = open(sys.argv[2], "rb").read()
|
|
print(hmac.new(secret, sys.argv[3].encode() + b"." + body, hashlib.sha256).hexdigest())
|
|
PY
|
|
)
|
|
python3 - "$target_url" "$curl_config" <<'PY'
|
|
import sys
|
|
|
|
url = sys.argv[1].replace("\\", "\\\\").replace('"', '\\"')
|
|
open(sys.argv[2], "w", encoding="utf-8").write(f'url = "{url}"\n')
|
|
PY
|
|
curl --fail --silent --show-error --max-time 10 --config "$curl_config" \
|
|
-H 'Content-Type: application/json' \
|
|
-H "X-Platform-Timestamp: $timestamp" \
|
|
-H "X-Platform-Signature: sha256=$signature" \
|
|
--data-binary "@$status_file" >/dev/null
|
|
}
|
|
|
|
if [[ -n "$platform_health_report_url" ]]; then
|
|
if [[ ! -r "$platform_health_report_secret_file" ]]; then
|
|
echo "Platform health reporting is configured but its HMAC secret is unreadable" >&2
|
|
exit 6
|
|
fi
|
|
post_signed_status "$platform_health_report_url" "$platform_health_report_secret_file"
|
|
fi
|
|
|
|
if [[ -n "$alert_webhook_url" && "$previous_status" != "$overall" ]]; then
|
|
if [[ ! -r "$alert_webhook_secret_file" ]]; then
|
|
record_check alert_delivery critical secret_missing
|
|
echo "Alert webhook is configured but its HMAC secret is unreadable" >&2
|
|
exit 5
|
|
fi
|
|
post_signed_status "$alert_webhook_url" "$alert_webhook_secret_file"
|
|
fi
|
|
|
|
printf 'platform_health status=%s critical=%s warning=%s status_file=%s\n' \
|
|
"$overall" "$critical_count" "$warning_count" "$status_file"
|
|
if (( critical_count > 0 )); then
|
|
exit 1
|
|
fi
|