wangcong/story
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add OSS offsite backup gate

Browse Source
This commit is contained in:
Codex Local
2026-08-26 20:55:35 +08:00
parent 0bf375155e
commit 1e12d329dc
10 changed files with 384 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/server/health-monitor.sh
+39 -2
View File
@@ -5,6 +5,7 @@ project_root=${PLATFORM_ROOT:-/opt/data-terminal-cloud/current}
backup_root=${BACKUP_ROOT:-/var/backups/data-terminal-cloud}
state_dir=${MONITOR_STATE_DIR:-/var/lib/data-terminal-cloud-monitor}
monitor_config=${MONITOR_CONFIG_FILE:-/etc/data-terminal-cloud/monitor.env}
offsite_config_file=${OFFSITE_CONFIG_FILE:-/etc/data-terminal-cloud/offsite-backup.env}
backup_max_age_seconds=${BACKUP_MAX_AGE_SECONDS:-129600}
certificate_min_seconds=${CERTIFICATE_MIN_SECONDS:-129600}
disk_warning_percent=${DISK_WARNING_PERCENT:-80}
@@ -17,8 +18,16 @@ if [[ -f "$monitor_config" ]]; then
fi
alert_webhook_url=${ALERT_WEBHOOK_URL:-}
alert_webhook_secret_file=${ALERT_WEBHOOK_SECRET_FILE:-/etc/data-terminal-cloud/monitor-webhook-secret}
if [[ -f "$offsite_config_file" ]]; then
# shellcheck disable=SC1090
source "$offsite_config_file"
fi
offsite_enabled=${OFFSITE_BACKUP_ENABLED:-false}
offsite_required=${OFFSITE_BACKUP_REQUIRED:-false}
offsite_state_file=${OFFSITE_STATE_FILE:-/var/lib/data-terminal-cloud-offsite/status.json}
offsite_max_age_seconds=${OFFSITE_MAX_AGE_SECONDS:-129600}
for numeric in "$backup_max_age_seconds" "$certificate_min_seconds" "$disk_warning_percent" "$disk_critical_percent"; do
for numeric in "$backup_max_age_seconds" "$certificate_min_seconds" "$disk_warning_percent" "$disk_critical_percent" "$offsite_max_age_seconds"; do
[[ "$numeric" =~ ^[0-9]+$ ]] || { echo "Monitor thresholds must be non-negative integers" >&2; exit 2; }
done
if (( disk_warning_percent >= disk_critical_percent || disk_critical_percent > 100 )); then
@@ -117,6 +126,30 @@ else
fi
fi
if [[ "$offsite_enabled" == true || "$offsite_required" == true ]]; then
if [[ ! -r "$offsite_state_file" ]]; then
record_check offsite_backup critical status_missing
else
offsite_evidence=$(python3 -c 'import json,sys
try:
d=json.load(open(sys.argv[1], encoding="utf-8")); print("|".join([str(d.get("status", "")), str(d.get("verified_epoch", 0)), str(d.get("source_archive", ""))]))
except Exception: print("invalid|0|")' "$offsite_state_file")
IFS='|' read -r offsite_status offsite_epoch offsite_source <<<"$offsite_evidence"
offsite_age=$(($(date +%s) - offsite_epoch))
if [[ "$offsite_status" != verified ]]; then
record_check offsite_backup critical "status=${offsite_status:-invalid}"
elif [[ "$offsite_source" != "$(basename "${latest_backup:-missing}")" ]]; then
record_check offsite_backup critical source_mismatch
elif (( offsite_epoch <= 0 || offsite_age > offsite_max_age_seconds )); then
record_check offsite_backup critical "age_seconds=$offsite_age"
else
record_check offsite_backup healthy "age_seconds=$offsite_age"
fi
fi
else
record_check offsite_backup healthy not_required
fi
backup_result=$(systemctl show platform-backup.service -p Result --value 2>/dev/null || true)
backup_exit=$(systemctl show platform-backup.service -p ExecMainStatus --value 2>/dev/null || true)
if [[ "$backup_result" == success && "$backup_exit" == 0 ]]; then
@@ -125,7 +158,11 @@ else
record_check backup_service critical "result=${backup_result:-unknown}/exit=${backup_exit:-unknown}"
fi
for timer in platform-backup.timer certbot-ip-renew.timer; do
timers=(platform-backup.timer certbot-ip-renew.timer)
if [[ -f "$offsite_config_file" ]]; then
timers+=(platform-offsite-backup.timer)
fi
for timer in "${timers[@]}"; do
timer_enabled=$(systemctl is-enabled "$timer" 2>/dev/null || true)
timer_active=$(systemctl is-active "$timer" 2>/dev/null || true)
if [[ "$timer_enabled" == enabled && "$timer_active" == active ]]; then