feat: add OSS offsite backup gate

2026-08-26 20:55:35 +08:00
parent 0bf375155e
commit 1e12d329dc
10 changed files with 384 additions and 4 deletions
@@ -55,3 +55,14 @@ python3 -m json.tool /var/lib/data-terminal-cloud-monitor/status.json
 ```

 监测任务每五分钟执行，正常状态为`healthy`。默认只记录结构化状态和systemd日志；在`/etc/data-terminal-cloud/monitor.env`配置`ALERT_WEBHOOK_URL`后，状态变化会发送带时间戳和HMAC-SHA256签名的JSON通知。完整检查项、签名规则、阈值和失败分支见[《单机云端健康监测与告警手册》](../../docs/server-health-monitoring.md)。
+
+## OSS异地备份适配
+
+```bash
+chmod 0755 deploy/server/install-ossutil.sh \
+  deploy/server/install-offsite-backup.sh \
+  deploy/server/sync-backup-offsite.sh
+sudo deploy/server/install-offsite-backup.sh
+```
+
+安装后默认为禁用状态，不会访问OSS。创建私有Bucket并绑定最小权限ECS RAM角色后，在`/etc/data-terminal-cloud/offsite-backup.env`填写Bucket和前缀、设置`OFFSITE_BACKUP_ENABLED=true`与`OFFSITE_BACKUP_REQUIRED=true`，再手工启动一次`platform-offsite-backup.service`。任务会上传加密归档及摘要、重新下载并复算SHA-256；健康检查在启用后强制要求最新本机归档已有对应远端验证。配置和真实验收步骤见[《阿里云OSS异地备份适配与门禁》](../../docs/offsite-backup.md)。
@@ -5,6 +5,7 @@ project_root=${PLATFORM_ROOT:-/opt/data-terminal-cloud/current}
 backup_root=${BACKUP_ROOT:-/var/backups/data-terminal-cloud}
 state_dir=${MONITOR_STATE_DIR:-/var/lib/data-terminal-cloud-monitor}
 monitor_config=${MONITOR_CONFIG_FILE:-/etc/data-terminal-cloud/monitor.env}
+offsite_config_file=${OFFSITE_CONFIG_FILE:-/etc/data-terminal-cloud/offsite-backup.env}
 backup_max_age_seconds=${BACKUP_MAX_AGE_SECONDS:-129600}
 certificate_min_seconds=${CERTIFICATE_MIN_SECONDS:-129600}
 disk_warning_percent=${DISK_WARNING_PERCENT:-80}
@@ -17,8 +18,16 @@ if [[ -f "$monitor_config" ]]; then
 fi
 alert_webhook_url=${ALERT_WEBHOOK_URL:-}
 alert_webhook_secret_file=${ALERT_WEBHOOK_SECRET_FILE:-/etc/data-terminal-cloud/monitor-webhook-secret}
+if [[ -f "$offsite_config_file" ]]; then
+  # shellcheck disable=SC1090
+  source "$offsite_config_file"
+fi
+offsite_enabled=${OFFSITE_BACKUP_ENABLED:-false}
+offsite_required=${OFFSITE_BACKUP_REQUIRED:-false}
+offsite_state_file=${OFFSITE_STATE_FILE:-/var/lib/data-terminal-cloud-offsite/status.json}
+offsite_max_age_seconds=${OFFSITE_MAX_AGE_SECONDS:-129600}

-for numeric in "$backup_max_age_seconds" "$certificate_min_seconds" "$disk_warning_percent" "$disk_critical_percent"; do
+for numeric in "$backup_max_age_seconds" "$certificate_min_seconds" "$disk_warning_percent" "$disk_critical_percent" "$offsite_max_age_seconds"; do
  [[ "$numeric" =~ ^[0-9]+$ ]] || { echo "Monitor thresholds must be non-negative integers" >&2; exit 2; }
 done
 if (( disk_warning_percent >= disk_critical_percent || disk_critical_percent > 100 )); then
@@ -117,6 +126,30 @@ else
  fi
 fi

+if [[ "$offsite_enabled" == true || "$offsite_required" == true ]]; then
+  if [[ ! -r "$offsite_state_file" ]]; then
+    record_check offsite_backup critical status_missing
+  else
+    offsite_evidence=$(python3 -c 'import json,sys
+try:
+ d=json.load(open(sys.argv[1], encoding="utf-8")); print("|".join([str(d.get("status", "")), str(d.get("verified_epoch", 0)), str(d.get("source_archive", ""))]))
+except Exception: print("invalid|0|")' "$offsite_state_file")
+    IFS='|' read -r offsite_status offsite_epoch offsite_source <<<"$offsite_evidence"
+    offsite_age=$(($(date +%s) - offsite_epoch))
+    if [[ "$offsite_status" != verified ]]; then
+      record_check offsite_backup critical "status=${offsite_status:-invalid}"
+    elif [[ "$offsite_source" != "$(basename "${latest_backup:-missing}")" ]]; then
+      record_check offsite_backup critical source_mismatch
+    elif (( offsite_epoch <= 0 || offsite_age > offsite_max_age_seconds )); then
+      record_check offsite_backup critical "age_seconds=$offsite_age"
+    else
+      record_check offsite_backup healthy "age_seconds=$offsite_age"
+    fi
+  fi
+else
+  record_check offsite_backup healthy not_required
+fi
+
 backup_result=$(systemctl show platform-backup.service -p Result --value 2>/dev/null || true)
 backup_exit=$(systemctl show platform-backup.service -p ExecMainStatus --value 2>/dev/null || true)
 if [[ "$backup_result" == success && "$backup_exit" == 0 ]]; then
@@ -125,7 +158,11 @@ else
  record_check backup_service critical "result=${backup_result:-unknown}/exit=${backup_exit:-unknown}"
 fi

-for timer in platform-backup.timer certbot-ip-renew.timer; do
+timers=(platform-backup.timer certbot-ip-renew.timer)
+if [[ -f "$offsite_config_file" ]]; then
+  timers+=(platform-offsite-backup.timer)
+fi
+for timer in "${timers[@]}"; do
  timer_enabled=$(systemctl is-enabled "$timer" 2>/dev/null || true)
  timer_active=$(systemctl is-active "$timer" 2>/dev/null || true)
  if [[ "$timer_enabled" == enabled && "$timer_active" == active ]]; then
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if [[ ${EUID:-$(id -u)} -ne 0 ]]; then
+  echo "Run this installer as root" >&2
+  exit 2
+fi
+
+project_root=${PLATFORM_ROOT:-/opt/data-terminal-cloud/current}
+"$project_root/deploy/server/install-ossutil.sh"
+install -d -m 0700 /etc/data-terminal-cloud
+install -d -m 0750 /var/lib/data-terminal-cloud-offsite
+if [[ ! -e /etc/data-terminal-cloud/offsite-backup.env ]]; then
+  cat >/etc/data-terminal-cloud/offsite-backup.env <<'EOF'
+OFFSITE_BACKUP_ENABLED=false
+OFFSITE_BACKUP_REQUIRED=false
+OFFSITE_OSS_BUCKET=
+OFFSITE_OSS_PREFIX=data-terminal-cloud/121.41.46.238
+OFFSITE_OSS_CONFIG_FILE=/etc/data-terminal-cloud/ossutilconfig
+EOF
+fi
+chmod 0600 /etc/data-terminal-cloud/offsite-backup.env
+install -m 0644 "$project_root/deploy/server/platform-offsite-backup.service" /etc/systemd/system/platform-offsite-backup.service
+install -m 0644 "$project_root/deploy/server/platform-offsite-backup.timer" /etc/systemd/system/platform-offsite-backup.timer
+systemctl daemon-reload
+systemctl enable --now platform-offsite-backup.timer
+systemctl is-active --quiet platform-offsite-backup.timer
+echo "platform-offsite-backup.timer enabled; upload remains disabled until OSS identity and bucket are configured"
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if [[ ${EUID:-$(id -u)} -ne 0 ]]; then
+  echo "Run this installer as root" >&2
+  exit 2
+fi
+
+version=2.3.0
+archive=ossutil-${version}-linux-amd64.zip
+expected_sha256=3ae4d9fc85a7a6e9f5654d1599766f1a3a42a3692870887b5ae9338d582ef65a
+download_url="https://gosspublic.alicdn.com/ossutil/v2/${version}/${archive}"
+
+if command -v ossutil >/dev/null 2>&1 && [[ $(ossutil version 2>/dev/null) == "$version" ]]; then
+  echo "ossutil $version already installed"
+  exit 0
+fi
+install_dir=$(mktemp -d /var/tmp/ossutil-install-XXXXXX)
+case "$install_dir" in /var/tmp/ossutil-install-*) ;; *) echo "Unsafe install directory" >&2; exit 9;; esac
+trap 'rm -rf -- "$install_dir"' EXIT
+curl -fsSL --max-time 120 -o "$install_dir/$archive" "$download_url"
+printf '%s  %s\n' "$expected_sha256" "$install_dir/$archive" | sha256sum -c -
+mkdir -p "$install_dir/unpacked"
+if command -v unzip >/dev/null 2>&1; then
+  unzip -q "$install_dir/$archive" -d "$install_dir/unpacked"
+elif command -v python3 >/dev/null 2>&1; then
+  python3 -m zipfile -e "$install_dir/$archive" "$install_dir/unpacked"
+else
+  echo "Neither unzip nor python3 is available to extract ossutil" >&2
+  exit 3
+fi
+binary=$(find "$install_dir/unpacked" -type f -name ossutil -print -quit)
+[[ -n "$binary" ]] || { echo "ossutil binary is missing from the verified archive" >&2; exit 4; }
+install -m 0755 "$binary" /usr/local/bin/ossutil
+[[ $(/usr/local/bin/ossutil version) == "$version" ]]
+echo "ossutil $version installed"
@@ -0,0 +1,18 @@
+[Unit]
+Description=Copy encrypted data terminal cloud backup to OSS
+Requires=network-online.target
+After=network-online.target platform-backup.service
+
+[Service]
+Type=oneshot
+User=root
+Group=root
+UMask=0077
+Environment=BACKUP_ROOT=/var/backups/data-terminal-cloud
+Environment=OFFSITE_STATE_DIR=/var/lib/data-terminal-cloud-offsite
+Environment=OFFSITE_CONFIG_FILE=/etc/data-terminal-cloud/offsite-backup.env
+ExecStart=/opt/data-terminal-cloud/current/deploy/server/sync-backup-offsite.sh
+Nice=10
+NoNewPrivileges=true
+PrivateTmp=true
+TimeoutStartSec=4h
@@ -0,0 +1,11 @@
+[Unit]
+Description=Copy encrypted data terminal cloud backup to OSS daily
+
+[Timer]
+OnCalendar=*-*-* 04:10:00
+RandomizedDelaySec=30m
+Persistent=true
+Unit=platform-offsite-backup.service
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+backup_root=${BACKUP_ROOT:-/var/backups/data-terminal-cloud}
+state_dir=${OFFSITE_STATE_DIR:-/var/lib/data-terminal-cloud-offsite}
+offsite_config_file=${OFFSITE_CONFIG_FILE:-/etc/data-terminal-cloud/offsite-backup.env}
+ossutil_bin=${OSSUTIL_BIN:-/usr/local/bin/ossutil}
+
+mkdir -p "$state_dir"
+chmod 0750 "$state_dir"
+umask 077
+status_file="$state_dir/status.json"
+status_tmp=$(mktemp "$state_dir/.status-XXXXXX")
+transfer_dir=$(mktemp -d "$state_dir/.transfer-XXXXXX")
+
+cleanup() {
+  rm -f -- "$status_tmp"
+  rm -rf -- "$transfer_dir"
+}
+trap cleanup EXIT INT TERM
+
+write_status() {
+  local status=$1
+  local reason=$2
+  local source_archive=${3:-}
+  local object_ref=${4:-}
+  local sha256=${5:-}
+  local size_bytes=${6:-0}
+  local verified_epoch=0
+  if [[ "$status" == verified ]]; then
+    verified_epoch=$(date +%s)
+  fi
+  python3 - "$status_tmp" "$status" "$reason" "$source_archive" "$object_ref" "$sha256" "$size_bytes" "$verified_epoch" <<'PY'
+import datetime
+import json
+import sys
+
+payload = {
+    "schema": "platform-offsite-backup-v1",
+    "status": sys.argv[2],
+    "reason": sys.argv[3],
+    "source_archive": sys.argv[4],
+    "object_ref": sys.argv[5],
+    "sha256": sys.argv[6],
+    "size_bytes": int(sys.argv[7]),
+    "verified_epoch": int(sys.argv[8]),
+    "observed_at": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
+}
+with open(sys.argv[1], "w", encoding="utf-8") as target:
+    json.dump(payload, target, ensure_ascii=False, separators=(",", ":"))
+    target.write("\n")
+PY
+  chmod 0644 "$status_tmp"
+  mv -f "$status_tmp" "$status_file"
+}
+
+fail_sync() {
+  local reason=$1
+  local code=${2:-1}
+  write_status failed "$reason" "${source_name:-}" "${remote_archive:-}" "${source_sha:-}" "${source_size:-0}"
+  echo "offsite_backup status=failed reason=$reason" >&2
+  exit "$code"
+}
+
+if [[ ! -f "$offsite_config_file" ]]; then
+  write_status disabled config_missing
+  echo "offsite_backup status=disabled reason=config_missing"
+  exit 0
+fi
+# The installer creates this file as root:root 0600.
+# shellcheck disable=SC1090
+source "$offsite_config_file"
+
+offsite_enabled=${OFFSITE_BACKUP_ENABLED:-false}
+bucket=${OFFSITE_OSS_BUCKET:-}
+prefix=${OFFSITE_OSS_PREFIX:-data-terminal-cloud}
+oss_config=${OFFSITE_OSS_CONFIG_FILE:-/etc/data-terminal-cloud/ossutilconfig}
+
+if [[ "$offsite_enabled" != true ]]; then
+  write_status disabled not_enabled
+  echo "offsite_backup status=disabled reason=not_enabled"
+  exit 0
+fi
+[[ "$bucket" =~ ^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$ ]] || fail_sync invalid_bucket 2
+[[ "$prefix" =~ ^[A-Za-z0-9._/-]+$ && "$prefix" != /* && "$prefix" != *..* ]] || fail_sync invalid_prefix 2
+[[ -x "$ossutil_bin" ]] || fail_sync ossutil_missing 3
+[[ -r "$oss_config" ]] || fail_sync oss_config_missing 4
+
+latest_backup=$(find "$backup_root" -maxdepth 1 -type f -name 'platform-*.tar.gz.enc' -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n1 | cut -d' ' -f2- || true)
+[[ -n "$latest_backup" && -f "$latest_backup" ]] || fail_sync local_backup_missing 5
+[[ -f "$latest_backup.sha256" ]] || fail_sync local_checksum_missing 5
+if ! (cd "$backup_root" && sha256sum -c "$(basename "$latest_backup").sha256" >/dev/null 2>&1); then
+  fail_sync local_checksum_failed 5
+fi
+
+source_name=$(basename "$latest_backup")
+source_sha=$(sha256sum "$latest_backup" | awk '{print $1}')
+source_size=$(stat -c %s "$latest_backup")
+prefix=${prefix%/}
+remote_archive="oss://$bucket/$prefix/$source_name"
+remote_checksum="$remote_archive.sha256"
+checkpoint_dir="$state_dir/checkpoints"
+mkdir -p "$checkpoint_dir"
+
+common=(-f -c "$oss_config" --job 1 --parallel 1 --checkpoint-dir "$checkpoint_dir" --output-format json)
+if ! "$ossutil_bin" cp "$latest_backup" "$remote_archive" "${common[@]}" --metadata "sha256=$source_sha" >/dev/null; then
+  fail_sync archive_upload_failed 6
+fi
+if ! "$ossutil_bin" cp "$latest_backup.sha256" "$remote_checksum" "${common[@]}" >/dev/null; then
+  fail_sync checksum_upload_failed 6
+fi
+
+downloaded="$transfer_dir/$source_name"
+downloaded_checksum="$downloaded.sha256"
+if ! "$ossutil_bin" cp "$remote_archive" "$downloaded" "${common[@]}" >/dev/null; then
+  fail_sync archive_download_verification_failed 7
+fi
+if ! "$ossutil_bin" cp "$remote_checksum" "$downloaded_checksum" "${common[@]}" >/dev/null; then
+  fail_sync checksum_download_verification_failed 7
+fi
+downloaded_sha=$(sha256sum "$downloaded" | awk '{print $1}')
+checksum_sha=$(awk 'NF >= 1 {print $1; exit}' "$downloaded_checksum")
+if [[ "$downloaded_sha" != "$source_sha" || "$checksum_sha" != "$source_sha" ]]; then
+  fail_sync remote_checksum_mismatch 8
+fi
+
+write_status verified upload_and_download_verified "$source_name" "$remote_archive" "$source_sha" "$source_size"
+printf 'offsite_backup status=verified source=%s object=%s size_bytes=%s\n' "$source_name" "$remote_archive" "$source_size"