feat: add OSS offsite backup gate

2026-08-26 20:55:35 +08:00
parent 0bf375155e
commit 1e12d329dc
10 changed files with 384 additions and 4 deletions
@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+backup_root=${BACKUP_ROOT:-/var/backups/data-terminal-cloud}
+state_dir=${OFFSITE_STATE_DIR:-/var/lib/data-terminal-cloud-offsite}
+offsite_config_file=${OFFSITE_CONFIG_FILE:-/etc/data-terminal-cloud/offsite-backup.env}
+ossutil_bin=${OSSUTIL_BIN:-/usr/local/bin/ossutil}
+
+mkdir -p "$state_dir"
+chmod 0750 "$state_dir"
+umask 077
+status_file="$state_dir/status.json"
+status_tmp=$(mktemp "$state_dir/.status-XXXXXX")
+transfer_dir=$(mktemp -d "$state_dir/.transfer-XXXXXX")
+
+cleanup() {
+  rm -f -- "$status_tmp"
+  rm -rf -- "$transfer_dir"
+}
+trap cleanup EXIT INT TERM
+
+write_status() {
+  local status=$1
+  local reason=$2
+  local source_archive=${3:-}
+  local object_ref=${4:-}
+  local sha256=${5:-}
+  local size_bytes=${6:-0}
+  local verified_epoch=0
+  if [[ "$status" == verified ]]; then
+    verified_epoch=$(date +%s)
+  fi
+  python3 - "$status_tmp" "$status" "$reason" "$source_archive" "$object_ref" "$sha256" "$size_bytes" "$verified_epoch" <<'PY'
+import datetime
+import json
+import sys
+
+payload = {
+    "schema": "platform-offsite-backup-v1",
+    "status": sys.argv[2],
+    "reason": sys.argv[3],
+    "source_archive": sys.argv[4],
+    "object_ref": sys.argv[5],
+    "sha256": sys.argv[6],
+    "size_bytes": int(sys.argv[7]),
+    "verified_epoch": int(sys.argv[8]),
+    "observed_at": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
+}
+with open(sys.argv[1], "w", encoding="utf-8") as target:
+    json.dump(payload, target, ensure_ascii=False, separators=(",", ":"))
+    target.write("\n")
+PY
+  chmod 0644 "$status_tmp"
+  mv -f "$status_tmp" "$status_file"
+}
+
+fail_sync() {
+  local reason=$1
+  local code=${2:-1}
+  write_status failed "$reason" "${source_name:-}" "${remote_archive:-}" "${source_sha:-}" "${source_size:-0}"
+  echo "offsite_backup status=failed reason=$reason" >&2
+  exit "$code"
+}
+
+if [[ ! -f "$offsite_config_file" ]]; then
+  write_status disabled config_missing
+  echo "offsite_backup status=disabled reason=config_missing"
+  exit 0
+fi
+# The installer creates this file as root:root 0600.
+# shellcheck disable=SC1090
+source "$offsite_config_file"
+
+offsite_enabled=${OFFSITE_BACKUP_ENABLED:-false}
+bucket=${OFFSITE_OSS_BUCKET:-}
+prefix=${OFFSITE_OSS_PREFIX:-data-terminal-cloud}
+oss_config=${OFFSITE_OSS_CONFIG_FILE:-/etc/data-terminal-cloud/ossutilconfig}
+
+if [[ "$offsite_enabled" != true ]]; then
+  write_status disabled not_enabled
+  echo "offsite_backup status=disabled reason=not_enabled"
+  exit 0
+fi
+[[ "$bucket" =~ ^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$ ]] || fail_sync invalid_bucket 2
+[[ "$prefix" =~ ^[A-Za-z0-9._/-]+$ && "$prefix" != /* && "$prefix" != *..* ]] || fail_sync invalid_prefix 2
+[[ -x "$ossutil_bin" ]] || fail_sync ossutil_missing 3
+[[ -r "$oss_config" ]] || fail_sync oss_config_missing 4
+
+latest_backup=$(find "$backup_root" -maxdepth 1 -type f -name 'platform-*.tar.gz.enc' -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n1 | cut -d' ' -f2- || true)
+[[ -n "$latest_backup" && -f "$latest_backup" ]] || fail_sync local_backup_missing 5
+[[ -f "$latest_backup.sha256" ]] || fail_sync local_checksum_missing 5
+if ! (cd "$backup_root" && sha256sum -c "$(basename "$latest_backup").sha256" >/dev/null 2>&1); then
+  fail_sync local_checksum_failed 5
+fi
+
+source_name=$(basename "$latest_backup")
+source_sha=$(sha256sum "$latest_backup" | awk '{print $1}')
+source_size=$(stat -c %s "$latest_backup")
+prefix=${prefix%/}
+remote_archive="oss://$bucket/$prefix/$source_name"
+remote_checksum="$remote_archive.sha256"
+checkpoint_dir="$state_dir/checkpoints"
+mkdir -p "$checkpoint_dir"
+
+common=(-f -c "$oss_config" --job 1 --parallel 1 --checkpoint-dir "$checkpoint_dir" --output-format json)
+if ! "$ossutil_bin" cp "$latest_backup" "$remote_archive" "${common[@]}" --metadata "sha256=$source_sha" >/dev/null; then
+  fail_sync archive_upload_failed 6
+fi
+if ! "$ossutil_bin" cp "$latest_backup.sha256" "$remote_checksum" "${common[@]}" >/dev/null; then
+  fail_sync checksum_upload_failed 6
+fi
+
+downloaded="$transfer_dir/$source_name"
+downloaded_checksum="$downloaded.sha256"
+if ! "$ossutil_bin" cp "$remote_archive" "$downloaded" "${common[@]}" >/dev/null; then
+  fail_sync archive_download_verification_failed 7
+fi
+if ! "$ossutil_bin" cp "$remote_checksum" "$downloaded_checksum" "${common[@]}" >/dev/null; then
+  fail_sync checksum_download_verification_failed 7
+fi
+downloaded_sha=$(sha256sum "$downloaded" | awk '{print $1}')
+checksum_sha=$(awk 'NF >= 1 {print $1; exit}' "$downloaded_checksum")
+if [[ "$downloaded_sha" != "$source_sha" || "$checksum_sha" != "$source_sha" ]]; then
+  fail_sync remote_checksum_mismatch 8
+fi
+
+write_status verified upload_and_download_verified "$source_name" "$remote_archive" "$source_sha" "$source_size"
+printf 'offsite_backup status=verified source=%s object=%s size_bytes=%s\n' "$source_name" "$remote_archive" "$source_size"