feat: add encrypted server backup verification

2026-08-26 20:34:49 +08:00
parent 8607ec33e7
commit fb4691c7b2
10 changed files with 429 additions and 4 deletions
@@ -1,6 +1,6 @@
 # 阿里云单机演示部署

-该配置用于单机公网演示，不是生产验收配置。只向公网发布HTTP 80和HTTPS 443；PostgreSQL、Redis、MinIO、Keycloak内部端口以及API/Worker端口均不映射到宿主机。当前阿里云演示机使用Let's Encrypt短期IP证书，`certbot-ip-renew.timer`负责自动续期。真实RTC、模型、内容安全、云KMS、备份恢复和监控告警仍按阶段8验收清单执行。
+该配置用于单机公网演示，不是生产验收配置。只向公网发布HTTP 80和HTTPS 443；PostgreSQL、Redis、MinIO、Keycloak内部端口以及API/Worker端口均不映射到宿主机。当前阿里云演示机使用Let's Encrypt短期IP证书，`certbot-ip-renew.timer`负责自动续期。单机备份覆盖PostgreSQL、Redis、MinIO对象和当前Keycloak数据目录，并执行数据库级隔离恢复校验；跨故障域备份、真实RTC、模型、内容安全、云KMS和生产监控告警仍按阶段8验收清单执行。

 服务器至少需要Docker Engine 29与Compose 2.40。首次部署先生成权限为600的服务器专用配置：

@@ -22,3 +22,24 @@ docker compose -f compose.yaml -f deploy/server/compose.override.yaml \
 ```

 安全组和主机防火墙只保留TCP 80、443以及受限来源的SSH 22。HTTP 80仅用于跳转到标准HTTPS 443。正式环境仍应配置自有域名、生产OIDC和真实外部服务，并切换到Kubernetes生产覆盖层。
+
+## 自动备份与恢复校验
+
+首次安装定时任务：
+
+```bash
+chmod 0755 deploy/server/backup.sh deploy/server/verify-backup.sh deploy/server/install-backup.sh
+sudo deploy/server/install-backup.sh
+sudo systemctl start platform-backup.service
+sudo systemctl status platform-backup.service --no-pager
+```
+
+备份以AES-256-CBC、PBKDF2和独立随机口令加密，默认保存在`/var/backups/data-terminal-cloud`并保留14天。每次定时任务会实际把PostgreSQL恢复到随机隔离数据库，对比迁移状态、客户组织、账号、表和外键计数，再删除临时数据库。Redis RDB、MinIO对象和暂停后的Keycloak数据目录会进入同一加密归档并进行逐文件SHA-256校验；当前单机配置不把这些文件级检查冒充跨故障域生产恢复。
+
+手工验证已有备份：
+
+```bash
+sudo deploy/server/verify-backup.sh /var/backups/data-terminal-cloud/platform-YYYYmmddTHHMMSSZ.tar.gz.enc
+```
+
+加密口令位于`/etc/data-terminal-cloud/backup-passphrase`且仅root可读。必须把该口令和加密备份复制到不同故障域；口令丢失将无法恢复备份。
@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+project_root=${PLATFORM_ROOT:-/opt/data-terminal-cloud/current}
+backup_root=${BACKUP_ROOT:-/var/backups/data-terminal-cloud}
+passphrase_file=${BACKUP_PASSPHRASE_FILE:-/etc/data-terminal-cloud/backup-passphrase}
+retention_days=${BACKUP_RETENTION_DAYS:-14}
+verify_after_backup=${VERIFY_AFTER_BACKUP:-false}
+
+case "$backup_root" in
+  /*) ;;
+  *) echo "BACKUP_ROOT must be an absolute path" >&2; exit 2 ;;
+esac
+if [[ "$backup_root" == "/" || "$backup_root" == "/var" || "$backup_root" == "/var/backups" ]]; then
+  echo "BACKUP_ROOT is too broad" >&2
+  exit 2
+fi
+if [[ ! "$retention_days" =~ ^[1-9][0-9]*$ ]]; then
+  echo "BACKUP_RETENTION_DAYS must be a positive integer" >&2
+  exit 2
+fi
+if [[ ! -r "$passphrase_file" ]]; then
+  echo "Backup passphrase file is missing or unreadable: $passphrase_file" >&2
+  exit 2
+fi
+
+compose=(docker compose -f "$project_root/compose.yaml" -f "$project_root/deploy/server/compose.override.yaml" --env-file "$project_root/deploy/server/.env")
+mkdir -p "$backup_root"
+chmod 0700 "$backup_root"
+umask 077
+
+exec 9>"$backup_root/.backup.lock"
+if ! flock -n 9; then
+  echo "Another backup is already running" >&2
+  exit 3
+fi
+
+stamp=$(date -u +%Y%m%dT%H%M%SZ)
+staging=$(mktemp -d "$backup_root/.staging-${stamp}-XXXXXX")
+plain_archive="$backup_root/.platform-${stamp}.tar.gz"
+encrypted_archive="$backup_root/platform-${stamp}.tar.gz.enc"
+checksum_file="$encrypted_archive.sha256"
+minio_container=""
+keycloak_container=""
+keycloak_paused=false
+
+cleanup() {
+  status=$?
+  if [[ "$keycloak_paused" == true && -n "$keycloak_container" ]]; then
+    docker unpause "$keycloak_container" >/dev/null 2>&1 || true
+  fi
+  if [[ -n "$minio_container" ]]; then
+    docker exec "$minio_container" rm -rf /tmp/platform-backup /tmp/platform-mc >/dev/null 2>&1 || true
+  fi
+  rm -rf -- "$staging"
+  rm -f -- "$plain_archive"
+  exit "$status"
+}
+trap cleanup EXIT INT TERM
+
+service_container() {
+  local service=$1
+  local container
+  container=$("${compose[@]}" ps -q "$service")
+  if [[ -z "$container" ]] || [[ $(docker inspect -f '{{.State.Running}}' "$container") != true ]]; then
+    echo "Required service is not running: $service" >&2
+    exit 4
+  fi
+  printf '%s' "$container"
+}
+
+postgres_container=$(service_container postgres)
+redis_container=$(service_container redis)
+minio_container=$(service_container minio)
+keycloak_container=$(service_container keycloak)
+redis_password=$(sed -n 's/^REDIS_PASSWORD=//p' "$project_root/deploy/server/.env")
+if [[ -z "$redis_password" ]]; then
+  echo "REDIS_PASSWORD is missing from the server environment" >&2
+  exit 4
+fi
+
+docker exec "$postgres_container" pg_dump -U voice -d voice_platform -Fc --no-owner --no-acl >"$staging/postgres.dump"
+docker exec "$postgres_container" psql -U voice -d voice_platform -Atc \
+  "SELECT version||'|'||dirty||'|'||(SELECT count(*) FROM tenant)||'|'||(SELECT count(*) FROM user_account)||'|'||(SELECT count(*) FROM information_schema.tables WHERE table_schema='public')||'|'||(SELECT count(*) FROM information_schema.table_constraints WHERE constraint_type='FOREIGN KEY') FROM schema_migrations;" \
+  >"$staging/database-evidence.txt"
+
+mkdir -p "$staging/redis"
+printf '%s\n' "$redis_password" | docker exec -i "$redis_container" sh -ec '
+  IFS= read -r REDISCLI_AUTH
+  export REDISCLI_AUTH
+  redis-cli --no-auth-warning SAVE >/dev/null
+'
+docker exec "$redis_container" redis-check-rdb /data/dump.rdb >/dev/null
+docker cp "$redis_container:/data/dump.rdb" "$staging/redis/dump.rdb" >/dev/null
+
+docker exec "$minio_container" sh -ec '
+  rm -rf /tmp/platform-backup /tmp/platform-mc
+  mkdir -p /tmp/platform-backup /tmp/platform-mc
+  mc --config-dir /tmp/platform-mc alias set local http://127.0.0.1:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
+  mc --config-dir /tmp/platform-mc mirror --overwrite --preserve local/ /tmp/platform-backup/
+'
+mkdir -p "$staging/object-store"
+docker cp "$minio_container:/tmp/platform-backup/." "$staging/object-store/" >/dev/null
+docker exec "$minio_container" rm -rf /tmp/platform-backup /tmp/platform-mc
+
+docker pause "$keycloak_container" >/dev/null
+keycloak_paused=true
+mkdir -p "$staging/keycloak"
+docker cp "$keycloak_container:/opt/keycloak/data/." "$staging/keycloak/" >/dev/null
+docker unpause "$keycloak_container" >/dev/null
+keycloak_paused=false
+
+object_files=$(find "$staging/object-store" -type f | wc -l | tr -d ' ')
+keycloak_files=$(find "$staging/keycloak" -type f | wc -l | tr -d ' ')
+cat >"$staging/manifest.properties" <<EOF
+format=platform-server-backup-v1
+created_at=${stamp}
+build_version=$(sed -n 's/^PLATFORM_BUILD_VERSION=//p' "$project_root/deploy/server/.env")
+build_commit=$(sed -n 's/^PLATFORM_BUILD_COMMIT=//p' "$project_root/deploy/server/.env")
+database_evidence=$(cat "$staging/database-evidence.txt")
+object_files=${object_files}
+keycloak_files=${keycloak_files}
+redis_snapshot=true
+keycloak_snapshot_mode=paused_filesystem_copy
+EOF
+
+(
+  cd "$staging"
+  find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 -r sha256sum >SHA256SUMS
+)
+tar -C "$staging" -czf "$plain_archive" .
+openssl enc -aes-256-cbc -salt -pbkdf2 -iter 200000 \
+  -pass "file:$passphrase_file" -in "$plain_archive" -out "$encrypted_archive"
+sha256sum "$encrypted_archive" >"$checksum_file"
+chmod 0600 "$encrypted_archive" "$checksum_file"
+
+while IFS= read -r -d '' expired; do
+  rm -f -- "$expired" "$expired.sha256"
+done < <(find "$backup_root" -maxdepth 1 -type f -name 'platform-*.tar.gz.enc' -mtime "+$retention_days" -print0)
+
+if [[ "$verify_after_backup" == true ]]; then
+  "$project_root/deploy/server/verify-backup.sh" "$encrypted_archive"
+fi
+
+printf '%s\n' "$encrypted_archive"
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if [[ ${EUID:-$(id -u)} -ne 0 ]]; then
+  echo "Run this installer as root" >&2
+  exit 2
+fi
+
+project_root=${PLATFORM_ROOT:-/opt/data-terminal-cloud/current}
+install -d -m 0700 /etc/data-terminal-cloud /var/backups/data-terminal-cloud
+if [[ ! -s /etc/data-terminal-cloud/backup-passphrase ]]; then
+  umask 077
+  openssl rand -base64 48 >/etc/data-terminal-cloud/backup-passphrase
+fi
+chmod 0600 /etc/data-terminal-cloud/backup-passphrase
+install -m 0644 "$project_root/deploy/server/platform-backup.service" /etc/systemd/system/platform-backup.service
+install -m 0644 "$project_root/deploy/server/platform-backup.timer" /etc/systemd/system/platform-backup.timer
+systemctl daemon-reload
+systemctl enable --now platform-backup.timer
+systemctl is-active --quiet platform-backup.timer
+echo "platform-backup.timer enabled"
@@ -0,0 +1,19 @@
+[Unit]
+Description=Encrypted data terminal cloud backup and restore verification
+Requires=docker.service
+After=docker.service network-online.target
+
+[Service]
+Type=oneshot
+User=root
+Group=root
+UMask=0077
+Environment=PLATFORM_ROOT=/opt/data-terminal-cloud/current
+Environment=BACKUP_ROOT=/var/backups/data-terminal-cloud
+Environment=BACKUP_RETENTION_DAYS=14
+Environment=VERIFY_AFTER_BACKUP=true
+ExecStart=/opt/data-terminal-cloud/current/deploy/server/backup.sh
+Nice=10
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+TimeoutStartSec=2h
@@ -0,0 +1,11 @@
+[Unit]
+Description=Run encrypted data terminal cloud backup daily
+
+[Timer]
+OnCalendar=*-*-* 02:40:00
+RandomizedDelaySec=45m
+Persistent=true
+Unit=platform-backup.service
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+project_root=${PLATFORM_ROOT:-/opt/data-terminal-cloud/current}
+backup_root=${BACKUP_ROOT:-/var/backups/data-terminal-cloud}
+passphrase_file=${BACKUP_PASSPHRASE_FILE:-/etc/data-terminal-cloud/backup-passphrase}
+archive=${1:-}
+
+if [[ -z "$archive" ]]; then
+  archive=$(find "$backup_root" -maxdepth 1 -type f -name 'platform-*.tar.gz.enc' -printf '%T@ %p\n' | sort -nr | head -n1 | cut -d' ' -f2-)
+fi
+if [[ -z "$archive" || ! -f "$archive" ]]; then
+  echo "Backup archive was not found" >&2
+  exit 2
+fi
+archive=$(realpath "$archive")
+backup_root=$(realpath "$backup_root")
+case "$archive" in
+  "$backup_root"/platform-*.tar.gz.enc) ;;
+  *) echo "Backup archive is outside the configured backup directory" >&2; exit 2 ;;
+esac
+if [[ ! -r "$passphrase_file" ]]; then
+  echo "Backup passphrase file is missing or unreadable" >&2
+  exit 2
+fi
+
+compose=(docker compose -f "$project_root/compose.yaml" -f "$project_root/deploy/server/compose.override.yaml" --env-file "$project_root/deploy/server/.env")
+postgres_container=$("${compose[@]}" ps -q postgres)
+if [[ -z "$postgres_container" ]] || [[ $(docker inspect -f '{{.State.Running}}' "$postgres_container") != true ]]; then
+  echo "PostgreSQL service is not running" >&2
+  exit 3
+fi
+redis_container=$("${compose[@]}" ps -q redis)
+if [[ -z "$redis_container" ]] || [[ $(docker inspect -f '{{.State.Running}}' "$redis_container") != true ]]; then
+  echo "Redis service is not running" >&2
+  exit 3
+fi
+
+umask 077
+work=$(mktemp -d "$backup_root/.verify-XXXXXX")
+plain="$work/backup.tar.gz"
+verify_token=$(openssl rand -hex 12)
+restore_database="voice_restore_verify_$verify_token"
+redis_verify_path="/tmp/platform-backup-$verify_token.rdb"
+database_created=false
+
+cleanup() {
+  status=$?
+  if [[ "$database_created" == true ]]; then
+    docker exec "$postgres_container" dropdb -U voice --if-exists "$restore_database" >/dev/null 2>&1 || true
+  fi
+  docker exec "$redis_container" rm -f -- "$redis_verify_path" >/dev/null 2>&1 || true
+  rm -rf -- "$work"
+  exit "$status"
+}
+trap cleanup EXIT INT TERM
+
+if [[ -f "$archive.sha256" ]]; then
+  (cd "$(dirname "$archive")" && sha256sum -c "$(basename "$archive").sha256") >/dev/null
+fi
+openssl enc -d -aes-256-cbc -pbkdf2 -iter 200000 \
+  -pass "file:$passphrase_file" -in "$archive" -out "$plain"
+tar -tzf "$plain" >"$work/archive-paths.txt"
+if grep -Eq '(^/|(^|/)\.\.(/|$))' "$work/archive-paths.txt"; then
+  echo "Backup contains an unsafe archive path" >&2
+  exit 4
+fi
+mkdir -p "$work/extracted"
+tar -xzf "$plain" --no-same-owner -C "$work/extracted"
+(
+  cd "$work/extracted"
+  sha256sum -c SHA256SUMS >/dev/null
+)
+
+expected=$(tr -d '\r\n' <"$work/extracted/database-evidence.txt")
+docker exec "$postgres_container" createdb -U voice "$restore_database"
+database_created=true
+docker exec -i "$postgres_container" pg_restore -U voice -d "$restore_database" --no-owner --no-acl <"$work/extracted/postgres.dump"
+actual=$(docker exec "$postgres_container" psql -U voice -d "$restore_database" -Atc \
+  "SELECT version||'|'||dirty||'|'||(SELECT count(*) FROM tenant)||'|'||(SELECT count(*) FROM user_account)||'|'||(SELECT count(*) FROM information_schema.tables WHERE table_schema='public')||'|'||(SELECT count(*) FROM information_schema.table_constraints WHERE constraint_type='FOREIGN KEY') FROM schema_migrations;")
+if [[ "$actual" != "$expected" ]]; then
+  echo "Restored database evidence does not match the backup: expected=$expected actual=$actual" >&2
+  exit 5
+fi
+
+[[ -s "$work/extracted/redis/dump.rdb" ]] || { echo "Redis snapshot is missing" >&2; exit 6; }
+[[ -d "$work/extracted/object-store" ]] || { echo "Object-store snapshot is missing" >&2; exit 6; }
+[[ -d "$work/extracted/keycloak" ]] || { echo "Keycloak snapshot is missing" >&2; exit 6; }
+expected_object_files=$(sed -n 's/^object_files=//p' "$work/extracted/manifest.properties")
+expected_keycloak_files=$(sed -n 's/^keycloak_files=//p' "$work/extracted/manifest.properties")
+actual_object_files=$(find "$work/extracted/object-store" -type f | wc -l | tr -d ' ')
+actual_keycloak_files=$(find "$work/extracted/keycloak" -type f | wc -l | tr -d ' ')
+if [[ ! "$expected_object_files" =~ ^[0-9]+$ || "$actual_object_files" != "$expected_object_files" ]]; then
+  echo "Object-store file count does not match the manifest" >&2
+  exit 6
+fi
+if [[ ! "$expected_keycloak_files" =~ ^[1-9][0-9]*$ || "$actual_keycloak_files" != "$expected_keycloak_files" ]]; then
+  echo "Keycloak file count does not match the manifest" >&2
+  exit 6
+fi
+docker cp "$work/extracted/redis/dump.rdb" "$redis_container:$redis_verify_path" >/dev/null
+docker exec "$redis_container" redis-check-rdb "$redis_verify_path" >/dev/null
+docker exec "$redis_container" rm -f -- "$redis_verify_path"
+
+printf 'backup_restore_verified archive=%s database_evidence=%s object_files=%s keycloak_files=%s redis_rdb=valid\n' \
+  "$archive" "$actual" "$actual_object_files" "$actual_keycloak_files"