story/deploy/server/backup.sh

#!/usr/bin/env bash
set -Eeuo pipefail

project_root=${PLATFORM_ROOT:-/opt/data-terminal-cloud/current}
backup_root=${BACKUP_ROOT:-/var/backups/data-terminal-cloud}
passphrase_file=${BACKUP_PASSPHRASE_FILE:-/etc/data-terminal-cloud/backup-passphrase}
retention_days=${BACKUP_RETENTION_DAYS:-14}
verify_after_backup=${VERIFY_AFTER_BACKUP:-false}

case "$backup_root" in
  /*) ;;
  *) echo "BACKUP_ROOT must be an absolute path" >&2; exit 2 ;;
esac
if [[ "$backup_root" == "/" || "$backup_root" == "/var" || "$backup_root" == "/var/backups" ]]; then
  echo "BACKUP_ROOT is too broad" >&2
  exit 2
fi
if [[ ! "$retention_days" =~ ^[1-9][0-9]*$ ]]; then
  echo "BACKUP_RETENTION_DAYS must be a positive integer" >&2
  exit 2
fi
if [[ ! -r "$passphrase_file" ]]; then
  echo "Backup passphrase file is missing or unreadable: $passphrase_file" >&2
  exit 2
fi

compose=(docker compose -f "$project_root/compose.yaml" -f "$project_root/deploy/server/compose.override.yaml" --env-file "$project_root/deploy/server/.env")
mkdir -p "$backup_root"
chmod 0700 "$backup_root"
umask 077

exec 9>"$backup_root/.backup.lock"
if ! flock -n 9; then
  echo "Another backup is already running" >&2
  exit 3
fi

stamp=$(date -u +%Y%m%dT%H%M%SZ)
staging=$(mktemp -d "$backup_root/.staging-${stamp}-XXXXXX")
plain_archive="$backup_root/.platform-${stamp}.tar.gz"
encrypted_archive="$backup_root/platform-${stamp}.tar.gz.enc"
checksum_file="$encrypted_archive.sha256"
minio_container=""
keycloak_container=""
keycloak_paused=false

cleanup() {
  status=$?
  if [[ "$keycloak_paused" == true && -n "$keycloak_container" ]]; then
    docker unpause "$keycloak_container" >/dev/null 2>&1 || true
  fi
  if [[ -n "$minio_container" ]]; then
    docker exec "$minio_container" rm -rf /tmp/platform-backup /tmp/platform-mc >/dev/null 2>&1 || true
  fi
  rm -rf -- "$staging"
  rm -f -- "$plain_archive"
  exit "$status"
}
trap cleanup EXIT INT TERM

service_container() {
  local service=$1
  local container
  container=$("${compose[@]}" ps -q "$service")
  if [[ -z "$container" ]] || [[ $(docker inspect -f '{{.State.Running}}' "$container") != true ]]; then
    echo "Required service is not running: $service" >&2
    exit 4
  fi
  printf '%s' "$container"
}

postgres_container=$(service_container postgres)
redis_container=$(service_container redis)
minio_container=$(service_container minio)
keycloak_container=$(service_container keycloak)
redis_password=$(sed -n 's/^REDIS_PASSWORD=//p' "$project_root/deploy/server/.env")
if [[ -z "$redis_password" ]]; then
  echo "REDIS_PASSWORD is missing from the server environment" >&2
  exit 4
fi

docker exec "$postgres_container" pg_dump -U voice -d voice_platform -Fc --no-owner --no-acl >"$staging/postgres.dump"
docker exec "$postgres_container" psql -U voice -d voice_platform -Atc \
  "SELECT version||'|'||dirty||'|'||(SELECT count(*) FROM tenant)||'|'||(SELECT count(*) FROM user_account)||'|'||(SELECT count(*) FROM information_schema.tables WHERE table_schema='public')||'|'||(SELECT count(*) FROM information_schema.table_constraints WHERE constraint_type='FOREIGN KEY') FROM schema_migrations;" \
  >"$staging/database-evidence.txt"

mkdir -p "$staging/redis"
printf '%s\n' "$redis_password" | docker exec -i "$redis_container" sh -ec '
  IFS= read -r REDISCLI_AUTH
  export REDISCLI_AUTH
  redis-cli --no-auth-warning SAVE >/dev/null
'
docker exec "$redis_container" redis-check-rdb /data/dump.rdb >/dev/null
docker cp "$redis_container:/data/dump.rdb" "$staging/redis/dump.rdb" >/dev/null

docker exec "$minio_container" sh -ec '
  rm -rf /tmp/platform-backup /tmp/platform-mc
  mkdir -p /tmp/platform-backup /tmp/platform-mc
  mc --config-dir /tmp/platform-mc alias set local http://127.0.0.1:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
  mc --config-dir /tmp/platform-mc mirror --overwrite --preserve local/ /tmp/platform-backup/
'
mkdir -p "$staging/object-store"
docker cp "$minio_container:/tmp/platform-backup/." "$staging/object-store/" >/dev/null
docker exec "$minio_container" rm -rf /tmp/platform-backup /tmp/platform-mc

docker pause "$keycloak_container" >/dev/null
keycloak_paused=true
mkdir -p "$staging/keycloak"
docker cp "$keycloak_container:/opt/keycloak/data/." "$staging/keycloak/" >/dev/null
docker unpause "$keycloak_container" >/dev/null
keycloak_paused=false

object_files=$(find "$staging/object-store" -type f | wc -l | tr -d ' ')
keycloak_files=$(find "$staging/keycloak" -type f | wc -l | tr -d ' ')
cat >"$staging/manifest.properties" <<EOF
format=platform-server-backup-v1
created_at=${stamp}
build_version=$(sed -n 's/^PLATFORM_BUILD_VERSION=//p' "$project_root/deploy/server/.env")
build_commit=$(sed -n 's/^PLATFORM_BUILD_COMMIT=//p' "$project_root/deploy/server/.env")
database_evidence=$(cat "$staging/database-evidence.txt")
object_files=${object_files}
keycloak_files=${keycloak_files}
redis_snapshot=true
keycloak_snapshot_mode=paused_filesystem_copy
EOF

(
  cd "$staging"
  find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 -r sha256sum >SHA256SUMS
)
tar -C "$staging" -czf "$plain_archive" .
openssl enc -aes-256-cbc -salt -pbkdf2 -iter 200000 \
  -pass "file:$passphrase_file" -in "$plain_archive" -out "$encrypted_archive"
sha256sum "$encrypted_archive" >"$checksum_file"
chmod 0600 "$encrypted_archive" "$checksum_file"

while IFS= read -r -d '' expired; do
  rm -f -- "$expired" "$expired.sha256"
done < <(find "$backup_root" -maxdepth 1 -type f -name 'platform-*.tar.gz.enc' -mtime "+$retention_days" -print0)

if [[ "$verify_after_backup" == true ]]; then
  "$project_root/deploy/server/verify-backup.sh" "$encrypted_archive"
fi

printf '%s\n' "$encrypted_archive"