107 lines
4.8 KiB
Bash
107 lines
4.8 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
set -Eeuo pipefail
|
||
|
|
|
||
|
|
project_root=${PLATFORM_ROOT:-/opt/data-terminal-cloud/current}
|
||
|
|
backup_root=${BACKUP_ROOT:-/var/backups/data-terminal-cloud}
|
||
|
|
passphrase_file=${BACKUP_PASSPHRASE_FILE:-/etc/data-terminal-cloud/backup-passphrase}
|
||
|
|
archive=${1:-}
|
||
|
|
|
||
|
|
if [[ -z "$archive" ]]; then
|
||
|
|
archive=$(find "$backup_root" -maxdepth 1 -type f -name 'platform-*.tar.gz.enc' -printf '%T@ %p\n' | sort -nr | head -n1 | cut -d' ' -f2-)
|
||
|
|
fi
|
||
|
|
if [[ -z "$archive" || ! -f "$archive" ]]; then
|
||
|
|
echo "Backup archive was not found" >&2
|
||
|
|
exit 2
|
||
|
|
fi
|
||
|
|
archive=$(realpath "$archive")
|
||
|
|
backup_root=$(realpath "$backup_root")
|
||
|
|
case "$archive" in
|
||
|
|
"$backup_root"/platform-*.tar.gz.enc) ;;
|
||
|
|
*) echo "Backup archive is outside the configured backup directory" >&2; exit 2 ;;
|
||
|
|
esac
|
||
|
|
if [[ ! -r "$passphrase_file" ]]; then
|
||
|
|
echo "Backup passphrase file is missing or unreadable" >&2
|
||
|
|
exit 2
|
||
|
|
fi
|
||
|
|
|
||
|
|
compose=(docker compose -f "$project_root/compose.yaml" -f "$project_root/deploy/server/compose.override.yaml" --env-file "$project_root/deploy/server/.env")
|
||
|
|
postgres_container=$("${compose[@]}" ps -q postgres)
|
||
|
|
if [[ -z "$postgres_container" ]] || [[ $(docker inspect -f '{{.State.Running}}' "$postgres_container") != true ]]; then
|
||
|
|
echo "PostgreSQL service is not running" >&2
|
||
|
|
exit 3
|
||
|
|
fi
|
||
|
|
redis_container=$("${compose[@]}" ps -q redis)
|
||
|
|
if [[ -z "$redis_container" ]] || [[ $(docker inspect -f '{{.State.Running}}' "$redis_container") != true ]]; then
|
||
|
|
echo "Redis service is not running" >&2
|
||
|
|
exit 3
|
||
|
|
fi
|
||
|
|
|
||
|
|
umask 077
|
||
|
|
work=$(mktemp -d "$backup_root/.verify-XXXXXX")
|
||
|
|
plain="$work/backup.tar.gz"
|
||
|
|
verify_token=$(openssl rand -hex 12)
|
||
|
|
restore_database="voice_restore_verify_$verify_token"
|
||
|
|
redis_verify_path="/tmp/platform-backup-$verify_token.rdb"
|
||
|
|
database_created=false
|
||
|
|
|
||
|
|
cleanup() {
|
||
|
|
status=$?
|
||
|
|
if [[ "$database_created" == true ]]; then
|
||
|
|
docker exec "$postgres_container" dropdb -U voice --if-exists "$restore_database" >/dev/null 2>&1 || true
|
||
|
|
fi
|
||
|
|
docker exec "$redis_container" rm -f -- "$redis_verify_path" >/dev/null 2>&1 || true
|
||
|
|
rm -rf -- "$work"
|
||
|
|
exit "$status"
|
||
|
|
}
|
||
|
|
trap cleanup EXIT INT TERM
|
||
|
|
|
||
|
|
if [[ -f "$archive.sha256" ]]; then
|
||
|
|
(cd "$(dirname "$archive")" && sha256sum -c "$(basename "$archive").sha256") >/dev/null
|
||
|
|
fi
|
||
|
|
openssl enc -d -aes-256-cbc -pbkdf2 -iter 200000 \
|
||
|
|
-pass "file:$passphrase_file" -in "$archive" -out "$plain"
|
||
|
|
tar -tzf "$plain" >"$work/archive-paths.txt"
|
||
|
|
if grep -Eq '(^/|(^|/)\.\.(/|$))' "$work/archive-paths.txt"; then
|
||
|
|
echo "Backup contains an unsafe archive path" >&2
|
||
|
|
exit 4
|
||
|
|
fi
|
||
|
|
mkdir -p "$work/extracted"
|
||
|
|
tar -xzf "$plain" --no-same-owner -C "$work/extracted"
|
||
|
|
(
|
||
|
|
cd "$work/extracted"
|
||
|
|
sha256sum -c SHA256SUMS >/dev/null
|
||
|
|
)
|
||
|
|
|
||
|
|
expected=$(tr -d '\r\n' <"$work/extracted/database-evidence.txt")
|
||
|
|
docker exec "$postgres_container" createdb -U voice "$restore_database"
|
||
|
|
database_created=true
|
||
|
|
docker exec -i "$postgres_container" pg_restore -U voice -d "$restore_database" --no-owner --no-acl <"$work/extracted/postgres.dump"
|
||
|
|
actual=$(docker exec "$postgres_container" psql -U voice -d "$restore_database" -Atc \
|
||
|
|
"SELECT version||'|'||dirty||'|'||(SELECT count(*) FROM tenant)||'|'||(SELECT count(*) FROM user_account)||'|'||(SELECT count(*) FROM information_schema.tables WHERE table_schema='public')||'|'||(SELECT count(*) FROM information_schema.table_constraints WHERE constraint_type='FOREIGN KEY') FROM schema_migrations;")
|
||
|
|
if [[ "$actual" != "$expected" ]]; then
|
||
|
|
echo "Restored database evidence does not match the backup: expected=$expected actual=$actual" >&2
|
||
|
|
exit 5
|
||
|
|
fi
|
||
|
|
|
||
|
|
[[ -s "$work/extracted/redis/dump.rdb" ]] || { echo "Redis snapshot is missing" >&2; exit 6; }
|
||
|
|
[[ -d "$work/extracted/object-store" ]] || { echo "Object-store snapshot is missing" >&2; exit 6; }
|
||
|
|
[[ -d "$work/extracted/keycloak" ]] || { echo "Keycloak snapshot is missing" >&2; exit 6; }
|
||
|
|
expected_object_files=$(sed -n 's/^object_files=//p' "$work/extracted/manifest.properties")
|
||
|
|
expected_keycloak_files=$(sed -n 's/^keycloak_files=//p' "$work/extracted/manifest.properties")
|
||
|
|
actual_object_files=$(find "$work/extracted/object-store" -type f | wc -l | tr -d ' ')
|
||
|
|
actual_keycloak_files=$(find "$work/extracted/keycloak" -type f | wc -l | tr -d ' ')
|
||
|
|
if [[ ! "$expected_object_files" =~ ^[0-9]+$ || "$actual_object_files" != "$expected_object_files" ]]; then
|
||
|
|
echo "Object-store file count does not match the manifest" >&2
|
||
|
|
exit 6
|
||
|
|
fi
|
||
|
|
if [[ ! "$expected_keycloak_files" =~ ^[1-9][0-9]*$ || "$actual_keycloak_files" != "$expected_keycloak_files" ]]; then
|
||
|
|
echo "Keycloak file count does not match the manifest" >&2
|
||
|
|
exit 6
|
||
|
|
fi
|
||
|
|
docker cp "$work/extracted/redis/dump.rdb" "$redis_container:$redis_verify_path" >/dev/null
|
||
|
|
docker exec "$redis_container" redis-check-rdb "$redis_verify_path" >/dev/null
|
||
|
|
docker exec "$redis_container" rm -f -- "$redis_verify_path"
|
||
|
|
|
||
|
|
printf 'backup_restore_verified archive=%s database_evidence=%s object_files=%s keycloak_files=%s redis_rdb=valid\n' \
|
||
|
|
"$archive" "$actual" "$actual_object_files" "$actual_keycloak_files"
|