wangcong/story
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: support ECS RAM role backups

Browse Source
This commit is contained in:
Codex Local
2026-08-26 22:27:49 +08:00
parent e4f7701439
commit 7435af7c07
8 changed files with 176 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/server/README.md
+1 -1
View File
@@ -65,4 +65,4 @@ chmod 0755 deploy/server/install-ossutil.sh \
sudo deploy/server/install-offsite-backup.sh
```
安装后默认为禁用状态，不会访问OSS。创建私有Bucket并绑定最小权限ECS RAM角色后，在`/etc/data-terminal-cloud/offsite-backup.env`填写Bucket和前缀、设置`OFFSITE_BACKUP_ENABLED=true`与`OFFSITE_BACKUP_REQUIRED=true`，再手工启动一次`platform-offsite-backup.service`。任务会上传加密归档及摘要、重新下载并复算SHA-256；健康检查在启用后强制要求最新本机归档已有对应远端验证。配置和真实验收步骤见[《阿里云OSS异地备份适配与门禁》](../../docs/offsite-backup.md)。
安装后默认为禁用状态，不会访问OSS。默认认证方式为`ecs_ram_role`，使用`cn-hangzhou`地域端点并强制先通过IMDSv2确认实例角色，不保存长期AccessKey。创建私有Bucket并绑定最小权限ECS RAM角色后，在`/etc/data-terminal-cloud/offsite-backup.env`填写Bucket和前缀、设置`OFFSITE_BACKUP_ENABLED=true`与`OFFSITE_BACKUP_REQUIRED=true`，再手工启动一次`platform-offsite-backup.service`。任务会上传加密归档及摘要、重新下载并复算SHA-256；健康检查在启用后强制要求最新本机归档已有对应远端验证。配置和真实验收步骤见[《阿里云OSS异地备份适配与门禁》](../../docs/offsite-backup.md)。
deploy/server/install-offsite-backup.sh
+3
View File
@@ -16,6 +16,9 @@ OFFSITE_BACKUP_ENABLED=false
OFFSITE_BACKUP_REQUIRED=false
OFFSITE_OSS_BUCKET=
OFFSITE_OSS_PREFIX=data-terminal-cloud/121.41.46.238
OFFSITE_OSS_AUTH_MODE=ecs_ram_role
OFFSITE_OSS_REGION=cn-hangzhou
OFFSITE_OSS_ENDPOINT=oss-cn-hangzhou.aliyuncs.com
OFFSITE_OSS_CONFIG_FILE=/etc/data-terminal-cloud/ossutilconfig
EOF
fi
deploy/server/sync-backup-offsite.sh
+33 -3
View File
@@ -74,7 +74,11 @@ source "$offsite_config_file"
offsite_enabled=${OFFSITE_BACKUP_ENABLED:-false}
bucket=${OFFSITE_OSS_BUCKET:-}
prefix=${OFFSITE_OSS_PREFIX:-data-terminal-cloud}
auth_mode=${OFFSITE_OSS_AUTH_MODE:-config}
oss_region=${OFFSITE_OSS_REGION:-}
oss_endpoint=${OFFSITE_OSS_ENDPOINT:-}
oss_config=${OFFSITE_OSS_CONFIG_FILE:-/etc/data-terminal-cloud/ossutilconfig}
ecs_metadata_base=${ECS_METADATA_BASE_URL:-http://100.100.100.200/latest}
if [[ "$offsite_enabled" != true ]]; then
write_status disabled not_enabled
@@ -84,7 +88,35 @@ fi
[[ "$bucket" =~ ^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$ ]] || fail_sync invalid_bucket 2
[[ "$prefix" =~ ^[A-Za-z0-9._/-]+$ && "$prefix" != /* && "$prefix" != *..* ]] || fail_sync invalid_prefix 2
[[ -x "$ossutil_bin" ]] || fail_sync ossutil_missing 3
[[ -r "$oss_config" ]] || fail_sync oss_config_missing 4
checkpoint_dir="$state_dir/checkpoints"
common=(-f --job 1 --parallel 1 --checkpoint-dir "$checkpoint_dir" --output-format json)
case "$auth_mode" in
config)
[[ -r "$oss_config" ]] || fail_sync oss_config_missing 4
common+=(-c "$oss_config")
;;
ecs_ram_role)
command -v curl >/dev/null 2>&1 || fail_sync curl_missing 3
[[ "$oss_region" =~ ^[a-z0-9][a-z0-9-]{1,62}$ ]] || fail_sync invalid_oss_region 2
if [[ -z "$oss_endpoint" ]]; then
oss_endpoint="oss-$oss_region.aliyuncs.com"
fi
[[ "$oss_endpoint" =~ ^[a-z0-9][a-z0-9.-]{1,251}[a-z0-9]$ && "$oss_endpoint" != *..* ]] || fail_sync invalid_oss_endpoint 2
metadata_token=$(curl --silent --show-error --fail --max-time 3 -X PUT \
-H 'X-aliyun-ecs-metadata-token-ttl-seconds: 60' "$ecs_metadata_base/api/token" || true)
[[ -n "$metadata_token" ]] || fail_sync ecs_metadata_v2_unavailable 4
role_response="$transfer_dir/ecs-role-name"
role_code=$(curl --silent --show-error --max-time 3 --output "$role_response" --write-out '%{http_code}' \
-H "X-aliyun-ecs-metadata-token: $metadata_token" "$ecs_metadata_base/meta-data/ram/security-credentials/" || true)
role_name=$(tr -d '\r\n' <"$role_response")
[[ "$role_code" == 200 && "$role_name" =~ ^[A-Za-z0-9@._-]{1,64}$ ]] || fail_sync ecs_ram_role_missing 4
common+=(--mode EcsRamRole --region "$oss_region" --endpoint "$oss_endpoint")
;;
*)
fail_sync invalid_auth_mode 2
;;
esac
latest_backup=$(find "$backup_root" -maxdepth 1 -type f -name 'platform-*.tar.gz.enc' -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n1 | cut -d' ' -f2- || true)
[[ -n "$latest_backup" && -f "$latest_backup" ]] || fail_sync local_backup_missing 5
@@ -99,10 +131,8 @@ source_size=$(stat -c %s "$latest_backup")
prefix=${prefix%/}
remote_archive="oss://$bucket/$prefix/$source_name"
remote_checksum="$remote_archive.sha256"
checkpoint_dir="$state_dir/checkpoints"
mkdir -p "$checkpoint_dir"
common=(-f -c "$oss_config" --job 1 --parallel 1 --checkpoint-dir "$checkpoint_dir" --output-format json)
if ! "$ossutil_bin" cp "$latest_backup" "$remote_archive" "${common[@]}" --metadata "sha256=$source_sha" >/dev/null; then
fail_sync archive_upload_failed 6
fi
deploy/server/test-sync-backup-offsite.sh
+121
View File
@@ -0,0 +1,121 @@
#!/usr/bin/env bash
set -Eeuo pipefail
project_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
candidate="$project_root/deploy/server/sync-backup-offsite.sh"
bash -n "$candidate"
test_root=$(mktemp -d /var/tmp/offsite-backup-test-XXXXXX)
case "$test_root" in /var/tmp/offsite-backup-test-*) ;; *) exit 90;; esac
metadata_pid=
cleanup() {
if [[ -n "$metadata_pid" ]]; then kill "$metadata_pid" 2>/dev/null || true; fi
rm -rf -- "$test_root"
}
trap cleanup EXIT INT TERM
mkdir -p "$test_root/backups" "$test_root/state" "$test_root/mock-oss"
role_mode_file="$test_root/role-mode"
port_file="$test_root/metadata-port"
printf 'missing\n' >"$role_mode_file"
python3 - "$port_file" "$role_mode_file" <<'PY' &
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import pathlib,sys
port_file=pathlib.Path(sys.argv[1])
role_mode=pathlib.Path(sys.argv[2])
class Handler(BaseHTTPRequestHandler):
def log_message(self,*args): pass
def do_PUT(self):
if self.path!='/latest/api/token': self.send_error(404); return
self.send_response(200); self.end_headers(); self.wfile.write(b'test-imdsv2-token')
def do_GET(self):
if self.path!='/latest/meta-data/ram/security-credentials/': self.send_error(404); return
if role_mode.read_text(encoding='utf-8').strip()!='attached': self.send_error(404); return
if self.headers.get('X-aliyun-ecs-metadata-token')!='test-imdsv2-token': self.send_error(401); return
self.send_response(200); self.end_headers(); self.wfile.write(b'DataTerminalCloudOssRole')
server=ThreadingHTTPServer(('127.0.0.1',0),Handler)
port_file.write_text(str(server.server_port),encoding='utf-8')
server.serve_forever()
PY
metadata_pid=$!
for _ in $(seq 1 50); do [[ -s "$port_file" ]] && break; sleep 0.1; done
[[ -s "$port_file" ]]
metadata_base="http://127.0.0.1:$(<"$port_file")/latest"
cat >"$test_root/enabled-role.env" <<'EOF'
OFFSITE_BACKUP_ENABLED=true
OFFSITE_BACKUP_REQUIRED=true
OFFSITE_OSS_BUCKET=valid-private-backup-bucket
OFFSITE_OSS_PREFIX=data-terminal-cloud/test
OFFSITE_OSS_AUTH_MODE=ecs_ram_role
OFFSITE_OSS_REGION=cn-hangzhou
OFFSITE_OSS_ENDPOINT=oss-cn-hangzhou.aliyuncs.com
EOF
set +e
ECS_METADATA_BASE_URL="$metadata_base" OFFSITE_CONFIG_FILE="$test_root/enabled-role.env" OFFSITE_STATE_DIR="$test_root/state" BACKUP_ROOT="$test_root/backups" \
"$candidate" >/dev/null 2>/dev/null
result=$?
set -e
test "$result" = 4
python3 - "$test_root/state/status.json" <<'PY'
import json,sys
d=json.load(open(sys.argv[1],encoding='utf-8'))
assert d['status']=='failed' and d['reason']=='ecs_ram_role_missing',d
PY
cat >"$test_root/disabled.env" <<'EOF'
OFFSITE_BACKUP_ENABLED=false
EOF
rm -f "$test_root/state/status.json"
OFFSITE_CONFIG_FILE="$test_root/disabled.env" OFFSITE_STATE_DIR="$test_root/state" BACKUP_ROOT="$test_root/backups" "$candidate" >/dev/null
printf 'encrypted-backup-fixture\n' >"$test_root/backups/platform-20260826T140000Z.tar.gz.enc"
(cd "$test_root/backups" && sha256sum platform-20260826T140000Z.tar.gz.enc >platform-20260826T140000Z.tar.gz.enc.sha256)
cat >"$test_root/mock-ossutil" <<'EOF'
#!/usr/bin/env bash
set -Eeuo pipefail
printf '%s\n' "$*" >>"$MOCK_OSS_LOG"
test "$1" = cp
source_path=$2
target_path=$3
map_path() {
if [[ "$1" == oss://* ]]; then printf '%s/%s' "$MOCK_OSS_ROOT" "${1#oss://}"; else printf '%s' "$1"; fi
}
source_local=$(map_path "$source_path")
target_local=$(map_path "$target_path")
mkdir -p "$(dirname "$target_local")"
cp "$source_local" "$target_local"
EOF
chmod 0700 "$test_root/mock-ossutil"
printf '[default]\nregion=cn-hangzhou\n' >"$test_root/ossutilconfig"
cat >"$test_root/config-mode.env" <<EOF
OFFSITE_BACKUP_ENABLED=true
OFFSITE_BACKUP_REQUIRED=true
OFFSITE_OSS_BUCKET=valid-private-backup-bucket
OFFSITE_OSS_PREFIX=data-terminal-cloud/config-test
OFFSITE_OSS_AUTH_MODE=config
OFFSITE_OSS_CONFIG_FILE=$test_root/ossutilconfig
EOF
rm -f "$test_root/state/status.json" "$test_root/ossutil.log"
MOCK_OSS_ROOT="$test_root/mock-oss" MOCK_OSS_LOG="$test_root/ossutil.log" OSSUTIL_BIN="$test_root/mock-ossutil" \
OFFSITE_CONFIG_FILE="$test_root/config-mode.env" OFFSITE_STATE_DIR="$test_root/state" BACKUP_ROOT="$test_root/backups" "$candidate" >/dev/null
python3 - "$test_root/state/status.json" <<'PY'
import json,sys
d=json.load(open(sys.argv[1],encoding='utf-8'))
assert d['status']=='verified' and d['reason']=='upload_and_download_verified',d
PY
printf 'attached\n' >"$role_mode_file"
rm -f "$test_root/state/status.json" "$test_root/ossutil.log"
MOCK_OSS_ROOT="$test_root/mock-oss" MOCK_OSS_LOG="$test_root/ossutil.log" OSSUTIL_BIN="$test_root/mock-ossutil" \
ECS_METADATA_BASE_URL="$metadata_base" OFFSITE_CONFIG_FILE="$test_root/enabled-role.env" OFFSITE_STATE_DIR="$test_root/state" \
BACKUP_ROOT="$test_root/backups" "$candidate" >/dev/null
python3 - "$test_root/state/status.json" <<'PY'
import json,sys
d=json.load(open(sys.argv[1],encoding='utf-8'))
assert d['status']=='verified' and d['reason']=='upload_and_download_verified',d
PY
grep -q -- '--mode EcsRamRole' "$test_root/ossutil.log"
grep -q -- '--region cn-hangzhou' "$test_root/ossutil.log"
grep -q -- '--endpoint oss-cn-hangzhou.aliyuncs.com' "$test_root/ossutil.log"
echo 'offsite_backup_smoke=ok disabled=ok missing_role=ok config_round_trip=ok ecs_role_round_trip=ok'