$ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true $projectRoot = Split-Path -Parent $PSScriptRoot Set-Location -LiteralPath $projectRoot $run = [Guid]::NewGuid().ToString('N') $aggregateId = [Guid]::NewGuid().ToString() $dedupeKey = "runtime-recovery-$run" $deviceId = '10000000-0000-0000-0000-000000000511' $deviceSecret = [Text.Encoding]::UTF8.GetBytes('local-simulated-device-secret-32-bytes-minimum') $workerStopped = $false $temporaryRoot = Join-Path ([System.IO.Path]::GetTempPath()) ('production-recovery-conformance-' + [Guid]::NewGuid().ToString('N')) $capturePath = Join-Path $temporaryRoot 'recovery-capture.json' $reportPath = Join-Path $temporaryRoot 'recovery-report.json' $recoveryStartedAt = [DateTime]::UtcNow function Invoke-Sql([string]$Sql) { $result = $Sql | docker compose exec -T postgres psql -U voice -d voice_platform -v ON_ERROR_STOP=1 -q -t -A if ($LASTEXITCODE -ne 0) { throw 'Runtime recovery SQL failed' } return ($result | Out-String).Trim() } function Wait-Ready([string]$Url) { $deadline = [DateTime]::UtcNow.AddSeconds(45) do { try { $response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 3 if ($response.StatusCode -eq 200) { return } } catch { } Start-Sleep -Milliseconds 500 } while ([DateTime]::UtcNow -lt $deadline) throw "Service did not become ready: $Url" } function Invoke-SignedDeviceConfig { $path = '/device/v2/config' $digest = [Convert]::ToHexString([Security.Cryptography.SHA256]::HashData([byte[]]::new(0))).ToLowerInvariant() $timestamp = [DateTime]::UtcNow.ToString('yyyy-MM-ddTHH:mm:ssZ') $nonce = [Guid]::NewGuid().ToString('N') $canonical = "GET`n$path`n$timestamp`n$nonce`n$digest" $hmac = [Security.Cryptography.HMACSHA256]::new($deviceSecret) try { $signature = [Convert]::ToBase64String($hmac.ComputeHash([Text.Encoding]::UTF8.GetBytes($canonical))).TrimEnd('=').Replace('+', '-').Replace('/', '_') } finally { $hmac.Dispose() } return Invoke-WebRequest -UseBasicParsing -Method Get -Uri ("http://localhost:58080" + $path) -Headers @{ 'X-Device-Id' = $deviceId 'X-Device-Timestamp' = $timestamp 'X-Device-Nonce' = $nonce 'X-Device-Content-SHA256' = $digest 'X-Device-Signature' = $signature } } try { New-Item -ItemType Directory -Path $temporaryRoot | Out-Null $databaseBefore = Invoke-Sql "SELECT (SELECT count(*) FROM tenant)||'|'||(SELECT count(*) FROM user_account)||'|'||(SELECT count(*) FROM household)||'|'||(SELECT count(*) FROM subject_profile)" $startedBefore = @{} foreach ($service in @('api','callbacks','worker')) { $container = (docker compose ps -q $service | Out-String).Trim() if (-not $container) { throw "$service is not running before recovery drill" } $startedBefore[$service] = (docker inspect -f '{{.State.StartedAt}}' $container | Out-String).Trim() } docker compose stop worker | Out-Null if ($LASTEXITCODE -ne 0) { throw 'Cannot stop worker for pending-task recovery drill' } $workerStopped = $true Invoke-Sql "INSERT INTO outbox_event(aggregate_type,aggregate_id,event_type,payload,deduplication_key) VALUES('recovery_probe','$aggregateId'::uuid,'platform.noop',jsonb_build_object('probe',true),'$dedupeKey')" | Out-Null $pending = Invoke-Sql "SELECT status FROM outbox_event WHERE aggregate_id='$aggregateId'::uuid" if ($pending -ne 'pending') { throw "Recovery event was not held pending while worker stopped: $pending" } docker compose restart api callbacks | Out-Null if ($LASTEXITCODE -ne 0) { throw 'API/callback restart failed' } docker compose start worker | Out-Null if ($LASTEXITCODE -ne 0) { throw 'Worker restart failed' } $workerStopped = $false Wait-Ready 'http://localhost:58080/health/ready' Wait-Ready 'http://localhost:58081/health/ready' Wait-Ready 'http://localhost:58082/health/ready' $deadline = [DateTime]::UtcNow.AddSeconds(30) do { $outboxStatus = Invoke-Sql "SELECT status||'|'||attempt_count||'|'||(completed_at IS NOT NULL) FROM outbox_event WHERE aggregate_id='$aggregateId'::uuid" if ($outboxStatus -eq 'completed|1|true') { break } Start-Sleep -Milliseconds 300 } while ([DateTime]::UtcNow -lt $deadline) if ($outboxStatus -ne 'completed|1|true') { throw "Pending outbox did not recover after worker restart: $outboxStatus" } $startedAfter = @{} foreach ($service in @('api','callbacks','worker')) { $container = (docker compose ps -q $service | Out-String).Trim() $startedAfter[$service] = (docker inspect -f '{{.State.StartedAt}}' $container | Out-String).Trim() if ($startedAfter[$service] -eq $startedBefore[$service]) { throw "$service start timestamp did not change" } } docker compose exec -T redis redis-cli SET "voice-platform:recovery:$run:lease" synthetic EX 300 | Out-Null docker compose exec -T redis redis-cli SET "voice-platform:recovery:$run:presence" synthetic EX 300 | Out-Null $keysBefore = [int]((docker compose exec -T redis redis-cli DBSIZE | Out-String).Trim()) if ($keysBefore -lt 2) { throw "Redis recovery fixtures were not created: $keysBefore" } $flushResult = (docker compose exec -T redis redis-cli FLUSHDB | Out-String).Trim() if ($flushResult -ne 'OK') { throw "Redis FLUSHDB failed: $flushResult" } $keysAfterFlush = [int]((docker compose exec -T redis redis-cli DBSIZE | Out-String).Trim()) if ($keysAfterFlush -ne 0) { throw "Redis was not cleared: $keysAfterFlush" } Wait-Ready 'http://localhost:58080/health/ready' $configResponse = Invoke-SignedDeviceConfig if ($configResponse.StatusCode -ne 200) { throw "Device config did not recover after Redis clear: $($configResponse.StatusCode)" } $keysAfterRequest = [int]((docker compose exec -T redis redis-cli DBSIZE | Out-String).Trim()) if ($keysAfterRequest -lt 1) { throw 'Device replay protection state was not recreated after Redis clear' } $databaseAfter = Invoke-Sql "SELECT (SELECT count(*) FROM tenant)||'|'||(SELECT count(*) FROM user_account)||'|'||(SELECT count(*) FROM household)||'|'||(SELECT count(*) FROM subject_profile)" if ($databaseAfter -ne $databaseBefore) { throw "Authoritative PostgreSQL counts changed across runtime recovery: before=$databaseBefore after=$databaseAfter" } $recoveryCompletedAt = [DateTime]::UtcNow $recoverySeconds = [Math]::Max(0, [int][Math]::Ceiling(($recoveryCompletedAt - $recoveryStartedAt).TotalSeconds)) $stateDigest = [Convert]::ToHexString([Security.Cryptography.SHA256]::HashData([Text.Encoding]::UTF8.GetBytes($databaseAfter))).ToLowerInvariant() $imageDigest = (docker image inspect data-terminal-cloud-api --format '{{.Id}}').Trim() $common = @{ started_at = $recoveryStartedAt.ToString('o'); completed_at = $recoveryCompletedAt.ToString('o'); observed_recovery_seconds = $recoverySeconds recovery_verified = $true; data_integrity_verified = $true; business_continuity_verified = $true; no_duplicate_effects = $true; cleanup_verified = $true before_state_digest = $stateDigest; after_state_digest = $stateDigest } $restartDrill = $common.Clone(); $restartDrill.drill_type = 'SERVICE_ROLLING_RESTART'; $restartDrill.scenario_ref = 'local-compose:api-callbacks-worker-restart'; $restartDrill.execution_ref = 'docker-compose:restart'; $restartDrill.validation_ref = "outbox:$outboxStatus"; $restartDrill.audit_ref = 'local-smoke:process-start-times' $redisDrill = $common.Clone(); $redisDrill.drill_type = 'REDIS_REBUILD'; $redisDrill.scenario_ref = 'local-redis:flush-and-signed-request-rebuild'; $redisDrill.execution_ref = "redis:flushdb:keys-before-$keysBefore"; $redisDrill.validation_ref = "redis:keys-after-request-$keysAfterRequest"; $redisDrill.audit_ref = 'local-smoke:postgres-authority-counts' $capture = @{ schema_version = 'production-recovery-capture-v1'; evidence_source = 'LOCAL'; candidate_version = '0.63.0-rc.local'; build_digest = $imageDigest environment = 'LOCAL'; infrastructure_ref = 'docker-compose:local'; failure_domain_ref = ''; change_ticket_ref = '' attested_by_ref = 'local-smoke:automated'; attested_at = $recoveryCompletedAt.ToString('o'); collected_at = $recoveryCompletedAt.ToString('o') drills = @($restartDrill, $redisDrill) } [System.IO.File]::WriteAllText($capturePath, ($capture | ConvertTo-Json -Depth 10), [System.Text.UTF8Encoding]::new($false)) go run ./cmd/recovery-conformance -capture $capturePath -candidate-version '0.63.0-rc.local' -build-digest $imageDigest -output $reportPath $report = Get-Content -Raw -LiteralPath $reportPath | ConvertFrom-Json if ($report.result -ne 'PASS' -or $report.evidence_scope -ne 'LOCAL_RECOVERY' -or -not $report.external_validation_required -or $report.drill_count -ne 2) { throw '本地恢复报告错误地改变了生产等价恢复门禁。' } Write-Host "Runtime recovery passed: services_restarted=true, pending_outbox=$outboxStatus, redis_after_flush=$keysAfterFlush, after_signed_request=$keysAfterRequest, report_scope=LOCAL_RECOVERY, production_equivalent_pending=true" } finally { if ($workerStopped) { docker compose start worker | Out-Null } try { Invoke-Sql "DELETE FROM outbox_event WHERE aggregate_id='$aggregateId'::uuid" | Out-Null } catch { Write-Warning "Recovery cleanup failed: $_" } if (Test-Path -LiteralPath $temporaryRoot) { Remove-Item -LiteralPath $temporaryRoot -Recurse -Force } }