chore: establish local platform baseline
This commit is contained in:
Codex Local
@@ -0,0 +1,141 @@
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$PSNativeCommandUseErrorActionPreference = $true
|
||||
|
||||
$projectRoot = Split-Path -Parent $PSScriptRoot
|
||||
Set-Location -LiteralPath $projectRoot
|
||||
|
||||
$run = [Guid]::NewGuid().ToString('N')
|
||||
$aggregateId = [Guid]::NewGuid().ToString()
|
||||
$dedupeKey = "runtime-recovery-$run"
|
||||
$deviceId = '10000000-0000-0000-0000-000000000511'
|
||||
$deviceSecret = [Text.Encoding]::UTF8.GetBytes('local-simulated-device-secret-32-bytes-minimum')
|
||||
$workerStopped = $false
|
||||
$temporaryRoot = Join-Path ([System.IO.Path]::GetTempPath()) ('production-recovery-conformance-' + [Guid]::NewGuid().ToString('N'))
|
||||
$capturePath = Join-Path $temporaryRoot 'recovery-capture.json'
|
||||
$reportPath = Join-Path $temporaryRoot 'recovery-report.json'
|
||||
$recoveryStartedAt = [DateTime]::UtcNow
|
||||
|
||||
function Invoke-Sql([string]$Sql) {
|
||||
$result = $Sql | docker compose exec -T postgres psql -U voice -d voice_platform -v ON_ERROR_STOP=1 -q -t -A
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Runtime recovery SQL failed' }
|
||||
return ($result | Out-String).Trim()
|
||||
}
|
||||
|
||||
function Wait-Ready([string]$Url) {
|
||||
$deadline = [DateTime]::UtcNow.AddSeconds(45)
|
||||
do {
|
||||
try {
|
||||
$response = Invoke-WebRequest -UseBasicParsing -Uri $Url -TimeoutSec 3
|
||||
if ($response.StatusCode -eq 200) { return }
|
||||
} catch { }
|
||||
Start-Sleep -Milliseconds 500
|
||||
} while ([DateTime]::UtcNow -lt $deadline)
|
||||
throw "Service did not become ready: $Url"
|
||||
}
|
||||
|
||||
function Invoke-SignedDeviceConfig {
|
||||
$path = '/device/v2/config'
|
||||
$digest = [Convert]::ToHexString([Security.Cryptography.SHA256]::HashData([byte[]]::new(0))).ToLowerInvariant()
|
||||
$timestamp = [DateTime]::UtcNow.ToString('yyyy-MM-ddTHH:mm:ssZ')
|
||||
$nonce = [Guid]::NewGuid().ToString('N')
|
||||
$canonical = "GET`n$path`n$timestamp`n$nonce`n$digest"
|
||||
$hmac = [Security.Cryptography.HMACSHA256]::new($deviceSecret)
|
||||
try {
|
||||
$signature = [Convert]::ToBase64String($hmac.ComputeHash([Text.Encoding]::UTF8.GetBytes($canonical))).TrimEnd('=').Replace('+', '-').Replace('/', '_')
|
||||
} finally { $hmac.Dispose() }
|
||||
return Invoke-WebRequest -UseBasicParsing -Method Get -Uri ("http://localhost:58080" + $path) -Headers @{
|
||||
'X-Device-Id' = $deviceId
|
||||
'X-Device-Timestamp' = $timestamp
|
||||
'X-Device-Nonce' = $nonce
|
||||
'X-Device-Content-SHA256' = $digest
|
||||
'X-Device-Signature' = $signature
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
New-Item -ItemType Directory -Path $temporaryRoot | Out-Null
|
||||
$databaseBefore = Invoke-Sql "SELECT (SELECT count(*) FROM tenant)||'|'||(SELECT count(*) FROM user_account)||'|'||(SELECT count(*) FROM household)||'|'||(SELECT count(*) FROM subject_profile)"
|
||||
$startedBefore = @{}
|
||||
foreach ($service in @('api','callbacks','worker')) {
|
||||
$container = (docker compose ps -q $service | Out-String).Trim()
|
||||
if (-not $container) { throw "$service is not running before recovery drill" }
|
||||
$startedBefore[$service] = (docker inspect -f '{{.State.StartedAt}}' $container | Out-String).Trim()
|
||||
}
|
||||
|
||||
docker compose stop worker | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Cannot stop worker for pending-task recovery drill' }
|
||||
$workerStopped = $true
|
||||
Invoke-Sql "INSERT INTO outbox_event(aggregate_type,aggregate_id,event_type,payload,deduplication_key) VALUES('recovery_probe','$aggregateId'::uuid,'platform.noop',jsonb_build_object('probe',true),'$dedupeKey')" | Out-Null
|
||||
$pending = Invoke-Sql "SELECT status FROM outbox_event WHERE aggregate_id='$aggregateId'::uuid"
|
||||
if ($pending -ne 'pending') { throw "Recovery event was not held pending while worker stopped: $pending" }
|
||||
|
||||
docker compose restart api callbacks | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) { throw 'API/callback restart failed' }
|
||||
docker compose start worker | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) { throw 'Worker restart failed' }
|
||||
$workerStopped = $false
|
||||
Wait-Ready 'http://localhost:58080/health/ready'
|
||||
Wait-Ready 'http://localhost:58081/health/ready'
|
||||
Wait-Ready 'http://localhost:58082/health/ready'
|
||||
|
||||
$deadline = [DateTime]::UtcNow.AddSeconds(30)
|
||||
do {
|
||||
$outboxStatus = Invoke-Sql "SELECT status||'|'||attempt_count||'|'||(completed_at IS NOT NULL) FROM outbox_event WHERE aggregate_id='$aggregateId'::uuid"
|
||||
if ($outboxStatus -eq 'completed|1|true') { break }
|
||||
Start-Sleep -Milliseconds 300
|
||||
} while ([DateTime]::UtcNow -lt $deadline)
|
||||
if ($outboxStatus -ne 'completed|1|true') { throw "Pending outbox did not recover after worker restart: $outboxStatus" }
|
||||
|
||||
$startedAfter = @{}
|
||||
foreach ($service in @('api','callbacks','worker')) {
|
||||
$container = (docker compose ps -q $service | Out-String).Trim()
|
||||
$startedAfter[$service] = (docker inspect -f '{{.State.StartedAt}}' $container | Out-String).Trim()
|
||||
if ($startedAfter[$service] -eq $startedBefore[$service]) { throw "$service start timestamp did not change" }
|
||||
}
|
||||
|
||||
docker compose exec -T redis redis-cli SET "voice-platform:recovery:$run:lease" synthetic EX 300 | Out-Null
|
||||
docker compose exec -T redis redis-cli SET "voice-platform:recovery:$run:presence" synthetic EX 300 | Out-Null
|
||||
$keysBefore = [int]((docker compose exec -T redis redis-cli DBSIZE | Out-String).Trim())
|
||||
if ($keysBefore -lt 2) { throw "Redis recovery fixtures were not created: $keysBefore" }
|
||||
$flushResult = (docker compose exec -T redis redis-cli FLUSHDB | Out-String).Trim()
|
||||
if ($flushResult -ne 'OK') { throw "Redis FLUSHDB failed: $flushResult" }
|
||||
$keysAfterFlush = [int]((docker compose exec -T redis redis-cli DBSIZE | Out-String).Trim())
|
||||
if ($keysAfterFlush -ne 0) { throw "Redis was not cleared: $keysAfterFlush" }
|
||||
|
||||
Wait-Ready 'http://localhost:58080/health/ready'
|
||||
$configResponse = Invoke-SignedDeviceConfig
|
||||
if ($configResponse.StatusCode -ne 200) { throw "Device config did not recover after Redis clear: $($configResponse.StatusCode)" }
|
||||
$keysAfterRequest = [int]((docker compose exec -T redis redis-cli DBSIZE | Out-String).Trim())
|
||||
if ($keysAfterRequest -lt 1) { throw 'Device replay protection state was not recreated after Redis clear' }
|
||||
$databaseAfter = Invoke-Sql "SELECT (SELECT count(*) FROM tenant)||'|'||(SELECT count(*) FROM user_account)||'|'||(SELECT count(*) FROM household)||'|'||(SELECT count(*) FROM subject_profile)"
|
||||
if ($databaseAfter -ne $databaseBefore) { throw "Authoritative PostgreSQL counts changed across runtime recovery: before=$databaseBefore after=$databaseAfter" }
|
||||
|
||||
$recoveryCompletedAt = [DateTime]::UtcNow
|
||||
$recoverySeconds = [Math]::Max(0, [int][Math]::Ceiling(($recoveryCompletedAt - $recoveryStartedAt).TotalSeconds))
|
||||
$stateDigest = [Convert]::ToHexString([Security.Cryptography.SHA256]::HashData([Text.Encoding]::UTF8.GetBytes($databaseAfter))).ToLowerInvariant()
|
||||
$imageDigest = (docker image inspect data-terminal-cloud-api --format '{{.Id}}').Trim()
|
||||
$common = @{
|
||||
started_at = $recoveryStartedAt.ToString('o'); completed_at = $recoveryCompletedAt.ToString('o'); observed_recovery_seconds = $recoverySeconds
|
||||
recovery_verified = $true; data_integrity_verified = $true; business_continuity_verified = $true; no_duplicate_effects = $true; cleanup_verified = $true
|
||||
before_state_digest = $stateDigest; after_state_digest = $stateDigest
|
||||
}
|
||||
$restartDrill = $common.Clone(); $restartDrill.drill_type = 'SERVICE_ROLLING_RESTART'; $restartDrill.scenario_ref = 'local-compose:api-callbacks-worker-restart'; $restartDrill.execution_ref = 'docker-compose:restart'; $restartDrill.validation_ref = "outbox:$outboxStatus"; $restartDrill.audit_ref = 'local-smoke:process-start-times'
|
||||
$redisDrill = $common.Clone(); $redisDrill.drill_type = 'REDIS_REBUILD'; $redisDrill.scenario_ref = 'local-redis:flush-and-signed-request-rebuild'; $redisDrill.execution_ref = "redis:flushdb:keys-before-$keysBefore"; $redisDrill.validation_ref = "redis:keys-after-request-$keysAfterRequest"; $redisDrill.audit_ref = 'local-smoke:postgres-authority-counts'
|
||||
$capture = @{
|
||||
schema_version = 'production-recovery-capture-v1'; evidence_source = 'LOCAL'; candidate_version = '0.63.0-rc.local'; build_digest = $imageDigest
|
||||
environment = 'LOCAL'; infrastructure_ref = 'docker-compose:local'; failure_domain_ref = ''; change_ticket_ref = ''
|
||||
attested_by_ref = 'local-smoke:automated'; attested_at = $recoveryCompletedAt.ToString('o'); collected_at = $recoveryCompletedAt.ToString('o')
|
||||
drills = @($restartDrill, $redisDrill)
|
||||
}
|
||||
[System.IO.File]::WriteAllText($capturePath, ($capture | ConvertTo-Json -Depth 10), [System.Text.UTF8Encoding]::new($false))
|
||||
go run ./cmd/recovery-conformance -capture $capturePath -candidate-version '0.63.0-rc.local' -build-digest $imageDigest -output $reportPath
|
||||
$report = Get-Content -Raw -LiteralPath $reportPath | ConvertFrom-Json
|
||||
if ($report.result -ne 'PASS' -or $report.evidence_scope -ne 'LOCAL_RECOVERY' -or -not $report.external_validation_required -or $report.drill_count -ne 2) { throw '本地恢复报告错误地改变了生产等价恢复门禁。' }
|
||||
|
||||
Write-Host "Runtime recovery passed: services_restarted=true, pending_outbox=$outboxStatus, redis_after_flush=$keysAfterFlush, after_signed_request=$keysAfterRequest, report_scope=LOCAL_RECOVERY, production_equivalent_pending=true"
|
||||
}
|
||||
finally {
|
||||
if ($workerStopped) { docker compose start worker | Out-Null }
|
||||
try { Invoke-Sql "DELETE FROM outbox_event WHERE aggregate_id='$aggregateId'::uuid" | Out-Null } catch { Write-Warning "Recovery cleanup failed: $_" }
|
||||
if (Test-Path -LiteralPath $temporaryRoot) { Remove-Item -LiteralPath $temporaryRoot -Recurse -Force }
|
||||
}
|
||||
Reference in New Issue
Block a user