param( [ValidateSet('RecoverySuccess', 'ABRollback')] [string]$Mode = 'RecoverySuccess' ) $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true $projectRoot = Split-Path -Parent $PSScriptRoot Set-Location -LiteralPath $projectRoot $api = 'http://localhost:58080' $tokenUrl = 'http://localhost:58180/realms/voice-local/protocol/openid-connect/token' $householdId = '10000000-0000-0000-0000-000000000211' $deviceId = '10000000-0000-0000-0000-000000000511' $keyId = 'smoke-sim-' + [Guid]::NewGuid().ToString('N') $tempFile = Join-Path ([IO.Path]::GetTempPath()) ('firmware-simulator-' + [Guid]::NewGuid().ToString('N') + '.bin') $stateFile = $tempFile + '.state.json' $evidenceFile = $tempFile + '.device-evidence.json' $cloudFile = $tempFile + '.cloud-rollout.json' $conformanceFile = $tempFile + '.conformance-report.json' $releaseId = $null $rolloutId = $null $objectRefs = @() $environmentNames = @('SIM_API_URL', 'SIM_DEVICE_ID', 'SIM_DEVICE_SECRET', 'SIM_SCENARIO', 'SIM_FIRMWARE_FAULT', 'SIM_FIRMWARE_STEP_DELAY', 'SIM_FIRMWARE_TRUSTED_KEYS', 'SIM_FIRMWARE_AB_UPDATE', 'SIM_FIRMWARE_STATE_FILE', 'SIM_FIRMWARE_BOOT_HEALTH_WINDOW', 'SIM_FIRMWARE_ACTIVE_SLOT', 'SIM_FIRMWARE_CURRENT_VERSION', 'SIM_FIRMWARE_EVIDENCE_FILE', 'SIM_FIRMWARE_EVIDENCE_SCENARIO', 'SIM_CANDIDATE_VERSION', 'SIM_BUILD_DIGEST', 'SIM_BOOTLOADER_VERSION', 'SIM_PARTITION_TABLE_SHA256', 'SIM_SECURE_BOOT_ENABLED') $previousEnvironment = @{} foreach ($name in $environmentNames) { $item = Get-Item -LiteralPath "Env:$name" -ErrorAction SilentlyContinue $previousEnvironment[$name] = if ($null -eq $item) { $null } else { $item.Value } } function Get-AccessToken { return (Invoke-RestMethod -Method Post -Uri $tokenUrl -ContentType 'application/x-www-form-urlencoded' -Body @{ grant_type = 'password' client_id = 'voice-platform-web' username = 'owner' password = 'LocalOnly123!' scope = 'openid' }).access_token } function Invoke-DatabaseScalar([string]$Sql) { return (docker compose exec -T postgres psql -U voice -d voice_platform -t -A -v ON_ERROR_STOP=1 -c $Sql | Out-String).Trim() } function Get-RolloutAssignment([hashtable]$Headers) { $response = Invoke-RestMethod -Method Get -Uri "$api/api/v1/firmware-rollouts/$rolloutId" -Headers $Headers return $response.data.assignments | Where-Object { $_.device_id -eq $deviceId } | Select-Object -First 1 } try { Invoke-RestMethod -Method Get -Uri "$api/health/ready" | Out-Null $activeCommands = Invoke-DatabaseScalar "SELECT count(*) FROM device_command dc JOIN device d ON d.id=dc.device_id WHERE d.public_id='$deviceId'::uuid AND dc.status IN ('QUEUED','DELIVERED','ACKNOWLEDGED')" if ([int]$activeCommands -ne 0) { throw "Simulator smoke requires no pre-existing active command for $deviceId; found $activeCommands" } [IO.File]::WriteAllBytes($tempFile, [Text.Encoding]::UTF8.GetBytes('simulator-firmware-' + [Guid]::NewGuid().ToString('N'))) $headers = @{ Authorization = 'Bearer ' + (Get-AccessToken) } $version = '9.7.' + (Get-Random -Minimum 10000 -Maximum 99999) $signed = go run ./cmd/firmware-sign simulated-terminal sim-rev-a $version $tempFile | ConvertFrom-Json $keyBody = @{ key_id = $keyId; public_key_base64 = $signed.public_key_base64 } | ConvertTo-Json -Compress Invoke-RestMethod -Method Post -Uri "$api/api/v1/households/$householdId/firmware-signing-keys" -Headers $headers -ContentType 'application/json' -Body $keyBody | Out-Null $releaseBody = @{ model = 'simulated-terminal' hardware_revision = 'sim-rev-a' version = $version expected_size = [int64]$signed.size expected_digest = $signed.sha256 signature = $signed.signature_base64 signing_key_id = $keyId minimum_security_version = '1.0.0' release_notes = 'simulator vertical smoke fixture' } | ConvertTo-Json -Compress $prepared = Invoke-RestMethod -Method Post -Uri "$api/api/v1/households/$householdId/firmware-releases" -Headers $headers -ContentType 'application/json' -Body $releaseBody $releaseId = $prepared.data.id Invoke-WebRequest -UseBasicParsing -Method Put -Uri $prepared.data.upload_url -InFile $tempFile -ContentType 'application/octet-stream' | Out-Null $ready = Invoke-RestMethod -Method Post -Uri "$api/api/v1/firmware-releases/$releaseId/finalize" -Headers $headers if ($ready.data.status -ne 'READY') { throw "Firmware release did not become READY: $($ready.data.status)" } $rolloutBody = @{ release_id = $releaseId name = if ($Mode -eq 'ABRollback') { 'simulator A-B rollback smoke' } else { 'simulator interruption recovery smoke' } target_percentage = 100 device_ids = @($deviceId) } | ConvertTo-Json -Compress $rollout = Invoke-RestMethod -Method Post -Uri "$api/api/v1/households/$householdId/firmware-rollouts" -Headers $headers -ContentType 'application/json' -Body $rolloutBody $rolloutId = $rollout.data.id $env:SIM_API_URL = $api $env:SIM_DEVICE_ID = $deviceId $env:SIM_DEVICE_SECRET = 'local-simulated-device-secret-32-bytes-minimum' $env:SIM_SCENARIO = 'commands' $env:SIM_FIRMWARE_STEP_DELAY = '0s' $env:SIM_FIRMWARE_TRUSTED_KEYS = @{ $keyId = $signed.public_key_base64 } | ConvertTo-Json -Compress if ($Mode -eq 'ABRollback') { $env:SIM_FIRMWARE_AB_UPDATE = 'true' $env:SIM_FIRMWARE_STATE_FILE = $stateFile $env:SIM_FIRMWARE_BOOT_HEALTH_WINDOW = '30s' $env:SIM_FIRMWARE_ACTIVE_SLOT = 'A' $env:SIM_FIRMWARE_CURRENT_VERSION = '0.2.0-sim' $env:SIM_FIRMWARE_EVIDENCE_FILE = $evidenceFile $env:SIM_FIRMWARE_EVIDENCE_SCENARIO = 'AB_ROLLBACK' $env:SIM_CANDIDATE_VERSION = 'local-ab-conformance-smoke' $env:SIM_BUILD_DIGEST = 'sha256:' + $signed.sha256 $env:SIM_BOOTLOADER_VERSION = 'sim-bootloader-1' $env:SIM_PARTITION_TABLE_SHA256 = $signed.sha256 $env:SIM_SECURE_BOOT_ENABLED = 'false' $env:SIM_FIRMWARE_FAULT = 'none' go run ./cmd/sim-device $armedState = Invoke-DatabaseScalar "SELECT dc.status||'|'||fa.status||'|'||fa.progress_percent||'|'||COALESCE(dc.result_summary->>'result_code','') FROM device_command dc JOIN firmware_assignment fa ON fa.command_id=dc.id JOIN firmware_rollout ro ON ro.id=fa.rollout_id WHERE ro.public_id='$rolloutId'::uuid" if ($armedState -ne 'ACKNOWLEDGED|INSTALLING|90|REBOOT_ARMED') { throw "A/B firmware update did not reach the reboot boundary: $armedState" } $localArmed = Get-Content -LiteralPath $stateFile -Raw | ConvertFrom-Json if ($localArmed.phase -ne 'REBOOT_PENDING' -or $localArmed.active_slot -ne 'A' -or $localArmed.target_slot -ne 'B' -or -not $localArmed.boot_attempt_id) { throw 'Local A/B journal did not durably arm the inactive slot' } $env:SIM_FIRMWARE_FAULT = 'boot_health_failed' go run ./cmd/sim-device $assignment = Get-RolloutAssignment $headers if (-not $assignment -or $assignment.status -ne 'FAILED' -or [int]$assignment.progress_percent -ne 95 -or -not [bool]$assignment.last_result.rollback_applied -or $assignment.last_result.rollback_version -ne '0.2.0-sim' -or $assignment.events.Count -ne 7) { throw 'Failed A/B boot did not expose an applied rollback to the prior version' } $eventPhases = ($assignment.events | ForEach-Object { $_.phase }) -join ',' if ($eventPhases -ne 'downloading,downloading,installing,staging,reboot_pending,boot_verifying,rolled_back') { throw "Cloud firmware event timeline was incomplete or out of order: $eventPhases" } $terminalState = Invoke-DatabaseScalar "SELECT dc.status||'|'||fa.status||'|'||fa.progress_percent||'|'||COALESCE(dc.result_summary->>'result_code','')||'|'||dc.delivery_attempts FROM device_command dc JOIN firmware_assignment fa ON fa.command_id=dc.id JOIN firmware_rollout ro ON ro.id=fa.rollout_id WHERE ro.public_id='$rolloutId'::uuid" if ($terminalState -ne 'FAILED|FAILED|95|BOOT_HEALTH_FAILED_ROLLED_BACK|2') { throw "A/B rollback did not converge to the expected terminal state: $terminalState" } $localRolledBack = Get-Content -LiteralPath $stateFile -Raw | ConvertFrom-Json if ($localRolledBack.phase -ne 'ROLLED_BACK' -or $localRolledBack.active_slot -ne 'A' -or $localRolledBack.target_version -ne '0.2.0-sim') { throw 'Local A/B journal did not return to the prior bootable slot' } $impact = Invoke-RestMethod -Method Get -Uri "$api/api/v1/firmware-releases/$releaseId/impact" -Headers $headers if ([int]$impact.data.devices.failed -ne 1 -or [int]$impact.data.devices.observing -ne 0 -or [int]$impact.data.commands.failed -ne 1 -or [int]$impact.data.rollouts.paused -ne 1) { throw 'Firmware impact summary did not converge after automatic A/B rollback' } $rolloutDetail = Invoke-RestMethod -Method Get -Uri "$api/api/v1/firmware-rollouts/$rolloutId" -Headers $headers $rolloutDetail | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $cloudFile -Encoding utf8NoBOM go run ./cmd/firmware-conformance -cloud $cloudFile -device $evidenceFile -output $conformanceFile $conformance = Get-Content -LiteralPath $conformanceFile -Raw | ConvertFrom-Json $failedChecks = @($conformance.checks | Where-Object { $_.status -ne 'PASS' }) if ($conformance.result -ne 'PASS' -or $conformance.evidence_scope -ne 'PROTOCOL_SIMULATION' -or -not [bool]$conformance.external_validation_required -or $failedChecks.Count -ne 0) { throw 'Firmware conformance report did not pass or incorrectly claimed external validation' } Write-Host "Simulated A/B firmware boot failure rollback smoke passed: release $releaseId, rollout $rolloutId" } else { $env:SIM_FIRMWARE_AB_UPDATE = 'false' $env:SIM_FIRMWARE_FAULT = 'interrupt_after_download' go run ./cmd/sim-device $interruptedState = Invoke-DatabaseScalar "SELECT dc.status||'|'||fa.status||'|'||fa.progress_percent||'|'||COALESCE(dc.result_summary->>'result_code','') FROM device_command dc JOIN firmware_assignment fa ON fa.command_id=dc.id JOIN firmware_rollout ro ON ro.id=fa.rollout_id WHERE ro.public_id='$rolloutId'::uuid" if ($interruptedState -ne 'ACKNOWLEDGED|DOWNLOADING|100|ARTIFACT_VERIFIED') { throw "Firmware interruption state was not recoverable: $interruptedState" } $env:SIM_FIRMWARE_FAULT = 'none' go run ./cmd/sim-device $assignment = Get-RolloutAssignment $headers if (-not $assignment -or $assignment.status -ne 'CONFIRMED' -or [int]$assignment.progress_percent -ne 100) { throw "Recovered firmware assignment did not converge to CONFIRMED/100: $($assignment.status)/$($assignment.progress_percent)" } $terminalState = Invoke-DatabaseScalar "SELECT dc.status||'|'||fa.status||'|'||fa.progress_percent||'|'||COALESCE(dc.result_summary->>'result_code','')||'|'||dc.delivery_attempts FROM device_command dc JOIN firmware_assignment fa ON fa.command_id=dc.id JOIN firmware_rollout ro ON ro.id=fa.rollout_id WHERE ro.public_id='$rolloutId'::uuid" if ($terminalState -ne 'SUCCEEDED|CONFIRMED|100|SIMULATED_OK|2') { throw "Recovered firmware command did not reach the expected terminal state: $terminalState" } $impact = Invoke-RestMethod -Method Get -Uri "$api/api/v1/firmware-releases/$releaseId/impact" -Headers $headers if ([int]$impact.data.devices.confirmed -ne 1 -or [int]$impact.data.devices.observing -ne 0 -or [int]$impact.data.commands.succeeded -ne 1 -or [int]$impact.data.rollouts.completed -ne 1) { throw 'Firmware impact summary did not converge after simulator recovery' } Write-Host "Simulated firmware interruption recovery smoke passed: release $releaseId, rollout $rolloutId" } } finally { if ($releaseId) { $refs = docker compose exec -T postgres psql -U voice -d voice_platform -t -A -c "SELECT object_ref FROM firmware_release WHERE public_id='$releaseId'::uuid" | Out-String $objectRefs = $refs -split "`r?`n" | Where-Object { $_.Trim() } } if ($releaseId) { $cleanupSql = @" BEGIN; CREATE TEMP TABLE smoke_releases AS SELECT id,public_id FROM firmware_release WHERE public_id='$releaseId'::uuid; CREATE TEMP TABLE smoke_rollouts AS SELECT id,public_id FROM firmware_rollout WHERE release_id IN(SELECT id FROM smoke_releases); CREATE TEMP TABLE smoke_commands AS SELECT dc.id,dc.public_id FROM device_command dc JOIN firmware_assignment fa ON fa.command_id=dc.id WHERE fa.rollout_id IN(SELECT id FROM smoke_rollouts); DELETE FROM audit_event WHERE resource_id IN(SELECT public_id FROM smoke_releases UNION SELECT public_id FROM smoke_rollouts UNION SELECT public_id FROM smoke_commands); DELETE FROM web_event WHERE resource_public_id IN(SELECT public_id FROM smoke_commands); DELETE FROM firmware_assignment WHERE rollout_id IN(SELECT id FROM smoke_rollouts); DELETE FROM firmware_rollout WHERE id IN(SELECT id FROM smoke_rollouts); DELETE FROM device_command WHERE id IN(SELECT id FROM smoke_commands); DELETE FROM firmware_release WHERE id IN(SELECT id FROM smoke_releases); DELETE FROM audit_event WHERE resource_type='firmware_signing_key' AND detail->>'key_id'='$keyId'; DELETE FROM firmware_signing_key WHERE key_id='$keyId'; COMMIT; "@ docker compose exec -T postgres psql -U voice -d voice_platform -v ON_ERROR_STOP=1 -q -c $cleanupSql | Out-Null } else { docker compose exec -T postgres psql -U voice -d voice_platform -v ON_ERROR_STOP=1 -q -c "DELETE FROM audit_event WHERE resource_type='firmware_signing_key' AND detail->>'key_id'='$keyId'; DELETE FROM firmware_signing_key WHERE key_id='$keyId';" | Out-Null } if ($objectRefs.Count -gt 0) { docker compose exec -T minio mc alias set smoke http://localhost:9000 local_minio local_minio_password | Out-Null foreach ($objectRef in $objectRefs) { docker compose exec -T minio mc rm --force "smoke/private-exports/$($objectRef.Trim())" | Out-Null } } if (Test-Path -LiteralPath $tempFile) { Remove-Item -LiteralPath $tempFile -Force } if (Test-Path -LiteralPath $stateFile) { Remove-Item -LiteralPath $stateFile -Force } foreach ($artifact in @($evidenceFile, $cloudFile, $conformanceFile)) { if (Test-Path -LiteralPath $artifact) { Remove-Item -LiteralPath $artifact -Force } } foreach ($name in $environmentNames) { if ($null -eq $previousEnvironment[$name]) { Remove-Item -LiteralPath "Env:$name" -ErrorAction SilentlyContinue } else { Set-Item -LiteralPath "Env:$name" -Value $previousEnvironment[$name] } } }