238 lines
15 KiB
PowerShell
238 lines
15 KiB
PowerShell
|
param(
|
||
|
|
[ValidateSet('RecoverySuccess', 'ABRollback')]
|
||
|
|
[string]$Mode = 'RecoverySuccess'
|
||
|
|
)
|
||
|
|
|
||
|
|
$ErrorActionPreference = 'Stop'
|
||
|
|
$PSNativeCommandUseErrorActionPreference = $true
|
||
|
|
|
||
|
|
$projectRoot = Split-Path -Parent $PSScriptRoot
|
||
|
|
Set-Location -LiteralPath $projectRoot
|
||
|
|
|
||
|
|
$api = 'http://localhost:58080'
|
||
|
|
$tokenUrl = 'http://localhost:58180/realms/voice-local/protocol/openid-connect/token'
|
||
|
|
$householdId = '10000000-0000-0000-0000-000000000211'
|
||
|
|
$deviceId = '10000000-0000-0000-0000-000000000511'
|
||
|
|
$keyId = 'smoke-sim-' + [Guid]::NewGuid().ToString('N')
|
||
|
|
$tempFile = Join-Path ([IO.Path]::GetTempPath()) ('firmware-simulator-' + [Guid]::NewGuid().ToString('N') + '.bin')
|
||
|
|
$stateFile = $tempFile + '.state.json'
|
||
|
|
$evidenceFile = $tempFile + '.device-evidence.json'
|
||
|
|
$cloudFile = $tempFile + '.cloud-rollout.json'
|
||
|
|
$conformanceFile = $tempFile + '.conformance-report.json'
|
||
|
|
$releaseId = $null
|
||
|
|
$rolloutId = $null
|
||
|
|
$objectRefs = @()
|
||
|
|
$environmentNames = @('SIM_API_URL', 'SIM_DEVICE_ID', 'SIM_DEVICE_SECRET', 'SIM_SCENARIO', 'SIM_FIRMWARE_FAULT', 'SIM_FIRMWARE_STEP_DELAY', 'SIM_FIRMWARE_TRUSTED_KEYS', 'SIM_FIRMWARE_AB_UPDATE', 'SIM_FIRMWARE_STATE_FILE', 'SIM_FIRMWARE_BOOT_HEALTH_WINDOW', 'SIM_FIRMWARE_ACTIVE_SLOT', 'SIM_FIRMWARE_CURRENT_VERSION', 'SIM_FIRMWARE_EVIDENCE_FILE', 'SIM_FIRMWARE_EVIDENCE_SCENARIO', 'SIM_CANDIDATE_VERSION', 'SIM_BUILD_DIGEST', 'SIM_BOOTLOADER_VERSION', 'SIM_PARTITION_TABLE_SHA256', 'SIM_SECURE_BOOT_ENABLED')
|
||
|
|
$previousEnvironment = @{}
|
||
|
|
foreach ($name in $environmentNames) {
|
||
|
|
$item = Get-Item -LiteralPath "Env:$name" -ErrorAction SilentlyContinue
|
||
|
|
$previousEnvironment[$name] = if ($null -eq $item) { $null } else { $item.Value }
|
||
|
|
}
|
||
|
|
|
||
|
|
function Get-AccessToken {
|
||
|
|
return (Invoke-RestMethod -Method Post -Uri $tokenUrl -ContentType 'application/x-www-form-urlencoded' -Body @{
|
||
|
|
grant_type = 'password'
|
||
|
|
client_id = 'voice-platform-web'
|
||
|
|
username = 'owner'
|
||
|
|
password = 'LocalOnly123!'
|
||
|
|
scope = 'openid'
|
||
|
|
}).access_token
|
||
|
|
}
|
||
|
|
|
||
|
|
function Invoke-DatabaseScalar([string]$Sql) {
|
||
|
|
return (docker compose exec -T postgres psql -U voice -d voice_platform -t -A -v ON_ERROR_STOP=1 -c $Sql | Out-String).Trim()
|
||
|
|
}
|
||
|
|
|
||
|
|
function Get-RolloutAssignment([hashtable]$Headers) {
|
||
|
|
$response = Invoke-RestMethod -Method Get -Uri "$api/api/v1/firmware-rollouts/$rolloutId" -Headers $Headers
|
||
|
|
return $response.data.assignments | Where-Object { $_.device_id -eq $deviceId } | Select-Object -First 1
|
||
|
|
}
|
||
|
|
|
||
|
|
try {
|
||
|
|
Invoke-RestMethod -Method Get -Uri "$api/health/ready" | Out-Null
|
||
|
|
$activeCommands = Invoke-DatabaseScalar "SELECT count(*) FROM device_command dc JOIN device d ON d.id=dc.device_id WHERE d.public_id='$deviceId'::uuid AND dc.status IN ('QUEUED','DELIVERED','ACKNOWLEDGED')"
|
||
|
|
if ([int]$activeCommands -ne 0) {
|
||
|
|
throw "Simulator smoke requires no pre-existing active command for $deviceId; found $activeCommands"
|
||
|
|
}
|
||
|
|
|
||
|
|
[IO.File]::WriteAllBytes($tempFile, [Text.Encoding]::UTF8.GetBytes('simulator-firmware-' + [Guid]::NewGuid().ToString('N')))
|
||
|
|
$headers = @{ Authorization = 'Bearer ' + (Get-AccessToken) }
|
||
|
|
$version = '9.7.' + (Get-Random -Minimum 10000 -Maximum 99999)
|
||
|
|
$signed = go run ./cmd/firmware-sign simulated-terminal sim-rev-a $version $tempFile | ConvertFrom-Json
|
||
|
|
|
||
|
|
$keyBody = @{ key_id = $keyId; public_key_base64 = $signed.public_key_base64 } | ConvertTo-Json -Compress
|
||
|
|
Invoke-RestMethod -Method Post -Uri "$api/api/v1/households/$householdId/firmware-signing-keys" -Headers $headers -ContentType 'application/json' -Body $keyBody | Out-Null
|
||
|
|
|
||
|
|
$releaseBody = @{
|
||
|
|
model = 'simulated-terminal'
|
||
|
|
hardware_revision = 'sim-rev-a'
|
||
|
|
version = $version
|
||
|
|
expected_size = [int64]$signed.size
|
||
|
|
expected_digest = $signed.sha256
|
||
|
|
signature = $signed.signature_base64
|
||
|
|
signing_key_id = $keyId
|
||
|
|
minimum_security_version = '1.0.0'
|
||
|
|
release_notes = 'simulator vertical smoke fixture'
|
||
|
|
} | ConvertTo-Json -Compress
|
||
|
|
$prepared = Invoke-RestMethod -Method Post -Uri "$api/api/v1/households/$householdId/firmware-releases" -Headers $headers -ContentType 'application/json' -Body $releaseBody
|
||
|
|
$releaseId = $prepared.data.id
|
||
|
|
Invoke-WebRequest -UseBasicParsing -Method Put -Uri $prepared.data.upload_url -InFile $tempFile -ContentType 'application/octet-stream' | Out-Null
|
||
|
|
$ready = Invoke-RestMethod -Method Post -Uri "$api/api/v1/firmware-releases/$releaseId/finalize" -Headers $headers
|
||
|
|
if ($ready.data.status -ne 'READY') {
|
||
|
|
throw "Firmware release did not become READY: $($ready.data.status)"
|
||
|
|
}
|
||
|
|
|
||
|
|
$rolloutBody = @{
|
||
|
|
release_id = $releaseId
|
||
|
|
name = if ($Mode -eq 'ABRollback') { 'simulator A-B rollback smoke' } else { 'simulator interruption recovery smoke' }
|
||
|
|
target_percentage = 100
|
||
|
|
device_ids = @($deviceId)
|
||
|
|
} | ConvertTo-Json -Compress
|
||
|
|
$rollout = Invoke-RestMethod -Method Post -Uri "$api/api/v1/households/$householdId/firmware-rollouts" -Headers $headers -ContentType 'application/json' -Body $rolloutBody
|
||
|
|
$rolloutId = $rollout.data.id
|
||
|
|
|
||
|
|
$env:SIM_API_URL = $api
|
||
|
|
$env:SIM_DEVICE_ID = $deviceId
|
||
|
|
$env:SIM_DEVICE_SECRET = 'local-simulated-device-secret-32-bytes-minimum'
|
||
|
|
$env:SIM_SCENARIO = 'commands'
|
||
|
|
$env:SIM_FIRMWARE_STEP_DELAY = '0s'
|
||
|
|
$env:SIM_FIRMWARE_TRUSTED_KEYS = @{ $keyId = $signed.public_key_base64 } | ConvertTo-Json -Compress
|
||
|
|
if ($Mode -eq 'ABRollback') {
|
||
|
|
$env:SIM_FIRMWARE_AB_UPDATE = 'true'
|
||
|
|
$env:SIM_FIRMWARE_STATE_FILE = $stateFile
|
||
|
|
$env:SIM_FIRMWARE_BOOT_HEALTH_WINDOW = '30s'
|
||
|
|
$env:SIM_FIRMWARE_ACTIVE_SLOT = 'A'
|
||
|
|
$env:SIM_FIRMWARE_CURRENT_VERSION = '0.2.0-sim'
|
||
|
|
$env:SIM_FIRMWARE_EVIDENCE_FILE = $evidenceFile
|
||
|
|
$env:SIM_FIRMWARE_EVIDENCE_SCENARIO = 'AB_ROLLBACK'
|
||
|
|
$env:SIM_CANDIDATE_VERSION = 'local-ab-conformance-smoke'
|
||
|
|
$env:SIM_BUILD_DIGEST = 'sha256:' + $signed.sha256
|
||
|
|
$env:SIM_BOOTLOADER_VERSION = 'sim-bootloader-1'
|
||
|
|
$env:SIM_PARTITION_TABLE_SHA256 = $signed.sha256
|
||
|
|
$env:SIM_SECURE_BOOT_ENABLED = 'false'
|
||
|
|
$env:SIM_FIRMWARE_FAULT = 'none'
|
||
|
|
go run ./cmd/sim-device
|
||
|
|
|
||
|
|
$armedState = Invoke-DatabaseScalar "SELECT dc.status||'|'||fa.status||'|'||fa.progress_percent||'|'||COALESCE(dc.result_summary->>'result_code','') FROM device_command dc JOIN firmware_assignment fa ON fa.command_id=dc.id JOIN firmware_rollout ro ON ro.id=fa.rollout_id WHERE ro.public_id='$rolloutId'::uuid"
|
||
|
|
if ($armedState -ne 'ACKNOWLEDGED|INSTALLING|90|REBOOT_ARMED') {
|
||
|
|
throw "A/B firmware update did not reach the reboot boundary: $armedState"
|
||
|
|
}
|
||
|
|
$localArmed = Get-Content -LiteralPath $stateFile -Raw | ConvertFrom-Json
|
||
|
|
if ($localArmed.phase -ne 'REBOOT_PENDING' -or $localArmed.active_slot -ne 'A' -or $localArmed.target_slot -ne 'B' -or -not $localArmed.boot_attempt_id) {
|
||
|
|
throw 'Local A/B journal did not durably arm the inactive slot'
|
||
|
|
}
|
||
|
|
|
||
|
|
$env:SIM_FIRMWARE_FAULT = 'boot_health_failed'
|
||
|
|
go run ./cmd/sim-device
|
||
|
|
$assignment = Get-RolloutAssignment $headers
|
||
|
|
if (-not $assignment -or $assignment.status -ne 'FAILED' -or [int]$assignment.progress_percent -ne 95 -or -not [bool]$assignment.last_result.rollback_applied -or $assignment.last_result.rollback_version -ne '0.2.0-sim' -or $assignment.events.Count -ne 7) {
|
||
|
|
throw 'Failed A/B boot did not expose an applied rollback to the prior version'
|
||
|
|
}
|
||
|
|
$eventPhases = ($assignment.events | ForEach-Object { $_.phase }) -join ','
|
||
|
|
if ($eventPhases -ne 'downloading,downloading,installing,staging,reboot_pending,boot_verifying,rolled_back') {
|
||
|
|
throw "Cloud firmware event timeline was incomplete or out of order: $eventPhases"
|
||
|
|
}
|
||
|
|
$terminalState = Invoke-DatabaseScalar "SELECT dc.status||'|'||fa.status||'|'||fa.progress_percent||'|'||COALESCE(dc.result_summary->>'result_code','')||'|'||dc.delivery_attempts FROM device_command dc JOIN firmware_assignment fa ON fa.command_id=dc.id JOIN firmware_rollout ro ON ro.id=fa.rollout_id WHERE ro.public_id='$rolloutId'::uuid"
|
||
|
|
if ($terminalState -ne 'FAILED|FAILED|95|BOOT_HEALTH_FAILED_ROLLED_BACK|2') {
|
||
|
|
throw "A/B rollback did not converge to the expected terminal state: $terminalState"
|
||
|
|
}
|
||
|
|
$localRolledBack = Get-Content -LiteralPath $stateFile -Raw | ConvertFrom-Json
|
||
|
|
if ($localRolledBack.phase -ne 'ROLLED_BACK' -or $localRolledBack.active_slot -ne 'A' -or $localRolledBack.target_version -ne '0.2.0-sim') {
|
||
|
|
throw 'Local A/B journal did not return to the prior bootable slot'
|
||
|
|
}
|
||
|
|
$impact = Invoke-RestMethod -Method Get -Uri "$api/api/v1/firmware-releases/$releaseId/impact" -Headers $headers
|
||
|
|
if ([int]$impact.data.devices.failed -ne 1 -or [int]$impact.data.devices.observing -ne 0 -or [int]$impact.data.commands.failed -ne 1 -or [int]$impact.data.rollouts.paused -ne 1) {
|
||
|
|
throw 'Firmware impact summary did not converge after automatic A/B rollback'
|
||
|
|
}
|
||
|
|
$rolloutDetail = Invoke-RestMethod -Method Get -Uri "$api/api/v1/firmware-rollouts/$rolloutId" -Headers $headers
|
||
|
|
$rolloutDetail | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $cloudFile -Encoding utf8NoBOM
|
||
|
|
go run ./cmd/firmware-conformance -cloud $cloudFile -device $evidenceFile -output $conformanceFile
|
||
|
|
$conformance = Get-Content -LiteralPath $conformanceFile -Raw | ConvertFrom-Json
|
||
|
|
$failedChecks = @($conformance.checks | Where-Object { $_.status -ne 'PASS' })
|
||
|
|
if ($conformance.result -ne 'PASS' -or $conformance.evidence_scope -ne 'PROTOCOL_SIMULATION' -or -not [bool]$conformance.external_validation_required -or $failedChecks.Count -ne 0) {
|
||
|
|
throw 'Firmware conformance report did not pass or incorrectly claimed external validation'
|
||
|
|
}
|
||
|
|
Write-Host "Simulated A/B firmware boot failure rollback smoke passed: release $releaseId, rollout $rolloutId"
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
$env:SIM_FIRMWARE_AB_UPDATE = 'false'
|
||
|
|
$env:SIM_FIRMWARE_FAULT = 'interrupt_after_download'
|
||
|
|
go run ./cmd/sim-device
|
||
|
|
|
||
|
|
$interruptedState = Invoke-DatabaseScalar "SELECT dc.status||'|'||fa.status||'|'||fa.progress_percent||'|'||COALESCE(dc.result_summary->>'result_code','') FROM device_command dc JOIN firmware_assignment fa ON fa.command_id=dc.id JOIN firmware_rollout ro ON ro.id=fa.rollout_id WHERE ro.public_id='$rolloutId'::uuid"
|
||
|
|
if ($interruptedState -ne 'ACKNOWLEDGED|DOWNLOADING|100|ARTIFACT_VERIFIED') {
|
||
|
|
throw "Firmware interruption state was not recoverable: $interruptedState"
|
||
|
|
}
|
||
|
|
|
||
|
|
$env:SIM_FIRMWARE_FAULT = 'none'
|
||
|
|
go run ./cmd/sim-device
|
||
|
|
|
||
|
|
$assignment = Get-RolloutAssignment $headers
|
||
|
|
if (-not $assignment -or $assignment.status -ne 'CONFIRMED' -or [int]$assignment.progress_percent -ne 100) {
|
||
|
|
throw "Recovered firmware assignment did not converge to CONFIRMED/100: $($assignment.status)/$($assignment.progress_percent)"
|
||
|
|
}
|
||
|
|
$terminalState = Invoke-DatabaseScalar "SELECT dc.status||'|'||fa.status||'|'||fa.progress_percent||'|'||COALESCE(dc.result_summary->>'result_code','')||'|'||dc.delivery_attempts FROM device_command dc JOIN firmware_assignment fa ON fa.command_id=dc.id JOIN firmware_rollout ro ON ro.id=fa.rollout_id WHERE ro.public_id='$rolloutId'::uuid"
|
||
|
|
if ($terminalState -ne 'SUCCEEDED|CONFIRMED|100|SIMULATED_OK|2') {
|
||
|
|
throw "Recovered firmware command did not reach the expected terminal state: $terminalState"
|
||
|
|
}
|
||
|
|
$impact = Invoke-RestMethod -Method Get -Uri "$api/api/v1/firmware-releases/$releaseId/impact" -Headers $headers
|
||
|
|
if ([int]$impact.data.devices.confirmed -ne 1 -or [int]$impact.data.devices.observing -ne 0 -or [int]$impact.data.commands.succeeded -ne 1 -or [int]$impact.data.rollouts.completed -ne 1) {
|
||
|
|
throw 'Firmware impact summary did not converge after simulator recovery'
|
||
|
|
}
|
||
|
|
|
||
|
|
Write-Host "Simulated firmware interruption recovery smoke passed: release $releaseId, rollout $rolloutId"
|
||
|
|
}
|
||
|
|
}
|
||
|
|
finally {
|
||
|
|
if ($releaseId) {
|
||
|
|
$refs = docker compose exec -T postgres psql -U voice -d voice_platform -t -A -c "SELECT object_ref FROM firmware_release WHERE public_id='$releaseId'::uuid" | Out-String
|
||
|
|
$objectRefs = $refs -split "`r?`n" | Where-Object { $_.Trim() }
|
||
|
|
}
|
||
|
|
if ($releaseId) {
|
||
|
|
$cleanupSql = @"
|
||
|
|
BEGIN;
|
||
|
|
CREATE TEMP TABLE smoke_releases AS SELECT id,public_id FROM firmware_release WHERE public_id='$releaseId'::uuid;
|
||
|
|
CREATE TEMP TABLE smoke_rollouts AS SELECT id,public_id FROM firmware_rollout WHERE release_id IN(SELECT id FROM smoke_releases);
|
||
|
|
CREATE TEMP TABLE smoke_commands AS SELECT dc.id,dc.public_id FROM device_command dc JOIN firmware_assignment fa ON fa.command_id=dc.id WHERE fa.rollout_id IN(SELECT id FROM smoke_rollouts);
|
||
|
|
DELETE FROM audit_event WHERE resource_id IN(SELECT public_id FROM smoke_releases UNION SELECT public_id FROM smoke_rollouts UNION SELECT public_id FROM smoke_commands);
|
||
|
|
DELETE FROM web_event WHERE resource_public_id IN(SELECT public_id FROM smoke_commands);
|
||
|
|
DELETE FROM firmware_assignment WHERE rollout_id IN(SELECT id FROM smoke_rollouts);
|
||
|
|
DELETE FROM firmware_rollout WHERE id IN(SELECT id FROM smoke_rollouts);
|
||
|
|
DELETE FROM device_command WHERE id IN(SELECT id FROM smoke_commands);
|
||
|
|
DELETE FROM firmware_release WHERE id IN(SELECT id FROM smoke_releases);
|
||
|
|
DELETE FROM audit_event WHERE resource_type='firmware_signing_key' AND detail->>'key_id'='$keyId';
|
||
|
|
DELETE FROM firmware_signing_key WHERE key_id='$keyId';
|
||
|
|
COMMIT;
|
||
|
|
"@
|
||
|
|
docker compose exec -T postgres psql -U voice -d voice_platform -v ON_ERROR_STOP=1 -q -c $cleanupSql | Out-Null
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
docker compose exec -T postgres psql -U voice -d voice_platform -v ON_ERROR_STOP=1 -q -c "DELETE FROM audit_event WHERE resource_type='firmware_signing_key' AND detail->>'key_id'='$keyId'; DELETE FROM firmware_signing_key WHERE key_id='$keyId';" | Out-Null
|
||
|
|
}
|
||
|
|
if ($objectRefs.Count -gt 0) {
|
||
|
|
docker compose exec -T minio mc alias set smoke http://localhost:9000 local_minio local_minio_password | Out-Null
|
||
|
|
foreach ($objectRef in $objectRefs) {
|
||
|
|
docker compose exec -T minio mc rm --force "smoke/private-exports/$($objectRef.Trim())" | Out-Null
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (Test-Path -LiteralPath $tempFile) {
|
||
|
|
Remove-Item -LiteralPath $tempFile -Force
|
||
|
|
}
|
||
|
|
if (Test-Path -LiteralPath $stateFile) {
|
||
|
|
Remove-Item -LiteralPath $stateFile -Force
|
||
|
|
}
|
||
|
|
foreach ($artifact in @($evidenceFile, $cloudFile, $conformanceFile)) {
|
||
|
|
if (Test-Path -LiteralPath $artifact) {
|
||
|
|
Remove-Item -LiteralPath $artifact -Force
|
||
|
|
}
|
||
|
|
}
|
||
|
|
foreach ($name in $environmentNames) {
|
||
|
|
if ($null -eq $previousEnvironment[$name]) {
|
||
|
|
Remove-Item -LiteralPath "Env:$name" -ErrorAction SilentlyContinue
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
Set-Item -LiteralPath "Env:$name" -Value $previousEnvironment[$name]
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|