$ErrorActionPreference='Stop' $PSNativeCommandUseErrorActionPreference=$true $root=Split-Path -Parent $PSScriptRoot Set-Location -LiteralPath $root $run=[Guid]::NewGuid().ToString('N') $prefix="provider-outage-$run" $tenant='10000000-0000-0000-0000-000000000001' $household='10000000-0000-0000-0000-000000000201' $subject='10000000-0000-0000-0000-000000000401' $providerHeaders=@{'X-Mock-Provider-Key'='local-mock-provider-key'} $conversationIds=@() function Sql([string]$statement){ $value=$statement|docker compose exec -T postgres psql -U voice -d voice_platform -v ON_ERROR_STOP=1 -q -t -A if($LASTEXITCODE-ne0){throw 'Provider outage SQL failed'} return ($value|Out-String).Trim() } function New-StartFixture([string]$suffix){ $task="$prefix-$suffix" $sql=@" WITH d AS ( INSERT INTO device(tenant_id,serial_digest,model,hardware_revision,credential_ref,minimum_security_version) VALUES('$tenant'::uuid,encode(digest('$task','sha256'),'hex'),'$prefix','fault','local-dev://simulated-terminal-1','local') RETURNING id ), cs AS ( INSERT INTO consent_snapshot(tenant_id,subject_id,snapshot_version,effective_consents,source_consent_ids) VALUES('$tenant'::uuid,'$subject'::uuid,1,'{}'::jsonb,jsonb_build_array('$task')) RETURNING id ), cx AS ( INSERT INTO context_snapshot(tenant_id,subject_id,profile_version,consent_snapshot_id,memory_versions,prompt_bundle_version,engine_mode,content_digest,sanitized_summary) SELECT '$tenant'::uuid,'$subject'::uuid,1,cs.id,'[]'::jsonb,'$prefix','s2s_hybrid',encode(digest('$task','sha256'),'hex'),'供应商故障合成夹具' FROM cs RETURNING id,consent_snapshot_id ), c AS ( INSERT INTO conversation(tenant_id,household_id,subject_id,device_id,consent_snapshot_id,context_snapshot_id,provider_task_ref,engine_mode,retention_mode,status,allocation_expires_at,lease_expires_at) SELECT '$tenant'::uuid,'$household'::uuid,'$subject'::uuid,d.id,cx.consent_snapshot_id,cx.id,'$task','s2s_hybrid','none','STARTING_AGENT',now()+interval '15 minutes',now()+interval '15 minutes' FROM d CROSS JOIN cx RETURNING id,public_id ), rt AS ( INSERT INTO rtc_task(conversation_id,provider,provider_task_ref,room_id,device_rtc_user_id,agent_rtc_user_id,status,lease_expires_at) SELECT id,'mock','$task','room-$task','device-$suffix','agent-$suffix','STARTING',now()+interval '15 minutes' FROM c ), history AS ( INSERT INTO conversation_state_history(conversation_id,from_status,to_status,reason,source) SELECT id,'ROOM_ISSUED','STARTING_AGENT','provider outage verification fixture','platform' FROM c ), queued AS ( INSERT INTO outbox_event(tenant_id,aggregate_type,aggregate_id,event_type,deduplication_key,payload) SELECT '$tenant'::uuid,'conversation',id,'conversation.agent_start_requested','provider-outage-start:'||id,'{}'::jsonb FROM c ) SELECT public_id::text FROM c; "@ $id=((Sql $sql)-split"`n"|Select-Object -Last 1).Trim() if($id-notmatch'^[0-9a-fA-F-]{36}$'){throw "Invalid outage fixture id: $id"} $script:conversationIds+=$id return $id } function Wait-Sql([string]$statement,[string]$expected,[int]$seconds=30){ $deadline=[DateTime]::UtcNow.AddSeconds($seconds) do{ $value=Sql $statement if($value-eq$expected){return $value} Start-Sleep -Milliseconds 500 }while([DateTime]::UtcNow-lt$deadline) throw "Timed out waiting for '$expected', last='$value'" } function Set-Provider([bool]$available){ $body=@{available=$available}|ConvertTo-Json -Compress Invoke-WebRequest -UseBasicParsing -Method Post -Uri 'http://localhost:59200/mock/v1/admin/availability' -Headers $providerHeaders -ContentType 'application/json' -Body $body|Out-Null } try{ Invoke-WebRequest -UseBasicParsing -Method Post -Uri 'http://localhost:59200/mock/v1/admin/reset' -Headers $providerHeaders|Out-Null $active=New-StartFixture 'active' Wait-Sql "SELECT status FROM conversation WHERE public_id='$active'::uuid" 'ACTIVE' 20|Out-Null Set-Provider $false $failing=New-StartFixture 'failing' Wait-Sql "SELECT status FROM provider_dependency_state WHERE tenant_id='$tenant'::uuid AND provider='mock'" 'UNAVAILABLE' 30|Out-Null $state=(Invoke-RestMethod -Method Get -Uri 'http://localhost:59200/mock/v1/admin/state' -Headers $providerHeaders) $callsBefore=[int]$state.start_calls $paused=New-StartFixture 'paused' Start-Sleep -Seconds 3 $state=(Invoke-RestMethod -Method Get -Uri 'http://localhost:59200/mock/v1/admin/state' -Headers $providerHeaders) if([int]$state.start_calls-ne$callsBefore){throw "New start reached unavailable provider after circuit opened: before=$callsBefore after=$($state.start_calls)"} $pausedEvidence=Sql "SELECT c.status||'|'||rt.start_attempts FROM conversation c JOIN rtc_task rt ON rt.conversation_id=c.id WHERE c.public_id='$paused'::uuid" if($pausedEvidence-ne'STARTING_AGENT|0'){throw "Paused start evidence mismatch: $pausedEvidence"} Sql "WITH c AS (UPDATE conversation SET status='STOPPING',stop_reason='provider_outage_probe',lease_expires_at=NULL,updated_at=now() WHERE public_id='$active'::uuid RETURNING id) UPDATE rtc_task SET status='STOPPING',lease_expires_at=NULL,updated_at=now() WHERE conversation_id IN(SELECT id FROM c); INSERT INTO outbox_event(tenant_id,aggregate_type,aggregate_id,event_type,deduplication_key,payload) SELECT '$tenant'::uuid,'conversation',id,'conversation.agent_stop_requested','provider-outage-stop:'||id,'{}'::jsonb FROM conversation WHERE public_id='$active'::uuid;"|Out-Null Wait-Sql "SELECT (stop_attempts>0)::text FROM rtc_task rt JOIN conversation c ON c.id=rt.conversation_id WHERE c.public_id='$active'::uuid" 'true' 15|Out-Null $token=(Invoke-RestMethod -Method Post -Uri 'http://localhost:58180/realms/voice-local/protocol/openid-connect/token' -ContentType 'application/x-www-form-urlencoded' -Body @{grant_type='password';client_id='voice-platform-web';username='owner';password='LocalOnly123!';scope='openid'}).access_token $history=Invoke-WebRequest -UseBasicParsing -Uri "http://localhost:58080/api/v1/conversations/$active" -Headers @{Authorization="Bearer $token"} if($history.StatusCode-ne200){throw 'Historical conversation became unavailable with provider'} Wait-Sql "SELECT count(*)::text FROM alert_incident WHERE tenant_id='$tenant'::uuid AND rule_code='PROVIDER_UNAVAILABLE' AND fingerprint='provider-unavailable:mock' AND status<>'RESOLVED'" '1' 15|Out-Null Set-Provider $true Wait-Sql "SELECT status FROM conversation WHERE public_id='$active'::uuid" 'COMPLETED' 30|Out-Null Wait-Sql "SELECT status FROM provider_dependency_state WHERE tenant_id='$tenant'::uuid AND provider='mock'" 'AVAILABLE' 15|Out-Null Wait-Sql "SELECT count(*)::text FROM conversation WHERE public_id IN('$failing'::uuid,'$paused'::uuid) AND status='ACTIVE'" '2' 30|Out-Null $stopAttempts=Sql "SELECT rt.stop_attempts::text FROM rtc_task rt JOIN conversation c ON c.id=rt.conversation_id WHERE c.public_id='$active'::uuid" Write-Host "Provider outage recovery passed: circuit_opened=true, paused_start_calls=0, history_http=200, alert=PROVIDER_UNAVAILABLE, cleanup_attempts=$stopAttempts, recovered=true" }finally{ try{Set-Provider $true}catch{} if($conversationIds.Count-gt0){ $ids=($conversationIds|ForEach-Object{"'$_'::uuid"})-join',' $cleanup=@" BEGIN; CREATE TEMP TABLE outage_c ON COMMIT DROP AS SELECT id,public_id,context_snapshot_id,consent_snapshot_id,device_id FROM conversation WHERE public_id IN($ids); DELETE FROM alert_incident WHERE tenant_id='$tenant'::uuid AND fingerprint='provider-unavailable:mock'; DELETE FROM web_event WHERE resource_public_id IN(SELECT public_id FROM outage_c); DELETE FROM audit_event WHERE resource_id IN(SELECT id FROM outage_c); DELETE FROM outbox_event WHERE aggregate_id IN(SELECT id FROM outage_c); DELETE FROM conversation_summary WHERE conversation_id IN(SELECT id FROM outage_c); DELETE FROM conversation_turn WHERE conversation_id IN(SELECT id FROM outage_c); DELETE FROM provider_event WHERE conversation_id IN(SELECT id FROM outage_c); DELETE FROM conversation_state_history WHERE conversation_id IN(SELECT id FROM outage_c); DELETE FROM rtc_task WHERE conversation_id IN(SELECT id FROM outage_c); DELETE FROM conversation WHERE id IN(SELECT id FROM outage_c); DELETE FROM context_snapshot WHERE id IN(SELECT context_snapshot_id FROM outage_c); DELETE FROM consent_snapshot WHERE id IN(SELECT consent_snapshot_id FROM outage_c); DELETE FROM device WHERE id IN(SELECT device_id FROM outage_c); DELETE FROM provider_dependency_state WHERE tenant_id='$tenant'::uuid AND provider='mock'; COMMIT; "@ try{Sql $cleanup|Out-Null}catch{Write-Warning $_} } try{Invoke-WebRequest -UseBasicParsing -Method Post -Uri 'http://localhost:59200/mock/v1/admin/reset' -Headers $providerHeaders|Out-Null}catch{} }