126 lines
8.7 KiB
PowerShell
126 lines
8.7 KiB
PowerShell
$ErrorActionPreference='Stop'
|
|
$PSNativeCommandUseErrorActionPreference=$true
|
|
$root=Split-Path -Parent $PSScriptRoot
|
|
Set-Location -LiteralPath $root
|
|
$run=[Guid]::NewGuid().ToString('N')
|
|
$prefix="provider-outage-$run"
|
|
$tenant='10000000-0000-0000-0000-000000000001'
|
|
$household='10000000-0000-0000-0000-000000000201'
|
|
$subject='10000000-0000-0000-0000-000000000401'
|
|
$providerHeaders=@{'X-Mock-Provider-Key'='local-mock-provider-key'}
|
|
$conversationIds=@()
|
|
|
|
function Sql([string]$statement){
|
|
$value=$statement|docker compose exec -T postgres psql -U voice -d voice_platform -v ON_ERROR_STOP=1 -q -t -A
|
|
if($LASTEXITCODE-ne0){throw 'Provider outage SQL failed'}
|
|
return ($value|Out-String).Trim()
|
|
}
|
|
|
|
function New-StartFixture([string]$suffix){
|
|
$task="$prefix-$suffix"
|
|
$sql=@"
|
|
WITH d AS (
|
|
INSERT INTO device(tenant_id,serial_digest,model,hardware_revision,credential_ref,minimum_security_version)
|
|
VALUES('$tenant'::uuid,encode(digest('$task','sha256'),'hex'),'$prefix','fault','local-dev://simulated-terminal-1','local') RETURNING id
|
|
), cs AS (
|
|
INSERT INTO consent_snapshot(tenant_id,subject_id,snapshot_version,effective_consents,source_consent_ids)
|
|
VALUES('$tenant'::uuid,'$subject'::uuid,1,'{}'::jsonb,jsonb_build_array('$task')) RETURNING id
|
|
), cx AS (
|
|
INSERT INTO context_snapshot(tenant_id,subject_id,profile_version,consent_snapshot_id,memory_versions,prompt_bundle_version,engine_mode,content_digest,sanitized_summary)
|
|
SELECT '$tenant'::uuid,'$subject'::uuid,1,cs.id,'[]'::jsonb,'$prefix','s2s_hybrid',encode(digest('$task','sha256'),'hex'),'供应商故障合成夹具' FROM cs RETURNING id,consent_snapshot_id
|
|
), c AS (
|
|
INSERT INTO conversation(tenant_id,household_id,subject_id,device_id,consent_snapshot_id,context_snapshot_id,provider_task_ref,engine_mode,retention_mode,status,allocation_expires_at,lease_expires_at)
|
|
SELECT '$tenant'::uuid,'$household'::uuid,'$subject'::uuid,d.id,cx.consent_snapshot_id,cx.id,'$task','s2s_hybrid','none','STARTING_AGENT',now()+interval '15 minutes',now()+interval '15 minutes' FROM d CROSS JOIN cx RETURNING id,public_id
|
|
), rt AS (
|
|
INSERT INTO rtc_task(conversation_id,provider,provider_task_ref,room_id,device_rtc_user_id,agent_rtc_user_id,status,lease_expires_at)
|
|
SELECT id,'mock','$task','room-$task','device-$suffix','agent-$suffix','STARTING',now()+interval '15 minutes' FROM c
|
|
), history AS (
|
|
INSERT INTO conversation_state_history(conversation_id,from_status,to_status,reason,source)
|
|
SELECT id,'ROOM_ISSUED','STARTING_AGENT','provider outage verification fixture','platform' FROM c
|
|
), queued AS (
|
|
INSERT INTO outbox_event(tenant_id,aggregate_type,aggregate_id,event_type,deduplication_key,payload)
|
|
SELECT '$tenant'::uuid,'conversation',id,'conversation.agent_start_requested','provider-outage-start:'||id,'{}'::jsonb FROM c
|
|
)
|
|
SELECT public_id::text FROM c;
|
|
"@
|
|
$id=((Sql $sql)-split"`n"|Select-Object -Last 1).Trim()
|
|
if($id-notmatch'^[0-9a-fA-F-]{36}$'){throw "Invalid outage fixture id: $id"}
|
|
$script:conversationIds+=$id
|
|
return $id
|
|
}
|
|
|
|
function Wait-Sql([string]$statement,[string]$expected,[int]$seconds=30){
|
|
$deadline=[DateTime]::UtcNow.AddSeconds($seconds)
|
|
do{
|
|
$value=Sql $statement
|
|
if($value-eq$expected){return $value}
|
|
Start-Sleep -Milliseconds 500
|
|
}while([DateTime]::UtcNow-lt$deadline)
|
|
throw "Timed out waiting for '$expected', last='$value'"
|
|
}
|
|
|
|
function Set-Provider([bool]$available){
|
|
$body=@{available=$available}|ConvertTo-Json -Compress
|
|
Invoke-WebRequest -UseBasicParsing -Method Post -Uri 'http://localhost:59200/mock/v1/admin/availability' -Headers $providerHeaders -ContentType 'application/json' -Body $body|Out-Null
|
|
}
|
|
|
|
try{
|
|
Invoke-WebRequest -UseBasicParsing -Method Post -Uri 'http://localhost:59200/mock/v1/admin/reset' -Headers $providerHeaders|Out-Null
|
|
$active=New-StartFixture 'active'
|
|
Wait-Sql "SELECT status FROM conversation WHERE public_id='$active'::uuid" 'ACTIVE' 20|Out-Null
|
|
|
|
Set-Provider $false
|
|
$failing=New-StartFixture 'failing'
|
|
Wait-Sql "SELECT status FROM provider_dependency_state WHERE tenant_id='$tenant'::uuid AND provider='mock'" 'UNAVAILABLE' 30|Out-Null
|
|
$state=(Invoke-RestMethod -Method Get -Uri 'http://localhost:59200/mock/v1/admin/state' -Headers $providerHeaders)
|
|
$callsBefore=[int]$state.start_calls
|
|
|
|
$paused=New-StartFixture 'paused'
|
|
Start-Sleep -Seconds 3
|
|
$state=(Invoke-RestMethod -Method Get -Uri 'http://localhost:59200/mock/v1/admin/state' -Headers $providerHeaders)
|
|
if([int]$state.start_calls-ne$callsBefore){throw "New start reached unavailable provider after circuit opened: before=$callsBefore after=$($state.start_calls)"}
|
|
$pausedEvidence=Sql "SELECT c.status||'|'||rt.start_attempts FROM conversation c JOIN rtc_task rt ON rt.conversation_id=c.id WHERE c.public_id='$paused'::uuid"
|
|
if($pausedEvidence-ne'STARTING_AGENT|0'){throw "Paused start evidence mismatch: $pausedEvidence"}
|
|
|
|
Sql "WITH c AS (UPDATE conversation SET status='STOPPING',stop_reason='provider_outage_probe',lease_expires_at=NULL,updated_at=now() WHERE public_id='$active'::uuid RETURNING id) UPDATE rtc_task SET status='STOPPING',lease_expires_at=NULL,updated_at=now() WHERE conversation_id IN(SELECT id FROM c); INSERT INTO outbox_event(tenant_id,aggregate_type,aggregate_id,event_type,deduplication_key,payload) SELECT '$tenant'::uuid,'conversation',id,'conversation.agent_stop_requested','provider-outage-stop:'||id,'{}'::jsonb FROM conversation WHERE public_id='$active'::uuid;"|Out-Null
|
|
Wait-Sql "SELECT (stop_attempts>0)::text FROM rtc_task rt JOIN conversation c ON c.id=rt.conversation_id WHERE c.public_id='$active'::uuid" 'true' 15|Out-Null
|
|
|
|
$token=(Invoke-RestMethod -Method Post -Uri 'http://localhost:58180/realms/voice-local/protocol/openid-connect/token' -ContentType 'application/x-www-form-urlencoded' -Body @{grant_type='password';client_id='voice-platform-web';username='owner';password='LocalOnly123!';scope='openid'}).access_token
|
|
$history=Invoke-WebRequest -UseBasicParsing -Uri "http://localhost:58080/api/v1/conversations/$active" -Headers @{Authorization="Bearer $token"}
|
|
if($history.StatusCode-ne200){throw 'Historical conversation became unavailable with provider'}
|
|
Wait-Sql "SELECT count(*)::text FROM alert_incident WHERE tenant_id='$tenant'::uuid AND rule_code='PROVIDER_UNAVAILABLE' AND fingerprint='provider-unavailable:mock' AND status<>'RESOLVED'" '1' 15|Out-Null
|
|
|
|
Set-Provider $true
|
|
Wait-Sql "SELECT status FROM conversation WHERE public_id='$active'::uuid" 'COMPLETED' 30|Out-Null
|
|
Wait-Sql "SELECT status FROM provider_dependency_state WHERE tenant_id='$tenant'::uuid AND provider='mock'" 'AVAILABLE' 15|Out-Null
|
|
Wait-Sql "SELECT count(*)::text FROM conversation WHERE public_id IN('$failing'::uuid,'$paused'::uuid) AND status='ACTIVE'" '2' 30|Out-Null
|
|
$stopAttempts=Sql "SELECT rt.stop_attempts::text FROM rtc_task rt JOIN conversation c ON c.id=rt.conversation_id WHERE c.public_id='$active'::uuid"
|
|
Write-Host "Provider outage recovery passed: circuit_opened=true, paused_start_calls=0, history_http=200, alert=PROVIDER_UNAVAILABLE, cleanup_attempts=$stopAttempts, recovered=true"
|
|
}finally{
|
|
try{Set-Provider $true}catch{}
|
|
if($conversationIds.Count-gt0){
|
|
$ids=($conversationIds|ForEach-Object{"'$_'::uuid"})-join','
|
|
$cleanup=@"
|
|
BEGIN;
|
|
CREATE TEMP TABLE outage_c ON COMMIT DROP AS SELECT id,public_id,context_snapshot_id,consent_snapshot_id,device_id FROM conversation WHERE public_id IN($ids);
|
|
DELETE FROM alert_incident WHERE tenant_id='$tenant'::uuid AND fingerprint='provider-unavailable:mock';
|
|
DELETE FROM web_event WHERE resource_public_id IN(SELECT public_id FROM outage_c);
|
|
DELETE FROM audit_event WHERE resource_id IN(SELECT id FROM outage_c);
|
|
DELETE FROM outbox_event WHERE aggregate_id IN(SELECT id FROM outage_c);
|
|
DELETE FROM conversation_summary WHERE conversation_id IN(SELECT id FROM outage_c);
|
|
DELETE FROM conversation_turn WHERE conversation_id IN(SELECT id FROM outage_c);
|
|
DELETE FROM provider_event WHERE conversation_id IN(SELECT id FROM outage_c);
|
|
DELETE FROM conversation_state_history WHERE conversation_id IN(SELECT id FROM outage_c);
|
|
DELETE FROM rtc_task WHERE conversation_id IN(SELECT id FROM outage_c);
|
|
DELETE FROM conversation WHERE id IN(SELECT id FROM outage_c);
|
|
DELETE FROM context_snapshot WHERE id IN(SELECT context_snapshot_id FROM outage_c);
|
|
DELETE FROM consent_snapshot WHERE id IN(SELECT consent_snapshot_id FROM outage_c);
|
|
DELETE FROM device WHERE id IN(SELECT device_id FROM outage_c);
|
|
DELETE FROM provider_dependency_state WHERE tenant_id='$tenant'::uuid AND provider='mock';
|
|
COMMIT;
|
|
"@
|
|
try{Sql $cleanup|Out-Null}catch{Write-Warning $_}
|
|
}
|
|
try{Invoke-WebRequest -UseBasicParsing -Method Post -Uri 'http://localhost:59200/mock/v1/admin/reset' -Headers $providerHeaders|Out-Null}catch{}
|
|
}
|