package orchestration import ( "context" "encoding/json" "io" "log/slog" "os" "testing" "time" "data-terminal-cloud/internal/database" "data-terminal-cloud/internal/deviceauth" "data-terminal-cloud/internal/outbox" "data-terminal-cloud/internal/voiceprovider" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" ) func TestRecoveryScannerExpiresAndSchedulesIdempotentReconciliation(t *testing.T) { databaseURL := os.Getenv("TEST_DATABASE_URL") if databaseURL == "" { t.Skip("TEST_DATABASE_URL is not set") } ctx := context.Background() pool, err := database.Open(ctx, databaseURL, "orchestration-recovery-test") if err != nil { t.Fatal(err) } defer pool.Close() identity := deviceauth.Identity{ ID: "10000000-0000-0000-0000-000000000501", PublicID: "10000000-0000-0000-0000-000000000511", TenantID: "10000000-0000-0000-0000-000000000001", Status: "registered", } now := time.Date(2026, 8, 23, 13, 0, 0, 0, time.UTC) publicIDs := make([]string, 0, 3) defer func() { cleanupRecoveryConversations(t, context.Background(), pool, publicIDs) }() roomIssued, roomInternalID := createRecoveryConversation(t, ctx, pool, identity, now.Add(-time.Minute), now.Add(-30*time.Second)) publicIDs = append(publicIDs, roomIssued) logger := slog.New(slog.NewTextHandler(io.Discard, nil)) scanner, err := NewRecoveryScanner(pool, outbox.NewStore(pool), logger, RecoveryOptions{ ScanInterval: time.Second, StartingAfter: 10 * time.Second, StoppingAfter: 10 * time.Second, BatchSize: 10, }) if err != nil { t.Fatal(err) } scanner.now = func() time.Time { return now } if count, err := scanner.ScanOnce(ctx); err != nil || count != 1 { t.Fatalf("expired room scan count=%d err=%v", count, err) } var status, taskStatus string if err := pool.QueryRow(ctx, `SELECT c.status, r.status FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id WHERE c.id=$1::uuid`, roomInternalID).Scan(&status, &taskStatus); err != nil { t.Fatal(err) } if status != "EXPIRED" || taskStatus != "COMPLETED" { t.Fatalf("expired room conversation=%s task=%s", status, taskStatus) } activePublicID, activeInternalID := createRecoveryConversation(t, ctx, pool, identity, now, now.Add(time.Minute)) publicIDs = append(publicIDs, activePublicID) if _, err := pool.Exec(ctx, `UPDATE conversation SET status='ACTIVE', lease_expires_at=$2, updated_at=$3 WHERE id=$1::uuid`, activeInternalID, now.Add(-time.Second), now.Add(-time.Minute)); err != nil { t.Fatal(err) } if _, err := pool.Exec(ctx, `UPDATE rtc_task SET status='ACTIVE', lease_expires_at=$2 WHERE conversation_id=$1::uuid`, activeInternalID, now.Add(-time.Second)); err != nil { t.Fatal(err) } if count, err := scanner.ScanOnce(ctx); err != nil || count != 1 { t.Fatalf("expired lease scan count=%d err=%v", count, err) } var stopEvents int if err := pool.QueryRow(ctx, `SELECT count(*) FROM outbox_event WHERE aggregate_id=$1::uuid AND event_type='conversation.agent_stop_requested'`, activeInternalID).Scan(&stopEvents); err != nil { t.Fatal(err) } if err := pool.QueryRow(ctx, `SELECT c.status, r.status FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id WHERE c.id=$1::uuid`, activeInternalID).Scan(&status, &taskStatus); err != nil { t.Fatal(err) } if status != "STOPPING" || taskStatus != "STOPPING" || stopEvents != 1 { t.Fatalf("lease recovery conversation=%s task=%s stopEvents=%d", status, taskStatus, stopEvents) } scanner.now = func() time.Time { return now.Add(11 * time.Second) } if count, err := scanner.ScanOnce(ctx); err != nil || count != 1 { t.Fatalf("stopping reconciliation count=%d err=%v", count, err) } var stopReconcileEvents int if err := pool.QueryRow(ctx, `SELECT count(*) FROM outbox_event WHERE aggregate_id=$1::uuid AND event_type='conversation.agent_stop_reconcile_requested'`, activeInternalID).Scan(&stopReconcileEvents); err != nil { t.Fatal(err) } if stopReconcileEvents != 1 { t.Fatalf("stop reconcile events=%d", stopReconcileEvents) } if _, err := pool.Exec(ctx, `UPDATE conversation SET status='COMPLETED', ended_at=$2 WHERE id=$1::uuid`, activeInternalID, now.Add(12*time.Second)); err != nil { t.Fatal(err) } if _, err := pool.Exec(ctx, `UPDATE rtc_task SET status='COMPLETED' WHERE conversation_id=$1::uuid`, activeInternalID); err != nil { t.Fatal(err) } startingPublicID, startingInternalID := createRecoveryConversation(t, ctx, pool, identity, now, now.Add(time.Minute)) publicIDs = append(publicIDs, startingPublicID) if _, err := pool.Exec(ctx, `UPDATE conversation SET status='STARTING_AGENT', updated_at=$2 WHERE id=$1::uuid`, startingInternalID, now.Add(-time.Minute)); err != nil { t.Fatal(err) } if _, err := pool.Exec(ctx, `UPDATE rtc_task SET status='STARTING' WHERE conversation_id=$1::uuid`, startingInternalID); err != nil { t.Fatal(err) } if count, err := scanner.ScanOnce(ctx); err != nil || count != 1 { t.Fatalf("starting reconciliation count=%d err=%v", count, err) } var startReconcileEvents, recoveryAudits int if err := pool.QueryRow(ctx, `SELECT count(*) FROM outbox_event WHERE aggregate_id=$1::uuid AND event_type='conversation.agent_start_reconcile_requested'`, startingInternalID).Scan(&startReconcileEvents); err != nil { t.Fatal(err) } if err := pool.QueryRow(ctx, `SELECT count(*) FROM audit_event WHERE action='conversation.recovery' AND resource_id IN ($1::uuid,$2::uuid,$3::uuid)`, roomInternalID, activeInternalID, startingInternalID).Scan(&recoveryAudits); err != nil { t.Fatal(err) } if startReconcileEvents != 1 || recoveryAudits != 4 { t.Fatalf("start reconcile=%d recovery audits=%d", startReconcileEvents, recoveryAudits) } var providerTaskRef string if err := pool.QueryRow(ctx, `SELECT provider_task_ref FROM conversation WHERE id=$1::uuid`, startingInternalID).Scan(&providerTaskRef); err != nil { t.Fatal(err) } processor, err := NewProviderEventProcessor(pool, time.Minute) if err != nil { t.Fatal(err) } processor.now = func() time.Time { return now.Add(12 * time.Second) } failedEvent := voiceprovider.CallbackEvent{ Provider: "mock", EventID: "recovery-test-provider-failed", TaskRef: providerTaskRef, ConversationID: startingPublicID, EventType: "voice_chat.failed", Status: "FAILED", OccurredAt: now.Add(12 * time.Second), } failedBody, _ := json.Marshal(failedEvent) failedResult, err := processor.Process(ctx, failedEvent, failedBody) if err != nil || failedResult.Status != "FAILED" || failedResult.Ignored { t.Fatalf("failed callback result=%#v err=%v", failedResult, err) } if err := pool.QueryRow(ctx, `SELECT c.status, r.status FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id WHERE c.id=$1::uuid`, startingInternalID).Scan(&status, &taskStatus); err != nil { t.Fatal(err) } if status != "FAILED" || taskStatus != "FAILED" { t.Fatalf("failed callback conversation=%s task=%s", status, taskStatus) } canceledContext, cancel := context.WithCancel(ctx) cancel() scanner.Run(canceledContext) } func createRecoveryConversation(t *testing.T, ctx context.Context, pool *pgxpool.Pool, identity deviceauth.Identity, now, allocationExpires time.Time) (string, string) { t.Helper() tx, err := pool.BeginTx(ctx, pgx.TxOptions{}) if err != nil { t.Fatal(err) } defer tx.Rollback(context.Background()) resume := true response, internalID, err := NewRepository().Create(ctx, tx, identity, CreateInput{ DeviceTime: now, Capabilities: Capabilities{RTC: true, AudioTracks: []string{"near"}, UploadResume: &resume}, Network: Network{Type: "test"}, RequestedGoal: "open_conversation", }, AllocationOptions{ Now: now, AllocationExpires: allocationExpires, TokenExpires: now.Add(time.Minute), RTCAppID: "test-app", EngineMode: "s2s_hybrid", HeartbeatInterval: 15, }) if err != nil { t.Fatal(err) } if err := tx.Commit(ctx); err != nil { t.Fatal(err) } return response.ConversationID, internalID } func cleanupRecoveryConversations(t *testing.T, ctx context.Context, pool *pgxpool.Pool, publicIDs []string) { t.Helper() for _, publicID := range publicIDs { var internalID, contextSnapshotID, consentSnapshotID string err := pool.QueryRow(ctx, `SELECT id::text, context_snapshot_id::text, consent_snapshot_id::text FROM conversation WHERE public_id=$1::uuid`, publicID).Scan(&internalID, &contextSnapshotID, &consentSnapshotID) if err != nil { continue } statements := []string{ `DELETE FROM provider_event WHERE conversation_id=$1::uuid`, `DELETE FROM conversation_state_history WHERE conversation_id=$1::uuid`, `DELETE FROM rtc_task WHERE conversation_id=$1::uuid`, `DELETE FROM outbox_event WHERE aggregate_id=$1::uuid`, `DELETE FROM audit_event WHERE resource_id=$1::uuid`, `DELETE FROM idempotency_record WHERE resource_id=$1::uuid`, `DELETE FROM conversation WHERE id=$1::uuid`, } for _, statement := range statements { if _, err := pool.Exec(ctx, statement, internalID); err != nil { t.Errorf("cleanup recovery conversation: %v", err) } } _, _ = pool.Exec(ctx, `DELETE FROM context_snapshot WHERE id=$1::uuid`, contextSnapshotID) _, _ = pool.Exec(ctx, `DELETE FROM consent_snapshot WHERE id=$1::uuid`, consentSnapshotID) } }