201 lines
9.0 KiB
Go
201 lines
9.0 KiB
Go
package orchestration
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"log/slog"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"data-terminal-cloud/internal/database"
|
|
"data-terminal-cloud/internal/deviceauth"
|
|
"data-terminal-cloud/internal/outbox"
|
|
"data-terminal-cloud/internal/voiceprovider"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
func TestRecoveryScannerExpiresAndSchedulesIdempotentReconciliation(t *testing.T) {
|
|
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
|
if databaseURL == "" {
|
|
t.Skip("TEST_DATABASE_URL is not set")
|
|
}
|
|
ctx := context.Background()
|
|
pool, err := database.Open(ctx, databaseURL, "orchestration-recovery-test")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer pool.Close()
|
|
identity := deviceauth.Identity{
|
|
ID: "10000000-0000-0000-0000-000000000501", PublicID: "10000000-0000-0000-0000-000000000511",
|
|
TenantID: "10000000-0000-0000-0000-000000000001", Status: "registered",
|
|
}
|
|
now := time.Date(2026, 8, 23, 13, 0, 0, 0, time.UTC)
|
|
publicIDs := make([]string, 0, 3)
|
|
defer func() { cleanupRecoveryConversations(t, context.Background(), pool, publicIDs) }()
|
|
|
|
roomIssued, roomInternalID := createRecoveryConversation(t, ctx, pool, identity, now.Add(-time.Minute), now.Add(-30*time.Second))
|
|
publicIDs = append(publicIDs, roomIssued)
|
|
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
|
scanner, err := NewRecoveryScanner(pool, outbox.NewStore(pool), logger, RecoveryOptions{
|
|
ScanInterval: time.Second, StartingAfter: 10 * time.Second, StoppingAfter: 10 * time.Second, BatchSize: 10,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
scanner.now = func() time.Time { return now }
|
|
if count, err := scanner.ScanOnce(ctx); err != nil || count != 1 {
|
|
t.Fatalf("expired room scan count=%d err=%v", count, err)
|
|
}
|
|
var status, taskStatus string
|
|
if err := pool.QueryRow(ctx, `SELECT c.status, r.status FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id WHERE c.id=$1::uuid`, roomInternalID).Scan(&status, &taskStatus); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if status != "EXPIRED" || taskStatus != "COMPLETED" {
|
|
t.Fatalf("expired room conversation=%s task=%s", status, taskStatus)
|
|
}
|
|
|
|
activePublicID, activeInternalID := createRecoveryConversation(t, ctx, pool, identity, now, now.Add(time.Minute))
|
|
publicIDs = append(publicIDs, activePublicID)
|
|
if _, err := pool.Exec(ctx, `UPDATE conversation SET status='ACTIVE', lease_expires_at=$2, updated_at=$3 WHERE id=$1::uuid`, activeInternalID, now.Add(-time.Second), now.Add(-time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := pool.Exec(ctx, `UPDATE rtc_task SET status='ACTIVE', lease_expires_at=$2 WHERE conversation_id=$1::uuid`, activeInternalID, now.Add(-time.Second)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if count, err := scanner.ScanOnce(ctx); err != nil || count != 1 {
|
|
t.Fatalf("expired lease scan count=%d err=%v", count, err)
|
|
}
|
|
var stopEvents int
|
|
if err := pool.QueryRow(ctx, `SELECT count(*) FROM outbox_event WHERE aggregate_id=$1::uuid AND event_type='conversation.agent_stop_requested'`, activeInternalID).Scan(&stopEvents); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := pool.QueryRow(ctx, `SELECT c.status, r.status FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id WHERE c.id=$1::uuid`, activeInternalID).Scan(&status, &taskStatus); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if status != "STOPPING" || taskStatus != "STOPPING" || stopEvents != 1 {
|
|
t.Fatalf("lease recovery conversation=%s task=%s stopEvents=%d", status, taskStatus, stopEvents)
|
|
}
|
|
scanner.now = func() time.Time { return now.Add(11 * time.Second) }
|
|
if count, err := scanner.ScanOnce(ctx); err != nil || count != 1 {
|
|
t.Fatalf("stopping reconciliation count=%d err=%v", count, err)
|
|
}
|
|
var stopReconcileEvents int
|
|
if err := pool.QueryRow(ctx, `SELECT count(*) FROM outbox_event WHERE aggregate_id=$1::uuid AND event_type='conversation.agent_stop_reconcile_requested'`, activeInternalID).Scan(&stopReconcileEvents); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if stopReconcileEvents != 1 {
|
|
t.Fatalf("stop reconcile events=%d", stopReconcileEvents)
|
|
}
|
|
|
|
if _, err := pool.Exec(ctx, `UPDATE conversation SET status='COMPLETED', ended_at=$2 WHERE id=$1::uuid`, activeInternalID, now.Add(12*time.Second)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := pool.Exec(ctx, `UPDATE rtc_task SET status='COMPLETED' WHERE conversation_id=$1::uuid`, activeInternalID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
startingPublicID, startingInternalID := createRecoveryConversation(t, ctx, pool, identity, now, now.Add(time.Minute))
|
|
publicIDs = append(publicIDs, startingPublicID)
|
|
if _, err := pool.Exec(ctx, `UPDATE conversation SET status='STARTING_AGENT', updated_at=$2 WHERE id=$1::uuid`, startingInternalID, now.Add(-time.Minute)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := pool.Exec(ctx, `UPDATE rtc_task SET status='STARTING' WHERE conversation_id=$1::uuid`, startingInternalID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if count, err := scanner.ScanOnce(ctx); err != nil || count != 1 {
|
|
t.Fatalf("starting reconciliation count=%d err=%v", count, err)
|
|
}
|
|
var startReconcileEvents, recoveryAudits int
|
|
if err := pool.QueryRow(ctx, `SELECT count(*) FROM outbox_event WHERE aggregate_id=$1::uuid AND event_type='conversation.agent_start_reconcile_requested'`, startingInternalID).Scan(&startReconcileEvents); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := pool.QueryRow(ctx, `SELECT count(*) FROM audit_event WHERE action='conversation.recovery' AND resource_id IN ($1::uuid,$2::uuid,$3::uuid)`, roomInternalID, activeInternalID, startingInternalID).Scan(&recoveryAudits); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if startReconcileEvents != 1 || recoveryAudits != 4 {
|
|
t.Fatalf("start reconcile=%d recovery audits=%d", startReconcileEvents, recoveryAudits)
|
|
}
|
|
var providerTaskRef string
|
|
if err := pool.QueryRow(ctx, `SELECT provider_task_ref FROM conversation WHERE id=$1::uuid`, startingInternalID).Scan(&providerTaskRef); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
processor, err := NewProviderEventProcessor(pool, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
processor.now = func() time.Time { return now.Add(12 * time.Second) }
|
|
failedEvent := voiceprovider.CallbackEvent{
|
|
Provider: "mock", EventID: "recovery-test-provider-failed", TaskRef: providerTaskRef,
|
|
ConversationID: startingPublicID, EventType: "voice_chat.failed", Status: "FAILED",
|
|
OccurredAt: now.Add(12 * time.Second),
|
|
}
|
|
failedBody, _ := json.Marshal(failedEvent)
|
|
failedResult, err := processor.Process(ctx, failedEvent, failedBody)
|
|
if err != nil || failedResult.Status != "FAILED" || failedResult.Ignored {
|
|
t.Fatalf("failed callback result=%#v err=%v", failedResult, err)
|
|
}
|
|
if err := pool.QueryRow(ctx, `SELECT c.status, r.status FROM conversation c JOIN rtc_task r ON r.conversation_id=c.id WHERE c.id=$1::uuid`, startingInternalID).Scan(&status, &taskStatus); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if status != "FAILED" || taskStatus != "FAILED" {
|
|
t.Fatalf("failed callback conversation=%s task=%s", status, taskStatus)
|
|
}
|
|
canceledContext, cancel := context.WithCancel(ctx)
|
|
cancel()
|
|
scanner.Run(canceledContext)
|
|
}
|
|
|
|
func createRecoveryConversation(t *testing.T, ctx context.Context, pool *pgxpool.Pool, identity deviceauth.Identity, now, allocationExpires time.Time) (string, string) {
|
|
t.Helper()
|
|
tx, err := pool.BeginTx(ctx, pgx.TxOptions{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer tx.Rollback(context.Background())
|
|
resume := true
|
|
response, internalID, err := NewRepository().Create(ctx, tx, identity, CreateInput{
|
|
DeviceTime: now, Capabilities: Capabilities{RTC: true, AudioTracks: []string{"near"}, UploadResume: &resume},
|
|
Network: Network{Type: "test"}, RequestedGoal: "open_conversation",
|
|
}, AllocationOptions{
|
|
Now: now, AllocationExpires: allocationExpires, TokenExpires: now.Add(time.Minute),
|
|
RTCAppID: "test-app", EngineMode: "s2s_hybrid", HeartbeatInterval: 15,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return response.ConversationID, internalID
|
|
}
|
|
|
|
func cleanupRecoveryConversations(t *testing.T, ctx context.Context, pool *pgxpool.Pool, publicIDs []string) {
|
|
t.Helper()
|
|
for _, publicID := range publicIDs {
|
|
var internalID, contextSnapshotID, consentSnapshotID string
|
|
err := pool.QueryRow(ctx, `SELECT id::text, context_snapshot_id::text, consent_snapshot_id::text FROM conversation WHERE public_id=$1::uuid`, publicID).Scan(&internalID, &contextSnapshotID, &consentSnapshotID)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
statements := []string{
|
|
`DELETE FROM provider_event WHERE conversation_id=$1::uuid`,
|
|
`DELETE FROM conversation_state_history WHERE conversation_id=$1::uuid`,
|
|
`DELETE FROM rtc_task WHERE conversation_id=$1::uuid`,
|
|
`DELETE FROM outbox_event WHERE aggregate_id=$1::uuid`,
|
|
`DELETE FROM audit_event WHERE resource_id=$1::uuid`,
|
|
`DELETE FROM idempotency_record WHERE resource_id=$1::uuid`,
|
|
`DELETE FROM conversation WHERE id=$1::uuid`,
|
|
}
|
|
for _, statement := range statements {
|
|
if _, err := pool.Exec(ctx, statement, internalID); err != nil {
|
|
t.Errorf("cleanup recovery conversation: %v", err)
|
|
}
|
|
}
|
|
_, _ = pool.Exec(ctx, `DELETE FROM context_snapshot WHERE id=$1::uuid`, contextSnapshotID)
|
|
_, _ = pool.Exec(ctx, `DELETE FROM consent_snapshot WHERE id=$1::uuid`, consentSnapshotID)
|
|
}
|
|
}
|