package orchestration import ( "context" "os" "testing" "time" "data-terminal-cloud/internal/database" ) func TestProviderHealthPersistsCircuitAndRecovers(t *testing.T) { databaseURL := os.Getenv("TEST_DATABASE_URL") if databaseURL == "" { t.Skip("TEST_DATABASE_URL is not set") } ctx := context.Background() pool, err := database.Open(ctx, databaseURL, "provider-health-test") if err != nil { t.Fatal(err) } defer pool.Close() tenantID := "10000000-0000-0000-0000-000000000001" provider := "provider-health-test" defer pool.Exec(context.Background(), `DELETE FROM provider_dependency_state WHERE tenant_id=$1::uuid AND provider=$2`, tenantID, provider) health, err := NewProviderHealth(pool, 3, time.Minute) if err != nil { t.Fatal(err) } now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) health.now = func() time.Time { return now } for attempt := 1; attempt <= 2; attempt++ { if err := health.RecordFailure(ctx, tenantID, provider, "PROVIDER_START_TEMPORARY"); err != nil { t.Fatal(err) } allowed, err := health.AllowStart(ctx, tenantID, provider) if err != nil || !allowed { t.Fatalf("attempt %d allowed=%v err=%v", attempt, allowed, err) } } if err := health.RecordFailure(ctx, tenantID, provider, "PROVIDER_START_TEMPORARY"); err != nil { t.Fatal(err) } allowed, err := health.AllowStart(ctx, tenantID, provider) if err != nil || allowed { t.Fatalf("open circuit allowed=%v err=%v", allowed, err) } var status, errorCode string var failures int if err := pool.QueryRow(ctx, `SELECT status,consecutive_failures,last_error_code FROM provider_dependency_state WHERE tenant_id=$1::uuid AND provider=$2`, tenantID, provider).Scan(&status, &failures, &errorCode); err != nil { t.Fatal(err) } if status != "UNAVAILABLE" || failures != 3 || errorCode != "PROVIDER_START_TEMPORARY" { t.Fatalf("state=%s failures=%d error=%s", status, failures, errorCode) } now = now.Add(61 * time.Second) allowed, err = health.AllowStart(ctx, tenantID, provider) if err != nil || !allowed { t.Fatalf("cooldown probe allowed=%v err=%v", allowed, err) } if err := health.RecordSuccess(ctx, tenantID, provider); err != nil { t.Fatal(err) } allowed, err = health.AllowStart(ctx, tenantID, provider) if err != nil || !allowed { t.Fatalf("recovered allowed=%v err=%v", allowed, err) } if err := pool.QueryRow(ctx, `SELECT status,consecutive_failures FROM provider_dependency_state WHERE tenant_id=$1::uuid AND provider=$2`, tenantID, provider).Scan(&status, &failures); err != nil { t.Fatal(err) } if status != "AVAILABLE" || failures != 0 { t.Fatalf("recovered state=%s failures=%d", status, failures) } }