76 lines
2.6 KiB
Go
76 lines
2.6 KiB
Go
package orchestration
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"data-terminal-cloud/internal/database"
|
|
)
|
|
|
|
func TestProviderHealthPersistsCircuitAndRecovers(t *testing.T) {
|
|
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
|
if databaseURL == "" {
|
|
t.Skip("TEST_DATABASE_URL is not set")
|
|
}
|
|
ctx := context.Background()
|
|
pool, err := database.Open(ctx, databaseURL, "provider-health-test")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer pool.Close()
|
|
tenantID := "10000000-0000-0000-0000-000000000001"
|
|
provider := "provider-health-test"
|
|
defer pool.Exec(context.Background(), `DELETE FROM provider_dependency_state WHERE tenant_id=$1::uuid AND provider=$2`, tenantID, provider)
|
|
|
|
health, err := NewProviderHealth(pool, 3, time.Minute)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC)
|
|
health.now = func() time.Time { return now }
|
|
for attempt := 1; attempt <= 2; attempt++ {
|
|
if err := health.RecordFailure(ctx, tenantID, provider, "PROVIDER_START_TEMPORARY"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
allowed, err := health.AllowStart(ctx, tenantID, provider)
|
|
if err != nil || !allowed {
|
|
t.Fatalf("attempt %d allowed=%v err=%v", attempt, allowed, err)
|
|
}
|
|
}
|
|
if err := health.RecordFailure(ctx, tenantID, provider, "PROVIDER_START_TEMPORARY"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
allowed, err := health.AllowStart(ctx, tenantID, provider)
|
|
if err != nil || allowed {
|
|
t.Fatalf("open circuit allowed=%v err=%v", allowed, err)
|
|
}
|
|
var status, errorCode string
|
|
var failures int
|
|
if err := pool.QueryRow(ctx, `SELECT status,consecutive_failures,last_error_code FROM provider_dependency_state WHERE tenant_id=$1::uuid AND provider=$2`, tenantID, provider).Scan(&status, &failures, &errorCode); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if status != "UNAVAILABLE" || failures != 3 || errorCode != "PROVIDER_START_TEMPORARY" {
|
|
t.Fatalf("state=%s failures=%d error=%s", status, failures, errorCode)
|
|
}
|
|
now = now.Add(61 * time.Second)
|
|
allowed, err = health.AllowStart(ctx, tenantID, provider)
|
|
if err != nil || !allowed {
|
|
t.Fatalf("cooldown probe allowed=%v err=%v", allowed, err)
|
|
}
|
|
if err := health.RecordSuccess(ctx, tenantID, provider); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
allowed, err = health.AllowStart(ctx, tenantID, provider)
|
|
if err != nil || !allowed {
|
|
t.Fatalf("recovered allowed=%v err=%v", allowed, err)
|
|
}
|
|
if err := pool.QueryRow(ctx, `SELECT status,consecutive_failures FROM provider_dependency_state WHERE tenant_id=$1::uuid AND provider=$2`, tenantID, provider).Scan(&status, &failures); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if status != "AVAILABLE" || failures != 0 {
|
|
t.Fatalf("recovered state=%s failures=%d", status, failures)
|
|
}
|
|
}
|