package orchestration import ( "context" "errors" "strings" "time" "github.com/jackc/pgx/v5/pgxpool" ) var ErrProviderUnavailable = errors.New("provider is temporarily unavailable") type ProviderHealth struct { pool *pgxpool.Pool failureThreshold int cooldown time.Duration now func() time.Time } func NewProviderHealth(pool *pgxpool.Pool, failureThreshold int, cooldown time.Duration) (*ProviderHealth, error) { if pool == nil || failureThreshold < 1 || cooldown <= 0 { return nil, errors.New("provider health requires a database, positive failure threshold and cooldown") } return &ProviderHealth{pool: pool, failureThreshold: failureThreshold, cooldown: cooldown, now: time.Now}, nil } func (h *ProviderHealth) AllowStart(ctx context.Context, tenantID, provider string) (bool, error) { now := h.now().UTC() var status string var openUntil *time.Time err := h.pool.QueryRow(ctx, ` INSERT INTO provider_dependency_state(tenant_id,provider,status,updated_at) VALUES($1::uuid,$2,'AVAILABLE',$3) ON CONFLICT (tenant_id,provider) DO UPDATE SET provider=EXCLUDED.provider RETURNING status,open_until`, tenantID, provider, now).Scan(&status, &openUntil) if err != nil { return false, err } return status != "UNAVAILABLE" || openUntil == nil || !now.Before(*openUntil), nil } func (h *ProviderHealth) RecordFailure(ctx context.Context, tenantID, provider, errorCode string) error { now := h.now().UTC() code := strings.ToUpper(strings.TrimSpace(errorCode)) if code == "" { code = "PROVIDER_TEMPORARY_ERROR" } _, err := h.pool.Exec(ctx, ` INSERT INTO provider_dependency_state( tenant_id,provider,status,consecutive_failures,open_until,last_error_code,last_failure_at,updated_at ) VALUES( $1::uuid,$2,CASE WHEN $3<=1 THEN 'UNAVAILABLE' ELSE 'AVAILABLE' END,1, CASE WHEN $3<=1 THEN $4::timestamptz ELSE NULL END,$5,$6,$6 ) ON CONFLICT (tenant_id,provider) DO UPDATE SET consecutive_failures=provider_dependency_state.consecutive_failures+1, status=CASE WHEN provider_dependency_state.consecutive_failures+1 >= $3 THEN 'UNAVAILABLE' ELSE provider_dependency_state.status END, open_until=CASE WHEN provider_dependency_state.consecutive_failures+1 >= $3 THEN $4::timestamptz ELSE provider_dependency_state.open_until END, last_error_code=$5,last_failure_at=$6,row_version=provider_dependency_state.row_version+1,updated_at=$6`, tenantID, provider, h.failureThreshold, now.Add(h.cooldown), code, now) return err } func (h *ProviderHealth) RecordSuccess(ctx context.Context, tenantID, provider string) error { now := h.now().UTC() _, err := h.pool.Exec(ctx, ` INSERT INTO provider_dependency_state(tenant_id,provider,status,last_success_at,updated_at) VALUES($1::uuid,$2,'AVAILABLE',$3,$3) ON CONFLICT (tenant_id,provider) DO UPDATE SET status='AVAILABLE',consecutive_failures=0,open_until=NULL,last_error_code=NULL, last_success_at=$3,row_version=provider_dependency_state.row_version+1,updated_at=$3`, tenantID, provider, now) return err }