75 lines
3.0 KiB
Go
75 lines
3.0 KiB
Go
package orchestration
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
var ErrProviderUnavailable = errors.New("provider is temporarily unavailable")
|
|
|
|
type ProviderHealth struct {
|
|
pool *pgxpool.Pool
|
|
failureThreshold int
|
|
cooldown time.Duration
|
|
now func() time.Time
|
|
}
|
|
|
|
func NewProviderHealth(pool *pgxpool.Pool, failureThreshold int, cooldown time.Duration) (*ProviderHealth, error) {
|
|
if pool == nil || failureThreshold < 1 || cooldown <= 0 {
|
|
return nil, errors.New("provider health requires a database, positive failure threshold and cooldown")
|
|
}
|
|
return &ProviderHealth{pool: pool, failureThreshold: failureThreshold, cooldown: cooldown, now: time.Now}, nil
|
|
}
|
|
|
|
func (h *ProviderHealth) AllowStart(ctx context.Context, tenantID, provider string) (bool, error) {
|
|
now := h.now().UTC()
|
|
var status string
|
|
var openUntil *time.Time
|
|
err := h.pool.QueryRow(ctx, `
|
|
INSERT INTO provider_dependency_state(tenant_id,provider,status,updated_at)
|
|
VALUES($1::uuid,$2,'AVAILABLE',$3)
|
|
ON CONFLICT (tenant_id,provider) DO UPDATE SET provider=EXCLUDED.provider
|
|
RETURNING status,open_until`, tenantID, provider, now).Scan(&status, &openUntil)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return status != "UNAVAILABLE" || openUntil == nil || !now.Before(*openUntil), nil
|
|
}
|
|
|
|
func (h *ProviderHealth) RecordFailure(ctx context.Context, tenantID, provider, errorCode string) error {
|
|
now := h.now().UTC()
|
|
code := strings.ToUpper(strings.TrimSpace(errorCode))
|
|
if code == "" {
|
|
code = "PROVIDER_TEMPORARY_ERROR"
|
|
}
|
|
_, err := h.pool.Exec(ctx, `
|
|
INSERT INTO provider_dependency_state(
|
|
tenant_id,provider,status,consecutive_failures,open_until,last_error_code,last_failure_at,updated_at
|
|
) VALUES(
|
|
$1::uuid,$2,CASE WHEN $3<=1 THEN 'UNAVAILABLE' ELSE 'AVAILABLE' END,1,
|
|
CASE WHEN $3<=1 THEN $4::timestamptz ELSE NULL END,$5,$6,$6
|
|
)
|
|
ON CONFLICT (tenant_id,provider) DO UPDATE SET
|
|
consecutive_failures=provider_dependency_state.consecutive_failures+1,
|
|
status=CASE WHEN provider_dependency_state.consecutive_failures+1 >= $3 THEN 'UNAVAILABLE' ELSE provider_dependency_state.status END,
|
|
open_until=CASE WHEN provider_dependency_state.consecutive_failures+1 >= $3 THEN $4::timestamptz ELSE provider_dependency_state.open_until END,
|
|
last_error_code=$5,last_failure_at=$6,row_version=provider_dependency_state.row_version+1,updated_at=$6`,
|
|
tenantID, provider, h.failureThreshold, now.Add(h.cooldown), code, now)
|
|
return err
|
|
}
|
|
|
|
func (h *ProviderHealth) RecordSuccess(ctx context.Context, tenantID, provider string) error {
|
|
now := h.now().UTC()
|
|
_, err := h.pool.Exec(ctx, `
|
|
INSERT INTO provider_dependency_state(tenant_id,provider,status,last_success_at,updated_at)
|
|
VALUES($1::uuid,$2,'AVAILABLE',$3,$3)
|
|
ON CONFLICT (tenant_id,provider) DO UPDATE SET
|
|
status='AVAILABLE',consecutive_failures=0,open_until=NULL,last_error_code=NULL,
|
|
last_success_at=$3,row_version=provider_dependency_state.row_version+1,updated_at=$3`, tenantID, provider, now)
|
|
return err
|
|
}
|