61 lines
2.6 KiB
Go
61 lines
2.6 KiB
Go
|
package externalprobe
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"os"
|
||
|
|
"testing"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
||
|
|
)
|
||
|
|
|
||
|
|
func TestPostgresObserverTracksFailureRecoveryAndReconciliation(t *testing.T) {
|
||
|
|
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
||
|
|
if databaseURL == "" {
|
||
|
|
t.Skip("TEST_DATABASE_URL is not set")
|
||
|
|
}
|
||
|
|
ctx := context.Background()
|
||
|
|
pool, err := pgxpool.New(ctx, databaseURL)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
defer pool.Close()
|
||
|
|
serviceName, dependency := "probe-test", "oidc_jwks"
|
||
|
|
defer pool.Exec(ctx, `DELETE FROM external_dependency_probe_state WHERE service_name=$1`, serviceName)
|
||
|
|
observer, err := PostgresObserver(pool, serviceName)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
checkedAt := time.Now().UTC().Truncate(time.Millisecond)
|
||
|
|
for index := 0; index < 2; index++ {
|
||
|
|
if err = observer(ctx, Observation{Dependency: dependency, CheckedAt: checkedAt.Add(time.Duration(index) * time.Second), Latency: 15 * time.Millisecond, Interval: 30 * time.Second, ErrorCode: "TLS_ERROR"}); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
var status, errorCode string
|
||
|
|
var failures int
|
||
|
|
if err = pool.QueryRow(ctx, `SELECT status,last_error_code,consecutive_failures FROM external_dependency_probe_state WHERE service_name=$1 AND dependency=$2`, serviceName, dependency).Scan(&status, &errorCode, &failures); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if status != "UNAVAILABLE" || errorCode != "TLS_ERROR" || failures != 2 {
|
||
|
|
t.Fatalf("failure state=%s code=%s consecutive=%d", status, errorCode, failures)
|
||
|
|
}
|
||
|
|
if err = observer(ctx, Observation{Dependency: dependency, Available: true, CheckedAt: checkedAt.Add(2 * time.Second), Latency: 4 * time.Millisecond, Interval: 30 * time.Second}); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
var errorCleared, hasSuccess, hasFailure bool
|
||
|
|
if err = pool.QueryRow(ctx, `SELECT status,last_error_code IS NULL,consecutive_failures,last_success_at IS NOT NULL,last_failure_at IS NOT NULL FROM external_dependency_probe_state WHERE service_name=$1 AND dependency=$2`, serviceName, dependency).Scan(&status, &errorCleared, &failures, &hasSuccess, &hasFailure); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if status != "AVAILABLE" || !errorCleared || !hasSuccess || !hasFailure || failures != 0 {
|
||
|
|
t.Fatalf("recovered state=%s error_cleared=%v success=%v failure=%v consecutive=%d", status, errorCleared, hasSuccess, hasFailure, failures)
|
||
|
|
}
|
||
|
|
if err = ReconcilePostgresState(ctx, pool, serviceName, nil); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
var remaining int
|
||
|
|
if err = pool.QueryRow(ctx, `SELECT count(*) FROM external_dependency_probe_state WHERE service_name=$1`, serviceName).Scan(&remaining); err != nil || remaining != 0 {
|
||
|
|
t.Fatalf("remaining=%d err=%v", remaining, err)
|
||
|
|
}
|
||
|
|
}
|