chore: establish local platform baseline

2026-08-26 06:41:54 +08:00
commit 29bdf4900f
575 changed files with 95644 additions and 0 deletions
@@ -0,0 +1,63 @@
+package outbox
+
+import (
+	"context"
+	"errors"
+	"net"
+
+	"github.com/jackc/pgx/v5/pgconn"
+)
+
+// retryableError is the explicit boundary between a temporary dependency
+// failure and a permanent job failure. Handlers must opt in by wrapping an
+// error with Retryable unless the infrastructure error is independently
+// recognizable as a timeout, network interruption or transient PostgreSQL
+// condition.
+type retryableError struct{ cause error }
+
+func (e retryableError) Error() string { return e.cause.Error() }
+func (e retryableError) Unwrap() error { return e.cause }
+
+// Retryable marks an error as safe to execute again with the same idempotency
+// boundary. A nil error remains nil so callers can wrap return values directly.
+func Retryable(err error) error {
+	if err == nil || IsRetryable(err) {
+		return err
+	}
+	return retryableError{cause: err}
+}
+
+// IsRetryable deliberately keeps the automatic allow-list narrow. Validation,
+// unsupported state and malformed payload errors are permanent by default.
+func IsRetryable(err error) bool {
+	if err == nil {
+		return false
+	}
+	var marked retryableError
+	if errors.As(err, &marked) {
+		return true
+	}
+	if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+		return true
+	}
+	var networkError net.Error
+	if errors.As(err, &networkError) && (networkError.Timeout() || networkError.Temporary()) {
+		return true
+	}
+	if pgconn.SafeToRetry(err) {
+		return true
+	}
+	var postgresError *pgconn.PgError
+	if !errors.As(err, &postgresError) {
+		return false
+	}
+	// Connection, transaction rollback, resource exhaustion, lock-not-available
+	// and administrator restart conditions can succeed on a later attempt.
+	class := ""
+	if len(postgresError.Code) >= 2 {
+		class = postgresError.Code[:2]
+	}
+	return class == "08" || class == "40" || class == "53" ||
+		postgresError.Code == "55P03" || postgresError.Code == "57P01" ||
+		postgresError.Code == "57P02" || postgresError.Code == "57P03"
+}
@@ -0,0 +1,32 @@
+package outbox
+
+import (
+	"context"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func RegisterMetrics(pool *pgxpool.Pool) {
+	for _, status := range []string{"pending", "processing", "dead"} {
+		status := status
+		prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+			Namespace: "voice_platform",
+			Subsystem: "outbox",
+			Name:      "events",
+			Help:      "Current outbox event count by status.",
+			ConstLabels: prometheus.Labels{
+				"status": status,
+			},
+		}, func() float64 {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+			defer cancel()
+			var count float64
+			if err := pool.QueryRow(ctx, "SELECT count(*)::float8 FROM outbox_event WHERE status = $1", status).Scan(&count); err != nil {
+				return 0
+			}
+			return count
+		}))
+	}
+}
@@ -0,0 +1,174 @@
+package outbox
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+var ErrDeduplicationConflict = errors.New("outbox deduplication key was reused with different payload")
+
+type Event struct {
+	ID               string
+	TenantID         *string
+	AggregateType    string
+	AggregateID      string
+	EventType        string
+	DeduplicationKey *string
+	Payload          json.RawMessage
+	AttemptCount     int
+}
+
+type NewEvent struct {
+	TenantID         *string
+	AggregateType    string
+	AggregateID      string
+	EventType        string
+	DeduplicationKey *string
+	Payload          json.RawMessage
+}
+
+type Store struct{ pool *pgxpool.Pool }
+
+func NewStore(pool *pgxpool.Pool) *Store { return &Store{pool: pool} }
+
+func (s *Store) Enqueue(ctx context.Context, tx pgx.Tx, event NewEvent) (string, error) {
+	if event.AggregateType == "" || event.AggregateID == "" || event.EventType == "" || !json.Valid(event.Payload) {
+		return "", errors.New("aggregate, event type and valid JSON payload are required")
+	}
+	payloadHash := hashPayload(event.Payload)
+	var eventID string
+	err := tx.QueryRow(ctx, `
+		INSERT INTO outbox_event (
+			tenant_id, aggregate_type, aggregate_id, event_type,
+			deduplication_key, payload, payload_hash
+		) VALUES ($1::uuid, $2, $3::uuid, $4, $5, $6::jsonb, $7)
+		ON CONFLICT DO NOTHING
+		RETURNING id::text`,
+		event.TenantID, event.AggregateType, event.AggregateID, event.EventType,
+		event.DeduplicationKey, event.Payload, payloadHash,
+	).Scan(&eventID)
+	if err == nil {
+		return eventID, nil
+	}
+	if !errors.Is(err, pgx.ErrNoRows) || event.DeduplicationKey == nil {
+		return "", fmt.Errorf("enqueue outbox event: %w", err)
+	}
+	var storedHash string
+	err = tx.QueryRow(ctx, `
+		SELECT id::text, payload_hash
+		FROM outbox_event
+		WHERE COALESCE(tenant_id, '00000000-0000-0000-0000-000000000000'::uuid) =
+			COALESCE($1::uuid, '00000000-0000-0000-0000-000000000000'::uuid)
+			AND event_type = $2 AND deduplication_key = $3
+		FOR UPDATE`, event.TenantID, event.EventType, event.DeduplicationKey,
+	).Scan(&eventID, &storedHash)
+	if err != nil {
+		return "", fmt.Errorf("read deduplicated outbox event: %w", err)
+	}
+	if storedHash != payloadHash {
+		return "", ErrDeduplicationConflict
+	}
+	return eventID, nil
+}
+
+func (s *Store) ClaimBatch(ctx context.Context, workerID string, batchSize int, lockTimeout time.Duration) ([]Event, error) {
+	rows, err := s.pool.Query(ctx, `
+		WITH candidates AS (
+			SELECT id
+			FROM outbox_event
+			WHERE (status = 'pending' AND available_at <= now())
+			   OR (status = 'processing' AND locked_at < now() - $3::interval)
+			ORDER BY available_at, created_at
+			FOR UPDATE SKIP LOCKED
+			LIMIT $2
+		)
+		UPDATE outbox_event event
+		SET status = 'processing', locked_at = now(), locked_by = $1,
+			attempt_count = event.attempt_count + 1, updated_at = now()
+		FROM candidates
+		WHERE event.id = candidates.id
+		RETURNING event.id::text, event.tenant_id::text, event.aggregate_type,
+			event.aggregate_id::text, event.event_type, event.deduplication_key,
+			event.payload, event.attempt_count`,
+		workerID, batchSize, postgresInterval(lockTimeout),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("claim outbox events: %w", err)
+	}
+	defer rows.Close()
+	events := make([]Event, 0, batchSize)
+	for rows.Next() {
+		var event Event
+		if err := rows.Scan(
+			&event.ID, &event.TenantID, &event.AggregateType, &event.AggregateID,
+			&event.EventType, &event.DeduplicationKey, &event.Payload, &event.AttemptCount,
+		); err != nil {
+			return nil, fmt.Errorf("scan outbox event: %w", err)
+		}
+		events = append(events, event)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate outbox events: %w", err)
+	}
+	return events, nil
+}
+
+func (s *Store) MarkCompleted(ctx context.Context, eventID, workerID string) error {
+	tag, err := s.pool.Exec(ctx, `
+		UPDATE outbox_event
+		SET status = 'completed', completed_at = now(), locked_at = NULL,
+			locked_by = NULL, last_error_code = NULL, updated_at = now()
+		WHERE id = $1::uuid AND status = 'processing' AND locked_by = $2`, eventID, workerID)
+	if err != nil {
+		return fmt.Errorf("complete outbox event: %w", err)
+	}
+	if tag.RowsAffected() != 1 {
+		return errors.New("outbox event lock was lost before completion")
+	}
+	return nil
+}
+
+func (s *Store) MarkFailed(
+	ctx context.Context,
+	event Event,
+	workerID, errorCode string,
+	maxAttempts int,
+	nextAttempt time.Time,
+) error {
+	status := "pending"
+	if event.AttemptCount >= maxAttempts {
+		status = "dead"
+	}
+	tag, err := s.pool.Exec(ctx, `
+		UPDATE outbox_event
+		SET status = $3::varchar, available_at = $4, locked_at = NULL, locked_by = NULL,
+			last_error_code = $5, dead_at = CASE WHEN $3::varchar = 'dead' THEN now() ELSE NULL END,
+			updated_at = now()
+		WHERE id = $1::uuid AND status = 'processing' AND locked_by = $2`,
+		event.ID, workerID, status, nextAttempt, errorCode,
+	)
+	if err != nil {
+		return fmt.Errorf("fail outbox event: %w", err)
+	}
+	if tag.RowsAffected() != 1 {
+		return errors.New("outbox event lock was lost before failure update")
+	}
+	return nil
+}
+
+func hashPayload(payload []byte) string {
+	digest := sha256.Sum256(payload)
+	return hex.EncodeToString(digest[:])
+}
+
+func postgresInterval(duration time.Duration) string {
+	return fmt.Sprintf("%f seconds", duration.Seconds())
+}
@@ -0,0 +1,144 @@
+package outbox
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"log/slog"
+	"math"
+	"time"
+)
+
+type Handler func(context.Context, Event) error
+
+type WorkerOptions struct {
+	PollInterval time.Duration
+	LockTimeout  time.Duration
+	BaseRetry    time.Duration
+	MaxRetry     time.Duration
+	BatchSize    int
+	MaxAttempts  int
+}
+
+type Worker struct {
+	store    *Store
+	logger   *slog.Logger
+	workerID string
+	options  WorkerOptions
+	handlers map[string]Handler
+}
+
+func NewWorker(store *Store, logger *slog.Logger, options WorkerOptions) *Worker {
+	if options.PollInterval <= 0 {
+		options.PollInterval = 2 * time.Second
+	}
+	if options.LockTimeout <= 0 {
+		options.LockTimeout = time.Minute
+	}
+	if options.BaseRetry <= 0 {
+		options.BaseRetry = time.Second
+	}
+	if options.MaxRetry <= 0 {
+		options.MaxRetry = 15 * time.Minute
+	}
+	if options.BatchSize <= 0 {
+		options.BatchSize = 20
+	}
+	if options.MaxAttempts <= 0 {
+		options.MaxAttempts = 8
+	}
+	return &Worker{
+		store: store, logger: logger, workerID: newWorkerID(), options: options,
+		handlers: make(map[string]Handler),
+	}
+}
+
+func (w *Worker) Register(eventType string, handler Handler) { w.handlers[eventType] = handler }
+
+func (w *Worker) Run(ctx context.Context) {
+	w.logger.Info("outbox worker started", "worker_id", w.workerID)
+	ticker := time.NewTicker(w.options.PollInterval)
+	defer ticker.Stop()
+	for {
+		if _, err := w.ProcessOnce(ctx); err != nil && !errors.Is(err, context.Canceled) {
+			w.logger.Error("outbox batch failed", "worker_id", w.workerID, "error", err)
+		}
+		select {
+		case <-ctx.Done():
+			w.logger.Info("outbox worker stopped", "worker_id", w.workerID)
+			return
+		case <-ticker.C:
+		}
+	}
+}
+
+func (w *Worker) ProcessOnce(ctx context.Context) (int, error) {
+	events, err := w.store.ClaimBatch(ctx, w.workerID, w.options.BatchSize, w.options.LockTimeout)
+	if err != nil {
+		return 0, err
+	}
+	for _, event := range events {
+		handler, ok := w.handlers[event.EventType]
+		if !ok {
+			if err := w.store.MarkFailed(
+				ctx, event, w.workerID, "HANDLER_NOT_REGISTERED", event.AttemptCount, time.Now(),
+			); err != nil {
+				return 0, err
+			}
+			continue
+		}
+		if err := handler(ctx, event); err != nil {
+			retryable := IsRetryable(err)
+			maxAttempts := event.AttemptCount
+			nextAttempt := time.Now()
+			errorCode := "HANDLER_PERMANENT_FAILURE"
+			if retryable {
+				maxAttempts = w.options.MaxAttempts
+				nextAttempt = w.nextAttempt(event.AttemptCount)
+				errorCode = "HANDLER_TEMPORARY_FAILURE"
+			}
+			w.logger.Warn("outbox handler failed", "event_id", event.ID, "event_type", event.EventType, "attempt", event.AttemptCount, "retryable", retryable)
+			if markErr := w.store.MarkFailed(
+				ctx, event, w.workerID, errorCode, maxAttempts, nextAttempt,
+			); markErr != nil {
+				return 0, markErr
+			}
+			continue
+		}
+		if err := w.store.MarkCompleted(ctx, event.ID, w.workerID); err != nil {
+			return 0, err
+		}
+	}
+	return len(events), nil
+}
+
+func (w *Worker) nextAttempt(attempt int) time.Time {
+	multiplier := math.Pow(2, float64(max(attempt-1, 0)))
+	delay := time.Duration(float64(w.options.BaseRetry) * multiplier)
+	if delay > w.options.MaxRetry {
+		delay = w.options.MaxRetry
+	}
+	// Jitter each worker independently to prevent a recovered dependency from
+	// receiving a synchronized retry wave. Keep the randomized delay at or
+	// below the configured maximum.
+	delay = time.Duration(float64(delay) * (0.5 + 0.5*randomFraction()))
+	return time.Now().Add(delay)
+}
+
+func randomFraction() float64 {
+	var value [8]byte
+	if _, err := rand.Read(value[:]); err != nil {
+		return 0.5
+	}
+	return float64(binary.BigEndian.Uint64(value[:])>>11) / (1 << 53)
+}
+
+func newWorkerID() string {
+	value := make([]byte, 8)
+	if _, err := rand.Read(value); err != nil {
+		return "worker-fallback"
+	}
+	return "worker-" + hex.EncodeToString(value)
+}
@@ -0,0 +1,154 @@
+package outbox
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+func TestWorkerProcessesAndDeduplicatesEvents(t *testing.T) {
+	pool := outboxIntegrationPool(t)
+	store := NewStore(pool)
+	worker := NewWorker(store, discardLogger(), WorkerOptions{MaxAttempts: 1, BaseRetry: time.Millisecond})
+	processed := 0
+	worker.Register("integration.noop", func(context.Context, Event) error {
+		processed++
+		return nil
+	})
+
+	deduplicationKey := fmt.Sprintf("integration-%d", time.Now().UnixNano())
+	tenantID := "10000000-0000-0000-0000-000000000001"
+	event := NewEvent{
+		TenantID: &tenantID, AggregateType: "household",
+		AggregateID: "10000000-0000-0000-0000-000000000201",
+		EventType:   "integration.noop", DeduplicationKey: &deduplicationKey,
+		Payload: json.RawMessage(`{"ok":true}`),
+	}
+	firstID := enqueueIntegrationEvent(t, pool, store, event)
+	t.Cleanup(func() { _, _ = pool.Exec(context.Background(), `DELETE FROM outbox_event WHERE id=$1::uuid`, firstID) })
+	secondID := enqueueIntegrationEvent(t, pool, store, event)
+	if firstID != secondID {
+		t.Fatalf("deduplicated event ids differ: %s %s", firstID, secondID)
+	}
+	event.Payload = json.RawMessage(`{"ok":false}`)
+	tx, err := pool.Begin(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := store.Enqueue(context.Background(), tx, event); !errors.Is(err, ErrDeduplicationConflict) {
+		_ = tx.Rollback(context.Background())
+		t.Fatalf("payload conflict = %v, want ErrDeduplicationConflict", err)
+	}
+	_ = tx.Rollback(context.Background())
+
+	count, err := worker.ProcessOnce(context.Background())
+	if err != nil || count < 1 || processed != 1 {
+		t.Fatalf("process result count=%d processed=%d err=%v", count, processed, err)
+	}
+	var status string
+	if err := pool.QueryRow(context.Background(), `SELECT status FROM outbox_event WHERE id = $1::uuid`, firstID).Scan(&status); err != nil {
+		t.Fatal(err)
+	}
+	if status != "completed" {
+		t.Fatalf("event status = %s, want completed", status)
+	}
+}
+
+func TestWorkerMovesUnknownEventToDead(t *testing.T) {
+	pool := outboxIntegrationPool(t)
+	store := NewStore(pool)
+	worker := NewWorker(store, discardLogger(), WorkerOptions{MaxAttempts: 1, BaseRetry: time.Millisecond})
+	deduplicationKey := fmt.Sprintf("unknown-%d", time.Now().UnixNano())
+	eventID := enqueueIntegrationEvent(t, pool, store, NewEvent{
+		AggregateType: "platform", AggregateID: "10000000-0000-0000-0000-000000000201",
+		EventType: "integration.unknown", DeduplicationKey: &deduplicationKey,
+		Payload: json.RawMessage(`{"test":true}`),
+	})
+	t.Cleanup(func() { _, _ = pool.Exec(context.Background(), `DELETE FROM outbox_event WHERE id=$1::uuid`, eventID) })
+	if _, err := worker.ProcessOnce(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	var status, errorCode string
+	var deadAt *time.Time
+	if err := pool.QueryRow(context.Background(), `
+		SELECT status, last_error_code, dead_at FROM outbox_event WHERE id = $1::uuid`, eventID,
+	).Scan(&status, &errorCode, &deadAt); err != nil {
+		t.Fatal(err)
+	}
+	if status != "dead" || errorCode != "HANDLER_NOT_REGISTERED" || deadAt == nil {
+		t.Fatalf("unknown event status=%s code=%s dead_at=%v", status, errorCode, deadAt)
+	}
+}
+
+func TestWorkerRetriesOnlyExplicitTemporaryErrors(t *testing.T) {
+	pool := outboxIntegrationPool(t)
+	store := NewStore(pool)
+	worker := NewWorker(store, discardLogger(), WorkerOptions{MaxAttempts: 4, BaseRetry: time.Millisecond})
+	worker.Register("integration.permanent", func(context.Context, Event) error { return errors.New("invalid payload") })
+	worker.Register("integration.temporary", func(context.Context, Event) error { return Retryable(errors.New("dependency unavailable")) })
+
+	permanentKey := fmt.Sprintf("permanent-%d", time.Now().UnixNano())
+	temporaryKey := fmt.Sprintf("temporary-%d", time.Now().UnixNano())
+	permanentID := enqueueIntegrationEvent(t, pool, store, NewEvent{AggregateType: "platform", AggregateID: "10000000-0000-0000-0000-000000000201", EventType: "integration.permanent", DeduplicationKey: &permanentKey, Payload: json.RawMessage(`{"test":true}`)})
+	temporaryID := enqueueIntegrationEvent(t, pool, store, NewEvent{AggregateType: "platform", AggregateID: "10000000-0000-0000-0000-000000000201", EventType: "integration.temporary", DeduplicationKey: &temporaryKey, Payload: json.RawMessage(`{"test":true}`)})
+	t.Cleanup(func() {
+		_, _ = pool.Exec(context.Background(), `DELETE FROM outbox_event WHERE id=ANY($1::uuid[])`, []string{permanentID, temporaryID})
+	})
+
+	if _, err := worker.ProcessOnce(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	for id, expected := range map[string]string{permanentID: "dead", temporaryID: "pending"} {
+		var status string
+		if err := pool.QueryRow(context.Background(), `SELECT status FROM outbox_event WHERE id=$1::uuid`, id).Scan(&status); err != nil {
+			t.Fatal(err)
+		}
+		if status != expected {
+			t.Fatalf("event %s status=%s, want %s", id, status, expected)
+		}
+	}
+}
+
+func enqueueIntegrationEvent(t *testing.T, pool *pgxpool.Pool, store *Store, event NewEvent) string {
+	t.Helper()
+	tx, err := pool.BeginTx(context.Background(), pgx.TxOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = tx.Rollback(context.Background()) }()
+	eventID, err := store.Enqueue(context.Background(), tx, event)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := tx.Commit(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	return eventID
+}
+
+func outboxIntegrationPool(t *testing.T) *pgxpool.Pool {
+	t.Helper()
+	databaseURL := os.Getenv("TEST_DATABASE_URL")
+	if databaseURL == "" {
+		t.Skip("TEST_DATABASE_URL is not set")
+	}
+	pool, err := pgxpool.New(context.Background(), databaseURL)
+	if err != nil {
+		t.Fatalf("connect integration database: %v", err)
+	}
+	t.Cleanup(pool.Close)
+	return pool
+}
+
+func discardLogger() *slog.Logger {
+	return slog.New(slog.NewTextHandler(io.Discard, nil))
+}
@@ -0,0 +1,71 @@
+package outbox
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgconn"
+)
+
+func TestWorkerDefaultsAndRetryCap(t *testing.T) {
+	worker := NewWorker(nil, discardLogger(), WorkerOptions{})
+	if worker.options.PollInterval != 2*time.Second || worker.options.BatchSize != 20 || worker.options.MaxAttempts != 8 {
+		t.Fatalf("unexpected worker defaults: %#v", worker.options)
+	}
+	worker.options.BaseRetry = time.Second
+	worker.options.MaxRetry = 3 * time.Second
+	delay := time.Until(worker.nextAttempt(10))
+	if delay < 1400*time.Millisecond || delay > 3100*time.Millisecond {
+		t.Fatalf("capped retry delay = %s", delay)
+	}
+	if !strings.HasPrefix(worker.workerID, "worker-") {
+		t.Fatalf("unexpected worker id: %s", worker.workerID)
+	}
+}
+
+func TestStoreRejectsInvalidEventBeforeDatabaseAccess(t *testing.T) {
+	store := NewStore(nil)
+	if _, err := store.Enqueue(context.Background(), nil, NewEvent{}); err == nil {
+		t.Fatal("empty event should be rejected")
+	}
+	if _, err := store.Enqueue(context.Background(), nil, NewEvent{
+		AggregateType: "test", AggregateID: "id", EventType: "type", Payload: json.RawMessage(`not-json`),
+	}); err == nil {
+		t.Fatal("invalid JSON payload should be rejected")
+	}
+}
+
+func TestHashPayloadIsStable(t *testing.T) {
+	first := hashPayload([]byte(`{"ok":true}`))
+	if first != hashPayload([]byte(`{"ok":true}`)) || first == hashPayload([]byte(`{"ok":false}`)) {
+		t.Fatal("payload hash is not stable")
+	}
+}
+
+func TestRetryableErrorAllowList(t *testing.T) {
+	permanent := errors.New("malformed payload")
+	if IsRetryable(permanent) {
+		t.Fatal("ordinary errors must be permanent by default")
+	}
+	if !IsRetryable(Retryable(permanent)) || !errors.Is(Retryable(permanent), permanent) {
+		t.Fatal("explicit retryable marker was not preserved")
+	}
+	if !IsRetryable(context.DeadlineExceeded) || !IsRetryable(&net.DNSError{IsTimeout: true}) {
+		t.Fatal("timeouts must remain retryable")
+	}
+	if !IsRetryable(&pgconn.PgError{Code: "40001"}) || IsRetryable(&pgconn.PgError{Code: "23505"}) {
+		t.Fatal("PostgreSQL transient and constraint errors were misclassified")
+	}
+	if Retryable(nil) != nil {
+		t.Fatal("wrapping nil must return nil")
+	}
+	if !IsRetryable(fmt.Errorf("dependency: %w", Retryable(permanent))) {
+		t.Fatal("wrapped marker must remain discoverable")
+	}
+}