package outbox import ( "context" "crypto/rand" "encoding/binary" "encoding/hex" "errors" "log/slog" "math" "time" ) type Handler func(context.Context, Event) error type WorkerOptions struct { PollInterval time.Duration LockTimeout time.Duration BaseRetry time.Duration MaxRetry time.Duration BatchSize int MaxAttempts int } type Worker struct { store *Store logger *slog.Logger workerID string options WorkerOptions handlers map[string]Handler } func NewWorker(store *Store, logger *slog.Logger, options WorkerOptions) *Worker { if options.PollInterval <= 0 { options.PollInterval = 2 * time.Second } if options.LockTimeout <= 0 { options.LockTimeout = time.Minute } if options.BaseRetry <= 0 { options.BaseRetry = time.Second } if options.MaxRetry <= 0 { options.MaxRetry = 15 * time.Minute } if options.BatchSize <= 0 { options.BatchSize = 20 } if options.MaxAttempts <= 0 { options.MaxAttempts = 8 } return &Worker{ store: store, logger: logger, workerID: newWorkerID(), options: options, handlers: make(map[string]Handler), } } func (w *Worker) Register(eventType string, handler Handler) { w.handlers[eventType] = handler } func (w *Worker) Run(ctx context.Context) { w.logger.Info("outbox worker started", "worker_id", w.workerID) ticker := time.NewTicker(w.options.PollInterval) defer ticker.Stop() for { if _, err := w.ProcessOnce(ctx); err != nil && !errors.Is(err, context.Canceled) { w.logger.Error("outbox batch failed", "worker_id", w.workerID, "error", err) } select { case <-ctx.Done(): w.logger.Info("outbox worker stopped", "worker_id", w.workerID) return case <-ticker.C: } } } func (w *Worker) ProcessOnce(ctx context.Context) (int, error) { events, err := w.store.ClaimBatch(ctx, w.workerID, w.options.BatchSize, w.options.LockTimeout) if err != nil { return 0, err } for _, event := range events { handler, ok := w.handlers[event.EventType] if !ok { if err := w.store.MarkFailed( ctx, event, w.workerID, "HANDLER_NOT_REGISTERED", event.AttemptCount, time.Now(), ); err != nil { return 0, err } continue } if err := handler(ctx, event); err != nil { retryable := IsRetryable(err) maxAttempts := event.AttemptCount nextAttempt := time.Now() errorCode := "HANDLER_PERMANENT_FAILURE" if retryable { maxAttempts = w.options.MaxAttempts nextAttempt = w.nextAttempt(event.AttemptCount) errorCode = "HANDLER_TEMPORARY_FAILURE" } w.logger.Warn("outbox handler failed", "event_id", event.ID, "event_type", event.EventType, "attempt", event.AttemptCount, "retryable", retryable) if markErr := w.store.MarkFailed( ctx, event, w.workerID, errorCode, maxAttempts, nextAttempt, ); markErr != nil { return 0, markErr } continue } if err := w.store.MarkCompleted(ctx, event.ID, w.workerID); err != nil { return 0, err } } return len(events), nil } func (w *Worker) nextAttempt(attempt int) time.Time { multiplier := math.Pow(2, float64(max(attempt-1, 0))) delay := time.Duration(float64(w.options.BaseRetry) * multiplier) if delay > w.options.MaxRetry { delay = w.options.MaxRetry } // Jitter each worker independently to prevent a recovered dependency from // receiving a synchronized retry wave. Keep the randomized delay at or // below the configured maximum. delay = time.Duration(float64(delay) * (0.5 + 0.5*randomFraction())) return time.Now().Add(delay) } func randomFraction() float64 { var value [8]byte if _, err := rand.Read(value[:]); err != nil { return 0.5 } return float64(binary.BigEndian.Uint64(value[:])>>11) / (1 << 53) } func newWorkerID() string { value := make([]byte, 8) if _, err := rand.Read(value); err != nil { return "worker-fallback" } return "worker-" + hex.EncodeToString(value) }