story/internal/outbox/worker.go

package outbox

import (
	"context"
	"crypto/rand"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"log/slog"
	"math"
	"time"
)

type Handler func(context.Context, Event) error

type WorkerOptions struct {
	PollInterval time.Duration
	LockTimeout  time.Duration
	BaseRetry    time.Duration
	MaxRetry     time.Duration
	BatchSize    int
	MaxAttempts  int
}

type Worker struct {
	store    *Store
	logger   *slog.Logger
	workerID string
	options  WorkerOptions
	handlers map[string]Handler
}

func NewWorker(store *Store, logger *slog.Logger, options WorkerOptions) *Worker {
	if options.PollInterval <= 0 {
		options.PollInterval = 2 * time.Second
	}
	if options.LockTimeout <= 0 {
		options.LockTimeout = time.Minute
	}
	if options.BaseRetry <= 0 {
		options.BaseRetry = time.Second
	}
	if options.MaxRetry <= 0 {
		options.MaxRetry = 15 * time.Minute
	}
	if options.BatchSize <= 0 {
		options.BatchSize = 20
	}
	if options.MaxAttempts <= 0 {
		options.MaxAttempts = 8
	}
	return &Worker{
		store: store, logger: logger, workerID: newWorkerID(), options: options,
		handlers: make(map[string]Handler),
	}
}

func (w *Worker) Register(eventType string, handler Handler) { w.handlers[eventType] = handler }

func (w *Worker) Run(ctx context.Context) {
	w.logger.Info("outbox worker started", "worker_id", w.workerID)
	ticker := time.NewTicker(w.options.PollInterval)
	defer ticker.Stop()
	for {
		if _, err := w.ProcessOnce(ctx); err != nil && !errors.Is(err, context.Canceled) {
			w.logger.Error("outbox batch failed", "worker_id", w.workerID, "error", err)
		}
		select {
		case <-ctx.Done():
			w.logger.Info("outbox worker stopped", "worker_id", w.workerID)
			return
		case <-ticker.C:
		}
	}
}

func (w *Worker) ProcessOnce(ctx context.Context) (int, error) {
	events, err := w.store.ClaimBatch(ctx, w.workerID, w.options.BatchSize, w.options.LockTimeout)
	if err != nil {
		return 0, err
	}
	for _, event := range events {
		handler, ok := w.handlers[event.EventType]
		if !ok {
			if err := w.store.MarkFailed(
				ctx, event, w.workerID, "HANDLER_NOT_REGISTERED", event.AttemptCount, time.Now(),
			); err != nil {
				return 0, err
			}
			continue
		}
		if err := handler(ctx, event); err != nil {
			retryable := IsRetryable(err)
			maxAttempts := event.AttemptCount
			nextAttempt := time.Now()
			errorCode := "HANDLER_PERMANENT_FAILURE"
			if retryable {
				maxAttempts = w.options.MaxAttempts
				nextAttempt = w.nextAttemptForError(event.AttemptCount, err)
				errorCode = "HANDLER_TEMPORARY_FAILURE"
			}
			w.logger.Warn("outbox handler failed", "event_id", event.ID, "event_type", event.EventType, "attempt", event.AttemptCount, "retryable", retryable)
			if markErr := w.store.MarkFailed(
				ctx, event, w.workerID, errorCode, maxAttempts, nextAttempt,
			); markErr != nil {
				return 0, markErr
			}
			continue
		}
		if err := w.store.MarkCompleted(ctx, event.ID, w.workerID); err != nil {
			return 0, err
		}
	}
	return len(events), nil
}

func (w *Worker) nextAttempt(attempt int) time.Time {
	multiplier := math.Pow(2, float64(max(attempt-1, 0)))
	delay := time.Duration(float64(w.options.BaseRetry) * multiplier)
	if delay > w.options.MaxRetry {
		delay = w.options.MaxRetry
	}
	// Jitter each worker independently to prevent a recovered dependency from
	// receiving a synchronized retry wave. Keep the randomized delay at or
	// below the configured maximum.
	delay = time.Duration(float64(delay) * (0.5 + 0.5*randomFraction()))
	return time.Now().Add(delay)
}

func (w *Worker) nextAttemptForError(attempt int, err error) time.Time {
	next := w.nextAttempt(attempt)
	if requested := time.Now().Add(RetryDelay(err)); requested.After(next) {
		return requested
	}
	return next
}

func randomFraction() float64 {
	var value [8]byte
	if _, err := rand.Read(value[:]); err != nil {
		return 0.5
	}
	return float64(binary.BigEndian.Uint64(value[:])>>11) / (1 << 53)
}

func newWorkerID() string {
	value := make([]byte, 8)
	if _, err := rand.Read(value); err != nil {
		return "worker-fallback"
	}
	return "worker-" + hex.EncodeToString(value)
}