story/internal/app/app.go

package app

import (
	"context"
	"fmt"
	"log/slog"
	"net/http"
	"os"
	"time"

	"data-terminal-cloud/internal/alertnotification"
	"data-terminal-cloud/internal/alerts"
	"data-terminal-cloud/internal/audit"
	"data-terminal-cloud/internal/billing"
	"data-terminal-cloud/internal/cache"
	"data-terminal-cloud/internal/config"
	"data-terminal-cloud/internal/contentpolicy"
	"data-terminal-cloud/internal/database"
	"data-terminal-cloud/internal/dataexport"
	"data-terminal-cloud/internal/device"
	"data-terminal-cloud/internal/deviceauth"
	"data-terminal-cloud/internal/diagnostics"
	"data-terminal-cloud/internal/embeddingprovider"
	"data-terminal-cloud/internal/externalprobe"
	"data-terminal-cloud/internal/extraction"
	"data-terminal-cloud/internal/health"
	"data-terminal-cloud/internal/httpapi"
	"data-terminal-cloud/internal/idempotency"
	"data-terminal-cloud/internal/identity"
	"data-terminal-cloud/internal/media"
	"data-terminal-cloud/internal/memoryindex"
	"data-terminal-cloud/internal/memorylifecycle"
	"data-terminal-cloud/internal/objectstore"
	"data-terminal-cloud/internal/observability"
	"data-terminal-cloud/internal/opsaccess"
	"data-terminal-cloud/internal/opsconfig"
	"data-terminal-cloud/internal/orchestration"
	"data-terminal-cloud/internal/outbox"
	"data-terminal-cloud/internal/pagination"
	"data-terminal-cloud/internal/privacy"
	"data-terminal-cloud/internal/ratelimit"
	"data-terminal-cloud/internal/rtc"
	"data-terminal-cloud/internal/saas"
	"data-terminal-cloud/internal/sharing"
	"data-terminal-cloud/internal/voiceprovider"
	"data-terminal-cloud/internal/webevents"

	"github.com/jackc/pgx/v5/pgxpool"
	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)

func Run(ctx context.Context, serviceName string) error {
	cfg, err := config.Load(serviceName)
	if err != nil {
		return fmt.Errorf("load configuration: %w", err)
	}
	logger := newLogger(cfg)
	traceProvider, err := observability.SetupTracing(ctx, serviceName, cfg.Environment, cfg.OTLPEndpoint)
	if err != nil {
		return err
	}
	defer func() {
		shutdownContext, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()
		if err := traceProvider.Shutdown(shutdownContext); err != nil {
			logger.Error("trace provider shutdown failed", "error", err)
		}
	}()
	databasePool, err := database.Open(ctx, cfg.DatabaseURL, serviceName)
	if err != nil {
		return err
	}
	defer databasePool.Close()
	redisClient := cache.Open(cfg.RedisAddr, cfg.RedisPassword, cfg.RedisDB)
	defer func() { _ = redisClient.Close() }()
	var exportStore *objectstore.Store
	var memoryEmbedding embeddingprovider.Provider
	if serviceName == "api" || serviceName == "worker" {
		memoryEmbedding, err = newMemoryEmbeddingProvider(cfg)
		if err != nil {
			return fmt.Errorf("create memory embedding provider: %w", err)
		}
		exportStore, err = objectstore.New(cfg.ObjectStoreEndpoint, cfg.ObjectStorePublic, cfg.ObjectStoreAccessKey, cfg.ObjectStoreSecretKey, cfg.ObjectStoreBucket, cfg.ObjectStoreRegion, cfg.ObjectStoreSecure, cfg.ObjectStorePublicSecure)
		if err != nil {
			return fmt.Errorf("create private export object store: %w", err)
		}
		ensureContext, cancel := context.WithTimeout(ctx, cfg.DependencyTimeout)
		err = exportStore.EnsureBucket(ensureContext)
		cancel()
		if err != nil {
			return fmt.Errorf("ensure private export bucket: %w", err)
		}
	}

	readinessChecks := map[string]health.CheckFunc{
		"postgres": observability.InstrumentDependencyCheck(serviceName, "postgres", databasePool.Ping),
		"redis": observability.InstrumentDependencyCheck(serviceName, "redis", func(checkContext context.Context) error {
			return redisClient.Ping(checkContext).Err()
		}),
	}
	if exportStore != nil {
		readinessChecks["object_store"] = observability.InstrumentDependencyCheck(serviceName, "object_store", exportStore.Ready)
	}
	readiness := health.NewChecker(cfg.DependencyTimeout, readinessChecks)
	externalProbeRunner, err := newExternalProbeRunner(cfg, logger, databasePool)
	if err != nil {
		return fmt.Errorf("create external dependency probes: %w", err)
	}
	tokenVerifier := identity.NewOIDCVerifier(ctx, cfg.AuthIssuerURL, cfg.AuthJWKSURL, cfg.AuthAudience)
	identityRepository := identity.NewRepository(databasePool)
	var deviceVerifier *deviceauth.Verifier
	var deviceRepository *device.Repository
	var deviceTelemetryService *device.TelemetryService
	var conversationCreator *orchestration.Service
	var conversationHeartbeater *orchestration.HeartbeatService
	var conversationTranscriptSubmitter *orchestration.TranscriptBatchService
	var providerEventProcessor *orchestration.ProviderEventProcessor
	var saasService *saas.Service
	var mediaService *media.Service
	var webEventService *webevents.Service
	var sharingService *sharing.Service
	var opsConfigService *opsconfig.Service
	var billingService *billing.Service
	var alertService *alerts.Service
	var diagnosticService *diagnostics.Service
	var opsAccessService *opsaccess.Service
	if serviceName == "api" || serviceName == "callbacks" {
		diagnosticService, err = diagnostics.NewService(databasePool, idempotency.NewExecutor(databasePool), audit.NewRepository())
		if err != nil {
			return fmt.Errorf("create diagnostic service: %w", err)
		}
	}
	if serviceName == "api" {
		webEventService = webevents.NewService(databasePool)
		cursorCodec, cursorErr := pagination.NewCodec([]byte(cfg.CursorSigningSecret))
		if cursorErr != nil {
			return fmt.Errorf("create pagination cursor codec: %w", cursorErr)
		}
		saasService = saas.NewService(databasePool, audit.NewRepository(), redisClient, cursorCodec)
		opsConfigService, err = opsconfig.NewService(databasePool, audit.NewRepository())
		if err != nil {
			return fmt.Errorf("create operations configuration service: %w", err)
		}
		billingService, err = billing.NewService(databasePool, audit.NewRepository())
		if err != nil {
			return fmt.Errorf("create usage billing service: %w", err)
		}
		alertService, err = alerts.NewService(databasePool, audit.NewRepository())
		if err != nil {
			return fmt.Errorf("create alert operations service: %w", err)
		}
		opsAccessService, err = opsaccess.NewService(databasePool, idempotency.NewExecutor(databasePool), audit.NewRepository(), exportStore, 5*time.Minute)
		if err != nil {
			return fmt.Errorf("create temporary sensitive access service: %w", err)
		}
		saasService.ConfigureExportSigning(exportStore, cfg.ExportDownloadTTL)
		saasService.ConfigureFirmwareStore(exportStore, 5*time.Minute)
		if exportStore != nil {
			mediaService, err = media.NewService(databasePool, outbox.NewStore(databasePool), audit.NewRepository(), exportStore, 15*time.Minute)
			if err != nil {
				return fmt.Errorf("create media service: %w", err)
			}
			sharingService, err = sharing.NewService(databasePool, audit.NewRepository(), exportStore, []byte(cfg.ShareTokenSecret), 5*time.Minute)
			if err != nil {
				return fmt.Errorf("create sharing service: %w", err)
			}
		}
		deviceIdentityRepository := deviceauth.NewRepository(databasePool)
		var deviceSecretResolver deviceauth.SecretResolver
		if cfg.DeviceSecretDirectory != "" {
			deviceSecretResolver, err = deviceauth.NewDirectorySecretResolver(cfg.DeviceSecretDirectory)
			if err != nil {
				return fmt.Errorf("create per-device secret resolver: %w", err)
			}
		} else {
			// The local environment provides one already-bound simulator and one
			// unbound simulator so the family portal's one-time claim flow can be
			// exercised without provisioning production device secrets.
			deviceSecretResolver = deviceauth.StaticSecretResolver{
				"local-dev://simulated-terminal-1": []byte(cfg.DeviceLocalSecret),
				"local-dev://unbound-terminal-2":   []byte(cfg.DeviceLocalSecret),
			}
		}
		deviceVerifier, err = deviceauth.NewVerifier(
			deviceIdentityRepository,
			deviceSecretResolver,
			deviceauth.NewRedisReplayGuard(redisClient, "voice-platform"),
			cfg.DeviceClockSkew,
		)
		if err != nil {
			return fmt.Errorf("create device verifier: %w", err)
		}
		deviceRepository = device.NewRepository(databasePool)
		deviceTelemetryService, err = device.NewTelemetryService(databasePool, idempotency.NewExecutor(databasePool))
		if err != nil {
			return fmt.Errorf("create device telemetry service: %w", err)
		}
		var tokenIssuer rtc.TokenIssuer
		if cfg.RTCTokenIssuer == "volcengine" {
			tokenIssuer, err = rtc.NewVolcengineTokenIssuer(cfg.RTCAppID, cfg.RTCAppKey, cfg.RTCTokenTTL)
		} else {
			tokenIssuer, err = rtc.NewLocalTokenIssuer([]byte(cfg.RTCTokenSecret))
		}
		if err != nil {
			return fmt.Errorf("create RTC token issuer: %w", err)
		}
		leaseCache, leaseErr := orchestration.NewRedisLeaseCache(redisClient, "voice-platform")
		if leaseErr != nil {
			return fmt.Errorf("create conversation lease cache: %w", leaseErr)
		}
		conversationCreator, err = orchestration.NewService(
			idempotency.NewExecutor(databasePool), orchestration.NewRepository(), audit.NewRepository(),
			outbox.NewStore(databasePool), tokenIssuer, leaseCache,
			orchestration.ServiceOptions{
				RTCAppID: cfg.RTCAppID, VoiceProvider: cfg.VoiceProvider, EngineMode: "s2s_hybrid", AllocationTTL: cfg.AllocationTTL,
				TokenTTL: cfg.RTCTokenTTL, HeartbeatInterval: cfg.HeartbeatInterval, RequirePublishedRouting: cfg.Environment == config.EnvironmentProd,
			},
		)
		if err != nil {
			return fmt.Errorf("create conversation orchestration service: %w", err)
		}
		if err = conversationCreator.ConfigureMemoryEmbedding(databasePool, memoryEmbedding); err != nil {
			return fmt.Errorf("configure conversation memory embedding: %w", err)
		}
		credentialRenewalWindow := cfg.RTCTokenTTL / 3
		minimumRenewalWindow := time.Duration(cfg.HeartbeatInterval*2) * time.Second
		if credentialRenewalWindow < minimumRenewalWindow {
			credentialRenewalWindow = minimumRenewalWindow
		}
		if credentialRenewalWindow >= cfg.RTCTokenTTL {
			credentialRenewalWindow = cfg.RTCTokenTTL / 2
		}
		conversationHeartbeater, err = orchestration.NewHeartbeatService(
			databasePool, orchestration.NewRepository(), leaseCache, tokenIssuer,
			orchestration.HeartbeatServiceOptions{
				RTCAppID: cfg.RTCAppID, LeaseTTL: time.Duration(cfg.HeartbeatInterval*5) * time.Second,
				TokenTTL: cfg.RTCTokenTTL, CredentialRenewalWindow: credentialRenewalWindow,
			},
		)
		if err != nil {
			return fmt.Errorf("create conversation heartbeat service: %w", err)
		}
		conversationTranscriptSubmitter, err = orchestration.NewTranscriptBatchService(
			databasePool, idempotency.NewExecutor(databasePool), audit.NewRepository(), outbox.NewStore(databasePool),
		)
		if err != nil {
			return fmt.Errorf("create conversation transcript batch service: %w", err)
		}
	}
	if serviceName == "callbacks" {
		providerEventProcessor, err = orchestration.NewProviderEventProcessor(databasePool, time.Duration(cfg.HeartbeatInterval*5)*time.Second)
		if err != nil {
			return fmt.Errorf("create provider event processor: %w", err)
		}
	}
	limiter, err := ratelimit.NewPolicies(redisClient, "voice-platform:"+serviceName, map[string]ratelimit.Policy{
		"network":        {RatePerSecond: cfg.RateLimitRPS, Burst: cfg.RateLimitBurst},
		"user":           {RatePerSecond: cfg.RateLimitUserRPS, Burst: cfg.RateLimitUserBurst},
		"tenant":         {RatePerSecond: cfg.RateLimitTenantRPS, Burst: cfg.RateLimitTenantBurst},
		"device":         {RatePerSecond: cfg.RateLimitDeviceRPS, Burst: cfg.RateLimitDeviceBurst},
		"provider_start": {RatePerSecond: cfg.ProviderStartRPS, Burst: cfg.ProviderStartBurst},
	})
	if err != nil {
		return fmt.Errorf("create rate limiter: %w", err)
	}
	if serviceName == "worker" {
		webEventService = webevents.NewService(databasePool)
		outbox.RegisterMetrics(databasePool)
		observability.RegisterBusinessMetrics(databasePool)
		outboxWorker := outbox.NewWorker(
			outbox.NewStore(databasePool),
			logger,
			outbox.WorkerOptions{PollInterval: cfg.WorkerPollInterval},
		)
		outboxWorker.Register("platform.noop", func(context.Context, outbox.Event) error { return nil })
		alertWebhookProcessor, processorErr := alertnotification.NewProcessor(cfg.AlertWebhookURL, cfg.AlertWebhookSecret, cfg.AlertWebhookTimeout)
		if processorErr != nil {
			return fmt.Errorf("create alert webhook processor: %w", processorErr)
		}
		outboxWorker.Register("alert.webhook_requested", alertWebhookProcessor.Handle)
		outboxWorker.Register("alert.webhook_test_requested", alertWebhookProcessor.Handle)
		outboxWorker.Register("conversation.room_issued", func(_ context.Context, event outbox.Event) error {
			logger.Info("conversation room issued event observed", "event_id", event.ID, "conversation_id", event.AggregateID)
			return nil
		})
		var providerClient voiceprovider.Client
		if cfg.VoiceProvider == "volcengine" {
			template, readErr := os.ReadFile(cfg.VolcVoiceChatTemplate)
			if readErr != nil {
				return fmt.Errorf("read Volcengine voice-chat template: %w", readErr)
			}
			providerClient, err = voiceprovider.NewVolcengineClient(voiceprovider.VolcengineOptions{
				BaseURL: cfg.VolcRTCBaseURL, AccessKeyID: cfg.VolcAccessKeyID, SecretAccessKey: cfg.VolcSecretAccessKey,
				SessionToken: cfg.VolcSessionToken, Region: cfg.VolcRTCRegion, AppID: cfg.RTCAppID,
				APIVersion: cfg.VolcRTCAPIVersion, Template: template, Timeout: cfg.ProviderRequestTimeout,
				RequireAllEngineModes: cfg.Environment == config.EnvironmentProd,
			})
		} else {
			providerClient, err = voiceprovider.NewHTTPClient(cfg.MockProviderBaseURL, cfg.MockProviderAPIKey, cfg.ProviderRequestTimeout)
		}
		if err != nil {
			return fmt.Errorf("create voice provider client: %w", err)
		}
		providerHealth, providerErr := orchestration.NewProviderHealth(databasePool, cfg.ProviderFailureThreshold, cfg.ProviderCircuitCooldown)
		if providerErr != nil {
			return fmt.Errorf("create voice provider health: %w", providerErr)
		}
		leaseTTL := time.Duration(cfg.HeartbeatInterval*5) * time.Second
		startProcessor, processorErr := orchestration.NewStartProcessor(databasePool, providerClient, providerHealth, leaseTTL)
		if processorErr != nil {
			return fmt.Errorf("create provider start processor: %w", processorErr)
		}
		if processorErr = startProcessor.ConfigureStartProtection(limiter, cfg.ProviderMaxActiveTasks); processorErr != nil {
			return fmt.Errorf("configure provider start protection: %w", processorErr)
		}
		outboxWorker.Register("conversation.agent_start_requested", startProcessor.Handle)
		outboxWorker.Register("conversation.agent_start_reconcile_requested", startProcessor.Handle)
		stopProcessor, processorErr := orchestration.NewStopProcessor(databasePool, providerClient, providerHealth)
		if processorErr != nil {
			return fmt.Errorf("create provider stop processor: %w", processorErr)
		}
		outboxWorker.Register("conversation.agent_stop_requested", stopProcessor.Handle)
		outboxWorker.Register("conversation.agent_stop_reconcile_requested", stopProcessor.Handle)
		var contentGenerator extraction.Generator = extraction.DeterministicGenerator{}
		if cfg.ContentGenerator == "ark" {
			arkGenerator, generatorErr := extraction.NewArkGenerator(cfg.ArkBaseURL, cfg.ArkAPIKey, cfg.ArkContentModel, cfg.ArkRequestTimeout)
			if generatorErr != nil {
				return fmt.Errorf("create Ark content generator: %w", generatorErr)
			}
			contentGenerator = arkGenerator
		}
		var contentSafety contentpolicy.Classifier = contentpolicy.LocalClassifier{}
		if cfg.ContentSafetyProvider == "managed_http" {
			managedSafety, safetyErr := contentpolicy.NewManagedClassifier(cfg.ContentSafetyEndpoint, cfg.ContentSafetyAPIKey, cfg.ContentSafetyPolicyRef, cfg.ContentSafetyTimeout)
			if safetyErr != nil {
				return fmt.Errorf("create managed content safety classifier: %w", safetyErr)
			}
			contentSafety = managedSafety
		}
		extractionProcessor, processorErr := extraction.NewProcessorWithSafety(databasePool, contentGenerator, contentSafety)
		if processorErr != nil {
			return fmt.Errorf("create content extraction processor: %w", processorErr)
		}
		outboxWorker.Register("conversation.content_extraction_requested", extractionProcessor.Handle)
		memoryIndexProcessor, processorErr := memoryindex.NewProcessor(databasePool, memoryEmbedding)
		if processorErr != nil {
			return fmt.Errorf("create memory index processor: %w", processorErr)
		}
		outboxWorker.Register("memory.index_requested", memoryIndexProcessor.Handle)
		memoryDeleteProcessor, processorErr := memorylifecycle.NewDeleteProcessor(databasePool)
		if processorErr != nil {
			return fmt.Errorf("create memory delete processor: %w", processorErr)
		}
		outboxWorker.Register("memory.delete_requested", memoryDeleteProcessor.Handle)
		privacyAccessProcessor, processorErr := privacy.NewAccessProcessor(databasePool)
		if processorErr != nil {
			return fmt.Errorf("create privacy access processor: %w", processorErr)
		}
		outboxWorker.Register("privacy.access_requested", privacyAccessProcessor.Handle)
		privacyDeleteProcessor, processorErr := privacy.NewDeleteProcessor(databasePool, redisClient, exportStore)
		if processorErr != nil {
			return fmt.Errorf("create privacy delete processor: %w", processorErr)
		}
		outboxWorker.Register("privacy.delete_requested", privacyDeleteProcessor.Handle)
		exportProcessor, processorErr := dataexport.NewProcessor(databasePool, exportStore, cfg.ExportObjectTTL)
		if processorErr != nil {
			return fmt.Errorf("create data export processor: %w", processorErr)
		}
		outboxWorker.Register("export.generate_requested", exportProcessor.Handle)
		if exportStore != nil {
			mediaProcessor, mediaErr := media.NewProcessor(databasePool, exportStore, audit.NewRepository(), cfg.MediaLocalDeleteGrace)
			if mediaErr != nil {
				return fmt.Errorf("create media verification processor: %w", mediaErr)
			}
			outboxWorker.Register("media.verify_requested", mediaProcessor.Handle)
		}
		expiryScanner, processorErr := dataexport.NewExpiryScanner(databasePool, exportStore, logger, cfg.WorkerPollInterval)
		if processorErr != nil {
			return fmt.Errorf("create export expiry scanner: %w", processorErr)
		}
		go expiryScanner.Run(ctx)
		go outboxWorker.Run(ctx)
		recoveryScanner, recoveryErr := orchestration.NewRecoveryScanner(
			databasePool, outbox.NewStore(databasePool), logger,
			orchestration.RecoveryOptions{
				ScanInterval: cfg.RecoveryScanInterval, StartingAfter: cfg.RecoveryStartingAfter,
				StoppingAfter: cfg.RecoveryStoppingAfter, BatchSize: 50,
			},
		)
		if recoveryErr != nil {
			return fmt.Errorf("create conversation recovery scanner: %w", recoveryErr)
		}
		go recoveryScanner.Run(ctx)
		alertScanner, alertErr := alerts.NewScanner(databasePool, logger, cfg.WorkerPollInterval)
		if alertErr != nil {
			return fmt.Errorf("create alert scanner: %w", alertErr)
		}
		go alertScanner.Run(ctx)
		go func() {
			ticker := time.NewTicker(time.Duration(cfg.HeartbeatInterval) * time.Second)
			defer ticker.Stop()
			for {
				select {
				case <-ctx.Done():
					return
				case <-ticker.C:
					staleBefore := time.Now().UTC().Add(-2 * leaseTTL)
					if _, err := webEventService.ExpireDevicePresence(ctx, staleBefore, 1000); err != nil {
						logger.Warn("device presence expiry scan failed", "error", err)
					}
					if _, err := webEventService.DeleteExpired(ctx, 5000); err != nil {
						logger.Warn("expired web event cleanup failed", "error", err)
					}
				}
			}
		}()
	}
	router := httpapi.NewRouter(httpapi.RouterOptions{
		ServiceName:                     serviceName,
		Logger:                          logger,
		RateLimiter:                     rateLimiterAdapter{limiter},
		Readiness:                       readiness,
		TokenVerifier:                   tokenVerifier,
		IdentityRepository:              identityRepository,
		SaaSService:                     saasService,
		DeviceVerifier:                  deviceVerifier,
		DeviceService:                   deviceRepository,
		DeviceTelemetrySubmitter:        deviceTelemetryService,
		MediaService:                    mediaService,
		WebEventService:                 webEventService,
		SharingService:                  sharingService,
		OpsConfigService:                opsConfigService,
		BillingService:                  billingService,
		AlertService:                    alertService,
		DiagnosticService:               diagnosticService,
		OpsAccessService:                opsAccessService,
		ConversationCreator:             conversationCreator,
		ConversationActivator:           conversationCreator,
		DeviceConversationStateReader:   conversationCreator,
		ConversationHeartbeater:         conversationHeartbeater,
		ConversationStopper:             conversationCreator,
		ConversationTranscriptSubmitter: conversationTranscriptSubmitter,
		ProviderCallbackSecret:          []byte(cfg.ProviderCallbackSecret),
		ProviderCallbackSkew:            cfg.ProviderCallbackSkew,
		ProviderCallbackProcessor:       providerEventProcessor,
		PlatformHealthCallbackSecret:    []byte(cfg.PlatformHealthSecret),
		PlatformHealthCallbackSkew:      cfg.PlatformHealthSkew,
		PlatformHealthProcessor:         diagnosticService,
	})
	server := &http.Server{
		Addr:              cfg.HTTPAddr,
		Handler:           otelhttp.NewHandler(httpapi.Middleware(logger, serviceName, 1<<20, rateLimiterAdapter{limiter}, router), serviceName),
		ReadHeaderTimeout: 5 * time.Second,
		ReadTimeout:       15 * time.Second,
		WriteTimeout:      30 * time.Second,
		IdleTimeout:       60 * time.Second,
	}

	serverError := make(chan error, 1)
	go externalProbeRunner.Run(ctx)
	go func() {
		logger.Info("service listening", "address", cfg.HTTPAddr, "environment", cfg.Environment)
		serverError <- server.ListenAndServe()
	}()

	select {
	case <-ctx.Done():
		shutdownContext, cancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout)
		defer cancel()
		logger.Info("service shutting down")
		return server.Shutdown(shutdownContext)
	case err := <-serverError:
		if err == http.ErrServerClosed {
			return nil
		}
		return err
	}
}

func newExternalProbeRunner(cfg config.Config, logger *slog.Logger, pool *pgxpool.Pool) (*externalprobe.Runner, error) {
	checks, err := externalProbeChecks(cfg)
	if err != nil {
		return nil, err
	}
	dependencies := make([]string, 0, len(checks))
	for dependency := range checks {
		dependencies = append(dependencies, dependency)
	}
	reconcileContext, cancel := context.WithTimeout(context.Background(), cfg.DependencyTimeout)
	err = externalprobe.ReconcilePostgresState(reconcileContext, pool, cfg.ServiceName, dependencies)
	cancel()
	if err != nil {
		return nil, err
	}
	observer, err := externalprobe.PostgresObserver(pool, cfg.ServiceName)
	if err != nil {
		return nil, err
	}
	return externalprobe.NewRunner(cfg.ExternalProbeInterval, cfg.DependencyTimeout, checks, logger, observer)
}

func externalProbeChecks(cfg config.Config) (map[string]externalprobe.CheckFunc, error) {
	checks := map[string]externalprobe.CheckFunc{}
	if cfg.ServiceName == "api" {
		jwksCheck, err := externalprobe.JWKS(cfg.AuthJWKSURL, nil)
		if err != nil {
			return nil, err
		}
		checks["oidc_jwks"] = observability.InstrumentDependencyCheck(cfg.ServiceName, "oidc_jwks", jwksCheck)
	}
	if (cfg.ServiceName == "api" || cfg.ServiceName == "worker") && cfg.EmbeddingProvider == "ark" {
		arkCheck, err := externalprobe.TLSReachability(cfg.ArkBaseURL, nil)
		if err != nil {
			return nil, err
		}
		checks["ark_tls"] = observability.InstrumentDependencyCheck(cfg.ServiceName, "ark_tls", arkCheck)
	}
	if cfg.ServiceName == "worker" && cfg.VoiceProvider == "volcengine" {
		rtcCheck, err := externalprobe.TLSReachability(cfg.VolcRTCBaseURL, nil)
		if err != nil {
			return nil, err
		}
		checks["volc_rtc_tls"] = observability.InstrumentDependencyCheck(cfg.ServiceName, "volc_rtc_tls", rtcCheck)
	}
	if cfg.ServiceName == "worker" && cfg.ContentSafetyProvider == "managed_http" {
		contentSafetyCheck, err := externalprobe.TLSReachability(cfg.ContentSafetyEndpoint, nil)
		if err != nil {
			return nil, err
		}
		checks["content_safety_tls"] = observability.InstrumentDependencyCheck(cfg.ServiceName, "content_safety_tls", contentSafetyCheck)
	}
	return checks, nil
}

func newMemoryEmbeddingProvider(cfg config.Config) (embeddingprovider.Provider, error) {
	if cfg.EmbeddingProvider == "ark" {
		return embeddingprovider.NewArk(cfg.ArkBaseURL, cfg.ArkAPIKey, cfg.ArkEmbeddingModel, cfg.ArkRequestTimeout)
	}
	return embeddingprovider.Local{}, nil
}

type rateLimiterAdapter struct{ limiter *ratelimit.Limiter }

func (a rateLimiterAdapter) Allow(ctx context.Context, scope, identifier string) (httpapi.RateLimitResult, error) {
	result, err := a.limiter.Allow(ctx, scope, identifier)
	return httpapi.RateLimitResult{Allowed: result.Allowed, Remaining: result.Remaining, RetryAfter: result.RetryAfter}, err
}

func newLogger(cfg config.Config) *slog.Logger {
	level := slog.LevelInfo
	switch cfg.LogLevel {
	case "debug":
		level = slog.LevelDebug
	case "warn":
		level = slog.LevelWarn
	case "error":
		level = slog.LevelError
	}
	return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: level})).With("service", cfg.ServiceName)
}