chore: establish local platform baseline

internal/externalprobe/postgres.go

@@ -0,0 +1,75 @@
+package externalprobe
+
+import (
+	"context"
+	"errors"
+	"regexp"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+var (
+	serviceNamePattern = regexp.MustCompile(`^[a-z][a-z0-9-]{1,31}$`)
+	dependencyPattern  = regexp.MustCompile(`^[a-z][a-z0-9_]{1,63}$`)
+)
+
+func PostgresObserver(pool *pgxpool.Pool, serviceName string) (Observer, error) {
+	if pool == nil || !serviceNamePattern.MatchString(serviceName) {
+		return nil, errors.New("external probe database and service name are required")
+	}
+	return func(ctx context.Context, observation Observation) error {
+		if !dependencyPattern.MatchString(observation.Dependency) {
+			return errors.New("external probe dependency name is invalid")
+		}
+		if observation.Interval <= 0 {
+			return errors.New("external probe interval is required")
+		}
+		latencyMS := observation.Latency.Milliseconds()
+		if latencyMS < 0 {
+			latencyMS = 0
+		}
+		intervalMS := observation.Interval.Milliseconds()
+		if intervalMS < 1 {
+			intervalMS = 1
+		}
+		checkedAt := observation.CheckedAt.UTC()
+		if checkedAt.IsZero() {
+			checkedAt = time.Now().UTC()
+		}
+		status := "AVAILABLE"
+		if !observation.Available {
+			status = "UNAVAILABLE"
+			if observation.ErrorCode == "" {
+				observation.ErrorCode = "PROBE_FAILED"
+			}
+		}
+		_, err := pool.Exec(ctx, `
+			INSERT INTO external_dependency_probe_state(service_name,dependency,status,checked_at,last_success_at,last_failure_at,last_error_code,latency_ms,probe_interval_ms,consecutive_failures)
+			VALUES($1,$2,$3::varchar,$4::timestamptz,CASE WHEN $3::varchar='AVAILABLE' THEN $4::timestamptz END,CASE WHEN $3::varchar='UNAVAILABLE' THEN $4::timestamptz END,NULLIF($5::varchar,''),$6,$7,CASE WHEN $3::varchar='UNAVAILABLE' THEN 1 ELSE 0 END)
+			ON CONFLICT(service_name,dependency) DO UPDATE SET
+				status=EXCLUDED.status,
+				checked_at=EXCLUDED.checked_at,
+				last_success_at=CASE WHEN EXCLUDED.status='AVAILABLE' THEN EXCLUDED.checked_at ELSE external_dependency_probe_state.last_success_at END,
+				last_failure_at=CASE WHEN EXCLUDED.status='UNAVAILABLE' THEN EXCLUDED.checked_at ELSE external_dependency_probe_state.last_failure_at END,
+				last_error_code=EXCLUDED.last_error_code,
+				latency_ms=EXCLUDED.latency_ms,
+				probe_interval_ms=EXCLUDED.probe_interval_ms,
+				consecutive_failures=CASE WHEN EXCLUDED.status='AVAILABLE' THEN 0 ELSE external_dependency_probe_state.consecutive_failures+1 END,
+				updated_at=now()`, serviceName, observation.Dependency, status, checkedAt, observation.ErrorCode, latencyMS, intervalMS)
+		return err
+	}, nil
+}
+
+func ReconcilePostgresState(ctx context.Context, pool *pgxpool.Pool, serviceName string, dependencies []string) error {
+	if pool == nil || !serviceNamePattern.MatchString(serviceName) {
+		return errors.New("external probe database and service name are required")
+	}
+	for _, dependency := range dependencies {
+		if !dependencyPattern.MatchString(dependency) {
+			return errors.New("external probe dependency name is invalid")
+		}
+	}
+	_, err := pool.Exec(ctx, `DELETE FROM external_dependency_probe_state WHERE service_name=$1 AND NOT (dependency=ANY(COALESCE($2::text[],ARRAY[]::text[])))`, serviceName, dependencies)
+	return err
+}

internal/externalprobe/postgres_integration_test.go

@@ -0,0 +1,60 @@
+package externalprobe
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+func TestPostgresObserverTracksFailureRecoveryAndReconciliation(t *testing.T) {
+	databaseURL := os.Getenv("TEST_DATABASE_URL")
+	if databaseURL == "" {
+		t.Skip("TEST_DATABASE_URL is not set")
+	}
+	ctx := context.Background()
+	pool, err := pgxpool.New(ctx, databaseURL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer pool.Close()
+	serviceName, dependency := "probe-test", "oidc_jwks"
+	defer pool.Exec(ctx, `DELETE FROM external_dependency_probe_state WHERE service_name=$1`, serviceName)
+	observer, err := PostgresObserver(pool, serviceName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	checkedAt := time.Now().UTC().Truncate(time.Millisecond)
+	for index := 0; index < 2; index++ {
+		if err = observer(ctx, Observation{Dependency: dependency, CheckedAt: checkedAt.Add(time.Duration(index) * time.Second), Latency: 15 * time.Millisecond, Interval: 30 * time.Second, ErrorCode: "TLS_ERROR"}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	var status, errorCode string
+	var failures int
+	if err = pool.QueryRow(ctx, `SELECT status,last_error_code,consecutive_failures FROM external_dependency_probe_state WHERE service_name=$1 AND dependency=$2`, serviceName, dependency).Scan(&status, &errorCode, &failures); err != nil {
+		t.Fatal(err)
+	}
+	if status != "UNAVAILABLE" || errorCode != "TLS_ERROR" || failures != 2 {
+		t.Fatalf("failure state=%s code=%s consecutive=%d", status, errorCode, failures)
+	}
+	if err = observer(ctx, Observation{Dependency: dependency, Available: true, CheckedAt: checkedAt.Add(2 * time.Second), Latency: 4 * time.Millisecond, Interval: 30 * time.Second}); err != nil {
+		t.Fatal(err)
+	}
+	var errorCleared, hasSuccess, hasFailure bool
+	if err = pool.QueryRow(ctx, `SELECT status,last_error_code IS NULL,consecutive_failures,last_success_at IS NOT NULL,last_failure_at IS NOT NULL FROM external_dependency_probe_state WHERE service_name=$1 AND dependency=$2`, serviceName, dependency).Scan(&status, &errorCleared, &failures, &hasSuccess, &hasFailure); err != nil {
+		t.Fatal(err)
+	}
+	if status != "AVAILABLE" || !errorCleared || !hasSuccess || !hasFailure || failures != 0 {
+		t.Fatalf("recovered state=%s error_cleared=%v success=%v failure=%v consecutive=%d", status, errorCleared, hasSuccess, hasFailure, failures)
+	}
+	if err = ReconcilePostgresState(ctx, pool, serviceName, nil); err != nil {
+		t.Fatal(err)
+	}
+	var remaining int
+	if err = pool.QueryRow(ctx, `SELECT count(*) FROM external_dependency_probe_state WHERE service_name=$1`, serviceName).Scan(&remaining); err != nil || remaining != 0 {
+		t.Fatalf("remaining=%d err=%v", remaining, err)
+	}
+}

internal/externalprobe/probe.go

@@ -0,0 +1,209 @@
+package externalprobe
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+)
+
+const maxJWKSBodyBytes = 1 << 20
+
+type CheckFunc func(context.Context) error
+
+type Observation struct {
+	ServiceName string
+	Dependency  string
+	Available   bool
+	CheckedAt   time.Time
+	Latency     time.Duration
+	Interval    time.Duration
+	ErrorCode   string
+}
+
+type Observer func(context.Context, Observation) error
+
+// Runner executes low-cost external dependency checks without making them part
+// of readiness. A temporary model or RTC network failure must remain visible to
+// operations without evicting otherwise healthy API and Worker pods.
+type Runner struct {
+	interval  time.Duration
+	timeout   time.Duration
+	checks    map[string]CheckFunc
+	logger    *slog.Logger
+	states    map[string]bool
+	observers []Observer
+	mu        sync.Mutex
+}
+
+func NewRunner(interval, timeout time.Duration, checks map[string]CheckFunc, logger *slog.Logger, observers ...Observer) (*Runner, error) {
+	if interval <= 0 || timeout <= 0 {
+		return nil, errors.New("external probe interval and timeout must be positive")
+	}
+	if logger == nil {
+		logger = slog.Default()
+	}
+	ownedChecks := make(map[string]CheckFunc, len(checks))
+	for name, check := range checks {
+		name = strings.TrimSpace(name)
+		if name == "" || check == nil {
+			return nil, errors.New("external probe name and check are required")
+		}
+		ownedChecks[name] = check
+	}
+	for _, observer := range observers {
+		if observer == nil {
+			return nil, errors.New("external probe observer is required")
+		}
+	}
+	return &Runner{interval: interval, timeout: timeout, checks: ownedChecks, logger: logger, states: map[string]bool{}, observers: append([]Observer(nil), observers...)}, nil
+}
+
+func (r *Runner) Run(ctx context.Context) {
+	r.runOnce(ctx)
+	ticker := time.NewTicker(r.interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			r.runOnce(ctx)
+		}
+	}
+}
+
+func (r *Runner) runOnce(ctx context.Context) {
+	for name, check := range r.checks {
+		checkContext, cancel := context.WithTimeout(ctx, r.timeout)
+		startedAt := time.Now()
+		err := check(checkContext)
+		checkedAt := time.Now().UTC()
+		latency := time.Since(startedAt)
+		cancel()
+		if latency < 0 {
+			latency = 0
+		}
+		r.mu.Lock()
+		previous, observed := r.states[name]
+		available := err == nil
+		r.states[name] = available
+		r.mu.Unlock()
+		if !available && (!observed || previous) {
+			r.logger.Warn("external dependency probe failed", "dependency", name, "error", err)
+		}
+		if available && observed && !previous {
+			r.logger.Info("external dependency probe recovered", "dependency", name)
+		}
+		observation := Observation{Dependency: name, Available: available, CheckedAt: checkedAt, Latency: latency, Interval: r.interval, ErrorCode: classifyError(err)}
+		for _, observer := range r.observers {
+			observerContext, observerCancel := context.WithTimeout(ctx, r.timeout)
+			observerErr := observer(observerContext, observation)
+			observerCancel()
+			if observerErr != nil && ctx.Err() == nil {
+				r.logger.Warn("external dependency probe observation failed", "dependency", name, "error", observerErr)
+			}
+		}
+	}
+}
+
+func classifyError(err error) string {
+	if err == nil {
+		return ""
+	}
+	if errors.Is(err, context.DeadlineExceeded) {
+		return "TIMEOUT"
+	}
+	if errors.Is(err, context.Canceled) {
+		return "CANCELED"
+	}
+	message := strings.ToLower(err.Error())
+	if strings.Contains(message, "certificate") || strings.Contains(message, "tls") || strings.Contains(message, "x509") {
+		return "TLS_ERROR"
+	}
+	if strings.Contains(message, "jwks") {
+		return "JWKS_INVALID"
+	}
+	var networkError net.Error
+	if errors.As(err, &networkError) {
+		return "NETWORK_ERROR"
+	}
+	return "PROBE_FAILED"
+}
+
+// JWKS validates that the configured endpoint returns a bounded JSON Web Key
+// Set with at least one key. It does not persist the keys or expose their body.
+func JWKS(endpoint string, client *http.Client) (CheckFunc, error) {
+	parsed, err := url.Parse(strings.TrimSpace(endpoint))
+	if err != nil || parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") {
+		return nil, errors.New("JWKS endpoint must be an absolute HTTP(S) URL")
+	}
+	if client == nil {
+		client = &http.Client{}
+	}
+	return func(ctx context.Context) error {
+		request, requestErr := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
+		if requestErr != nil {
+			return fmt.Errorf("create JWKS request: %w", requestErr)
+		}
+		request.Header.Set("Accept", "application/json")
+		response, requestErr := client.Do(request)
+		if requestErr != nil {
+			return fmt.Errorf("request JWKS: %w", requestErr)
+		}
+		defer response.Body.Close()
+		if response.StatusCode < 200 || response.StatusCode >= 300 {
+			_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4*1024))
+			return fmt.Errorf("JWKS endpoint returned HTTP %d", response.StatusCode)
+		}
+		var document struct {
+			Keys []json.RawMessage `json:"keys"`
+		}
+		decoder := json.NewDecoder(io.LimitReader(response.Body, maxJWKSBodyBytes+1))
+		if decodeErr := decoder.Decode(&document); decodeErr != nil {
+			return fmt.Errorf("decode JWKS document: %w", decodeErr)
+		}
+		if len(document.Keys) == 0 {
+			return errors.New("JWKS document contains no keys")
+		}
+		return nil
+	}, nil
+}
+
+// TLSReachability performs only DNS, TCP and certificate-verified TLS setup.
+// It intentionally sends no provider API request and therefore consumes no
+// model tokens, voice minutes or provider task quota.
+func TLSReachability(endpoint string, rootCAs *x509.CertPool) (CheckFunc, error) {
+	parsed, err := url.Parse(strings.TrimSpace(endpoint))
+	if err != nil || parsed.Scheme != "https" || parsed.Hostname() == "" {
+		return nil, errors.New("TLS probe endpoint must be an absolute HTTPS URL")
+	}
+	port := parsed.Port()
+	if port == "" {
+		port = "443"
+	}
+	address := net.JoinHostPort(parsed.Hostname(), port)
+	config := &tls.Config{
+		MinVersion: tls.VersionTLS12,
+		ServerName: parsed.Hostname(),
+		RootCAs:    rootCAs,
+	}
+	return func(ctx context.Context) error {
+		dialer := &tls.Dialer{NetDialer: &net.Dialer{}, Config: config.Clone()}
+		connection, dialErr := dialer.DialContext(ctx, "tcp", address)
+		if dialErr != nil {
+			return fmt.Errorf("establish verified TLS connection: %w", dialErr)
+		}
+		return connection.Close()
+	}, nil
+}

internal/externalprobe/probe_test.go

@@ -0,0 +1,75 @@
+package externalprobe
+
+import (
+	"context"
+	"crypto/x509"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestJWKSRequiresSuccessfulDocumentWithKeys(t *testing.T) {
+	tests := []struct {
+		name    string
+		status  int
+		body    string
+		wantErr bool
+	}{
+		{name: "valid", status: http.StatusOK, body: `{"keys":[{"kty":"RSA","kid":"one"}]}`},
+		{name: "empty", status: http.StatusOK, body: `{"keys":[]}`, wantErr: true},
+		{name: "invalid", status: http.StatusOK, body: `{`, wantErr: true},
+		{name: "unavailable", status: http.StatusServiceUnavailable, body: `{}`, wantErr: true},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
+				writer.WriteHeader(test.status)
+				_, _ = writer.Write([]byte(test.body))
+			}))
+			defer server.Close()
+			check, err := JWKS(server.URL, server.Client())
+			if err != nil {
+				t.Fatal(err)
+			}
+			err = check(context.Background())
+			if (err != nil) != test.wantErr {
+				t.Fatalf("check() error=%v, wantErr=%v", err, test.wantErr)
+			}
+		})
+	}
+}
+
+func TestTLSReachabilityValidatesCertificate(t *testing.T) {
+	server := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
+	defer server.Close()
+	roots := x509.NewCertPool()
+	roots.AddCert(server.Certificate())
+	check, err := TLSReachability(server.URL, roots)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	if err = check(ctx); err != nil {
+		t.Fatalf("TLS check failed: %v", err)
+	}
+
+	untrusted, err := TLSReachability(server.URL, x509.NewCertPool())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = untrusted(context.Background()); err == nil || !strings.Contains(err.Error(), "TLS") {
+		t.Fatalf("untrusted TLS check error=%v", err)
+	}
+}
+
+func TestRunnerRejectsInvalidDefinitions(t *testing.T) {
+	if _, err := NewRunner(0, time.Second, nil, nil); err == nil {
+		t.Fatal("zero interval should be rejected")
+	}
+	if _, err := NewRunner(time.Second, time.Second, map[string]CheckFunc{"": func(context.Context) error { return nil }}, nil); err == nil {
+		t.Fatal("empty probe name should be rejected")
+	}
+}