chore: establish local platform baseline

2026-08-26 06:41:54 +08:00
commit 29bdf4900f
575 changed files with 95644 additions and 0 deletions
@@ -0,0 +1,209 @@
+package externalprobe
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+)
+
+const maxJWKSBodyBytes = 1 << 20
+
+type CheckFunc func(context.Context) error
+
+type Observation struct {
+	ServiceName string
+	Dependency  string
+	Available   bool
+	CheckedAt   time.Time
+	Latency     time.Duration
+	Interval    time.Duration
+	ErrorCode   string
+}
+
+type Observer func(context.Context, Observation) error
+
+// Runner executes low-cost external dependency checks without making them part
+// of readiness. A temporary model or RTC network failure must remain visible to
+// operations without evicting otherwise healthy API and Worker pods.
+type Runner struct {
+	interval  time.Duration
+	timeout   time.Duration
+	checks    map[string]CheckFunc
+	logger    *slog.Logger
+	states    map[string]bool
+	observers []Observer
+	mu        sync.Mutex
+}
+
+func NewRunner(interval, timeout time.Duration, checks map[string]CheckFunc, logger *slog.Logger, observers ...Observer) (*Runner, error) {
+	if interval <= 0 || timeout <= 0 {
+		return nil, errors.New("external probe interval and timeout must be positive")
+	}
+	if logger == nil {
+		logger = slog.Default()
+	}
+	ownedChecks := make(map[string]CheckFunc, len(checks))
+	for name, check := range checks {
+		name = strings.TrimSpace(name)
+		if name == "" || check == nil {
+			return nil, errors.New("external probe name and check are required")
+		}
+		ownedChecks[name] = check
+	}
+	for _, observer := range observers {
+		if observer == nil {
+			return nil, errors.New("external probe observer is required")
+		}
+	}
+	return &Runner{interval: interval, timeout: timeout, checks: ownedChecks, logger: logger, states: map[string]bool{}, observers: append([]Observer(nil), observers...)}, nil
+}
+
+func (r *Runner) Run(ctx context.Context) {
+	r.runOnce(ctx)
+	ticker := time.NewTicker(r.interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			r.runOnce(ctx)
+		}
+	}
+}
+
+func (r *Runner) runOnce(ctx context.Context) {
+	for name, check := range r.checks {
+		checkContext, cancel := context.WithTimeout(ctx, r.timeout)
+		startedAt := time.Now()
+		err := check(checkContext)
+		checkedAt := time.Now().UTC()
+		latency := time.Since(startedAt)
+		cancel()
+		if latency < 0 {
+			latency = 0
+		}
+		r.mu.Lock()
+		previous, observed := r.states[name]
+		available := err == nil
+		r.states[name] = available
+		r.mu.Unlock()
+		if !available && (!observed || previous) {
+			r.logger.Warn("external dependency probe failed", "dependency", name, "error", err)
+		}
+		if available && observed && !previous {
+			r.logger.Info("external dependency probe recovered", "dependency", name)
+		}
+		observation := Observation{Dependency: name, Available: available, CheckedAt: checkedAt, Latency: latency, Interval: r.interval, ErrorCode: classifyError(err)}
+		for _, observer := range r.observers {
+			observerContext, observerCancel := context.WithTimeout(ctx, r.timeout)
+			observerErr := observer(observerContext, observation)
+			observerCancel()
+			if observerErr != nil && ctx.Err() == nil {
+				r.logger.Warn("external dependency probe observation failed", "dependency", name, "error", observerErr)
+			}
+		}
+	}
+}
+
+func classifyError(err error) string {
+	if err == nil {
+		return ""
+	}
+	if errors.Is(err, context.DeadlineExceeded) {
+		return "TIMEOUT"
+	}
+	if errors.Is(err, context.Canceled) {
+		return "CANCELED"
+	}
+	message := strings.ToLower(err.Error())
+	if strings.Contains(message, "certificate") || strings.Contains(message, "tls") || strings.Contains(message, "x509") {
+		return "TLS_ERROR"
+	}
+	if strings.Contains(message, "jwks") {
+		return "JWKS_INVALID"
+	}
+	var networkError net.Error
+	if errors.As(err, &networkError) {
+		return "NETWORK_ERROR"
+	}
+	return "PROBE_FAILED"
+}
+
+// JWKS validates that the configured endpoint returns a bounded JSON Web Key
+// Set with at least one key. It does not persist the keys or expose their body.
+func JWKS(endpoint string, client *http.Client) (CheckFunc, error) {
+	parsed, err := url.Parse(strings.TrimSpace(endpoint))
+	if err != nil || parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") {
+		return nil, errors.New("JWKS endpoint must be an absolute HTTP(S) URL")
+	}
+	if client == nil {
+		client = &http.Client{}
+	}
+	return func(ctx context.Context) error {
+		request, requestErr := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
+		if requestErr != nil {
+			return fmt.Errorf("create JWKS request: %w", requestErr)
+		}
+		request.Header.Set("Accept", "application/json")
+		response, requestErr := client.Do(request)
+		if requestErr != nil {
+			return fmt.Errorf("request JWKS: %w", requestErr)
+		}
+		defer response.Body.Close()
+		if response.StatusCode < 200 || response.StatusCode >= 300 {
+			_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4*1024))
+			return fmt.Errorf("JWKS endpoint returned HTTP %d", response.StatusCode)
+		}
+		var document struct {
+			Keys []json.RawMessage `json:"keys"`
+		}
+		decoder := json.NewDecoder(io.LimitReader(response.Body, maxJWKSBodyBytes+1))
+		if decodeErr := decoder.Decode(&document); decodeErr != nil {
+			return fmt.Errorf("decode JWKS document: %w", decodeErr)
+		}
+		if len(document.Keys) == 0 {
+			return errors.New("JWKS document contains no keys")
+		}
+		return nil
+	}, nil
+}
+
+// TLSReachability performs only DNS, TCP and certificate-verified TLS setup.
+// It intentionally sends no provider API request and therefore consumes no
+// model tokens, voice minutes or provider task quota.
+func TLSReachability(endpoint string, rootCAs *x509.CertPool) (CheckFunc, error) {
+	parsed, err := url.Parse(strings.TrimSpace(endpoint))
+	if err != nil || parsed.Scheme != "https" || parsed.Hostname() == "" {
+		return nil, errors.New("TLS probe endpoint must be an absolute HTTPS URL")
+	}
+	port := parsed.Port()
+	if port == "" {
+		port = "443"
+	}
+	address := net.JoinHostPort(parsed.Hostname(), port)
+	config := &tls.Config{
+		MinVersion: tls.VersionTLS12,
+		ServerName: parsed.Hostname(),
+		RootCAs:    rootCAs,
+	}
+	return func(ctx context.Context) error {
+		dialer := &tls.Dialer{NetDialer: &net.Dialer{}, Config: config.Clone()}
+		connection, dialErr := dialer.DialContext(ctx, "tcp", address)
+		if dialErr != nil {
+			return fmt.Errorf("establish verified TLS connection: %w", dialErr)
+		}
+		return connection.Close()
+	}, nil
+}