chore: establish local platform baseline

2026-08-26 06:41:54 +08:00
commit 29bdf4900f
575 changed files with 95644 additions and 0 deletions
@@ -0,0 +1,80 @@
+package observability
+
+import (
+	"context"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// RegisterBusinessMetrics exposes bounded-cardinality workflow state for
+// operational alerting. It is registered by the single worker process so
+// horizontally scaled API instances do not publish duplicate database gauges.
+func RegisterBusinessMetrics(pool *pgxpool.Pool) {
+	registerStatusCounts(pool, "conversations", "Current conversations by lifecycle status.", "conversation", []string{"PREPARING", "ROOM_ISSUED", "STARTING_AGENT", "ACTIVE", "STOPPING", "COMPLETED", "FAILED", "EXPIRED"})
+	registerStatusCounts(pool, "privacy_requests", "Current privacy requests by status.", "privacy_request", []string{"SUBMITTED", "VERIFYING", "EXECUTING", "COMPLETED", "PARTIAL_FAILED"})
+	registerStatusCounts(pool, "firmware_rollouts", "Current firmware rollouts by status.", "firmware_rollout", []string{"DRAFT", "RUNNING", "PAUSED", "COMPLETED", "ROLLED_BACK"})
+	registerStatusCounts(pool, "firmware_assignments", "Current firmware assignments by status.", "firmware_assignment", []string{"ASSIGNED", "DOWNLOADING", "INSTALLING", "CONFIRMED", "FAILED", "ROLLED_BACK"})
+	prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Namespace: "voice_platform",
+		Subsystem: "firmware",
+		Name:      "stalled_rollouts",
+		Help:      "Running firmware rollouts not updated for at least 15 minutes.",
+	}, func() float64 {
+		return queryCount(pool, "SELECT count(*)::float8 FROM firmware_rollout WHERE status='RUNNING' AND updated_at < now()-interval '15 minutes'")
+	}))
+	prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Namespace: "voice_platform", Subsystem: "provider", Name: "notification_backlog",
+		Help: "Provider callback events pending or failed for more than one minute.",
+	}, func() float64 {
+		return queryCount(pool, "SELECT count(*)::float8 FROM provider_event WHERE processing_status IN ('pending','failed') AND received_at<now()-interval '1 minute'")
+	}))
+	for _, window := range []struct {
+		label string
+		days  int
+	}{{"7d", 7}, {"30d", 30}} {
+		window := window
+		prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+			Namespace: "voice_platform", Subsystem: "security", Name: "certificates_expiring",
+			Help:        "Active managed certificate references expiring in the selected window.",
+			ConstLabels: prometheus.Labels{"window": window.label},
+		}, func() float64 {
+			return queryCount(pool, "SELECT count(*)::float8 FROM ops_resource_reference WHERE resource_type='CERTIFICATE' AND status='ACTIVE' AND expires_at<=now()+$1*interval '1 day'", window.days)
+		}))
+	}
+	for _, category := range []string{"BUSINESS", "TECHNICAL", "SECURITY", "COST"} {
+		category := category
+		prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+			Namespace: "voice_platform", Name: "alert_incidents",
+			Help:        "Current non-resolved alert incident count by category.",
+			ConstLabels: prometheus.Labels{"category": category},
+		}, func() float64 {
+			return queryCount(pool, "SELECT count(*)::float8 FROM alert_incident WHERE category=$1 AND status<>'RESOLVED'", category)
+		}))
+	}
+}
+
+func registerStatusCounts(pool *pgxpool.Pool, name, help, table string, statuses []string) {
+	for _, status := range statuses {
+		status := status
+		prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+			Namespace:   "voice_platform",
+			Name:        name,
+			Help:        help,
+			ConstLabels: prometheus.Labels{"status": status},
+		}, func() float64 {
+			return queryCount(pool, "SELECT count(*)::float8 FROM "+table+" WHERE status=$1", status)
+		}))
+	}
+}
+
+func queryCount(pool *pgxpool.Pool, query string, args ...any) float64 {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	var count float64
+	if err := pool.QueryRow(ctx, query, args...).Scan(&count); err != nil {
+		return 0
+	}
+	return count
+}
@@ -0,0 +1,35 @@
+package observability
+
+import (
+	"context"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	dependencyUp = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "voice_platform", Subsystem: "dependency", Name: "up",
+		Help: "Whether the last dependency check succeeded.",
+	}, []string{"service", "dependency"})
+	dependencyCheckDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: "voice_platform", Subsystem: "dependency", Name: "check_duration_seconds",
+		Help:    "Dependency check duration.",
+		Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2},
+	}, []string{"service", "dependency"})
+)
+
+func InstrumentDependencyCheck(serviceName, dependency string, check func(context.Context) error) func(context.Context) error {
+	return func(ctx context.Context) error {
+		startedAt := time.Now()
+		err := check(ctx)
+		dependencyCheckDuration.WithLabelValues(serviceName, dependency).Observe(time.Since(startedAt).Seconds())
+		if err != nil {
+			dependencyUp.WithLabelValues(serviceName, dependency).Set(0)
+			return err
+		}
+		dependencyUp.WithLabelValues(serviceName, dependency).Set(1)
+		return nil
+	}
+}
@@ -0,0 +1,15 @@
+package observability
+
+import (
+	"context"
+	"errors"
+	"testing"
+)
+
+func TestInstrumentDependencyCheckPreservesResult(t *testing.T) {
+	expected := errors.New("unavailable")
+	check := InstrumentDependencyCheck("test", "dependency", func(context.Context) error { return expected })
+	if err := check(context.Background()); !errors.Is(err, expected) {
+		t.Fatalf("error = %v", err)
+	}
+}
@@ -0,0 +1,49 @@
+package observability
+
+import (
+	"context"
+	"fmt"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+type TraceProvider struct{ provider *sdktrace.TracerProvider }
+
+func SetupTracing(ctx context.Context, serviceName, environment, endpoint string) (*TraceProvider, error) {
+	res, err := resource.New(ctx, resource.WithAttributes(
+		attribute.String("service.name", serviceName),
+		attribute.String("deployment.environment.name", environment),
+	))
+	if err != nil {
+		return nil, fmt.Errorf("create telemetry resource: %w", err)
+	}
+	options := []sdktrace.TracerProviderOption{sdktrace.WithResource(res)}
+	if endpoint != "" {
+		exporter, err := otlptracegrpc.New(ctx,
+			otlptracegrpc.WithEndpoint(endpoint),
+			otlptracegrpc.WithInsecure(),
+		)
+		if err != nil {
+			return nil, fmt.Errorf("create OTLP trace exporter: %w", err)
+		}
+		options = append(options, sdktrace.WithBatcher(exporter))
+	}
+	provider := sdktrace.NewTracerProvider(options...)
+	otel.SetTracerProvider(provider)
+	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
+		propagation.TraceContext{}, propagation.Baggage{},
+	))
+	return &TraceProvider{provider: provider}, nil
+}
+
+func (p *TraceProvider) Shutdown(ctx context.Context) error {
+	if p == nil || p.provider == nil {
+		return nil
+	}
+	return p.provider.Shutdown(ctx)
+}
@@ -0,0 +1,16 @@
+package observability
+
+import (
+	"context"
+	"testing"
+)
+
+func TestSetupTracingWithoutExporter(t *testing.T) {
+	provider, err := SetupTracing(context.Background(), "test-service", "test", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := provider.Shutdown(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+}