chore: establish local platform baseline

2026-08-26 06:41:54 +08:00
commit 29bdf4900f
575 changed files with 95644 additions and 0 deletions
@@ -0,0 +1,80 @@
+package observability
+
+import (
+	"context"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// RegisterBusinessMetrics exposes bounded-cardinality workflow state for
+// operational alerting. It is registered by the single worker process so
+// horizontally scaled API instances do not publish duplicate database gauges.
+func RegisterBusinessMetrics(pool *pgxpool.Pool) {
+	registerStatusCounts(pool, "conversations", "Current conversations by lifecycle status.", "conversation", []string{"PREPARING", "ROOM_ISSUED", "STARTING_AGENT", "ACTIVE", "STOPPING", "COMPLETED", "FAILED", "EXPIRED"})
+	registerStatusCounts(pool, "privacy_requests", "Current privacy requests by status.", "privacy_request", []string{"SUBMITTED", "VERIFYING", "EXECUTING", "COMPLETED", "PARTIAL_FAILED"})
+	registerStatusCounts(pool, "firmware_rollouts", "Current firmware rollouts by status.", "firmware_rollout", []string{"DRAFT", "RUNNING", "PAUSED", "COMPLETED", "ROLLED_BACK"})
+	registerStatusCounts(pool, "firmware_assignments", "Current firmware assignments by status.", "firmware_assignment", []string{"ASSIGNED", "DOWNLOADING", "INSTALLING", "CONFIRMED", "FAILED", "ROLLED_BACK"})
+	prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Namespace: "voice_platform",
+		Subsystem: "firmware",
+		Name:      "stalled_rollouts",
+		Help:      "Running firmware rollouts not updated for at least 15 minutes.",
+	}, func() float64 {
+		return queryCount(pool, "SELECT count(*)::float8 FROM firmware_rollout WHERE status='RUNNING' AND updated_at < now()-interval '15 minutes'")
+	}))
+	prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Namespace: "voice_platform", Subsystem: "provider", Name: "notification_backlog",
+		Help: "Provider callback events pending or failed for more than one minute.",
+	}, func() float64 {
+		return queryCount(pool, "SELECT count(*)::float8 FROM provider_event WHERE processing_status IN ('pending','failed') AND received_at<now()-interval '1 minute'")
+	}))
+	for _, window := range []struct {
+		label string
+		days  int
+	}{{"7d", 7}, {"30d", 30}} {
+		window := window
+		prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+			Namespace: "voice_platform", Subsystem: "security", Name: "certificates_expiring",
+			Help:        "Active managed certificate references expiring in the selected window.",
+			ConstLabels: prometheus.Labels{"window": window.label},
+		}, func() float64 {
+			return queryCount(pool, "SELECT count(*)::float8 FROM ops_resource_reference WHERE resource_type='CERTIFICATE' AND status='ACTIVE' AND expires_at<=now()+$1*interval '1 day'", window.days)
+		}))
+	}
+	for _, category := range []string{"BUSINESS", "TECHNICAL", "SECURITY", "COST"} {
+		category := category
+		prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+			Namespace: "voice_platform", Name: "alert_incidents",
+			Help:        "Current non-resolved alert incident count by category.",
+			ConstLabels: prometheus.Labels{"category": category},
+		}, func() float64 {
+			return queryCount(pool, "SELECT count(*)::float8 FROM alert_incident WHERE category=$1 AND status<>'RESOLVED'", category)
+		}))
+	}
+}
+
+func registerStatusCounts(pool *pgxpool.Pool, name, help, table string, statuses []string) {
+	for _, status := range statuses {
+		status := status
+		prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+			Namespace:   "voice_platform",
+			Name:        name,
+			Help:        help,
+			ConstLabels: prometheus.Labels{"status": status},
+		}, func() float64 {
+			return queryCount(pool, "SELECT count(*)::float8 FROM "+table+" WHERE status=$1", status)
+		}))
+	}
+}
+
+func queryCount(pool *pgxpool.Pool, query string, args ...any) float64 {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	var count float64
+	if err := pool.QueryRow(ctx, query, args...).Scan(&count); err != nil {
+		return 0
+	}
+	return count
+}