wangcong/story
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: establish local platform baseline

Browse Source
This commit is contained in:
Codex Local
2026-08-26 06:41:54 +08:00
commit 29bdf4900f
575 changed files with 95644 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/alerts/delivery.go
+156
View File
@@ -0,0 +1,156 @@
package alerts
import (
"context"
"encoding/json"
"errors"
"strings"
"time"
"data-terminal-cloud/internal/audit"
"data-terminal-cloud/internal/idempotency"
"data-terminal-cloud/internal/identity"
"data-terminal-cloud/internal/outbox"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
const alertDeliveryTestEventType = "alert.webhook_test_requested"
type DeliveryTestInput struct {
Reason string `json:"reason"`
}
type DeliveryTest struct {
ID string `json:"id"`
TestRef string `json:"test_ref"`
RoutingPolicyID string `json:"routing_policy_id"`
Category string `json:"category"`
Severity string `json:"severity"`
ReceiverGroup string `json:"receiver_group"`
RunbookRef string `json:"runbook_ref"`
Status string `json:"status"`
AttemptCount int `json:"attempt_count"`
ManualRetryCount int `json:"manual_retry_count"`
LastErrorCode string `json:"last_error_code,omitempty"`
LastManualRetryAt *time.Time `json:"last_manual_retry_at,omitempty"`
CreatedAt time.Time `json:"created_at"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
DeadAt *time.Time `json:"dead_at,omitempty"`
}
type deliveryTestPayload struct {
Action string `json:"action"`
TestRef string `json:"test_ref"`
TenantID string `json:"tenant_id"`
RoutingPolicyID string `json:"routing_policy_id"`
Category string `json:"category"`
Severity string `json:"severity"`
ReceiverGroup string `json:"receiver_group"`
RunbookRef string `json:"runbook_ref"`
Title string `json:"title"`
Summary string `json:"summary"`
OccurredAt time.Time `json:"occurred_at"`
}
func (s *Service) CreateDeliveryTest(ctx context.Context, principal identity.Principal, tenantPublicID, routingPolicyPublicID, key, requestID string, input DeliveryTestInput) (DeliveryTest, bool, error) {
input.Reason = strings.TrimSpace(input.Reason)
if !validUUID(routingPolicyPublicID) || input.Reason == "" || len([]rune(input.Reason)) > 500 {
return DeliveryTest{}, false, ErrInvalid
}
a, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_operator", "ops_approver", "ops_admin")
if err != nil {
return DeliveryTest{}, false, err
}
requestBody, _ := json.Marshal(struct {
RoutingPolicyID string `json:"routing_policy_id"`
Reason string `json:"reason"`
}{routingPolicyPublicID, input.Reason})
result, err := s.idem.Execute(ctx, idempotency.Request{
TenantID: &a.tenantID, ActorKey: "ops:" + a.userPublicID,
Scope: "POST /api/v1/ops/alert-routing-policies/{id}/delivery-tests", Key: key,
RequestHash: idempotency.Hash(requestBody), ExpiresAfter: 24 * time.Hour,
}, func(op context.Context, tx pgx.Tx) (idempotency.Result, error) {
var routeInternalID, category, severity, receiverGroup, runbookRef string
err := tx.QueryRow(op, `SELECT id::text,category,severity,receiver_group,runbook_ref FROM alert_routing_policy WHERE tenant_id=$1::uuid AND public_id=$2::uuid AND status='ACTIVE'`, a.tenantID, routingPolicyPublicID).
Scan(&routeInternalID, &category, &severity, &receiverGroup, &runbookRef)
if errors.Is(err, pgx.ErrNoRows) {
return idempotency.Result{}, ErrNotFound
}
if err != nil {
return idempotency.Result{}, err
}
testRef := uuid.NewString()
payload, _ := json.Marshal(deliveryTestPayload{
Action: "TEST", TestRef: testRef, TenantID: a.tenantPublicID, RoutingPolicyID: routingPolicyPublicID,
Category: category, Severity: severity, ReceiverGroup: receiverGroup, RunbookRef: runbookRef,
Title: "告警接收器联通测试", Summary: "这是一条由运营工作台发起的受控测试通知，不代表真实故障。", OccurredAt: time.Now().UTC(),
})
dedupe := "alert-delivery-test:" + testRef
eventID, err := outbox.NewStore(s.pool).Enqueue(op, tx, outbox.NewEvent{
TenantID: &a.tenantID, AggregateType: "alert_delivery_test", AggregateID: testRef,
EventType: alertDeliveryTestEventType, DeduplicationKey: &dedupe, Payload: payload,
})
if err != nil {
return idempotency.Result{}, err
}
if err = s.audit.Record(op, tx, audit.Event{
TenantID: &a.tenantID, ActorType: "user", ActorPublicID: &a.userPublicID,
Action: "alert.delivery_test.create", ResourceType: "alert_delivery_test", ResourceID: &eventID,
RequestID: requestID, Purpose: input.Reason, Result: "success",
Detail: map[string]any{"routing_policy_id": routingPolicyPublicID, "receiver_group": receiverGroup, "test_ref": testRef},
}); err != nil {
return idempotency.Result{}, err
}
item, err := getDeliveryTestTx(op, tx, a.tenantID, eventID)
if err != nil {
return idempotency.Result{}, err
}
body, _ := json.Marshal(item)
return idempotency.Result{StatusCode: 201, Body: body, ResourceID: &eventID}, nil
})
if err != nil {
return DeliveryTest{}, false, err
}
var item DeliveryTest
if err = json.Unmarshal(result.Body, &item); err != nil {
return DeliveryTest{}, false, err
}
return item, !result.Replayed, nil
}
func (s *Service) ListDeliveryTests(ctx context.Context, principal identity.Principal, tenantPublicID string) ([]DeliveryTest, error) {
a, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_viewer", "ops_operator", "ops_approver", "ops_admin")
if err != nil {
return nil, err
}
rows, err := s.pool.Query(ctx, deliveryTestSelect+` WHERE o.tenant_id=$1::uuid AND o.event_type=$2 ORDER BY o.created_at DESC LIMIT 50`, a.tenantID, alertDeliveryTestEventType)
if err != nil {
return nil, err
}
defer rows.Close()
items := []DeliveryTest{}
for rows.Next() {
item, scanErr := scanDeliveryTest(rows)
if scanErr != nil {
return nil, scanErr
}
items = append(items, item)
}
return items, rows.Err()
}
const deliveryTestSelect = `SELECT o.public_id::text,o.aggregate_id::text,COALESCE(o.payload->>'routing_policy_id',''),COALESCE(o.payload->>'category',''),COALESCE(o.payload->>'severity',''),COALESCE(o.payload->>'receiver_group',''),COALESCE(o.payload->>'runbook_ref',''),CASE o.status WHEN 'pending' THEN 'PENDING' WHEN 'processing' THEN 'PROCESSING' WHEN 'completed' THEN 'DELIVERED' ELSE 'FAILED' END,o.attempt_count,o.manual_retry_count,COALESCE(o.last_error_code,''),o.last_manual_retry_at,o.created_at,o.completed_at,o.dead_at FROM outbox_event o`
type deliveryTestScanner interface{ Scan(...any) error }
func scanDeliveryTest(row deliveryTestScanner) (DeliveryTest, error) {
var item DeliveryTest
err := row.Scan(&item.ID, &item.TestRef, &item.RoutingPolicyID, &item.Category, &item.Severity, &item.ReceiverGroup, &item.RunbookRef, &item.Status, &item.AttemptCount, &item.ManualRetryCount, &item.LastErrorCode, &item.LastManualRetryAt, &item.CreatedAt, &item.CompletedAt, &item.DeadAt)
return item, err
}
func getDeliveryTestTx(ctx context.Context, tx pgx.Tx, tenantID, eventID string) (DeliveryTest, error) {
return scanDeliveryTest(tx.QueryRow(ctx, deliveryTestSelect+` WHERE o.tenant_id=$1::uuid AND o.id=$2::uuid AND o.event_type=$3`, tenantID, eventID, alertDeliveryTestEventType))
}
internal/alerts/detection_integration_test.go
+104
View File
@@ -0,0 +1,104 @@
package alerts
import (
"context"
"errors"
"io"
"log/slog"
"os"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestScannerDetectionDedupSilenceAndReopenLifecycle(t *testing.T) {
databaseURL := os.Getenv("TEST_DATABASE_URL")
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
scanner, err := NewScanner(pool, logger, time.Second)
if err != nil {
t.Fatal(err)
}
if _, err := NewScanner(nil, logger, time.Second); err == nil {
t.Fatal("nil scanner pool accepted")
}
if _, err := NewScanner(pool, nil, time.Second); err == nil {
t.Fatal("nil scanner logger accepted")
}
if _, err := NewScanner(pool, logger, 0); err == nil {
t.Fatal("zero scanner interval accepted")
}
now := time.Now().UTC().Truncate(time.Second)
tenantID := "10000000-0000-0000-0000-000000000001"
resourceID := uuid.NewString()
fingerprint := "core-detect-" + uuid.NewString()
silenceID := uuid.NewString()
defer func() {
_, _ = pool.Exec(ctx, `DELETE FROM alert_incident WHERE tenant_id=$1::uuid AND fingerprint=$2`, tenantID, fingerprint)
_, _ = pool.Exec(ctx, `DELETE FROM alert_silence WHERE id=$1::uuid`, silenceID)
}()
if err := scanner.detect(ctx, Signal{TenantID: tenantID, RuleCode: "bad code", Category: "TECHNICAL", Severity: "LOW", Fingerprint: fingerprint, Title: "标题", Summary: "摘要"}, now); !errors.Is(err, ErrInvalid) {
t.Fatalf("invalid signal error=%v", err)
}
signal := Signal{
TenantID: tenantID, RuleCode: " core_detect_test ", Category: "TECHNICAL", Severity: "LOW",
Fingerprint: fingerprint, Title: "核心告警发现", Summary: "首次发现合成告警。",
ResourceType: "device", ResourcePublicID: resourceID, RequestID: "alerts-core-detect",
}
if err := scanner.detect(ctx, signal, now); err != nil {
t.Fatal(err)
}
signal.Summary = "相同指纹再次命中。"
if err := scanner.detect(ctx, signal, now.Add(time.Minute)); err != nil {
t.Fatal(err)
}
var status, summary string
var occurrences, rowVersion int64
if err := pool.QueryRow(ctx, `SELECT status,summary,occurrence_count,row_version FROM alert_incident WHERE tenant_id=$1::uuid AND fingerprint=$2`, tenantID, fingerprint).Scan(&status, &summary, &occurrences, &rowVersion); err != nil {
t.Fatal(err)
}
if status != "OPEN" || summary != signal.Summary || occurrences != 2 || rowVersion != 2 {
t.Fatalf("open status=%s summary=%q occurrences=%d version=%d", status, summary, occurrences, rowVersion)
}
if _, err := pool.Exec(ctx, `
INSERT INTO alert_silence(id,public_id,tenant_id,rule_code,category,resource_type,resource_public_id,matcher,reason,starts_at,ends_at,created_by)
VALUES($1::uuid,$2::uuid,$3::uuid,'CORE_DETECT_TEST','TECHNICAL','device',$4::uuid,'{}'::jsonb,'核心检测静默窗口',$5,$6,'10000000-0000-0000-0000-000000000101'::uuid)`,
silenceID, uuid.NewString(), tenantID, resourceID, now.Add(-time.Minute), now.Add(time.Hour)); err != nil {
t.Fatal(err)
}
signal.Summary = "静默窗口内再次命中。"
if err := scanner.detect(ctx, signal, now.Add(2*time.Minute)); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `UPDATE alert_silence SET revoked_at=$2,revoked_by='10000000-0000-0000-0000-000000000101'::uuid WHERE id=$1::uuid`, silenceID, now.Add(3*time.Minute)); err != nil {
t.Fatal(err)
}
signal.Summary = "静默撤销后重新进入队列。"
if err := scanner.detect(ctx, signal, now.Add(4*time.Minute)); err != nil {
t.Fatal(err)
}
var actionCount int
if err := pool.QueryRow(ctx, `SELECT ai.status,ai.summary,ai.occurrence_count,ai.row_version,count(aa.id) FROM alert_incident ai LEFT JOIN alert_action aa ON aa.incident_id=ai.id WHERE ai.tenant_id=$1::uuid AND ai.fingerprint=$2 GROUP BY ai.id`, tenantID, fingerprint).Scan(&status, &summary, &occurrences, &rowVersion, &actionCount); err != nil {
t.Fatal(err)
}
if status != "OPEN" || summary != signal.Summary || occurrences != 4 || rowVersion != 4 || actionCount != 3 {
t.Fatalf("reopened status=%s summary=%q occurrences=%d version=%d actions=%d", status, summary, occurrences, rowVersion, actionCount)
}
cancelled, cancel := context.WithCancel(context.Background())
cancel()
scanner.Run(cancelled)
if nullUUID("") != nil || nullUUID(resourceID) != resourceID {
t.Fatal("null UUID normalization failed")
}
}
internal/alerts/scanner.go
+339
View File
@@ -0,0 +1,339 @@
package alerts
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"strings"
"time"
"data-terminal-cloud/internal/outbox"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
type Signal struct {
TenantID, RuleCode, Category, Severity, Fingerprint, Title, Summary string
ResourceType, ResourcePublicID, RequestID string
}
type Scanner struct {
pool *pgxpool.Pool
logger *slog.Logger
interval time.Duration
now func() time.Time
outbox *outbox.Store
}
func NewScanner(pool *pgxpool.Pool, logger *slog.Logger, interval time.Duration) (*Scanner, error) {
if pool == nil || logger == nil || interval <= 0 {
return nil, errors.New("alert scanner dependencies are required")
}
return &Scanner{pool: pool, logger: logger, interval: interval, now: time.Now, outbox: outbox.NewStore(pool)}, nil
}
func (s *Scanner) Run(ctx context.Context) {
ticker := time.NewTicker(s.interval)
defer ticker.Stop()
for {
if count, err := s.ScanOnce(ctx); err != nil && !errors.Is(err, context.Canceled) {
s.logger.Error("alert scan failed", "error", err)
} else if count > 0 {
s.logger.Info("alert signals processed", "count", count)
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
func (s *Scanner) ScanOnce(ctx context.Context) (int, error) {
now := s.now().UTC()
signals := []Signal{}
queries := []struct {
sql string
build func(row pgx.Row) (Signal, error)
}{
{`SELECT tenant_id::text,count(*) FROM outbox_event WHERE status='dead' GROUP BY tenant_id`, func(r pgx.Row) (Signal, error) {
var tenant string
var count int64
e := r.Scan(&tenant, &count)
return Signal{TenantID: tenant, RuleCode: "OUTBOX_DEAD_LETTER", Category: "TECHNICAL", Severity: "HIGH", Fingerprint: "outbox-dead", Title: "可靠任务出现死信", Summary: fmt.Sprintf("当前客户组织存在%d条死信待办，请按错误码修复并受控重放。", count), ResourceType: "outbox_queue"}, e
}},
{`SELECT tenant_id::text,public_id::text,status FROM privacy_request WHERE status='PARTIAL_FAILED' OR (request_type='DELETE' AND status<>'COMPLETED' AND created_at<now()-interval '1 hour')`, func(r pgx.Row) (Signal, error) {
var tenant, id, status string
e := r.Scan(&tenant, &id, &status)
return Signal{TenantID: tenant, RuleCode: "PRIVACY_DELETE_OVERDUE", Category: "BUSINESS", Severity: "CRITICAL", Fingerprint: "privacy-delete:" + id, Title: "隐私删除需要人工处置", Summary: "主体删除未在预期时间内完成全部子系统对账，必须立即核查失败子任务。", ResourceType: "privacy_request", ResourcePublicID: id}, e
}},
{`SELECT tenant_id::text,public_id::text,COALESCE(last_error_code,'UNKNOWN') FROM upload_batch WHERE status='FAILED'`, func(r pgx.Row) (Signal, error) {
var tenant, id, code string
e := r.Scan(&tenant, &id, &code)
return Signal{TenantID: tenant, RuleCode: "MEDIA_UPLOAD_FAILED", Category: "BUSINESS", Severity: "MEDIUM", Fingerprint: "media-upload:" + id, Title: "录音上传批次校验失败", Summary: "录音上传批次进入失败终态，错误码：" + code + "。", ResourceType: "upload_batch", ResourcePublicID: id}, e
}},
{`SELECT fa.tenant_id::text,fr.public_id::text,count(*) FROM firmware_assignment fa JOIN firmware_rollout fr ON fr.id=fa.rollout_id WHERE fa.status='FAILED' GROUP BY fa.tenant_id,fr.public_id`, func(r pgx.Row) (Signal, error) {
var tenant, id string
var count int64
e := r.Scan(&tenant, &id, &count)
return Signal{TenantID: tenant, RuleCode: "FIRMWARE_ROLLOUT_FAILURE", Category: "TECHNICAL", Severity: "HIGH", Fingerprint: "firmware-rollout:" + id, Title: "固件灰度出现失败设备", Summary: fmt.Sprintf("固件灰度批次已有%d台设备失败，应暂停后续分配并核查回滚条件。", count), ResourceType: "firmware_rollout", ResourcePublicID: id}, e
}},
{`WITH spend AS (SELECT qp.tenant_id,qp.id,qp.currency,qp.daily_limit_micros,COALESCE(SUM(ccl.cost_micros) FILTER(WHERE cue.occurred_at>=date_trunc('day',now() AT TIME ZONE qp.billing_timezone) AT TIME ZONE qp.billing_timezone),0)::bigint used FROM usage_quota_policy qp LEFT JOIN conversation_cost_line ccl ON ccl.tenant_id=qp.tenant_id AND ccl.currency=qp.currency LEFT JOIN conversation_usage_event cue ON cue.id=ccl.usage_event_id WHERE qp.scope_type='TENANT' AND qp.status='ACTIVE' GROUP BY qp.tenant_id,qp.id) SELECT tenant_id::text,currency,used,daily_limit_micros FROM spend WHERE used*100>=daily_limit_micros*80`, func(r pgx.Row) (Signal, error) {
var tenant, currency string
var used, limit int64
e := r.Scan(&tenant, &currency, &used, &limit)
return Signal{TenantID: tenant, RuleCode: "COST_DAILY_QUOTA_NEAR", Category: "COST", Severity: "MEDIUM", Fingerprint: "cost-daily-quota", Title: "日费用接近硬额度", Summary: fmt.Sprintf("本账期已使用%s %.6f，达到日额度的80%%以上。", currency, float64(used)/1e6), ResourceType: "usage_quota_policy"}, e
}},
{`SELECT tenant_id::text,public_id::text,status FROM conversation WHERE (status='STARTING_AGENT' AND updated_at<now()-interval '5 minutes') OR (status='STOPPING' AND updated_at<now()-interval '5 minutes')`, func(r pgx.Row) (Signal, error) {
var tenant, id, status string
e := r.Scan(&tenant, &id, &status)
return Signal{TenantID: tenant, RuleCode: "VOICE_TASK_STUCK", Category: "TECHNICAL", Severity: "HIGH", Fingerprint: "voice-task-stuck:" + id, Title: "语音任务长时间停留在中间状态", Summary: "通话持续停留在" + status + "，自动恢复未及时收敛，需要检查供应商任务和恢复记录。", ResourceType: "conversation", ResourcePublicID: id}, e
}},
{`SELECT tenant_id::text,provider,COALESCE(last_error_code,'PROVIDER_TEMPORARY_ERROR'),consecutive_failures FROM provider_dependency_state WHERE status='UNAVAILABLE'`, func(r pgx.Row) (Signal, error) {
var tenant, provider, code string
var failures int
e := r.Scan(&tenant, &provider, &code, &failures)
return Signal{TenantID: tenant, RuleCode: "PROVIDER_UNAVAILABLE", Category: "TECHNICAL", Severity: "CRITICAL", Fingerprint: "provider-unavailable:" + provider, Title: "语音供应商暂时不可用", Summary: fmt.Sprintf("供应商%s连续临时失败%d次，错误码%s；新启动已暂停，停止与回收仍继续重试。", provider, failures, code), ResourceType: "provider_dependency"}, e
}},
{`SELECT tenant_id::text,provider,count(*),min(received_at) FROM provider_event WHERE processing_status IN ('pending','failed') AND received_at<now()-interval '1 minute' GROUP BY tenant_id,provider`, func(r pgx.Row) (Signal, error) {
var tenant, provider string
var count int64
var oldest time.Time
e := r.Scan(&tenant, &provider, &count, &oldest)
return Signal{TenantID: tenant, RuleCode: "PROVIDER_NOTIFICATION_BACKLOG", Category: "TECHNICAL", Severity: "HIGH", Fingerprint: "provider-notification-backlog:" + provider, Title: "供应商状态通知处理积压", Summary: fmt.Sprintf("供应商%s有%d条通知超过1分钟仍未完成处理，最早接收时间为%s。", provider, count, oldest.UTC().Format(time.RFC3339)), ResourceType: "provider_event_queue"}, e
}},
{`SELECT tenant_id::text,public_id::text,display_name,expires_at FROM ops_resource_reference WHERE resource_type='CERTIFICATE' AND status='ACTIVE' AND expires_at<=now()+interval '30 days'`, func(r pgx.Row) (Signal, error) {
var tenant, id, name string
var expires time.Time
e := r.Scan(&tenant, &id, &name, &expires)
severity := "HIGH"
if !expires.After(now.Add(7 * 24 * time.Hour)) {
severity = "CRITICAL"
}
return Signal{TenantID: tenant, RuleCode: "CERTIFICATE_EXPIRING", Category: "SECURITY", Severity: severity, Fingerprint: "certificate-expiring:" + id, Title: "受控证书即将到期", Summary: fmt.Sprintf("证书%s将在%s到期，请按轮换手册更新引用并验证回退路径。", name, expires.UTC().Format(time.RFC3339)), ResourceType: "ops_resource_reference", ResourcePublicID: id}, e
}},
}
for _, q := range queries {
rows, err := s.pool.Query(ctx, q.sql)
if err != nil {
return len(signals), err
}
for rows.Next() {
signal, scanErr := q.build(rows)
if scanErr != nil {
rows.Close()
return len(signals), scanErr
}
signals = append(signals, signal)
}
if err = rows.Err(); err != nil {
rows.Close()
return len(signals), err
}
rows.Close()
}
processed := 0
for _, signal := range signals {
if err := s.detect(ctx, signal, now); err != nil {
return processed, err
}
processed++
}
if err := s.escalate(ctx, now); err != nil {
return processed, err
}
return processed, nil
}
func (s *Scanner) detect(ctx context.Context, signal Signal, now time.Time) error {
signal.RuleCode = strings.ToUpper(strings.TrimSpace(signal.RuleCode))
if !rulePattern.MatchString(signal.RuleCode) || !validCategory(signal.Category) || !validSeverity(signal.Severity) || signal.Fingerprint == "" || signal.Title == "" || signal.Summary == "" {
return ErrInvalid
}
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return err
}
defer tx.Rollback(context.Background())
if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, signal.TenantID+":"+signal.Fingerprint); err != nil {
return err
}
var tenantPublicID string
if err = tx.QueryRow(ctx, `SELECT public_id::text FROM tenant WHERE id=$1::uuid`, signal.TenantID).Scan(&tenantPublicID); err != nil {
return err
}
var routeID, receiverGroup, runbookRef *string
_ = tx.QueryRow(ctx, `SELECT id::text,receiver_group,runbook_ref FROM alert_routing_policy WHERE tenant_id=$1::uuid AND category=$2 AND severity=$3 AND status='ACTIVE'`, signal.TenantID, signal.Category, signal.Severity).Scan(&routeID, &receiverGroup, &runbookRef)
var silenceUntil *time.Time
err = tx.QueryRow(ctx, `SELECT max(ends_at) FROM alert_silence WHERE tenant_id=$1::uuid AND revoked_at IS NULL AND starts_at<=$2 AND ends_at>$2 AND (rule_code IS NULL OR rule_code=$3) AND (category IS NULL OR category=$4) AND (resource_public_id IS NULL OR resource_public_id=$5::uuid)`, signal.TenantID, now, signal.RuleCode, signal.Category, nullUUID(signal.ResourcePublicID)).Scan(&silenceUntil)
if err != nil {
return err
}
status := "OPEN"
if silenceUntil != nil {
status = "SILENCED"
}
var incidentID, previousStatus string
err = tx.QueryRow(ctx, `SELECT id::text,status FROM alert_incident WHERE tenant_id=$1::uuid AND fingerprint=$2 AND status<>'RESOLVED' FOR UPDATE`, signal.TenantID, signal.Fingerprint).Scan(&incidentID, &previousStatus)
if errors.Is(err, pgx.ErrNoRows) {
incidentPublicID := uuid.NewString()
err = tx.QueryRow(ctx, `INSERT INTO alert_incident(public_id,tenant_id,rule_code,category,severity,status,fingerprint,title,summary,resource_type,resource_public_id,correlation_request_id,routing_policy_id,first_detected_at,last_detected_at,silenced_until) VALUES($1::uuid,$2::uuid,$3,$4,$5,$6,$7,$8,$9,NULLIF($10,''),$11::uuid,NULLIF($12,''),$13::uuid,$14,$14,$15) RETURNING id::text`, incidentPublicID, signal.TenantID, signal.RuleCode, signal.Category, signal.Severity, status, signal.Fingerprint, signal.Title, signal.Summary, signal.ResourceType, nullUUID(signal.ResourcePublicID), signal.RequestID, routeID, now, silenceUntil).Scan(&incidentID)
if err != nil {
return err
}
_, err = tx.Exec(ctx, `INSERT INTO alert_action(tenant_id,incident_id,action_type,to_status,actor_type,note) VALUES($1::uuid,$2::uuid,'DETECTED',$3,'system',$4)`, signal.TenantID, incidentID, status, "监测规则 "+signal.RuleCode+" 首次命中")
if err != nil {
return err
}
if status == "SILENCED" {
if _, err = tx.Exec(ctx, `INSERT INTO alert_action(tenant_id,incident_id,action_type,to_status,actor_type,note) VALUES($1::uuid,$2::uuid,'SILENCED','SILENCED','system','首次命中时已匹配有效静默策略')`, signal.TenantID, incidentID); err != nil {
return err
}
} else if err = s.enqueueNotification(ctx, tx, alertNotification{
Action: "DETECTED", IncidentID: incidentPublicID, TenantID: tenantPublicID,
RuleCode: signal.RuleCode, Category: signal.Category, Severity: signal.Severity,
Status: status, Title: signal.Title, Summary: signal.Summary,
ResourceType: signal.ResourceType, ResourceID: signal.ResourcePublicID,
ReceiverGroup: valueOrEmpty(receiverGroup), RunbookRef: valueOrEmpty(runbookRef),
OccurredAt: now,
}); err != nil {
return err
}
return tx.Commit(ctx)
}
if err != nil {
return err
}
nextStatus := previousStatus
if silenceUntil != nil {
nextStatus = "SILENCED"
} else if previousStatus == "SILENCED" {
nextStatus = "OPEN"
}
_, err = tx.Exec(ctx, `UPDATE alert_incident SET status=$2,last_detected_at=$3,occurrence_count=occurrence_count+1,summary=$4,routing_policy_id=COALESCE($5::uuid,routing_policy_id),silenced_until=$6,row_version=row_version+1,updated_at=$3 WHERE id=$1::uuid`, incidentID, nextStatus, now, signal.Summary, routeID, silenceUntil)
if err != nil {
return err
}
if nextStatus != previousStatus {
action := "REOPENED"
note := "告警静默窗口已结束，重新进入待处置队列"
if nextStatus == "SILENCED" {
action = "SILENCED"
note = "告警命中有效静默策略"
}
if _, err = tx.Exec(ctx, `INSERT INTO alert_action(tenant_id,incident_id,action_type,from_status,to_status,actor_type,note) VALUES($1::uuid,$2::uuid,$3,$4,$5,'system',$6)`, signal.TenantID, incidentID, action, previousStatus, nextStatus, note); err != nil {
return err
}
}
return tx.Commit(ctx)
}
func (s *Scanner) escalate(ctx context.Context, now time.Time) error {
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
if err != nil {
return err
}
defer tx.Rollback(context.Background())
rows, err := tx.Query(ctx, `
WITH due AS (
SELECT ai.id,ai.tenant_id,ai.public_id,ai.rule_code,ai.category,ai.severity,ai.status,
ai.title,ai.summary,ai.resource_type,ai.resource_public_id,
t.public_id AS tenant_public_id,arp.escalation_receiver_group,arp.runbook_ref
FROM alert_incident ai
JOIN alert_routing_policy arp ON arp.id=ai.routing_policy_id
JOIN tenant t ON t.id=ai.tenant_id
WHERE ai.status IN('OPEN','ACKNOWLEDGED') AND ai.escalated_at IS NULL
AND ai.first_detected_at+(arp.escalation_after_minutes||' minutes')::interval<=$1
FOR UPDATE OF ai
), updated AS (
UPDATE alert_incident ai SET escalated_at=$1,row_version=row_version+1,updated_at=$1
FROM due WHERE ai.id=due.id
RETURNING ai.id,ai.tenant_id,ai.public_id,ai.rule_code,ai.category,ai.severity,ai.status,
ai.title,ai.summary,ai.resource_type,ai.resource_public_id
), actions AS (
INSERT INTO alert_action(tenant_id,incident_id,action_type,from_status,to_status,actor_type,note)
SELECT tenant_id,id,'ESCALATED',status,status,'system','超过升级时限，已进入升级接收组' FROM updated
RETURNING incident_id
)
SELECT u.public_id::text,d.tenant_public_id::text,u.rule_code,u.category,u.severity,u.status,
u.title,u.summary,COALESCE(u.resource_type,''),COALESCE(u.resource_public_id::text,''),
d.escalation_receiver_group,d.runbook_ref
FROM updated u JOIN due d ON d.id=u.id JOIN actions a ON a.incident_id=u.id`, now)
if err != nil {
return err
}
notifications := []alertNotification{}
for rows.Next() {
var item alertNotification
item.Action = "ESCALATED"
item.OccurredAt = now
if err = rows.Scan(&item.IncidentID, &item.TenantID, &item.RuleCode, &item.Category, &item.Severity, &item.Status, &item.Title, &item.Summary, &item.ResourceType, &item.ResourceID, &item.ReceiverGroup, &item.RunbookRef); err != nil {
rows.Close()
return err
}
notifications = append(notifications, item)
}
if err = rows.Err(); err != nil {
rows.Close()
return err
}
rows.Close()
for _, item := range notifications {
if err = s.enqueueNotification(ctx, tx, item); err != nil {
return err
}
}
return tx.Commit(ctx)
}
type alertNotification struct {
Action string `json:"action"`
IncidentID string `json:"incident_id"`
TenantID string `json:"tenant_id"`
RuleCode string `json:"rule_code"`
Category string `json:"category"`
Severity string `json:"severity"`
Status string `json:"status"`
Title string `json:"title"`
Summary string `json:"summary"`
ResourceType string `json:"resource_type,omitempty"`
ResourceID string `json:"resource_id,omitempty"`
ReceiverGroup string `json:"receiver_group,omitempty"`
RunbookRef string `json:"runbook_ref,omitempty"`
OccurredAt time.Time `json:"occurred_at"`
}
func (s *Scanner) enqueueNotification(ctx context.Context, tx pgx.Tx, item alertNotification) error {
payload, err := json.Marshal(item)
if err != nil {
return err
}
tenantInternalID := item.TenantID
if err = tx.QueryRow(ctx, `SELECT id::text FROM tenant WHERE public_id=$1::uuid`, item.TenantID).Scan(&tenantInternalID); err != nil {
return err
}
deduplicationKey := "alert-webhook:" + strings.ToLower(item.Action) + ":" + item.IncidentID
_, err = s.outbox.Enqueue(ctx, tx, outbox.NewEvent{
TenantID: &tenantInternalID, AggregateType: "alert_incident", AggregateID: item.IncidentID,
EventType: "alert.webhook_requested", DeduplicationKey: &deduplicationKey, Payload: payload,
})
return err
}
func valueOrEmpty(value *string) string {
if value == nil {
return ""
}
return *value
}
func nullUUID(value string) any {
if strings.TrimSpace(value) == "" {
return nil
}
return value
}
internal/alerts/scanner_integration_test.go
+57
View File
@@ -0,0 +1,57 @@
package alerts
import (
"context"
"io"
"log/slog"
"os"
"testing"
"time"
"data-terminal-cloud/internal/database"
"github.com/google/uuid"
)
func TestScannerDetectsExpiringCertificateInventory(t *testing.T) {
databaseURL := os.Getenv("TEST_DATABASE_URL")
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := database.Open(ctx, databaseURL, "certificate-alert-test")
if err != nil {
t.Fatal(err)
}
defer pool.Close()
tenantID := "10000000-0000-0000-0000-000000000001"
var userID string
if err = pool.QueryRow(ctx, `SELECT id::text FROM user_account WHERE tenant_id=$1::uuid ORDER BY created_at LIMIT 1`, tenantID).Scan(&userID); err != nil {
t.Fatal(err)
}
publicID := uuid.NewString()
name := "cert-manager://integration/" + publicID
now := time.Now().UTC().Truncate(time.Second)
if _, err = pool.Exec(ctx, `INSERT INTO ops_resource_reference(public_id,tenant_id,resource_type,reference_name,display_name,verified_at,expires_at,created_by) VALUES($1::uuid,$2::uuid,'CERTIFICATE',$3,'集成测试证书',$4,$5,$6::uuid)`, publicID, tenantID, name, now.Add(-time.Hour), now.Add(5*24*time.Hour), userID); err != nil {
t.Fatal(err)
}
defer func() {
_, _ = pool.Exec(context.Background(), `DELETE FROM alert_incident WHERE tenant_id=$1::uuid AND fingerprint=$2`, tenantID, "certificate-expiring:"+publicID)
_, _ = pool.Exec(context.Background(), `DELETE FROM ops_resource_reference WHERE tenant_id=$1::uuid AND public_id=$2::uuid`, tenantID, publicID)
}()
scanner, err := NewScanner(pool, slog.New(slog.NewTextHandler(io.Discard, nil)), time.Second)
if err != nil {
t.Fatal(err)
}
scanner.now = func() time.Time { return now }
if _, err = scanner.ScanOnce(ctx); err != nil {
t.Fatal(err)
}
var rule, category, severity, resourceID string
if err = pool.QueryRow(ctx, `SELECT rule_code,category,severity,resource_public_id::text FROM alert_incident WHERE tenant_id=$1::uuid AND fingerprint=$2`, tenantID, "certificate-expiring:"+publicID).Scan(&rule, &category, &severity, &resourceID); err != nil {
t.Fatal(err)
}
if rule != "CERTIFICATE_EXPIRING" || category != "SECURITY" || severity != "CRITICAL" || resourceID != publicID {
t.Fatalf("rule=%s category=%s severity=%s resource=%s", rule, category, severity, resourceID)
}
}
internal/alerts/service.go
+528
View File
@@ -0,0 +1,528 @@
package alerts
import (
"context"
"encoding/json"
"errors"
"regexp"
"strings"
"time"
"data-terminal-cloud/internal/audit"
"data-terminal-cloud/internal/idempotency"
"data-terminal-cloud/internal/identity"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
var (
ErrInvalid = errors.New("alert input is invalid")
ErrAccessDenied = errors.New("alert operations access denied")
ErrNotFound = errors.New("alert resource not found")
ErrStateConflict = errors.New("alert state conflict")
ErrVersionConflict = errors.New("alert version conflict")
)
type RoutingInput struct {
Category string `json:"category"`
Severity string `json:"severity"`
ReceiverGroup string `json:"receiver_group"`
ResponseMinutes int `json:"response_minutes"`
EscalationAfterMinutes int `json:"escalation_after_minutes"`
EscalationReceiverGroup string `json:"escalation_receiver_group"`
RunbookRef string `json:"runbook_ref"`
ChangeSummary string `json:"change_summary"`
}
type RoutingPolicy struct {
ID string `json:"id"`
TenantID string `json:"tenant_id"`
Category string `json:"category"`
Severity string `json:"severity"`
VersionNo int `json:"version_no"`
Status string `json:"status"`
ReceiverGroup string `json:"receiver_group"`
ResponseMinutes int `json:"response_minutes"`
EscalationAfterMinutes int `json:"escalation_after_minutes"`
EscalationReceiverGroup string `json:"escalation_receiver_group"`
RunbookRef string `json:"runbook_ref"`
ChangeSummary string `json:"change_summary"`
CreatedBy string `json:"created_by"`
RetiredAt *time.Time `json:"retired_at,omitempty"`
RowVersion int64 `json:"row_version"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
type SilenceInput struct {
RuleCode string `json:"rule_code,omitempty"`
Category string `json:"category,omitempty"`
ResourceType string `json:"resource_type,omitempty"`
ResourcePublicID string `json:"resource_id,omitempty"`
Matcher map[string]any `json:"matcher"`
Reason string `json:"reason"`
StartsAt time.Time `json:"starts_at"`
EndsAt time.Time `json:"ends_at"`
}
type Silence struct {
ID string `json:"id"`
RuleCode *string `json:"rule_code,omitempty"`
Category *string `json:"category,omitempty"`
ResourceType *string `json:"resource_type,omitempty"`
ResourcePublicID *string `json:"resource_id,omitempty"`
Matcher map[string]any `json:"matcher"`
Reason string `json:"reason"`
StartsAt time.Time `json:"starts_at"`
EndsAt time.Time `json:"ends_at"`
CreatedBy string `json:"created_by"`
RevokedAt *time.Time `json:"revoked_at,omitempty"`
RowVersion int64 `json:"row_version"`
CreatedAt time.Time `json:"created_at"`
}
type Action struct {
ID string `json:"id"`
ActionType string `json:"action_type"`
FromStatus *string `json:"from_status,omitempty"`
ToStatus *string `json:"to_status,omitempty"`
ActorType string `json:"actor_type"`
ActorUser *string `json:"actor_user_id,omitempty"`
Note string `json:"note"`
CreatedAt time.Time `json:"created_at"`
}
type Incident struct {
ID string `json:"id"`
RuleCode string `json:"rule_code"`
Category string `json:"category"`
Severity string `json:"severity"`
Status string `json:"status"`
Fingerprint string `json:"fingerprint"`
Title string `json:"title"`
Summary string `json:"summary"`
ResourceType *string `json:"resource_type,omitempty"`
ResourcePublicID *string `json:"resource_id,omitempty"`
CorrelationRequestID *string `json:"request_id,omitempty"`
ReceiverGroup *string `json:"receiver_group,omitempty"`
RunbookRef *string `json:"runbook_ref,omitempty"`
CurrentOwner *string `json:"current_owner,omitempty"`
FirstDetectedAt time.Time `json:"first_detected_at"`
LastDetectedAt time.Time `json:"last_detected_at"`
OccurrenceCount int64 `json:"occurrence_count"`
AcknowledgedAt *time.Time `json:"acknowledged_at,omitempty"`
MitigatedAt *time.Time `json:"mitigated_at,omitempty"`
ResolvedAt *time.Time `json:"resolved_at,omitempty"`
SilencedUntil *time.Time `json:"silenced_until,omitempty"`
EscalatedAt *time.Time `json:"escalated_at,omitempty"`
RowVersion int64 `json:"row_version"`
Actions []Action `json:"actions"`
}
type actor struct{ tenantID, tenantPublicID, userID, userPublicID, role string }
type Service struct {
pool *pgxpool.Pool
audit *audit.Repository
idem *idempotency.Executor
}
func NewService(pool *pgxpool.Pool, auditRepository *audit.Repository) (*Service, error) {
if pool == nil || auditRepository == nil {
return nil, errors.New("alert service dependencies are required")
}
return &Service{pool: pool, audit: auditRepository, idem: idempotency.NewExecutor(pool)}, nil
}
func (s *Service) CreateRoutingPolicy(ctx context.Context, principal identity.Principal, tenantPublicID, key, requestID string, input RoutingInput) (RoutingPolicy, bool, error) {
normalizeRouting(&input)
if !validRoutingInput(input) {
return RoutingPolicy{}, false, ErrInvalid
}
a, err := s.resolveActor(ctx, principal, tenantPublicID, "ops_approver", "ops_admin")
if err != nil {
return RoutingPolicy{}, false, err
}
body, _ := json.Marshal(input)
result, err := s.idem.Execute(ctx, idempotency.Request{TenantID: &a.tenantID, ActorKey: "ops:" + a.userPublicID, Scope: "POST /api/v1/ops/alert-routing-policies", Key: key, RequestHash: idempotency.Hash(body), ExpiresAfter: 24 * time.Hour}, func(op context.Context, tx pgx.Tx) (idempotency.Result, error) {
if _, e := tx.Exec(op, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, a.tenantID+":"+input.Category+":"+input.Severity); e != nil {
return idempotency.Result{}, e
}
var version int
if e := tx.QueryRow(op, `SELECT COALESCE(MAX(version_no),0)+1 FROM alert_routing_policy WHERE tenant_id=$1::uuid AND category=$2 AND severity=$3`, a.tenantID, input.Category, input.Severity).Scan(&version); e != nil {
return idempotency.Result{}, e
}
if _, e := tx.Exec(op, `UPDATE alert_routing_policy SET status='RETIRED',retired_at=now(),row_version=row_version+1,updated_at=now() WHERE tenant_id=$1::uuid AND category=$2 AND severity=$3 AND status='ACTIVE'`, a.tenantID, input.Category, input.Severity); e != nil {
return idempotency.Result{}, e
}
publicID := uuid.NewString()
var internalID string
if e := tx.QueryRow(op, `INSERT INTO alert_routing_policy(public_id,tenant_id,category,severity,version_no,receiver_group,response_minutes,escalation_after_minutes,escalation_receiver_group,runbook_ref,change_summary,created_by) VALUES($1::uuid,$2::uuid,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12::uuid) RETURNING id::text`, publicID, a.tenantID, input.Category, input.Severity, version, input.ReceiverGroup, input.ResponseMinutes, input.EscalationAfterMinutes, input.EscalationReceiverGroup, input.RunbookRef, input.ChangeSummary, a.userID).Scan(&internalID); e != nil {
return idempotency.Result{}, e
}
if e := s.audit.Record(op, tx, audit.Event{TenantID: &a.tenantID, ActorType: "user", ActorPublicID: &a.userPublicID, Action: "alert.routing.activate", ResourceType: "alert_routing_policy", ResourceID: &internalID, RequestID: requestID, Purpose: input.ChangeSummary, Result: "success", Detail: map[string]any{"category": input.Category, "severity": input.Severity, "version_no": version}}); e != nil {
return idempotency.Result{}, e
}
item, e := getRoutingTx(op, tx, a.tenantID, publicID)
if e != nil {
return idempotency.Result{}, e
}
encoded, _ := json.Marshal(item)
return idempotency.Result{StatusCode: 201, Body: encoded, ResourceID: &internalID}, nil
})
if err != nil {
return RoutingPolicy{}, false, err
}
var item RoutingPolicy
if err = json.Unmarshal(result.Body, &item); err != nil {
return RoutingPolicy{}, false, err
}
return item, !result.Replayed, nil
}
func (s *Service) ListRoutingPolicies(ctx context.Context, principal identity.Principal, tenantPublicID string) ([]RoutingPolicy, error) {
a, e := s.resolveActor(ctx, principal, tenantPublicID, "ops_viewer", "ops_operator", "ops_approver", "ops_admin")
if e != nil {
return nil, e
}
rows, e := s.pool.Query(ctx, routingSelect+` WHERE arp.tenant_id=$1::uuid ORDER BY CASE arp.status WHEN 'ACTIVE' THEN 1 ELSE 2 END,arp.category,arp.severity,arp.version_no DESC`, a.tenantID)
if e != nil {
return nil, e
}
defer rows.Close()
items := []RoutingPolicy{}
for rows.Next() {
item, se := scanRouting(rows)
if se != nil {
return nil, se
}
items = append(items, item)
}
return items, rows.Err()
}
func (s *Service) ListIncidents(ctx context.Context, principal identity.Principal, tenantPublicID, status string) ([]Incident, error) {
a, e := s.resolveActor(ctx, principal, tenantPublicID, "ops_viewer", "ops_operator", "ops_approver", "ops_admin")
if e != nil {
return nil, e
}
status = strings.ToUpper(strings.TrimSpace(status))
if status != "" && !validStatus(status) {
return nil, ErrInvalid
}
rows, e := s.pool.Query(ctx, incidentSelect+` WHERE ai.tenant_id=$1::uuid AND (NULLIF($2,'') IS NULL OR ai.status=$2) ORDER BY CASE ai.severity WHEN 'CRITICAL' THEN 1 WHEN 'HIGH' THEN 2 WHEN 'MEDIUM' THEN 3 ELSE 4 END,ai.last_detected_at DESC LIMIT 1000`, a.tenantID, status)
if e != nil {
return nil, e
}
defer rows.Close()
items := []Incident{}
for rows.Next() {
item, se := scanIncident(rows)
if se != nil {
return nil, se
}
items = append(items, item)
}
return items, rows.Err()
}
func (s *Service) GetIncident(ctx context.Context, principal identity.Principal, tenantPublicID, publicID string) (Incident, error) {
a, e := s.resolveActor(ctx, principal, tenantPublicID, "ops_viewer", "ops_operator", "ops_approver", "ops_admin")
if e != nil {
return Incident{}, e
}
if !validUUID(publicID) {
return Incident{}, ErrInvalid
}
tx, e := s.pool.BeginTx(ctx, pgx.TxOptions{AccessMode: pgx.ReadOnly})
if e != nil {
return Incident{}, e
}
defer tx.Rollback(context.Background())
item, e := getIncidentTx(ctx, tx, a.tenantID, publicID)
if errors.Is(e, pgx.ErrNoRows) {
return Incident{}, ErrNotFound
}
if e == nil {
e = tx.Commit(ctx)
}
return item, e
}
func (s *Service) CreateSilence(ctx context.Context, principal identity.Principal, tenantPublicID, key, requestID string, input SilenceInput) (Silence, bool, error) {
normalizeSilence(&input)
if !validSilenceInput(input, time.Now().UTC()) {
return Silence{}, false, ErrInvalid
}
a, e := s.resolveActor(ctx, principal, tenantPublicID, "ops_operator", "ops_approver", "ops_admin")
if e != nil {
return Silence{}, false, e
}
encoded, _ := json.Marshal(input)
result, e := s.idem.Execute(ctx, idempotency.Request{TenantID: &a.tenantID, ActorKey: "ops:" + a.userPublicID, Scope: "POST /api/v1/ops/alert-silences", Key: key, RequestHash: idempotency.Hash(encoded), ExpiresAfter: 24 * time.Hour}, func(op context.Context, tx pgx.Tx) (idempotency.Result, error) {
matcher, _ := json.Marshal(input.Matcher)
publicID := uuid.NewString()
var internalID string
e := tx.QueryRow(op, `INSERT INTO alert_silence(public_id,tenant_id,rule_code,category,resource_type,resource_public_id,matcher,reason,starts_at,ends_at,created_by) VALUES($1::uuid,$2::uuid,NULLIF($3,''),NULLIF($4,''),NULLIF($5,''),$6::uuid,$7::jsonb,$8,$9,$10,$11::uuid) RETURNING id::text`, publicID, a.tenantID, input.RuleCode, input.Category, input.ResourceType, nullStringUUID(input.ResourcePublicID), matcher, input.Reason, input.StartsAt.UTC(), input.EndsAt.UTC(), a.userID).Scan(&internalID)
if e != nil {
return idempotency.Result{}, e
}
if e = s.audit.Record(op, tx, audit.Event{TenantID: &a.tenantID, ActorType: "user", ActorPublicID: &a.userPublicID, Action: "alert.silence.create", ResourceType: "alert_silence", ResourceID: &internalID, RequestID: requestID, Purpose: input.Reason, Result: "success"}); e != nil {
return idempotency.Result{}, e
}
item, e := getSilenceTx(op, tx, a.tenantID, publicID)
if e != nil {
return idempotency.Result{}, e
}
body, _ := json.Marshal(item)
return idempotency.Result{StatusCode: 201, Body: body, ResourceID: &internalID}, nil
})
if e != nil {
return Silence{}, false, e
}
var item Silence
if e = json.Unmarshal(result.Body, &item); e != nil {
return Silence{}, false, e
}
return item, !result.Replayed, nil
}
func (s *Service) ListSilences(ctx context.Context, principal identity.Principal, tenantPublicID string) ([]Silence, error) {
a, e := s.resolveActor(ctx, principal, tenantPublicID, "ops_viewer", "ops_operator", "ops_approver", "ops_admin")
if e != nil {
return nil, e
}
rows, e := s.pool.Query(ctx, silenceSelect+` WHERE als.tenant_id=$1::uuid ORDER BY CASE WHEN revoked_at IS NULL AND ends_at>now() THEN 1 ELSE 2 END,ends_at DESC LIMIT 1000`, a.tenantID)
if e != nil {
return nil, e
}
defer rows.Close()
items := []Silence{}
for rows.Next() {
item, se := scanSilence(rows)
if se != nil {
return nil, se
}
items = append(items, item)
}
return items, rows.Err()
}
func (s *Service) RevokeSilence(ctx context.Context, principal identity.Principal, tenantPublicID, publicID, requestID, reason string, expected int64) (Silence, error) {
reason = strings.TrimSpace(reason)
if reason == "" || len([]rune(reason)) > 500 {
return Silence{}, ErrInvalid
}
a, e := s.resolveActor(ctx, principal, tenantPublicID, "ops_operator", "ops_approver", "ops_admin")
if e != nil {
return Silence{}, e
}
tx, e := s.pool.BeginTx(ctx, pgx.TxOptions{})
if e != nil {
return Silence{}, e
}
defer tx.Rollback(context.Background())
var id string
var version int64
var revoked *time.Time
e = tx.QueryRow(ctx, `SELECT id::text,row_version,revoked_at FROM alert_silence WHERE tenant_id=$1::uuid AND public_id=$2::uuid FOR UPDATE`, a.tenantID, publicID).Scan(&id, &version, &revoked)
if errors.Is(e, pgx.ErrNoRows) {
return Silence{}, ErrNotFound
}
if e != nil {
return Silence{}, e
}
if version != expected {
return Silence{}, ErrVersionConflict
}
if revoked != nil {
return Silence{}, ErrStateConflict
}
_, e = tx.Exec(ctx, `UPDATE alert_silence SET revoked_by=$2::uuid,revoked_at=now(),row_version=row_version+1,updated_at=now() WHERE id=$1::uuid`, id, a.userID)
if e != nil {
return Silence{}, e
}
if e = s.audit.Record(ctx, tx, audit.Event{TenantID: &a.tenantID, ActorType: "user", ActorPublicID: &a.userPublicID, Action: "alert.silence.revoke", ResourceType: "alert_silence", ResourceID: &id, RequestID: requestID, Purpose: reason, Result: "success"}); e != nil {
return Silence{}, e
}
item, e := getSilenceTx(ctx, tx, a.tenantID, publicID)
if e == nil {
e = tx.Commit(ctx)
}
return item, e
}
func (s *Service) TransitionIncident(ctx context.Context, principal identity.Principal, tenantPublicID, publicID, requestID, action, note string, expected int64) (Incident, error) {
action = strings.ToUpper(strings.TrimSpace(action))
note = strings.TrimSpace(note)
if note == "" || len([]rune(note)) > 1000 {
return Incident{}, ErrInvalid
}
targets := map[string]string{"ACKNOWLEDGE": "ACKNOWLEDGED", "MITIGATE": "MITIGATED", "RESOLVE": "RESOLVED"}
target, ok := targets[action]
if !ok {
return Incident{}, ErrInvalid
}
a, e := s.resolveActor(ctx, principal, tenantPublicID, "ops_operator", "ops_approver", "ops_admin")
if e != nil {
return Incident{}, e
}
tx, e := s.pool.BeginTx(ctx, pgx.TxOptions{})
if e != nil {
return Incident{}, e
}
defer tx.Rollback(context.Background())
var id, status string
var version int64
e = tx.QueryRow(ctx, `SELECT id::text,status,row_version FROM alert_incident WHERE tenant_id=$1::uuid AND public_id=$2::uuid FOR UPDATE`, a.tenantID, publicID).Scan(&id, &status, &version)
if errors.Is(e, pgx.ErrNoRows) {
return Incident{}, ErrNotFound
}
if e != nil {
return Incident{}, e
}
if version != expected {
return Incident{}, ErrVersionConflict
}
if status == "RESOLVED" || status == target {
return Incident{}, ErrStateConflict
}
now := time.Now().UTC()
_, e = tx.Exec(ctx, `UPDATE alert_incident SET status=$2::varchar,current_owner_id=COALESCE(current_owner_id,$3::uuid),acknowledged_at=CASE WHEN $2::varchar='ACKNOWLEDGED' THEN $4::timestamptz ELSE acknowledged_at END,mitigated_at=CASE WHEN $2::varchar='MITIGATED' THEN $4::timestamptz ELSE mitigated_at END,resolved_at=CASE WHEN $2::varchar='RESOLVED' THEN $4::timestamptz ELSE resolved_at END,row_version=row_version+1,updated_at=$4::timestamptz WHERE id=$1::uuid`, id, target, a.userID, now)
if e != nil {
return Incident{}, e
}
_, e = tx.Exec(ctx, `INSERT INTO alert_action(tenant_id,incident_id,action_type,from_status,to_status,actor_type,actor_user_id,note) VALUES($1::uuid,$2::uuid,$3,$4,$5,'user',$6::uuid,$7)`, a.tenantID, id, target, status, target, a.userID, note)
if e != nil {
return Incident{}, e
}
if e = s.audit.Record(ctx, tx, audit.Event{TenantID: &a.tenantID, ActorType: "user", ActorPublicID: &a.userPublicID, Action: "alert.incident." + strings.ToLower(action), ResourceType: "alert_incident", ResourceID: &id, RequestID: requestID, Purpose: note, Result: "success"}); e != nil {
return Incident{}, e
}
item, e := getIncidentTx(ctx, tx, a.tenantID, publicID)
if e == nil {
e = tx.Commit(ctx)
}
return item, e
}
func (s *Service) resolveActor(ctx context.Context, principal identity.Principal, tenantPublicID string, roles ...string) (actor, error) {
if _, e := uuid.Parse(tenantPublicID); e != nil {
return actor{}, ErrInvalid
}
var a actor
e := s.pool.QueryRow(ctx, `SELECT t.id::text,t.public_id::text,ua.id::text,ua.public_id::text,ua.platform_role FROM user_account ua JOIN tenant t ON t.id=ua.tenant_id AND t.status='active' WHERE t.public_id=$1::uuid AND ua.oidc_issuer=$2 AND ua.oidc_subject=$3 AND ua.status='active'`, tenantPublicID, principal.Issuer, principal.Subject).Scan(&a.tenantID, &a.tenantPublicID, &a.userID, &a.userPublicID, &a.role)
if errors.Is(e, pgx.ErrNoRows) {
return actor{}, ErrAccessDenied
}
if e != nil {
return actor{}, e
}
for _, role := range roles {
if a.role == role {
return a, nil
}
}
return actor{}, ErrAccessDenied
}
func normalizeRouting(i *RoutingInput) {
i.Category = strings.ToUpper(strings.TrimSpace(i.Category))
i.Severity = strings.ToUpper(strings.TrimSpace(i.Severity))
i.ReceiverGroup = strings.TrimSpace(i.ReceiverGroup)
i.EscalationReceiverGroup = strings.TrimSpace(i.EscalationReceiverGroup)
i.RunbookRef = strings.TrimSpace(i.RunbookRef)
i.ChangeSummary = strings.TrimSpace(i.ChangeSummary)
}
func validRoutingInput(i RoutingInput) bool {
return validCategory(i.Category) && validSeverity(i.Severity) && i.ReceiverGroup != "" && len([]rune(i.ReceiverGroup)) <= 120 && i.ResponseMinutes >= 1 && i.ResponseMinutes <= 10080 && i.EscalationAfterMinutes >= i.ResponseMinutes && i.EscalationAfterMinutes <= 10080 && i.EscalationReceiverGroup != "" && len([]rune(i.EscalationReceiverGroup)) <= 120 && i.RunbookRef != "" && len([]rune(i.RunbookRef)) <= 300 && i.ChangeSummary != "" && len([]rune(i.ChangeSummary)) <= 500
}
func normalizeSilence(i *SilenceInput) {
i.RuleCode = strings.ToUpper(strings.TrimSpace(i.RuleCode))
i.Category = strings.ToUpper(strings.TrimSpace(i.Category))
i.ResourceType = strings.TrimSpace(i.ResourceType)
i.ResourcePublicID = strings.TrimSpace(i.ResourcePublicID)
i.Reason = strings.TrimSpace(i.Reason)
}
func validSilenceInput(i SilenceInput, now time.Time) bool {
return !(i.RuleCode == "" && i.Category == "" && i.ResourcePublicID == "") && (i.RuleCode == "" || rulePattern.MatchString(i.RuleCode)) && (i.Category == "" || validCategory(i.Category)) && (i.ResourcePublicID == "" || validUUID(i.ResourcePublicID)) && i.Reason != "" && len([]rune(i.Reason)) <= 500 && !i.StartsAt.IsZero() && i.EndsAt.After(i.StartsAt) && i.EndsAt.Sub(i.StartsAt) <= 30*24*time.Hour && !i.EndsAt.Before(now)
}
func validCategory(v string) bool {
return v == "BUSINESS" || v == "TECHNICAL" || v == "SECURITY" || v == "COST"
}
func validSeverity(v string) bool {
return v == "CRITICAL" || v == "HIGH" || v == "MEDIUM" || v == "LOW"
}
func validStatus(v string) bool {
return v == "OPEN" || v == "ACKNOWLEDGED" || v == "MITIGATED" || v == "RESOLVED" || v == "SILENCED"
}
func validUUID(v string) bool { _, e := uuid.Parse(v); return e == nil }
func nullStringUUID(v string) any {
if strings.TrimSpace(v) == "" {
return nil
}
return v
}
var rulePattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]{2,99}$`)
type scanner interface{ Scan(...any) error }
const routingSelect = `SELECT arp.public_id::text,t.public_id::text,arp.category,arp.severity,arp.version_no,arp.status,arp.receiver_group,arp.response_minutes,arp.escalation_after_minutes,arp.escalation_receiver_group,arp.runbook_ref,arp.change_summary,ua.public_id::text,arp.retired_at,arp.row_version,arp.created_at,arp.updated_at FROM alert_routing_policy arp JOIN tenant t ON t.id=arp.tenant_id JOIN user_account ua ON ua.id=arp.created_by`
func scanRouting(r scanner) (RoutingPolicy, error) {
var i RoutingPolicy
e := r.Scan(&i.ID, &i.TenantID, &i.Category, &i.Severity, &i.VersionNo, &i.Status, &i.ReceiverGroup, &i.ResponseMinutes, &i.EscalationAfterMinutes, &i.EscalationReceiverGroup, &i.RunbookRef, &i.ChangeSummary, &i.CreatedBy, &i.RetiredAt, &i.RowVersion, &i.CreatedAt, &i.UpdatedAt)
return i, e
}
func getRoutingTx(ctx context.Context, tx pgx.Tx, tenantID, publicID string) (RoutingPolicy, error) {
return scanRouting(tx.QueryRow(ctx, routingSelect+` WHERE arp.tenant_id=$1::uuid AND arp.public_id=$2::uuid`, tenantID, publicID))
}
const incidentSelect = `SELECT ai.public_id::text,ai.rule_code,ai.category,ai.severity,ai.status,ai.fingerprint,ai.title,ai.summary,ai.resource_type,ai.resource_public_id::text,ai.correlation_request_id,arp.receiver_group,arp.runbook_ref,owner.public_id::text,ai.first_detected_at,ai.last_detected_at,ai.occurrence_count,ai.acknowledged_at,ai.mitigated_at,ai.resolved_at,ai.silenced_until,ai.escalated_at,ai.row_version FROM alert_incident ai LEFT JOIN alert_routing_policy arp ON arp.id=ai.routing_policy_id LEFT JOIN user_account owner ON owner.id=ai.current_owner_id`
func scanIncident(r scanner) (Incident, error) {
var i Incident
e := r.Scan(&i.ID, &i.RuleCode, &i.Category, &i.Severity, &i.Status, &i.Fingerprint, &i.Title, &i.Summary, &i.ResourceType, &i.ResourcePublicID, &i.CorrelationRequestID, &i.ReceiverGroup, &i.RunbookRef, &i.CurrentOwner, &i.FirstDetectedAt, &i.LastDetectedAt, &i.OccurrenceCount, &i.AcknowledgedAt, &i.MitigatedAt, &i.ResolvedAt, &i.SilencedUntil, &i.EscalatedAt, &i.RowVersion)
i.Actions = []Action{}
return i, e
}
func getIncidentTx(ctx context.Context, tx pgx.Tx, tenantID, publicID string) (Incident, error) {
i, e := scanIncident(tx.QueryRow(ctx, incidentSelect+` WHERE ai.tenant_id=$1::uuid AND ai.public_id=$2::uuid`, tenantID, publicID))
if e != nil {
return i, e
}
rows, e := tx.Query(ctx, `SELECT aa.public_id::text,aa.action_type,aa.from_status,aa.to_status,aa.actor_type,ua.public_id::text,aa.note,aa.created_at FROM alert_action aa LEFT JOIN user_account ua ON ua.id=aa.actor_user_id WHERE aa.incident_id=(SELECT id FROM alert_incident WHERE tenant_id=$1::uuid AND public_id=$2::uuid) ORDER BY aa.created_at,aa.id`, tenantID, publicID)
if e != nil {
return i, e
}
defer rows.Close()
for rows.Next() {
var a Action
if e = rows.Scan(&a.ID, &a.ActionType, &a.FromStatus, &a.ToStatus, &a.ActorType, &a.ActorUser, &a.Note, &a.CreatedAt); e != nil {
return i, e
}
i.Actions = append(i.Actions, a)
}
return i, rows.Err()
}
const silenceSelect = `SELECT als.public_id::text,als.rule_code,als.category,als.resource_type,als.resource_public_id::text,als.matcher,als.reason,als.starts_at,als.ends_at,ua.public_id::text,als.revoked_at,als.row_version,als.created_at FROM alert_silence als JOIN user_account ua ON ua.id=als.created_by`
func scanSilence(r scanner) (Silence, error) {
var i Silence
var raw []byte
e := r.Scan(&i.ID, &i.RuleCode, &i.Category, &i.ResourceType, &i.ResourcePublicID, &raw, &i.Reason, &i.StartsAt, &i.EndsAt, &i.CreatedBy, &i.RevokedAt, &i.RowVersion, &i.CreatedAt)
if e == nil {
e = json.Unmarshal(raw, &i.Matcher)
}
if i.Matcher == nil {
i.Matcher = map[string]any{}
}
return i, e
}
func getSilenceTx(ctx context.Context, tx pgx.Tx, tenantID, publicID string) (Silence, error) {
return scanSilence(tx.QueryRow(ctx, silenceSelect+` WHERE als.tenant_id=$1::uuid AND als.public_id=$2::uuid`, tenantID, publicID))
}
internal/alerts/service_integration_test.go
+132
View File
@@ -0,0 +1,132 @@
package alerts
import (
"context"
"errors"
"os"
"testing"
"time"
"data-terminal-cloud/internal/audit"
"data-terminal-cloud/internal/identity"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestAlertRoutingSilenceAndIncidentLifecycle(t *testing.T) {
databaseURL := os.Getenv("TEST_DATABASE_URL")
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if _, err := NewService(nil, audit.NewRepository()); err == nil {
t.Fatal("nil database pool was accepted")
}
service, err := NewService(pool, audit.NewRepository())
if err != nil {
t.Fatal(err)
}
admin := identity.Principal{Issuer: "http://localhost:58180/realms/voice-local", Subject: "00000000-0000-0000-0000-000000000101"}
approver := identity.Principal{Issuer: admin.Issuer, Subject: "00000000-0000-0000-0000-000000000102"}
outsider := identity.Principal{Issuer: admin.Issuer, Subject: "00000000-0000-0000-0000-000000000103"}
tenantID := "10000000-0000-0000-0000-000000000010"
routeKeys := []string{"alerts-integration-route1-" + uuid.NewString(), "alerts-integration-route2-" + uuid.NewString()}
silenceKey := "alerts-integration-silence-" + uuid.NewString()
incidentID, incidentPublicID := uuid.NewString(), uuid.NewString()
fingerprint := "alerts-integration-" + uuid.NewString()
defer func() {
_, _ = pool.Exec(ctx, `DELETE FROM audit_event WHERE request_id LIKE 'alerts-integration-%'`)
_, _ = pool.Exec(ctx, `DELETE FROM idempotency_record WHERE idempotency_key=ANY($1::text[])`, append(routeKeys, silenceKey))
_, _ = pool.Exec(ctx, `DELETE FROM alert_incident WHERE id=$1::uuid`, incidentID)
_, _ = pool.Exec(ctx, `DELETE FROM alert_silence WHERE reason='集成测试维护窗口'`)
_, _ = pool.Exec(ctx, `DELETE FROM alert_routing_policy WHERE category='BUSINESS' AND severity='LOW'`)
}()
routeInput := RoutingInput{Category: "BUSINESS", Severity: "LOW", ReceiverGroup: "integration-oncall", ResponseMinutes: 30, EscalationAfterMinutes: 60, EscalationReceiverGroup: "integration-command", RunbookRef: "runbook://integration/alert", ChangeSummary: "集成告警路由第一版"}
route1, created, err := service.CreateRoutingPolicy(ctx, admin, tenantID, routeKeys[0], "alerts-integration-route1", routeInput)
if err != nil || !created || route1.Status != "ACTIVE" || route1.VersionNo != 1 {
t.Fatalf("route1=%+v created=%v err=%v", route1, created, err)
}
replayed, created, err := service.CreateRoutingPolicy(ctx, admin, tenantID, routeKeys[0], "alerts-integration-route1-replay", routeInput)
if err != nil || created || replayed.ID != route1.ID {
t.Fatalf("route replay=%+v created=%v err=%v", replayed, created, err)
}
routeInput.ReceiverGroup = "integration-oncall-v2"
routeInput.ChangeSummary = "集成告警路由第二版"
route2, created, err := service.CreateRoutingPolicy(ctx, approver, tenantID, routeKeys[1], "alerts-integration-route2", routeInput)
if err != nil || !created || route2.VersionNo != 2 || route2.Status != "ACTIVE" {
t.Fatalf("route2=%+v created=%v err=%v", route2, created, err)
}
routes, err := service.ListRoutingPolicies(ctx, approver, tenantID)
if err != nil || len(routes) != 2 || routes[0].ID != route2.ID || routes[1].Status != "RETIRED" {
t.Fatalf("routes=%+v err=%v", routes, err)
}
now := time.Now().UTC()
silenceInput := SilenceInput{RuleCode: "INTEGRATION_TEST_ALERT", Matcher: map[string]any{"environment": "integration"}, Reason: "集成测试维护窗口", StartsAt: now.Add(-time.Minute), EndsAt: now.Add(time.Hour)}
silence, created, err := service.CreateSilence(ctx, admin, tenantID, silenceKey, "alerts-integration-silence", silenceInput)
if err != nil || !created || silence.RowVersion != 1 || silence.RevokedAt != nil {
t.Fatalf("silence=%+v created=%v err=%v", silence, created, err)
}
if _, err := service.RevokeSilence(ctx, admin, tenantID, silence.ID, "alerts-integration-silence-stale", "旧版本", 2); !errors.Is(err, ErrVersionConflict) {
t.Fatalf("stale silence error=%v", err)
}
silence, err = service.RevokeSilence(ctx, admin, tenantID, silence.ID, "alerts-integration-silence-revoke", "维护结束", 1)
if err != nil || silence.RevokedAt == nil || silence.RowVersion != 2 {
t.Fatalf("revoked silence=%+v err=%v", silence, err)
}
silences, err := service.ListSilences(ctx, approver, tenantID)
if err != nil || len(silences) != 1 || silences[0].ID != silence.ID {
t.Fatalf("silences=%+v err=%v", silences, err)
}
var routeInternalID string
if err := pool.QueryRow(ctx, `SELECT id::text FROM alert_routing_policy WHERE public_id=$1::uuid`, route2.ID).Scan(&routeInternalID); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `
INSERT INTO alert_incident(id,public_id,tenant_id,rule_code,category,severity,status,fingerprint,title,summary,routing_policy_id,first_detected_at,last_detected_at)
VALUES($1::uuid,$2::uuid,'10000000-0000-0000-0000-000000000001','INTEGRATION_TEST_ALERT','BUSINESS','LOW','OPEN',$3,'集成告警','合成告警状态机',$4::uuid,now(),now())`, incidentID, incidentPublicID, fingerprint, routeInternalID); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `INSERT INTO alert_action(tenant_id,incident_id,action_type,from_status,to_status,actor_type,note) VALUES('10000000-0000-0000-0000-000000000001',$1::uuid,'DETECTED',NULL,'OPEN','system','集成信号发现')`, incidentID); err != nil {
t.Fatal(err)
}
open, err := service.ListIncidents(ctx, approver, tenantID, "OPEN")
if err != nil || len(open) != 1 || open[0].ID != incidentPublicID {
t.Fatalf("open incidents=%+v err=%v", open, err)
}
if _, err := service.GetIncident(ctx, outsider, tenantID, incidentPublicID); !errors.Is(err, ErrAccessDenied) {
t.Fatalf("outsider incident error=%v", err)
}
incident, err := service.GetIncident(ctx, approver, tenantID, incidentPublicID)
if err != nil || len(incident.Actions) != 1 {
t.Fatalf("incident=%+v err=%v", incident, err)
}
if _, err := service.TransitionIncident(ctx, admin, tenantID, incidentPublicID, "alerts-integration-invalid", "PAUSE", "无效动作", 1); !errors.Is(err, ErrInvalid) {
t.Fatalf("invalid transition error=%v", err)
}
if _, err := service.TransitionIncident(ctx, admin, tenantID, incidentPublicID, "alerts-integration-stale", "ACKNOWLEDGE", "旧版本", 2); !errors.Is(err, ErrVersionConflict) {
t.Fatalf("stale transition error=%v", err)
}
incident, err = service.TransitionIncident(ctx, admin, tenantID, incidentPublicID, "alerts-integration-ack", "ACKNOWLEDGE", "值班人员确认", 1)
if err != nil || incident.Status != "ACKNOWLEDGED" || incident.RowVersion != 2 || incident.CurrentOwner == nil {
t.Fatalf("ack incident=%+v err=%v", incident, err)
}
incident, err = service.TransitionIncident(ctx, admin, tenantID, incidentPublicID, "alerts-integration-mitigate", "MITIGATE", "临时措施已执行", 2)
if err != nil || incident.Status != "MITIGATED" || incident.RowVersion != 3 {
t.Fatalf("mitigate incident=%+v err=%v", incident, err)
}
incident, err = service.TransitionIncident(ctx, admin, tenantID, incidentPublicID, "alerts-integration-resolve", "RESOLVE", "根因已消除", 3)
if err != nil || incident.Status != "RESOLVED" || incident.RowVersion != 4 || len(incident.Actions) != 4 {
t.Fatalf("resolve incident=%+v err=%v", incident, err)
}
if _, err := service.TransitionIncident(ctx, admin, tenantID, incidentPublicID, "alerts-integration-resolve-again", "RESOLVE", "重复关闭", 4); !errors.Is(err, ErrStateConflict) {
t.Fatalf("duplicate resolve error=%v", err)
}
}
internal/alerts/service_test.go
+79
View File
@@ -0,0 +1,79 @@
package alerts
import (
"testing"
"time"
)
func TestAlertEnumsAndRuleCode(t *testing.T) {
t.Parallel()
for _, value := range []string{"BUSINESS", "TECHNICAL", "SECURITY", "COST"} {
if !validCategory(value) {
t.Fatalf("expected valid category %q", value)
}
}
for _, value := range []string{"CRITICAL", "HIGH", "MEDIUM", "LOW"} {
if !validSeverity(value) {
t.Fatalf("expected valid severity %q", value)
}
}
for _, value := range []string{"OPEN", "ACKNOWLEDGED", "MITIGATED", "RESOLVED", "SILENCED"} {
if !validStatus(value) {
t.Fatalf("expected valid status %q", value)
}
}
for _, value := range []string{"OUTBOX_DEAD_LETTER", "VOICE_TASK_STUCK", "COST_DAILY_QUOTA_NEAR"} {
if !rulePattern.MatchString(value) {
t.Fatalf("expected valid rule code %q", value)
}
}
for _, value := range []string{"", "ab", "lower_case", "-BROKEN"} {
if rulePattern.MatchString(value) {
t.Fatalf("expected invalid rule code %q", value)
}
}
}
func TestNormalizeRouting(t *testing.T) {
t.Parallel()
input := RoutingInput{
Category: " technical ",
Severity: " high ",
ReceiverGroup: " on-call ",
EscalationReceiverGroup: " incident-command ",
RunbookRef: " runbook://outbox/dead-letter ",
ChangeSummary: " initial route ",
}
normalizeRouting(&input)
if input.Category != "TECHNICAL" || input.Severity != "HIGH" || input.ReceiverGroup != "on-call" || input.EscalationReceiverGroup != "incident-command" || input.RunbookRef != "runbook://outbox/dead-letter" || input.ChangeSummary != "initial route" {
t.Fatalf("unexpected normalized routing input: %#v", input)
}
input.ResponseMinutes = 10
input.EscalationAfterMinutes = 30
if !validRoutingInput(input) {
t.Fatal("expected complete routing policy to be valid")
}
input.EscalationAfterMinutes = 5
if validRoutingInput(input) {
t.Fatal("escalation deadline before response deadline must be rejected")
}
}
func TestSilenceWindowBoundaries(t *testing.T) {
t.Parallel()
start := time.Date(2026, 8, 23, 10, 0, 0, 0, time.UTC)
input := SilenceInput{RuleCode: " outbox_dead_letter ", Reason: " planned maintenance ", StartsAt: start, EndsAt: start.Add(30 * 24 * time.Hour)}
normalizeSilence(&input)
if !validSilenceInput(input, start) {
t.Fatal("expected normalized 30-day silence to be valid")
}
input.EndsAt = start.Add(30*24*time.Hour + time.Second)
if validSilenceInput(input, start) {
t.Fatal("silence longer than 30 days must be rejected")
}
input.EndsAt = start.Add(time.Hour)
input.RuleCode = ""
if validSilenceInput(input, start) {
t.Fatal("silence without rule, category or resource scope must be rejected")
}
}