133 lines
7.5 KiB
Go
133 lines
7.5 KiB
Go
package alerts
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"data-terminal-cloud/internal/audit"
|
|
"data-terminal-cloud/internal/identity"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
func TestAlertRoutingSilenceAndIncidentLifecycle(t *testing.T) {
|
|
databaseURL := os.Getenv("TEST_DATABASE_URL")
|
|
if databaseURL == "" {
|
|
t.Skip("TEST_DATABASE_URL is not set")
|
|
}
|
|
ctx := context.Background()
|
|
pool, err := pgxpool.New(ctx, databaseURL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer pool.Close()
|
|
if _, err := NewService(nil, audit.NewRepository()); err == nil {
|
|
t.Fatal("nil database pool was accepted")
|
|
}
|
|
service, err := NewService(pool, audit.NewRepository())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
admin := identity.Principal{Issuer: "http://localhost:58180/realms/voice-local", Subject: "00000000-0000-0000-0000-000000000101"}
|
|
approver := identity.Principal{Issuer: admin.Issuer, Subject: "00000000-0000-0000-0000-000000000102"}
|
|
outsider := identity.Principal{Issuer: admin.Issuer, Subject: "00000000-0000-0000-0000-000000000103"}
|
|
tenantID := "10000000-0000-0000-0000-000000000010"
|
|
routeKeys := []string{"alerts-integration-route1-" + uuid.NewString(), "alerts-integration-route2-" + uuid.NewString()}
|
|
silenceKey := "alerts-integration-silence-" + uuid.NewString()
|
|
incidentID, incidentPublicID := uuid.NewString(), uuid.NewString()
|
|
fingerprint := "alerts-integration-" + uuid.NewString()
|
|
defer func() {
|
|
_, _ = pool.Exec(ctx, `DELETE FROM audit_event WHERE request_id LIKE 'alerts-integration-%'`)
|
|
_, _ = pool.Exec(ctx, `DELETE FROM idempotency_record WHERE idempotency_key=ANY($1::text[])`, append(routeKeys, silenceKey))
|
|
_, _ = pool.Exec(ctx, `DELETE FROM alert_incident WHERE id=$1::uuid`, incidentID)
|
|
_, _ = pool.Exec(ctx, `DELETE FROM alert_silence WHERE reason='集成测试维护窗口'`)
|
|
_, _ = pool.Exec(ctx, `DELETE FROM alert_routing_policy WHERE category='BUSINESS' AND severity='LOW'`)
|
|
}()
|
|
routeInput := RoutingInput{Category: "BUSINESS", Severity: "LOW", ReceiverGroup: "integration-oncall", ResponseMinutes: 30, EscalationAfterMinutes: 60, EscalationReceiverGroup: "integration-command", RunbookRef: "runbook://integration/alert", ChangeSummary: "集成告警路由第一版"}
|
|
route1, created, err := service.CreateRoutingPolicy(ctx, admin, tenantID, routeKeys[0], "alerts-integration-route1", routeInput)
|
|
if err != nil || !created || route1.Status != "ACTIVE" || route1.VersionNo != 1 {
|
|
t.Fatalf("route1=%+v created=%v err=%v", route1, created, err)
|
|
}
|
|
replayed, created, err := service.CreateRoutingPolicy(ctx, admin, tenantID, routeKeys[0], "alerts-integration-route1-replay", routeInput)
|
|
if err != nil || created || replayed.ID != route1.ID {
|
|
t.Fatalf("route replay=%+v created=%v err=%v", replayed, created, err)
|
|
}
|
|
routeInput.ReceiverGroup = "integration-oncall-v2"
|
|
routeInput.ChangeSummary = "集成告警路由第二版"
|
|
route2, created, err := service.CreateRoutingPolicy(ctx, approver, tenantID, routeKeys[1], "alerts-integration-route2", routeInput)
|
|
if err != nil || !created || route2.VersionNo != 2 || route2.Status != "ACTIVE" {
|
|
t.Fatalf("route2=%+v created=%v err=%v", route2, created, err)
|
|
}
|
|
routes, err := service.ListRoutingPolicies(ctx, approver, tenantID)
|
|
if err != nil || len(routes) != 2 || routes[0].ID != route2.ID || routes[1].Status != "RETIRED" {
|
|
t.Fatalf("routes=%+v err=%v", routes, err)
|
|
}
|
|
|
|
now := time.Now().UTC()
|
|
silenceInput := SilenceInput{RuleCode: "INTEGRATION_TEST_ALERT", Matcher: map[string]any{"environment": "integration"}, Reason: "集成测试维护窗口", StartsAt: now.Add(-time.Minute), EndsAt: now.Add(time.Hour)}
|
|
silence, created, err := service.CreateSilence(ctx, admin, tenantID, silenceKey, "alerts-integration-silence", silenceInput)
|
|
if err != nil || !created || silence.RowVersion != 1 || silence.RevokedAt != nil {
|
|
t.Fatalf("silence=%+v created=%v err=%v", silence, created, err)
|
|
}
|
|
if _, err := service.RevokeSilence(ctx, admin, tenantID, silence.ID, "alerts-integration-silence-stale", "旧版本", 2); !errors.Is(err, ErrVersionConflict) {
|
|
t.Fatalf("stale silence error=%v", err)
|
|
}
|
|
silence, err = service.RevokeSilence(ctx, admin, tenantID, silence.ID, "alerts-integration-silence-revoke", "维护结束", 1)
|
|
if err != nil || silence.RevokedAt == nil || silence.RowVersion != 2 {
|
|
t.Fatalf("revoked silence=%+v err=%v", silence, err)
|
|
}
|
|
silences, err := service.ListSilences(ctx, approver, tenantID)
|
|
if err != nil || len(silences) != 1 || silences[0].ID != silence.ID {
|
|
t.Fatalf("silences=%+v err=%v", silences, err)
|
|
}
|
|
|
|
var routeInternalID string
|
|
if err := pool.QueryRow(ctx, `SELECT id::text FROM alert_routing_policy WHERE public_id=$1::uuid`, route2.ID).Scan(&routeInternalID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := pool.Exec(ctx, `
|
|
INSERT INTO alert_incident(id,public_id,tenant_id,rule_code,category,severity,status,fingerprint,title,summary,routing_policy_id,first_detected_at,last_detected_at)
|
|
VALUES($1::uuid,$2::uuid,'10000000-0000-0000-0000-000000000001','INTEGRATION_TEST_ALERT','BUSINESS','LOW','OPEN',$3,'集成告警','合成告警状态机',$4::uuid,now(),now())`, incidentID, incidentPublicID, fingerprint, routeInternalID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := pool.Exec(ctx, `INSERT INTO alert_action(tenant_id,incident_id,action_type,from_status,to_status,actor_type,note) VALUES('10000000-0000-0000-0000-000000000001',$1::uuid,'DETECTED',NULL,'OPEN','system','集成信号发现')`, incidentID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
open, err := service.ListIncidents(ctx, approver, tenantID, "OPEN")
|
|
if err != nil || len(open) != 1 || open[0].ID != incidentPublicID {
|
|
t.Fatalf("open incidents=%+v err=%v", open, err)
|
|
}
|
|
if _, err := service.GetIncident(ctx, outsider, tenantID, incidentPublicID); !errors.Is(err, ErrAccessDenied) {
|
|
t.Fatalf("outsider incident error=%v", err)
|
|
}
|
|
incident, err := service.GetIncident(ctx, approver, tenantID, incidentPublicID)
|
|
if err != nil || len(incident.Actions) != 1 {
|
|
t.Fatalf("incident=%+v err=%v", incident, err)
|
|
}
|
|
if _, err := service.TransitionIncident(ctx, admin, tenantID, incidentPublicID, "alerts-integration-invalid", "PAUSE", "无效动作", 1); !errors.Is(err, ErrInvalid) {
|
|
t.Fatalf("invalid transition error=%v", err)
|
|
}
|
|
if _, err := service.TransitionIncident(ctx, admin, tenantID, incidentPublicID, "alerts-integration-stale", "ACKNOWLEDGE", "旧版本", 2); !errors.Is(err, ErrVersionConflict) {
|
|
t.Fatalf("stale transition error=%v", err)
|
|
}
|
|
incident, err = service.TransitionIncident(ctx, admin, tenantID, incidentPublicID, "alerts-integration-ack", "ACKNOWLEDGE", "值班人员确认", 1)
|
|
if err != nil || incident.Status != "ACKNOWLEDGED" || incident.RowVersion != 2 || incident.CurrentOwner == nil {
|
|
t.Fatalf("ack incident=%+v err=%v", incident, err)
|
|
}
|
|
incident, err = service.TransitionIncident(ctx, admin, tenantID, incidentPublicID, "alerts-integration-mitigate", "MITIGATE", "临时措施已执行", 2)
|
|
if err != nil || incident.Status != "MITIGATED" || incident.RowVersion != 3 {
|
|
t.Fatalf("mitigate incident=%+v err=%v", incident, err)
|
|
}
|
|
incident, err = service.TransitionIncident(ctx, admin, tenantID, incidentPublicID, "alerts-integration-resolve", "RESOLVE", "根因已消除", 3)
|
|
if err != nil || incident.Status != "RESOLVED" || incident.RowVersion != 4 || len(incident.Actions) != 4 {
|
|
t.Fatalf("resolve incident=%+v err=%v", incident, err)
|
|
}
|
|
if _, err := service.TransitionIncident(ctx, admin, tenantID, incidentPublicID, "alerts-integration-resolve-again", "RESOLVE", "重复关闭", 4); !errors.Is(err, ErrStateConflict) {
|
|
t.Fatalf("duplicate resolve error=%v", err)
|
|
}
|
|
}
|