From 19daf4ad3cf724808fd6f473c07df96faca384e1 Mon Sep 17 00:00:00 2001 From: naiba Date: Wed, 2 Sep 2026 13:30:46 +0000 Subject: [PATCH] fix(alert): prevent persisted rule crash loops --- cmd/dashboard/controller/alertrule.go | 12 ++ .../controller/alertrule_security_test.go | 67 ++++++++++ model/alertrule.go | 79 +++++++++-- model/alertrule_security_test.go | 123 ++++++++++++++++++ model/alertrule_test.go | 20 ++- model/rule.go | 89 +++++++++++-- service/singleton/alertsentinel.go | 110 +++++++++------- service/singleton/alertsentinel_test.go | 50 +++++++ 8 files changed, 479 insertions(+), 71 deletions(-) create mode 100644 cmd/dashboard/controller/alertrule_security_test.go create mode 100644 model/alertrule_security_test.go diff --git a/cmd/dashboard/controller/alertrule.go b/cmd/dashboard/controller/alertrule.go index 3c54f4a2..85021a47 100644 --- a/cmd/dashboard/controller/alertrule.go +++ b/cmd/dashboard/controller/alertrule.go @@ -172,6 +172,12 @@ func validateRule(c *gin.Context, r *model.AlertRule) error { } if len(r.Rules) > 0 { for _, rule := range r.Rules { + if rule == nil { + return singleton.Localizer.ErrorT("rule is not set") + } + if !rule.IsSupportedType() { + return singleton.Localizer.ErrorT("unsupported rule type") + } switch rule.Cover { case model.RuleCoverAll, model.RuleCoverIgnoreAll: default: @@ -182,10 +188,16 @@ func validateRule(c *gin.Context, r *model.AlertRule) error { if rule.Duration < 3 { return singleton.Localizer.ErrorT("duration need to be at least 3") } + if rule.Duration > model.MaxAlertRuleDuration { + return singleton.Localizer.ErrorT("duration is too large") + } } else { if rule.CycleInterval < 1 { return singleton.Localizer.ErrorT("cycle_interval need to be at least 1") } + if rule.CycleInterval > model.MaxAlertRuleCycleInterval { + return singleton.Localizer.ErrorT("cycle_interval is too large") + } if rule.CycleStart == nil { return singleton.Localizer.ErrorT("cycle_start is not set") } diff --git a/cmd/dashboard/controller/alertrule_security_test.go b/cmd/dashboard/controller/alertrule_security_test.go new file mode 100644 index 00000000..7d68d388 --- /dev/null +++ b/cmd/dashboard/controller/alertrule_security_test.go @@ -0,0 +1,67 @@ +package controller + +import ( + "math" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/nezhahq/nezha/model" +) + +func TestValidateRuleRejectsCrashPayloadsFromMember(t *testing.T) { + ctx := newMemberValidationContext(t) + cycleStart := time.Now().Add(-time.Hour) + tests := []struct { + name string + rule *model.Rule + }{ + { + name: "unknown rule type", + rule: &model.Rule{Type: "attacker_controlled", Duration: 3, Cover: model.RuleCoverAll}, + }, + { + name: "duration overflows int", + rule: &model.Rule{Type: "offline", Duration: math.MaxUint64, Cover: model.RuleCoverAll}, + }, + { + name: "cycle interval overflows int", + rule: &model.Rule{ + Type: "transfer_in_cycle", + CycleStart: &cycleStart, + CycleInterval: math.MaxUint64, + Cover: model.RuleCoverAll, + }, + }, + { + name: "nil rule", + rule: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + alert := &model.AlertRule{ + Common: model.Common{UserID: 200}, + Name: "security regression", + Rules: []*model.Rule{test.rule}, + } + require.Error(t, validateRule(ctx, alert)) + }) + } +} + +func TestValidateRuleAcceptsSafeDurationBoundary(t *testing.T) { + ctx := newMemberValidationContext(t) + alert := &model.AlertRule{ + Common: model.Common{UserID: 200}, + Name: "duration boundary", + Rules: []*model.Rule{{ + Type: "offline", + Duration: model.MaxAlertRuleDuration, + Cover: model.RuleCoverAll, + }}, + } + require.NoError(t, validateRule(ctx, alert)) +} diff --git a/model/alertrule.go b/model/alertrule.go index 37e734c6..886ea0ce 100644 --- a/model/alertrule.go +++ b/model/alertrule.go @@ -64,6 +64,36 @@ func (r *AlertRule) Enabled() bool { return r.Enable != nil && *r.Enable } +// IsSafeToEvaluate validates persisted rule data before it reaches the alert +// goroutine. API validation protects new writes; this second boundary protects +// upgrades from malformed or deliberately poisoned rows already in the DB. +func (r *AlertRule) IsSafeToEvaluate() bool { + if r == nil || len(r.Rules) == 0 { + return false + } + for _, rule := range r.Rules { + if rule == nil || !rule.IsSupportedType() { + return false + } + switch rule.Cover { + case RuleCoverAll, RuleCoverIgnoreAll: + default: + return false + } + if rule.IsTransferDurationRule() { + if !rule.HasSafeCycleConfiguration() { + return false + } + continue + } + duration, ok := rule.DurationInt() + if !ok || duration < 3 { + return false + } + } + return true +} + // HasPermission extends the default owner/admin check with PAT // server_ids whitelist enforcement. AlertRule.Snapshot fans out across // every owner-visible server filtered only by Rule.Ignore semantics @@ -125,6 +155,12 @@ func (r *AlertRule) Snapshot(cycleTransferStats *CycleTransferStats, server *Ser point := make([]bool, len(r.Rules)) for i, rule := range r.Rules { + if rule == nil || !rule.IsSupportedType() { + // Invalid persisted rules are ignored instead of being interpreted as + // a failed condition or allowed to panic the sentinel. + point[i] = true + continue + } point[i] = rule.Snapshot(cycleTransferStats, server, db) } return point @@ -134,10 +170,14 @@ func (r *AlertRule) Snapshot(cycleTransferStats *CycleTransferStats, server *Ser func (r *AlertRule) Check(points [][]bool) (int, bool) { var hasPassedRule bool durations := make([]int, len(r.Rules)) + validRules := 0 for ruleIndex, rule := range r.Rules { - duration := int(rule.Duration) + if rule == nil || !rule.IsSupportedType() { + continue + } if rule.IsTransferDurationRule() { + validRules++ // 循环区间流量报警 if durations[ruleIndex] < 1 { durations[ruleIndex] = 1 @@ -146,10 +186,21 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) { continue } // 只要最后一次检查超出了规则范围 就认为检查未通过 - if len(points) > 0 && points[len(points)-1][ruleIndex] { - hasPassedRule = true + if len(points) > 0 { + lastPoint := points[len(points)-1] + if ruleIndex >= len(lastPoint) || lastPoint[ruleIndex] { + hasPassedRule = true + } } - } else if rule.IsOfflineRule() { + continue + } + + duration, ok := rule.DurationInt() + if !ok || duration <= 0 { + continue + } + validRules++ + if rule.IsOfflineRule() { // 离线报警，检查直到最后一次在线的离线采样点是否大于 duration if hasPassedRule = boundCheck(len(points), duration, hasPassedRule); hasPassedRule { continue @@ -157,7 +208,7 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) { var fail int for _, point := range slices.Backward(points[len(points)-duration:]) { fail++ - if point[ruleIndex] { + if ruleIndex >= len(point) || point[ruleIndex] { hasPassedRule = true break } @@ -168,11 +219,7 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) { // 常规报警 // duration<=0 是无意义的规则（持续 0 秒）：直接跳过该规则， // 既不污染 hasPassedRule（否则会连带跳过同一 alert 里其它有效 - // 规则），也避免下方 fail*100/total 在 total=0 时整数除零 panic - // —— checkStatus 无 recover，一次 panic 会拖垮整个告警 goroutine。 - if duration <= 0 { - continue - } + // 规则），也避免下方百分比计算在 total=0 时除零。 if hasPassedRule = boundCheck(len(points), duration, hasPassedRule); hasPassedRule { continue } @@ -181,17 +228,20 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) { } total, fail := duration, 0 for timeTick := len(points) - duration; timeTick < len(points); timeTick++ { - if !points[timeTick][ruleIndex] { + if ruleIndex >= len(points[timeTick]) || !points[timeTick][ruleIndex] { fail++ } } // 当70%以上的采样点未通过规则判断时 才认为当前检查未通过 - if fail*100/total <= 70 { + if float64(fail)*100/float64(total) <= 70 { hasPassedRule = true } } } + if validRules == 0 { + return 0, true + } // 仅当所有检查均未通过时 才触发告警 return slices.Max(durations), hasPassedRule } @@ -205,10 +255,13 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) { func (r *AlertRule) RetentionWindow() int { window := 0 for _, rule := range r.Rules { + if rule == nil || !rule.IsSupportedType() { + continue + } var need int if rule.IsTransferDurationRule() { need = 1 - } else if d := int(rule.Duration); d > 0 { + } else if d, ok := rule.DurationInt(); ok && d > 0 { need = d } if need > window { diff --git a/model/alertrule_security_test.go b/model/alertrule_security_test.go new file mode 100644 index 00000000..8ded87b5 --- /dev/null +++ b/model/alertrule_security_test.go @@ -0,0 +1,123 @@ +package model + +import ( + "math" + "testing" + "time" +) + +func TestRuleSnapshotIgnoresEmptyMetricSamples(t *testing.T) { + tests := []struct { + name string + ruleType string + hostState *HostState + }{ + { + name: "GPU list is empty", + ruleType: "gpu_max", + hostState: &HostState{}, + }, + { + name: "all temperature samples are filtered", + ruleType: "temperature_max", + hostState: &HostState{Temperatures: []SensorTemperature{ + {Name: "coretemp", Temperature: 0}, + }}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := &Server{Common: Common{ID: 1}, State: test.hostState, Host: &Host{}} + rule := &Rule{Type: test.ruleType, Max: 1, Duration: 3, Cover: RuleCoverAll} + + defer func() { + if recovered := recover(); recovered != nil { + t.Fatalf("Snapshot panicked for missing metric data: %v", recovered) + } + }() + if passed := rule.Snapshot(nil, server, nil); !passed { + t.Fatal("missing metric data must be ignored instead of treated as a failed threshold") + } + }) + } +} + +func TestAlertRuleCheckRejectsOverflowingPersistedDuration(t *testing.T) { + rule := &AlertRule{Rules: []*Rule{{ + Type: "offline", + Duration: math.MaxUint64, + Cover: RuleCoverAll, + }}} + + defer func() { + if recovered := recover(); recovered != nil { + t.Fatalf("Check panicked after uint64-to-int overflow: %v", recovered) + } + }() + duration, passed := rule.Check([][]bool{{false}}) + if duration != 0 || !passed { + t.Fatalf("invalid persisted duration must be ignored safely, got duration=%d passed=%v", duration, passed) + } + if window := rule.RetentionWindow(); window != 0 { + t.Fatalf("invalid persisted duration must not create a retention window, got %d", window) + } +} + +func TestAlertRulePersistedDataSafety(t *testing.T) { + cycleStart := time.Now().Add(-time.Hour) + tests := []struct { + name string + rule *AlertRule + want bool + }{ + { + name: "normal rule", + rule: &AlertRule{Rules: []*Rule{{ + Type: "cpu", Duration: 3, Cover: RuleCoverAll, + }}}, + want: true, + }, + { + name: "normal cycle rule", + rule: &AlertRule{Rules: []*Rule{{ + Type: "transfer_in_cycle", CycleStart: &cycleStart, CycleInterval: 1, Cover: RuleCoverAll, + }}}, + want: true, + }, + { + name: "unknown type", + rule: &AlertRule{Rules: []*Rule{{ + Type: "attacker_controlled", Duration: 3, Cover: RuleCoverAll, + }}}, + }, + { + name: "overflowing duration", + rule: &AlertRule{Rules: []*Rule{{ + Type: "offline", Duration: math.MaxUint64, Cover: RuleCoverAll, + }}}, + }, + { + name: "cycle without start", + rule: &AlertRule{Rules: []*Rule{{ + Type: "transfer_in_cycle", CycleInterval: 1, Cover: RuleCoverAll, + }}}, + }, + { + name: "nil rule", + rule: &AlertRule{Rules: []*Rule{nil}}, + }, + { + name: "empty rule list", + rule: &AlertRule{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := test.rule.IsSafeToEvaluate(); got != test.want { + t.Fatalf("IsSafeToEvaluate()=%v, want %v", got, test.want) + } + }) + } +} diff --git a/model/alertrule_test.go b/model/alertrule_test.go index 72cd18df..81ae7b5b 100644 --- a/model/alertrule_test.go +++ b/model/alertrule_test.go @@ -26,7 +26,7 @@ func testCycleRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { - Type: "_cycle", + Type: "transfer_in_cycle", }, }, }, @@ -39,7 +39,7 @@ func testCycleRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { - Type: "_cycle", + Type: "transfer_in_cycle", }, }, }, @@ -130,6 +130,7 @@ func testGeneralRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, }, @@ -143,6 +144,7 @@ func testGeneralRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, }, @@ -156,6 +158,7 @@ func testGeneralRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, }, @@ -169,6 +172,7 @@ func testGeneralRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, }, @@ -182,6 +186,7 @@ func testGeneralRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, }, @@ -210,6 +215,7 @@ func testCombinedRules(t *testing.T) { Duration: 10, }, { + Type: "cpu", Duration: 10, }, }, @@ -227,6 +233,7 @@ func testCombinedRules(t *testing.T) { Duration: 10, }, { + Type: "cpu", Duration: 10, }, }, @@ -240,6 +247,7 @@ func testCombinedRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, { @@ -257,9 +265,11 @@ func testCombinedRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, { + Type: "memory", Duration: 30, }, }, @@ -273,9 +283,11 @@ func testCombinedRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, { + Type: "memory", Duration: 30, }, }, @@ -416,7 +428,7 @@ func TestAlertRule_RetentionWindow(t *testing.T) { {"zero duration only", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 0}}}, 0}, {"mixed picks max", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 0}, {Type: "cpu", Duration: 7}}}, 7}, {"offline keeps Duration", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 30}}}, 30}, - {"cycle looks back one", &AlertRule{Rules: []*Rule{{Type: "net_in_speed_cycle"}}}, 1}, + {"cycle looks back one", &AlertRule{Rules: []*Rule{{Type: "transfer_in_cycle"}}}, 1}, } for _, c := range cases { if got := c.rule.RetentionWindow(); got != c.want { @@ -469,7 +481,7 @@ func TestAlertRule_CombinedRuleAccumulatesSamples(t *testing.T) { }{ {"general3+general10", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 3}, {Type: "memory", Duration: 10}}}, []bool{false, false}, 9, 10}, {"offline5+general10", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 5}, {Type: "cpu", Duration: 10}}}, []bool{false, false}, 9, 10}, - {"transfer+general8", &AlertRule{Rules: []*Rule{{Type: "net_in_speed_cycle"}, {Type: "cpu", Duration: 8}}}, []bool{false, false}, 7, 8}, + {"transfer+general8", &AlertRule{Rules: []*Rule{{Type: "transfer_in_cycle"}, {Type: "cpu", Duration: 8}}}, []bool{false, false}, 7, 8}, {"offline3+offline12", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 3}, {Type: "offline", Duration: 12}}}, []bool{false, false}, 11, 12}, } for _, c := range cases { diff --git a/model/rule.go b/model/rule.go index e069eef5..2d492b10 100644 --- a/model/rule.go +++ b/model/rule.go @@ -15,12 +15,23 @@ const ( RuleCoverIgnoreAll ) +// MaxAlertRuleDuration is the largest duration that can be converted to int +// on every architecture supported by Go. Alert rules are persisted as uint64, +// but their sampling windows use int indexes; keeping the public bound at +// MaxInt32 prevents a crafted value from wrapping during that conversion. +const MaxAlertRuleDuration uint64 = 1<<31 - 1 + +// MaxAlertRuleCycleInterval also leaves room for the largest integer +// multiplication performed by the calendar helpers (weeks become 7*interval) +// on 32-bit targets. +const MaxAlertRuleCycleInterval uint64 = (1<<31 - 1) / 7 + type NResult struct { N uint64 } type Rule struct { - // 指标类型，cpu、memory、swap、disk、net_in_speed、net_out_speed + // 指标类型，cpu、gpu/gpu_max、memory、swap、disk、net_in_speed、net_out_speed // net_all_speed、transfer_in、transfer_out、transfer_all、offline // transfer_in_cycle、transfer_out_cycle、transfer_all_cycle Type string `json:"type"` @@ -45,8 +56,52 @@ func percentage(used, total uint64) float64 { return float64(used) * 100 / float64(total) } +// IsSupportedType reports whether the rule type has an evaluator. Keep this +// list in lockstep with Snapshot's switch so untrusted strings cannot enter the +// persisted alert pipeline and silently acquire fallback semantics. +func (u *Rule) IsSupportedType() bool { + if u == nil { + return false + } + switch u.Type { + case "cpu", "gpu", "gpu_max", "memory", "swap", "disk", + "net_in_speed", "net_out_speed", "net_all_speed", + "transfer_in", "transfer_out", "transfer_all", "offline", + "transfer_in_cycle", "transfer_out_cycle", "transfer_all_cycle", + "load1", "load5", "load15", "tcp_conn_count", "udp_conn_count", + "process_count", "temperature_max": + return true + default: + return false + } +} + +// DurationInt converts the persisted duration without allowing uint64-to-int +// truncation. Callers must handle ok=false as an invalid rule. +func (u *Rule) DurationInt() (duration int, ok bool) { + if u == nil || u.Duration > MaxAlertRuleDuration { + return 0, false + } + return int(u.Duration), true +} + +// HasSafeCycleConfiguration guards every value that the cycle evaluator later +// converts or dereferences. It intentionally does not reject an empty unit, +// which is the documented legacy spelling for hours. +func (u *Rule) HasSafeCycleConfiguration() bool { + return u != nil && u.IsTransferDurationRule() && u.CycleStart != nil && + u.CycleInterval > 0 && u.CycleInterval <= MaxAlertRuleCycleInterval +} + // Snapshot 未通过规则返回 false, 通过返回 true func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) bool { + if u == nil || server == nil || !u.IsSupportedType() { + return true + } + if u.IsTransferDurationRule() && (!u.HasSafeCycleConfiguration() || cycleTransferStats == nil) { + return true + } + // 监控全部但是排除了此服务器 if u.Cover == RuleCoverAll && u.Ignore[server.ID] { return true @@ -71,7 +126,10 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, switch u.Type { case "cpu": src = float64(state.CPU) - case "gpu_max": + case "gpu", "gpu_max": + if len(state.GPU) == 0 { + return true + } src = slices.Max(state.GPU) case "memory": if runtime.Host == nil { @@ -141,14 +199,17 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, src = float64(state.ProcessCount) case "temperature_max": var temp []float64 - if state.Temperatures != nil { - for _, tempStat := range state.Temperatures { - if tempStat.Temperature != 0 { - temp = append(temp, tempStat.Temperature) - } + for _, tempStat := range state.Temperatures { + if tempStat.Temperature != 0 { + temp = append(temp, tempStat.Temperature) } - src = slices.Max(temp) } + if len(temp) == 0 { + return true + } + src = slices.Max(temp) + default: + return true } // 循环区间流量检测 · 更新下次需要检测时间 @@ -187,11 +248,19 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, // IsTransferDurationRule 判断该规则是否属于周期流量规则 属于则返回true func (u *Rule) IsTransferDurationRule() bool { - return strings.HasSuffix(u.Type, "_cycle") + if u == nil { + return false + } + switch u.Type { + case "transfer_in_cycle", "transfer_out_cycle", "transfer_all_cycle": + return true + default: + return false + } } func (u *Rule) IsOfflineRule() bool { - return u.Type == "offline" + return u != nil && u.Type == "offline" } // GetTransferDurationStart 获取周期流量的起始时间 diff --git a/service/singleton/alertsentinel.go b/service/singleton/alertsentinel.go index b17aeed8..82f56846 100644 --- a/service/singleton/alertsentinel.go +++ b/service/singleton/alertsentinel.go @@ -33,7 +33,7 @@ var ( // addCycleTransferStatsInfo 向AlertsCycleTransferStatsStore中添加周期流量报警统计信息 func addCycleTransferStatsInfo(alert *model.AlertRule) { - if !alert.Enabled() { + if alert == nil || !alert.Enabled() || !alert.IsSafeToEvaluate() { return } for _, rule := range alert.Rules { @@ -67,8 +67,16 @@ func AlertSentinelStart() { panic(err) } for _, alert := range Alerts { + if alert == nil { + log.Printf("NEZHA>> Skipping invalid nil alert rule loaded from database") + continue + } alertsStore[alert.ID] = make(map[uint64][][]bool) alertsPrevState[alert.ID] = make(map[uint64]uint8) + if !alert.IsSafeToEvaluate() { + log.Printf("NEZHA>> Skipping invalid alert rule %d loaded from database", alert.ID) + continue + } addCycleTransferStatsInfo(alert) } AlertsLock.Unlock() @@ -136,7 +144,7 @@ func checkStatus() { for _, alert := range Alerts { // 跳过未启用 - if !alert.Enabled() { + if alert == nil || !alert.Enabled() || !alert.IsSafeToEvaluate() { continue } for _, server := range m { @@ -152,48 +160,62 @@ func checkStatus() { if alert.UserID != server.GetUserID() && !role.IsAdmin() { continue } - alertsStore[alert.ID][server.ID] = append(alertsStore[alert. - ID][server.ID], alert.Snapshot(AlertsCycleTransferStatsStore[alert.ID], server, DB)) - // 发送通知，分为触发报警和恢复通知 - _, passed := alert.Check(alertsStore[alert.ID][server.ID]) - // 保存当前服务器状态信息 - curServer := model.Server{} - copier.Copy(&curServer, server) - - // 本次未通过检查 - if !passed { - // 始终触发模式或上次检查不为失败时触发报警（跳过单次触发+上次失败的情况） - if alert.TriggerMode == model.ModeAlwaysTrigger || alertsPrevState[alert.ID][server.ID] != _RuleCheckFail { - alertsPrevState[alert.ID][server.ID] = _RuleCheckFail - message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Incident"), - server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name) - go CronShared.SendTriggerTasks(alert.FailTriggerTasks, curServer.ID, alert.UserID) - go NotificationShared.SendNotification(alert.NotificationGroupID, message, NotificationMuteLabel.ServerIncident(server.ID, alert.ID), &curServer) - // 清除恢复通知的静音缓存 - NotificationShared.UnMuteNotification(alert.NotificationGroupID, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID)) - } - } else { - // 本次通过检查但上一次的状态为失败，则发送恢复通知 - if alertsPrevState[alert.ID][server.ID] == _RuleCheckFail { - message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Resolved"), - server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name) - go CronShared.SendTriggerTasks(alert.RecoverTriggerTasks, curServer.ID, alert.UserID) - go NotificationShared.SendNotification(alert.NotificationGroupID, message, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), &curServer) - // 清除失败通知的静音缓存 - NotificationShared.UnMuteNotification(alert.NotificationGroupID, NotificationMuteLabel.ServerIncident(server.ID, alert.ID)) - } - alertsPrevState[alert.ID][server.ID] = _RuleCheckPass - } - // 清理旧数据：保留窗口由规则定义决定（各规则 Duration 的最大值）， - // 而非 Check 的判定结果。window==0 表示没有任何有效规则需要回看历史 - // （例如全部 Duration<=0），此时清空采样避免切片无限增长。 - window := alert.RetentionWindow() - samples := alertsStore[alert.ID][server.ID] - if window <= 0 { - alertsStore[alert.ID][server.ID] = samples[:0] - } else if window < len(samples) { - alertsStore[alert.ID][server.ID] = samples[len(samples)-window:] - } + checkStatusForServer(alert, server) } } } + +// checkStatusForServer isolates each alert/server evaluation. Input validation +// and model-level guards handle known malformed states; this recovery boundary +// ensures an unexpected evaluator panic cannot terminate the dashboard process +// or prevent unrelated servers from being checked on the same tick. +func checkStatusForServer(alert *model.AlertRule, server *model.Server) { + defer func() { + if recovered := recover(); recovered != nil { + log.Printf("NEZHA>> Recovered panic evaluating alert rule %d for server %d: %v", alert.ID, server.ID, recovered) + } + }() + + alertsStore[alert.ID][server.ID] = append(alertsStore[alert. + ID][server.ID], alert.Snapshot(AlertsCycleTransferStatsStore[alert.ID], server, DB)) + // 发送通知，分为触发报警和恢复通知 + _, passed := alert.Check(alertsStore[alert.ID][server.ID]) + // 保存当前服务器状态信息 + curServer := model.Server{} + copier.Copy(&curServer, server) + + // 本次未通过检查 + if !passed { + // 始终触发模式或上次检查不为失败时触发报警（跳过单次触发+上次失败的情况） + if alert.TriggerMode == model.ModeAlwaysTrigger || alertsPrevState[alert.ID][server.ID] != _RuleCheckFail { + alertsPrevState[alert.ID][server.ID] = _RuleCheckFail + message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Incident"), + server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name) + go CronShared.SendTriggerTasks(alert.FailTriggerTasks, curServer.ID, alert.UserID) + go NotificationShared.SendNotification(alert.NotificationGroupID, message, NotificationMuteLabel.ServerIncident(server.ID, alert.ID), &curServer) + // 清除恢复通知的静音缓存 + NotificationShared.UnMuteNotification(alert.NotificationGroupID, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID)) + } + } else { + // 本次通过检查但上一次的状态为失败，则发送恢复通知 + if alertsPrevState[alert.ID][server.ID] == _RuleCheckFail { + message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Resolved"), + server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name) + go CronShared.SendTriggerTasks(alert.RecoverTriggerTasks, curServer.ID, alert.UserID) + go NotificationShared.SendNotification(alert.NotificationGroupID, message, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), &curServer) + // 清除失败通知的静音缓存 + NotificationShared.UnMuteNotification(alert.NotificationGroupID, NotificationMuteLabel.ServerIncident(server.ID, alert.ID)) + } + alertsPrevState[alert.ID][server.ID] = _RuleCheckPass + } + // 清理旧数据：保留窗口由规则定义决定（各规则 Duration 的最大值）， + // 而非 Check 的判定结果。window==0 表示没有任何有效规则需要回看历史 + // （例如全部 Duration<=0），此时清空采样避免切片无限增长。 + window := alert.RetentionWindow() + samples := alertsStore[alert.ID][server.ID] + if window <= 0 { + alertsStore[alert.ID][server.ID] = samples[:0] + } else if window < len(samples) { + alertsStore[alert.ID][server.ID] = samples[len(samples)-window:] + } +} diff --git a/service/singleton/alertsentinel_test.go b/service/singleton/alertsentinel_test.go index 1d290523..e227e57b 100644 --- a/service/singleton/alertsentinel_test.go +++ b/service/singleton/alertsentinel_test.go @@ -117,3 +117,53 @@ func TestCheckStatus_SampleMemoryBounded(t *testing.T) { t.Fatalf("sample capacity grew unbounded: peak cap %d exceeds 4x window %d", peakCap, duration*4) } } + +func TestCheckStatusForServerRecoversUnexpectedEvaluatorPanic(t *testing.T) { + originalStore := alertsStore + originalPrevState := alertsPrevState + originalCycleStore := AlertsCycleTransferStatsStore + alertsStore = map[uint64]map[uint64][][]bool{} + alertsPrevState = map[uint64]map[uint64]uint8{} + AlertsCycleTransferStatsStore = map[uint64]*model.CycleTransferStats{} + t.Cleanup(func() { + alertsStore = originalStore + alertsPrevState = originalPrevState + AlertsCycleTransferStatsStore = originalCycleStore + }) + + enabled := true + alert := &model.AlertRule{ + Common: model.Common{ID: 77, UserID: 2}, + Enable: &enabled, + Rules: []*model.Rule{{Type: "cpu", Duration: 3, Cover: model.RuleCoverAll}}, + } + server := &model.Server{Common: model.Common{ID: 1, UserID: 2}} + model.InitServer(server) + + // The deliberately missing alertsStore[alert.ID] map panics at assignment + // after evaluation. The per-alert/server recovery boundary must contain it. + checkStatusForServer(alert, server) +} + +func TestAddCycleTransferStatsSkipsPoisonedPersistedRule(t *testing.T) { + original := AlertsCycleTransferStatsStore + AlertsCycleTransferStatsStore = map[uint64]*model.CycleTransferStats{} + t.Cleanup(func() { AlertsCycleTransferStatsStore = original }) + + enabled := true + alert := &model.AlertRule{ + Common: model.Common{ID: 88}, + Enable: &enabled, + Rules: []*model.Rule{{ + Type: "transfer_in_cycle", + CycleInterval: 1, + Cover: model.RuleCoverAll, + // CycleStart intentionally nil, as it may be in a poisoned legacy DB row. + }}, + } + + addCycleTransferStatsInfo(alert) + if _, exists := AlertsCycleTransferStatsStore[alert.ID]; exists { + t.Fatal("invalid persisted cycle rule must not be scheduled") + } +}