diff --git a/model/alertrule.go b/model/alertrule.go index 29819900..e47668b2 100644 --- a/model/alertrule.go +++ b/model/alertrule.go @@ -196,6 +196,25 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) { return slices.Max(durations), hasPassedRule } +// RetentionWindow 返回保留历史采样所需的长度(各规则窗口的最大值),只依赖 +// 规则定义而非 Check 的判定结果——否则窗口未填满时 Check 返回的 max=0 会被 +// 误判为"无需历史"而清空采样,使规则永远攒不够样本。 +func (r *AlertRule) RetentionWindow() int { + window := 0 + for _, rule := range r.Rules { + var need int + if rule.IsTransferDurationRule() || rule.IsOfflineRule() { + need = 1 + } else if d := int(rule.Duration); d > 0 { + need = d + } + if need > window { + window = need + } + } + return window +} + func boundCheck(length, duration int, passed bool) bool { if passed { return true diff --git a/model/alertrule_test.go b/model/alertrule_test.go index 5b0c1b35..393c16bc 100644 --- a/model/alertrule_test.go +++ b/model/alertrule_test.go @@ -360,3 +360,67 @@ func TestAlertRule_ZeroDurationMixedWithValidRule(t *testing.T) { t.Fatalf("the valid Duration:3 rule must still set max=3, got %d", maxD) } } + +// trimSamples mirrors singleton.checkStatus retention: keep the most recent +// `window` samples, clear when window<=0. window comes from RetentionWindow(), +// the production code under test. +func trimSamples(samples [][]bool, window int) [][]bool { + if window <= 0 { + return samples[:0] + } else if window < len(samples) { + return samples[len(samples)-window:] + } + return samples +} + +// TestAlertRule_GeneralRuleAccumulatesSamples is a regression guard: a normal +// Duration>1 general rule must be able to fire. checkStatus appends one sample +// per tick then trims to the retention window; if the window is derived from +// Check's verdict (which is 0 while the rule is still filling) the history is +// wiped every tick, the window never reaches Duration, and the alert never +// raises. RetentionWindow() must keep enough samples for the rule to converge. +func TestAlertRule_GeneralRuleAccumulatesSamples(t *testing.T) { + const duration = 10 + rule := &AlertRule{ + Rules: []*Rule{{Type: "cpu", Duration: duration}}, + } + + var samples [][]bool + var lastPassed bool + maxLen := 0 + for tick := 0; tick < duration*3; tick++ { + samples = append(samples, []bool{false}) // failing sample + _, lastPassed = rule.Check(samples) + samples = trimSamples(samples, rule.RetentionWindow()) + if len(samples) > maxLen { + maxLen = len(samples) + } + } + + if maxLen < duration { + t.Fatalf("samples never accumulated to Duration: max window reached %d, want >= %d", maxLen, duration) + } + if lastPassed { + t.Fatalf("a server failing every tick must eventually fail the check (passed=false), got passed=true") + } +} + +// TestAlertRule_RetentionWindow pins the retention contract directly. +func TestAlertRule_RetentionWindow(t *testing.T) { + cases := []struct { + msg string + rule *AlertRule + want int + }{ + {"single general", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 10}}}, 10}, + {"zero duration only", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 0}}}, 0}, + {"mixed picks max", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 0}, {Type: "cpu", Duration: 7}}}, 7}, + {"offline looks back one", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 30}}}, 1}, + {"cycle looks back one", &AlertRule{Rules: []*Rule{{Type: "net_in_speed_cycle"}}}, 1}, + } + for _, c := range cases { + if got := c.rule.RetentionWindow(); got != c.want { + t.Fatalf("%s: RetentionWindow()=%d want %d", c.msg, got, c.want) + } + } +} diff --git a/service/singleton/alertsentinel.go b/service/singleton/alertsentinel.go index 6af6c6fe..b17aeed8 100644 --- a/service/singleton/alertsentinel.go +++ b/service/singleton/alertsentinel.go @@ -155,7 +155,7 @@ func checkStatus() { alertsStore[alert.ID][server.ID] = append(alertsStore[alert. ID][server.ID], alert.Snapshot(AlertsCycleTransferStatsStore[alert.ID], server, DB)) // 发送通知,分为触发报警和恢复通知 - max, passed := alert.Check(alertsStore[alert.ID][server.ID]) + _, passed := alert.Check(alertsStore[alert.ID][server.ID]) // 保存当前服务器状态信息 curServer := model.Server{} copier.Copy(&curServer, server) @@ -184,15 +184,15 @@ func checkStatus() { } alertsPrevState[alert.ID][server.ID] = _RuleCheckPass } - // 清理旧数据:只需保留最近 max 个采样点(各规则 Duration 的最大值)。 - // max==0 表示该 alert 没有任何有效规则会回看历史(例如全部 - // Duration<=0 被跳过),此时历史采样无用,必须清空——否则每个 - // tick 都 append 而永不裁剪,切片会无限增长成内存泄漏。 + // 清理旧数据:保留窗口由规则定义决定(各规则 Duration 的最大值), + // 而非 Check 的判定结果。window==0 表示没有任何有效规则需要回看历史 + // (例如全部 Duration<=0),此时清空采样避免切片无限增长。 + window := alert.RetentionWindow() samples := alertsStore[alert.ID][server.ID] - if max <= 0 { + if window <= 0 { alertsStore[alert.ID][server.ID] = samples[:0] - } else if max < len(samples) { - alertsStore[alert.ID][server.ID] = samples[len(samples)-max:] + } else if window < len(samples) { + alertsStore[alert.ID][server.ID] = samples[len(samples)-window:] } } } diff --git a/service/singleton/alertsentinel_test.go b/service/singleton/alertsentinel_test.go new file mode 100644 index 00000000..c2ff8e34 --- /dev/null +++ b/service/singleton/alertsentinel_test.go @@ -0,0 +1,87 @@ +package singleton + +import ( + "testing" + + "github.com/nezhahq/nezha/model" +) + +// notifyDecision replays the exact send-gate from checkStatus (lines 164-186) +// for a single (alert, server) pair: given the current Check verdict and the +// previous stored state, it reports whether an incident or recovery notification +// would be dispatched and what the next stored state becomes. Kept in lockstep +// with checkStatus so the end-to-end "does it actually notify" path is testable +// without the DB/global singletons checkStatus pulls in. +func notifyDecision(triggerMode uint8, passed bool, prev uint8) (incident, recover bool, next uint8) { + if !passed { + if triggerMode == model.ModeAlwaysTrigger || prev != _RuleCheckFail { + return true, false, _RuleCheckFail + } + return false, false, _RuleCheckFail + } + if prev == _RuleCheckFail { + return false, true, _RuleCheckPass + } + return false, false, _RuleCheckPass +} + +// driveCheckStatus simulates the checkStatus tick loop end-to-end: each tick +// appends one sample, runs the real Check, applies the real RetentionWindow +// trim, then runs the send-gate. It returns how many incident notifications +// would have been dispatched across all ticks and the final sample window size. +func driveCheckStatus(rule *model.AlertRule, triggerMode uint8, ticks int, sample []bool) (incidents int, finalWindow int) { + var samples [][]bool + prev := uint8(_RuleCheckNoData) + for i := 0; i < ticks; i++ { + samples = append(samples, append([]bool(nil), sample...)) + _, passed := rule.Check(samples) + w := rule.RetentionWindow() + if w <= 0 { + samples = samples[:0] + } else if w < len(samples) { + samples = samples[len(samples)-w:] + } + incident, _, next := notifyDecision(triggerMode, passed, prev) + if incident { + incidents++ + } + prev = next + } + return incidents, len(samples) +} + +// TestCheckStatus_GeneralRuleFiresIncident is the end-to-end guard for the +// regression: a Duration:10 rule on a server that fails every tick must, +// after the window fills, reach passed=false and actually dispatch an incident +// notification. Before the fix the window was wiped each tick, so passed never +// became false and zero notifications were sent. +func TestCheckStatus_GeneralRuleFiresIncident(t *testing.T) { + rule := &model.AlertRule{Rules: []*model.Rule{{Type: "cpu", Duration: 10}}} + + t.Run("AlwaysTrigger fires repeatedly once window fills", func(t *testing.T) { + incidents, window := driveCheckStatus(rule, model.ModeAlwaysTrigger, 30, []bool{false}) + if window < 10 { + t.Fatalf("window never filled: got %d want >= 10", window) + } + if incidents == 0 { + t.Fatalf("AlwaysTrigger rule never dispatched an incident notification") + } + }) + + t.Run("OnetimeTrigger fires exactly once", func(t *testing.T) { + incidents, _ := driveCheckStatus(rule, model.ModeOnetimeTrigger, 30, []bool{false}) + if incidents != 1 { + t.Fatalf("OnetimeTrigger must dispatch exactly one incident, got %d", incidents) + } + }) +} + +// TestCheckStatus_HealthyServerStaysSilent guards the other direction: a server +// passing every tick must never dispatch an incident. +func TestCheckStatus_HealthyServerStaysSilent(t *testing.T) { + rule := &model.AlertRule{Rules: []*model.Rule{{Type: "cpu", Duration: 10}}} + incidents, _ := driveCheckStatus(rule, model.ModeAlwaysTrigger, 30, []bool{true}) + if incidents != 0 { + t.Fatalf("a healthy server must never trigger an incident, got %d", incidents) + } +}