fix(alert): keep sample history by rule-defined retention window

Trim alert samples using AlertRule.RetentionWindow() (max rule duration)
instead of Check()'s verdict max, which is 0 while a rule's window is still
filling. The old max<=0 trim wiped history every tick, so Duration>=2 general
rules never accumulated enough samples and never fired a notification.

Add model + end-to-end singleton regression tests covering sample
accumulation, the retention-window contract, and the actual notify path.
This commit is contained in:
naiba
2026-06-07 16:11:43 +00:00
parent 66de244c2e
commit e756b4540c
4 changed files with 178 additions and 8 deletions
+8 -8
View File
@@ -155,7 +155,7 @@ func checkStatus() {
alertsStore[alert.ID][server.ID] = append(alertsStore[alert.
ID][server.ID], alert.Snapshot(AlertsCycleTransferStatsStore[alert.ID], server, DB))
// 发送通知,分为触发报警和恢复通知
max, passed := alert.Check(alertsStore[alert.ID][server.ID])
_, passed := alert.Check(alertsStore[alert.ID][server.ID])
// 保存当前服务器状态信息
curServer := model.Server{}
copier.Copy(&curServer, server)
@@ -184,15 +184,15 @@ func checkStatus() {
}
alertsPrevState[alert.ID][server.ID] = _RuleCheckPass
}
// 清理旧数据:只需保留最近 max 个采样点(各规则 Duration 的最大值)
// max==0 表示该 alert 没有任何有效规则回看历史(例如全部
// Duration<=0 被跳过),此时历史采样无用,必须清空——否则每个
// tick 都 append 而永不裁剪,切片会无限增长成内存泄漏。
// 清理旧数据:保留窗口由规则定义决定(各规则 Duration 的最大值)
// 而非 Check 的判定结果。window==0 表示没有任何有效规则需要回看历史
// (例如全部 Duration<=0),此时清空采样避免切片无限增长。
window := alert.RetentionWindow()
samples := alertsStore[alert.ID][server.ID]
if max <= 0 {
if window <= 0 {
alertsStore[alert.ID][server.ID] = samples[:0]
} else if max < len(samples) {
alertsStore[alert.ID][server.ID] = samples[len(samples)-max:]
} else if window < len(samples) {
alertsStore[alert.ID][server.ID] = samples[len(samples)-window:]
}
}
}
+87
View File
@@ -0,0 +1,87 @@
package singleton
import (
"testing"
"github.com/nezhahq/nezha/model"
)
// notifyDecision replays the exact send-gate from checkStatus (lines 164-186)
// for a single (alert, server) pair: given the current Check verdict and the
// previous stored state, it reports whether an incident or recovery notification
// would be dispatched and what the next stored state becomes. Kept in lockstep
// with checkStatus so the end-to-end "does it actually notify" path is testable
// without the DB/global singletons checkStatus pulls in.
func notifyDecision(triggerMode uint8, passed bool, prev uint8) (incident, recover bool, next uint8) {
if !passed {
if triggerMode == model.ModeAlwaysTrigger || prev != _RuleCheckFail {
return true, false, _RuleCheckFail
}
return false, false, _RuleCheckFail
}
if prev == _RuleCheckFail {
return false, true, _RuleCheckPass
}
return false, false, _RuleCheckPass
}
// driveCheckStatus simulates the checkStatus tick loop end-to-end: each tick
// appends one sample, runs the real Check, applies the real RetentionWindow
// trim, then runs the send-gate. It returns how many incident notifications
// would have been dispatched across all ticks and the final sample window size.
func driveCheckStatus(rule *model.AlertRule, triggerMode uint8, ticks int, sample []bool) (incidents int, finalWindow int) {
var samples [][]bool
prev := uint8(_RuleCheckNoData)
for i := 0; i < ticks; i++ {
samples = append(samples, append([]bool(nil), sample...))
_, passed := rule.Check(samples)
w := rule.RetentionWindow()
if w <= 0 {
samples = samples[:0]
} else if w < len(samples) {
samples = samples[len(samples)-w:]
}
incident, _, next := notifyDecision(triggerMode, passed, prev)
if incident {
incidents++
}
prev = next
}
return incidents, len(samples)
}
// TestCheckStatus_GeneralRuleFiresIncident is the end-to-end guard for the
// regression: a Duration:10 rule on a server that fails every tick must,
// after the window fills, reach passed=false and actually dispatch an incident
// notification. Before the fix the window was wiped each tick, so passed never
// became false and zero notifications were sent.
func TestCheckStatus_GeneralRuleFiresIncident(t *testing.T) {
rule := &model.AlertRule{Rules: []*model.Rule{{Type: "cpu", Duration: 10}}}
t.Run("AlwaysTrigger fires repeatedly once window fills", func(t *testing.T) {
incidents, window := driveCheckStatus(rule, model.ModeAlwaysTrigger, 30, []bool{false})
if window < 10 {
t.Fatalf("window never filled: got %d want >= 10", window)
}
if incidents == 0 {
t.Fatalf("AlwaysTrigger rule never dispatched an incident notification")
}
})
t.Run("OnetimeTrigger fires exactly once", func(t *testing.T) {
incidents, _ := driveCheckStatus(rule, model.ModeOnetimeTrigger, 30, []bool{false})
if incidents != 1 {
t.Fatalf("OnetimeTrigger must dispatch exactly one incident, got %d", incidents)
}
})
}
// TestCheckStatus_HealthyServerStaysSilent guards the other direction: a server
// passing every tick must never dispatch an incident.
func TestCheckStatus_HealthyServerStaysSilent(t *testing.T) {
rule := &model.AlertRule{Rules: []*model.Rule{{Type: "cpu", Duration: 10}}}
incidents, _ := driveCheckStatus(rule, model.ModeAlwaysTrigger, 30, []bool{true})
if incidents != 0 {
t.Fatalf("a healthy server must never trigger an incident, got %d", incidents)
}
}