mirror of
https://github.com/Buriburizaem0n/nezha_domains.git
synced 2026-09-21 18:50:13 +00:00
fix(alert): keep sample history by rule-defined retention window
Trim alert samples using AlertRule.RetentionWindow() (max rule duration) instead of Check()'s verdict max, which is 0 while a rule's window is still filling. The old max<=0 trim wiped history every tick, so Duration>=2 general rules never accumulated enough samples and never fired a notification. Add model + end-to-end singleton regression tests covering sample accumulation, the retention-window contract, and the actual notify path.
This commit is contained in:
@@ -196,6 +196,25 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) {
|
|||||||
return slices.Max(durations), hasPassedRule
|
return slices.Max(durations), hasPassedRule
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RetentionWindow 返回保留历史采样所需的长度(各规则窗口的最大值),只依赖
|
||||||
|
// 规则定义而非 Check 的判定结果——否则窗口未填满时 Check 返回的 max=0 会被
|
||||||
|
// 误判为"无需历史"而清空采样,使规则永远攒不够样本。
|
||||||
|
func (r *AlertRule) RetentionWindow() int {
|
||||||
|
window := 0
|
||||||
|
for _, rule := range r.Rules {
|
||||||
|
var need int
|
||||||
|
if rule.IsTransferDurationRule() || rule.IsOfflineRule() {
|
||||||
|
need = 1
|
||||||
|
} else if d := int(rule.Duration); d > 0 {
|
||||||
|
need = d
|
||||||
|
}
|
||||||
|
if need > window {
|
||||||
|
window = need
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return window
|
||||||
|
}
|
||||||
|
|
||||||
func boundCheck(length, duration int, passed bool) bool {
|
func boundCheck(length, duration int, passed bool) bool {
|
||||||
if passed {
|
if passed {
|
||||||
return true
|
return true
|
||||||
|
|||||||
@@ -360,3 +360,67 @@ func TestAlertRule_ZeroDurationMixedWithValidRule(t *testing.T) {
|
|||||||
t.Fatalf("the valid Duration:3 rule must still set max=3, got %d", maxD)
|
t.Fatalf("the valid Duration:3 rule must still set max=3, got %d", maxD)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// trimSamples mirrors singleton.checkStatus retention: keep the most recent
|
||||||
|
// `window` samples, clear when window<=0. window comes from RetentionWindow(),
|
||||||
|
// the production code under test.
|
||||||
|
func trimSamples(samples [][]bool, window int) [][]bool {
|
||||||
|
if window <= 0 {
|
||||||
|
return samples[:0]
|
||||||
|
} else if window < len(samples) {
|
||||||
|
return samples[len(samples)-window:]
|
||||||
|
}
|
||||||
|
return samples
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAlertRule_GeneralRuleAccumulatesSamples is a regression guard: a normal
|
||||||
|
// Duration>1 general rule must be able to fire. checkStatus appends one sample
|
||||||
|
// per tick then trims to the retention window; if the window is derived from
|
||||||
|
// Check's verdict (which is 0 while the rule is still filling) the history is
|
||||||
|
// wiped every tick, the window never reaches Duration, and the alert never
|
||||||
|
// raises. RetentionWindow() must keep enough samples for the rule to converge.
|
||||||
|
func TestAlertRule_GeneralRuleAccumulatesSamples(t *testing.T) {
|
||||||
|
const duration = 10
|
||||||
|
rule := &AlertRule{
|
||||||
|
Rules: []*Rule{{Type: "cpu", Duration: duration}},
|
||||||
|
}
|
||||||
|
|
||||||
|
var samples [][]bool
|
||||||
|
var lastPassed bool
|
||||||
|
maxLen := 0
|
||||||
|
for tick := 0; tick < duration*3; tick++ {
|
||||||
|
samples = append(samples, []bool{false}) // failing sample
|
||||||
|
_, lastPassed = rule.Check(samples)
|
||||||
|
samples = trimSamples(samples, rule.RetentionWindow())
|
||||||
|
if len(samples) > maxLen {
|
||||||
|
maxLen = len(samples)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if maxLen < duration {
|
||||||
|
t.Fatalf("samples never accumulated to Duration: max window reached %d, want >= %d", maxLen, duration)
|
||||||
|
}
|
||||||
|
if lastPassed {
|
||||||
|
t.Fatalf("a server failing every tick must eventually fail the check (passed=false), got passed=true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAlertRule_RetentionWindow pins the retention contract directly.
|
||||||
|
func TestAlertRule_RetentionWindow(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
msg string
|
||||||
|
rule *AlertRule
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{"single general", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 10}}}, 10},
|
||||||
|
{"zero duration only", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 0}}}, 0},
|
||||||
|
{"mixed picks max", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 0}, {Type: "cpu", Duration: 7}}}, 7},
|
||||||
|
{"offline looks back one", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 30}}}, 1},
|
||||||
|
{"cycle looks back one", &AlertRule{Rules: []*Rule{{Type: "net_in_speed_cycle"}}}, 1},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
if got := c.rule.RetentionWindow(); got != c.want {
|
||||||
|
t.Fatalf("%s: RetentionWindow()=%d want %d", c.msg, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -155,7 +155,7 @@ func checkStatus() {
|
|||||||
alertsStore[alert.ID][server.ID] = append(alertsStore[alert.
|
alertsStore[alert.ID][server.ID] = append(alertsStore[alert.
|
||||||
ID][server.ID], alert.Snapshot(AlertsCycleTransferStatsStore[alert.ID], server, DB))
|
ID][server.ID], alert.Snapshot(AlertsCycleTransferStatsStore[alert.ID], server, DB))
|
||||||
// 发送通知,分为触发报警和恢复通知
|
// 发送通知,分为触发报警和恢复通知
|
||||||
max, passed := alert.Check(alertsStore[alert.ID][server.ID])
|
_, passed := alert.Check(alertsStore[alert.ID][server.ID])
|
||||||
// 保存当前服务器状态信息
|
// 保存当前服务器状态信息
|
||||||
curServer := model.Server{}
|
curServer := model.Server{}
|
||||||
copier.Copy(&curServer, server)
|
copier.Copy(&curServer, server)
|
||||||
@@ -184,15 +184,15 @@ func checkStatus() {
|
|||||||
}
|
}
|
||||||
alertsPrevState[alert.ID][server.ID] = _RuleCheckPass
|
alertsPrevState[alert.ID][server.ID] = _RuleCheckPass
|
||||||
}
|
}
|
||||||
// 清理旧数据:只需保留最近 max 个采样点(各规则 Duration 的最大值)。
|
// 清理旧数据:保留窗口由规则定义决定(各规则 Duration 的最大值),
|
||||||
// max==0 表示该 alert 没有任何有效规则会回看历史(例如全部
|
// 而非 Check 的判定结果。window==0 表示没有任何有效规则需要回看历史
|
||||||
// Duration<=0 被跳过),此时历史采样无用,必须清空——否则每个
|
// (例如全部 Duration<=0),此时清空采样避免切片无限增长。
|
||||||
// tick 都 append 而永不裁剪,切片会无限增长成内存泄漏。
|
window := alert.RetentionWindow()
|
||||||
samples := alertsStore[alert.ID][server.ID]
|
samples := alertsStore[alert.ID][server.ID]
|
||||||
if max <= 0 {
|
if window <= 0 {
|
||||||
alertsStore[alert.ID][server.ID] = samples[:0]
|
alertsStore[alert.ID][server.ID] = samples[:0]
|
||||||
} else if max < len(samples) {
|
} else if window < len(samples) {
|
||||||
alertsStore[alert.ID][server.ID] = samples[len(samples)-max:]
|
alertsStore[alert.ID][server.ID] = samples[len(samples)-window:]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,87 @@
|
|||||||
|
package singleton
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/nezhahq/nezha/model"
|
||||||
|
)
|
||||||
|
|
||||||
|
// notifyDecision replays the exact send-gate from checkStatus (lines 164-186)
|
||||||
|
// for a single (alert, server) pair: given the current Check verdict and the
|
||||||
|
// previous stored state, it reports whether an incident or recovery notification
|
||||||
|
// would be dispatched and what the next stored state becomes. Kept in lockstep
|
||||||
|
// with checkStatus so the end-to-end "does it actually notify" path is testable
|
||||||
|
// without the DB/global singletons checkStatus pulls in.
|
||||||
|
func notifyDecision(triggerMode uint8, passed bool, prev uint8) (incident, recover bool, next uint8) {
|
||||||
|
if !passed {
|
||||||
|
if triggerMode == model.ModeAlwaysTrigger || prev != _RuleCheckFail {
|
||||||
|
return true, false, _RuleCheckFail
|
||||||
|
}
|
||||||
|
return false, false, _RuleCheckFail
|
||||||
|
}
|
||||||
|
if prev == _RuleCheckFail {
|
||||||
|
return false, true, _RuleCheckPass
|
||||||
|
}
|
||||||
|
return false, false, _RuleCheckPass
|
||||||
|
}
|
||||||
|
|
||||||
|
// driveCheckStatus simulates the checkStatus tick loop end-to-end: each tick
|
||||||
|
// appends one sample, runs the real Check, applies the real RetentionWindow
|
||||||
|
// trim, then runs the send-gate. It returns how many incident notifications
|
||||||
|
// would have been dispatched across all ticks and the final sample window size.
|
||||||
|
func driveCheckStatus(rule *model.AlertRule, triggerMode uint8, ticks int, sample []bool) (incidents int, finalWindow int) {
|
||||||
|
var samples [][]bool
|
||||||
|
prev := uint8(_RuleCheckNoData)
|
||||||
|
for i := 0; i < ticks; i++ {
|
||||||
|
samples = append(samples, append([]bool(nil), sample...))
|
||||||
|
_, passed := rule.Check(samples)
|
||||||
|
w := rule.RetentionWindow()
|
||||||
|
if w <= 0 {
|
||||||
|
samples = samples[:0]
|
||||||
|
} else if w < len(samples) {
|
||||||
|
samples = samples[len(samples)-w:]
|
||||||
|
}
|
||||||
|
incident, _, next := notifyDecision(triggerMode, passed, prev)
|
||||||
|
if incident {
|
||||||
|
incidents++
|
||||||
|
}
|
||||||
|
prev = next
|
||||||
|
}
|
||||||
|
return incidents, len(samples)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCheckStatus_GeneralRuleFiresIncident is the end-to-end guard for the
|
||||||
|
// regression: a Duration:10 rule on a server that fails every tick must,
|
||||||
|
// after the window fills, reach passed=false and actually dispatch an incident
|
||||||
|
// notification. Before the fix the window was wiped each tick, so passed never
|
||||||
|
// became false and zero notifications were sent.
|
||||||
|
func TestCheckStatus_GeneralRuleFiresIncident(t *testing.T) {
|
||||||
|
rule := &model.AlertRule{Rules: []*model.Rule{{Type: "cpu", Duration: 10}}}
|
||||||
|
|
||||||
|
t.Run("AlwaysTrigger fires repeatedly once window fills", func(t *testing.T) {
|
||||||
|
incidents, window := driveCheckStatus(rule, model.ModeAlwaysTrigger, 30, []bool{false})
|
||||||
|
if window < 10 {
|
||||||
|
t.Fatalf("window never filled: got %d want >= 10", window)
|
||||||
|
}
|
||||||
|
if incidents == 0 {
|
||||||
|
t.Fatalf("AlwaysTrigger rule never dispatched an incident notification")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("OnetimeTrigger fires exactly once", func(t *testing.T) {
|
||||||
|
incidents, _ := driveCheckStatus(rule, model.ModeOnetimeTrigger, 30, []bool{false})
|
||||||
|
if incidents != 1 {
|
||||||
|
t.Fatalf("OnetimeTrigger must dispatch exactly one incident, got %d", incidents)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCheckStatus_HealthyServerStaysSilent guards the other direction: a server
|
||||||
|
// passing every tick must never dispatch an incident.
|
||||||
|
func TestCheckStatus_HealthyServerStaysSilent(t *testing.T) {
|
||||||
|
rule := &model.AlertRule{Rules: []*model.Rule{{Type: "cpu", Duration: 10}}}
|
||||||
|
incidents, _ := driveCheckStatus(rule, model.ModeAlwaysTrigger, 30, []bool{true})
|
||||||
|
if incidents != 0 {
|
||||||
|
t.Fatalf("a healthy server must never trigger an incident, got %d", incidents)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user