fix(alert): prevent persisted rule crash loops

This commit is contained in:
naiba committed 2026-09-02 13:30:46 +00:00
1 parent 8f1147c800
commit 19daf4ad3c
8 files changed
+479 -71

No files matched your search

+66 -13
View File
@@ -64,6 +64,36 @@ func (r *AlertRule) Enabled() bool {
return r.Enable != nil && *r.Enable
}
// IsSafeToEvaluate validates persisted rule data before it reaches the alert
// goroutine. API validation protects new writes; this second boundary protects
// upgrades from malformed or deliberately poisoned rows already in the DB.
func (r *AlertRule) IsSafeToEvaluate() bool {
if r == nil || len(r.Rules) == 0 {
return false
}
for _, rule := range r.Rules {
if rule == nil || !rule.IsSupportedType() {
return false
}
switch rule.Cover {
case RuleCoverAll, RuleCoverIgnoreAll:
default:
return false
}
if rule.IsTransferDurationRule() {
if !rule.HasSafeCycleConfiguration() {
return false
}
continue
}
duration, ok := rule.DurationInt()
if !ok || duration < 3 {
return false
}
}
return true
}
// HasPermission extends the default owner/admin check with PAT
// server_ids whitelist enforcement. AlertRule.Snapshot fans out across
// every owner-visible server filtered only by Rule.Ignore semantics
@@ -125,6 +155,12 @@ func (r *AlertRule) Snapshot(cycleTransferStats *CycleTransferStats, server *Ser
point := make([]bool, len(r.Rules))
for i, rule := range r.Rules {
if rule == nil || !rule.IsSupportedType() {
// Invalid persisted rules are ignored instead of being interpreted as
// a failed condition or allowed to panic the sentinel.
point[i] = true
continue
}
point[i] = rule.Snapshot(cycleTransferStats, server, db)
}
return point
@@ -134,10 +170,14 @@ func (r *AlertRule) Snapshot(cycleTransferStats *CycleTransferStats, server *Ser
func (r *AlertRule) Check(points [][]bool) (int, bool) {
var hasPassedRule bool
durations := make([]int, len(r.Rules))
validRules := 0
for ruleIndex, rule := range r.Rules {
duration := int(rule.Duration)
if rule == nil || !rule.IsSupportedType() {
continue
}
if rule.IsTransferDurationRule() {
validRules++
// 循环区间流量报警
if durations[ruleIndex] < 1 {
durations[ruleIndex] = 1
@@ -146,10 +186,21 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) {
continue
}
// 只要最后一次检查超出了规则范围 就认为检查未通过
if len(points) > 0 && points[len(points)-1][ruleIndex] {
hasPassedRule = true
if len(points) > 0 {
lastPoint := points[len(points)-1]
if ruleIndex >= len(lastPoint) || lastPoint[ruleIndex] {
hasPassedRule = true
}
}
} else if rule.IsOfflineRule() {
continue
}
duration, ok := rule.DurationInt()
if !ok || duration <= 0 {
continue
}
validRules++
if rule.IsOfflineRule() {
// 离线报警,检查直到最后一次在线的离线采样点是否大于 duration
if hasPassedRule = boundCheck(len(points), duration, hasPassedRule); hasPassedRule {
continue
@@ -157,7 +208,7 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) {
var fail int
for _, point := range slices.Backward(points[len(points)-duration:]) {
fail++
if point[ruleIndex] {
if ruleIndex >= len(point) || point[ruleIndex] {
hasPassedRule = true
break
}
@@ -168,11 +219,7 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) {
// 常规报警
// duration<=0 是无意义的规则(持续 0 秒):直接跳过该规则,
// 既不污染 hasPassedRule(否则会连带跳过同一 alert 里其它有效
// 规则),也避免下方 fail*100/total 在 total=0 时整数除零 panic
// —— checkStatus 无 recover,一次 panic 会拖垮整个告警 goroutine。
if duration <= 0 {
continue
}
// 规则),也避免下方百分比计算在 total=0 时除零。
if hasPassedRule = boundCheck(len(points), duration, hasPassedRule); hasPassedRule {
continue
}
@@ -181,17 +228,20 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) {
}
total, fail := duration, 0
for timeTick := len(points) - duration; timeTick < len(points); timeTick++ {
if !points[timeTick][ruleIndex] {
if ruleIndex >= len(points[timeTick]) || !points[timeTick][ruleIndex] {
fail++
}
}
// 当70%以上的采样点未通过规则判断时 才认为当前检查未通过
if fail*100/total <= 70 {
if float64(fail)*100/float64(total) <= 70 {
hasPassedRule = true
}
}
}
if validRules == 0 {
return 0, true
}
// 仅当所有检查均未通过时 才触发告警
return slices.Max(durations), hasPassedRule
}
@@ -205,10 +255,13 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) {
func (r *AlertRule) RetentionWindow() int {
window := 0
for _, rule := range r.Rules {
if rule == nil || !rule.IsSupportedType() {
continue
}
var need int
if rule.IsTransferDurationRule() {
need = 1
} else if d := int(rule.Duration); d > 0 {
} else if d, ok := rule.DurationInt(); ok && d > 0 {
need = d
}
if need > window {
+123
View File
@@ -0,0 +1,123 @@
package model
import (
"math"
"testing"
"time"
)
func TestRuleSnapshotIgnoresEmptyMetricSamples(t *testing.T) {
tests := []struct {
name string
ruleType string
hostState *HostState
}{
{
name: "GPU list is empty",
ruleType: "gpu_max",
hostState: &HostState{},
},
{
name: "all temperature samples are filtered",
ruleType: "temperature_max",
hostState: &HostState{Temperatures: []SensorTemperature{
{Name: "coretemp", Temperature: 0},
}},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
server := &Server{Common: Common{ID: 1}, State: test.hostState, Host: &Host{}}
rule := &Rule{Type: test.ruleType, Max: 1, Duration: 3, Cover: RuleCoverAll}
defer func() {
if recovered := recover(); recovered != nil {
t.Fatalf("Snapshot panicked for missing metric data: %v", recovered)
}
}()
if passed := rule.Snapshot(nil, server, nil); !passed {
t.Fatal("missing metric data must be ignored instead of treated as a failed threshold")
}
})
}
}
func TestAlertRuleCheckRejectsOverflowingPersistedDuration(t *testing.T) {
rule := &AlertRule{Rules: []*Rule{{
Type: "offline",
Duration: math.MaxUint64,
Cover: RuleCoverAll,
}}}
defer func() {
if recovered := recover(); recovered != nil {
t.Fatalf("Check panicked after uint64-to-int overflow: %v", recovered)
}
}()
duration, passed := rule.Check([][]bool{{false}})
if duration != 0 || !passed {
t.Fatalf("invalid persisted duration must be ignored safely, got duration=%d passed=%v", duration, passed)
}
if window := rule.RetentionWindow(); window != 0 {
t.Fatalf("invalid persisted duration must not create a retention window, got %d", window)
}
}
func TestAlertRulePersistedDataSafety(t *testing.T) {
cycleStart := time.Now().Add(-time.Hour)
tests := []struct {
name string
rule *AlertRule
want bool
}{
{
name: "normal rule",
rule: &AlertRule{Rules: []*Rule{{
Type: "cpu", Duration: 3, Cover: RuleCoverAll,
}}},
want: true,
},
{
name: "normal cycle rule",
rule: &AlertRule{Rules: []*Rule{{
Type: "transfer_in_cycle", CycleStart: &cycleStart, CycleInterval: 1, Cover: RuleCoverAll,
}}},
want: true,
},
{
name: "unknown type",
rule: &AlertRule{Rules: []*Rule{{
Type: "attacker_controlled", Duration: 3, Cover: RuleCoverAll,
}}},
},
{
name: "overflowing duration",
rule: &AlertRule{Rules: []*Rule{{
Type: "offline", Duration: math.MaxUint64, Cover: RuleCoverAll,
}}},
},
{
name: "cycle without start",
rule: &AlertRule{Rules: []*Rule{{
Type: "transfer_in_cycle", CycleInterval: 1, Cover: RuleCoverAll,
}}},
},
{
name: "nil rule",
rule: &AlertRule{Rules: []*Rule{nil}},
},
{
name: "empty rule list",
rule: &AlertRule{},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := test.rule.IsSafeToEvaluate(); got != test.want {
t.Fatalf("IsSafeToEvaluate()=%v, want %v", got, test.want)
}
})
}
}
+16 -4
View File
@@ -26,7 +26,7 @@ func testCycleRules(t *testing.T) {
rule: &AlertRule{
Rules: []*Rule{
{
Type: "_cycle",
Type: "transfer_in_cycle",
},
},
},
@@ -39,7 +39,7 @@ func testCycleRules(t *testing.T) {
rule: &AlertRule{
Rules: []*Rule{
{
Type: "_cycle",
Type: "transfer_in_cycle",
},
},
},
@@ -130,6 +130,7 @@ func testGeneralRules(t *testing.T) {
rule: &AlertRule{
Rules: []*Rule{
{
Type: "cpu",
Duration: 10,
},
},
@@ -143,6 +144,7 @@ func testGeneralRules(t *testing.T) {
rule: &AlertRule{
Rules: []*Rule{
{
Type: "cpu",
Duration: 10,
},
},
@@ -156,6 +158,7 @@ func testGeneralRules(t *testing.T) {
rule: &AlertRule{
Rules: []*Rule{
{
Type: "cpu",
Duration: 10,
},
},
@@ -169,6 +172,7 @@ func testGeneralRules(t *testing.T) {
rule: &AlertRule{
Rules: []*Rule{
{
Type: "cpu",
Duration: 10,
},
},
@@ -182,6 +186,7 @@ func testGeneralRules(t *testing.T) {
rule: &AlertRule{
Rules: []*Rule{
{
Type: "cpu",
Duration: 10,
},
},
@@ -210,6 +215,7 @@ func testCombinedRules(t *testing.T) {
Duration: 10,
},
{
Type: "cpu",
Duration: 10,
},
},
@@ -227,6 +233,7 @@ func testCombinedRules(t *testing.T) {
Duration: 10,
},
{
Type: "cpu",
Duration: 10,
},
},
@@ -240,6 +247,7 @@ func testCombinedRules(t *testing.T) {
rule: &AlertRule{
Rules: []*Rule{
{
Type: "cpu",
Duration: 10,
},
{
@@ -257,9 +265,11 @@ func testCombinedRules(t *testing.T) {
rule: &AlertRule{
Rules: []*Rule{
{
Type: "cpu",
Duration: 10,
},
{
Type: "memory",
Duration: 30,
},
},
@@ -273,9 +283,11 @@ func testCombinedRules(t *testing.T) {
rule: &AlertRule{
Rules: []*Rule{
{
Type: "cpu",
Duration: 10,
},
{
Type: "memory",
Duration: 30,
},
},
@@ -416,7 +428,7 @@ func TestAlertRule_RetentionWindow(t *testing.T) {
{"zero duration only", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 0}}}, 0},
{"mixed picks max", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 0}, {Type: "cpu", Duration: 7}}}, 7},
{"offline keeps Duration", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 30}}}, 30},
{"cycle looks back one", &AlertRule{Rules: []*Rule{{Type: "net_in_speed_cycle"}}}, 1},
{"cycle looks back one", &AlertRule{Rules: []*Rule{{Type: "transfer_in_cycle"}}}, 1},
}
for _, c := range cases {
if got := c.rule.RetentionWindow(); got != c.want {
@@ -469,7 +481,7 @@ func TestAlertRule_CombinedRuleAccumulatesSamples(t *testing.T) {
}{
{"general3+general10", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 3}, {Type: "memory", Duration: 10}}}, []bool{false, false}, 9, 10},
{"offline5+general10", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 5}, {Type: "cpu", Duration: 10}}}, []bool{false, false}, 9, 10},
{"transfer+general8", &AlertRule{Rules: []*Rule{{Type: "net_in_speed_cycle"}, {Type: "cpu", Duration: 8}}}, []bool{false, false}, 7, 8},
{"transfer+general8", &AlertRule{Rules: []*Rule{{Type: "transfer_in_cycle"}, {Type: "cpu", Duration: 8}}}, []bool{false, false}, 7, 8},
{"offline3+offline12", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 3}, {Type: "offline", Duration: 12}}}, []bool{false, false}, 11, 12},
}
for _, c := range cases {
+79 -10
View File
@@ -15,12 +15,23 @@ const (
RuleCoverIgnoreAll
)
// MaxAlertRuleDuration is the largest duration that can be converted to int
// on every architecture supported by Go. Alert rules are persisted as uint64,
// but their sampling windows use int indexes; keeping the public bound at
// MaxInt32 prevents a crafted value from wrapping during that conversion.
const MaxAlertRuleDuration uint64 = 1<<31 - 1
// MaxAlertRuleCycleInterval also leaves room for the largest integer
// multiplication performed by the calendar helpers (weeks become 7*interval)
// on 32-bit targets.
const MaxAlertRuleCycleInterval uint64 = (1<<31 - 1) / 7
type NResult struct {
N uint64
}
type Rule struct {
// 指标类型,cpu、memory、swap、disk、net_in_speed、net_out_speed
// 指标类型,cpu、gpu/gpu_max、memory、swap、disk、net_in_speed、net_out_speed
// net_all_speed、transfer_in、transfer_out、transfer_all、offline
// transfer_in_cycle、transfer_out_cycle、transfer_all_cycle
Type string `json:"type"`
@@ -45,8 +56,52 @@ func percentage(used, total uint64) float64 {
return float64(used) * 100 / float64(total)
}
// IsSupportedType reports whether the rule type has an evaluator. Keep this
// list in lockstep with Snapshot's switch so untrusted strings cannot enter the
// persisted alert pipeline and silently acquire fallback semantics.
func (u *Rule) IsSupportedType() bool {
if u == nil {
return false
}
switch u.Type {
case "cpu", "gpu", "gpu_max", "memory", "swap", "disk",
"net_in_speed", "net_out_speed", "net_all_speed",
"transfer_in", "transfer_out", "transfer_all", "offline",
"transfer_in_cycle", "transfer_out_cycle", "transfer_all_cycle",
"load1", "load5", "load15", "tcp_conn_count", "udp_conn_count",
"process_count", "temperature_max":
return true
default:
return false
}
}
// DurationInt converts the persisted duration without allowing uint64-to-int
// truncation. Callers must handle ok=false as an invalid rule.
func (u *Rule) DurationInt() (duration int, ok bool) {
if u == nil || u.Duration > MaxAlertRuleDuration {
return 0, false
}
return int(u.Duration), true
}
// HasSafeCycleConfiguration guards every value that the cycle evaluator later
// converts or dereferences. It intentionally does not reject an empty unit,
// which is the documented legacy spelling for hours.
func (u *Rule) HasSafeCycleConfiguration() bool {
return u != nil && u.IsTransferDurationRule() && u.CycleStart != nil &&
u.CycleInterval > 0 && u.CycleInterval <= MaxAlertRuleCycleInterval
}
// Snapshot 未通过规则返回 false, 通过返回 true
func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) bool {
if u == nil || server == nil || !u.IsSupportedType() {
return true
}
if u.IsTransferDurationRule() && (!u.HasSafeCycleConfiguration() || cycleTransferStats == nil) {
return true
}
// 监控全部但是排除了此服务器
if u.Cover == RuleCoverAll && u.Ignore[server.ID] {
return true
@@ -71,7 +126,10 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server,
switch u.Type {
case "cpu":
src = float64(state.CPU)
case "gpu_max":
case "gpu", "gpu_max":
if len(state.GPU) == 0 {
return true
}
src = slices.Max(state.GPU)
case "memory":
if runtime.Host == nil {
@@ -141,14 +199,17 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server,
src = float64(state.ProcessCount)
case "temperature_max":
var temp []float64
if state.Temperatures != nil {
for _, tempStat := range state.Temperatures {
if tempStat.Temperature != 0 {
temp = append(temp, tempStat.Temperature)
}
for _, tempStat := range state.Temperatures {
if tempStat.Temperature != 0 {
temp = append(temp, tempStat.Temperature)
}
src = slices.Max(temp)
}
if len(temp) == 0 {
return true
}
src = slices.Max(temp)
default:
return true
}
// 循环区间流量检测 · 更新下次需要检测时间
@@ -187,11 +248,19 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server,
// IsTransferDurationRule 判断该规则是否属于周期流量规则 属于则返回true
func (u *Rule) IsTransferDurationRule() bool {
return strings.HasSuffix(u.Type, "_cycle")
if u == nil {
return false
}
switch u.Type {
case "transfer_in_cycle", "transfer_out_cycle", "transfer_all_cycle":
return true
default:
return false
}
}
func (u *Rule) IsOfflineRule() bool {
return u.Type == "offline"
return u != nil && u.Type == "offline"
}
// GetTransferDurationStart 获取周期流量的起始时间