mirror of
https://github.com/Buriburizaem0n/nezha_domains.git
synced 2026-11-04 00:40:20 +00:00
fix(alert): prevent persisted rule crash loops
This commit is contained in:
1 parent
8f1147c800
commit
19daf4ad3c
8 files changed
+479
-71
No files matched your search
+66
-13
@@ -64,6 +64,36 @@ func (r *AlertRule) Enabled() bool {
|
||||
return r.Enable != nil && *r.Enable
|
||||
}
|
||||
|
||||
// IsSafeToEvaluate validates persisted rule data before it reaches the alert
|
||||
// goroutine. API validation protects new writes; this second boundary protects
|
||||
// upgrades from malformed or deliberately poisoned rows already in the DB.
|
||||
func (r *AlertRule) IsSafeToEvaluate() bool {
|
||||
if r == nil || len(r.Rules) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, rule := range r.Rules {
|
||||
if rule == nil || !rule.IsSupportedType() {
|
||||
return false
|
||||
}
|
||||
switch rule.Cover {
|
||||
case RuleCoverAll, RuleCoverIgnoreAll:
|
||||
default:
|
||||
return false
|
||||
}
|
||||
if rule.IsTransferDurationRule() {
|
||||
if !rule.HasSafeCycleConfiguration() {
|
||||
return false
|
||||
}
|
||||
continue
|
||||
}
|
||||
duration, ok := rule.DurationInt()
|
||||
if !ok || duration < 3 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// HasPermission extends the default owner/admin check with PAT
|
||||
// server_ids whitelist enforcement. AlertRule.Snapshot fans out across
|
||||
// every owner-visible server filtered only by Rule.Ignore semantics
|
||||
@@ -125,6 +155,12 @@ func (r *AlertRule) Snapshot(cycleTransferStats *CycleTransferStats, server *Ser
|
||||
point := make([]bool, len(r.Rules))
|
||||
|
||||
for i, rule := range r.Rules {
|
||||
if rule == nil || !rule.IsSupportedType() {
|
||||
// Invalid persisted rules are ignored instead of being interpreted as
|
||||
// a failed condition or allowed to panic the sentinel.
|
||||
point[i] = true
|
||||
continue
|
||||
}
|
||||
point[i] = rule.Snapshot(cycleTransferStats, server, db)
|
||||
}
|
||||
return point
|
||||
@@ -134,10 +170,14 @@ func (r *AlertRule) Snapshot(cycleTransferStats *CycleTransferStats, server *Ser
|
||||
func (r *AlertRule) Check(points [][]bool) (int, bool) {
|
||||
var hasPassedRule bool
|
||||
durations := make([]int, len(r.Rules))
|
||||
validRules := 0
|
||||
|
||||
for ruleIndex, rule := range r.Rules {
|
||||
duration := int(rule.Duration)
|
||||
if rule == nil || !rule.IsSupportedType() {
|
||||
continue
|
||||
}
|
||||
if rule.IsTransferDurationRule() {
|
||||
validRules++
|
||||
// 循环区间流量报警
|
||||
if durations[ruleIndex] < 1 {
|
||||
durations[ruleIndex] = 1
|
||||
@@ -146,10 +186,21 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) {
|
||||
continue
|
||||
}
|
||||
// 只要最后一次检查超出了规则范围 就认为检查未通过
|
||||
if len(points) > 0 && points[len(points)-1][ruleIndex] {
|
||||
hasPassedRule = true
|
||||
if len(points) > 0 {
|
||||
lastPoint := points[len(points)-1]
|
||||
if ruleIndex >= len(lastPoint) || lastPoint[ruleIndex] {
|
||||
hasPassedRule = true
|
||||
}
|
||||
}
|
||||
} else if rule.IsOfflineRule() {
|
||||
continue
|
||||
}
|
||||
|
||||
duration, ok := rule.DurationInt()
|
||||
if !ok || duration <= 0 {
|
||||
continue
|
||||
}
|
||||
validRules++
|
||||
if rule.IsOfflineRule() {
|
||||
// 离线报警,检查直到最后一次在线的离线采样点是否大于 duration
|
||||
if hasPassedRule = boundCheck(len(points), duration, hasPassedRule); hasPassedRule {
|
||||
continue
|
||||
@@ -157,7 +208,7 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) {
|
||||
var fail int
|
||||
for _, point := range slices.Backward(points[len(points)-duration:]) {
|
||||
fail++
|
||||
if point[ruleIndex] {
|
||||
if ruleIndex >= len(point) || point[ruleIndex] {
|
||||
hasPassedRule = true
|
||||
break
|
||||
}
|
||||
@@ -168,11 +219,7 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) {
|
||||
// 常规报警
|
||||
// duration<=0 是无意义的规则(持续 0 秒):直接跳过该规则,
|
||||
// 既不污染 hasPassedRule(否则会连带跳过同一 alert 里其它有效
|
||||
// 规则),也避免下方 fail*100/total 在 total=0 时整数除零 panic
|
||||
// —— checkStatus 无 recover,一次 panic 会拖垮整个告警 goroutine。
|
||||
if duration <= 0 {
|
||||
continue
|
||||
}
|
||||
// 规则),也避免下方百分比计算在 total=0 时除零。
|
||||
if hasPassedRule = boundCheck(len(points), duration, hasPassedRule); hasPassedRule {
|
||||
continue
|
||||
}
|
||||
@@ -181,17 +228,20 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) {
|
||||
}
|
||||
total, fail := duration, 0
|
||||
for timeTick := len(points) - duration; timeTick < len(points); timeTick++ {
|
||||
if !points[timeTick][ruleIndex] {
|
||||
if ruleIndex >= len(points[timeTick]) || !points[timeTick][ruleIndex] {
|
||||
fail++
|
||||
}
|
||||
}
|
||||
// 当70%以上的采样点未通过规则判断时 才认为当前检查未通过
|
||||
if fail*100/total <= 70 {
|
||||
if float64(fail)*100/float64(total) <= 70 {
|
||||
hasPassedRule = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if validRules == 0 {
|
||||
return 0, true
|
||||
}
|
||||
// 仅当所有检查均未通过时 才触发告警
|
||||
return slices.Max(durations), hasPassedRule
|
||||
}
|
||||
@@ -205,10 +255,13 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) {
|
||||
func (r *AlertRule) RetentionWindow() int {
|
||||
window := 0
|
||||
for _, rule := range r.Rules {
|
||||
if rule == nil || !rule.IsSupportedType() {
|
||||
continue
|
||||
}
|
||||
var need int
|
||||
if rule.IsTransferDurationRule() {
|
||||
need = 1
|
||||
} else if d := int(rule.Duration); d > 0 {
|
||||
} else if d, ok := rule.DurationInt(); ok && d > 0 {
|
||||
need = d
|
||||
}
|
||||
if need > window {
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRuleSnapshotIgnoresEmptyMetricSamples(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
ruleType string
|
||||
hostState *HostState
|
||||
}{
|
||||
{
|
||||
name: "GPU list is empty",
|
||||
ruleType: "gpu_max",
|
||||
hostState: &HostState{},
|
||||
},
|
||||
{
|
||||
name: "all temperature samples are filtered",
|
||||
ruleType: "temperature_max",
|
||||
hostState: &HostState{Temperatures: []SensorTemperature{
|
||||
{Name: "coretemp", Temperature: 0},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
server := &Server{Common: Common{ID: 1}, State: test.hostState, Host: &Host{}}
|
||||
rule := &Rule{Type: test.ruleType, Max: 1, Duration: 3, Cover: RuleCoverAll}
|
||||
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
t.Fatalf("Snapshot panicked for missing metric data: %v", recovered)
|
||||
}
|
||||
}()
|
||||
if passed := rule.Snapshot(nil, server, nil); !passed {
|
||||
t.Fatal("missing metric data must be ignored instead of treated as a failed threshold")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertRuleCheckRejectsOverflowingPersistedDuration(t *testing.T) {
|
||||
rule := &AlertRule{Rules: []*Rule{{
|
||||
Type: "offline",
|
||||
Duration: math.MaxUint64,
|
||||
Cover: RuleCoverAll,
|
||||
}}}
|
||||
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
t.Fatalf("Check panicked after uint64-to-int overflow: %v", recovered)
|
||||
}
|
||||
}()
|
||||
duration, passed := rule.Check([][]bool{{false}})
|
||||
if duration != 0 || !passed {
|
||||
t.Fatalf("invalid persisted duration must be ignored safely, got duration=%d passed=%v", duration, passed)
|
||||
}
|
||||
if window := rule.RetentionWindow(); window != 0 {
|
||||
t.Fatalf("invalid persisted duration must not create a retention window, got %d", window)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertRulePersistedDataSafety(t *testing.T) {
|
||||
cycleStart := time.Now().Add(-time.Hour)
|
||||
tests := []struct {
|
||||
name string
|
||||
rule *AlertRule
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "normal rule",
|
||||
rule: &AlertRule{Rules: []*Rule{{
|
||||
Type: "cpu", Duration: 3, Cover: RuleCoverAll,
|
||||
}}},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "normal cycle rule",
|
||||
rule: &AlertRule{Rules: []*Rule{{
|
||||
Type: "transfer_in_cycle", CycleStart: &cycleStart, CycleInterval: 1, Cover: RuleCoverAll,
|
||||
}}},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "unknown type",
|
||||
rule: &AlertRule{Rules: []*Rule{{
|
||||
Type: "attacker_controlled", Duration: 3, Cover: RuleCoverAll,
|
||||
}}},
|
||||
},
|
||||
{
|
||||
name: "overflowing duration",
|
||||
rule: &AlertRule{Rules: []*Rule{{
|
||||
Type: "offline", Duration: math.MaxUint64, Cover: RuleCoverAll,
|
||||
}}},
|
||||
},
|
||||
{
|
||||
name: "cycle without start",
|
||||
rule: &AlertRule{Rules: []*Rule{{
|
||||
Type: "transfer_in_cycle", CycleInterval: 1, Cover: RuleCoverAll,
|
||||
}}},
|
||||
},
|
||||
{
|
||||
name: "nil rule",
|
||||
rule: &AlertRule{Rules: []*Rule{nil}},
|
||||
},
|
||||
{
|
||||
name: "empty rule list",
|
||||
rule: &AlertRule{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := test.rule.IsSafeToEvaluate(); got != test.want {
|
||||
t.Fatalf("IsSafeToEvaluate()=%v, want %v", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+16
-4
@@ -26,7 +26,7 @@ func testCycleRules(t *testing.T) {
|
||||
rule: &AlertRule{
|
||||
Rules: []*Rule{
|
||||
{
|
||||
Type: "_cycle",
|
||||
Type: "transfer_in_cycle",
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -39,7 +39,7 @@ func testCycleRules(t *testing.T) {
|
||||
rule: &AlertRule{
|
||||
Rules: []*Rule{
|
||||
{
|
||||
Type: "_cycle",
|
||||
Type: "transfer_in_cycle",
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -130,6 +130,7 @@ func testGeneralRules(t *testing.T) {
|
||||
rule: &AlertRule{
|
||||
Rules: []*Rule{
|
||||
{
|
||||
Type: "cpu",
|
||||
Duration: 10,
|
||||
},
|
||||
},
|
||||
@@ -143,6 +144,7 @@ func testGeneralRules(t *testing.T) {
|
||||
rule: &AlertRule{
|
||||
Rules: []*Rule{
|
||||
{
|
||||
Type: "cpu",
|
||||
Duration: 10,
|
||||
},
|
||||
},
|
||||
@@ -156,6 +158,7 @@ func testGeneralRules(t *testing.T) {
|
||||
rule: &AlertRule{
|
||||
Rules: []*Rule{
|
||||
{
|
||||
Type: "cpu",
|
||||
Duration: 10,
|
||||
},
|
||||
},
|
||||
@@ -169,6 +172,7 @@ func testGeneralRules(t *testing.T) {
|
||||
rule: &AlertRule{
|
||||
Rules: []*Rule{
|
||||
{
|
||||
Type: "cpu",
|
||||
Duration: 10,
|
||||
},
|
||||
},
|
||||
@@ -182,6 +186,7 @@ func testGeneralRules(t *testing.T) {
|
||||
rule: &AlertRule{
|
||||
Rules: []*Rule{
|
||||
{
|
||||
Type: "cpu",
|
||||
Duration: 10,
|
||||
},
|
||||
},
|
||||
@@ -210,6 +215,7 @@ func testCombinedRules(t *testing.T) {
|
||||
Duration: 10,
|
||||
},
|
||||
{
|
||||
Type: "cpu",
|
||||
Duration: 10,
|
||||
},
|
||||
},
|
||||
@@ -227,6 +233,7 @@ func testCombinedRules(t *testing.T) {
|
||||
Duration: 10,
|
||||
},
|
||||
{
|
||||
Type: "cpu",
|
||||
Duration: 10,
|
||||
},
|
||||
},
|
||||
@@ -240,6 +247,7 @@ func testCombinedRules(t *testing.T) {
|
||||
rule: &AlertRule{
|
||||
Rules: []*Rule{
|
||||
{
|
||||
Type: "cpu",
|
||||
Duration: 10,
|
||||
},
|
||||
{
|
||||
@@ -257,9 +265,11 @@ func testCombinedRules(t *testing.T) {
|
||||
rule: &AlertRule{
|
||||
Rules: []*Rule{
|
||||
{
|
||||
Type: "cpu",
|
||||
Duration: 10,
|
||||
},
|
||||
{
|
||||
Type: "memory",
|
||||
Duration: 30,
|
||||
},
|
||||
},
|
||||
@@ -273,9 +283,11 @@ func testCombinedRules(t *testing.T) {
|
||||
rule: &AlertRule{
|
||||
Rules: []*Rule{
|
||||
{
|
||||
Type: "cpu",
|
||||
Duration: 10,
|
||||
},
|
||||
{
|
||||
Type: "memory",
|
||||
Duration: 30,
|
||||
},
|
||||
},
|
||||
@@ -416,7 +428,7 @@ func TestAlertRule_RetentionWindow(t *testing.T) {
|
||||
{"zero duration only", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 0}}}, 0},
|
||||
{"mixed picks max", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 0}, {Type: "cpu", Duration: 7}}}, 7},
|
||||
{"offline keeps Duration", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 30}}}, 30},
|
||||
{"cycle looks back one", &AlertRule{Rules: []*Rule{{Type: "net_in_speed_cycle"}}}, 1},
|
||||
{"cycle looks back one", &AlertRule{Rules: []*Rule{{Type: "transfer_in_cycle"}}}, 1},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := c.rule.RetentionWindow(); got != c.want {
|
||||
@@ -469,7 +481,7 @@ func TestAlertRule_CombinedRuleAccumulatesSamples(t *testing.T) {
|
||||
}{
|
||||
{"general3+general10", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 3}, {Type: "memory", Duration: 10}}}, []bool{false, false}, 9, 10},
|
||||
{"offline5+general10", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 5}, {Type: "cpu", Duration: 10}}}, []bool{false, false}, 9, 10},
|
||||
{"transfer+general8", &AlertRule{Rules: []*Rule{{Type: "net_in_speed_cycle"}, {Type: "cpu", Duration: 8}}}, []bool{false, false}, 7, 8},
|
||||
{"transfer+general8", &AlertRule{Rules: []*Rule{{Type: "transfer_in_cycle"}, {Type: "cpu", Duration: 8}}}, []bool{false, false}, 7, 8},
|
||||
{"offline3+offline12", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 3}, {Type: "offline", Duration: 12}}}, []bool{false, false}, 11, 12},
|
||||
}
|
||||
for _, c := range cases {
|
||||
|
||||
+79
-10
@@ -15,12 +15,23 @@ const (
|
||||
RuleCoverIgnoreAll
|
||||
)
|
||||
|
||||
// MaxAlertRuleDuration is the largest duration that can be converted to int
|
||||
// on every architecture supported by Go. Alert rules are persisted as uint64,
|
||||
// but their sampling windows use int indexes; keeping the public bound at
|
||||
// MaxInt32 prevents a crafted value from wrapping during that conversion.
|
||||
const MaxAlertRuleDuration uint64 = 1<<31 - 1
|
||||
|
||||
// MaxAlertRuleCycleInterval also leaves room for the largest integer
|
||||
// multiplication performed by the calendar helpers (weeks become 7*interval)
|
||||
// on 32-bit targets.
|
||||
const MaxAlertRuleCycleInterval uint64 = (1<<31 - 1) / 7
|
||||
|
||||
type NResult struct {
|
||||
N uint64
|
||||
}
|
||||
|
||||
type Rule struct {
|
||||
// 指标类型,cpu、memory、swap、disk、net_in_speed、net_out_speed
|
||||
// 指标类型,cpu、gpu/gpu_max、memory、swap、disk、net_in_speed、net_out_speed
|
||||
// net_all_speed、transfer_in、transfer_out、transfer_all、offline
|
||||
// transfer_in_cycle、transfer_out_cycle、transfer_all_cycle
|
||||
Type string `json:"type"`
|
||||
@@ -45,8 +56,52 @@ func percentage(used, total uint64) float64 {
|
||||
return float64(used) * 100 / float64(total)
|
||||
}
|
||||
|
||||
// IsSupportedType reports whether the rule type has an evaluator. Keep this
|
||||
// list in lockstep with Snapshot's switch so untrusted strings cannot enter the
|
||||
// persisted alert pipeline and silently acquire fallback semantics.
|
||||
func (u *Rule) IsSupportedType() bool {
|
||||
if u == nil {
|
||||
return false
|
||||
}
|
||||
switch u.Type {
|
||||
case "cpu", "gpu", "gpu_max", "memory", "swap", "disk",
|
||||
"net_in_speed", "net_out_speed", "net_all_speed",
|
||||
"transfer_in", "transfer_out", "transfer_all", "offline",
|
||||
"transfer_in_cycle", "transfer_out_cycle", "transfer_all_cycle",
|
||||
"load1", "load5", "load15", "tcp_conn_count", "udp_conn_count",
|
||||
"process_count", "temperature_max":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// DurationInt converts the persisted duration without allowing uint64-to-int
|
||||
// truncation. Callers must handle ok=false as an invalid rule.
|
||||
func (u *Rule) DurationInt() (duration int, ok bool) {
|
||||
if u == nil || u.Duration > MaxAlertRuleDuration {
|
||||
return 0, false
|
||||
}
|
||||
return int(u.Duration), true
|
||||
}
|
||||
|
||||
// HasSafeCycleConfiguration guards every value that the cycle evaluator later
|
||||
// converts or dereferences. It intentionally does not reject an empty unit,
|
||||
// which is the documented legacy spelling for hours.
|
||||
func (u *Rule) HasSafeCycleConfiguration() bool {
|
||||
return u != nil && u.IsTransferDurationRule() && u.CycleStart != nil &&
|
||||
u.CycleInterval > 0 && u.CycleInterval <= MaxAlertRuleCycleInterval
|
||||
}
|
||||
|
||||
// Snapshot 未通过规则返回 false, 通过返回 true
|
||||
func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) bool {
|
||||
if u == nil || server == nil || !u.IsSupportedType() {
|
||||
return true
|
||||
}
|
||||
if u.IsTransferDurationRule() && (!u.HasSafeCycleConfiguration() || cycleTransferStats == nil) {
|
||||
return true
|
||||
}
|
||||
|
||||
// 监控全部但是排除了此服务器
|
||||
if u.Cover == RuleCoverAll && u.Ignore[server.ID] {
|
||||
return true
|
||||
@@ -71,7 +126,10 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server,
|
||||
switch u.Type {
|
||||
case "cpu":
|
||||
src = float64(state.CPU)
|
||||
case "gpu_max":
|
||||
case "gpu", "gpu_max":
|
||||
if len(state.GPU) == 0 {
|
||||
return true
|
||||
}
|
||||
src = slices.Max(state.GPU)
|
||||
case "memory":
|
||||
if runtime.Host == nil {
|
||||
@@ -141,14 +199,17 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server,
|
||||
src = float64(state.ProcessCount)
|
||||
case "temperature_max":
|
||||
var temp []float64
|
||||
if state.Temperatures != nil {
|
||||
for _, tempStat := range state.Temperatures {
|
||||
if tempStat.Temperature != 0 {
|
||||
temp = append(temp, tempStat.Temperature)
|
||||
}
|
||||
for _, tempStat := range state.Temperatures {
|
||||
if tempStat.Temperature != 0 {
|
||||
temp = append(temp, tempStat.Temperature)
|
||||
}
|
||||
src = slices.Max(temp)
|
||||
}
|
||||
if len(temp) == 0 {
|
||||
return true
|
||||
}
|
||||
src = slices.Max(temp)
|
||||
default:
|
||||
return true
|
||||
}
|
||||
|
||||
// 循环区间流量检测 · 更新下次需要检测时间
|
||||
@@ -187,11 +248,19 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server,
|
||||
|
||||
// IsTransferDurationRule 判断该规则是否属于周期流量规则 属于则返回true
|
||||
func (u *Rule) IsTransferDurationRule() bool {
|
||||
return strings.HasSuffix(u.Type, "_cycle")
|
||||
if u == nil {
|
||||
return false
|
||||
}
|
||||
switch u.Type {
|
||||
case "transfer_in_cycle", "transfer_out_cycle", "transfer_all_cycle":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (u *Rule) IsOfflineRule() bool {
|
||||
return u.Type == "offline"
|
||||
return u != nil && u.Type == "offline"
|
||||
}
|
||||
|
||||
// GetTransferDurationStart 获取周期流量的起始时间
|
||||
|
||||
Reference in new issue
Block a user