fix(alerts): evaluate fresh reports and order notification delivery (#1242)

* fix(alerts): evaluate fresh reports and make notification transitions reliable

* fix(alerts): identify equal-timestamp reports by sequence

* fix(alerts): preserve notification transition semantics

---------

Co-authored-by: naiba <hi@nai.ba>

model/alert_evaluation_test.go

@@ -0,0 +1,205 @@
+package model
+
+import (
+	"math"
+	"testing"
+	"time"
+
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+)
+
+func TestAlertEvaluationDistinguishesUnknownFromBreach(t *testing.T) {
+	server := &Server{Common: Common{ID: 1}}
+	rule := &Rule{Type: "cpu", Max: 80, Cover: RuleCoverAll}
+	if passed, known := rule.Evaluate(nil, server, nil); !passed || known {
+		t.Fatalf("missing state = (%v, %v), want unknown", passed, known)
+	}
+	lease := server.AttachStateStream(runtimeOwnershipStream{})
+	if !lease.UpdateState(&HostState{CPU: 99}, time.Now()) {
+		t.Fatal("state update rejected")
+	}
+	if passed, known := rule.Evaluate(nil, server, nil); passed || !known {
+		t.Fatalf("fresh high CPU = (%v, %v), want known breach", passed, known)
+	}
+	if !lease.Clear() {
+		t.Fatal("stream clear rejected")
+	}
+	// The raw metric remains available for historical display. The sentinel
+	// enforces freshness and must not count this value after disconnect.
+	if !server.RuntimeSnapshot().LastActive.IsZero() {
+		t.Fatal("disconnect did not clear online visibility")
+	}
+}
+
+func newCycleTestDB(t *testing.T, migrate bool) *gorm.DB {
+	t.Helper()
+	db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if migrate {
+		if err := db.AutoMigrate(&Transfer{}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return db
+}
+
+func newCycleTestStats() *CycleTransferStats {
+	return &CycleTransferStats{
+		ServerName: make(map[uint64]string),
+		Transfer:   make(map[uint64]uint64),
+		NextUpdate: make(map[uint64]time.Time),
+	}
+}
+
+func TestCycleRuleRejectsInvalidConfiguration(t *testing.T) {
+	start := time.Now().Add(-time.Hour)
+	rule := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Max: 100}
+	if !rule.HasSafeCycleConfiguration() {
+		t.Fatal("normal hourly cycle rejected")
+	}
+	for _, tc := range []struct {
+		name   string
+		mutate func(*Rule)
+	}{
+		{"unknown unit", func(r *Rule) { r.CycleUnit = "fortnight" }},
+		{"negative limit", func(r *Rule) { r.Max = -1 }},
+		{"NaN limit", func(r *Rule) { r.Max = math.NaN() }},
+		{"reversed limits", func(r *Rule) { r.Min, r.Max = 200, 100 }},
+		{"future start", func(r *Rule) { future := time.Now().Add(time.Hour); r.CycleStart = &future }},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			copy := *rule
+			tc.mutate(&copy)
+			if copy.HasSafeCycleConfiguration() {
+				t.Fatal("unsafe cycle accepted")
+			}
+		})
+	}
+}
+
+func TestCycleBoundsUseOneClockAtBoundary(t *testing.T) {
+	start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
+	rule := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Max: 100}
+	before := start.Add(59*time.Minute + 59*time.Second)
+	from, to := rule.transferDurationBounds(before)
+	if !from.Equal(start) || !to.Equal(start.Add(time.Hour)) {
+		t.Fatalf("before boundary = %v..%v", from, to)
+	}
+	from, to = rule.transferDurationBounds(start.Add(time.Hour))
+	if !from.Equal(start.Add(time.Hour)) || !to.Equal(start.Add(2*time.Hour)) {
+		t.Fatalf("at boundary = %v..%v", from, to)
+	}
+}
+
+func TestCycleQueryFailureIsUnknownAndDoesNotCache(t *testing.T) {
+	start := time.Now().Add(-time.Hour)
+	rule := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Max: 50}
+	server := &Server{Common: Common{ID: 1}, State: &HostState{}}
+	passed, known := rule.Evaluate(newCycleTestStats(), server, newCycleTestDB(t, false))
+	if !passed || known || len(rule.NextTransferAt) != 0 {
+		t.Fatalf("SQL error yielded passed=%v known=%v cache=%v", passed, known, rule.NextTransferAt)
+	}
+}
+
+func TestCycleQueryExcludesFutureRowsAndHandlesMinOnly(t *testing.T) {
+	start := time.Now().Add(-90 * time.Minute)
+	db := newCycleTestDB(t, true)
+	server := &Server{Common: Common{ID: 1}, State: &HostState{}}
+	for _, row := range []Transfer{
+		{Common: Common{CreatedAt: time.Now().Add(-time.Minute)}, ServerID: 1, Out: 10},
+		{Common: Common{CreatedAt: time.Now().Add(10 * time.Minute)}, ServerID: 1, Out: 100},
+	} {
+		if err := db.Create(&row).Error; err != nil {
+			t.Fatal(err)
+		}
+	}
+	rule := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Max: 50}
+	stats := newCycleTestStats()
+	passed, known := rule.Evaluate(stats, server, db)
+	if !passed || !known || stats.Transfer[1] != 10 {
+		t.Fatalf("future row included: passed=%v known=%v amount=%d", passed, known, stats.Transfer[1])
+	}
+	minOnly := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Min: 20}
+	passed, known = minOnly.Evaluate(newCycleTestStats(), server, db)
+	if passed || !known {
+		t.Fatalf("min-only cycle = (%v,%v), want known breach", passed, known)
+	}
+}
+
+func TestAlertEvaluationAllNetworkSpeedUsesBothDirections(t *testing.T) {
+	for _, tc := range []struct {
+		name    string
+		in, out uint64
+		breach  bool
+	}{
+		{"inbound only", 100, 0, true},
+		{"outbound below combined limit", 0, 30, false},
+		{"both directions", 25, 30, true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			server := &Server{Common: Common{ID: 1}, State: &HostState{NetInSpeed: tc.in, NetOutSpeed: tc.out}}
+			rule := &Rule{Type: "net_all_speed", Max: 50, Cover: RuleCoverAll}
+			passed, known := rule.Evaluate(nil, server, nil)
+			if !known || passed == tc.breach {
+				t.Fatalf("got passed=%v known=%v, want breach=%v", passed, known, tc.breach)
+			}
+		})
+	}
+}
+
+func TestAlertEvaluationUnavailableSensorsAreUnknown(t *testing.T) {
+	tests := []struct {
+		name  string
+		rule  string
+		state *HostState
+		host  *Host
+	}{
+		{"missing memory host", "memory", &HostState{MemUsed: 1}, nil},
+		{"zero memory total", "memory", &HostState{MemUsed: 1}, &Host{}},
+		{"missing GPU", "gpu_max", &HostState{}, &Host{}},
+		{"missing temperature", "temperature_max", &HostState{}, &Host{}},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			server := &Server{Common: Common{ID: 1}, State: tc.state, Host: tc.host}
+			passed, known := (&Rule{Type: tc.rule, Max: 50, Cover: RuleCoverAll}).Evaluate(nil, server, nil)
+			if !passed || known {
+				t.Fatalf("got passed=%v known=%v, want unknown", passed, known)
+			}
+		})
+	}
+}
+
+func TestOfflineRuleDoesNotRequireMetricState(t *testing.T) {
+	server := &Server{Common: Common{ID: 1}}
+	InitServer(server)
+	passed, known := (&Rule{Type: "offline", Cover: RuleCoverAll}).Evaluate(nil, server, nil)
+	if passed || !known {
+		t.Fatalf("offline without state = (%v, %v), want known breach", passed, known)
+	}
+}
+
+func TestCompoundAlertUsesOneImmutableAgentReport(t *testing.T) {
+	server := &Server{Common: Common{ID: 1}}
+	alert := &AlertRule{Rules: []*Rule{
+		{Type: "cpu", Max: 80, Cover: RuleCoverAll},
+		{Type: "memory", Max: 80, Cover: RuleCoverAll},
+	}}
+	oldReport := RuntimeSnapshot{
+		State: &HostState{CPU: 99, MemUsed: 0},
+		Host:  &Host{MemTotal: 100},
+	}
+	newReport := RuntimeSnapshot{
+		State: &HostState{CPU: 0, MemUsed: 99},
+		Host:  &Host{MemTotal: 100},
+	}
+	for _, report := range []RuntimeSnapshot{oldReport, newReport} {
+		point, known := alert.SnapshotStatusWithRuntime(nil, server, report, nil)
+		if !known || len(point) != 2 || point[0] == point[1] {
+			t.Fatalf("compound point combined incompatible reports: known=%v point=%v", known, point)
+		}
+	}
+}

model/alert_timed_test.go

@@ -0,0 +1,104 @@
+package model
+
+import (
+	"testing"
+	"time"
+)
+
+func timedRule(kind string, duration uint64) *AlertRule {
+	return &AlertRule{Rules: []*Rule{{Type: kind, Duration: duration, Max: 80, Cover: RuleCoverAll}}}
+}
+
+func timedPoints(start time.Time, values []bool, spacing time.Duration) []TimedAlertPoint {
+	points := make([]TimedAlertPoint, len(values))
+	for i, value := range values {
+		points[i] = TimedAlertPoint{At: start.Add(time.Duration(i) * spacing), Values: []bool{value}}
+	}
+	return points
+}
+
+func TestTimedAlertNeedsElapsedWindowAndDistinctReports(t *testing.T) {
+	start := time.Unix(1_700_000_000, 0)
+	rule := timedRule("cpu", 5)
+	points := timedPoints(start, []bool{false, false, false, false}, AlertSampleInterval)
+	if known, _ := rule.CheckTimed(points, points[len(points)-1].At); known {
+		t.Fatal("four reports decided a five-tick window")
+	}
+	points = append(points, TimedAlertPoint{At: start.Add(12 * time.Second), Values: []bool{false}})
+	if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || passed {
+		t.Fatalf("complete high-CPU window: known=%v passed=%v", known, passed)
+	}
+	// Repeating the same observation does not advance wall-clock coverage.
+	if known, _ := rule.CheckTimed(points, start.Add(30*time.Second)); known {
+		t.Fatal("stale sample created a new complete window")
+	}
+}
+
+func TestTimedAlertGapIsUnknownAndCannotResolve(t *testing.T) {
+	start := time.Unix(1_700_000_000, 0)
+	rule := timedRule("cpu", 5)
+	points := []TimedAlertPoint{
+		{At: start, Values: []bool{false}},
+		{At: start.Add(30 * time.Second), Values: []bool{true}},
+	}
+	if known, _ := rule.CheckTimed(points, start.Add(30*time.Second)); known {
+		t.Fatal("telemetry gap must not be interpreted as healthy")
+	}
+}
+
+func TestTimedAlertIrregularReportsUseElapsedTime(t *testing.T) {
+	start := time.Unix(1_700_000_000, 0)
+	rule := timedRule("cpu", 6)
+	points := timedPoints(start, []bool{false, false, false, false}, 5*time.Second)
+	if known, passed := rule.CheckTimed(points, start.Add(15*time.Second)); !known || passed {
+		t.Fatalf("irregular reports cover the wall-clock window: known=%v passed=%v", known, passed)
+	}
+}
+
+func TestTimedAlertExactSeventyPercentIsNotIncident(t *testing.T) {
+	start := time.Unix(1_700_000_000, 0)
+	rule := timedRule("cpu", 10)
+	values := []bool{false, false, false, false, false, false, false, true, true, true}
+	points := timedPoints(start, values, AlertSampleInterval)
+	if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || !passed {
+		t.Fatalf("70%% failed is below the >70%% trigger: known=%v passed=%v", known, passed)
+	}
+}
+
+func TestTimedAlertTenTickWindowDoesNotDecideAtNine(t *testing.T) {
+	start := time.Unix(1_700_000_000, 0)
+	rule := timedRule("cpu", 10)
+	points := timedPoints(start, []bool{false, false, false, false, false, false, false, false, false}, AlertSampleInterval)
+	if known, _ := rule.CheckTimed(points, points[len(points)-1].At); known {
+		t.Fatal("nine reports decided a ten-tick window one tick early")
+	}
+	points = append(points, TimedAlertPoint{At: start.Add(27 * time.Second), Values: []bool{false}})
+	if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || passed {
+		t.Fatalf("full ten-tick breach: known=%v passed=%v", known, passed)
+	}
+}
+
+func TestTimedOfflineRequiresSustainedAbsence(t *testing.T) {
+	start := time.Unix(1_700_000_000, 0)
+	rule := timedRule("offline", 5)
+	points := timedPoints(start, []bool{false, false, false, false, false}, AlertSampleInterval)
+	if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || passed {
+		t.Fatalf("continuous offline interval: known=%v passed=%v", known, passed)
+	}
+	points = append(points, TimedAlertPoint{At: start.Add(15 * time.Second), Values: []bool{true}})
+	if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || !passed {
+		t.Fatalf("online observation should resolve offline: known=%v passed=%v", known, passed)
+	}
+}
+
+func TestTimedCompoundRuleWaitsForAllKnown(t *testing.T) {
+	start := time.Unix(1_700_000_000, 0)
+	rule := &AlertRule{Rules: []*Rule{
+		{Type: "cpu", Duration: 5, Max: 80, Cover: RuleCoverAll},
+		{Type: "memory", Duration: 5, Max: 80, Cover: RuleCoverAll},
+	}}
+	points := timedPoints(start, []bool{false, false, false, false, false}, AlertSampleInterval)
+	if known, _ := rule.CheckTimed(points, points[len(points)-1].At); known {
+		t.Fatal("missing second condition treated as decisive")
+	}
+}

model/alertrule.go

@@ -2,12 +2,134 @@ package model
 
 import (
 	"slices"
+	"time"
 
 	"github.com/gin-gonic/gin"
 	"github.com/goccy/go-json"
 	"gorm.io/gorm"
 )
 
+const (
+	AlertSampleInterval = 3 * time.Second
+	AlertSampleMaxAge   = 6 * time.Second
+)
+
+// TimedAlertPoint records one distinct observation. Its timestamp is the
+// dashboard receipt time of the Agent report, or the sentinel tick for an
+// offline-only rule.
+type TimedAlertPoint struct {
+	At     time.Time
+	Values []bool
+}
+
+// CheckTimed evaluates the legacy Duration as Duration * 3 seconds, avoiding
+// both repeated stale samples and cadence-dependent early incidents. Unknown
+// coverage cannot create an incident or a recovery.
+func (r *AlertRule) CheckTimed(points []TimedAlertPoint, now time.Time) (known, passed bool) {
+	if r == nil || len(r.Rules) == 0 {
+		return false, false
+	}
+	allFailed := true
+	for index, rule := range r.Rules {
+		if rule == nil || !rule.IsSupportedType() {
+			return false, false
+		}
+		ruleKnown, rulePassed := rule.checkTimed(points, index, now)
+		if !ruleKnown {
+			return false, false
+		}
+		if rulePassed {
+			allFailed = false
+		}
+	}
+	return true, !allFailed
+}
+
+func (rule *Rule) checkTimed(points []TimedAlertPoint, index int, now time.Time) (known, passed bool) {
+	if len(points) == 0 {
+		return false, false
+	}
+	if rule.IsTransferDurationRule() {
+		last := points[len(points)-1]
+		if index >= len(last.Values) || now.Sub(last.At) > AlertSampleMaxAge {
+			return false, false
+		}
+		return true, last.Values[index]
+	}
+	window := time.Duration(rule.Duration) * AlertSampleInterval
+	if window <= 0 {
+		return false, false
+	}
+	end := now.Add(AlertSampleInterval) // the current observation represents one tick
+	start := end.Add(-window)
+	if points[0].At.After(start) {
+		return false, false // the configured wall-clock window has not elapsed
+	}
+	var covered, failed time.Duration
+	for i, point := range points {
+		if index >= len(point.Values) {
+			return false, false
+		}
+		segmentStart := point.At
+		if start.After(segmentStart) {
+			segmentStart = start
+		}
+		segmentEnd := point.At.Add(AlertSampleMaxAge)
+		if end.Before(segmentEnd) {
+			segmentEnd = end
+		}
+		if i+1 < len(points) {
+			if points[i+1].At.Before(segmentEnd) {
+				segmentEnd = points[i+1].At
+			}
+		} else {
+			if tickEnd := point.At.Add(AlertSampleInterval); tickEnd.Before(segmentEnd) {
+				segmentEnd = tickEnd
+			}
+		}
+		if !segmentEnd.After(segmentStart) {
+			continue
+		}
+		span := segmentEnd.Sub(segmentStart)
+		covered += span
+		if !point.Values[index] {
+			failed += span
+		}
+	}
+	if covered < window-window/10 {
+		return false, false
+	}
+	if rule.IsOfflineRule() {
+		if covered-failed >= AlertSampleInterval {
+			return true, true
+		}
+		if failed >= window-AlertSampleInterval && covered >= window-AlertSampleInterval {
+			return true, false
+		}
+		return false, false
+	}
+	if float64(failed) > float64(window)*0.70 {
+		return true, false
+	}
+	if float64(failed+window-covered) <= float64(window)*0.70 {
+		return true, true
+	}
+	return false, false
+}
+
+func (r *AlertRule) TimedRetention() time.Duration {
+	longest := AlertSampleMaxAge
+	for _, rule := range r.Rules {
+		if rule == nil || rule.IsTransferDurationRule() {
+			continue
+		}
+		if duration, ok := rule.DurationInt(); ok {
+			longest = max(longest, time.Duration(duration)*AlertSampleInterval+AlertSampleMaxAge)
+		}
+	}
+	return longest
+}
+
 const (
 	ModeAlwaysTrigger  = 0
 	ModeOnetimeTrigger = 1
@@ -75,6 +197,9 @@ func (r *AlertRule) IsSafeToEvaluate() bool {
 		if rule == nil || !rule.IsSupportedType() {
 			return false
 		}
+		if !rule.HasSafeThresholds() {
+			return false
+		}
 		switch rule.Cover {
 		case RuleCoverAll, RuleCoverIgnoreAll:
 		default:
@@ -152,18 +277,38 @@ func (r *AlertRule) HasPermission(ctx *gin.Context) bool {
 
 // Snapshot 对传入的Server进行该报警规则下所有type的检查 返回每项检查结果
 func (r *AlertRule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) []bool {
+	point, _ := r.SnapshotStatus(cycleTransferStats, server, db)
+	return point
+}
+
+// SnapshotStatus preserves the distinction between a healthy measurement and
+// telemetry that is unavailable. A compound alert cannot be decided while any
+// of its applicable conditions is unknown.
+func (r *AlertRule) SnapshotStatus(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) ([]bool, bool) {
+	if server == nil {
+		return nil, false
+	}
+	return r.SnapshotStatusWithRuntime(cycleTransferStats, server, server.RuntimeSnapshot(), db)
+}
+
+// SnapshotStatusWithRuntime uses one immutable report for every condition.
+func (r *AlertRule) SnapshotStatusWithRuntime(cycleTransferStats *CycleTransferStats, server *Server, runtime RuntimeSnapshot, db *gorm.DB) ([]bool, bool) {
 	point := make([]bool, len(r.Rules))
+	known := true
 
 	for i, rule := range r.Rules {
 		if rule == nil || !rule.IsSupportedType() {
 			// Invalid persisted rules are ignored instead of being interpreted as
 			// a failed condition or allowed to panic the sentinel.
 			point[i] = true
+			known = false
 			continue
 		}
-		point[i] = rule.Snapshot(cycleTransferStats, server, db)
+		var valid bool
+		point[i], valid = rule.EvaluateRuntime(cycleTransferStats, server, runtime, db)
+		known = known && valid
 	}
-	return point
+	return point, known
 }
 
 // Check 传入包含当前报警规则下所有type检查结果 返回报警持续时间与是否通过报警检查(通过则返回true)

model/notification.go

@@ -180,63 +180,65 @@ func (ns *NotificationServerBundle) replaceParamsInString(str string, message st
 
 	if ns.Server != nil {
 		runtime := ns.Server.RuntimeSnapshot()
-		if runtime.State == nil || runtime.Host == nil {
-			return str
-		}
-		state := runtime.State
-		host := runtime.Host
 		replacements = append(replacements,
 			"#SERVER.NAME#", mod(ns.Server.Name),
 			"#SERVER.ID#", mod(fmt.Sprintf("%d", ns.Server.ID)),
-
-			// Converted metrics
-			"#SERVER.CPU#", mod(ns.formatUsage(false, state.CPU)),
-			"#SERVER.MEM#", mod(ns.formatUsage(true, float64(state.MemUsed)/float64(host.MemTotal))),
-			"#SERVER.SWAP#", mod(ns.formatUsage(true, float64(state.SwapUsed)/float64(host.SwapTotal))),
-			"#SERVER.DISK#", mod(ns.formatUsage(true, float64(state.DiskUsed)/float64(host.DiskTotal))),
-			"#SERVER.SPEEDIN#", mod(fmt.Sprintf("%s/s", ns.formatSize(state.NetInSpeed))),
-			"#SERVER.SPEEDOUT#", mod(fmt.Sprintf("%s/s", ns.formatSize(state.NetOutSpeed))),
-			"#SERVER.TRANSFERIN#", mod(ns.formatSize(state.NetInTransfer)),
-			"#SERVER.TRANSFEROUT#", mod(ns.formatSize(state.NetOutTransfer)),
-
-			// Raw metrics
-			"#SERVER.CPUUSED#", mod(fmt.Sprintf("%f", state.CPU)),
-			"#SERVER.MEMUSED#", mod(fmt.Sprintf("%d", state.MemUsed)),
-			"#SERVER.SWAPUSED#", mod(fmt.Sprintf("%d", state.SwapUsed)),
-			"#SERVER.DISKUSED#", mod(fmt.Sprintf("%d", state.DiskUsed)),
-			"#SERVER.MEMTOTAL#", mod(fmt.Sprintf("%d", host.MemTotal)),
-			"#SERVER.SWAPTOTAL#", mod(fmt.Sprintf("%d", host.SwapTotal)),
-			"#SERVER.DISKTOTAL#", mod(fmt.Sprintf("%d", host.DiskTotal)),
-			"#SERVER.NETINSPEED#", mod(fmt.Sprintf("%d", state.NetInSpeed)),
-			"#SERVER.NETOUTSPEED#", mod(fmt.Sprintf("%d", state.NetOutSpeed)),
-			"#SERVER.NETINTRANSFER#", mod(fmt.Sprintf("%d", state.NetInTransfer)),
-			"#SERVER.NETOUTTRANSFER#", mod(fmt.Sprintf("%d", state.NetOutTransfer)),
-			"#SERVER.LOAD1#", mod(fmt.Sprintf("%f", state.Load1)),
-			"#SERVER.LOAD5#", mod(fmt.Sprintf("%f", state.Load5)),
-			"#SERVER.LOAD15#", mod(fmt.Sprintf("%f", state.Load15)),
-			"#SERVER.TCPCONNCOUNT#", mod(fmt.Sprintf("%d", state.TcpConnCount)),
-			"#SERVER.UDPCONNCOUNT#", mod(fmt.Sprintf("%d", state.UdpConnCount)),
 		)
+		if runtime.State != nil && runtime.Host != nil {
+			state := runtime.State
+			host := runtime.Host
+			replacements = append(replacements,
+				// Converted metrics
+				"#SERVER.CPU#", mod(ns.formatUsage(false, state.CPU)),
+				"#SERVER.MEM#", mod(ns.formatUsage(true, float64(state.MemUsed)/float64(host.MemTotal))),
+				"#SERVER.SWAP#", mod(ns.formatUsage(true, float64(state.SwapUsed)/float64(host.SwapTotal))),
+				"#SERVER.DISK#", mod(ns.formatUsage(true, float64(state.DiskUsed)/float64(host.DiskTotal))),
+				"#SERVER.SPEEDIN#", mod(fmt.Sprintf("%s/s", ns.formatSize(state.NetInSpeed))),
+				"#SERVER.SPEEDOUT#", mod(fmt.Sprintf("%s/s", ns.formatSize(state.NetOutSpeed))),
+				"#SERVER.TRANSFERIN#", mod(ns.formatSize(state.NetInTransfer)),
+				"#SERVER.TRANSFEROUT#", mod(ns.formatSize(state.NetOutTransfer)),
 
-		var ipv4, ipv6, validIP string
-		ip := ns.Server.GeoIP.IP
-		if ip.IPv4Addr != "" && ip.IPv6Addr != "" {
-			ipv4 = ip.IPv4Addr
-			ipv6 = ip.IPv6Addr
-			validIP = ipv4
-		} else if ip.IPv4Addr != "" {
-			ipv4 = ip.IPv4Addr
-			validIP = ipv4
-		} else {
-			ipv6 = ip.IPv6Addr
-			validIP = ipv6
+				// Raw metrics
+				"#SERVER.CPUUSED#", mod(fmt.Sprintf("%f", state.CPU)),
+				"#SERVER.MEMUSED#", mod(fmt.Sprintf("%d", state.MemUsed)),
+				"#SERVER.SWAPUSED#", mod(fmt.Sprintf("%d", state.SwapUsed)),
+				"#SERVER.DISKUSED#", mod(fmt.Sprintf("%d", state.DiskUsed)),
+				"#SERVER.MEMTOTAL#", mod(fmt.Sprintf("%d", host.MemTotal)),
+				"#SERVER.SWAPTOTAL#", mod(fmt.Sprintf("%d", host.SwapTotal)),
+				"#SERVER.DISKTOTAL#", mod(fmt.Sprintf("%d", host.DiskTotal)),
+				"#SERVER.NETINSPEED#", mod(fmt.Sprintf("%d", state.NetInSpeed)),
+				"#SERVER.NETOUTSPEED#", mod(fmt.Sprintf("%d", state.NetOutSpeed)),
+				"#SERVER.NETINTRANSFER#", mod(fmt.Sprintf("%d", state.NetInTransfer)),
+				"#SERVER.NETOUTTRANSFER#", mod(fmt.Sprintf("%d", state.NetOutTransfer)),
+				"#SERVER.LOAD1#", mod(fmt.Sprintf("%f", state.Load1)),
+				"#SERVER.LOAD5#", mod(fmt.Sprintf("%f", state.Load5)),
+				"#SERVER.LOAD15#", mod(fmt.Sprintf("%f", state.Load15)),
+				"#SERVER.TCPCONNCOUNT#", mod(fmt.Sprintf("%d", state.TcpConnCount)),
+				"#SERVER.UDPCONNCOUNT#", mod(fmt.Sprintf("%d", state.UdpConnCount)),
+			)
 		}
 
-		replacements = append(replacements,
-			"#SERVER.IP#", mod(validIP),
-			"#SERVER.IPV4#", mod(ipv4),
-			"#SERVER.IPV6#", mod(ipv6),
-		)
+		var ipv4, ipv6, validIP string
+		if ns.Server.GeoIP != nil {
+			ip := ns.Server.GeoIP.IP
+			if ip.IPv4Addr != "" && ip.IPv6Addr != "" {
+				ipv4 = ip.IPv4Addr
+				ipv6 = ip.IPv6Addr
+				validIP = ipv4
+			} else if ip.IPv4Addr != "" {
+				ipv4 = ip.IPv4Addr
+				validIP = ipv4
+			} else {
+				ipv6 = ip.IPv6Addr
+				validIP = ipv6
+			}
+
+			replacements = append(replacements,
+				"#SERVER.IP#", mod(validIP),
+				"#SERVER.IPV4#", mod(ipv4),
+				"#SERVER.IPV6#", mod(ipv6),
+			)
+		}
 	}
 
 	replacer := strings.NewReplacer(replacements...)

model/rule.go

@@ -1,6 +1,8 @@
 package model
 
 import (
+	"log"
+	"math"
 	"slices"
 	"strings"
 	"time"
@@ -90,36 +92,80 @@ func (u *Rule) DurationInt() (duration int, ok bool) {
 // which is the documented legacy spelling for hours.
 func (u *Rule) HasSafeCycleConfiguration() bool {
 	return u != nil && u.IsTransferDurationRule() && u.CycleStart != nil &&
-		u.CycleInterval > 0 && u.CycleInterval <= MaxAlertRuleCycleInterval
+		!u.CycleStart.After(time.Now()) && u.CycleInterval > 0 && u.CycleInterval <= MaxAlertRuleCycleInterval &&
+		u.HasSafeThresholds() && u.HasSafeCycleUnit()
 }
 
-// Snapshot 未通过规则返回 false, 通过返回 true
-func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) bool {
-	if u == nil || server == nil || !u.IsSupportedType() {
+func (u *Rule) HasSafeCycleUnit() bool {
+	if u == nil {
+		return false
+	}
+	switch strings.ToLower(u.CycleUnit) {
+	case "", "hour", "day", "week", "month", "year":
 		return true
+	default:
+		return false
+	}
+}
+
+func (u *Rule) HasSafeThresholds() bool {
+	if u == nil || math.IsNaN(u.Min) || math.IsNaN(u.Max) || math.IsInf(u.Min, 0) || math.IsInf(u.Max, 0) || u.Min < 0 || u.Max < 0 {
+		return false
+	}
+	return u.Min == 0 || u.Max == 0 || u.Min <= u.Max
+}
+
+// Snapshot is retained for callers which only need a boolean result. The alert
+// sentinel uses Evaluate so missing telemetry is not mistaken for a breach.
+func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) bool {
+	passed, _ := u.Evaluate(cycleTransferStats, server, db)
+	return passed
+}
+
+// Evaluate returns known=false when a metric cannot be evaluated from the
+// current report. Unknown must neither trigger nor resolve an incident.
+func (u *Rule) Evaluate(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) (passed, known bool) {
+	if server == nil {
+		return true, false
+	}
+	return u.EvaluateRuntime(cycleTransferStats, server, server.RuntimeSnapshot(), db)
+}
+
+// EvaluateRuntime evaluates all conditions against the same immutable Agent
+// report. Callers evaluating a compound rule must share this snapshot.
+func (u *Rule) EvaluateRuntime(cycleTransferStats *CycleTransferStats, server *Server, runtime RuntimeSnapshot, db *gorm.DB) (passed, known bool) {
+	if u == nil || server == nil || !u.IsSupportedType() {
+		return true, false
 	}
 	if u.IsTransferDurationRule() && (!u.HasSafeCycleConfiguration() || cycleTransferStats == nil) {
-		return true
+		return true, false
 	}
 
 	// 监控全部但是排除了此服务器
 	if u.Cover == RuleCoverAll && u.Ignore[server.ID] {
-		return true
+		return true, true
 	}
 	// 忽略全部但是指定监控了此服务器
 	if u.Cover == RuleCoverIgnoreAll && !u.Ignore[server.ID] {
-		return true
+		return true, true
 	}
 
+	evalNow := time.Now()
+	var cycleStart, cycleEnd time.Time
+	if u.IsTransferDurationRule() {
+		cycleStart, cycleEnd = u.transferDurationBounds(evalNow)
+	}
 	// 循环区间流量检测 · 短期无需重复检测
-	if u.IsTransferDurationRule() && u.NextTransferAt[server.ID].After(time.Now()) {
-		return u.LastCycleStatus[server.ID]
+	if u.IsTransferDurationRule() && u.NextTransferAt[server.ID].After(evalNow) {
+		return u.LastCycleStatus[server.ID], true
 	}
 
 	var src float64
-	runtime := server.RuntimeSnapshot()
+	if u.IsOfflineRule() {
+		return !runtime.LastActive.IsZero() && time.Since(runtime.LastActive) <= 6*time.Second, true
+	}
 	if runtime.State == nil {
-		return false
+		return true, false
 	}
 	state := runtime.State
 
@@ -128,22 +174,31 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server,
 		src = float64(state.CPU)
 	case "gpu", "gpu_max":
 		if len(state.GPU) == 0 {
-			return true
+			return true, false
 		}
 		src = slices.Max(state.GPU)
 	case "memory":
 		if runtime.Host == nil {
-			return false
+			return true, false
+		}
+		if runtime.Host.MemTotal == 0 {
+			return true, false
 		}
 		src = percentage(state.MemUsed, runtime.Host.MemTotal)
 	case "swap":
 		if runtime.Host == nil {
-			return false
+			return true, false
+		}
+		if runtime.Host.SwapTotal == 0 {
+			return true, false
 		}
 		src = percentage(state.SwapUsed, runtime.Host.SwapTotal)
 	case "disk":
 		if runtime.Host == nil {
-			return false
+			return true, false
+		}
+		if runtime.Host.DiskTotal == 0 {
+			return true, false
 		}
 		src = percentage(state.DiskUsed, runtime.Host.DiskTotal)
 	case "net_in_speed":
@@ -151,38 +206,50 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server,
 	case "net_out_speed":
 		src = float64(state.NetOutSpeed)
 	case "net_all_speed":
-		src = float64(state.NetOutSpeed + state.NetOutSpeed)
+		src = float64(state.NetInSpeed) + float64(state.NetOutSpeed)
 	case "transfer_in":
 		src = float64(state.NetInTransfer)
 	case "transfer_out":
 		src = float64(state.NetOutTransfer)
 	case "transfer_all":
 		src = float64(state.NetOutTransfer + state.NetInTransfer)
-	case "offline":
-		if runtime.LastActive.IsZero() {
-			src = 0
-		} else {
-			src = float64(runtime.LastActive.Unix())
-		}
 	case "transfer_in_cycle":
+		if db == nil {
+			return true, false
+		}
 		src = float64(utils.SubUintChecked(state.NetInTransfer, runtime.PrevTransferInSnapshot))
 		if u.CycleInterval != 0 {
 			var res NResult
-			db.Model(&Transfer{}).Select("SUM(`in`) AS n").Where("datetime(`created_at`) >= datetime(?) AND server_id = ?", u.GetTransferDurationStart().UTC(), server.ID).Scan(&res)
+			if err := db.Model(&Transfer{}).Select("SUM(`in`) AS n").Where("datetime(`created_at`) >= datetime(?) AND datetime(`created_at`) < datetime(?) AND datetime(`created_at`) <= datetime(?) AND server_id = ?", cycleStart.UTC(), cycleEnd.UTC(), evalNow.UTC(), server.ID).Scan(&res).Error; err != nil {
+				log.Printf("NEZHA>> Alert cycle transfer query failed for rule %s server %d: %v", u.Type, server.ID, err)
+				return true, false
+			}
 			src += float64(res.N)
 		}
 	case "transfer_out_cycle":
+		if db == nil {
+			return true, false
+		}
 		src = float64(utils.SubUintChecked(state.NetOutTransfer, runtime.PrevTransferOutSnapshot))
 		if u.CycleInterval != 0 {
 			var res NResult
-			db.Model(&Transfer{}).Select("SUM(`out`) AS n").Where("datetime(`created_at`) >= datetime(?) AND server_id = ?", u.GetTransferDurationStart().UTC(), server.ID).Scan(&res)
+			if err := db.Model(&Transfer{}).Select("SUM(`out`) AS n").Where("datetime(`created_at`) >= datetime(?) AND datetime(`created_at`) < datetime(?) AND datetime(`created_at`) <= datetime(?) AND server_id = ?", cycleStart.UTC(), cycleEnd.UTC(), evalNow.UTC(), server.ID).Scan(&res).Error; err != nil {
+				log.Printf("NEZHA>> Alert cycle transfer query failed for rule %s server %d: %v", u.Type, server.ID, err)
+				return true, false
+			}
 			src += float64(res.N)
 		}
 	case "transfer_all_cycle":
+		if db == nil {
+			return true, false
+		}
 		src = float64(utils.SubUintChecked(state.NetOutTransfer, runtime.PrevTransferOutSnapshot) + utils.SubUintChecked(state.NetInTransfer, runtime.PrevTransferInSnapshot))
 		if u.CycleInterval != 0 {
 			var res NResult
-			db.Model(&Transfer{}).Select("SUM(`in`+`out`) AS n").Where("datetime(`created_at`) >= datetime(?) AND server_id = ?", u.GetTransferDurationStart().UTC(), server.ID).Scan(&res)
+			if err := db.Model(&Transfer{}).Select("SUM(`in`+`out`) AS n").Where("datetime(`created_at`) >= datetime(?) AND datetime(`created_at`) < datetime(?) AND datetime(`created_at`) <= datetime(?) AND server_id = ?", cycleStart.UTC(), cycleEnd.UTC(), evalNow.UTC(), server.ID).Scan(&res).Error; err != nil {
+				log.Printf("NEZHA>> Alert cycle transfer query failed for rule %s server %d: %v", u.Type, server.ID, err)
+				return true, false
+			}
 			src += float64(res.N)
 		}
 	case "load1":
@@ -205,23 +272,32 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server,
 			}
 		}
 		if len(temp) == 0 {
-			return true
+			return true, false
 		}
 		src = slices.Max(temp)
 	default:
-		return true
+		return true, false
 	}
 
 	// 循环区间流量检测 · 更新下次需要检测时间
 	if u.IsTransferDurationRule() {
-		seconds := max(1800*((u.Max-src)/u.Max), 180)
+		seconds := float64(180)
+		if u.Max > 0 {
+			seconds = max(1800*((u.Max-src)/u.Max), 180)
+		} else if u.Min > 0 {
+			seconds = 1800
+		}
 		if u.NextTransferAt == nil {
 			u.NextTransferAt = make(map[uint64]time.Time)
 		}
 		if u.LastCycleStatus == nil {
 			u.LastCycleStatus = make(map[uint64]bool)
 		}
-		u.NextTransferAt[server.ID] = time.Now().Add(time.Second * time.Duration(seconds))
+		nextAt := evalNow.Add(time.Second * time.Duration(seconds))
+		if nextAt.After(cycleEnd) {
+			nextAt = cycleEnd
+		}
+		u.NextTransferAt[server.ID] = nextAt
 		if (u.Max > 0 && src > u.Max) || (u.Min > 0 && src < u.Min) {
 			u.LastCycleStatus[server.ID] = false
 		} else {
@@ -233,17 +309,15 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server,
 		cycleTransferStats.Transfer[server.ID] = uint64(src)
 		cycleTransferStats.NextUpdate[server.ID] = u.NextTransferAt[server.ID]
 		// 自动更新周期流量展示起止时间
-		cycleTransferStats.From = u.GetTransferDurationStart()
-		cycleTransferStats.To = u.GetTransferDurationEnd()
+		cycleTransferStats.From = cycleStart
+		cycleTransferStats.To = cycleEnd
 	}
 
-	if u.Type == "offline" && float64(time.Now().Unix())-src > 6 {
-		return false
-	} else if (u.Max > 0 && src > u.Max) || (u.Min > 0 && src < u.Min) {
-		return false
+	if (u.Max > 0 && src > u.Max) || (u.Min > 0 && src < u.Min) {
+		return false, true
 	}
 
-	return true
+	return true, true
 }
 
 // IsTransferDurationRule 判断该规则是否属于周期流量规则 属于则返回true
@@ -263,83 +337,49 @@ func (u *Rule) IsOfflineRule() bool {
 	return u != nil && u.Type == "offline"
 }
 
-// GetTransferDurationStart 获取周期流量的起始时间
+// transferDurationBounds computes both edges from the same clock reading.
+// Separate time.Now calls can otherwise straddle a cycle boundary.
+func (u *Rule) transferDurationBounds(now time.Time) (time.Time, time.Time) {
+	if !u.HasSafeCycleConfiguration() {
+		return time.Time{}, time.Time{}
+	}
+	start := *u.CycleStart
+	var end time.Time
+	switch strings.ToLower(u.CycleUnit) {
+	case "year":
+		end = start.AddDate(int(u.CycleInterval), 0, 0)
+		for !now.Before(end) {
+			start, end = end, end.AddDate(int(u.CycleInterval), 0, 0)
+		}
+	case "month":
+		end = start.AddDate(0, int(u.CycleInterval), 0)
+		for !now.Before(end) {
+			start, end = end, end.AddDate(0, int(u.CycleInterval), 0)
+		}
+	case "week":
+		end = start.AddDate(0, 0, 7*int(u.CycleInterval))
+		for !now.Before(end) {
+			start, end = end, end.AddDate(0, 0, 7*int(u.CycleInterval))
+		}
+	case "day":
+		end = start.AddDate(0, 0, int(u.CycleInterval))
+		for !now.Before(end) {
+			start, end = end, end.AddDate(0, 0, int(u.CycleInterval))
+		}
+	default: // empty is the legacy spelling for hour
+		interval := 3600 * int64(u.CycleInterval)
+		start = time.Unix(u.CycleStart.Unix()+(now.Unix()-u.CycleStart.Unix())/interval*interval, 0)
+		end = time.Unix(start.Unix()+interval, 0)
+	}
+	return start, end
+}
+
 func (u *Rule) GetTransferDurationStart() time.Time {
-	// Accept uppercase and lowercase
-	unit := strings.ToLower(u.CycleUnit)
-	startTime := *u.CycleStart
-	var nextTime time.Time
-	switch unit {
-	case "year":
-		nextTime = startTime.AddDate(int(u.CycleInterval), 0, 0)
-		for time.Now().After(nextTime) {
-			startTime = nextTime
-			nextTime = nextTime.AddDate(int(u.CycleInterval), 0, 0)
-		}
-	case "month":
-		nextTime = startTime.AddDate(0, int(u.CycleInterval), 0)
-		for time.Now().After(nextTime) {
-			startTime = nextTime
-			nextTime = nextTime.AddDate(0, int(u.CycleInterval), 0)
-		}
-	case "week":
-		nextTime = startTime.AddDate(0, 0, 7*int(u.CycleInterval))
-		for time.Now().After(nextTime) {
-			startTime = nextTime
-			nextTime = nextTime.AddDate(0, 0, 7*int(u.CycleInterval))
-		}
-	case "day":
-		nextTime = startTime.AddDate(0, 0, int(u.CycleInterval))
-		for time.Now().After(nextTime) {
-			startTime = nextTime
-			nextTime = nextTime.AddDate(0, 0, int(u.CycleInterval))
-		}
-	default:
-		// For hour unit or not set.
-		interval := 3600 * int64(u.CycleInterval)
-		startTime = time.Unix(u.CycleStart.Unix()+(time.Now().Unix()-u.CycleStart.Unix())/interval*interval, 0)
-	}
-
-	return startTime
+	start, _ := u.transferDurationBounds(time.Now())
+	return start
 }
 
-// GetTransferDurationEnd 获取周期流量结束时间
 func (u *Rule) GetTransferDurationEnd() time.Time {
-	// Accept uppercase and lowercase
-	unit := strings.ToLower(u.CycleUnit)
-	startTime := *u.CycleStart
-	var nextTime time.Time
-	switch unit {
-	case "year":
-		nextTime = startTime.AddDate(int(u.CycleInterval), 0, 0)
-		for time.Now().After(nextTime) {
-			startTime = nextTime
-			nextTime = nextTime.AddDate(int(u.CycleInterval), 0, 0)
-		}
-	case "month":
-		nextTime = startTime.AddDate(0, int(u.CycleInterval), 0)
-		for time.Now().After(nextTime) {
-			startTime = nextTime
-			nextTime = nextTime.AddDate(0, int(u.CycleInterval), 0)
-		}
-	case "week":
-		nextTime = startTime.AddDate(0, 0, 7*int(u.CycleInterval))
-		for time.Now().After(nextTime) {
-			startTime = nextTime
-			nextTime = nextTime.AddDate(0, 0, 7*int(u.CycleInterval))
-		}
-	case "day":
-		nextTime = startTime.AddDate(0, 0, int(u.CycleInterval))
-		for time.Now().After(nextTime) {
-			startTime = nextTime
-			nextTime = nextTime.AddDate(0, 0, int(u.CycleInterval))
-		}
-	default:
-		// For hour unit or not set.
-		interval := 3600 * int64(u.CycleInterval)
-		startTime = time.Unix(u.CycleStart.Unix()+(time.Now().Unix()-u.CycleStart.Unix())/interval*interval, 0)
-		nextTime = time.Unix(startTime.Unix()+interval, 0)
-	}
-
-	return nextTime
+	_, end := u.transferDurationBounds(time.Now())
+	return end
 }

model/server.go

@@ -16,6 +16,7 @@ import (
 )
 
 var runtimeHolderInitMu sync.Mutex
+var nextServerReportSequence atomic.Uint64
 
 type Server struct {
 	Common
@@ -73,15 +74,16 @@ type taskStreamHolder struct {
 }
 
 type serverRuntimeHolder struct {
-	mu         sync.Mutex
-	canonical  *Server
-	stream     pb.NezhaService_ReportSystemStateServer
-	generation uint64
-	state      *HostState
-	host       *Host
-	lastActive time.Time
-	prevIn     uint64
-	prevOut    uint64
+	mu             sync.Mutex
+	canonical      *Server
+	stream         pb.NezhaService_ReportSystemStateServer
+	generation     uint64
+	state          *HostState
+	host           *Host
+	lastActive     time.Time
+	reportSequence uint64
+	prevIn         uint64
+	prevOut        uint64
 }
 
 type StateStreamLease struct {
@@ -204,6 +206,7 @@ func (lease StateStreamLease) updateState(receiver *Server, state *HostState, la
 			return false
 		}
 	}
+	lease.holder.reportSequence = nextServerReportSequence.Add(1)
 	return true
 }
 
@@ -334,6 +337,7 @@ type RuntimeSnapshot struct {
 	State                   *HostState
 	Host                    *Host
 	LastActive              time.Time
+	ReportSequence          uint64
 	PrevTransferInSnapshot  uint64
 	PrevTransferOutSnapshot uint64
 }
@@ -360,7 +364,7 @@ func (s *Server) RuntimeSnapshot() RuntimeSnapshot {
 			holder.host = cloneHost(s.Host)
 		}
 	}
-	return RuntimeSnapshot{State: cloneHostState(holder.state), Host: cloneHost(holder.host), LastActive: holder.lastActive, PrevTransferInSnapshot: holder.prevIn, PrevTransferOutSnapshot: holder.prevOut}
+	return RuntimeSnapshot{State: cloneHostState(holder.state), Host: cloneHost(holder.host), LastActive: holder.lastActive, ReportSequence: holder.reportSequence, PrevTransferInSnapshot: holder.prevIn, PrevTransferOutSnapshot: holder.prevOut}
 }
 
 func (s *Server) SetTransferSnapshots(inbound, outbound uint64) bool {