Merge upstream monitoring features preserving domain extensions and require fixed Go toolchain

service/singleton/alertsentinel.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/jinzhu/copier"
@@ -26,14 +27,125 @@ type NotificationHistory struct {
 var (
 	AlertsLock                    sync.RWMutex
 	Alerts                        []*model.AlertRule
-	alertsStore                   map[uint64]map[uint64][][]bool       // [alert_id][server_id] -> [timeTick][ruleId] 时间点对应的rule的检查结果
-	alertsPrevState               map[uint64]map[uint64]uint8          // [alert_id][server_id] -> 对应报警规则的上一次报警状态
-	AlertsCycleTransferStatsStore map[uint64]*model.CycleTransferStats // [alert_id] -> 对应报警规则的周期流量统计
+	alertsStore                   map[uint64]map[uint64][]model.TimedAlertPoint // distinct reports per alert/server
+	alertsPrevState               map[uint64]map[uint64]uint8                   // [alert_id][server_id] -> 对应报警规则的上一次报警状态
+	alertsLastMetricSeq           map[uint64]map[uint64]uint64                  // last distinct Agent metric report per alert/server
+	AlertsCycleTransferStatsStore map[uint64]*model.CycleTransferStats          // [alert_id] -> 对应报警规则的周期流量统计
+	alertDeliveryMu               sync.Mutex
+	alertDeliveries               map[uint64]map[uint64]*alertDeliveryEntry
+	alertEventSequence            atomic.Uint64
 )
 
+type alertDeliveryEntry struct {
+	cancel chan struct{}
+	done   chan struct{}
+	sendMu *sync.Mutex
+}
+
+type alertDeliveryPhase uint8
+
+const (
+	alertDeliveryIncident alertDeliveryPhase = iota
+	alertDeliveryRecovery
+)
+
+func cancelAlertDeliveries(alertID uint64) {
+	alertDeliveryMu.Lock()
+	defer alertDeliveryMu.Unlock()
+	for _, entry := range alertDeliveries[alertID] {
+		close(entry.cancel)
+	}
+	delete(alertDeliveries, alertID)
+}
+
+func cancelAlertDeliveriesForServers(serverIDs []uint64) {
+	alertDeliveryMu.Lock()
+	defer alertDeliveryMu.Unlock()
+	for alertID, byServer := range alertDeliveries {
+		for _, serverID := range serverIDs {
+			if entry := byServer[serverID]; entry != nil {
+				close(entry.cancel)
+				delete(byServer, serverID)
+			}
+		}
+		if len(byServer) == 0 {
+			delete(alertDeliveries, alertID)
+		}
+	}
+}
+
+func runAlertDelivery(cancel <-chan struct{}, send func() bool, always bool, retryDelay time.Duration) {
+	for {
+		select {
+		case <-cancel:
+			return
+		default:
+		}
+		delivered := send()
+		if delivered && !always {
+			return
+		}
+		delay := retryDelay
+		if delivered {
+			delay = firstNotificationDelay
+		}
+		timer := time.NewTimer(delay)
+		select {
+		case <-cancel:
+			timer.Stop()
+			return
+		case <-timer.C:
+		}
+	}
+}
+
+func startAlertDelivery(alert *model.AlertRule, server *model.Server, message, muteLabel string, phase alertDeliveryPhase) {
+	// A mute entry belongs to one transition, not all later incidents of the
+	// same phase. Concurrent in-flight sends may finish after a recovery.
+	muteLabel = fmt.Sprintf("%s:event-%d", muteLabel, alertEventSequence.Add(1))
+	alertDeliveryMu.Lock()
+	if alertDeliveries == nil {
+		alertDeliveries = make(map[uint64]map[uint64]*alertDeliveryEntry)
+	}
+	if alertDeliveries[alert.ID] == nil {
+		alertDeliveries[alert.ID] = make(map[uint64]*alertDeliveryEntry)
+	}
+	sendMu := new(sync.Mutex)
+	if old := alertDeliveries[alert.ID][server.ID]; old != nil {
+		close(old.cancel)
+		sendMu = old.sendMu
+	}
+	entry := &alertDeliveryEntry{cancel: make(chan struct{}), done: make(chan struct{}), sendMu: sendMu}
+	alertDeliveries[alert.ID][server.ID] = entry
+	alertDeliveryMu.Unlock()
+
+	groupID := alert.NotificationGroupID
+	always := phase == alertDeliveryIncident && alert.TriggerMode == model.ModeAlwaysTrigger
+	go func() {
+		defer close(entry.done)
+		runAlertDelivery(entry.cancel, func() bool {
+			// Serialize incident/recovery delivery for this alert/server pair.
+			// A recovery can cancel a pending incident, but never overtake an
+			// already-running network request.
+			entry.sendMu.Lock()
+			defer entry.sendMu.Unlock()
+			select {
+			case <-entry.cancel:
+				return true
+			default:
+			}
+			return NotificationShared.SendNotification(groupID, message, muteLabel, server)
+		}, always, 30*time.Second)
+	}()
+}
+
+func shouldRunAlertFailTasks(triggerMode uint8, newIncident bool) bool {
+	return newIncident || triggerMode == model.ModeAlwaysTrigger
+}
+
 // addCycleTransferStatsInfo 向AlertsCycleTransferStatsStore中添加周期流量报警统计信息
 func addCycleTransferStatsInfo(alert *model.AlertRule) {
-	if !alert.Enabled() {
+	if alert == nil || !alert.Enabled() || !alert.IsSafeToEvaluate() {
 		return
 	}
 	for _, rule := range alert.Rules {
@@ -59,16 +171,34 @@ func addCycleTransferStatsInfo(alert *model.AlertRule) {
 
 // AlertSentinelStart 报警器启动
 func AlertSentinelStart() {
-	alertsStore = make(map[uint64]map[uint64][][]bool)
+	alertDeliveryMu.Lock()
+	for _, byServer := range alertDeliveries {
+		for _, entry := range byServer {
+			close(entry.cancel)
+		}
+	}
+	alertDeliveries = make(map[uint64]map[uint64]*alertDeliveryEntry)
+	alertDeliveryMu.Unlock()
+	alertsStore = make(map[uint64]map[uint64][]model.TimedAlertPoint)
 	alertsPrevState = make(map[uint64]map[uint64]uint8)
+	alertsLastMetricSeq = make(map[uint64]map[uint64]uint64)
 	AlertsCycleTransferStatsStore = make(map[uint64]*model.CycleTransferStats)
 	AlertsLock.Lock()
 	if err := DB.Find(&Alerts).Error; err != nil {
 		panic(err)
 	}
 	for _, alert := range Alerts {
-		alertsStore[alert.ID] = make(map[uint64][][]bool)
+		if alert == nil {
+			log.Printf("NEZHA>> Skipping invalid nil alert rule loaded from database")
+			continue
+		}
+		alertsStore[alert.ID] = make(map[uint64][]model.TimedAlertPoint)
 		alertsPrevState[alert.ID] = make(map[uint64]uint8)
+		alertsLastMetricSeq[alert.ID] = make(map[uint64]uint64)
+		if !alert.IsSafeToEvaluate() {
+			log.Printf("NEZHA>> Skipping invalid alert rule %d loaded from database", alert.ID)
+			continue
+		}
 		addCycleTransferStatsInfo(alert)
 	}
 	AlertsLock.Unlock()
@@ -93,8 +223,10 @@ func AlertSentinelStart() {
 func OnRefreshOrAddAlert(alert *model.AlertRule) {
 	AlertsLock.Lock()
 	defer AlertsLock.Unlock()
+	cancelAlertDeliveries(alert.ID)
 	delete(alertsStore, alert.ID)
 	delete(alertsPrevState, alert.ID)
+	delete(alertsLastMetricSeq, alert.ID)
 	var isEdit bool
 	for i := range Alerts {
 		if Alerts[i].ID == alert.ID {
@@ -105,8 +237,9 @@ func OnRefreshOrAddAlert(alert *model.AlertRule) {
 	if !isEdit {
 		Alerts = append(Alerts, alert)
 	}
-	alertsStore[alert.ID] = make(map[uint64][][]bool)
+	alertsStore[alert.ID] = make(map[uint64][]model.TimedAlertPoint)
 	alertsPrevState[alert.ID] = make(map[uint64]uint8)
+	alertsLastMetricSeq[alert.ID] = make(map[uint64]uint64)
 	delete(AlertsCycleTransferStatsStore, alert.ID)
 	addCycleTransferStatsInfo(alert)
 }
@@ -115,8 +248,10 @@ func OnDeleteAlert(id []uint64) {
 	AlertsLock.Lock()
 	defer AlertsLock.Unlock()
 	for _, i := range id {
+		cancelAlertDeliveries(i)
 		delete(alertsStore, i)
 		delete(alertsPrevState, i)
+		delete(alertsLastMetricSeq, i)
 		currentAlerts := Alerts[:0]
 		for _, alert := range Alerts {
 			if alert.ID != i {
@@ -136,7 +271,7 @@ func checkStatus() {
 
 	for _, alert := range Alerts {
 		// 跳过未启用
-		if !alert.Enabled() {
+		if alert == nil || !alert.Enabled() || !alert.IsSafeToEvaluate() {
 			continue
 		}
 		for _, server := range m {
@@ -152,48 +287,92 @@ func checkStatus() {
 			if alert.UserID != server.GetUserID() && !role.IsAdmin() {
 				continue
 			}
-			alertsStore[alert.ID][server.ID] = append(alertsStore[alert.
-				ID][server.ID], alert.Snapshot(AlertsCycleTransferStatsStore[alert.ID], server, DB))
-			// 发送通知，分为触发报警和恢复通知
-			_, passed := alert.Check(alertsStore[alert.ID][server.ID])
-			// 保存当前服务器状态信息
-			curServer := model.Server{}
-			copier.Copy(&curServer, server)
-
-			// 本次未通过检查
-			if !passed {
-				// 始终触发模式或上次检查不为失败时触发报警（跳过单次触发+上次失败的情况）
-				if alert.TriggerMode == model.ModeAlwaysTrigger || alertsPrevState[alert.ID][server.ID] != _RuleCheckFail {
-					alertsPrevState[alert.ID][server.ID] = _RuleCheckFail
-					message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Incident"),
-						server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name)
-					go CronShared.SendTriggerTasks(alert.FailTriggerTasks, curServer.ID, alert.UserID)
-					go NotificationShared.SendNotification(alert.NotificationGroupID, message, NotificationMuteLabel.ServerIncident(server.ID, alert.ID), &curServer)
-					// 清除恢复通知的静音缓存
-					NotificationShared.UnMuteNotification(alert.NotificationGroupID, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID))
-				}
-			} else {
-				// 本次通过检查但上一次的状态为失败，则发送恢复通知
-				if alertsPrevState[alert.ID][server.ID] == _RuleCheckFail {
-					message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Resolved"),
-						server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name)
-					go CronShared.SendTriggerTasks(alert.RecoverTriggerTasks, curServer.ID, alert.UserID)
-					go NotificationShared.SendNotification(alert.NotificationGroupID, message, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), &curServer)
-					// 清除失败通知的静音缓存
-					NotificationShared.UnMuteNotification(alert.NotificationGroupID, NotificationMuteLabel.ServerIncident(server.ID, alert.ID))
-				}
-				alertsPrevState[alert.ID][server.ID] = _RuleCheckPass
-			}
-			// 清理旧数据：保留窗口由规则定义决定（各规则 Duration 的最大值），
-			// 而非 Check 的判定结果。window==0 表示没有任何有效规则需要回看历史
-			// （例如全部 Duration<=0），此时清空采样避免切片无限增长。
-			window := alert.RetentionWindow()
-			samples := alertsStore[alert.ID][server.ID]
-			if window <= 0 {
-				alertsStore[alert.ID][server.ID] = samples[:0]
-			} else if window < len(samples) {
-				alertsStore[alert.ID][server.ID] = samples[len(samples)-window:]
-			}
+			checkStatusForServer(alert, server)
 		}
 	}
 }
+
+// checkStatusForServer isolates each alert/server evaluation. Input validation
+// and model-level guards handle known malformed states; this recovery boundary
+// ensures an unexpected evaluator panic cannot terminate the dashboard process
+// or prevent unrelated servers from being checked on the same tick.
+func checkStatusForServer(alert *model.AlertRule, server *model.Server) {
+	defer func() {
+		if recovered := recover(); recovered != nil {
+			log.Printf("NEZHA>> Recovered panic evaluating alert rule %d for server %d: %v", alert.ID, server.ID, recovered)
+		}
+	}()
+
+	// Offline-only rules deliberately observe elapsed time on every tick. Metric
+	// rules, including compound rules, must never count the same Agent report
+	// repeatedly or treat a disconnected Agent's retained State as fresh data.
+	metricRule := false
+	runtime := server.RuntimeSnapshot()
+	for _, rule := range alert.Rules {
+		if !rule.IsOfflineRule() {
+			metricRule = true
+			break
+		}
+	}
+	if metricRule {
+		if runtime.LastActive.IsZero() || runtime.State == nil || time.Since(runtime.LastActive) > model.AlertSampleMaxAge || time.Since(runtime.LastActive) < 0 {
+			alertsStore[alert.ID][server.ID] = nil
+			return
+		}
+		if runtime.ReportSequence == 0 || runtime.ReportSequence == alertsLastMetricSeq[alert.ID][server.ID] {
+			return
+		}
+		alertsLastMetricSeq[alert.ID][server.ID] = runtime.ReportSequence
+	}
+	point, known := alert.SnapshotStatusWithRuntime(AlertsCycleTransferStatsStore[alert.ID], server, runtime, DB)
+	if !known {
+		alertsStore[alert.ID][server.ID] = nil
+		return
+	}
+	sampledAt := time.Now()
+	if metricRule {
+		sampledAt = runtime.LastActive
+	}
+	alertsStore[alert.ID][server.ID] = append(alertsStore[alert.ID][server.ID], model.TimedAlertPoint{At: sampledAt, Values: point})
+	// Bound memory even while coverage is insufficient to decide a state.
+	cutoff := time.Now().Add(-alert.TimedRetention())
+	samples := alertsStore[alert.ID][server.ID]
+	first := 0
+	for first < len(samples) && samples[first].At.Before(cutoff) {
+		first++
+	}
+	if first > 0 {
+		alertsStore[alert.ID][server.ID] = append([]model.TimedAlertPoint(nil), samples[first:]...)
+	}
+	// 发送通知，分为触发报警和恢复通知
+	known, passed := alert.CheckTimed(alertsStore[alert.ID][server.ID], time.Now())
+	if !known {
+		return
+	}
+	// 保存当前服务器状态信息
+	curServer := model.Server{}
+	copier.Copy(&curServer, server)
+
+	// 本次未通过检查
+	if !passed {
+		newIncident := alertsPrevState[alert.ID][server.ID] != _RuleCheckFail
+		if newIncident {
+			alertsPrevState[alert.ID][server.ID] = _RuleCheckFail
+			message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Incident"),
+				server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name)
+			startAlertDelivery(alert, &curServer, message, NotificationMuteLabel.ServerIncident(server.ID, alert.ID), alertDeliveryIncident)
+		}
+		if shouldRunAlertFailTasks(alert.TriggerMode, newIncident) {
+			go CronShared.SendTriggerTasks(alert.FailTriggerTasks, curServer.ID, alert.UserID)
+		}
+	} else {
+		// 本次通过检查但上一次的状态为失败，则发送恢复通知
+		if alertsPrevState[alert.ID][server.ID] == _RuleCheckFail {
+			message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Resolved"),
+				server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name)
+			go CronShared.SendTriggerTasks(alert.RecoverTriggerTasks, curServer.ID, alert.UserID)
+			startAlertDelivery(alert, &curServer, message, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), alertDeliveryRecovery)
+		}
+		alertsPrevState[alert.ID][server.ID] = _RuleCheckPass
+	}
+}

service/singleton/alertsentinel_freshness_test.go

@@ -0,0 +1,281 @@
+package singleton
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/patrickmn/go-cache"
+	"google.golang.org/grpc/metadata"
+
+	"github.com/nezhahq/nezha/model"
+	pb "github.com/nezhahq/nezha/proto"
+)
+
+type alertTestStateStream struct{}
+
+func (alertTestStateStream) Send(*pb.Receipt) error       { return nil }
+func (alertTestStateStream) Recv() (*pb.State, error)     { return nil, context.Canceled }
+func (alertTestStateStream) SetHeader(metadata.MD) error  { return nil }
+func (alertTestStateStream) SendHeader(metadata.MD) error { return nil }
+func (alertTestStateStream) SetTrailer(metadata.MD)       {}
+func (alertTestStateStream) Context() context.Context     { return context.Background() }
+func (alertTestStateStream) SendMsg(any) error            { return nil }
+func (alertTestStateStream) RecvMsg(any) error            { return nil }
+
+func TestAlertSentinelCountsOnlyFreshDistinctMetricReports(t *testing.T) {
+	oldStore, oldPrev, oldLast, oldCycle := alertsStore, alertsPrevState, alertsLastMetricSeq, AlertsCycleTransferStatsStore
+	alertsStore = map[uint64]map[uint64][]model.TimedAlertPoint{7: {1: nil}}
+	alertsPrevState = map[uint64]map[uint64]uint8{7: {}}
+	alertsLastMetricSeq = map[uint64]map[uint64]uint64{7: {}}
+	AlertsCycleTransferStatsStore = map[uint64]*model.CycleTransferStats{}
+	t.Cleanup(func() {
+		alertsStore, alertsPrevState, alertsLastMetricSeq, AlertsCycleTransferStatsStore = oldStore, oldPrev, oldLast, oldCycle
+	})
+
+	alert := &model.AlertRule{
+		Common: model.Common{ID: 7},
+		Rules:  []*model.Rule{{Type: "cpu", Max: 80, Duration: 100, Cover: model.RuleCoverAll}},
+	}
+	server := &model.Server{Common: model.Common{ID: 1}}
+	model.InitServer(server)
+	lease := server.AttachStateStream(alertTestStateStream{})
+	fixedReportTime := time.Now()
+	if !lease.UpdateState(&model.HostState{CPU: 99}, fixedReportTime) {
+		t.Fatal("state update rejected")
+	}
+	checkStatusForServer(alert, server)
+	if got := len(alertsStore[7][1]); got != 1 {
+		t.Fatalf("first report produced %d samples, want 1", got)
+	}
+	for i := 0; i < 100; i++ {
+		checkStatusForServer(alert, server)
+	}
+	if got := len(alertsStore[7][1]); got != 1 {
+		t.Fatalf("same report was counted %d times, want 1", got)
+	}
+	if !lease.Clear() {
+		t.Fatal("stream clear rejected")
+	}
+	checkStatusForServer(alert, server)
+	if got := len(alertsStore[7][1]); got != 0 {
+		t.Fatalf("disconnect retained %d high samples, want 0", got)
+	}
+	alertsPrevState[7][1] = _RuleCheckFail // incident was active before disconnect
+	newLease := server.AttachStateStream(alertTestStateStream{})
+	// Distinct reports may share a clock timestamp on coarse-resolution hosts.
+	if !newLease.UpdateState(&model.HostState{CPU: 99}, fixedReportTime) {
+		t.Fatal("reconnect state update rejected")
+	}
+	checkStatusForServer(alert, server)
+	if got := len(alertsStore[7][1]); got != 1 || alertsStore[7][1][0].Values[0] {
+		t.Fatalf("reconnect window = %v, want one high sample", alertsStore[7][1])
+	}
+	if alertsPrevState[7][1] != _RuleCheckFail {
+		t.Fatal("incomplete warm-up falsely resolved active incident")
+	}
+	replacement := &model.Server{Common: model.Common{ID: 1}}
+	model.InitServer(replacement)
+	replacementLease := replacement.AttachStateStream(alertTestStateStream{})
+	if !replacementLease.UpdateState(&model.HostState{CPU: 99}, fixedReportTime) {
+		t.Fatal("replacement server state update rejected")
+	}
+	checkStatusForServer(alert, replacement)
+	if got := len(alertsStore[7][1]); got != 2 {
+		t.Fatalf("replacement holder reused a report identity; samples=%d", got)
+	}
+}
+
+func TestAlertNewIncidentNotMutedByEarlierInflightIncident(t *testing.T) {
+	oldCache, oldNotifications := Cache, NotificationShared
+	Cache = cache.New(time.Minute, time.Minute)
+	n := &model.Notification{Common: model.Common{ID: 601}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 603}, Name: "alert-test"}, []uint64{n.ID})
+	NotificationShared = nc
+	t.Cleanup(func() {
+		cancelAlertDeliveries(602)
+		NotificationShared, Cache = oldNotifications, oldCache
+	})
+	started := make(chan struct{})
+	release := make(chan struct{})
+	delivered := make(chan string, 3)
+	nc.sendForTest = func(_ *model.Notification, desc string, _ *model.Server) error {
+		if desc == "first incident" {
+			close(started)
+			<-release
+		}
+		delivered <- desc
+		return nil
+	}
+	alert := &model.AlertRule{
+		Common:              model.Common{ID: 602},
+		NotificationGroupID: 603,
+		TriggerMode:         model.ModeOnetimeTrigger,
+	}
+	server := &model.Server{Common: model.Common{ID: 604}}
+	label := NotificationMuteLabel.ServerIncident(server.ID, alert.ID)
+	startAlertDelivery(alert, server, "first incident", label, alertDeliveryIncident)
+	<-started
+	startAlertDelivery(alert, server, "recovered", NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), alertDeliveryRecovery)
+	startAlertDelivery(alert, server, "second incident", label, alertDeliveryIncident)
+	close(release)
+	timer := time.NewTimer(time.Second)
+	defer timer.Stop()
+	seen := map[string]bool{}
+	for len(seen) < 2 {
+		select {
+		case message := <-delivered:
+			seen[message] = true
+		case <-timer.C:
+			t.Fatalf("new incident was suppressed after old send completed: %v", seen)
+		}
+	}
+	if !seen["first incident"] || !seen["second incident"] || seen["recovered"] {
+		t.Fatalf("unexpected transition deliveries: %v", seen)
+	}
+	alertDeliveryMu.Lock()
+	done := alertDeliveries[alert.ID][server.ID].done
+	alertDeliveryMu.Unlock()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("new incident delivery did not finish")
+	}
+}
+
+func TestAlertRecoveryDeliveryStopsAfterSuccessInAlwaysMode(t *testing.T) {
+	oldCache, oldNotifications := Cache, NotificationShared
+	Cache = cache.New(time.Minute, time.Minute)
+	n := &model.Notification{Common: model.Common{ID: 611}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 613}, Name: "recovery-test"}, []uint64{n.ID})
+	NotificationShared = nc
+	t.Cleanup(func() {
+		cancelAlertDeliveries(612)
+		NotificationShared, Cache = oldNotifications, oldCache
+	})
+
+	delivered := make(chan struct{}, 2)
+	nc.sendForTest = func(*model.Notification, string, *model.Server) error {
+		delivered <- struct{}{}
+		return nil
+	}
+	alert := &model.AlertRule{
+		Common:              model.Common{ID: 612},
+		NotificationGroupID: 613,
+		TriggerMode:         model.ModeAlwaysTrigger,
+	}
+	server := &model.Server{Common: model.Common{ID: 614}}
+	startAlertDelivery(alert, server, "recovered", "recovery-event", alertDeliveryRecovery)
+
+	select {
+	case <-delivered:
+	case <-time.After(time.Second):
+		t.Fatal("recovery notification was not delivered")
+	}
+	alertDeliveryMu.Lock()
+	done := alertDeliveries[alert.ID][server.ID].done
+	alertDeliveryMu.Unlock()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("successful recovery delivery kept repeating in always mode")
+	}
+	select {
+	case <-delivered:
+		t.Fatal("recovery notification was delivered more than once")
+	default:
+	}
+}
+
+func TestServerDeleteCancelsAlertDelivery(t *testing.T) {
+	alertDeliveryMu.Lock()
+	oldDeliveries := alertDeliveries
+	cancel := make(chan struct{})
+	alertDeliveries = map[uint64]map[uint64]*alertDeliveryEntry{
+		621: {622: {cancel: cancel, done: make(chan struct{})}},
+	}
+	alertDeliveryMu.Unlock()
+	t.Cleanup(func() {
+		alertDeliveryMu.Lock()
+		alertDeliveries = oldDeliveries
+		alertDeliveryMu.Unlock()
+	})
+
+	servers := &ServerClass{
+		class: class[uint64, *model.Server]{
+			list: map[uint64]*model.Server{
+				622: {Common: model.Common{ID: 622}, UUID: "delete-alert-worker"},
+			},
+		},
+		uuidToID: map[string]uint64{"delete-alert-worker": 622},
+	}
+	servers.Delete([]uint64{622})
+
+	select {
+	case <-cancel:
+	default:
+		t.Fatal("server deletion did not cancel its alert delivery")
+	}
+	alertDeliveryMu.Lock()
+	_, retained := alertDeliveries[621]
+	alertDeliveryMu.Unlock()
+	if retained {
+		t.Fatal("server deletion retained an empty alert delivery map")
+	}
+}
+
+func TestAlwaysAlertContinuesFailureTriggerTasks(t *testing.T) {
+	if !shouldRunAlertFailTasks(model.ModeAlwaysTrigger, false) {
+		t.Fatal("always mode stopped failure trigger tasks after the initial incident")
+	}
+	if shouldRunAlertFailTasks(model.ModeOnetimeTrigger, false) {
+		t.Fatal("one-time mode repeated failure trigger tasks")
+	}
+	if !shouldRunAlertFailTasks(model.ModeOnetimeTrigger, true) {
+		t.Fatal("one-time mode skipped failure trigger tasks for a new incident")
+	}
+}
+
+func TestAlertDeliveryRetriesFailureAndStopsAfterSuccess(t *testing.T) {
+	cancel := make(chan struct{})
+	done := make(chan struct{})
+	attempts := 0
+	go func() {
+		runAlertDelivery(cancel, func() bool {
+			attempts++
+			return attempts == 2
+		}, false, time.Millisecond)
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		close(cancel)
+		t.Fatal("delivery did not retry and finish")
+	}
+	if attempts != 2 {
+		t.Fatalf("got %d attempts, want 2", attempts)
+	}
+}
+
+func TestAlertDeliveryCancelsAfterRecovery(t *testing.T) {
+	cancel := make(chan struct{})
+	done := make(chan struct{})
+	started := make(chan struct{})
+	go func() {
+		runAlertDelivery(cancel, func() bool {
+			close(started)
+			return true
+		}, true, time.Millisecond)
+		close(done)
+	}()
+	<-started
+	close(cancel)
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("recovered incident delivery was not cancelled")
+	}
+}

service/singleton/alertsentinel_test.go

@@ -117,3 +117,53 @@ func TestCheckStatus_SampleMemoryBounded(t *testing.T) {
 		t.Fatalf("sample capacity grew unbounded: peak cap %d exceeds 4x window %d", peakCap, duration*4)
 	}
 }
+
+func TestCheckStatusForServerRecoversUnexpectedEvaluatorPanic(t *testing.T) {
+	originalStore := alertsStore
+	originalPrevState := alertsPrevState
+	originalCycleStore := AlertsCycleTransferStatsStore
+	alertsStore = map[uint64]map[uint64][]model.TimedAlertPoint{}
+	alertsPrevState = map[uint64]map[uint64]uint8{}
+	AlertsCycleTransferStatsStore = map[uint64]*model.CycleTransferStats{}
+	t.Cleanup(func() {
+		alertsStore = originalStore
+		alertsPrevState = originalPrevState
+		AlertsCycleTransferStatsStore = originalCycleStore
+	})
+
+	enabled := true
+	alert := &model.AlertRule{
+		Common: model.Common{ID: 77, UserID: 2},
+		Enable: &enabled,
+		Rules:  []*model.Rule{{Type: "cpu", Duration: 3, Cover: model.RuleCoverAll}},
+	}
+	server := &model.Server{Common: model.Common{ID: 1, UserID: 2}}
+	model.InitServer(server)
+
+	// The deliberately missing alertsStore[alert.ID] map panics at assignment
+	// after evaluation. The per-alert/server recovery boundary must contain it.
+	checkStatusForServer(alert, server)
+}
+
+func TestAddCycleTransferStatsSkipsPoisonedPersistedRule(t *testing.T) {
+	original := AlertsCycleTransferStatsStore
+	AlertsCycleTransferStatsStore = map[uint64]*model.CycleTransferStats{}
+	t.Cleanup(func() { AlertsCycleTransferStatsStore = original })
+
+	enabled := true
+	alert := &model.AlertRule{
+		Common: model.Common{ID: 88},
+		Enable: &enabled,
+		Rules: []*model.Rule{{
+			Type:          "transfer_in_cycle",
+			CycleInterval: 1,
+			Cover:         model.RuleCoverAll,
+			// CycleStart intentionally nil, as it may be in a poisoned legacy DB row.
+		}},
+	}
+
+	addCycleTransferStatsInfo(alert)
+	if _, exists := AlertsCycleTransferStatsStore[alert.ID]; exists {
+		t.Fatal("invalid persisted cycle rule must not be scheduled")
+	}
+}

service/singleton/config.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/nezhahq/nezha/model"
 	"github.com/nezhahq/nezha/pkg/utils"
@@ -14,6 +15,8 @@ var Conf *ConfigClass
 type ConfigClass struct {
 	*model.Config
 
+	onlineHistoryRetentionMu sync.RWMutex
+
 	IgnoredIPNotificationServerIDs map[uint64]bool `json:"ignored_ip_notification_server_ids,omitempty"`
 	Oauth2Providers                []string        `json:"oauth2_providers,omitempty"`
 }
@@ -59,3 +62,39 @@ func (c *ConfigClass) updateIgnoredIPNotificationID() {
 		}
 	}
 }
+
+// OnlineHistoryRetentionDays returns the retention policy visible to queries and pruning.
+func (c *ConfigClass) OnlineHistoryRetentionDays() int {
+	c.onlineHistoryRetentionMu.RLock()
+	defer c.onlineHistoryRetentionMu.RUnlock()
+	return c.OnlineStatsDays
+}
+
+// CommitOnlineHistoryRetention applies a retention change with the settings save.
+func (c *ConfigClass) CommitOnlineHistoryRetention(days *int, persist func() error) error {
+	// Readers must not capture a shorter cutoff while Save can still fail.
+	// Serialize assignment, persistence and rollback with the policy snapshot.
+	c.onlineHistoryRetentionMu.Lock()
+	defer c.onlineHistoryRetentionMu.Unlock()
+	previous := c.OnlineStatsDays
+	if days != nil {
+		c.OnlineStatsDays = *days
+	}
+	if err := persist(); err != nil {
+		c.OnlineStatsDays = previous
+		return err
+	}
+	return nil
+}
+
+// SettingsSnapshot copies response fields without copying the retention mutex.
+func (c *ConfigClass) SettingsSnapshot() model.Setting {
+	c.onlineHistoryRetentionMu.RLock()
+	defer c.onlineHistoryRetentionMu.RUnlock()
+	return model.Setting{
+		ConfigForGuests:                c.ConfigForGuests,
+		ConfigDashboard:                c.ConfigDashboard,
+		IgnoredIPNotificationServerIDs: c.IgnoredIPNotificationServerIDs,
+		Oauth2Providers:                c.Oauth2Providers,
+	}
+}

service/singleton/frontend-templates.yaml

@@ -2,24 +2,20 @@
   name: "OfficialAdmin"
   repository: "https://github.com/nezhahq/admin-frontend"
   author: "nezhahq"
-  version: "v2.3.4"
+  version: "v2.4.0"
   is_admin: true
   is_official: true
 - path: "user-dist"
   name: "Official"
   repository: "https://github.com/hamster1963/nezha-dash-v2"
   author: "hamster1963"
-  version: "v2.4.2"
+  version: "v2.5.1"
   is_official: true
 - path: "nezha-pixel-dist"
   name: "Nezha-Pixel"
   repository: "https://github.com/karllao/nezha-pixel"
   author: "karllao"
   version: "v1.6.0"
-# Third-party user themes consume the opaque Server.PublicNote field. Theme
-# maintainers must validate URL schemes (including after decoding) before using
-# values such as customData.orderLink in href or window.open; the Dashboard
-# backend and admin frontend do not execute those fields.
 - path: "nazhua-dist"
   name: "Nazhua"
   repository: "https://github.com/hi2shark/nazhua"
@@ -29,7 +25,7 @@
   name: "Aobobo"
   repository: "https://github.com/hi2shark/aobobo"
   author: "hi2shark"
-  version: "v1.5.1"
+  version: "v1.5.4"
 - path: "nezha-ascii-dist"
   name: "Nezha-ASCII"
   repository: "https://github.com/hamster1963/nezha-ascii"

service/singleton/notification.go

@@ -58,8 +58,228 @@ type NotificationClass struct {
 	groupToIDList map[uint64]map[uint64]*model.Notification
 	idToGroupList map[uint64]map[uint64]struct{}
 
-	groupList map[uint64]string
-	groupMu   sync.RWMutex
+	groupList       map[uint64]string
+	groupMu         sync.RWMutex
+	deliveryMu      sync.Mutex
+	deliveryKeys    map[string]*notificationKeyLock
+	asyncInFlight   map[string]struct{}
+	asyncRetryAfter map[string]time.Time
+	asyncRetryDelay map[string]time.Duration
+	muteGeneration  map[string]uint64
+	serviceDelivery map[uint64]*orderedServiceDelivery
+	sendForTest     func(*model.Notification, string, *model.Server) error
+}
+
+type serviceDeliveryRequest struct {
+	groupID uint64
+	desc    string
+	label   string
+}
+
+type orderedServiceDelivery struct {
+	current serviceDeliveryRequest
+	pending *serviceDeliveryRequest
+	update  chan struct{}
+	cancel  chan struct{}
+	done    chan struct{}
+}
+
+// SendServiceState preserves incident→recovery order without allowing a
+// slow webhook to accumulate one goroutine per probe result. Only the latest
+// pending phase is retained; a failed current phase retries with backoff.
+func (c *NotificationClass) SendServiceState(serviceID, groupID uint64, desc, label string) {
+	req := serviceDeliveryRequest{groupID: groupID, desc: desc, label: label}
+	c.deliveryMu.Lock()
+	if c.serviceDelivery == nil {
+		c.serviceDelivery = make(map[uint64]*orderedServiceDelivery)
+	}
+	state := c.serviceDelivery[serviceID]
+	if state == nil {
+		state = &orderedServiceDelivery{current: req, update: make(chan struct{}, 1), cancel: make(chan struct{}), done: make(chan struct{})}
+		c.serviceDelivery[serviceID] = state
+		c.deliveryMu.Unlock()
+		go c.runServiceDelivery(serviceID, state)
+		return
+	}
+	if state.pending == nil && state.current.label == label {
+		c.deliveryMu.Unlock()
+		return
+	}
+	if state.pending == nil || state.pending.label != label {
+		state.pending = &req
+		select {
+		case state.update <- struct{}{}:
+		default:
+		}
+	}
+	c.deliveryMu.Unlock()
+}
+
+func (c *NotificationClass) runServiceDelivery(serviceID uint64, state *orderedServiceDelivery) {
+	defer close(state.done)
+	delay := 30 * time.Second
+	for {
+		select {
+		case <-state.cancel:
+			return
+		default:
+		}
+		delivered := c.SendNotification(state.current.groupID, state.current.desc, state.current.label)
+		c.deliveryMu.Lock()
+		// Do not advance to a later phase until every recipient has received the
+		// current one. Otherwise a failed incident can be discarded as soon as a
+		// recovery is queued, leaving that recipient with a recovery but no incident.
+		if delivered && state.pending != nil {
+			next := state.pending
+			state.current = *next
+			state.pending = nil
+			delay = 30 * time.Second
+			select {
+			case <-state.update:
+			default:
+			}
+			c.deliveryMu.Unlock()
+			continue
+		}
+		if delivered {
+			if c.serviceDelivery[serviceID] == state {
+				delete(c.serviceDelivery, serviceID)
+			}
+			c.deliveryMu.Unlock()
+			return
+		}
+		c.deliveryMu.Unlock()
+		timer := time.NewTimer(delay)
+		select {
+		case <-state.cancel:
+			timer.Stop()
+			return
+		case <-state.update:
+			timer.Stop()
+		case <-timer.C:
+			delay = min(delay*2, 15*time.Minute)
+		}
+	}
+}
+
+func (c *NotificationClass) CancelServiceState(serviceID uint64) {
+	if c == nil {
+		return
+	}
+	c.deliveryMu.Lock()
+	if state := c.serviceDelivery[serviceID]; state != nil {
+		close(state.cancel)
+		delete(c.serviceDelivery, serviceID)
+	}
+	c.deliveryMu.Unlock()
+}
+
+// SendNotificationAsync coalesces concurrent service-probe reports for the
+// same event before allocating a webhook goroutine. Alert transitions use the
+// synchronous SendNotification path so they retain their retry semantics.
+func (c *NotificationClass) SendNotificationAsync(groupID uint64, desc, muteLabel string, ext ...*model.Server) {
+	c.sendNotificationAsync(groupID, desc, muteLabel, nil, ext...)
+}
+
+func (c *NotificationClass) sendNotificationAsync(groupID uint64, desc, muteLabel string, onSuccess func(), ext ...*model.Server) {
+	key := fmt.Sprintf("%d:%s", groupID, muteLabel)
+	c.deliveryMu.Lock()
+	if c.asyncInFlight == nil {
+		c.asyncInFlight = make(map[string]struct{})
+	}
+	if _, busy := c.asyncInFlight[key]; busy {
+		c.deliveryMu.Unlock()
+		return
+	}
+	if time.Now().Before(c.asyncRetryAfter[key]) {
+		c.deliveryMu.Unlock()
+		return
+	}
+	c.asyncInFlight[key] = struct{}{}
+	c.deliveryMu.Unlock()
+	go func() {
+		delivered := c.SendNotification(groupID, desc, muteLabel, ext...)
+		if delivered && onSuccess != nil {
+			onSuccess()
+		}
+		c.deliveryMu.Lock()
+		if delivered {
+			delete(c.asyncRetryAfter, key)
+			delete(c.asyncRetryDelay, key)
+		} else {
+			if c.asyncRetryAfter == nil {
+				c.asyncRetryAfter = make(map[string]time.Time)
+				c.asyncRetryDelay = make(map[string]time.Duration)
+			}
+			delay := c.asyncRetryDelay[key]
+			if delay == 0 {
+				delay = 30 * time.Second
+			} else {
+				delay = min(delay*2, 15*time.Minute)
+			}
+			c.asyncRetryDelay[key] = delay
+			c.asyncRetryAfter[key] = time.Now().Add(delay)
+			c.pruneAsyncBackoffLocked()
+		}
+		delete(c.asyncInFlight, key)
+		c.deliveryMu.Unlock()
+	}()
+}
+
+const maxAsyncBackoffKeys = 4096
+
+// Bound memory when a failing webhook sees many short-lived event labels.
+// Evicted old event keys are no longer active after their phase changes.
+func (c *NotificationClass) pruneAsyncBackoffLocked() {
+	if len(c.asyncRetryAfter) <= maxAsyncBackoffKeys {
+		return
+	}
+	now := time.Now()
+	for key, until := range c.asyncRetryAfter {
+		if !now.Before(until) {
+			delete(c.asyncRetryAfter, key)
+			delete(c.asyncRetryDelay, key)
+		}
+	}
+	for key := range c.asyncRetryAfter {
+		if len(c.asyncRetryAfter) <= maxAsyncBackoffKeys {
+			break
+		}
+		delete(c.asyncRetryAfter, key)
+		delete(c.asyncRetryDelay, key)
+	}
+}
+
+type notificationKeyLock struct {
+	mu   sync.Mutex
+	refs int
+}
+
+func (c *NotificationClass) lockDeliveryKey(key string) func() {
+	if key == "" {
+		return func() {}
+	}
+	c.deliveryMu.Lock()
+	if c.deliveryKeys == nil {
+		c.deliveryKeys = make(map[string]*notificationKeyLock)
+	}
+	entry := c.deliveryKeys[key]
+	if entry == nil {
+		entry = new(notificationKeyLock)
+		c.deliveryKeys[key] = entry
+	}
+	entry.refs++
+	c.deliveryMu.Unlock()
+	entry.mu.Lock()
+	return func() {
+		entry.mu.Unlock()
+		c.deliveryMu.Lock()
+		entry.refs--
+		if entry.refs == 0 {
+			delete(c.deliveryKeys, key)
+		}
+		c.deliveryMu.Unlock()
+	}
 }
 
 func NewNotificationClass() *NotificationClass {
@@ -117,20 +337,29 @@ func NewNotificationClass() *NotificationClass {
 }
 
 func (c *NotificationClass) Update(n *model.Notification) {
-	c.listMu.Lock()
+	func() {
+		c.listMu.Lock()
+		defer c.listMu.Unlock()
 
-	_, ok := c.list[n.ID]
-	c.list[n.ID] = n
-
-	if ok {
-		if gids, ok := c.idToGroupList[n.ID]; ok {
-			for gid := range gids {
-				c.groupToIDList[gid][n.ID] = n
-			}
+		_, ok := c.list[n.ID]
+		c.list[n.ID] = n
+		if !ok {
+			return
 		}
-	}
 
-	c.listMu.Unlock()
+		gids := c.idToGroupList[n.ID]
+		for gid := range gids {
+			group, exists := c.groupToIDList[gid]
+			if !exists {
+				delete(gids, gid)
+				continue
+			}
+			group[n.ID] = n
+		}
+		if len(gids) == 0 {
+			delete(c.idToGroupList, n.ID)
+		}
+	}()
 	c.sortList()
 }
 
@@ -138,71 +367,69 @@ func (c *NotificationClass) UpdateGroup(ng *model.NotificationGroup, ngn []uint6
 	c.groupMu.Lock()
 	defer c.groupMu.Unlock()
 
-	_, ok := c.groupList[ng.ID]
 	c.groupList[ng.ID] = ng.Name
 
 	c.listMu.Lock()
 	defer c.listMu.Unlock()
-	if !ok {
-		c.groupToIDList[ng.ID] = make(map[uint64]*model.Notification, len(ngn))
-		for _, n := range ngn {
-			if c.idToGroupList[n] == nil {
-				c.idToGroupList[n] = make(map[uint64]struct{})
-			}
-			c.idToGroupList[n][ng.ID] = struct{}{}
-			c.groupToIDList[ng.ID][n] = c.list[n]
-		}
-	} else {
-		oldList := make(map[uint64]struct{})
-		for nid := range c.groupToIDList[ng.ID] {
-			oldList[nid] = struct{}{}
-		}
 
-		c.groupToIDList[ng.ID] = make(map[uint64]*model.Notification)
-		for _, nid := range ngn {
-			c.groupToIDList[ng.ID][nid] = c.list[nid]
-			if c.idToGroupList[nid] == nil {
-				c.idToGroupList[nid] = make(map[uint64]struct{})
-			}
-			c.idToGroupList[nid][ng.ID] = struct{}{}
+	oldList := c.groupToIDList[ng.ID]
+	newList := make(map[uint64]*model.Notification, len(ngn))
+	for _, nid := range ngn {
+		n, ok := c.list[nid]
+		if !ok {
+			continue
 		}
+		newList[nid] = n
+		if c.idToGroupList[nid] == nil {
+			c.idToGroupList[nid] = make(map[uint64]struct{})
+		}
+		c.idToGroupList[nid][ng.ID] = struct{}{}
+	}
 
-		for oldID := range oldList {
-			if _, ok := c.groupToIDList[ng.ID][oldID]; !ok {
-				delete(c.groupToIDList[oldID], ng.ID)
-				if len(c.idToGroupList[oldID]) == 0 {
-					delete(c.idToGroupList, oldID)
-				}
-			}
+	for oldID := range oldList {
+		if _, ok := newList[oldID]; ok {
+			continue
+		}
+		delete(c.idToGroupList[oldID], ng.ID)
+		if len(c.idToGroupList[oldID]) == 0 {
+			delete(c.idToGroupList, oldID)
 		}
 	}
+	c.groupToIDList[ng.ID] = newList
 }
 
 func (c *NotificationClass) Delete(idList []uint64) {
-	c.listMu.Lock()
+	func() {
+		c.listMu.Lock()
+		defer c.listMu.Unlock()
 
-	for _, id := range idList {
-		delete(c.list, id)
-		// 如果绑定了通知组才删除
-		if gids, ok := c.idToGroupList[id]; ok {
-			for gid := range gids {
-				delete(c.groupToIDList[gid], id)
+		for _, id := range idList {
+			delete(c.list, id)
+			// 如果绑定了通知组才删除
+			if gids, ok := c.idToGroupList[id]; ok {
+				for gid := range gids {
+					delete(c.groupToIDList[gid], id)
+				}
 				delete(c.idToGroupList, id)
 			}
 		}
-	}
-
-	c.listMu.Unlock()
+	}()
 	c.sortList()
 }
 
 func (c *NotificationClass) DeleteGroup(gids []uint64) {
-	c.listMu.Lock()
-	defer c.listMu.Unlock()
 	c.groupMu.Lock()
 	defer c.groupMu.Unlock()
+	c.listMu.Lock()
+	defer c.listMu.Unlock()
 
 	for _, gid := range gids {
+		for nid := range c.groupToIDList[gid] {
+			delete(c.idToGroupList[nid], gid)
+			if len(c.idToGroupList[nid]) == 0 {
+				delete(c.idToGroupList, nid)
+			}
+		}
 		delete(c.groupList, gid)
 		delete(c.groupToIDList, gid)
 	}
@@ -231,66 +458,115 @@ func (c *NotificationClass) sortList() {
 
 func (c *NotificationClass) UnMuteNotification(notificationGroupID uint64, muteLabel string) {
 	fullMuteLabel := NotificationMuteLabel.AppendNotificationGroupName(muteLabel, c.GetGroupName(notificationGroupID))
-	Cache.Delete(fullMuteLabel)
+	c.listMu.RLock()
+	for id := range c.groupToIDList[notificationGroupID] {
+		key := notificationMethodMuteLabel(fullMuteLabel, id)
+		c.deliveryMu.Lock()
+		if c.muteGeneration == nil {
+			c.muteGeneration = make(map[string]uint64)
+		}
+		c.muteGeneration[key]++
+		Cache.Delete(key)
+		c.deliveryMu.Unlock()
+	}
+	c.listMu.RUnlock()
+}
+
+func notificationMethodMuteLabel(label string, id uint64) string {
+	return fmt.Sprintf("%s:method-%d", label, id)
 }
 
 // SendNotification 向指定的通知方式组的所有通知方式发送通知
-func (c *NotificationClass) SendNotification(notificationGroupID uint64, desc string, muteLabel string, ext ...*model.Server) {
+func (c *NotificationClass) SendNotification(notificationGroupID uint64, desc string, muteLabel string, ext ...*model.Server) bool {
+	fullMuteLabel := ""
 	if muteLabel != "" {
-		// 将通知方式组名称加入静音标志
-		muteLabel := NotificationMuteLabel.AppendNotificationGroupName(muteLabel, c.GetGroupName(notificationGroupID))
-		// 通知防骚扰策略
-		var flag bool
-		if cacheN, has := Cache.Get(muteLabel); has {
-			nHistory := cacheN.(NotificationHistory)
-			// 每次提醒都增加一倍等待时间，最后每天最多提醒一次
-			if time.Now().After(nHistory.Until) {
-				flag = true
-				nHistory.Duration *= 2
-				if nHistory.Duration > time.Hour*24 {
-					nHistory.Duration = time.Hour * 24
-				}
-				nHistory.Until = time.Now().Add(nHistory.Duration)
-				// 缓存有效期加 10 分钟
-				Cache.Set(muteLabel, nHistory, nHistory.Duration+time.Minute*10)
-			}
-		} else {
-			// 新提醒直接通知
-			flag = true
-			Cache.Set(muteLabel, NotificationHistory{
-				Duration: firstNotificationDelay,
-				Until:    time.Now().Add(firstNotificationDelay),
-			}, firstNotificationDelay+time.Minute*10)
-		}
-
-		if !flag {
-			if Conf.Debug {
-				log.Println("NEZHA>> Muted repeated notification", desc, muteLabel)
-			}
-			return
-		}
+		fullMuteLabel = NotificationMuteLabel.AppendNotificationGroupName(muteLabel, c.GetGroupName(notificationGroupID))
 	}
-	// 向该通知方式组的所有通知方式发出通知
+	// Copy the group under the lock. Webhook delivery can take minutes and must
+	// not block notification or notification-group updates while it is in flight.
 	c.listMu.RLock()
-	defer c.listMu.RUnlock()
+	notifications := make([]*model.Notification, 0, len(c.groupToIDList[notificationGroupID]))
 	for _, n := range c.groupToIDList[notificationGroupID] {
-		log.Printf("NEZHA>> Try to notify %s", n.Name)
+		notifications = append(notifications, n)
 	}
-	for _, n := range c.groupToIDList[notificationGroupID] {
-		ns := model.NotificationServerBundle{
-			Notification: n,
-			Server:       nil,
-			Loc:          Loc,
-		}
+	c.listMu.RUnlock()
+
+	if len(notifications) == 0 {
+		log.Printf("NEZHA>> Notification group %d has no recipients", notificationGroupID)
+		return false
+	}
+	allDelivered := true
+	for _, n := range notifications {
+		var server *model.Server
 		if len(ext) > 0 {
-			ns.Server = ext[0]
+			server = ext[0]
 		}
-		if err := ns.Send(desc); err != nil {
-			log.Printf("NEZHA>> Sending notification to %s failed: %v", n.Name, err)
-		} else {
-			log.Printf("NEZHA>> Sending notification to %s succeeded", n.Name)
+		if !c.sendRecipient(n, desc, fullMuteLabel, server) {
+			allDelivered = false
 		}
 	}
+	return allDelivered
+}
+
+func (c *NotificationClass) sendRecipient(n *model.Notification, desc, fullMuteLabel string, server *model.Server) bool {
+	if n == nil {
+		return false
+	}
+	cacheKey := ""
+	if fullMuteLabel != "" {
+		cacheKey = notificationMethodMuteLabel(fullMuteLabel, n.ID)
+	}
+	unlock := c.lockDeliveryKey(cacheKey)
+	defer unlock()
+	var generation uint64
+	if cacheKey != "" {
+		c.deliveryMu.Lock()
+		generation = c.muteGeneration[cacheKey]
+		c.deliveryMu.Unlock()
+	}
+	var history NotificationHistory
+	if cacheKey != "" {
+		if value, exists := Cache.Get(cacheKey); exists {
+			history = value.(NotificationHistory)
+			if time.Now().Before(history.Until) {
+				return true // this recipient already received this phase
+			}
+		}
+	}
+	log.Printf("NEZHA>> Try to notify %s", n.Name)
+	var err error
+	if c.sendForTest != nil {
+		err = c.sendForTest(n, desc, server)
+	} else {
+		err = (&model.NotificationServerBundle{Notification: n, Server: server, Loc: Loc}).Send(desc)
+	}
+	if err != nil {
+		// Webhook errors can contain a credential-bearing request URL. Keep
+		// failure type without ever writing the URL/token to dashboard logs.
+		log.Printf("NEZHA>> Sending notification to %s failed (%s)", n.Name, notificationFailureSummary(err))
+		return false
+	}
+	log.Printf("NEZHA>> Sending notification to %s succeeded", n.Name)
+	if cacheKey != "" {
+		if history.Duration == 0 {
+			history.Duration = firstNotificationDelay
+		} else {
+			history.Duration = min(history.Duration*2, 24*time.Hour)
+		}
+		history.Until = time.Now().Add(history.Duration)
+		c.deliveryMu.Lock()
+		if c.muteGeneration[cacheKey] == generation {
+			Cache.Set(cacheKey, history, history.Duration+10*time.Minute)
+		}
+		c.deliveryMu.Unlock()
+	}
+	return true
+}
+
+func notificationFailureSummary(err error) string {
+	// Even wrapped URL errors can include credentials in path/query. Log only
+	// the error type; the HTTP status path already returns a dedicated type.
+	return fmt.Sprintf("%T", err)
 }
 
 func sendSMTPAdminNotification(message string) error {

service/singleton/notification_test.go

@@ -0,0 +1,519 @@
+package singleton
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/patrickmn/go-cache"
+
+	"github.com/nezhahq/nezha/model"
+)
+
+func TestNotificationFailureDoesNotMuteRetryOrResendSuccessfulRecipient(t *testing.T) {
+	oldCache := Cache
+	Cache = cache.New(time.Minute, time.Minute)
+	t.Cleanup(func() { Cache = oldCache })
+	first := &model.Notification{Common: model.Common{ID: 301}, Name: "first"}
+	second := &model.Notification{Common: model.Common{ID: 302}, Name: "second"}
+	nc := newNotificationClassWithItems(first, second)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 303}, Name: "retry-test"}, []uint64{first.ID, second.ID})
+	counts := map[uint64]int{}
+	nc.sendForTest = func(n *model.Notification, _ string, _ *model.Server) error {
+		counts[n.ID]++
+		if n.ID == second.ID && counts[n.ID] == 1 {
+			return errors.New("temporary failure")
+		}
+		return nil
+	}
+	label := "alert-retry-test"
+	nc.UnMuteNotification(303, label)
+	if nc.SendNotification(303, "incident", label) {
+		t.Fatal("partial delivery must report failure")
+	}
+	if !nc.SendNotification(303, "incident", label) {
+		t.Fatal("failed recipient should be retried successfully")
+	}
+	if counts[first.ID] != 1 || counts[second.ID] != 2 {
+		t.Fatalf("delivery attempts = %v, want first=1 second=2", counts)
+	}
+	nc.UnMuteNotification(303, label)
+}
+
+func TestNotificationConcurrentSameRecipientSendsOnce(t *testing.T) {
+	oldCache := Cache
+	Cache = cache.New(time.Minute, time.Minute)
+	t.Cleanup(func() { Cache = oldCache })
+	n := &model.Notification{Common: model.Common{ID: 401}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 402}, Name: "concurrent-test"}, []uint64{n.ID})
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	var calls atomic.Int32
+	nc.sendForTest = func(*model.Notification, string, *model.Server) error {
+		if calls.Add(1) == 1 {
+			close(entered)
+			<-release
+		}
+		return nil
+	}
+	done := make(chan bool, 2)
+	go func() { done <- nc.SendNotification(402, "incident", "same-event") }()
+	<-entered
+	go func() { done <- nc.SendNotification(402, "incident", "same-event") }()
+	close(release)
+	if !<-done || !<-done {
+		t.Fatal("concurrent delivery returned failure")
+	}
+	if calls.Load() != 1 {
+		t.Fatalf("same event delivered %d times, want 1", calls.Load())
+	}
+}
+
+func TestNotificationAsyncCoalescesBlockedServiceReports(t *testing.T) {
+	oldCache := Cache
+	Cache = cache.New(time.Minute, time.Minute)
+	t.Cleanup(func() { Cache = oldCache })
+	n := &model.Notification{Common: model.Common{ID: 501}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 502}, Name: "service-test"}, []uint64{n.ID})
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	var calls atomic.Int32
+	nc.sendForTest = func(*model.Notification, string, *model.Server) error {
+		if calls.Add(1) == 1 {
+			close(entered)
+			<-release
+		}
+		return nil
+	}
+	nc.SendNotificationAsync(502, "down", "service-down")
+	<-entered
+	for i := 0; i < 1000; i++ {
+		nc.SendNotificationAsync(502, "down", "service-down")
+	}
+	nc.deliveryMu.Lock()
+	inFlight := len(nc.asyncInFlight)
+	nc.deliveryMu.Unlock()
+	if inFlight != 1 || calls.Load() != 1 {
+		t.Fatalf("in-flight sends=%d attempts=%d, want one each", inFlight, calls.Load())
+	}
+	close(release)
+	deadline := time.After(time.Second)
+	for {
+		nc.deliveryMu.Lock()
+		remaining := len(nc.asyncInFlight)
+		nc.deliveryMu.Unlock()
+		if remaining == 0 {
+			break
+		}
+		select {
+		case <-deadline:
+			t.Fatal("async notification did not finish")
+		case <-time.After(time.Millisecond):
+		}
+	}
+}
+
+func TestNotificationFailureSummaryOmitsCredentialURL(t *testing.T) {
+	err := &url.Error{Op: "Post", URL: "https://example.test/bot-secret-token/send?key=secret-query", Err: errors.New("timeout")}
+	if summary := notificationFailureSummary(err); strings.Contains(summary, "secret") || strings.Contains(summary, "example.test") {
+		t.Fatalf("credential-bearing URL leaked: %s", summary)
+	}
+}
+
+func TestNotificationAsyncBacksOffFastFailures(t *testing.T) {
+	oldCache := Cache
+	Cache = cache.New(time.Minute, time.Minute)
+	t.Cleanup(func() { Cache = oldCache })
+	n := &model.Notification{Common: model.Common{ID: 511}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 512}, Name: "backoff-test"}, []uint64{n.ID})
+	var calls atomic.Int32
+	nc.sendForTest = func(*model.Notification, string, *model.Server) error {
+		calls.Add(1)
+		return errors.New("fast failure")
+	}
+	key := "512:service-down"
+	nc.SendNotificationAsync(512, "down", "service-down")
+	deadline := time.After(time.Second)
+	for {
+		nc.deliveryMu.Lock()
+		busy := len(nc.asyncInFlight) != 0
+		after := nc.asyncRetryAfter[key]
+		nc.deliveryMu.Unlock()
+		if !busy && !after.IsZero() {
+			break
+		}
+		select {
+		case <-deadline:
+			t.Fatal("first failed send did not record backoff")
+		case <-time.After(time.Millisecond):
+		}
+	}
+	for i := 0; i < 1000; i++ {
+		nc.SendNotificationAsync(512, "down", "service-down")
+	}
+	if calls.Load() != 1 {
+		t.Fatalf("fast failures caused %d attempts before retry-after", calls.Load())
+	}
+}
+
+func TestUnMuteDuringInflightSendCannotRemuteRecovery(t *testing.T) {
+	oldCache := Cache
+	Cache = cache.New(time.Minute, time.Minute)
+	t.Cleanup(func() { Cache = oldCache })
+	n := &model.Notification{Common: model.Common{ID: 521}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 522}, Name: "generation-test"}, []uint64{n.ID})
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	var calls atomic.Int32
+	nc.sendForTest = func(*model.Notification, string, *model.Server) error {
+		if calls.Add(1) == 1 {
+			close(entered)
+			<-release
+		}
+		return nil
+	}
+	done := make(chan bool, 1)
+	go func() { done <- nc.SendNotification(522, "first failure", "static-label") }()
+	<-entered
+	nc.UnMuteNotification(522, "static-label")
+	close(release)
+	if !<-done {
+		t.Fatal("first send failed unexpectedly")
+	}
+	if !nc.SendNotification(522, "next failure", "static-label") || calls.Load() != 2 {
+		t.Fatalf("old in-flight send re-muted next failure: calls=%d", calls.Load())
+	}
+}
+
+func TestAsyncBackoffMemoryIsBounded(t *testing.T) {
+	nc := &NotificationClass{
+		asyncRetryAfter: make(map[string]time.Time),
+		asyncRetryDelay: make(map[string]time.Duration),
+	}
+	for i := 0; i < maxAsyncBackoffKeys+100; i++ {
+		key := fmt.Sprintf("old-event-%d", i)
+		nc.asyncRetryAfter[key] = time.Now().Add(time.Minute)
+		nc.asyncRetryDelay[key] = time.Minute
+	}
+	nc.pruneAsyncBackoffLocked()
+	if len(nc.asyncRetryAfter) > maxAsyncBackoffKeys || len(nc.asyncRetryDelay) != len(nc.asyncRetryAfter) {
+		t.Fatalf("unbounded backoff maps: after=%d delay=%d", len(nc.asyncRetryAfter), len(nc.asyncRetryDelay))
+	}
+}
+
+func TestServiceStateNotificationsKeepIncidentRecoveryOrder(t *testing.T) {
+	oldCache := Cache
+	Cache = cache.New(time.Minute, time.Minute)
+	t.Cleanup(func() { Cache = oldCache })
+	n := &model.Notification{Common: model.Common{ID: 531}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 532}, Name: "ordered-test"}, []uint64{n.ID})
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	delivered := make(chan string, 3)
+	nc.sendForTest = func(_ *model.Notification, desc string, _ *model.Server) error {
+		if desc == "down" {
+			close(entered)
+			<-release
+		}
+		delivered <- desc
+		return nil
+	}
+	nc.SendServiceState(533, 532, "down", "state-down-1")
+	<-entered
+	nc.SendServiceState(533, 532, "good", "state-good-2")
+	select {
+	case message := <-delivered:
+		t.Fatalf("service transition overtook blocked incident: %s", message)
+	default:
+	}
+	close(release)
+	for _, want := range []string{"down", "good"} {
+		select {
+		case got := <-delivered:
+			if got != want {
+				t.Fatalf("delivery order got %s, want %s", got, want)
+			}
+		case <-time.After(time.Second):
+			t.Fatalf("missing %s delivery", want)
+		}
+	}
+	deadline := time.After(time.Second)
+	for {
+		nc.deliveryMu.Lock()
+		remaining := len(nc.serviceDelivery)
+		nc.deliveryMu.Unlock()
+		if remaining == 0 {
+			break
+		}
+		select {
+		case <-deadline:
+			t.Fatal("ordered delivery worker did not exit")
+		case <-time.After(time.Millisecond):
+		}
+	}
+}
+
+func TestServiceStateRetriesFailedIncidentBeforePendingRecovery(t *testing.T) {
+	oldCache := Cache
+	Cache = cache.New(time.Minute, time.Minute)
+	t.Cleanup(func() { Cache = oldCache })
+	n := &model.Notification{Common: model.Common{ID: 535}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 536}, Name: "retry-order-test"}, []uint64{n.ID})
+	t.Cleanup(func() { nc.CancelServiceState(537) })
+
+	attempts := make(chan string, 3)
+	releaseFirst := make(chan struct{})
+	var downAttempts atomic.Int32
+	nc.sendForTest = func(_ *model.Notification, desc string, _ *model.Server) error {
+		attempts <- desc
+		if desc == "down" && downAttempts.Add(1) == 1 {
+			<-releaseFirst
+			return errors.New("temporary incident failure")
+		}
+		return nil
+	}
+
+	nc.SendServiceState(537, 536, "down", "state-down-1")
+	nc.deliveryMu.Lock()
+	state := nc.serviceDelivery[537]
+	nc.deliveryMu.Unlock()
+	select {
+	case got := <-attempts:
+		if got != "down" {
+			t.Fatalf("first delivery = %q, want down", got)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("incident delivery did not start")
+	}
+	nc.SendServiceState(537, 536, "good", "state-good-2")
+	close(releaseFirst)
+
+	for _, want := range []string{"down", "good"} {
+		select {
+		case got := <-attempts:
+			if got != want {
+				t.Fatalf("delivery after failed incident = %q, want %q", got, want)
+			}
+		case <-time.After(time.Second):
+			t.Fatalf("missing %s delivery", want)
+		}
+	}
+	select {
+	case <-state.done:
+	case <-time.After(time.Second):
+		t.Fatal("ordered delivery worker did not finish")
+	}
+}
+
+func TestServiceStateCancelStopsOldConfiguration(t *testing.T) {
+	oldCache := Cache
+	Cache = cache.New(time.Minute, time.Minute)
+	t.Cleanup(func() { Cache = oldCache })
+	n := &model.Notification{Common: model.Common{ID: 541}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 542}, Name: "old-group"}, []uint64{n.ID})
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	var calls atomic.Int32
+	nc.sendForTest = func(*model.Notification, string, *model.Server) error {
+		if calls.Add(1) == 1 {
+			close(entered)
+			<-release
+		}
+		return errors.New("old webhook failed")
+	}
+	nc.SendServiceState(543, 542, "down", "old-event")
+	<-entered
+	nc.deliveryMu.Lock()
+	state := nc.serviceDelivery[543]
+	nc.deliveryMu.Unlock()
+	nc.CancelServiceState(543)
+	nc.deliveryMu.Lock()
+	remaining := len(nc.serviceDelivery)
+	nc.deliveryMu.Unlock()
+	if remaining != 0 {
+		t.Fatal("old service delivery worker remained registered after cancellation")
+	}
+	close(release)
+	select {
+	case <-state.done:
+	case <-time.After(time.Second):
+		t.Fatal("canceled old delivery worker did not exit")
+	}
+}
+
+func TestServiceUpdateCancelsOldRecipientRetry(t *testing.T) {
+	ss := newServiceMonitorSecurityHarness(t)
+	service := &model.Service{
+		Common:              model.Common{ID: 551, UserID: 1},
+		Name:                "generic-probe",
+		Type:                model.TaskTypeTCPPing,
+		Target:              "example.invalid:443",
+		Duration:            3600,
+		Notify:              true,
+		NotificationGroupID: 552,
+		Cover:               model.ServiceCoverIgnoreAll,
+	}
+	addServiceMonitorSecurityService(t, ss, service)
+	n := &model.Notification{Common: model.Common{ID: 553}, Name: "old-hook"}
+	NotificationShared.InsertForTest(n)
+	NotificationShared.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 552}, Name: "old-group"}, []uint64{n.ID})
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	NotificationShared.sendForTest = func(*model.Notification, string, *model.Server) error {
+		close(entered)
+		<-release
+		return errors.New("failed old recipient")
+	}
+	NotificationShared.SendServiceState(service.ID, 552, "old down", "old-state")
+	<-entered
+	NotificationShared.deliveryMu.Lock()
+	state := NotificationShared.serviceDelivery[service.ID]
+	NotificationShared.deliveryMu.Unlock()
+	updated := *service
+	updated.Notify = false
+	updated.NotificationGroupID = 554
+	if err := ss.Update(&updated); err != nil {
+		t.Fatal(err)
+	}
+	close(release)
+	select {
+	case <-state.done:
+	case <-time.After(time.Second):
+		t.Fatal("old recipient retry did not stop after service update")
+	}
+	NotificationShared.deliveryMu.Lock()
+	remaining := len(NotificationShared.serviceDelivery)
+	NotificationShared.deliveryMu.Unlock()
+	if remaining != 0 {
+		t.Fatal("old recipient retained after service update")
+	}
+}
+
+func newNotificationClassWithItems(items ...*model.Notification) *NotificationClass {
+	nc := NewEmptyNotificationClassForTest()
+	for _, item := range items {
+		nc.InsertForTest(item)
+	}
+	return nc
+}
+
+func TestNotificationClassDeleteGroupCleansReverseIndex(t *testing.T) {
+	n := &model.Notification{Common: model.Common{ID: 1, UserID: 2}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	group := &model.NotificationGroup{Common: model.Common{ID: 10, UserID: 2}, Name: "group"}
+	nc.UpdateGroup(group, []uint64{n.ID})
+
+	nc.DeleteGroup([]uint64{group.ID})
+
+	if _, ok := nc.groupToIDList[group.ID]; ok {
+		t.Fatalf("forward index still contains deleted group %d", group.ID)
+	}
+	if groups, ok := nc.idToGroupList[n.ID]; ok {
+		if _, stale := groups[group.ID]; stale {
+			t.Fatalf("reverse index for notification %d still contains deleted group %d", n.ID, group.ID)
+		}
+	}
+}
+
+func TestNotificationClassUpdateGroupCleansRemovedMembership(t *testing.T) {
+	first := &model.Notification{Common: model.Common{ID: 1, UserID: 2}, Name: "first"}
+	second := &model.Notification{Common: model.Common{ID: 2, UserID: 2}, Name: "second"}
+	nc := newNotificationClassWithItems(first, second)
+	group := &model.NotificationGroup{Common: model.Common{ID: 10, UserID: 2}, Name: "group"}
+	nc.UpdateGroup(group, []uint64{first.ID, second.ID})
+
+	nc.UpdateGroup(group, []uint64{second.ID})
+
+	if groups, ok := nc.idToGroupList[first.ID]; ok {
+		if _, stale := groups[group.ID]; stale {
+			t.Fatalf("reverse index for notification %d still contains group %d", first.ID, group.ID)
+		}
+	}
+	if _, ok := nc.idToGroupList[second.ID][group.ID]; !ok {
+		t.Fatalf("reverse index lost retained membership of notification %d in group %d", second.ID, group.ID)
+	}
+}
+
+func TestNotificationClassUpdateRepairsOrphanedReverseIndex(t *testing.T) {
+	n := &model.Notification{Common: model.Common{ID: 1, UserID: 2}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.idToGroupList[n.ID] = map[uint64]struct{}{99: {}}
+
+	n.Name = "updated"
+	nc.Update(n)
+
+	if groups, ok := nc.idToGroupList[n.ID]; ok && len(groups) != 0 {
+		t.Fatalf("orphaned reverse memberships were not removed: %v", groups)
+	}
+}
+
+func TestNotificationClassGroupMutationsDoNotDeadlock(t *testing.T) {
+	nc := NewEmptyNotificationClassForTest()
+	nc.listMu.RLock()
+	readLockHeld := true
+	defer func() {
+		if readLockHeld {
+			nc.listMu.RUnlock()
+		}
+	}()
+
+	deleteDone := make(chan struct{})
+	go func() {
+		nc.DeleteGroup([]uint64{10})
+		close(deleteDone)
+	}()
+
+	deadline := time.Now().Add(2 * time.Second)
+	for nc.listMu.TryRLock() {
+		nc.listMu.RUnlock()
+		if time.Now().After(deadline) {
+			t.Fatal("DeleteGroup did not queue for listMu")
+		}
+		time.Sleep(time.Millisecond)
+	}
+
+	updateDone := make(chan struct{})
+	go func() {
+		nc.UpdateGroup(&model.NotificationGroup{
+			Common: model.Common{ID: 10, UserID: 2},
+			Name:   "group",
+		}, nil)
+		close(updateDone)
+	}()
+
+	deadline = time.Now().Add(2 * time.Second)
+	for nc.groupMu.TryLock() {
+		nc.groupMu.Unlock()
+		if time.Now().After(deadline) {
+			t.Fatal("neither group mutation acquired groupMu")
+		}
+		time.Sleep(time.Millisecond)
+	}
+
+	nc.listMu.RUnlock()
+	readLockHeld = false
+
+	timer := time.NewTimer(2 * time.Second)
+	defer timer.Stop()
+	for deleteDone != nil || updateDone != nil {
+		select {
+		case <-deleteDone:
+			deleteDone = nil
+		case <-updateDone:
+			updateDone = nil
+		case <-timer.C:
+			t.Fatal("concurrent UpdateGroup and DeleteGroup deadlocked")
+		}
+	}
+}

service/singleton/online_history.go

@@ -0,0 +1,348 @@
+package singleton
+
+import (
+	"fmt"
+	"log"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/nezhahq/nezha/model"
+)
+
+// OnlineHistoryRecorder 维护每台服务器当前进行中的在线会话。会话的打开与
+// 关闭由 agent 状态流的建立/断开驱动（service/rpc），并由定时清扫兜底关闭
+// 失联（未正常断流）的会话；dashboard 重启时通过数据库中 Ongoing 会话恢复。
+type OnlineHistoryRecorder struct {
+	mu       sync.Mutex
+	restored bool
+	open     map[uint64]*onlineSessionRuntime
+	// closing 是关库失败后的待关闭状态：断线边界已被观察到，会话 ID 与
+	// 结束时间在此固定、不可被后续上报覆盖；重试只为旧会话完成持久化，
+	// 重连不得把它续接成活跃会话（同服务器可能同时有多条待关闭会话）。
+	closing map[uint64][]*onlineSessionClosing
+	// Preserve the first disconnect while startup reads are unavailable.
+	recoveryDisconnects map[uint64]time.Time
+}
+
+type onlineSessionRuntime struct {
+	id           uint64
+	lastObserved time.Time
+	revision     uint64
+	// lastPersist 是 LastActiveAt 最近一次落库时间，用于节流写库。
+	lastPersist time.Time
+}
+
+type onlineSessionClosing struct {
+	id uint64
+	// end 是已观察到的会话结束时刻，重试持久化时原样使用。
+	end time.Time
+}
+
+var OnlineHistoryShared *OnlineHistoryRecorder
+
+func NewOnlineHistoryRecorder() *OnlineHistoryRecorder {
+	r := &OnlineHistoryRecorder{
+		open:                make(map[uint64]*onlineSessionRuntime),
+		closing:             make(map[uint64][]*onlineSessionClosing),
+		recoveryDisconnects: make(map[uint64]time.Time),
+	}
+	r.mu.Lock()
+	r.restoreOngoingSessions()
+	r.mu.Unlock()
+	return r
+}
+
+// restoreOngoingSessions 每台服务器仅允许最新且仍在在线窗口内的会话续接，
+// 其余会话按最后持久化的活跃时间结转。崩溃会丢失内存中的待关闭边界，
+// 恢复时采用 LastActiveAt 保守结转，不推测未持久化的断线时间。
+func (r *OnlineHistoryRecorder) restoreOngoingSessions() bool {
+	if DB == nil {
+		return false
+	}
+	var ongoing []model.OnlineSession
+	if err := DB.Where("ongoing = ?", true).
+		Order("server_id ASC").Order("started_at DESC").Order("id DESC").Find(&ongoing).Error; err != nil {
+		log.Printf("NEZHA>> Failed to load ongoing online sessions: %v", err)
+		return false
+	}
+	now := time.Now()
+	seenServers := make(map[uint64]bool)
+	for _, s := range ongoing {
+		// 关闭失败后重连可留下同服务器多条 Ongoing 行；直接赋值 open 会覆盖
+		// 旧 ID，导致其永久脱管。按开始时间（同时间按 ID）选最新行，其余
+		// 必须关闭或进入 closing 重试，即使最新行本身已经过期也不能复活旧行。
+		duplicate := seenServers[s.ServerID]
+		seenServers[s.ServerID] = true
+		disconnect, observedDisconnect := r.recoveryDisconnects[s.ServerID]
+		observedDisconnect = observedDisconnect && !duplicate && !disconnect.Before(s.StartedAt)
+		if observedDisconnect || duplicate || s.LastActiveAt.Before(now.Add(-model.OnlineActiveWindow)) {
+			end := s.LastActiveAt
+			if observedDisconnect && disconnect.After(end) {
+				end = disconnect
+			}
+			if end.Before(s.StartedAt) {
+				end = s.StartedAt
+			}
+			if err := closeOnlineSessionRow(s.ID, end); err != nil {
+				log.Printf("NEZHA>> Failed to close recovered online session %d: %v", s.ID, err)
+				// 关库失败不能跳过，否则会留下永不闭合、也无人重试的
+				// Ongoing 行；进入待关闭状态交由清扫周期重试。过期或
+				// 被更新会话替代的行均不得被重连续接为活跃会话
+				r.closing[s.ServerID] = append(r.closing[s.ServerID],
+					&onlineSessionClosing{id: s.ID, end: end})
+			}
+			continue
+		}
+		r.open[s.ServerID] = &onlineSessionRuntime{id: s.ID, lastPersist: s.LastActiveAt, lastObserved: s.LastActiveAt}
+	}
+	clear(r.recoveryDisconnects)
+	r.restored = true
+	return true
+}
+
+// flushClosing 尽力重试关闭该服务器的待关闭会话：按入列时固定的结束时间
+// 持久化，成功才移除；失败的条目原样保留，结束时间不得被后续上报改写。
+func (r *OnlineHistoryRecorder) flushClosing(serverID uint64) {
+	pendings, ok := r.closing[serverID]
+	if !ok {
+		return
+	}
+	kept := pendings[:0]
+	for _, pc := range pendings {
+		if err := closeOnlineSessionRow(pc.id, pc.end); err != nil {
+			log.Printf("NEZHA>> Failed to retry closing online session %d: %v", pc.id, err)
+			kept = append(kept, pc)
+		}
+	}
+	if len(kept) == 0 {
+		delete(r.closing, serverID)
+	} else {
+		r.closing[serverID] = kept
+	}
+}
+
+// RecordOnline 记录一次状态上报：无进行中会话时开启新会话（重启遗留且仍在
+// 窗口内的 Ongoing 会话直接续接）；已有会话时节流落库 LastActiveAt。
+func (r *OnlineHistoryRecorder) RecordOnline(serverID uint64, at time.Time) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	// A failed startup read is not an empty history: retry before accepting reports.
+	if !r.restored && !r.restoreOngoingSessions() {
+		return
+	}
+	at = at.UTC()
+	r.flushClosing(serverID)
+
+	// A report can arrive after expiry but before the next sweep. Close at the
+	// previous observation before overwriting it, preserving failed-close retries.
+	if rt := r.open[serverID]; rt != nil && at.Sub(rt.lastObserved) > model.OnlineActiveWindow {
+		r.recordOfflineLocked(serverID, rt.lastObserved)
+	}
+	if rt, ok := r.open[serverID]; ok {
+		rt.lastObserved = at
+		rt.revision++
+		if at.Sub(rt.lastPersist) >= model.OnlineActiveWindow {
+			if err := DB.Model(&model.OnlineSession{}).Where("id = ?", rt.id).Update("last_active_at", at).Error; err != nil {
+				log.Printf("NEZHA>> Failed to persist online session last_active_at: %v", err)
+			} else {
+				// Only committed writes advance the throttle; failures retry on the next report.
+				rt.lastPersist = at
+			}
+		}
+		return
+	}
+
+	// All ongoing rows are owned by open or closing after successful recovery.
+
+	s := model.OnlineSession{ServerID: serverID, StartedAt: at, LastActiveAt: at, Ongoing: true}
+	if err := DB.Create(&s).Error; err != nil {
+		log.Printf("NEZHA>> Failed to create online session: %v", err)
+		return
+	}
+	r.open[serverID] = &onlineSessionRuntime{id: s.ID, lastPersist: at, lastObserved: at}
+}
+
+// RecordOffline 关闭该服务器当前进行中的会话。at 是会话结束时刻：正常断流
+// 为当前时间，清扫/结转为最后一次已知活跃时间。
+// 关库失败时把会话转入待关闭状态而非保留为活跃条目：会话 ID 与已观察到的
+// 结束时刻（at 与 lastPersist 的较大者）就此固定，后续上报不得覆盖；重试
+// 只为旧会话完成持久化，重连则开启新会话——断线后的离线间隔不得计作在线。
+func (r *OnlineHistoryRecorder) RecordOffline(serverID uint64, at time.Time) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	// Disconnect may be the first event after startup storage recovers.
+	if !r.restored && !r.restoreOngoingSessions() {
+		// Retrying SELECT alone loses confirmed offline gaps on a quick reconnect.
+		// Keep the first boundary; subsequent disconnected streams cannot extend it.
+		if r.recoveryDisconnects == nil {
+			r.recoveryDisconnects = make(map[uint64]time.Time)
+		}
+		if _, exists := r.recoveryDisconnects[serverID]; !exists {
+			r.recoveryDisconnects[serverID] = at.UTC()
+		}
+		return
+	}
+	r.recordOfflineLocked(serverID, at)
+}
+
+func (r *OnlineHistoryRecorder) recordOfflineLocked(serverID uint64, at time.Time) {
+	rt, ok := r.open[serverID]
+	if !ok {
+		return
+	}
+	end := at
+	if end.Before(rt.lastPersist) {
+		end = rt.lastPersist
+	}
+	if err := closeOnlineSessionRow(rt.id, end); err != nil {
+		log.Printf("NEZHA>> Failed to close online session %d: %v", rt.id, err)
+		// 结束时刻已观察到，不能因写库失败丢失，也不能把行继续当活跃
+		// 会话：转待关闭状态，由 RecordOnline 冲洗与清扫周期重试
+		r.closing[serverID] = append(r.closing[serverID], &onlineSessionClosing{id: rt.id, end: end})
+		delete(r.open, serverID)
+		return
+	}
+	delete(r.open, serverID)
+}
+
+// SweepStale 关闭超过在线窗口仍无状态上报的进行中会话。lastActiveFn 返回
+// 服务器的运行时最后活跃时间与是否存在。
+func (r *OnlineHistoryRecorder) SweepStale(now time.Time, lastActiveFn func(serverID uint64) (time.Time, bool)) {
+	// Never call the runtime observer while holding mu: reports acquire the
+	// runtime lease before this mutex. Snapshot identities, then revalidate them.
+	r.mu.Lock()
+	if !r.restored && !r.restoreOngoingSessions() {
+		r.mu.Unlock()
+		return
+	}
+	candidates := make(map[uint64]onlineSessionRuntime, len(r.open))
+	for id, rt := range r.open {
+		candidates[id] = *rt
+	}
+	for id := range r.closing {
+		r.flushClosing(id)
+	}
+	r.mu.Unlock()
+	for id, candidate := range candidates {
+		lastActive, ok := lastActiveFn(id)
+		r.mu.Lock()
+		rt := r.open[id]
+		if rt != nil && rt.id == candidate.id && rt.revision == candidate.revision {
+			if lastActive.Before(rt.lastObserved) {
+				lastActive = rt.lastObserved
+			}
+			if (!ok || lastActive.Before(now.Add(-model.OnlineActiveWindow))) && rt.lastObserved.Before(now.Add(-model.OnlineActiveWindow)) {
+				r.recordOfflineLocked(id, lastActive)
+			}
+		}
+		r.mu.Unlock()
+	}
+
+}
+
+// SweepOnlineSessions 是定时清扫入口，接上服务器运行时的最后活跃时间。
+func SweepOnlineSessions() {
+	if OnlineHistoryShared == nil || ServerShared == nil || DB == nil {
+		return
+	}
+	OnlineHistoryShared.SweepStale(time.Now(), onlineServerLastActive)
+}
+
+func onlineServerLastActive(serverID uint64) (time.Time, bool) {
+	server, ok := ServerShared.Get(serverID)
+	if !ok || server == nil {
+		return time.Time{}, false
+	}
+	lastActive := server.RuntimeSnapshot().LastActive
+	return lastActive, !lastActive.IsZero()
+}
+
+// PruneOnlineSessions 清理超出统计窗口的已结束会话与已删除服务器的残留记录。
+// 时间列直接比较（与写入时的绑定格式同构）以走 ended_at 索引，并用分批
+// DELETE 替换单条巨型删除，避免大表上长时间占用 sqlite 写锁、阻塞上报路径。
+func PruneOnlineSessions() {
+	if DB == nil {
+		return
+	}
+	keepDays := Conf.OnlineHistoryRetentionDays()
+	if keepDays < model.MinOnlineStatsDays || keepDays > model.MaxOnlineStatsDays {
+		keepDays = model.DefaultOnlineStatsDays
+	}
+	cutoff := model.OnlineHistoryWindowStart(time.Now(), keepDays, Loc)
+	pruneOnlineSessionsBatch("ongoing = ? AND ended_at < ?", false, cutoff)
+	pruneOnlineSessionsBatch("server_id NOT IN (SELECT `id` FROM servers)")
+}
+
+// pruneOnlineSessionsBatch 按主键分批删除匹配的行，每批之间释放写锁。
+func pruneOnlineSessionsBatch(where string, args ...any) {
+	const batchSize = 10000
+	for {
+		var ids []uint64
+		if err := DB.Model(&model.OnlineSession{}).Unscoped().
+			Where(where, args...).Limit(batchSize).Pluck("id", &ids).Error; err != nil {
+			log.Printf("NEZHA>> Failed to select online sessions for pruning: %v", err)
+			return
+		}
+		if len(ids) == 0 {
+			return
+		}
+		if err := DB.Unscoped().Delete(&model.OnlineSession{}, ids).Error; err != nil {
+			log.Printf("NEZHA>> Failed to prune online sessions: %v", err)
+			return
+		}
+		if len(ids) < batchSize {
+			return
+		}
+	}
+}
+
+func closeOnlineSessionRow(id uint64, end time.Time) error {
+	return DB.Model(&model.OnlineSession{}).Where("id = ?", id).Updates(map[string]any{
+		"ended_at": end.UTC(),
+		"ongoing":  false,
+	}).Error
+}
+
+// Sessions snapshots rows and volatile close boundaries under the same lock.
+// Otherwise a failed close can count confirmed offline time as ongoing, or a
+// successful retry can remove the boundary after the reader fetched a stale row.
+func (r *OnlineHistoryRecorder) Sessions(serverID uint64, start, end time.Time) ([]model.OnlineSession, error) {
+	if r != nil {
+		r.mu.Lock()
+		defer r.mu.Unlock()
+		if !r.restored && !r.restoreOngoingSessions() {
+			return nil, fmt.Errorf("online history recovery unavailable")
+		}
+	}
+	var sessions []model.OnlineSession
+	// Separate branches use (server_id, ongoing, ended_at) and
+	// (server_id, ongoing, started_at), avoiding a scan of retained closed rows.
+	for _, ongoing := range []bool{false, true} {
+		var rows []model.OnlineSession
+		q := DB.Where("server_id = ? AND ongoing = ?", serverID, ongoing)
+		if ongoing {
+			q = q.Where("started_at < ?", end.UTC())
+		} else {
+			q = q.Where("ended_at > ?", start.UTC())
+		}
+		if err := q.Find(&rows).Error; err != nil {
+			return nil, err
+		}
+		sessions = append(sessions, rows...)
+	}
+	if r != nil {
+		endings := make(map[uint64]time.Time)
+		for _, pending := range r.closing[serverID] {
+			endings[pending.id] = pending.end
+		}
+		for i := range sessions {
+			if end, ok := endings[sessions[i].ID]; ok {
+				sessions[i].Ongoing = false
+				sessions[i].EndedAt = end
+			}
+		}
+	}
+	sort.Slice(sessions, func(i, j int) bool { return sessions[i].StartedAt.Before(sessions[j].StartedAt) })
+	return sessions, nil
+}

service/singleton/online_history_failure_test.go

@@ -0,0 +1,224 @@
+package singleton
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/nezhahq/nezha/model"
+	"github.com/stretchr/testify/require"
+	"gorm.io/gorm"
+)
+
+func TestOnlineHistoryRetriesFailedHeartbeatImmediately(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	r := NewOnlineHistoryRecorder()
+	start := time.Now().UTC().Truncate(time.Second)
+	r.RecordOnline(1, start)
+	id := r.open[1].id
+	require.NoError(t, DB.Exec(`CREATE TRIGGER reject_heartbeat BEFORE UPDATE OF last_active_at ON online_sessions BEGIN SELECT RAISE(ABORT, 'heartbeat failure'); END`).Error)
+	r.RecordOnline(1, start.Add(30*time.Second))
+	require.NoError(t, DB.Exec("DROP TRIGGER reject_heartbeat").Error)
+	r.RecordOnline(1, start.Add(31*time.Second))
+	require.True(t, getOnlineSession(t, id).LastActiveAt.Equal(start.Add(31*time.Second)))
+}
+
+func TestOnlineHistoryRetriesStartupReadBeforeNewReport(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	start := time.Now().UTC().Add(-time.Hour).Truncate(time.Second)
+	old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true}
+	require.NoError(t, DB.Create(&old).Error)
+	require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("reject_history_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) }))
+	r := NewOnlineHistoryRecorder()
+	require.NoError(t, DB.Callback().Query().Remove("reject_history_restore"))
+	r.RecordOnline(1, time.Now())
+	stored := getOnlineSession(t, old.ID)
+	require.False(t, stored.Ongoing)
+	require.True(t, stored.EndedAt.Equal(start))
+	require.NotEqual(t, old.ID, r.open[1].id)
+	require.Equal(t, int64(1), countOnlineSessions(t, "ongoing = ?", true))
+}
+
+func TestOnlineHistorySweepKeepsSessionReconnectedDuringObservation(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	r := NewOnlineHistoryRecorder()
+	start := time.Now().Add(-time.Hour)
+	r.RecordOnline(1, start)
+	observed := make(chan struct{})
+	release := make(chan struct{})
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		r.SweepStale(time.Now(), func(uint64) (time.Time, bool) { close(observed); <-release; return start, true })
+	}()
+	<-observed
+	reconnected := make(chan struct{})
+	go func() { r.RecordOffline(1, start.Add(time.Second)); r.RecordOnline(1, time.Now()); close(reconnected) }()
+	// The observer must not hold the recorder lock: RPC updates take the runtime
+	// lock first and then enter the recorder. Holding both in reverse order deadlocks.
+	select {
+	case <-reconnected:
+	case <-time.After(time.Second):
+		close(release)
+		<-done
+		<-reconnected
+		t.Fatal("sweep holds recorder lock while consulting runtime state")
+	}
+	id := r.open[1].id
+	close(release)
+	<-done
+	require.Contains(t, r.open, uint64(1))
+	require.Equal(t, id, r.open[1].id)
+	require.True(t, getOnlineSession(t, id).Ongoing)
+}
+
+func TestOnlineHistoryQueryNormalizesOffsetsAndUsesWindowIndexes(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	east := time.FixedZone("UTC+8", 8*3600)
+	start := time.Date(2025, 1, 1, 12, 0, 0, 0, east)
+	r := NewOnlineHistoryRecorder()
+	r.RecordOnline(1, start)
+	r.RecordOffline(1, start.Add(time.Hour))
+	rows, err := r.Sessions(1, start.UTC().Add(-time.Minute), start.UTC().Add(2*time.Hour))
+	require.NoError(t, err)
+	require.Len(t, rows, 1)
+	require.True(t, rows[0].StartedAt.Equal(start))
+	require.True(t, rows[0].EndedAt.Equal(start.Add(time.Hour)))
+	for _, test := range []struct {
+		query, index string
+		args         []any
+	}{
+		{"EXPLAIN QUERY PLAN SELECT * FROM online_sessions WHERE server_id = ? AND ongoing = ? AND ended_at > ?", "idx_online_sessions_server_closed", []any{1, false, start.UTC()}},
+		{"EXPLAIN QUERY PLAN SELECT * FROM online_sessions WHERE server_id = ? AND ongoing = ? AND started_at < ?", "idx_online_sessions_server_open", []any{1, true, start.Add(2 * time.Hour).UTC()}},
+	} {
+		var plan []struct{ Detail string }
+		require.NoError(t, DB.Raw(test.query, test.args...).Scan(&plan).Error)
+		require.NotEmpty(t, plan)
+		require.Contains(t, plan[0].Detail, test.index)
+	}
+}
+
+func TestOnlineHistorySweepKeepsReportReceivedDuringObservation(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	r := NewOnlineHistoryRecorder()
+	start := time.Now().Add(-time.Hour)
+	r.RecordOnline(1, start)
+	id := r.open[1].id
+	r.SweepStale(time.Now(), func(uint64) (time.Time, bool) {
+		r.RecordOnline(1, time.Now())
+		return start, true
+	})
+	require.Contains(t, r.open, uint64(1))
+	// A report after an hour starts a new session; the stale sweep must not close it.
+	require.NotEqual(t, id, r.open[1].id)
+	require.False(t, getOnlineSession(t, id).Ongoing)
+	require.True(t, getOnlineSession(t, r.open[1].id).Ongoing)
+}
+
+func TestOnlineHistoryStartupReadFailureBlocksQueryAndSweepRetries(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	start := time.Now().Add(-time.Hour)
+	old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true}
+	require.NoError(t, DB.Create(&old).Error)
+	require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("reject_history_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) }))
+	r := NewOnlineHistoryRecorder()
+	_, err := r.Sessions(1, start, time.Now())
+	require.ErrorContains(t, err, "recovery unavailable")
+	r.RecordOnline(1, time.Now())
+	require.Empty(t, r.open)
+	require.NoError(t, DB.Callback().Query().Remove("reject_history_restore"))
+	r.SweepStale(time.Now(), func(uint64) (time.Time, bool) { return time.Time{}, false })
+	require.False(t, getOnlineSession(t, old.ID).Ongoing)
+	require.Equal(t, int64(0), countOnlineSessions(t, "ongoing = ?", true))
+}
+
+func TestOnlineHistoryDisconnectRetriesStartupRead(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	start := time.Now().Add(-10 * time.Second)
+	old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true}
+	require.NoError(t, DB.Create(&old).Error)
+	require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("reject_history_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) }))
+	r := NewOnlineHistoryRecorder()
+	require.NoError(t, DB.Callback().Query().Remove("reject_history_restore"))
+	end := start.Add(time.Second)
+	r.RecordOffline(1, end)
+	require.False(t, getOnlineSession(t, old.ID).Ongoing)
+	require.True(t, getOnlineSession(t, old.ID).EndedAt.Equal(end))
+	r.RecordOnline(1, time.Now())
+	require.NotEqual(t, old.ID, r.open[1].id)
+}
+
+func TestOnlineHistoryReportAfterTimeoutSplitsBeforeSweep(t *testing.T) {
+	for _, failClose := range []bool{false, true} {
+		t.Run(map[bool]string{false: "persisted", true: "retry_close"}[failClose], func(t *testing.T) {
+			setupOnlineHistoryTestDB(t)
+			r := NewOnlineHistoryRecorder()
+			start := time.Now().UTC().Truncate(time.Second)
+			r.RecordOnline(1, start)
+			r.RecordOnline(1, start.Add(time.Second))
+			first := r.open[1].id
+			r.SweepStale(start.Add(30*time.Second), func(uint64) (time.Time, bool) { return start.Add(time.Second), true })
+			require.Equal(t, first, r.open[1].id)
+			if failClose {
+				injectCloseUpdateFailure(t)
+			}
+			r.RecordOnline(1, start.Add(45*time.Second))
+			require.NotEqual(t, first, r.open[1].id)
+			second := r.open[1].id
+			rows, err := r.Sessions(1, start, start.Add(46*time.Second))
+			require.NoError(t, err)
+			require.Len(t, rows, 2)
+			require.False(t, rows[0].Ongoing)
+			require.True(t, rows[0].EndedAt.Equal(start.Add(time.Second)))
+			require.True(t, rows[1].StartedAt.Equal(start.Add(45*time.Second)))
+			if failClose {
+				require.Len(t, r.closing[1], 1)
+				dropCloseUpdateFailure(t)
+			}
+			r.SweepStale(start.Add(46*time.Second), func(uint64) (time.Time, bool) { return start.Add(45 * time.Second), true })
+			require.Empty(t, r.closing)
+			require.False(t, getOnlineSession(t, first).Ongoing)
+			require.True(t, getOnlineSession(t, first).EndedAt.Equal(start.Add(time.Second)))
+			require.True(t, getOnlineSession(t, second).Ongoing)
+		})
+	}
+}
+
+func TestOnlineHistoryDisconnectSurvivesUnavailableRecovery(t *testing.T) {
+	for _, failClose := range []bool{false, true} {
+		t.Run(map[bool]string{false: "close_success", true: "close_retry"}[failClose], func(t *testing.T) {
+			setupOnlineHistoryTestDB(t)
+			start := time.Now().UTC().Add(-10 * time.Second)
+			old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true}
+			require.NoError(t, DB.Create(&old).Error)
+			require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("unavailable_disconnect_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) }))
+			r := NewOnlineHistoryRecorder()
+			end := start.Add(time.Second)
+			r.RecordOffline(1, end)
+			// Later disconnects while recovery is unavailable must not extend the first session.
+			r.RecordOffline(1, end.Add(time.Second))
+			require.NoError(t, DB.Callback().Query().Remove("unavailable_disconnect_restore"))
+			if failClose {
+				injectCloseUpdateFailure(t)
+			}
+			reconnect := start.Add(5 * time.Second)
+			r.RecordOnline(1, reconnect)
+			require.NotEqual(t, old.ID, r.open[1].id)
+			rows, err := r.Sessions(1, start, reconnect.Add(time.Second))
+			require.NoError(t, err)
+			require.Len(t, rows, 2)
+			require.False(t, rows[0].Ongoing)
+			require.True(t, rows[0].EndedAt.Equal(end))
+			require.True(t, rows[1].StartedAt.Equal(reconnect))
+			if failClose {
+				require.Len(t, r.closing[1], 1)
+				dropCloseUpdateFailure(t)
+			}
+			r.SweepStale(reconnect.Add(time.Second), func(uint64) (time.Time, bool) { return reconnect, true })
+			require.Empty(t, r.closing)
+			require.False(t, getOnlineSession(t, old.ID).Ongoing)
+			require.True(t, getOnlineSession(t, old.ID).EndedAt.Equal(end))
+			require.True(t, getOnlineSession(t, r.open[1].id).Ongoing)
+		})
+	}
+}

service/singleton/online_history_restore_test.go

@@ -0,0 +1,73 @@
+package singleton
+
+import (
+	"testing"
+	"time"
+
+	"github.com/nezhahq/nezha/model"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRestoreOngoingSessionsReconcilesRecentDuplicates(t *testing.T) {
+	for _, failClose := range []bool{false, true} {
+		name := "successful_close"
+		if failClose {
+			name = "retry_failed_close"
+		}
+		t.Run(name, func(t *testing.T) {
+			setupOnlineHistoryTestDB(t)
+			base := time.Now().Add(-10 * time.Second).Truncate(time.Second)
+			// Insert out of chronological order: choosing by ID alone would keep the wrong session.
+			newest := model.OnlineSession{ServerID: 1, StartedAt: base.Add(4 * time.Second), LastActiveAt: base.Add(5 * time.Second), Ongoing: true}
+			oldest := model.OnlineSession{ServerID: 1, StartedAt: base, LastActiveAt: base.Add(time.Second), Ongoing: true}
+			middle := model.OnlineSession{ServerID: 1, StartedAt: base.Add(2 * time.Second), LastActiveAt: base.Add(3 * time.Second), Ongoing: true}
+			for _, session := range []*model.OnlineSession{&newest, &oldest, &middle} {
+				require.NoError(t, DB.Create(session).Error)
+			}
+			if failClose {
+				injectCloseUpdateFailure(t)
+			}
+			r := NewOnlineHistoryRecorder()
+			require.Len(t, r.open, 1)
+			require.Equal(t, newest.ID, r.open[1].id)
+			if failClose {
+				require.Len(t, r.closing[1], 2)
+				require.Equal(t, int64(3), countOnlineSessions(t, "ongoing = ?", true))
+				dropCloseUpdateFailure(t)
+			}
+			r.SweepStale(base.Add(6*time.Second), func(uint64) (time.Time, bool) { return newest.LastActiveAt, true })
+			require.Empty(t, r.closing)
+			require.True(t, getOnlineSession(t, newest.ID).Ongoing)
+			for _, previous := range []model.OnlineSession{oldest, middle} {
+				stored := getOnlineSession(t, previous.ID)
+				require.False(t, stored.Ongoing)
+				require.True(t, stored.EndedAt.Equal(previous.LastActiveAt))
+			}
+			r.RecordOnline(1, base.Add(7*time.Second))
+			require.Equal(t, newest.ID, r.open[1].id)
+			require.Equal(t, int64(3), countOnlineSessions(t, "server_id = ?", 1))
+			r.SweepStale(base.Add(2*model.OnlineActiveWindow), func(uint64) (time.Time, bool) { return time.Time{}, false })
+			require.Empty(t, r.open)
+			require.Empty(t, r.closing)
+			require.Equal(t, int64(0), countOnlineSessions(t, "ongoing = ?", true))
+		})
+	}
+}
+
+func TestRestoreOngoingSessionsUsesIDToBreakStartTimeTies(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	start := time.Now().Add(-5 * time.Second)
+	first := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true}
+	second := first
+	other := model.OnlineSession{ServerID: 2, StartedAt: start, LastActiveAt: start, Ongoing: true}
+	for _, session := range []*model.OnlineSession{&first, &second, &other} {
+		require.NoError(t, DB.Create(session).Error)
+	}
+	r := NewOnlineHistoryRecorder()
+	require.Len(t, r.open, 2)
+	require.Equal(t, second.ID, r.open[1].id)
+	require.Equal(t, other.ID, r.open[2].id)
+	require.False(t, getOnlineSession(t, first.ID).Ongoing)
+	require.True(t, getOnlineSession(t, first.ID).EndedAt.Equal(start))
+	require.Equal(t, int64(2), countOnlineSessions(t, "ongoing = ?", true))
+}

service/singleton/online_history_retention_test.go

@@ -0,0 +1,63 @@
+package singleton
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/nezhahq/nezha/model"
+	"github.com/stretchr/testify/require"
+)
+
+func TestOnlineHistoryPruneWaitsForRetentionCommit(t *testing.T) {
+	for _, saveFails := range []bool{true, false} {
+		t.Run(map[bool]string{true: "failed_save", false: "successful_save"}[saveFails], func(t *testing.T) {
+			setupOnlineHistoryTestDB(t)
+			previous := Conf
+			Conf = &ConfigClass{Config: &model.Config{}}
+			Conf.OnlineStatsDays = 30
+			t.Cleanup(func() { Conf = previous })
+			now := time.Now().UTC()
+			row := model.OnlineSession{ServerID: 1, StartedAt: now.AddDate(0, 0, -15), EndedAt: now.AddDate(0, 0, -14)}
+			require.NoError(t, DB.Create(&row).Error)
+			saveEntered := make(chan struct{})
+			releaseSave := make(chan struct{})
+			saved := make(chan error, 1)
+			days := 7
+			go func() {
+				saved <- Conf.CommitOnlineHistoryRetention(&days, func() error {
+					close(saveEntered)
+					<-releaseSave
+					if saveFails {
+						return errors.New("retention save unavailable")
+					}
+					return nil
+				})
+			}()
+			<-saveEntered
+			pruneStarted := make(chan struct{})
+			pruned := make(chan struct{})
+			go func() { close(pruneStarted); PruneOnlineSessions(); close(pruned) }()
+			<-pruneStarted
+			premature := false
+			select {
+			case <-pruned:
+				premature = true
+			case <-time.After(100 * time.Millisecond):
+			}
+			close(releaseSave)
+			err := <-saved
+			<-pruned
+			require.False(t, premature, "pruning must not read an uncommitted retention policy")
+			if saveFails {
+				require.ErrorContains(t, err, "retention save unavailable")
+				require.Equal(t, 30, Conf.OnlineHistoryRetentionDays())
+				require.Equal(t, int64(1), countOnlineSessions(t, "id = ?", row.ID))
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, 7, Conf.OnlineHistoryRetentionDays())
+				require.Equal(t, int64(0), countOnlineSessions(t, "id = ?", row.ID))
+			}
+		})
+	}
+}

service/singleton/online_history_test.go

@@ -0,0 +1,316 @@
+package singleton
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"gorm.io/gorm"
+
+	"github.com/nezhahq/nezha/model"
+)
+
+func setupOnlineHistoryTestDB(t *testing.T) {
+	t.Helper()
+
+	previousDB := DB
+	var err error
+	DB, err = gorm.Open(openSQLiteDialector(filepath.Join(t.TempDir(), "dashboard.sqlite")), &gorm.Config{})
+	require.NoError(t, err)
+	sqlDB, err := DB.DB()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		DB = previousDB
+		if err := sqlDB.Close(); err != nil {
+			t.Errorf("close online history test database: %v", err)
+		}
+	})
+
+	require.NoError(t, DB.AutoMigrate(&model.Server{}, &model.OnlineSession{}))
+	require.NoError(t, DB.Exec("INSERT INTO servers (id, name, uuid) VALUES (1, 'server-1', 'online-history-test-1')").Error)
+}
+
+func getOnlineSession(t *testing.T, id uint64) model.OnlineSession {
+	t.Helper()
+	var s model.OnlineSession
+	require.NoError(t, DB.First(&s, id).Error)
+	return s
+}
+
+func countOnlineSessions(t *testing.T, cond string, args ...any) int64 {
+	t.Helper()
+	var count int64
+	require.NoError(t, DB.Model(&model.OnlineSession{}).Where(cond, args...).Count(&count).Error)
+	return count
+}
+
+// injectCloseUpdateFailure 用 SQLite 触发器注入"SELECT 正常、仅关闭 UPDATE 失败"：
+// 只拦截把 ongoing 置 0 的更新，LastActiveAt 节流写与 INSERT 均不受影响，
+// 使后续 RecordOnline/恢复/清扫全部走真实执行路径。
+func injectCloseUpdateFailure(t *testing.T) {
+	t.Helper()
+	require.NoError(t, DB.Exec(`CREATE TRIGGER fail_online_session_close
+BEFORE UPDATE ON online_sessions
+WHEN NEW.ongoing = 0
+BEGIN
+    SELECT RAISE(ABORT, 'injected close failure');
+END`).Error)
+	t.Cleanup(func() { dropCloseUpdateFailure(t) })
+}
+
+func dropCloseUpdateFailure(t *testing.T) {
+	t.Helper()
+	require.NoError(t, DB.Exec("DROP TRIGGER IF EXISTS fail_online_session_close").Error)
+}
+
+// 评审回归（P2：关库失败丢边界/重连续接待关闭会话）：RecordOffline 关库失败时
+// 必须转入待关闭状态——会话 ID 与已观察到的断线时刻固定、后续上报不得覆盖；
+// 重连开启的是新会话而非续接旧行；失败恢复后由清扫周期按固定边界完成持久化，
+// 离线间隔不计入在线。时间自单一基准单调推进。
+func TestRecordOfflineCloseFailurePreservesBoundary(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	r := NewOnlineHistoryRecorder()
+
+	base := time.Now().Add(-time.Hour).Truncate(time.Second)
+	atStart := base
+	atLastActive := atStart.Add(model.OnlineActiveWindow) // 恰好达到节流窗口，触发落库
+	atOffline := atLastActive.Add(15 * time.Second)       // 观察到的断线时刻
+	atReconnect := atOffline.Add(10 * time.Minute)
+
+	r.RecordOnline(1, atStart)
+	r.RecordOnline(1, atLastActive)
+	require.Len(t, r.open, 1)
+	first := getOnlineSession(t, r.open[1].id)
+	require.WithinDuration(t, atLastActive, first.LastActiveAt, time.Second)
+
+	// 注入关闭失败：行保持 Ongoing 等待重试，会话转入待关闭而非继续当活跃会话
+	injectCloseUpdateFailure(t)
+	r.RecordOffline(1, atOffline)
+	require.NotContains(t, r.open, uint64(1))
+	stuck := getOnlineSession(t, first.ID)
+	require.True(t, stuck.Ongoing)
+	require.True(t, stuck.EndedAt.IsZero(), "关库失败时旧行必须保持未闭合等待重试")
+
+	// 失败未恢复时 agent 重连：旧行不得被续接，必须开启新会话
+	r.RecordOnline(1, atReconnect)
+	require.Contains(t, r.open, uint64(1))
+	require.NotEqual(t, first.ID, r.open[1].id, "待关闭会话不得被续接为活跃会话")
+	afterRetry := getOnlineSession(t, first.ID)
+	require.True(t, afterRetry.Ongoing, "失败未恢复时重试仍失败，旧行保持待关闭")
+	second := getOnlineSession(t, r.open[1].id)
+	require.True(t, second.Ongoing)
+	require.WithinDuration(t, atReconnect, second.StartedAt, time.Second)
+
+	// 失败恢复后清扫：旧行按观察到的断线时刻闭合，新会话仍活跃不受影响
+	dropCloseUpdateFailure(t)
+	r.SweepStale(atReconnect.Add(time.Second), func(uint64) (time.Time, bool) {
+		return atReconnect, true
+	})
+
+	closed := getOnlineSession(t, first.ID)
+	require.False(t, closed.Ongoing)
+	require.WithinDuration(t, atOffline, closed.EndedAt, time.Second, "旧行必须按观察到的断线时刻闭合")
+	require.WithinDuration(t, atStart, closed.StartedAt, time.Second)
+	stillOngoing := getOnlineSession(t, second.ID)
+	require.True(t, stillOngoing.Ongoing)
+
+	// 离线间隔不计入在线：两行在 [atStart, atReconnect) 内的在线合计止于断线时刻
+	var rows []model.OnlineSession
+	require.NoError(t, DB.Where("server_id = ?", 1).Order("id").Find(&rows).Error)
+	require.Len(t, rows, 2)
+	var online time.Duration
+	for _, s := range rows {
+		end := s.EndedAt
+		if s.Ongoing {
+			end = atReconnect
+		}
+		online += end.Sub(s.StartedAt)
+	}
+	require.Equal(t, atOffline.Sub(atStart), online, "离线间隔不得计作在线")
+}
+
+// 评审回归（P2 启动恢复路径）：重启恢复对超窗会话结转关库失败时，条目必须
+// 进入待关闭状态（经真实 NewOnlineHistoryRecorder 恢复路径验证，而非手工
+// 构造内存状态）；重连不得续接该行，清扫周期按固定边界时刻完成持久化。
+func TestRestoreStaleCloseFailureEntersPendingClose(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+
+	// 预置一条超窗 Ongoing 行（dashboard 停机前遗留），时间自单一基准单调推进
+	base := time.Now().Add(-3 * time.Hour).Truncate(time.Second)
+	staleAt := base.Add(time.Hour)
+	atReconnect := base.Add(2 * time.Hour)
+	legacy := model.OnlineSession{ServerID: 1, StartedAt: base, LastActiveAt: staleAt, Ongoing: true}
+	require.NoError(t, DB.Create(&legacy).Error)
+
+	injectCloseUpdateFailure(t)
+	r := NewOnlineHistoryRecorder()
+
+	stuck := getOnlineSession(t, legacy.ID)
+	require.True(t, stuck.Ongoing, "恢复结转关库失败时行必须保持 Ongoing 等待重试")
+
+	// 重连：待关闭的超窗会话不得被续接，开启新会话
+	r.RecordOnline(1, atReconnect)
+	require.Contains(t, r.open, uint64(1))
+	require.NotEqual(t, legacy.ID, r.open[1].id, "待关闭会话不得被续接为活跃会话")
+
+	dropCloseUpdateFailure(t)
+	r.SweepStale(atReconnect.Add(time.Second), func(uint64) (time.Time, bool) {
+		return atReconnect, true
+	})
+
+	closed := getOnlineSession(t, legacy.ID)
+	require.False(t, closed.Ongoing)
+	require.WithinDuration(t, staleAt, closed.EndedAt, time.Second, "旧行必须按恢复时观察到的边界闭合")
+	require.Equal(t, int64(2), countOnlineSessions(t, "server_id = ?", 1))
+	require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
+}
+
+func TestRecordOnlineOpensSingleSession(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	r := NewOnlineHistoryRecorder()
+
+	now := time.Now()
+	r.RecordOnline(1, now)
+	r.RecordOnline(1, now.Add(time.Second))
+
+	require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ?", 1))
+	require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
+}
+
+func TestRecordOnlinePersistsLastActiveThrottled(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	r := NewOnlineHistoryRecorder()
+
+	now := time.Now()
+	r.RecordOnline(1, now)
+	// 窗口内的重复上报不应触发落库
+	r.RecordOnline(1, now.Add(5*time.Second))
+	s := r.open[1]
+	stored := getOnlineSession(t, s.id)
+	require.WithinDuration(t, now, stored.LastActiveAt, time.Second)
+
+	// 超过窗口后落库
+	r.RecordOnline(1, now.Add(model.OnlineActiveWindow+time.Second))
+	stored = getOnlineSession(t, s.id)
+	require.WithinDuration(t, now.Add(model.OnlineActiveWindow+time.Second), stored.LastActiveAt, time.Second)
+}
+
+func TestRecordOfflineClosesSession(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	r := NewOnlineHistoryRecorder()
+
+	start := time.Now()
+	r.RecordOnline(1, start)
+	r.RecordOffline(1, start.Add(10*time.Minute))
+	// 重复关闭应为无操作
+	r.RecordOffline(1, start.Add(11*time.Minute))
+
+	require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ?", 1))
+	require.Equal(t, int64(0), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
+	var s model.OnlineSession
+	require.NoError(t, DB.First(&s).Error)
+	require.WithinDuration(t, start.Add(10*time.Minute), s.EndedAt, time.Second)
+	require.WithinDuration(t, start, s.StartedAt, time.Second)
+}
+
+func TestRecordOnlineContinuesSessionAfterRestart(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+
+	// 模拟重启前遗留的进行中会话（最后活跃时间在在线窗口内：dashboard 只是短暂重启）
+	now := time.Now()
+	legacy := model.OnlineSession{ServerID: 1, StartedAt: now.Add(-time.Minute), LastActiveAt: now.Add(-5 * time.Second), Ongoing: true}
+	require.NoError(t, DB.Create(&legacy).Error)
+
+	// 新 recorder（构造时从库恢复）：新近会话应保留并续接，不产生新会话
+	r := NewOnlineHistoryRecorder()
+	r.RecordOnline(1, now)
+
+	require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ?", 1))
+	continued := getOnlineSession(t, legacy.ID)
+	require.True(t, continued.Ongoing)
+	require.WithinDuration(t, now.Add(-time.Minute), continued.StartedAt, time.Second)
+}
+
+func TestRestoreOngoingSessionsClosesStaleSessions(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+
+	// 上次活跃时间远早于在线窗口：重启恢复时按最后已知活跃时刻结转
+	staleStart := time.Now().Add(-time.Hour)
+	stale := model.OnlineSession{ServerID: 1, StartedAt: staleStart, LastActiveAt: staleStart.Add(30 * time.Minute), Ongoing: true}
+	require.NoError(t, DB.Create(&stale).Error)
+
+	r := NewOnlineHistoryRecorder()
+	r.RecordOnline(1, time.Now())
+
+	closed := getOnlineSession(t, stale.ID)
+	require.False(t, closed.Ongoing)
+	require.WithinDuration(t, staleStart.Add(30*time.Minute), closed.EndedAt, time.Second)
+	// 结转后新的上报开启新会话
+	require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
+}
+
+func TestSweepStaleClosesOnlyStaleSessions(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	r := NewOnlineHistoryRecorder()
+
+	now := time.Now()
+	r.RecordOnline(1, now.Add(-time.Hour))
+	// 第二台服务器：会话在 DB 但没有 servers 记录（服务器已被删除），手工放进内存 map
+	require.NoError(t, DB.Exec("INSERT INTO online_sessions (server_id, started_at, last_active_at, ongoing, created_at, updated_at) VALUES (2, ?, ?, 1, ?, ?)", now.Add(-time.Hour), now.Add(-time.Hour), now, now).Error)
+	var second model.OnlineSession
+	require.NoError(t, DB.Where("server_id = ?", 2).First(&second).Error)
+	r.mu.Lock()
+	r.open[2] = &onlineSessionRuntime{id: second.ID, lastPersist: now.Add(-time.Hour)}
+	r.mu.Unlock()
+
+	// 服务器 1 活跃，服务器 2 不存在 → 仅关闭服务器 2 的会话
+	lastActive := func(serverID uint64) (time.Time, bool) {
+		if serverID == 1 {
+			return now, true
+		}
+		return time.Time{}, false
+	}
+	r.SweepStale(now, lastActive)
+
+	require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
+	require.Equal(t, int64(0), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 2, true))
+
+	// 服务器 1 失联后也会被关闭，结束时刻为最后已知活跃时间
+	r.SweepStale(now.Add(model.OnlineActiveWindow*2), func(uint64) (time.Time, bool) {
+		return now, true
+	})
+	require.Equal(t, int64(0), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
+	var closedFirst = getOnlineSession(t, func() uint64 {
+		var s model.OnlineSession
+		require.NoError(t, DB.Where("server_id = ?", 1).First(&s).Error)
+		return s.ID
+	}())
+	require.WithinDuration(t, now, closedFirst.EndedAt, time.Second)
+}
+
+func TestPruneOnlineSessions(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	previousConf := Conf
+	Conf = &ConfigClass{Config: &model.Config{}}
+	Conf.OnlineStatsDays = 7
+	t.Cleanup(func() { Conf = previousConf })
+
+	now := time.Now()
+	old := now.AddDate(0, 0, -30)
+	rows := []model.OnlineSession{
+		{ServerID: 1, StartedAt: old, EndedAt: old.Add(time.Hour), LastActiveAt: old.Add(time.Hour)},   // 超窗口 → 清理
+		{ServerID: 1, StartedAt: old, EndedAt: now.Add(-time.Hour), LastActiveAt: now.Add(-time.Hour)}, // 跨窗口边界 → 保留
+		{ServerID: 1, StartedAt: now.Add(-time.Hour), EndedAt: now, LastActiveAt: now},                 // 窗口内 → 保留
+		{ServerID: 1, StartedAt: now, Ongoing: true, LastActiveAt: now},                                // 进行中 → 保留
+		{ServerID: 99, StartedAt: now.Add(-time.Hour), EndedAt: now, LastActiveAt: now},                // 服务器已删除 → 清理
+	}
+	for i := range rows {
+		require.NoError(t, DB.Create(&rows[i]).Error)
+	}
+
+	PruneOnlineSessions()
+
+	require.Equal(t, int64(3), countOnlineSessions(t, "1 = 1"))
+	require.Equal(t, int64(1), countOnlineSessions(t, "ongoing = ?", true))
+}

service/singleton/server.go

@@ -117,6 +117,7 @@ func (c *ServerClass) Update(s *model.Server, uuid string) {
 func (c *ServerClass) Delete(idList []uint64) {
 	c.lockLifecycleWrite()
 	defer c.unlockLifecycleWrite()
+	cancelAlertDeliveriesForServers(idList)
 
 	c.listMu.Lock()
 

service/singleton/server_transfer.go

@@ -152,6 +152,8 @@ type ServerTransferClass struct {
 
 	subMu     sync.Mutex
 	subs      map[uint64]chan *model.ServerTransfer
+	subOwners map[uint64]uint64
+	subCounts map[uint64]int
 	nextSubID uint64
 
 	timeout  time.Duration
@@ -174,6 +176,18 @@ var ErrServerAlreadyTransferring = errors.New("server already has an in-flight t
 // deferred to OnAgentReconnect / PushIfOnline.
 var ErrAgentTooOldForTransfer = fmt.Errorf("agent build older than %s does not support server transfer (TaskTypeServerTransferApply)", MinServerTransferAgentVersion)
 
+const (
+	// A normal dashboard needs one subscription per open transfer page. Keep a
+	// small per-user allowance for tabs/devices plus a process-wide hard stop.
+	maxServerTransferSubscribersPerUser = 8
+	maxServerTransferSubscribersGlobal  = 1024
+)
+
+var (
+	errTooManyServerTransferSubscribersForUser = errors.New("too many concurrent server transfer subscribers for this user")
+	errTooManyServerTransferSubscribersGlobal  = errors.New("too many concurrent server transfer subscribers")
+)
+
 // agentSupportsTransfer reports whether s has reported a build version >=
 // MinServerTransferAgentVersion. Returns true when version is unknown (agent
 // never reported) so callers can defer the decision; PushIfOnline re-checks
@@ -210,6 +224,8 @@ func NewServerTransferClass() *ServerTransferClass {
 		verifiedHandshakes:     make(map[uint64]string),
 		initiating:             make(map[uint64]bool),
 		subs:                   make(map[uint64]chan *model.ServerTransfer),
+		subOwners:              make(map[uint64]uint64),
+		subCounts:              make(map[uint64]int),
 		timeout:                defaultServerTransferTimeout,
 		stopCh:                 make(chan struct{}),
 	}
@@ -1410,20 +1426,37 @@ func (c *ServerTransferClass) sweepTimeouts() {
 // Subscribe registers a channel that will receive every transfer transition
 // event from this point forward. The caller MUST Unsubscribe when done or
 // the broker will block forever if the channel is unbuffered or full.
-func (c *ServerTransferClass) Subscribe() (uint64, <-chan *model.ServerTransfer) {
+func (c *ServerTransferClass) Subscribe(userID uint64) (uint64, <-chan *model.ServerTransfer, error) {
 	c.subMu.Lock()
 	defer c.subMu.Unlock()
+	if len(c.subs) >= maxServerTransferSubscribersGlobal {
+		return 0, nil, errTooManyServerTransferSubscribersGlobal
+	}
+	if c.subCounts[userID] >= maxServerTransferSubscribersPerUser {
+		return 0, nil, errTooManyServerTransferSubscribersForUser
+	}
 
 	id := atomic.AddUint64(&c.nextSubID, 1)
 	ch := make(chan *model.ServerTransfer, 16)
 	c.subs[id] = ch
-	return id, ch
+	c.subOwners[id] = userID
+	c.subCounts[userID]++
+	return id, ch, nil
 }
 
 func (c *ServerTransferClass) Unsubscribe(id uint64) {
 	c.subMu.Lock()
 	ch, ok := c.subs[id]
+	ownerID, ownerOK := c.subOwners[id]
 	delete(c.subs, id)
+	delete(c.subOwners, id)
+	if ownerOK {
+		if c.subCounts[ownerID] <= 1 {
+			delete(c.subCounts, ownerID)
+		} else {
+			c.subCounts[ownerID]--
+		}
+	}
 	c.subMu.Unlock()
 	if ok {
 		close(ch)

service/singleton/server_transfer_test.go

@@ -455,7 +455,8 @@ func TestServerTransferBroadcastReachesSubscribers(t *testing.T) {
 	defer cleanup()
 	seedServerForTransfer(t, 1, 100)
 
-	id, ch := c.Subscribe()
+	id, ch, err := c.Subscribe(100)
+	require.NoError(t, err)
 	defer c.Unsubscribe(id)
 
 	tr := initiateAndRegister(t, c, 1, 100, 200, 1)
@@ -469,7 +470,7 @@ func TestServerTransferBroadcastReachesSubscribers(t *testing.T) {
 		t.Fatal("expected Pending broadcast within 1s")
 	}
 
-	_, _, err := markPendingVerified(t, c, 1)
+	_, _, err = markPendingVerified(t, c, 1)
 	require.NoError(t, err)
 	select {
 	case ev := <-ch:
@@ -479,6 +480,57 @@ func TestServerTransferBroadcastReachesSubscribers(t *testing.T) {
 	}
 }
 
+func TestServerTransferSubscribeEnforcesPerUserLimitAndReleasesQuota(t *testing.T) {
+	c, cleanup := setupTransferFixture(t)
+	defer cleanup()
+
+	ids := make([]uint64, 0, maxServerTransferSubscribersPerUser)
+	for range maxServerTransferSubscribersPerUser {
+		id, ch, err := c.Subscribe(100)
+		require.NoError(t, err)
+		require.NotNil(t, ch)
+		ids = append(ids, id)
+	}
+
+	_, _, err := c.Subscribe(100)
+	require.ErrorIs(t, err, errTooManyServerTransferSubscribersForUser)
+
+	otherID, _, err := c.Subscribe(200)
+	require.NoError(t, err, "one user's quota must not block another user")
+	c.Unsubscribe(otherID)
+
+	c.Unsubscribe(ids[0])
+	replacementID, _, err := c.Subscribe(100)
+	require.NoError(t, err, "disconnecting must release the user's quota")
+	c.Unsubscribe(replacementID)
+	for _, id := range ids[1:] {
+		c.Unsubscribe(id)
+	}
+}
+
+func TestServerTransferSubscribeEnforcesGlobalLimit(t *testing.T) {
+	c, cleanup := setupTransferFixture(t)
+	defer cleanup()
+
+	ids := make([]uint64, 0, maxServerTransferSubscribersGlobal)
+	for userID := uint64(1); len(ids) < maxServerTransferSubscribersGlobal; userID++ {
+		for range maxServerTransferSubscribersPerUser {
+			if len(ids) == maxServerTransferSubscribersGlobal {
+				break
+			}
+			id, _, err := c.Subscribe(userID)
+			require.NoError(t, err)
+			ids = append(ids, id)
+		}
+	}
+
+	_, _, err := c.Subscribe(999999)
+	require.ErrorIs(t, err, errTooManyServerTransferSubscribersGlobal)
+	for _, id := range ids {
+		c.Unsubscribe(id)
+	}
+}
+
 // The "one active transfer per server" invariant must hold under concurrent
 // callers (two operators batch-moving the same server, or batch-move racing
 // retry). The old flow had a TOCTOU between HasPending() and Initiate() that
@@ -1626,7 +1678,8 @@ func TestOnServersDeletedTerminatesPendingTransfersAndClearsIndexes(t *testing.T
 	tr := initiateAndRegister(t, c, 1, 100, 200, 1)
 	require.True(t, c.HasPending(1))
 
-	subID, ch := c.Subscribe()
+	subID, ch, err := c.Subscribe(100)
+	require.NoError(t, err)
 	defer c.Unsubscribe(subID)
 
 	require.NoError(t, DB.Unscoped().Delete(&model.Server{}, tr.ServerID).Error)

service/singleton/servicesentinel.go

@@ -2,6 +2,7 @@ package singleton
 
 import (
 	"cmp"
+	"crypto/sha256"
 	"fmt"
 	"iter"
 	"log"
@@ -9,6 +10,7 @@ import (
 	"slices"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/gin-gonic/gin"
@@ -22,9 +24,11 @@ import (
 )
 
 const (
-	_CurrentStatusSize = 30 // 统计 15 分钟内的数据为当前状态
+	_CurrentStatusSize = 30 // initial capacity for the rolling service window
 )
 
+var serviceEventSequence atomic.Uint64
+
 type serviceResponseItem struct {
 	model.ServiceResponseItem
 
@@ -46,9 +50,101 @@ type _TodayStatsOfService struct {
 type serviceResponseData = _TodayStatsOfService
 
 type serviceTaskStatus struct {
-	lastStatus uint8
-	t          time.Time
-	result     []*pb.TaskResult
+	lastStatus    uint8
+	stateEpoch    uint64
+	lastHistoryAt time.Time
+	history       serviceResponseData
+	lastData      string
+	result        []serviceWindowBucket
+}
+
+type serviceProbeSample struct {
+	successful bool
+	failed     bool
+	delay      float64
+}
+
+type serviceWindowBucket struct {
+	at        time.Time
+	reporters map[uint64]serviceProbeSample
+}
+
+func (status *serviceTaskStatus) appendResult(now time.Time, reporter uint64, result *pb.TaskResult) {
+	bucketAt := now.Truncate(30 * time.Second)
+	if len(status.result) > 0 && bucketAt.Before(status.result[len(status.result)-1].at) {
+		status.result = nil // wall-clock moved backwards; old buckets are not comparable
+	}
+	if len(status.result) == 0 || !status.result[len(status.result)-1].at.Equal(bucketAt) {
+		status.result = append(status.result, serviceWindowBucket{at: bucketAt, reporters: make(map[uint64]serviceProbeSample)})
+	}
+	bucket := &status.result[len(status.result)-1]
+	sample := bucket.reporters[reporter]
+	if result.Successful {
+		sample.successful = true
+		sample.delay = float64(result.Delay)
+	} else {
+		sample.failed = true
+	}
+	bucket.reporters[reporter] = sample
+	cutoff := now.Add(-15 * time.Minute)
+	first := 0
+	for first < len(status.result) && status.result[first].at.Before(cutoff) {
+		first++
+	}
+	if first > 0 {
+		status.result = append([]serviceWindowBucket(nil), status.result[first:]...)
+	}
+	if result.Successful {
+		status.history.Up++
+		status.history.Delay = (status.history.Delay*float64(status.history.Up-1) + float64(result.Delay)) / float64(status.history.Up)
+	} else {
+		status.history.Down++
+	}
+	status.lastData = result.Data
+}
+
+// flushServiceHistory persists the previous non-overlapping reporting period
+// before the next observation is added, so long probe gaps cannot lose or
+// double-count samples in the 15-minute availability history.
+func (ss *ServiceSentinel) flushServiceHistory(serviceID uint64, status *serviceTaskStatus, now time.Time) {
+	if status.lastHistoryAt.IsZero() || now.Before(status.lastHistoryAt.Add(15*time.Minute)) {
+		return
+	}
+	if status.history.Up+status.history.Down == 0 {
+		status.lastHistoryAt = now
+		return
+	}
+	if !TSDBEnabled() {
+		if err := DB.Create(&model.ServiceHistory{
+			CreatedAt: status.lastHistoryAt,
+			ServiceID: serviceID,
+			AvgDelay:  status.history.Delay,
+			Data:      status.lastData,
+			Up:        status.history.Up,
+			Down:      status.history.Down,
+		}).Error; err != nil {
+			log.Printf("NEZHA>> Failed to save service monitor metrics: %v", err)
+			return
+		}
+	}
+	status.history = serviceResponseData{}
+	status.lastHistoryAt = now
+}
+
+func (status *serviceTaskStatus) responseData() serviceResponseData {
+	var data serviceResponseData
+	for _, bucket := range status.result {
+		for _, sample := range bucket.reporters {
+			if sample.successful {
+				data.Up++
+				data.Delay = (data.Delay*float64(data.Up-1) + sample.delay) / float64(data.Up)
+			}
+			if sample.failed {
+				data.Down++
+			}
+		}
+	}
+	return data
 }
 
 type pingStore struct {
@@ -228,7 +324,7 @@ func (ss *ServiceSentinel) loadServiceHistory() error {
 		}
 		ss.services[service.ID] = service
 		ss.serviceCurrentStatusData[service.ID] = new(serviceTaskStatus)
-		ss.serviceCurrentStatusData[service.ID].result = make([]*pb.TaskResult, 0, _CurrentStatusSize)
+		ss.serviceCurrentStatusData[service.ID].result = make([]serviceWindowBucket, 0, _CurrentStatusSize)
 		ss.serviceStatusToday[service.ID] = &_TodayStatsOfService{}
 		validServices = append(validServices, service)
 	}
@@ -335,10 +431,18 @@ func (ss *ServiceSentinel) loadTodayStats(today time.Time) {
 		totalDelay := make(map[uint64]float64)
 		totalDelayCount := make(map[uint64]int)
 		for _, mh := range mhs {
-			ss.serviceStatusToday[mh.ServiceID].Up += mh.Up
-			ss.monthlyStatus[mh.ServiceID].TotalUp += mh.Up
-			ss.serviceStatusToday[mh.ServiceID].Down += mh.Down
-			ss.monthlyStatus[mh.ServiceID].TotalDown += mh.Down
+			todayStats := ss.serviceStatusToday[mh.ServiceID]
+			monthlyStats := ss.monthlyStatus[mh.ServiceID]
+			// History can outlive a deleted service or belong to a quarantined
+			// service. Neither has startup status entries; skip its rows before
+			// accumulating counts or delays so retained history cannot panic startup.
+			if todayStats == nil || monthlyStats == nil {
+				continue
+			}
+			todayStats.Up += mh.Up
+			monthlyStats.TotalUp += mh.Up
+			todayStats.Down += mh.Down
+			monthlyStats.TotalDown += mh.Down
 			totalDelay[mh.ServiceID] += mh.AvgDelay
 			totalDelayCount[mh.ServiceID]++
 		}
@@ -371,6 +475,9 @@ func (ss *ServiceSentinel) Update(m *model.Service) error {
 	if err != nil {
 		return err
 	}
+	// A service edit may disable notifications or change their recipients.
+	// Cancel any retry worker holding the previous service configuration.
+	NotificationShared.CancelServiceState(m.ID)
 	if ss.services[m.ID] != nil {
 		// 停掉旧任务
 		CronShared.Remove(ss.services[m.ID].CronJobID)
@@ -387,7 +494,7 @@ func (ss *ServiceSentinel) Update(m *model.Service) error {
 		if ss.serviceCurrentStatusData[m.ID] == nil {
 			ss.serviceCurrentStatusData[m.ID] = new(serviceTaskStatus)
 		}
-		ss.serviceCurrentStatusData[m.ID].result = make([]*pb.TaskResult, 0, _CurrentStatusSize)
+		ss.serviceCurrentStatusData[m.ID].result = make([]serviceWindowBucket, 0, _CurrentStatusSize)
 		ss.serviceStatusToday[m.ID] = &_TodayStatsOfService{}
 	}
 	// 更新这个任务
@@ -404,6 +511,7 @@ func (ss *ServiceSentinel) Delete(ids []uint64) {
 	defer ss.servicesLock.Unlock()
 
 	for _, id := range ids {
+		NotificationShared.CancelServiceState(id)
 		delete(ss.serviceCurrentStatusData, id)
 		delete(ss.serviceResponseDataStore, id)
 		delete(ss.serviceResponsePing, id)
@@ -669,32 +777,16 @@ func (ss *ServiceSentinel) processReport(r ReportData, serverShared *ServerClass
 	}
 
 	currentTime := time.Now()
-	if serviceCurrentStatusData.t.IsZero() {
-		serviceCurrentStatusData.t = currentTime
-	}
-
-	// 写入当前数据
-	if serviceCurrentStatusData.t.Before(currentTime) {
-		serviceCurrentStatusData.t = currentTime.Add(30 * time.Second)
-		serviceCurrentStatusData.result = append(serviceCurrentStatusData.result, mh)
+	ss.flushServiceHistory(mh.GetId(), serviceCurrentStatusData, currentTime)
+	// Include every accepted reporter's result. A fixed count of samples is
+	// not a 15-minute window when probes have different intervals or reporters.
+	serviceCurrentStatusData.appendResult(currentTime, r.Reporter, mh)
+	if serviceCurrentStatusData.lastHistoryAt.IsZero() {
+		serviceCurrentStatusData.lastHistoryAt = currentTime
 	}
 
 	// 更新当前状态
-	ss.serviceResponseDataStore[mh.GetId()] = serviceResponseData{}
-
-	// 永远是最新的 30 个数据的状态 [01:00, 02:00, 03:00] -> [04:00, 02:00, 03: 00]
-	for _, cs := range serviceCurrentStatusData.result {
-		if cs.GetId() > 0 {
-			rd := ss.serviceResponseDataStore[mh.GetId()]
-			if cs.Successful {
-				rd.Up++
-				rd.Delay = (rd.Delay*float64(rd.Up-1) + float64(cs.Delay)) / float64(rd.Up)
-			} else {
-				rd.Down++
-			}
-			ss.serviceResponseDataStore[mh.GetId()] = rd
-		}
-	}
+	ss.serviceResponseDataStore[mh.GetId()] = serviceCurrentStatusData.responseData()
 
 	// 计算在线率，
 	var stateCode uint8
@@ -704,24 +796,11 @@ func (ss *ServiceSentinel) processReport(r ReportData, serverShared *ServerClass
 		if rd.Down+rd.Up > 0 {
 			upPercent = rd.Up * 100 / (rd.Down + rd.Up)
 		}
-		stateCode = GetStatusCode(upPercent)
-	}
-
-	if len(serviceCurrentStatusData.result) == _CurrentStatusSize {
-		serviceCurrentStatusData.t = currentTime
-		if !TSDBEnabled() {
-			rd := ss.serviceResponseDataStore[mh.GetId()]
-			if err := DB.Create(&model.ServiceHistory{
-				ServiceID: mh.GetId(),
-				AvgDelay:  rd.Delay,
-				Data:      mh.Data,
-				Up:        rd.Up,
-				Down:      rd.Down,
-			}).Error; err != nil {
-				log.Printf("NEZHA>> Failed to save service monitor metrics: %v", err)
-			}
+		if rd.Down+rd.Up == 0 {
+			stateCode = StatusNoData
+		} else {
+			stateCode = GetStatusCode(upPercent)
 		}
-		serviceCurrentStatusData.result = serviceCurrentStatusData.result[:0]
 	}
 
 	// 延迟报警
@@ -734,77 +813,100 @@ func (ss *ServiceSentinel) processReport(r ReportData, serverShared *ServerClass
 		lastStatus := serviceCurrentStatusData.lastStatus
 		// 存储新的状态值
 		serviceCurrentStatusData.lastStatus = stateCode
-
-		notifyCheck(&r, m, cs, mh, lastStatus, stateCode)
-	}
-
-	// TLS 证书报警
-	if ss.serviceReportBeforeTLSSideEffectsHook != nil {
-		ss.serviceReportBeforeTLSSideEffectsHook(mh.GetId())
-	}
-	var errMsg string
-	if strings.HasPrefix(mh.Data, "SSL证书错误：") {
-		// i/o timeout、connection timeout、EOF 错误
-		if !strings.HasSuffix(mh.Data, "timeout") &&
-			!strings.HasSuffix(mh.Data, "EOF") &&
-			!strings.HasSuffix(mh.Data, "timed out") {
-			errMsg = mh.Data
-			if cs.Notify {
-				muteLabel := NotificationMuteLabel.ServiceTLS(mh.GetId(), "network")
-				go NotificationShared.SendNotification(cs.NotificationGroupID, Localizer.Tf("[TLS] Fetch cert info failed, Reporter: %s, Error: %s", cs.Name, errMsg), muteLabel)
-			}
+		if stateCode != lastStatus {
+			serviceCurrentStatusData.stateEpoch = serviceEventSequence.Add(1)
 		}
-	} else {
-		// 清除网络错误静音缓存
-		NotificationShared.UnMuteNotification(cs.NotificationGroupID, NotificationMuteLabel.ServiceTLS(mh.GetId(), "network"))
 
-		var newCert = strings.Split(mh.Data, "|")
-		if len(newCert) > 1 {
-			enableNotify := cs.Notify
+		notifyCheck(&r, m, cs, mh, lastStatus, stateCode, serviceCurrentStatusData.stateEpoch)
+	}
 
-			// 首次获取证书信息时，缓存证书信息
-			if ss.tlsCertCache[mh.GetId()] == "" {
-				ss.tlsCertCache[mh.GetId()] = mh.Data
+	// TLS 证书报警仅适用于 HTTPS 探针。其它探针的任意响应文本
+	// 不得被当作证书资料解析。
+	if mh.Type == model.TaskTypeHTTPGet && strings.HasPrefix(strings.ToLower(cs.Target), "https://") {
+		if ss.serviceReportBeforeTLSSideEffectsHook != nil {
+			ss.serviceReportBeforeTLSSideEffectsHook(mh.GetId())
+		}
+		var errMsg string
+		if strings.HasPrefix(mh.Data, "SSL证书错误：") {
+			// i/o timeout、connection timeout、EOF 错误
+			if !strings.HasSuffix(mh.Data, "timeout") &&
+				!strings.HasSuffix(mh.Data, "EOF") &&
+				!strings.HasSuffix(mh.Data, "timed out") {
+				errMsg = mh.Data
+				if cs.Notify {
+					muteLabel := NotificationMuteLabel.ServiceTLS(mh.GetId(), "network")
+					NotificationShared.SendNotificationAsync(cs.NotificationGroupID, Localizer.Tf("[TLS] Fetch cert info failed, Reporter: %s, Error: %s", cs.Name, errMsg), muteLabel)
+				}
 			}
+		} else {
+			// 清除网络错误静音缓存
+			NotificationShared.UnMuteNotification(cs.NotificationGroupID, NotificationMuteLabel.ServiceTLS(mh.GetId(), "network"))
 
-			oldCert := strings.Split(ss.tlsCertCache[mh.GetId()], "|")
-			isCertChanged := false
-			expiresOld, _ := time.Parse("2006-01-02 15:04:05 -0700 MST", oldCert[1])
-			expiresNew, _ := time.Parse("2006-01-02 15:04:05 -0700 MST", newCert[1])
-
-			// 证书变更时，更新缓存
-			if oldCert[0] != newCert[0] && !expiresNew.Equal(expiresOld) {
-				isCertChanged = true
-				ss.tlsCertCache[mh.GetId()] = mh.Data
-			}
-
-			notificationGroupID := cs.NotificationGroupID
-			serviceName := cs.Name
-
-			// 需要发送提醒
-			if enableNotify {
-				// 证书过期提醒
-				if expiresNew.Before(time.Now().AddDate(0, 0, 7)) {
-					expiresTimeStr := expiresNew.Format("2006-01-02 15:04:05")
-					errMsg = Localizer.Tf(
-						"The TLS certificate will expire within seven days. Expiration time: %s",
-						expiresTimeStr,
-					)
-
-					// 静音规则： 服务id+证书过期时间
-					// 用于避免多个监测点对相同证书同时报警
-					muteLabel := NotificationMuteLabel.ServiceTLS(mh.GetId(), fmt.Sprintf("expire_%s", expiresTimeStr))
-					go NotificationShared.SendNotification(notificationGroupID, fmt.Sprintf("[TLS] %s %s", serviceName, errMsg), muteLabel)
+			var newCert = strings.Split(mh.Data, "|")
+			if len(newCert) > 1 {
+				enableNotify := cs.Notify
+				expiresNew, newErr := time.Parse("2006-01-02 15:04:05 -0700 MST", newCert[1])
+				if newErr != nil {
+					log.Printf("NEZHA>> Ignoring malformed TLS certificate expiry for service %d", mh.GetId())
+					return
 				}
 
-				// 证书变更提醒
-				if isCertChanged {
-					errMsg = Localizer.Tf(
-						"TLS certificate changed, old: issuer %s, expires at %s; new: issuer %s, expires at %s",
-						oldCert[0], expiresOld.Format("2006-01-02 15:04:05"), newCert[0], expiresNew.Format("2006-01-02 15:04:05"))
+				// 首次获取证书信息时，缓存证书信息
+				if ss.tlsCertCache[mh.GetId()] == "" {
+					ss.tlsCertCache[mh.GetId()] = mh.Data
+				}
 
-					// 证书变更后会自动更新缓存，所以不需要静音
-					go NotificationShared.SendNotification(notificationGroupID, fmt.Sprintf("[TLS] %s %s", serviceName, errMsg), "")
+				oldCert := strings.Split(ss.tlsCertCache[mh.GetId()], "|")
+				isCertChanged := false
+				expiresOld, oldErr := time.Parse("2006-01-02 15:04:05 -0700 MST", oldCert[1])
+				if oldErr != nil {
+					log.Printf("NEZHA>> Ignoring malformed TLS certificate expiry for service %d", mh.GetId())
+					return
+				}
+
+				// 只有通知成功后才推进基线；失败时下一次探针结果会重试。
+				if oldCert[0] != newCert[0] || !expiresNew.Equal(expiresOld) {
+					isCertChanged = true
+					if !enableNotify {
+						ss.tlsCertCache[mh.GetId()] = mh.Data
+					}
+				}
+
+				notificationGroupID := cs.NotificationGroupID
+				serviceName := cs.Name
+
+				// 需要发送提醒
+				if enableNotify {
+					// 证书过期提醒
+					if expiresNew.Before(time.Now().AddDate(0, 0, 7)) {
+						expiresTimeStr := expiresNew.Format("2006-01-02 15:04:05")
+						errMsg = Localizer.Tf(
+							"The TLS certificate will expire within seven days. Expiration time: %s",
+							expiresTimeStr,
+						)
+
+						// 静音规则： 服务id+证书过期时间
+						// 用于避免多个监测点对相同证书同时报警
+						muteLabel := NotificationMuteLabel.ServiceTLS(mh.GetId(), fmt.Sprintf("expire_%s", expiresTimeStr))
+						NotificationShared.SendNotificationAsync(notificationGroupID, fmt.Sprintf("[TLS] %s %s", serviceName, errMsg), muteLabel)
+					}
+
+					// 证书变更提醒
+					if isCertChanged {
+						errMsg = Localizer.Tf(
+							"TLS certificate changed, old: issuer %s, expires at %s; new: issuer %s, expires at %s",
+							oldCert[0], expiresOld.Format("2006-01-02 15:04:05"), newCert[0], expiresNew.Format("2006-01-02 15:04:05"))
+
+						oldRaw, newRaw, serviceID := ss.tlsCertCache[mh.GetId()], mh.Data, mh.GetId()
+						muteLabel := NotificationMuteLabel.ServiceTLS(serviceID, fmt.Sprintf("change_%x", sha256.Sum256([]byte(newRaw))))
+						NotificationShared.sendNotificationAsync(notificationGroupID, fmt.Sprintf("[TLS] %s %s", serviceName, errMsg), muteLabel, func() {
+							ss.serviceResponseDataStoreLock.Lock()
+							if ss.serviceCurrentStatusData[serviceID] != nil && ss.tlsCertCache[serviceID] == oldRaw {
+								ss.tlsCertCache[serviceID] = newRaw
+							}
+							ss.serviceResponseDataStoreLock.Unlock()
+						})
+					}
 				}
 			}
 		}
@@ -832,11 +934,11 @@ func delayCheck(r *ReportData, m map[uint64]*model.Server, ss *model.Service, mh
 	if mh.Delay > ss.MaxLatency {
 		// 延迟超过最大值
 		msg := Localizer.Tf("[Latency] %s %2f > %2f, Reporter: %s", ss.Name, mh.Delay, ss.MaxLatency, reporterServer.Name)
-		go NotificationShared.SendNotification(notificationGroupID, msg, minMuteLabel)
+		NotificationShared.SendNotificationAsync(notificationGroupID, msg, minMuteLabel)
 	} else if mh.Delay < ss.MinLatency {
 		// 延迟低于最小值
 		msg := Localizer.Tf("[Latency] %s %2f < %2f, Reporter: %s", ss.Name, mh.Delay, ss.MinLatency, reporterServer.Name)
-		go NotificationShared.SendNotification(notificationGroupID, msg, maxMuteLabel)
+		NotificationShared.SendNotificationAsync(notificationGroupID, msg, maxMuteLabel)
 	} else {
 		// 正常延迟， 清除静音缓存
 		NotificationShared.UnMuteNotification(notificationGroupID, minMuteLabel)
@@ -845,7 +947,7 @@ func delayCheck(r *ReportData, m map[uint64]*model.Server, ss *model.Service, mh
 }
 
 func notifyCheck(r *ReportData, m map[uint64]*model.Server,
-	ss *model.Service, mh *pb.TaskResult, lastStatus, stateCode uint8) {
+	ss *model.Service, mh *pb.TaskResult, lastStatus, stateCode uint8, stateEpoch uint64) {
 	// GHSA-jx78-55p5-rwv5: guard against concurrent server deletion (same TOCTOU
 	// class as the 2026-07-21 fix, a few dozen lines lower in the same worker).
 	// ServerShared has its own lock; m is a snapshot taken outside
@@ -858,14 +960,9 @@ func notifyCheck(r *ReportData, m map[uint64]*model.Server,
 	if isNeedSendNotification && reporterServer != nil {
 		notificationGroupID := ss.NotificationGroupID
 		notificationMsg := Localizer.Tf("[%s] %s Reporter: %s, Error: %s", StatusCodeToString(stateCode), ss.Name, reporterServer.Name, mh.Data)
-		muteLabel := NotificationMuteLabel.ServiceStateChanged(mh.GetId())
+		muteLabel := fmt.Sprintf("%s:%d:%d", NotificationMuteLabel.ServiceStateChanged(mh.GetId()), stateCode, stateEpoch)
 
-		// 状态变更时，清除静音缓存
-		if stateCode != lastStatus {
-			NotificationShared.UnMuteNotification(notificationGroupID, muteLabel)
-		}
-
-		go NotificationShared.SendNotification(notificationGroupID, notificationMsg, muteLabel)
+		NotificationShared.SendServiceState(mh.GetId(), notificationGroupID, notificationMsg, muteLabel)
 	}
 
 	// 判断是否需要触发任务
@@ -890,9 +987,6 @@ const (
 )
 
 func GetStatusCode[T constraints.Float | constraints.Integer](percent T) uint8 {
-	if percent == 0 {
-		return StatusNoData
-	}
 	if percent > 95 {
 		return StatusGood
 	}

service/singleton/servicesentinel_startup_history_test.go

@@ -0,0 +1,94 @@
+package singleton
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/nezhahq/nezha/model"
+)
+
+func TestServiceSentinelStartupWithRetainedHistory(t *testing.T) {
+	for _, serviceState := range []string{"retained", "deleted", "quarantined"} {
+		t.Run(serviceState, func(t *testing.T) {
+			initial := newServiceMonitorSecurityHarness(t)
+			initial.Close()
+			require.False(t, TSDBEnabled())
+
+			activeService := &model.Service{Name: "active monitor", Type: model.TaskTypeTCPPing}
+			historicalService := &model.Service{Name: "historical monitor", Type: model.TaskTypeHTTPGet}
+			require.NoError(t, DB.Create(activeService).Error)
+			require.NoError(t, DB.Create(historicalService).Error)
+
+			year, month, day := time.Now().Date()
+			today := time.Date(year, month, day, 0, 0, 0, 0, Loc)
+			// Use the production writer so retained rows have the same shape as
+			// ordinary aggregated history, including server_id = 0.
+			historicalStatus := &serviceTaskStatus{
+				lastHistoryAt: today,
+				history:       serviceResponseData{Up: 7, Down: 2, Delay: 90},
+			}
+			initial.flushServiceHistory(historicalService.ID, historicalStatus, today.Add(15*time.Minute))
+			var retainedHistory []model.ServiceHistory
+			require.NoError(t, DB.Where("service_id = ?", historicalService.ID).Find(&retainedHistory).Error)
+			require.Len(t, retainedHistory, 1)
+			require.Zero(t, retainedHistory[0].ServerID)
+			require.Equal(t, uint64(7), retainedHistory[0].Up)
+			require.Equal(t, uint64(2), retainedHistory[0].Down)
+			require.Equal(t, float64(90), retainedHistory[0].AvgDelay)
+
+			require.NoError(t, DB.Create(&[]model.ServiceHistory{
+				{ServiceID: activeService.ID, CreatedAt: today, Up: 5, Down: 1, AvgDelay: 10},
+				{ServiceID: activeService.ID, CreatedAt: today, Up: 3, Down: 2, AvgDelay: 30},
+				{ServiceID: activeService.ID, CreatedAt: today.Add(-time.Hour), Up: 4, Down: 1, AvgDelay: 15},
+				{ServiceID: activeService.ID, ServerID: 100, CreatedAt: today, Up: 100, AvgDelay: 500},
+			}).Error)
+
+			switch serviceState {
+			case "deleted":
+				// Match the controller's hard delete without cleaning history: old
+				// databases must remain loadable even if deletion cleanup changes.
+				result := DB.Unscoped().Delete(&model.Service{}, "id in (?)", []uint64{historicalService.ID})
+				require.NoError(t, result.Error)
+				require.EqualValues(t, 1, result.RowsAffected)
+			case "quarantined":
+				// Legacy invalid service types are deliberately excluded at startup.
+				result := DB.Model(historicalService).UpdateColumn("type", model.TaskTypeCommand)
+				require.NoError(t, result.Error)
+				require.EqualValues(t, 1, result.RowsAffected)
+			}
+			var historyCount int64
+			require.NoError(t, DB.Model(&model.ServiceHistory{}).Where("service_id = ?", historicalService.ID).Count(&historyCount).Error)
+			require.EqualValues(t, 1, historyCount)
+
+			var restarted *ServiceSentinel
+			var startupErr error
+			require.NotPanics(t, func() {
+				restarted, startupErr = NewServiceSentinel(make(chan *model.Service, 1))
+			})
+			if restarted != nil {
+				t.Cleanup(restarted.Close)
+			}
+			require.NoError(t, startupErr)
+			require.NotNil(t, restarted)
+			require.Equal(t, &_TodayStatsOfService{Up: 8, Down: 3, Delay: 20}, restarted.serviceStatusToday[activeService.ID])
+			monthly := restarted.monthlyStatus[activeService.ID]
+			require.NotNil(t, monthly)
+			require.Equal(t, uint64(12), monthly.TotalUp)
+			require.Equal(t, uint64(4), monthly.TotalDown)
+
+			if serviceState == "retained" {
+				require.Len(t, restarted.services, 2)
+				require.Equal(t, &_TodayStatsOfService{Up: 7, Down: 2, Delay: 90}, restarted.serviceStatusToday[historicalService.ID])
+				require.Equal(t, uint64(7), restarted.monthlyStatus[historicalService.ID].TotalUp)
+				require.Equal(t, uint64(2), restarted.monthlyStatus[historicalService.ID].TotalDown)
+			} else {
+				require.Len(t, restarted.services, 1)
+				require.NotContains(t, restarted.services, historicalService.ID)
+				require.NotContains(t, restarted.serviceStatusToday, historicalService.ID)
+				require.NotContains(t, restarted.monthlyStatus, historicalService.ID)
+			}
+		})
+	}
+}

service/singleton/servicesentinel_window_test.go

@@ -0,0 +1,63 @@
+package singleton
+
+import (
+	"testing"
+	"time"
+
+	pb "github.com/nezhahq/nezha/proto"
+)
+
+func TestServiceWindowCountsFirstFailureAsDown(t *testing.T) {
+	status := new(serviceTaskStatus)
+	status.appendResult(time.Unix(1000, 0), 1, &pb.TaskResult{Successful: false})
+	data := status.responseData()
+	if data.Down != 1 || data.Up != 0 || GetStatusCode(uint64(0)) != StatusDown {
+		t.Fatalf("first failed report = %+v, status %d; want Down", data, GetStatusCode(uint64(0)))
+	}
+}
+
+func TestServiceWindowExpiresOldReportsAndDoesNotResetAtThirty(t *testing.T) {
+	status := new(serviceTaskStatus)
+	start := time.Unix(1000, 0)
+	for i := 0; i < 30; i++ {
+		status.appendResult(start.Add(time.Duration(i)*30*time.Second), 1, &pb.TaskResult{Successful: true})
+	}
+	if data := status.responseData(); data.Up != 30 {
+		t.Fatalf("30th report reset window: %+v", data)
+	}
+	status.appendResult(start.Add(29*30*time.Second+time.Second), 2, &pb.TaskResult{Successful: false})
+	if data := status.responseData(); data.Up != 30 || data.Down != 1 {
+		t.Fatalf("31st report corrupted window: %+v", data)
+	}
+	status.appendResult(start.Add(31*time.Minute), 1, &pb.TaskResult{Successful: false})
+	if data := status.responseData(); data.Up != 0 || data.Down != 1 {
+		t.Fatalf("expired reports still influence status: %+v", data)
+	}
+}
+
+func TestServiceWindowDeduplicatesFloodWithinBucket(t *testing.T) {
+	status := new(serviceTaskStatus)
+	now := time.Unix(1000, 0)
+	for i := 0; i < 10000; i++ {
+		status.appendResult(now, 1, &pb.TaskResult{Successful: false, Data: "ignored payload"})
+	}
+	if len(status.result) != 1 || len(status.result[0].reporters) != 1 {
+		t.Fatalf("flood grew live window: buckets=%d", len(status.result))
+	}
+	if data := status.responseData(); data.Down != 1 {
+		t.Fatalf("duplicate reporter skewed status: %+v", data)
+	}
+	if status.history.Down != 10000 {
+		t.Fatalf("history lost raw reports: %+v", status.history)
+	}
+}
+
+func TestServiceWindowDoesNotEraseFailureWithLaterSuccess(t *testing.T) {
+	status := new(serviceTaskStatus)
+	now := time.Unix(1000, 0)
+	status.appendResult(now, 1, &pb.TaskResult{Successful: false})
+	status.appendResult(now.Add(time.Second), 1, &pb.TaskResult{Successful: true, Delay: 20})
+	if data := status.responseData(); data.Down != 1 || data.Up != 1 {
+		t.Fatalf("later success erased failure in same bucket: %+v", data)
+	}
+}

service/singleton/singleton.go

@@ -63,6 +63,7 @@ func LoadSingleton(bus chan<- *model.Service) (err error) {
 	ServerShared = NewServerClass()
 	CronShared = NewCronClass()
 	ServerTransferShared = NewServerTransferClass()
+	OnlineHistoryShared = NewOnlineHistoryRecorder()
 	// 最后初始化 ServiceSentinel
 	ServiceSentinelShared, err = NewServiceSentinel(bus)
 	if err == nil {
@@ -97,7 +98,7 @@ func InitDBFromPath(path string) error {
 		model.Cron{}, model.Transfer{}, model.ServerGroupServer{},
 		model.NAT{}, model.DDNSProfile{}, model.NotificationGroupNotification{},
 		model.WAF{}, model.Oauth2Bind{}, model.Domain{}, model.ServerTransfer{}, model.JWTSession{},
-		model.APIToken{}, model.MCPAuditLog{})
+		model.APIToken{}, model.MCPAuditLog{}, model.OnlineSession{})
 
 	if err != nil {
 		return err
@@ -171,7 +172,7 @@ func CleanMonitorHistory() {
 	for _, alert := range alerts {
 		for _, rule := range alert.Rules {
 			// 是不是流量记录规则
-			if !rule.IsTransferDurationRule() {
+			if !rule.HasSafeCycleConfiguration() {
 				continue
 			}
 			dataCouldRemoveBefore := rule.GetTransferDurationStart().UTC()

service/singleton/tsdb.go

@@ -1,6 +1,7 @@
 package singleton
 
 import (
+	"errors"
 	"log"
 	"time"
 
@@ -10,6 +11,8 @@ import (
 
 var TSDBShared *tsdb.TSDB
 
+var openTSDB = tsdb.Open
+
 func InitTSDB() error {
 	config := &tsdb.Config{
 		RetentionDays:      30,
@@ -44,11 +47,21 @@ func InitTSDB() error {
 		return nil
 	}
 
-	var err error
-	TSDBShared, err = tsdb.Open(config)
+	TSDBShared = nil
+	db, err := openTSDB(config)
 	if err != nil {
+		if errors.Is(err, tsdb.ErrDiskFull) {
+			log.Printf("NEZHA>> Warning: TSDB is unavailable because its disk is full; dashboard and alerts will continue without TSDB writes: %v", err)
+			if DB != nil {
+				if migrateErr := DB.AutoMigrate(model.ServiceHistory{}); migrateErr != nil {
+					log.Printf("NEZHA>> Warning: failed to prepare SQLite service history fallback: %v", migrateErr)
+				}
+			}
+			return nil
+		}
 		return err
 	}
+	TSDBShared = db
 
 	log.Println("NEZHA>> TSDB initialized successfully")
 

service/singleton/tsdb_disk_guard_test.go

@@ -0,0 +1,58 @@
+package singleton
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/nezhahq/nezha/model"
+	"github.com/nezhahq/nezha/pkg/tsdb"
+)
+
+func TestInitTSDBContinuesWhenDiskIsFull(t *testing.T) {
+	originalConf := Conf
+	originalDB := DB
+	originalShared := TSDBShared
+	originalOpen := openTSDB
+	t.Cleanup(func() {
+		Conf = originalConf
+		DB = originalDB
+		TSDBShared = originalShared
+		openTSDB = originalOpen
+	})
+
+	Conf = &ConfigClass{Config: &model.Config{
+		TSDB: model.TSDBConf{DataPath: "/full/tsdb"},
+	}}
+	DB = nil
+	TSDBShared = nil
+	openTSDB = func(*tsdb.Config) (*tsdb.TSDB, error) {
+		return nil, fmt.Errorf("open failed: %w", tsdb.ErrDiskFull)
+	}
+
+	require.NoError(t, InitTSDB())
+	require.Nil(t, TSDBShared)
+}
+
+func TestInitTSDBStillReturnsNonDiskErrors(t *testing.T) {
+	originalConf := Conf
+	originalShared := TSDBShared
+	originalOpen := openTSDB
+	t.Cleanup(func() {
+		Conf = originalConf
+		TSDBShared = originalShared
+		openTSDB = originalOpen
+	})
+
+	Conf = &ConfigClass{Config: &model.Config{
+		TSDB: model.TSDBConf{DataPath: "/broken/tsdb"},
+	}}
+	TSDBShared = nil
+	openTSDB = func(*tsdb.Config) (*tsdb.TSDB, error) {
+		return nil, fmt.Errorf("permission denied")
+	}
+
+	require.ErrorContains(t, InitTSDB(), "permission denied")
+	require.Nil(t, TSDBShared)
+}