fix(service): skip retained history for unavailable monitors at startup

Ignore history rows without initialized daily or monthly state so deleted and quarantined services cannot panic dashboard startup. Cover startup and statistics preservation with regression tests.

Thanks to Zachariah Mithani (@zachariah-mithani, AploSec) for privately reporting the retained service history startup panic.

Co-authored-by: naiba/CloudCode <hi+cloudcode@nai.ba>

service/singleton/servicesentinel.go

@@ -431,10 +431,18 @@ func (ss *ServiceSentinel) loadTodayStats(today time.Time) {
 		totalDelay := make(map[uint64]float64)
 		totalDelayCount := make(map[uint64]int)
 		for _, mh := range mhs {
-			ss.serviceStatusToday[mh.ServiceID].Up += mh.Up
-			ss.monthlyStatus[mh.ServiceID].TotalUp += mh.Up
-			ss.serviceStatusToday[mh.ServiceID].Down += mh.Down
-			ss.monthlyStatus[mh.ServiceID].TotalDown += mh.Down
+			todayStats := ss.serviceStatusToday[mh.ServiceID]
+			monthlyStats := ss.monthlyStatus[mh.ServiceID]
+			// History can outlive a deleted service or belong to a quarantined
+			// service. Neither has startup status entries; skip its rows before
+			// accumulating counts or delays so retained history cannot panic startup.
+			if todayStats == nil || monthlyStats == nil {
+				continue
+			}
+			todayStats.Up += mh.Up
+			monthlyStats.TotalUp += mh.Up
+			todayStats.Down += mh.Down
+			monthlyStats.TotalDown += mh.Down
 			totalDelay[mh.ServiceID] += mh.AvgDelay
 			totalDelayCount[mh.ServiceID]++
 		}

service/singleton/servicesentinel_startup_history_test.go

@@ -0,0 +1,94 @@
+package singleton
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/nezhahq/nezha/model"
+)
+
+func TestServiceSentinelStartupWithRetainedHistory(t *testing.T) {
+	for _, serviceState := range []string{"retained", "deleted", "quarantined"} {
+		t.Run(serviceState, func(t *testing.T) {
+			initial := newServiceMonitorSecurityHarness(t)
+			initial.Close()
+			require.False(t, TSDBEnabled())
+
+			activeService := &model.Service{Name: "active monitor", Type: model.TaskTypeTCPPing}
+			historicalService := &model.Service{Name: "historical monitor", Type: model.TaskTypeHTTPGet}
+			require.NoError(t, DB.Create(activeService).Error)
+			require.NoError(t, DB.Create(historicalService).Error)
+
+			year, month, day := time.Now().Date()
+			today := time.Date(year, month, day, 0, 0, 0, 0, Loc)
+			// Use the production writer so retained rows have the same shape as
+			// ordinary aggregated history, including server_id = 0.
+			historicalStatus := &serviceTaskStatus{
+				lastHistoryAt: today,
+				history:       serviceResponseData{Up: 7, Down: 2, Delay: 90},
+			}
+			initial.flushServiceHistory(historicalService.ID, historicalStatus, today.Add(15*time.Minute))
+			var retainedHistory []model.ServiceHistory
+			require.NoError(t, DB.Where("service_id = ?", historicalService.ID).Find(&retainedHistory).Error)
+			require.Len(t, retainedHistory, 1)
+			require.Zero(t, retainedHistory[0].ServerID)
+			require.Equal(t, uint64(7), retainedHistory[0].Up)
+			require.Equal(t, uint64(2), retainedHistory[0].Down)
+			require.Equal(t, float64(90), retainedHistory[0].AvgDelay)
+
+			require.NoError(t, DB.Create(&[]model.ServiceHistory{
+				{ServiceID: activeService.ID, CreatedAt: today, Up: 5, Down: 1, AvgDelay: 10},
+				{ServiceID: activeService.ID, CreatedAt: today, Up: 3, Down: 2, AvgDelay: 30},
+				{ServiceID: activeService.ID, CreatedAt: today.Add(-time.Hour), Up: 4, Down: 1, AvgDelay: 15},
+				{ServiceID: activeService.ID, ServerID: 100, CreatedAt: today, Up: 100, AvgDelay: 500},
+			}).Error)
+
+			switch serviceState {
+			case "deleted":
+				// Match the controller's hard delete without cleaning history: old
+				// databases must remain loadable even if deletion cleanup changes.
+				result := DB.Unscoped().Delete(&model.Service{}, "id in (?)", []uint64{historicalService.ID})
+				require.NoError(t, result.Error)
+				require.EqualValues(t, 1, result.RowsAffected)
+			case "quarantined":
+				// Legacy invalid service types are deliberately excluded at startup.
+				result := DB.Model(historicalService).UpdateColumn("type", model.TaskTypeCommand)
+				require.NoError(t, result.Error)
+				require.EqualValues(t, 1, result.RowsAffected)
+			}
+			var historyCount int64
+			require.NoError(t, DB.Model(&model.ServiceHistory{}).Where("service_id = ?", historicalService.ID).Count(&historyCount).Error)
+			require.EqualValues(t, 1, historyCount)
+
+			var restarted *ServiceSentinel
+			var startupErr error
+			require.NotPanics(t, func() {
+				restarted, startupErr = NewServiceSentinel(make(chan *model.Service, 1))
+			})
+			if restarted != nil {
+				t.Cleanup(restarted.Close)
+			}
+			require.NoError(t, startupErr)
+			require.NotNil(t, restarted)
+			require.Equal(t, &_TodayStatsOfService{Up: 8, Down: 3, Delay: 20}, restarted.serviceStatusToday[activeService.ID])
+			monthly := restarted.monthlyStatus[activeService.ID]
+			require.NotNil(t, monthly)
+			require.Equal(t, uint64(12), monthly.TotalUp)
+			require.Equal(t, uint64(4), monthly.TotalDown)
+
+			if serviceState == "retained" {
+				require.Len(t, restarted.services, 2)
+				require.Equal(t, &_TodayStatsOfService{Up: 7, Down: 2, Delay: 90}, restarted.serviceStatusToday[historicalService.ID])
+				require.Equal(t, uint64(7), restarted.monthlyStatus[historicalService.ID].TotalUp)
+				require.Equal(t, uint64(2), restarted.monthlyStatus[historicalService.ID].TotalDown)
+			} else {
+				require.Len(t, restarted.services, 1)
+				require.NotContains(t, restarted.services, historicalService.ID)
+				require.NotContains(t, restarted.serviceStatusToday, historicalService.ID)
+				require.NotContains(t, restarted.monthlyStatus, historicalService.ID)
+			}
+		})
+	}
+}