From efd3e07b901d6acf7ef2ab3b26458cc07e7605dc Mon Sep 17 00:00:00 2001 From: naiba Date: Wed, 21 Oct 2026 13:30:33 +0000 Subject: [PATCH] fix(service): skip retained history for unavailable monitors at startup Ignore history rows without initialized daily or monthly state so deleted and quarantined services cannot panic dashboard startup. Cover startup and statistics preservation with regression tests. Thanks to Zachariah Mithani (@zachariah-mithani, AploSec) for privately reporting the retained service history startup panic. Co-authored-by: naiba/CloudCode --- service/singleton/servicesentinel.go | 16 +++- .../servicesentinel_startup_history_test.go | 94 +++++++++++++++++++ 2 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 service/singleton/servicesentinel_startup_history_test.go diff --git a/service/singleton/servicesentinel.go b/service/singleton/servicesentinel.go index 3745c915..9d6c66fa 100644 --- a/service/singleton/servicesentinel.go +++ b/service/singleton/servicesentinel.go @@ -431,10 +431,18 @@ func (ss *ServiceSentinel) loadTodayStats(today time.Time) { totalDelay := make(map[uint64]float64) totalDelayCount := make(map[uint64]int) for _, mh := range mhs { - ss.serviceStatusToday[mh.ServiceID].Up += mh.Up - ss.monthlyStatus[mh.ServiceID].TotalUp += mh.Up - ss.serviceStatusToday[mh.ServiceID].Down += mh.Down - ss.monthlyStatus[mh.ServiceID].TotalDown += mh.Down + todayStats := ss.serviceStatusToday[mh.ServiceID] + monthlyStats := ss.monthlyStatus[mh.ServiceID] + // History can outlive a deleted service or belong to a quarantined + // service. Neither has startup status entries; skip its rows before + // accumulating counts or delays so retained history cannot panic startup. + if todayStats == nil || monthlyStats == nil { + continue + } + todayStats.Up += mh.Up + monthlyStats.TotalUp += mh.Up + todayStats.Down += mh.Down + monthlyStats.TotalDown += mh.Down totalDelay[mh.ServiceID] += mh.AvgDelay totalDelayCount[mh.ServiceID]++ } diff --git a/service/singleton/servicesentinel_startup_history_test.go b/service/singleton/servicesentinel_startup_history_test.go new file mode 100644 index 00000000..8656c880 --- /dev/null +++ b/service/singleton/servicesentinel_startup_history_test.go @@ -0,0 +1,94 @@ +package singleton + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/nezhahq/nezha/model" +) + +func TestServiceSentinelStartupWithRetainedHistory(t *testing.T) { + for _, serviceState := range []string{"retained", "deleted", "quarantined"} { + t.Run(serviceState, func(t *testing.T) { + initial := newServiceMonitorSecurityHarness(t) + initial.Close() + require.False(t, TSDBEnabled()) + + activeService := &model.Service{Name: "active monitor", Type: model.TaskTypeTCPPing} + historicalService := &model.Service{Name: "historical monitor", Type: model.TaskTypeHTTPGet} + require.NoError(t, DB.Create(activeService).Error) + require.NoError(t, DB.Create(historicalService).Error) + + year, month, day := time.Now().Date() + today := time.Date(year, month, day, 0, 0, 0, 0, Loc) + // Use the production writer so retained rows have the same shape as + // ordinary aggregated history, including server_id = 0. + historicalStatus := &serviceTaskStatus{ + lastHistoryAt: today, + history: serviceResponseData{Up: 7, Down: 2, Delay: 90}, + } + initial.flushServiceHistory(historicalService.ID, historicalStatus, today.Add(15*time.Minute)) + var retainedHistory []model.ServiceHistory + require.NoError(t, DB.Where("service_id = ?", historicalService.ID).Find(&retainedHistory).Error) + require.Len(t, retainedHistory, 1) + require.Zero(t, retainedHistory[0].ServerID) + require.Equal(t, uint64(7), retainedHistory[0].Up) + require.Equal(t, uint64(2), retainedHistory[0].Down) + require.Equal(t, float64(90), retainedHistory[0].AvgDelay) + + require.NoError(t, DB.Create(&[]model.ServiceHistory{ + {ServiceID: activeService.ID, CreatedAt: today, Up: 5, Down: 1, AvgDelay: 10}, + {ServiceID: activeService.ID, CreatedAt: today, Up: 3, Down: 2, AvgDelay: 30}, + {ServiceID: activeService.ID, CreatedAt: today.Add(-time.Hour), Up: 4, Down: 1, AvgDelay: 15}, + {ServiceID: activeService.ID, ServerID: 100, CreatedAt: today, Up: 100, AvgDelay: 500}, + }).Error) + + switch serviceState { + case "deleted": + // Match the controller's hard delete without cleaning history: old + // databases must remain loadable even if deletion cleanup changes. + result := DB.Unscoped().Delete(&model.Service{}, "id in (?)", []uint64{historicalService.ID}) + require.NoError(t, result.Error) + require.EqualValues(t, 1, result.RowsAffected) + case "quarantined": + // Legacy invalid service types are deliberately excluded at startup. + result := DB.Model(historicalService).UpdateColumn("type", model.TaskTypeCommand) + require.NoError(t, result.Error) + require.EqualValues(t, 1, result.RowsAffected) + } + var historyCount int64 + require.NoError(t, DB.Model(&model.ServiceHistory{}).Where("service_id = ?", historicalService.ID).Count(&historyCount).Error) + require.EqualValues(t, 1, historyCount) + + var restarted *ServiceSentinel + var startupErr error + require.NotPanics(t, func() { + restarted, startupErr = NewServiceSentinel(make(chan *model.Service, 1)) + }) + if restarted != nil { + t.Cleanup(restarted.Close) + } + require.NoError(t, startupErr) + require.NotNil(t, restarted) + require.Equal(t, &_TodayStatsOfService{Up: 8, Down: 3, Delay: 20}, restarted.serviceStatusToday[activeService.ID]) + monthly := restarted.monthlyStatus[activeService.ID] + require.NotNil(t, monthly) + require.Equal(t, uint64(12), monthly.TotalUp) + require.Equal(t, uint64(4), monthly.TotalDown) + + if serviceState == "retained" { + require.Len(t, restarted.services, 2) + require.Equal(t, &_TodayStatsOfService{Up: 7, Down: 2, Delay: 90}, restarted.serviceStatusToday[historicalService.ID]) + require.Equal(t, uint64(7), restarted.monthlyStatus[historicalService.ID].TotalUp) + require.Equal(t, uint64(2), restarted.monthlyStatus[historicalService.ID].TotalDown) + } else { + require.Len(t, restarted.services, 1) + require.NotContains(t, restarted.services, historicalService.ID) + require.NotContains(t, restarted.serviceStatusToday, historicalService.ID) + require.NotContains(t, restarted.monthlyStatus, historicalService.ID) + } + }) + } +}