nezha_domains/service/singleton/servicesentinel_lifecycle_test.go

package singleton

import (
	"context"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/nezhahq/nezha/model"
)

// Regression markers for Finding 1 and Finding 2 of GHSA-jx78-55p5-rwv5
// (incomplete fix of GHSA-qjpp-gffx-2wm9).
const (
	concurrentServerDeleteSuccessMarker = "ghsa-jx78-55p5-rwv5-finding1-no-crash"
	deleteUnknownIDSuccessMarker        = "ghsa-jx78-55p5-rwv5-finding2-no-zombie"
)

const serviceSentinelLifecycleSuccessMarker = "service-sentinel-stale-report-lifecycle-success"

func TestServiceSentinelReporterDeleteWaitsForSynchronousReportProcessing(t *testing.T) {
	ctx, cancel := context.WithTimeout(t.Context(), time.Second)
	defer cancel()
	ss := newServiceMonitorSecurityHarness(t,
		&model.Server{Common: model.Common{ID: 1, UserID: 1}, Name: "reporter"},
	)
	service := &model.Service{
		Common:      model.Common{ID: 10, UserID: 1},
		Name:        "lifecycle-service",
		Type:        model.TaskTypeTCPPing,
		Target:      "lifecycle.example.invalid:443",
		Duration:    3600,
		Cover:       model.ServiceCoverIgnoreAll,
		SkipServers: map[uint64]bool{1: true},
	}
	addServiceMonitorSecurityService(t, ss, service)

	reportValidated := make(chan struct{})
	releaseReport := make(chan struct{})
	var releaseOnce sync.Once
	release := func() { releaseOnce.Do(func() { close(releaseReport) }) }
	ss.serviceReportValidatedHook = func(serviceID uint64) {
		if serviceID == service.ID {
			close(reportValidated)
			<-releaseReport
		}
	}
	t.Cleanup(func() {
		release()
		ss.Close()
	})

	ss.Dispatch(serviceMonitorResult(1, service.ID, model.TaskTypeTCPPing, true))
	select {
	case <-reportValidated:
	case <-ctx.Done():
		t.Fatal(ctx.Err())
	}

	deleteDone := make(chan struct{})
	go func() {
		ServerShared.Delete([]uint64{1})
		close(deleteDone)
	}()
	select {
	case <-deleteDone:
		t.Fatal("server deletion returned before the accepted report completed")
	case <-time.After(25 * time.Millisecond):
	}

	release()
	select {
	case <-deleteDone:
	case <-ctx.Done():
		t.Fatal(ctx.Err())
	}

	var historyCount int64
	if err := DB.Model(&model.ServiceHistory{}).
		Where("service_id = ? AND server_id = ?", service.ID, 1).
		Count(&historyCount).Error; err != nil {
		t.Fatal(err)
	}
	if historyCount != 1 {
		t.Fatalf("expected report side effects before deletion returned, got %d history rows", historyCount)
	}
	if _, ok := ServerShared.Get(1); ok {
		t.Fatal("expected reporter to be deleted after the report completed")
	}
}

func TestServiceSentinelWorkerRejectsReportAfterReporterDeletion(t *testing.T) {
	ss := newServiceMonitorSecurityHarness(t,
		&model.Server{Common: model.Common{ID: 1, UserID: 1}, Name: "reporter"},
	)
	service := &model.Service{
		Common:      model.Common{ID: 10, UserID: 1},
		Name:        "deleted-reporter-service",
		Type:        model.TaskTypeTCPPing,
		Target:      "deleted-reporter.example.invalid:443",
		Duration:    3600,
		Cover:       model.ServiceCoverIgnoreAll,
		SkipServers: map[uint64]bool{1: true},
	}
	addServiceMonitorSecurityService(t, ss, service)

	ServerShared.Delete([]uint64{1})
	ss.Dispatch(serviceMonitorResult(1, service.ID, model.TaskTypeTCPPing, true))
	ss.Close()

	var historyCount int64
	if err := DB.Model(&model.ServiceHistory{}).
		Where("service_id = ? AND server_id = ?", service.ID, 1).
		Count(&historyCount).Error; err != nil {
		t.Fatal(err)
	}
	if historyCount != 0 {
		t.Fatalf("expected no history after reporter deletion, got %d rows", historyCount)
	}
	ss.serviceResponseDataStoreLock.RLock()
	_, pingCached := ss.serviceResponsePing[service.ID]
	_, responseCached := ss.serviceResponseDataStore[service.ID]
	stats := ss.serviceStatusToday[service.ID]
	ss.serviceResponseDataStoreLock.RUnlock()
	if pingCached {
		t.Fatal("expected no ping cache side effect after reporter deletion")
	}
	if responseCached {
		t.Fatal("expected no response cache side effect after reporter deletion")
	}
	if stats == nil || stats.Up != 0 || stats.Down != 0 {
		t.Fatalf("expected no stats side effect after reporter deletion, got %+v", stats)
	}
}

func TestServiceSentinelWorkerRecoversPerReportPanic(t *testing.T) {
	ctx, cancel := context.WithTimeout(t.Context(), time.Second)
	defer cancel()
	ss := newServiceMonitorSecurityHarness(t,
		&model.Server{Common: model.Common{ID: 1, UserID: 1}, Name: "reporter"},
	)
	panicService := &model.Service{
		Common:      model.Common{ID: 10, UserID: 1},
		Name:        "panic-service",
		Type:        model.TaskTypeHTTPGet,
		Target:      "https://panic.example.invalid",
		Duration:    3600,
		Cover:       model.ServiceCoverIgnoreAll,
		SkipServers: map[uint64]bool{1: true},
	}
	validService := &model.Service{
		Common:      model.Common{ID: 20, UserID: 1},
		Name:        "valid-service",
		Type:        model.TaskTypeTCPPing,
		Target:      "valid.example.invalid:443",
		Duration:    3600,
		Cover:       model.ServiceCoverIgnoreAll,
		SkipServers: map[uint64]bool{1: true},
	}
	addServiceMonitorSecurityService(t, ss, panicService)
	addServiceMonitorSecurityService(t, ss, validService)
	ss.serviceReportBeforeTLSSideEffectsHook = func(serviceID uint64) {
		if serviceID == panicService.ID {
			panic("test service report panic")
		}
	}

	ss.Dispatch(serviceMonitorResult(1, panicService.ID, model.TaskTypeHTTPGet, true))
	ss.Dispatch(serviceMonitorResult(1, validService.ID, model.TaskTypeTCPPing, true))
	waitForServiceHistory(t, validService.ID, 1)
	ss.Close()
	if !ss.serviceResponseDataStoreLock.TryLock() {
		t.Fatal("panic leaked the service response lock")
	}
	ss.serviceResponseDataStoreLock.Unlock()

	deleteDone := make(chan struct{})
	go func() {
		ServerShared.Delete([]uint64{1})
		close(deleteDone)
	}()
	select {
	case <-deleteDone:
	case <-ctx.Done():
		t.Fatal("panic leaked a lifecycle lock: " + ctx.Err().Error())
	}
}

func TestServiceSentinelWorkerIgnoresStaleReportAfterDeletion(t *testing.T) {
	if os.Getenv("NEZHA_SERVICE_SENTINEL_LIFECYCLE_CHILD") == "1" {
		testServiceSentinelWorkerIgnoresStaleReportAfterDeletionChild(t)
		return
	}

	// Given
	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
	defer cancel()
	child := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestServiceSentinelWorkerIgnoresStaleReportAfterDeletion$")
	child.Env = append(os.Environ(), "NEZHA_SERVICE_SENTINEL_LIFECYCLE_CHILD=1")

	// When
	output, err := child.CombinedOutput()

	// Then
	if ctx.Err() != nil {
		t.Fatalf("service sentinel lifecycle child timed out: %v\n%s", ctx.Err(), output)
	}
	if err != nil {
		t.Fatalf("service sentinel lifecycle child failed: %v\n%s", err, output)
	}
	if !strings.Contains(string(output), serviceSentinelLifecycleSuccessMarker) {
		t.Fatalf("service sentinel lifecycle child did not report success:\n%s", output)
	}
}

func testServiceSentinelWorkerIgnoresStaleReportAfterDeletionChild(t *testing.T) {
	// Given
	ctx, cancel := context.WithTimeout(t.Context(), time.Second)
	defer cancel()
	ss := newServiceMonitorSecurityHarness(t,
		&model.Server{Common: model.Common{ID: 1, UserID: 1}, Name: "reporter"},
	)
	for _, service := range []*model.Service{
		{
			Common:      model.Common{ID: 10, UserID: 1},
			Name:        "stale-service",
			Type:        model.TaskTypeTCPPing,
			Target:      "stale.example.invalid:443",
			Duration:    3600,
			Cover:       model.ServiceCoverIgnoreAll,
			SkipServers: map[uint64]bool{1: true},
		},
		{
			Common:      model.Common{ID: 20, UserID: 1},
			Name:        "valid-service",
			Type:        model.TaskTypeTCPPing,
			Target:      "valid.example.invalid:443",
			Duration:    3600,
			Cover:       model.ServiceCoverIgnoreAll,
			SkipServers: map[uint64]bool{1: true},
		},
	} {
		addServiceMonitorSecurityService(t, ss, service)
	}
	acceptedStaleReport := make(chan struct{})
	releaseWorker := make(chan struct{})
	var releaseOnce sync.Once
	releaseWorkerHook := func() {
		releaseOnce.Do(func() { close(releaseWorker) })
	}
	ss.serviceReportValidatedHook = func(serviceID uint64) {
		if serviceID == 10 {
			close(acceptedStaleReport)
			<-releaseWorker
		}
	}
	t.Cleanup(func() {
		releaseWorkerHook()
		ss.Close()
	})

	// When
	ss.Dispatch(serviceMonitorResult(1, 10, model.TaskTypeTCPPing, true))
	select {
	case <-acceptedStaleReport:
	case <-ctx.Done():
		t.Fatal(ctx.Err())
	}
	ss.Delete([]uint64{10})
	releaseWorkerHook()
	ss.Dispatch(serviceMonitorResult(1, 20, model.TaskTypeTCPPing, true))
	ss.Close()

	// Then
	var staleHistoryCount int64
	if err := DB.Model(&model.ServiceHistory{}).
		Where("service_id = ? AND server_id = ?", 10, 1).
		Count(&staleHistoryCount).Error; err != nil {
		t.Fatal(err)
	}
	if staleHistoryCount != 0 {
		t.Fatalf("expected stale service to write zero per-reporter history rows, got %d", staleHistoryCount)
	}
	var validHistoryCount int64
	if err := DB.Model(&model.ServiceHistory{}).
		Where("service_id = ? AND server_id = ?", 20, 1).
		Count(&validHistoryCount).Error; err != nil {
		t.Fatal(err)
	}
	if validHistoryCount != 1 {
		t.Fatalf("expected exactly one valid service history row, got %d", validHistoryCount)
	}
	ss.serviceResponseDataStoreLock.RLock()
	_, stalePingCached := ss.serviceResponsePing[10]
	validStats := ss.serviceStatusToday[20]
	ss.serviceResponseDataStoreLock.RUnlock()
	if stalePingCached {
		t.Fatal("expected stale service ping cache to be deleted")
	}
	if validStats == nil || validStats.Up != 1 || validStats.Down != 0 {
		t.Fatalf("expected valid service stats up=1 down=0, got %+v", validStats)
	}
	if _, err := fmt.Fprintln(os.Stdout, serviceSentinelLifecycleSuccessMarker); err != nil {
		t.Fatal(err)
	}
}

func TestServiceSentinelWorkerRevalidatesReportAfterUpdate(t *testing.T) {
	// Given
	ctx, cancel := context.WithTimeout(t.Context(), time.Second)
	defer cancel()
	ss := newServiceMonitorSecurityHarness(t,
		&model.Server{Common: model.Common{ID: 1, UserID: 1}, Name: "reporter"},
	)
	service := &model.Service{
		Common:      model.Common{ID: 10, UserID: 1},
		Name:        "updatable-service",
		Type:        model.TaskTypeTCPPing,
		Target:      "updatable.example.invalid:443",
		Duration:    3600,
		Cover:       model.ServiceCoverIgnoreAll,
		SkipServers: map[uint64]bool{1: true},
	}
	addServiceMonitorSecurityService(t, ss, service)
	ss.serviceResponseDataStoreLock.Lock()
	ss.serviceStatusToday[service.ID] = &_TodayStatsOfService{Up: 7, Down: 3, Delay: 12.5}
	ss.serviceResponseDataStoreLock.Unlock()
	acceptedReport := make(chan struct{})
	releaseWorker := make(chan struct{})
	var releaseOnce sync.Once
	releaseWorkerHook := func() {
		releaseOnce.Do(func() { close(releaseWorker) })
	}
	ss.serviceReportValidatedHook = func(serviceID uint64) {
		if serviceID == service.ID {
			close(acceptedReport)
			<-releaseWorker
		}
	}
	t.Cleanup(func() {
		releaseWorkerHook()
		ss.Close()
	})

	// When
	ss.Dispatch(serviceMonitorResult(1, service.ID, model.TaskTypeTCPPing, true))
	select {
	case <-acceptedReport:
	case <-ctx.Done():
		t.Fatal(ctx.Err())
	}
	updatedService := *service
	updatedService.Name = "updated-service"
	updatedService.SkipServers = map[uint64]bool{}
	if err := ss.Update(&updatedService); err != nil {
		t.Fatal(err)
	}
	releaseWorkerHook()
	ss.Close()

	// Then
	var historyCount int64
	if err := DB.Model(&model.ServiceHistory{}).
		Where("service_id = ? AND server_id = ?", service.ID, 1).
		Count(&historyCount).Error; err != nil {
		t.Fatal(err)
	}
	if historyCount != 0 {
		t.Fatalf("expected updated service to write zero per-reporter history rows, got %d", historyCount)
	}
	ss.serviceResponseDataStoreLock.RLock()
	_, pingCached := ss.serviceResponsePing[service.ID]
	stats := ss.serviceStatusToday[service.ID]
	ss.serviceResponseDataStoreLock.RUnlock()
	if pingCached {
		t.Fatal("expected updated service report to leave no ping cache entry")
	}
	if stats == nil || stats.Up != 7 || stats.Down != 3 || stats.Delay != 12.5 {
		t.Fatalf("expected existing service stats to remain unchanged, got %+v", stats)
	}
	currentService, ok := ss.Get(service.ID)
	if !ok || currentService.Name != updatedService.Name || currentService.SkipServers[1] {
		t.Fatalf("expected updated service configuration, got %+v", currentService)
	}
}

func TestServiceSentinelLoadStatsFollowsLifecycleLockOrder(t *testing.T) {
	// Given
	ss := &ServiceSentinel{
		serviceStatusToday:       make(map[uint64]*_TodayStatsOfService),
		serviceResponseDataStore: make(map[uint64]serviceResponseData),
		services:                 make(map[uint64]*model.Service),
		monthlyStatus:            make(map[uint64]*serviceResponseItem),
	}
	ss.loadStatsResponseLockedHook = func() {
		if ss.serviceResponseDataStoreLock.TryLock() {
			ss.serviceResponseDataStoreLock.Unlock()
			t.Fatal("LoadStats invoked the hook before acquiring the response read lock")
		}
		if !ss.monthlyStatusLock.TryLock() {
			t.Fatal("LoadStats acquired monthlyStatusLock before the response lock hook")
		}
		ss.monthlyStatusLock.Unlock()
		if !ss.servicesLock.TryLock() {
			t.Fatal("LoadStats acquired servicesLock before the response lock hook")
		}
		ss.servicesLock.Unlock()
	}

	// When / Then
	ss.LoadStats()
}

func TestServiceSentinelWorkerHoldsResponseLockDuringTLSSideEffects(t *testing.T) {
	// Given
	ctx, cancel := context.WithTimeout(t.Context(), time.Second)
	defer cancel()
	ss := newServiceMonitorSecurityHarness(t,
		&model.Server{Common: model.Common{ID: 1, UserID: 1}, Name: "reporter"},
	)
	service := &model.Service{
		Common:      model.Common{ID: 10, UserID: 1},
		Name:        "tls-service",
		Type:        model.TaskTypeHTTPGet,
		Target:      "https://tls.example.invalid",
		Duration:    3600,
		Cover:       model.ServiceCoverIgnoreAll,
		SkipServers: map[uint64]bool{1: true},
	}
	addServiceMonitorSecurityService(t, ss, service)
	tlsSideEffectsReady := make(chan struct{})
	releaseTLSSideEffects := make(chan struct{})
	var releaseOnce sync.Once
	releaseTLSSideEffectsHook := func() {
		releaseOnce.Do(func() { close(releaseTLSSideEffects) })
	}
	ss.serviceReportBeforeTLSSideEffectsHook = func(serviceID uint64) {
		if serviceID == service.ID {
			close(tlsSideEffectsReady)
			<-releaseTLSSideEffects
		}
	}
	t.Cleanup(func() {
		releaseTLSSideEffectsHook()
		ss.Close()
	})
	report := serviceMonitorResult(1, service.ID, model.TaskTypeHTTPGet, true)
	report.Data.Data = "issuer|2030-01-02 15:04:05 +0000 UTC"

	// When
	ss.Dispatch(report)
	select {
	case <-tlsSideEffectsReady:
	case <-ctx.Done():
		t.Fatal(ctx.Err())
	}
	responseLockAcquired := ss.serviceResponseDataStoreLock.TryLock()
	if responseLockAcquired {
		ss.serviceResponseDataStoreLock.Unlock()
		t.Fatal("worker released the response lock before TLS side effects")
	}
	releaseTLSSideEffectsHook()
	ss.Close()

	// Then
	ss.serviceResponseDataStoreLock.RLock()
	cachedCertificate := ss.tlsCertCache[service.ID]
	ss.serviceResponseDataStoreLock.RUnlock()
	if cachedCertificate != report.Data.Data {
		t.Fatalf("expected TLS cache %q, got %q", report.Data.Data, cachedCertificate)
	}
}

// TestServiceSentinelWorkerSurvivesConcurrentReporterServerDelete is a
// regression test for GHSA-jx78-55p5-rwv5 Finding 1 (incomplete fix of
// GHSA-qjpp-gffx-2wm9).
//
// The vulnerability: after the 2026-07-21 fix, the worker re-validates the
// service under serviceResponseDataStoreLock, but then takes a fresh snapshot
// m := ServerShared.GetList() with no guard. A concurrent batch-delete of the
// reporter's own server removes it between the pre-lock validation and the
// GetList call, so m[r.Reporter] is nil.  delayCheck and notifyCheck then
// dereference m[r.Reporter].Name unconditionally — SIGSEGV.
//
// The subprocess-isolation pattern is used because the pre-fix code path
// panicked (nil pointer dereference in an unrecovered goroutine), which would
// crash the whole test binary rather than simply failing a single test.
func TestServiceSentinelWorkerSurvivesConcurrentReporterServerDelete(t *testing.T) {
	if os.Getenv("NEZHA_SENTINEL_CONCURRENT_DELETE_CHILD") == "1" {
		testServiceSentinelWorkerSurvivesConcurrentReporterServerDeleteChild(t)
		return
	}

	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
	defer cancel()
	child := exec.CommandContext(ctx, os.Args[0],
		"-test.run=^TestServiceSentinelWorkerSurvivesConcurrentReporterServerDelete$",
		"-test.v",
	)
	child.Env = append(os.Environ(), "NEZHA_SENTINEL_CONCURRENT_DELETE_CHILD=1")

	output, err := child.CombinedOutput()
	if ctx.Err() != nil {
		t.Fatalf("child process timed out: %v\n%s", ctx.Err(), output)
	}
	if err != nil {
		t.Fatalf("child process crashed (likely nil deref in delayCheck/notifyCheck): %v\n%s", err, output)
	}
	if !strings.Contains(string(output), concurrentServerDeleteSuccessMarker) {
		t.Fatalf("child did not print success marker:\n%s", output)
	}
}

func testServiceSentinelWorkerSurvivesConcurrentReporterServerDeleteChild(t *testing.T) {
	// Given: a reporter server and a service with latency-alerting enabled so
	// that delayCheck (the vulnerable sink at line 785) is exercised on every
	// dispatch.  MaxLatency=1 ensures delay=12 always exceeds the threshold and
	// the notification branch (not just the mute-clear branch) is taken.
	ss := newServiceMonitorSecurityHarness(t,
		&model.Server{Common: model.Common{ID: 1, UserID: 1}, Name: "reporter"},
	)
	service := &model.Service{
		Common:        model.Common{ID: 10, UserID: 1},
		Name:          "latency-service",
		Type:          model.TaskTypeTCPPing,
		Target:        "example.invalid:443",
		Duration:      3600,
		Cover:         model.ServiceCoverIgnoreAll,
		SkipServers:   map[uint64]bool{1: true},
		LatencyNotify: true,
		MaxLatency:    1,
	}
	addServiceMonitorSecurityService(t, ss, service)

	reportProcessing := make(chan struct{})
	releaseWorker := make(chan struct{})
	var releaseOnce sync.Once
	releaseWorkerFn := func() { releaseOnce.Do(func() { close(releaseWorker) }) }

	// serviceReportValidatedHook runs while the report holds the lifecycle read
	// lock. Deletion must therefore run in another goroutine and wait until this
	// hook releases; attempting Delete here would try to upgrade the RWMutex.
	ss.serviceReportValidatedHook = func(serviceID uint64) {
		if serviceID == service.ID {
			close(reportProcessing)
			<-releaseWorker
		}
	}
	t.Cleanup(func() {
		releaseWorkerFn()
		ss.Close()
	})

	// When
	ss.Dispatch(serviceMonitorResult(1, service.ID, model.TaskTypeTCPPing, true))
	select {
	case <-reportProcessing:
	case <-t.Context().Done():
		t.Fatal(t.Context().Err())
	}
	deleteDone := make(chan struct{})
	go func() {
		ServerShared.Delete([]uint64{1})
		close(deleteDone)
	}()
	releaseWorkerFn()
	select {
	case <-deleteDone:
	case <-t.Context().Done():
		t.Fatal(t.Context().Err())
	}
	ss.Close()

	// Then: no crash; the worker handled the nil reporter gracefully.
	if _, err := fmt.Fprintln(os.Stdout, concurrentServerDeleteSuccessMarker); err != nil {
		t.Fatal(err)
	}
}

// TestServiceSentinelDeleteWithUnknownIDDoesNotLeaveZombies is a regression
// test for GHSA-jx78-55p5-rwv5 Finding 2 (low severity).
//
// The vulnerability: ServiceSentinel.Delete iterates the caller-supplied id
// slice and does CronShared.Remove(ss.services[id].CronJobID) without checking
// whether id is present in ss.services.  CheckPermission returns vacuously
// true for unknown ids, so the controller layer cannot block this path.
// ss.services[unknownID] returns nil, and .CronJobID panics.  Because the
// panic aborts the loop, every id ordered AFTER the bogus one is never removed
// from the in-memory registry even though its database row was already deleted,
// producing zombie services that keep dispatching cron probes.
func TestServiceSentinelDeleteWithUnknownIDDoesNotLeaveZombies(t *testing.T) {
	// Given: one legitimate service (ID 10) registered in the sentinel.
	ss := newServiceMonitorSecurityHarness(t,
		&model.Server{Common: model.Common{ID: 1, UserID: 1}, Name: "reporter"},
	)
	service := &model.Service{
		Common:      model.Common{ID: 10, UserID: 1},
		Name:        "real-service",
		Type:        model.TaskTypeTCPPing,
		Target:      "example.invalid:443",
		Duration:    3600,
		Cover:       model.ServiceCoverIgnoreAll,
		SkipServers: map[uint64]bool{1: true},
	}
	addServiceMonitorSecurityService(t, ss, service)

	// When: Delete is called with a bogus ID first, then the real service ID.
	// Before the fix this panicked on ss.services[99999].CronJobID and left
	// service 10 as a zombie.
	ss.Delete([]uint64{99999, service.ID})

	// Then: the real service must be fully removed from every in-memory map.
	ss.serviceResponseDataStoreLock.RLock()
	_, todayPresent := ss.serviceStatusToday[service.ID]
	_, pingPresent := ss.serviceResponsePing[service.ID]
	ss.serviceResponseDataStoreLock.RUnlock()

	ss.servicesLock.RLock()
	_, servicePresent := ss.services[service.ID]
	ss.servicesLock.RUnlock()

	ss.monthlyStatusLock.Lock()
	_, monthlyPresent := ss.monthlyStatus[service.ID]
	ss.monthlyStatusLock.Unlock()

	if todayPresent {
		t.Error("zombie: serviceStatusToday still contains the deleted service")
	}
	if pingPresent {
		t.Error("zombie: serviceResponsePing still contains the deleted service")
	}
	if servicePresent {
		t.Error("zombie: services map still contains the deleted service")
	}
	if monthlyPresent {
		t.Error("zombie: monthlyStatus still contains the deleted service")
	}

	if _, err := fmt.Fprintln(os.Stdout, deleteUnknownIDSuccessMarker); err != nil {
		t.Fatal(err)
	}
}