Merge upstream monitoring features preserving domain extensions and require fixed Go toolchain

Ignore history rows without initialized daily or monthly state so deleted and quarantined services cannot panic dashboard startup. Cover startup and statistics preservation with regression tests.

Thanks to Zachariah Mithani (@zachariah-mithani, AploSec) for privately reporting the retained service history startup panic.

Co-authored-by: naiba/CloudCode <hi+cloudcode@nai.ba>

.github/workflows/release.yml

@@ -52,7 +52,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
-          go-version: "1.26.x"
+          go-version: "^1.26.10"
 
       - name: generate swagger docs
         run: |

.github/workflows/test.yml

@@ -35,7 +35,7 @@ jobs:
           persist-credentials: false
       - uses: actions/setup-go@v7
         with:
-          go-version: "1.26.x"
+          go-version: "^1.26.10"
           cache: false
 
       - name: Generate Swagger docs
@@ -61,7 +61,7 @@ jobs:
           persist-credentials: false
       - uses: actions/setup-go@v7
         with:
-          go-version: "1.26.x"
+          go-version: "^1.26.10"
           cache: false
 
       - name: Generate Swagger docs
@@ -113,7 +113,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
-          go-version: "1.26.x"
+          go-version: "^1.26.10"
           cache: false
       - name: Prepare Dashboard build inputs
         working-directory: nezha

SECURITY.md

@@ -6,4 +6,4 @@ Code in `master` branch.
 
 ## Reporting a Vulnerability
 
-Thank you for your contribution to open source security, please submit on https://github.com/nezhahq/nezha/security
+Thank you for your contribution to open source security. Please report vulnerabilities privately at https://github.com/nezhahq/nezha/security or by email to [hi@nai.ba](mailto:hi@nai.ba).

cmd/dashboard/controller/alertrule.go

@@ -172,6 +172,15 @@ func validateRule(c *gin.Context, r *model.AlertRule) error {
 	}
 	if len(r.Rules) > 0 {
 		for _, rule := range r.Rules {
+			if rule == nil {
+				return singleton.Localizer.ErrorT("rule is not set")
+			}
+			if !rule.IsSupportedType() {
+				return singleton.Localizer.ErrorT("unsupported rule type")
+			}
+			if !rule.HasSafeThresholds() {
+				return singleton.Localizer.ErrorT("invalid rule thresholds")
+			}
 			switch rule.Cover {
 			case model.RuleCoverAll, model.RuleCoverIgnoreAll:
 			default:
@@ -182,10 +191,19 @@ func validateRule(c *gin.Context, r *model.AlertRule) error {
 				if rule.Duration < 3 {
 					return singleton.Localizer.ErrorT("duration need to be at least 3")
 				}
+				if rule.Duration > model.MaxAlertRuleDuration {
+					return singleton.Localizer.ErrorT("duration is too large")
+				}
 			} else {
+				if !rule.HasSafeCycleUnit() {
+					return singleton.Localizer.ErrorT("invalid cycle unit")
+				}
 				if rule.CycleInterval < 1 {
 					return singleton.Localizer.ErrorT("cycle_interval need to be at least 1")
 				}
+				if rule.CycleInterval > model.MaxAlertRuleCycleInterval {
+					return singleton.Localizer.ErrorT("cycle_interval is too large")
+				}
 				if rule.CycleStart == nil {
 					return singleton.Localizer.ErrorT("cycle_start is not set")
 				}

cmd/dashboard/controller/alertrule_security_test.go

@@ -0,0 +1,67 @@
+package controller
+
+import (
+	"math"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/nezhahq/nezha/model"
+)
+
+func TestValidateRuleRejectsCrashPayloadsFromMember(t *testing.T) {
+	ctx := newMemberValidationContext(t)
+	cycleStart := time.Now().Add(-time.Hour)
+	tests := []struct {
+		name string
+		rule *model.Rule
+	}{
+		{
+			name: "unknown rule type",
+			rule: &model.Rule{Type: "attacker_controlled", Duration: 3, Cover: model.RuleCoverAll},
+		},
+		{
+			name: "duration overflows int",
+			rule: &model.Rule{Type: "offline", Duration: math.MaxUint64, Cover: model.RuleCoverAll},
+		},
+		{
+			name: "cycle interval overflows int",
+			rule: &model.Rule{
+				Type:          "transfer_in_cycle",
+				CycleStart:    &cycleStart,
+				CycleInterval: math.MaxUint64,
+				Cover:         model.RuleCoverAll,
+			},
+		},
+		{
+			name: "nil rule",
+			rule: nil,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			alert := &model.AlertRule{
+				Common: model.Common{UserID: 200},
+				Name:   "security regression",
+				Rules:  []*model.Rule{test.rule},
+			}
+			require.Error(t, validateRule(ctx, alert))
+		})
+	}
+}
+
+func TestValidateRuleAcceptsSafeDurationBoundary(t *testing.T) {
+	ctx := newMemberValidationContext(t)
+	alert := &model.AlertRule{
+		Common: model.Common{UserID: 200},
+		Name:   "duration boundary",
+		Rules: []*model.Rule{{
+			Type:     "offline",
+			Duration: model.MaxAlertRuleDuration,
+			Cover:    model.RuleCoverAll,
+		}},
+	}
+	require.NoError(t, validateRule(ctx, alert))
+}

cmd/dashboard/controller/controller.go

@@ -57,6 +57,9 @@ func routers(r *gin.Engine, frontendDist fs.FS) {
 	if err := authMiddleware.MiddlewareInit(); err != nil {
 		log.Fatal("authMiddleware.MiddlewareInit Error:" + err.Error())
 	}
+	// Signature-only view of the auth middleware for IsDashboardCredential:
+	// same key and pinned algorithm, claims validation deliberately skipped.
+	dashboardCredentialJWTParser = newDashboardCredentialJWTParser(authMiddleware)
 	// /mcp — Model Context Protocol endpoint, authenticated by PAT only (闸 1 + 闸 2)。
 	// 不放在 /api/v1 下：MCP client 配置 URL 更短，且 MCP transport 协议演进与 REST API
 	// 解耦。鉴权一律走 apiTokenAuthMiddleware；不接受 JWT 以避免浏览器误触。
@@ -101,6 +104,7 @@ func routers(r *gin.Engine, frontendDist fs.FS) {
 	optionalAuth.GET("/service/:id/history", restScopeMiddleware(model.ScopeServiceRead), commonHandler(getServiceHistory))
 	optionalAuth.GET("/server/:id/service", restScopeMiddleware(model.ScopeServiceRead), commonHandler(listServerServices))
 	optionalAuth.GET("/server/:id/metrics", restScopeMiddleware(model.ScopeServerRead), commonHandler(getServerMetrics))
+	optionalAuth.GET("/server/:id/online-history", restScopeMiddleware(model.ScopeServerRead), commonHandler(getServerOnlineHistory))
 
 	// CSRF middleware applies group-wide. Safe methods short-circuit and
 	// PAT bearer requests bypass — so the only callers gated are

cmd/dashboard/controller/credential_redaction_test.go

@@ -109,12 +109,14 @@ func TestListDDNS_RedactsCredentials(t *testing.T) {
 	defer setupTenancyTest(t)()
 
 	p := model.DDNSProfile{
-		Common:         model.Common{UserID: 10},
-		Name:           "cf",
-		Provider:       "cloudflare",
-		AccessID:       "id",
-		AccessSecret:   "super-secret-token",
-		WebhookHeaders: `{"Authorization":"Bearer xxx"}`,
+		Common:             model.Common{UserID: 10},
+		Name:               "cf",
+		Provider:           "cloudflare",
+		AccessID:           "id",
+		AccessSecret:       "super-secret-token",
+		WebhookURL:         "https://ddns.example/update?token=embedded-secret",
+		WebhookRequestBody: `{"token":"embedded-secret"}`,
+		WebhookHeaders:     `{"Authorization":"Bearer xxx"}`,
 	}
 	require.NoError(t, singleton.DB.Create(&p).Error)
 	singleton.DDNSShared.InsertForTest(&p)
@@ -124,12 +126,18 @@ func TestListDDNS_RedactsCredentials(t *testing.T) {
 	require.NoError(t, err)
 	require.Len(t, out, 1)
 	require.Empty(t, out[0].AccessSecret, "access_secret must be redacted in list response")
+	require.Empty(t, out[0].WebhookURL, "webhook_url must be redacted in list response")
+	require.Empty(t, out[0].WebhookRequestBody, "webhook_request_body must be redacted in list response")
 	require.Empty(t, out[0].WebhookHeaders, "webhook_headers must be redacted in list response")
 	require.Equal(t, "id", out[0].AccessID, "non-secret fields must be preserved")
 
 	var stored model.DDNSProfile
 	require.NoError(t, singleton.DB.First(&stored, p.ID).Error)
 	require.Equal(t, "super-secret-token", stored.AccessSecret, "redaction must not mutate stored data")
+	require.Equal(t, "https://ddns.example/update?token=embedded-secret", stored.WebhookURL,
+		"redaction must not mutate stored webhook URL")
+	require.Equal(t, `{"token":"embedded-secret"}`, stored.WebhookRequestBody,
+		"redaction must not mutate stored webhook body")
 }
 
 func TestListNotification_RedactsCredentials(t *testing.T) {
@@ -164,14 +172,15 @@ func TestUpdateDDNS_EmptySecretPreservesStored(t *testing.T) {
 	defer setupTenancyTest(t)()
 
 	existing := model.DDNSProfile{
-		Common:         model.Common{UserID: 10},
-		Name:           "cf",
-		Provider:       "webhook",
-		AccessID:       "id",
-		AccessSecret:   "keep-me",
-		WebhookURL:     "http://127.0.0.1/",
-		WebhookMethod:  1,
-		WebhookHeaders: `{"X-Token":"keep-header"}`,
+		Common:             model.Common{UserID: 10},
+		Name:               "cf",
+		Provider:           "webhook",
+		AccessID:           "id",
+		AccessSecret:       "keep-me",
+		WebhookURL:         "http://127.0.0.1/",
+		WebhookRequestBody: `{"X-Token":"keep-body"}`,
+		WebhookMethod:      1,
+		WebhookHeaders:     `{"X-Token":"keep-header"}`,
 	}
 	require.NoError(t, singleton.DB.Create(&existing).Error)
 	singleton.DDNSShared.InsertForTest(&existing)
@@ -181,10 +190,11 @@ func TestUpdateDDNS_EmptySecretPreservesStored(t *testing.T) {
 		"provider":             "webhook",
 		"access_id":            "id",
 		"access_secret":        "",
-		"webhook_url":          "http://127.0.0.1/",
+		"webhook_url":          "",
 		"webhook_method":       1,
 		"webhook_request_type": 1,
 		"webhook_headers":      "",
+		"webhook_request_body": "",
 		"max_retries":          3,
 	})
 	c.Params = gin.Params{{Key: "id", Value: itoa(existing.ID)}}
@@ -197,6 +207,10 @@ func TestUpdateDDNS_EmptySecretPreservesStored(t *testing.T) {
 	require.Equal(t, "keep-me", after.AccessSecret, "empty submitted secret must preserve stored value")
 	require.Equal(t, `{"X-Token":"keep-header"}`, after.WebhookHeaders,
 		"empty submitted headers must preserve stored value")
+	require.Equal(t, "http://127.0.0.1/", after.WebhookURL,
+		"empty submitted webhook URL must preserve stored value")
+	require.Equal(t, `{"X-Token":"keep-body"}`, after.WebhookRequestBody,
+		"empty submitted webhook body must preserve stored value")
 }
 
 func TestUpdateDDNS_NonEmptySecretOverwrites(t *testing.T) {

cmd/dashboard/controller/dashboard_credential.go

@@ -0,0 +1,162 @@
+package controller
+
+import (
+	"context"
+	"log"
+	"strings"
+
+	jwt "github.com/golang-jwt/jwt/v4"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+
+	ginjwt "github.com/appleboy/gin-jwt/v2"
+	"github.com/nezhahq/nezha/model"
+	"github.com/nezhahq/nezha/service/singleton"
+)
+
+// dashboardCredentialJWTParser verifies that a JWT was signed by this
+// dashboard. It is derived from the auth middleware created in routers() and
+// exists only for dashboard-credential classification; it must never
+// authenticate dashboard requests on its own.
+var dashboardCredentialJWTParser *ginjwt.GinJWTMiddleware
+
+// newDashboardCredentialJWTParser derives a signature-only parser from the
+// dashboard auth middleware. It keeps the exact key value and the pinned HS256
+// signing algorithm of initParams() — there is no second read of
+// singleton.Conf.JWTSecretKey — and disables claims validation so expiry does
+// not change the verdict: an expired token is still proof the dashboard issued
+// it, and keeping expired dashboard tokens out of the NAT tunnel prevents a
+// careless backend from treating "any JWT" as its own credential. The live
+// auth middleware keeps full claims validation; dashboard authentication is
+// not relaxed by this parser in any way.
+func newDashboardCredentialJWTParser(authMiddleware *ginjwt.GinJWTMiddleware) *ginjwt.GinJWTMiddleware {
+	parser := *authMiddleware
+	parser.ParseOptions = []jwt.ParserOption{jwt.WithoutClaimsValidation()}
+	return &parser
+}
+
+// IsDashboardCredential reports whether an Authorization header value was
+// issued by this dashboard: a panel-signed JWT or a panel API token (PAT).
+//
+// NAT ingress uses this to enforce the tunnel credential invariant: values
+// classified as dashboard-issued are stripped before the request is tunneled
+// to the agent, while foreign credentials are forwarded untouched because the
+// NAT backend may authenticate with credentials of its own.
+//
+// Decision policy mirrors the dashboard's own auth middlewares: only the
+// case-sensitive "Bearer " scheme is considered; the value after the scheme is
+// classified by IsDashboardCredentialValue, which is also the entry point for
+// the panel's scheme-less TokenLookup channels. Only a deterministic proof
+// that the value was not issued by this dashboard (scheme mismatch, unknown
+// PAT hash, failed signature verification) returns false.
+func IsDashboardCredential(authz string) bool {
+	raw := strings.TrimSpace(authz)
+	if !strings.HasPrefix(raw, "Bearer ") {
+		return false
+	}
+	return IsDashboardCredentialValue(strings.TrimPrefix(raw, "Bearer "))
+}
+
+// IsDashboardCredentialValue reports whether a raw credential value — the part
+// after the "Bearer " scheme, exactly as the panel's other TokenLookup
+// channels carry it (the ?token= query parameter and the nz-jwt cookie, see
+// initParams) — was issued by this dashboard: a panel-signed JWT or a panel
+// API token (PAT).
+//
+// Failure policy is identical to IsDashboardCredential: whenever the
+// classification cannot be completed (parser or config not ready, database
+// failure) the value is conservatively reported as a dashboard credential so
+// the caller strips it; only a deterministic proof that the value was not
+// issued by this dashboard (unknown PAT hash, failed signature verification)
+// returns false.
+func IsDashboardCredentialValue(value string) bool {
+	classified := ClassifyDashboardCredentialValues(context.Background(), []string{value})
+	if len(classified) != 1 {
+		return true
+	}
+	return classified[0]
+}
+
+// ClassifyDashboardCredentialValues classifies a bounded request batch. JWTs
+// are checked independently, while all PAT-shaped values are hashed and
+// resolved with one silent IN query. This prevents attacker-controlled NAT
+// query parameters or cookies from amplifying one HTTP request into thousands
+// of SQLite queries and record-not-found log entries.
+func ClassifyDashboardCredentialValues(ctx context.Context, values []string) []bool {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	results := make([]bool, len(values))
+	patIndexes := make(map[string][]int)
+	for i, value := range values {
+		plaintext := strings.TrimSpace(value)
+		if strings.HasPrefix(plaintext, model.APITokenPrefix) {
+			hash := model.HashAPIToken(plaintext)
+			patIndexes[hash] = append(patIndexes[hash], i)
+			continue
+		}
+		results[i] = isDashboardSignedJWT(plaintext)
+	}
+	if len(patIndexes) == 0 {
+		return results
+	}
+	if singleton.DB == nil {
+		logDashboardCredentialUnclassifiable("api token lookup unavailable")
+		for _, indexes := range patIndexes {
+			for _, i := range indexes {
+				results[i] = true
+			}
+		}
+		return results
+	}
+
+	hashes := make([]string, 0, len(patIndexes))
+	for hash := range patIndexes {
+		hashes = append(hashes, hash)
+	}
+	type tokenHashRow struct {
+		TokenHash string
+	}
+	var rows []tokenHashRow
+	err := singleton.DB.WithContext(ctx).
+		Session(&gorm.Session{Logger: logger.Discard}).
+		Model(&model.APIToken{}).
+		Select("token_hash").
+		Where("token_hash IN ?", hashes).
+		Find(&rows).Error
+	if err != nil {
+		logDashboardCredentialUnclassifiable("api token lookup failed")
+		for _, indexes := range patIndexes {
+			for _, i := range indexes {
+				results[i] = true
+			}
+		}
+		return results
+	}
+	for _, row := range rows {
+		for _, i := range patIndexes[row.TokenHash] {
+			results[i] = true
+		}
+	}
+	return results
+}
+
+// isDashboardSignedJWT verifies the token signature with the dashboard's
+// pinned algorithm and key. Claims validation is disabled on purpose (see
+// newDashboardCredentialJWTParser): expiry does not change who issued the
+// token.
+func isDashboardSignedJWT(token string) bool {
+	if dashboardCredentialJWTParser == nil {
+		// routers() has not run yet (startup incomplete): fail closed.
+		logDashboardCredentialUnclassifiable("dashboard credential parser unavailable")
+		return true
+	}
+	_, err := dashboardCredentialJWTParser.ParseTokenString(token)
+	return err == nil
+}
+
+// logDashboardCredentialUnclassifiable records a classification failure. The
+// credential value itself is never included in the message.
+func logDashboardCredentialUnclassifiable(reason string) {
+	log.Printf("NEZHA>> NAT ingress: credential value could not be classified (%s); stripping it as a precaution", reason)
+}

cmd/dashboard/controller/dashboard_credential_nat_flow_test.go

@@ -0,0 +1,250 @@
+package controller
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/goccy/go-json"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/metadata"
+
+	"github.com/nezhahq/nezha/cmd/dashboard/rpc"
+	"github.com/nezhahq/nezha/model"
+	"github.com/nezhahq/nezha/proto"
+	rpcService "github.com/nezhahq/nezha/service/rpc"
+	"github.com/nezhahq/nezha/service/singleton"
+)
+
+// setupNATFlowTest extends setupDashboardCredentialTest with the minimal
+// ServeNAT environment: a registered server with a task stream and a fresh
+// handler singleton, mirroring the rpc package's serveNAT fixture.
+func setupNATFlowTest(t *testing.T) (*rpcService.NezhaHandler, *model.Server, *natFlowTaskStream) {
+	t.Helper()
+	setupDashboardCredentialTest(t)
+
+	require.NoError(t, singleton.DB.AutoMigrate(&model.Server{}))
+	originalServerShared, originalHandler := singleton.ServerShared, rpcService.NezhaHandlerSingleton
+	t.Cleanup(func() {
+		singleton.ServerShared, rpcService.NezhaHandlerSingleton = originalServerShared, originalHandler
+	})
+
+	serverRow := &model.Server{Common: model.Common{ID: 7}, UUID: "nat-credential-flow-test", Name: "nat-credential-flow-test"}
+	require.NoError(t, singleton.DB.Create(serverRow).Error)
+	singleton.ServerShared = singleton.NewServerClass()
+	handler := rpcService.NewNezhaHandler()
+	rpcService.NezhaHandlerSingleton = handler
+
+	server, ok := singleton.ServerShared.Get(serverRow.ID)
+	require.True(t, ok)
+	taskStream := &natFlowTaskStream{}
+	server.SetTaskStream(taskStream)
+
+	// Wire the gate exactly as cmd/dashboard/main.go does, with the real
+	// classifiers built by setupDashboardCredentialTest.
+	rpc.SetNATDashboardCredentialGate(ClassifyDashboardCredentialValues)
+	t.Cleanup(func() { rpc.SetNATDashboardCredentialGate(nil) })
+
+	return handler, server, taskStream
+}
+
+// TestServeNATWithRealClassifierStripsDashboardCredentialsFromAllChannels pins
+// the main() wiring contract end to end: with the real classifiers wired as
+// main() does, ServeNAT forwards a request whose dashboard-issued credentials
+// — Authorization header, ?token= query parameter, nz-jwt cookie — are all
+// stripped, while foreign values and untouched query/cookie data survive
+// byte-for-byte.
+func TestServeNATWithRealClassifierStripsDashboardCredentialsFromAllChannels(t *testing.T) {
+	handler, server, taskStream := setupNATFlowTest(t)
+
+	dashboardJWT := mintDashboardCredentialJWT(t, time.Hour)
+	foreignJWT := mintForeignJWT(t)
+	// No api_tokens row hashes to this PAT, so the classifier proves it was
+	// not issued by this dashboard and it must survive the tunnel.
+	foreignPAT := model.APITokenPrefix + "foreign-issued-pat"
+	dashboardPAT := model.APITokenPrefix + "dashboard-issued-pat"
+	require.NoError(t, singleton.DB.Create(&model.APIToken{
+		UserID:    1,
+		Name:      "nat-flow",
+		TokenHash: model.HashAPIToken(dashboardPAT),
+	}).Error)
+
+	connection := &natFlowConn{closed: make(chan struct{})}
+	agent := &natFlowAgent{writeDone: make(chan struct{})}
+	writer := &natFlowWriter{conn: connection}
+	request := &http.Request{
+		Method: http.MethodPost,
+		URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + dashboardJWT + "&token=" + url.QueryEscape(foreignJWT) +
+			"&token=" + foreignPAT + "&token=" + dashboardPAT},
+		Header: make(http.Header),
+		Body:   http.NoBody,
+	}
+	request.Header.Set("Authorization", "Bearer "+dashboardJWT)
+	request.Header.Set("Cookie", "sid=abc; nz-jwt="+dashboardJWT+"; nz-jwt="+foreignJWT+"; nz-jwt="+foreignPAT)
+	request.Header.Set("X-Ordinary-Nat", "ordinary")
+	taskStream.onSend = func(task *proto.Task) error {
+		var nat model.TaskNAT
+		if err := json.Unmarshal([]byte(task.Data), &nat); err != nil {
+			return err
+		}
+		return handler.AgentConnected(nat.StreamID, agent)
+	}
+
+	done := make(chan struct{})
+	go func() {
+		rpc.ServeNAT(writer, request, &model.NAT{ServerID: server.ID, Host: "target.example"})
+		close(done)
+	}()
+	select {
+	case <-agent.writeDone:
+	case <-time.After(time.Second):
+		t.Fatal("agent did not receive the transferred NAT request")
+	}
+	require.NoError(t, connection.Close())
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("ServeNAT did not finish")
+	}
+
+	forwarded := string(agent.writtenBytes())
+	// Dashboard credentials are gone from every channel.
+	require.NotContains(t, forwarded, "Authorization:")
+	require.NotContains(t, forwarded, dashboardJWT)
+	require.NotContains(t, forwarded, dashboardPAT)
+	// Foreign credentials and untouched data survive byte-for-byte.
+	require.Contains(t, forwarded, "POST /nat?keep=1&token="+url.QueryEscape(foreignJWT)+"&token="+foreignPAT+" HTTP/1.1")
+	require.Contains(t, forwarded, "Cookie: sid=abc; nz-jwt="+foreignJWT+"; nz-jwt="+foreignPAT)
+	require.Contains(t, forwarded, "X-Ordinary-Nat: ordinary")
+}
+
+// TestServeNATWithRealClassifierKeepsForeignCredentials pins that the real
+// classifier deterministically proves foreign values were not issued by this
+// dashboard and leaves every channel untouched.
+func TestServeNATWithRealClassifierKeepsForeignCredentials(t *testing.T) {
+	handler, server, taskStream := setupNATFlowTest(t)
+
+	foreignJWT := mintForeignJWT(t)
+	connection := &natFlowConn{closed: make(chan struct{})}
+	agent := &natFlowAgent{writeDone: make(chan struct{})}
+	writer := &natFlowWriter{conn: connection}
+	request := &http.Request{
+		Method: http.MethodPost,
+		URL:    &url.URL{Path: "/nat", RawQuery: "token=" + foreignJWT},
+		Header: make(http.Header),
+		Body:   http.NoBody,
+	}
+	request.Header.Set("Authorization", "Bearer "+foreignJWT)
+	request.Header.Set("Cookie", "nz-jwt="+foreignJWT)
+	taskStream.onSend = func(task *proto.Task) error {
+		var nat model.TaskNAT
+		if err := json.Unmarshal([]byte(task.Data), &nat); err != nil {
+			return err
+		}
+		return handler.AgentConnected(nat.StreamID, agent)
+	}
+
+	done := make(chan struct{})
+	go func() {
+		rpc.ServeNAT(writer, request, &model.NAT{ServerID: server.ID, Host: "target.example"})
+		close(done)
+	}()
+	select {
+	case <-agent.writeDone:
+	case <-time.After(time.Second):
+		t.Fatal("agent did not receive the transferred NAT request")
+	}
+	require.NoError(t, connection.Close())
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("ServeNAT did not finish")
+	}
+
+	forwarded := string(agent.writtenBytes())
+	require.Contains(t, forwarded, "POST /nat?token="+foreignJWT+" HTTP/1.1")
+	require.Contains(t, forwarded, "Authorization: Bearer "+foreignJWT)
+	require.Contains(t, forwarded, "Cookie: nz-jwt="+foreignJWT)
+}
+
+type natFlowTaskStream struct {
+	onSend func(*proto.Task) error
+}
+
+func (stream *natFlowTaskStream) Send(task *proto.Task) error {
+	if stream.onSend != nil {
+		return stream.onSend(task)
+	}
+	return nil
+}
+func (*natFlowTaskStream) Recv() (*proto.TaskResult, error) { return nil, io.EOF }
+func (*natFlowTaskStream) SetHeader(metadata.MD) error      { return nil }
+func (*natFlowTaskStream) SendHeader(metadata.MD) error     { return nil }
+func (*natFlowTaskStream) SetTrailer(metadata.MD)           {}
+func (*natFlowTaskStream) Context() context.Context         { return context.Background() }
+func (*natFlowTaskStream) SendMsg(any) error                { return nil }
+func (*natFlowTaskStream) RecvMsg(any) error                { return io.EOF }
+
+type natFlowWriter struct{ conn *natFlowConn }
+
+func (writer *natFlowWriter) Header() http.Header            { return http.Header{} }
+func (writer *natFlowWriter) Write(data []byte) (int, error) { return len(data), nil }
+func (writer *natFlowWriter) WriteHeader(int)                {}
+func (writer *natFlowWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	go func() { _, _ = writer.conn.Read(make([]byte, 1)) }()
+	return writer.conn, bufio.NewReadWriter(bufio.NewReader(writer.conn), bufio.NewWriter(writer.conn)), nil
+}
+
+type natFlowConn struct {
+	closed    chan struct{}
+	closeOnce sync.Once
+}
+
+func (conn *natFlowConn) Read([]byte) (int, error) {
+	<-conn.closed
+	return 0, io.EOF
+}
+func (conn *natFlowConn) Write(data []byte) (int, error) { return len(data), nil }
+func (conn *natFlowConn) Close() error {
+	conn.closeOnce.Do(func() { close(conn.closed) })
+	return nil
+}
+func (conn *natFlowConn) LocalAddr() net.Addr              { return natFlowAddr{} }
+func (conn *natFlowConn) RemoteAddr() net.Addr             { return natFlowAddr{} }
+func (conn *natFlowConn) SetDeadline(time.Time) error      { return nil }
+func (conn *natFlowConn) SetReadDeadline(time.Time) error  { return nil }
+func (conn *natFlowConn) SetWriteDeadline(time.Time) error { return nil }
+
+type natFlowAddr struct{}
+
+func (natFlowAddr) Network() string { return "test" }
+func (natFlowAddr) String() string  { return "test" }
+
+type natFlowAgent struct {
+	mu        sync.Mutex
+	written   bytes.Buffer
+	writeDone chan struct{}
+	once      sync.Once
+}
+
+func (agent *natFlowAgent) Read([]byte) (int, error) { return 0, io.EOF }
+func (agent *natFlowAgent) Write(data []byte) (int, error) {
+	agent.mu.Lock()
+	agent.written.Write(data)
+	agent.mu.Unlock()
+	agent.once.Do(func() { close(agent.writeDone) })
+	return len(data), nil
+}
+func (*natFlowAgent) Close() error { return nil }
+func (agent *natFlowAgent) writtenBytes() []byte {
+	agent.mu.Lock()
+	defer agent.mu.Unlock()
+	return append([]byte(nil), agent.written.Bytes()...)
+}

cmd/dashboard/controller/dashboard_credential_test.go

@@ -0,0 +1,226 @@
+package controller
+
+import (
+	"bytes"
+	"context"
+	"log"
+	"testing"
+	"time"
+
+	jwt "github.com/appleboy/gin-jwt/v2"
+	"github.com/stretchr/testify/require"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+
+	"github.com/nezhahq/nezha/model"
+	"github.com/nezhahq/nezha/service/singleton"
+)
+
+func setupDashboardCredentialTest(t *testing.T) {
+	t.Helper()
+	originalConf, originalDB, originalParser := singleton.Conf, singleton.DB, dashboardCredentialJWTParser
+	t.Cleanup(func() {
+		singleton.Conf, singleton.DB, dashboardCredentialJWTParser = originalConf, originalDB, originalParser
+	})
+
+	singleton.Conf = &singleton.ConfigClass{Config: &model.Config{}}
+	singleton.Conf.JWTSecretKey = "dashboard-credential-classification-test-key"
+	singleton.Conf.JWTTimeout = 1
+
+	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+	require.NoError(t, err)
+	sqlDB, err := db.DB()
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sqlDB.Close() })
+	require.NoError(t, db.AutoMigrate(&model.APIToken{}))
+	singleton.DB = db
+
+	// Same construction path as routers(): jwt.New(initParams()).
+	authMiddleware, err := jwt.New(initParams())
+	require.NoError(t, err)
+	dashboardCredentialJWTParser = newDashboardCredentialJWTParser(authMiddleware)
+}
+
+// mintDashboardCredentialJWT signs a JWT with the dashboard key through
+// initParams(), the same key source the production verifier uses. A negative
+// timeout mints an already-expired token.
+func mintDashboardCredentialJWT(t *testing.T, timeout time.Duration) string {
+	t.Helper()
+	params := initParams()
+	params.Timeout = timeout
+	mw, err := jwt.New(params)
+	require.NoError(t, err)
+	token, _, err := mw.TokenGenerator(nil)
+	require.NoError(t, err)
+	return token
+}
+
+func mintForeignJWT(t *testing.T) string {
+	t.Helper()
+	params := initParams()
+	params.Key = []byte("foreign-signing-key-not-held-by-the-dashboard")
+	mw, err := jwt.New(params)
+	require.NoError(t, err)
+	token, _, err := mw.TokenGenerator(nil)
+	require.NoError(t, err)
+	return token
+}
+
+func TestIsDashboardCredentialClassification(t *testing.T) {
+	setupDashboardCredentialTest(t)
+
+	dashboardJWT := mintDashboardCredentialJWT(t, time.Hour)
+	expiredDashboardJWT := mintDashboardCredentialJWT(t, -time.Hour)
+	foreignJWT := mintForeignJWT(t)
+
+	dashboardPAT := model.APITokenPrefix + "dashboard-credential-pat"
+	require.NoError(t, singleton.DB.Create(&model.APIToken{
+		UserID:    1,
+		Name:      "nat-classification",
+		TokenHash: model.HashAPIToken(dashboardPAT),
+	}).Error)
+
+	t.Run("dashboard signed jwt is a dashboard credential", func(t *testing.T) {
+		require.True(t, IsDashboardCredential("Bearer "+dashboardJWT))
+	})
+
+	t.Run("expired dashboard signed jwt is still a dashboard credential", func(t *testing.T) {
+		// Signature verification ignores exp on purpose: an expired token is
+		// still proof the dashboard issued it.
+		require.True(t, IsDashboardCredential("Bearer "+expiredDashboardJWT))
+	})
+
+	t.Run("dashboard api token is a dashboard credential", func(t *testing.T) {
+		require.True(t, IsDashboardCredential("Bearer "+dashboardPAT))
+	})
+
+	t.Run("foreign signed jwt is not a dashboard credential", func(t *testing.T) {
+		require.False(t, IsDashboardCredential("Bearer "+foreignJWT))
+	})
+
+	t.Run("unknown api token is not a dashboard credential", func(t *testing.T) {
+		require.False(t, IsDashboardCredential("Bearer "+model.APITokenPrefix+"unknown-to-the-dashboard"))
+	})
+
+	t.Run("random string is not a dashboard credential", func(t *testing.T) {
+		require.False(t, IsDashboardCredential("Bearer random-not-a-token"))
+	})
+
+	t.Run("non bearer scheme is not a dashboard credential", func(t *testing.T) {
+		require.False(t, IsDashboardCredential("Basic "+dashboardJWT))
+		// Scheme matching is case-sensitive, as in both dashboard auth middlewares.
+		require.False(t, IsDashboardCredential("bearer "+dashboardJWT))
+	})
+
+	t.Run("empty or missing header is not a dashboard credential", func(t *testing.T) {
+		require.False(t, IsDashboardCredential(""))
+		require.False(t, IsDashboardCredential("Bearer "))
+	})
+}
+
+// TestIsDashboardCredentialValueClassification pins the scheme-less classifier
+// that backs IsDashboardCredential and drives the ?token= query parameter and
+// nz-jwt cookie channels at NAT ingress (see IsDashboardCredentialValue).
+func TestIsDashboardCredentialValueClassification(t *testing.T) {
+	setupDashboardCredentialTest(t)
+
+	dashboardJWT := mintDashboardCredentialJWT(t, time.Hour)
+	expiredDashboardJWT := mintDashboardCredentialJWT(t, -time.Hour)
+	foreignJWT := mintForeignJWT(t)
+
+	dashboardPAT := model.APITokenPrefix + "dashboard-credential-value-pat"
+	require.NoError(t, singleton.DB.Create(&model.APIToken{
+		UserID:    1,
+		Name:      "nat-value-classification",
+		TokenHash: model.HashAPIToken(dashboardPAT),
+	}).Error)
+
+	t.Run("dashboard signed jwt is a dashboard credential", func(t *testing.T) {
+		require.True(t, IsDashboardCredentialValue(dashboardJWT))
+	})
+	t.Run("expired dashboard signed jwt is still a dashboard credential", func(t *testing.T) {
+		require.True(t, IsDashboardCredentialValue(expiredDashboardJWT))
+	})
+	t.Run("dashboard api token is a dashboard credential", func(t *testing.T) {
+		require.True(t, IsDashboardCredentialValue(dashboardPAT))
+	})
+	t.Run("foreign signed jwt is not a dashboard credential", func(t *testing.T) {
+		require.False(t, IsDashboardCredentialValue(foreignJWT))
+	})
+	t.Run("unknown api token is not a dashboard credential", func(t *testing.T) {
+		require.False(t, IsDashboardCredentialValue(model.APITokenPrefix+"unknown-to-the-dashboard"))
+	})
+	t.Run("random string is not a dashboard credential", func(t *testing.T) {
+		require.False(t, IsDashboardCredentialValue("random-not-a-token"))
+	})
+	t.Run("empty value is not a dashboard credential", func(t *testing.T) {
+		require.False(t, IsDashboardCredentialValue(""))
+	})
+	t.Run("bearer scheme is not part of the value contract", func(t *testing.T) {
+		// The query and cookie channels carry the bare value; a value that
+		// itself starts with "Bearer " is not something the panel issues and
+		// fails signature verification.
+		require.False(t, IsDashboardCredentialValue("Bearer "+dashboardJWT))
+	})
+}
+
+func TestClassifyDashboardCredentialValuesBatchesPATLookup(t *testing.T) {
+	setupDashboardCredentialTest(t)
+
+	dashboardPAT := model.APITokenPrefix + "dashboard-batch-pat"
+	require.NoError(t, singleton.DB.Create(&model.APIToken{
+		UserID:    1,
+		Name:      "nat-batch-classification",
+		TokenHash: model.HashAPIToken(dashboardPAT),
+	}).Error)
+
+	queryCount := 0
+	const callbackName = "test:count-dashboard-credential-batch-query"
+	require.NoError(t, singleton.DB.Callback().Query().Before("gorm:query").Register(callbackName, func(*gorm.DB) {
+		queryCount++
+	}))
+	t.Cleanup(func() { singleton.DB.Callback().Query().Remove(callbackName) })
+
+	values := []string{
+		dashboardPAT,
+		model.APITokenPrefix + "unknown-one",
+		dashboardPAT,
+		model.APITokenPrefix + "unknown-two",
+	}
+	classified := ClassifyDashboardCredentialValues(context.Background(), values)
+
+	require.Equal(t, []bool{true, false, true, false}, classified)
+	require.Equal(t, 1, queryCount, "all PAT hashes in one request must use one database query")
+}
+
+func TestIsDashboardCredentialFailsClosedWhenParserUnavailable(t *testing.T) {
+	setupDashboardCredentialTest(t)
+	dashboardCredentialJWTParser = nil
+
+	token := mintDashboardCredentialJWT(t, time.Hour)
+	require.True(t, IsDashboardCredential("Bearer "+token))
+}
+
+func TestIsDashboardCredentialFailsClosedWhenAPITokenLookupUnavailable(t *testing.T) {
+	setupDashboardCredentialTest(t)
+	singleton.DB = nil
+
+	require.True(t, IsDashboardCredential("Bearer "+model.APITokenPrefix+"unresolvable"))
+}
+
+func TestIsDashboardCredentialFailsClosedOnDatabaseError(t *testing.T) {
+	setupDashboardCredentialTest(t)
+	sqlDB, err := singleton.DB.DB()
+	require.NoError(t, err)
+	require.NoError(t, sqlDB.Close())
+
+	var logs bytes.Buffer
+	originalOutput := log.Writer()
+	log.SetOutput(&logs)
+	t.Cleanup(func() { log.SetOutput(originalOutput) })
+
+	plaintext := model.APITokenPrefix + "unresolvable-after-db-failure"
+	require.True(t, IsDashboardCredential("Bearer "+plaintext))
+	// The credential content itself must never reach the log.
+	require.NotContains(t, logs.String(), plaintext)
+}

cmd/dashboard/controller/ddns.go

@@ -34,6 +34,8 @@ func listDDNS(c *gin.Context) ([]*model.DDNSProfile, error) {
 	// 不影响 singleton 内原始数据。
 	for _, p := range ddnsProfiles {
 		p.AccessSecret = ""
+		p.WebhookURL = ""
+		p.WebhookRequestBody = ""
 		p.WebhookHeaders = ""
 	}
 
@@ -144,10 +146,8 @@ func updateDDNS(c *gin.Context) (any, error) {
 	p.Provider = df.Provider
 	p.Domains = df.Domains
 	p.AccessID = df.AccessID
-	p.WebhookURL = df.WebhookURL
 	p.WebhookMethod = df.WebhookMethod
 	p.WebhookRequestType = df.WebhookRequestType
-	p.WebhookRequestBody = df.WebhookRequestBody
 
 	// 凭据在列表接口已脱敏，前端无法回填；空值视为"不修改"，保留旧值避免误清空。
 	if df.AccessSecret != "" {
@@ -156,6 +156,12 @@ func updateDDNS(c *gin.Context) (any, error) {
 	if df.WebhookHeaders != "" {
 		p.WebhookHeaders = df.WebhookHeaders
 	}
+	if df.WebhookURL != "" {
+		p.WebhookURL = df.WebhookURL
+	}
+	if df.WebhookRequestBody != "" {
+		p.WebhookRequestBody = df.WebhookRequestBody
+	}
 
 	for n, domain := range p.Domains {
 		// IDN to ASCII

cmd/dashboard/controller/fm.go

@@ -15,6 +15,19 @@ import (
 	"github.com/nezhahq/nezha/service/singleton"
 )
 
+// The official file-manager client uploads in 1 MiB WebSocket messages. Cap
+// each complete message at that protocol boundary so gorilla/websocket cannot
+// buffer an attacker-sized frame before the IO stream relay sees it.
+const fileManagerWebSocketInputLimit int64 = 1024 * 1024
+
+type websocketReadLimiter interface {
+	SetReadLimit(limit int64)
+}
+
+func limitFileManagerWebSocketInput(conn websocketReadLimiter) {
+	conn.SetReadLimit(fileManagerWebSocketInputLimit)
+}
+
 // Create FM session
 // @Summary Create FM session
 // @Description Create an "attached" FM. It is advised to only call this within a terminal session.
@@ -103,6 +116,7 @@ func fmStream(c *gin.Context) (any, error) {
 	if err != nil {
 		return nil, newWsError("%v", err)
 	}
+	limitFileManagerWebSocketInput(wsConn)
 	conn := websocketx.NewConn(wsConn)
 	pingTransport := newWebsocketPingTransport(conn, wsConn.Close)
 	stopPing := startWebsocketPingTicker(c.Request.Context(), time.Second*10, pingTransport)

cmd/dashboard/controller/jwt.go

@@ -149,7 +149,7 @@ func identityHandler() func(c *gin.Context) any {
 		}
 
 		currentIP := c.GetString(model.CtxKeyRealIPStr)
-		if sess.IP != currentIP {
+		if !singleton.Conf.JWTIPChangeAllowed() && sess.IP != currentIP {
 			c.Set(model.CtxKeyIsIPMismatch, true)
 			return nil
 		}

cmd/dashboard/controller/jwt_session_test.go

@@ -193,6 +193,61 @@ func TestIdentityHandlerFlagsIPMismatch(t *testing.T) {
 	assert.True(t, verify.GetBool(model.CtxKeyIsIPMismatch))
 }
 
+func TestIdentityHandlerAllowsIPMismatchWhenConfigured(t *testing.T) {
+	cleanup := setupJWTSessionTest(t)
+	defer cleanup()
+	singleton.Conf.SetJWTIPChangeAllowed(true)
+
+	ctx := newCtxForUser(0, "1.2.3.4", "ua")
+	user := model.User{Common: model.Common{ID: 100}, TokenVersion: 7}
+	claims, err := issueJWTSession(ctx, &user, 1)
+	require.NoError(t, err)
+
+	verify := newCtxForUser(0, "9.9.9.9", "ua")
+	verify.Set("JWT_PAYLOAD", jwt.MapClaims{
+		jwtClaimUserID: claims[jwtClaimUserID],
+		jwtClaimKeyID:  claims[jwtClaimKeyID],
+	})
+
+	identity := identityHandler()(verify)
+	require.NotNil(t, identity, "IP mismatch must be accepted when allow_jwt_ip_change is enabled")
+	assert.False(t, verify.GetBool(model.CtxKeyIsIPMismatch))
+}
+
+func TestIdentityHandlerConcurrentIPChangeSetting(t *testing.T) {
+	cleanup := setupJWTSessionTest(t)
+	defer cleanup()
+
+	ctx := newCtxForUser(0, "1.2.3.4", "ua")
+	user := model.User{Common: model.Common{ID: 100}, TokenVersion: 7}
+	claims, err := issueJWTSession(ctx, &user, 1)
+	require.NoError(t, err)
+
+	stop, done := make(chan struct{}), make(chan struct{})
+	go func() {
+		defer close(done)
+		for {
+			select {
+			case <-stop:
+				return
+			default:
+				singleton.Conf.SetJWTIPChangeAllowed(true)
+				singleton.Conf.SetJWTIPChangeAllowed(false)
+			}
+		}
+	}()
+	defer func() {
+		close(stop)
+		<-done
+	}()
+
+	for i := 0; i < 100; i++ {
+		verify := newCtxForUser(0, "1.2.3.4", "ua")
+		verify.Set("JWT_PAYLOAD", jwt.MapClaims(claims))
+		require.NotNil(t, identityHandler()(verify), "matching IP must pass during setting updates")
+	}
+}
+
 func TestIdentityHandlerRejectsUnknownKeyID(t *testing.T) {
 	cleanup := setupJWTSessionTest(t)
 	defer cleanup()

cmd/dashboard/controller/mcp_test.go

@@ -39,7 +39,12 @@ func setupMCPTest(t *testing.T) (func(), uint64) {
 	require.NoError(t, err)
 	require.NoError(t, db.AutoMigrate(&model.User{}, &model.APIToken{}, &model.MCPAuditLog{}, &model.Server{}, &model.WAF{}))
 	singleton.DB = db
-	singleton.Conf = &singleton.ConfigClass{Config: &model.Config{JWTTimeout: 1}}
+	singleton.Conf = &singleton.ConfigClass{Config: &model.Config{
+		JWTTimeout: 1,
+		ConfigDashboard: model.ConfigDashboard{
+			DashboardHost: "example.com",
+		},
+	}}
 	singleton.Conf.SetMCPEnabled(true)
 
 	user := model.User{Common: model.Common{ID: 100}, Username: "alice", Role: model.RoleMember}

cmd/dashboard/controller/mcp_transfer.go

@@ -13,6 +13,7 @@ import (
 	"io"
 	"math"
 	"net/http"
+	"net/url"
 	"strconv"
 	"strings"
 	"sync"
@@ -91,6 +92,37 @@ var (
 	transferSecretVal string
 )
 
+var errMCPTransferHostNotConfigured = errors.New("MCP transfer URL host is not an operator-declared dashboard host")
+
+// mcpTransferURLBase never reflects an untrusted request Host into a bearer
+// transfer URL. DashboardHost is the canonical public authority; deployments
+// that intentionally serve multiple names must declare them as reserved hosts.
+func mcpTransferURLBase(c *gin.Context) (string, error) {
+	if c == nil || c.Request == nil || singleton.Conf == nil {
+		return "", errMCPTransferHostNotConfigured
+	}
+
+	requestHost := strings.TrimSpace(c.Request.Host)
+	host := strings.TrimSpace(singleton.Conf.DashboardHost)
+	if host == "" {
+		if !singleton.IsReservedDashboardHost(requestHost) {
+			return "", errMCPTransferHostNotConfigured
+		}
+		host = requestHost
+	}
+
+	parsed, err := url.Parse("//" + host)
+	if err != nil || parsed.Host == "" || parsed.Host != host || parsed.User != nil || parsed.Path != "" || parsed.RawQuery != "" || parsed.Fragment != "" {
+		return "", errMCPTransferHostNotConfigured
+	}
+
+	scheme := "https"
+	if c.Request.TLS == nil && c.Request.Header.Get("X-Forwarded-Proto") != "https" {
+		scheme = "http"
+	}
+	return scheme + "://" + host, nil
+}
+
 // transferHMACSecret 返回进程内随机生成的 HMAC key。
 // 这是有意设计：transferEntries 本身也只活在内存 sync.Map 里，dashboard
 // 重启等价于全部 token 失效；让 secret 也随进程随机，可以避免“secret 来自
@@ -315,6 +347,10 @@ func mintTransferTool(c *gin.Context, serverID uint64, path string, ttlSeconds i
 	if err := validateTransferPath(path); err != nil {
 		return nil, err
 	}
+	baseURL, err := mcpTransferURLBase(c)
+	if err != nil {
+		return nil, err
+	}
 	ttl := time.Duration(ttlSeconds) * time.Second
 	if ttl <= 0 {
 		ttl = transferTokenTTLDefault
@@ -348,14 +384,9 @@ func mintTransferTool(c *gin.Context, serverID uint64, path string, ttlSeconds i
 	if err != nil {
 		return nil, err
 	}
-	scheme := "https"
-	if c.Request.TLS == nil && c.Request.Header.Get("X-Forwarded-Proto") != "https" {
-		scheme = "http"
-	}
-	host := c.Request.Host
-	url := fmt.Sprintf("%s://%s/mcp/%s/%s", scheme, host, dir, t)
+	transferURL := fmt.Sprintf("%s/mcp/%s/%s", baseURL, dir, t)
 	return map[string]any{
-		"url":        url,
+		"url":        transferURL,
 		"method":     map[transferDirection]string{transferDirDownload: "GET", transferDirUpload: "POST"}[dir],
 		"expires_at": entry.ExpiresAt,
 	}, nil

cmd/dashboard/controller/mcp_transfer_host_test.go

@@ -0,0 +1,80 @@
+package controller
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/nezhahq/nezha/model"
+	"github.com/nezhahq/nezha/service/singleton"
+)
+
+func TestMCPTransferURLPinsUntrustedRequestHostToDashboardHost(t *testing.T) {
+	cleanup, uid := setupMCPTest(t)
+	defer cleanup()
+	PurgeTransferEntries()
+	t.Cleanup(func() { PurgeTransferEntries() })
+	tok, _ := mkToken(t, uid, []string{model.ScopeServerRead}, nil)
+
+	c, _ := mcpCallCtx(t, tok, uid, nil)
+	c.Request.Host = "attacker.example"
+	c.Request.Header.Set("X-Forwarded-Proto", "https")
+
+	out, err := mintTransferTool(c, 7, "/srv/file", 60, transferDirDownload, transferEntry{})
+	require.NoError(t, err)
+	result := out.(map[string]any)
+	require.Contains(t, result["url"], "https://example.com/mcp/download/")
+	require.NotContains(t, result["url"], "attacker.example")
+}
+
+func TestMCPTransferURLRejectsUntrustedHostWithoutCanonicalHost(t *testing.T) {
+	cleanup, uid := setupMCPTest(t)
+	defer cleanup()
+	PurgeTransferEntries()
+	t.Cleanup(func() { PurgeTransferEntries() })
+	tok, _ := mkToken(t, uid, []string{model.ScopeServerRead}, nil)
+	singleton.Conf.DashboardHost = ""
+	singleton.Conf.InstallHost = ""
+	singleton.Conf.ListenHost = ""
+	singleton.Conf.ReservedHosts = ""
+
+	c, _ := mcpCallCtx(t, tok, uid, nil)
+	c.Request = c.Request.Clone(c.Request.Context())
+	c.Request.Method = http.MethodPost
+	c.Request.Host = "attacker.example"
+
+	_, err := mintTransferTool(c, 7, "/srv/file", 60, transferDirDownload, transferEntry{})
+	require.ErrorIs(t, err, errMCPTransferHostNotConfigured)
+	entries := 0
+	transferEntries.Range(func(_, _ any) bool {
+		entries++
+		return true
+	})
+	require.Zero(t, entries, "host validation must happen before a bearer transfer token is minted")
+}
+
+func TestMCPTransferURLAllowsOperatorReservedHost(t *testing.T) {
+	cleanup, _ := setupMCPTest(t)
+	defer cleanup()
+	singleton.Conf.DashboardHost = ""
+	singleton.Conf.ReservedHosts = "mcp.example.com"
+
+	c, _ := mcpCallCtx(t, nil, 0, nil)
+	c.Request.Host = "mcp.example.com"
+	c.Request.Header.Set("X-Forwarded-Proto", "https")
+
+	base, err := mcpTransferURLBase(c)
+	require.NoError(t, err)
+	require.Equal(t, "https://mcp.example.com", base)
+}
+
+func TestMCPTransferURLRejectsMalformedConfiguredHost(t *testing.T) {
+	cleanup, _ := setupMCPTest(t)
+	defer cleanup()
+	singleton.Conf.DashboardHost = "https://example.com/path"
+
+	c, _ := mcpCallCtx(t, nil, 0, nil)
+	_, err := mcpTransferURLBase(c)
+	require.ErrorIs(t, err, errMCPTransferHostNotConfigured)
+}

cmd/dashboard/controller/notification.go

@@ -1,6 +1,7 @@
 package controller
 
 import (
+	"net/http"
 	"slices"
 	"strconv"
 
@@ -12,6 +13,8 @@ import (
 	"github.com/nezhahq/nezha/service/singleton"
 )
 
+const notificationBatchDeleteMaxBodyBytes = 1 << 20
+
 // List notification
 // @Summary List notification
 // @Security BearerAuth
@@ -176,6 +179,9 @@ func updateNotification(c *gin.Context) (any, error) {
 // @Router /batch-delete/notification [post]
 func batchDeleteNotification(c *gin.Context) (any, error) {
 	var n []uint64
+	if c.Request != nil && c.Request.Body != nil {
+		c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, notificationBatchDeleteMaxBodyBytes)
+	}
 	if err := c.ShouldBindJSON(&n); err != nil {
 		return nil, err
 	}

cmd/dashboard/controller/notification_body_limit_test.go

@@ -0,0 +1,25 @@
+package controller
+
+import (
+	"bytes"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+)
+
+func TestBatchDeleteNotificationCapsRequestBody(t *testing.T) {
+	body := append(bytes.Repeat([]byte{' '}, notificationBatchDeleteMaxBodyBytes+1), '[', ']')
+	req := httptest.NewRequest(http.MethodPost, "/api/v1/batch-delete/notification", bytes.NewReader(body))
+	req.Header.Set("Content-Type", "application/json")
+	w := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(w)
+	c.Request = req
+
+	_, err := batchDeleteNotification(c)
+	if err == nil || !strings.Contains(err.Error(), "request body too large") {
+		t.Fatalf("oversized request body was not rejected by MaxBytesReader: %v", err)
+	}
+}

cmd/dashboard/controller/online_history.go

@@ -0,0 +1,121 @@
+package controller
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/gin-gonic/gin"
+
+	"github.com/nezhahq/nezha/model"
+	"github.com/nezhahq/nezha/service/singleton"
+)
+
+// Get server online history
+// @Summary Get server online history
+// @Security BearerAuth
+// @Schemes
+// @Description Get online periods and aggregated uptime statistics of a specific server
+// @Tags common
+// @Param id path uint true "Server ID"
+// @Param days query int false "Query window in days (default: configured online_stats_days, clamped to it)"
+// @Produce json
+// @Success 200 {object} model.CommonResponse[model.OnlineHistoryResponse]
+// @Router /server/{id}/online-history [get]
+func getServerOnlineHistory(c *gin.Context) (*model.OnlineHistoryResponse, error) {
+	idStr := c.Param("id")
+	serverID, err := strconv.ParseUint(idStr, 10, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	server, ok := singleton.ServerShared.Get(serverID)
+	if !ok || server == nil {
+		return nil, singleton.Localizer.ErrorT("server not found")
+	}
+	if !userCanViewServer(c, server) {
+		return nil, singleton.Localizer.ErrorT("unauthorized")
+	}
+	_, isMember := c.Get(model.CtxKeyAuthorizedUser)
+
+	maxDays := singleton.Conf.OnlineHistoryRetentionDays()
+	if maxDays < model.MinOnlineStatsDays || maxDays > model.MaxOnlineStatsDays {
+		maxDays = model.DefaultOnlineStatsDays
+	}
+	days := maxDays
+	if daysStr := c.Query("days"); daysStr != "" {
+		d, err := strconv.Atoi(daysStr)
+		if err != nil || d < 1 {
+			return nil, singleton.Localizer.ErrorT("invalid days parameter")
+		}
+		// 查询窗口收敛到后台设置的统计范围内
+		if d > maxDays {
+			d = maxDays
+		}
+		days = d
+	}
+
+	if !isMember && days > 1 {
+		return nil, singleton.Localizer.ErrorT("unauthorized: only 1d data available for guests")
+	}
+
+	now := time.Now()
+	windowEnd := now
+	windowStart := model.OnlineHistoryWindowStart(now, days, singleton.Loc)
+
+	sessions, err := singleton.OnlineHistoryShared.Sessions(serverID, windowStart, windowEnd)
+	if err != nil {
+		return nil, newGormError("%v", err)
+	}
+
+	periods := model.ClipOnlineSessions(sessions, windowStart, windowEnd)
+	daily := model.DailyOnlineBreakdown(periods, windowStart, windowEnd, singleton.Loc)
+	// 总时长取各日之和，与 daily[] 严格对账（每日累计原始时长后统一取整，
+	// 口径见 DailyOnlineBreakdown）
+	var totalSeconds uint64
+	for _, d := range daily {
+		totalSeconds += d.OnlineSeconds
+	}
+	windowSeconds := uint64(windowEnd.Sub(windowStart).Seconds())
+	onlineRate := 0.0
+	if windowSeconds > 0 {
+		onlineRate = float64(totalSeconds) / float64(windowSeconds) * 100
+	}
+
+	runtime := server.RuntimeSnapshot()
+	currentlyOnline := !runtime.LastActive.IsZero() && now.Sub(runtime.LastActive) <= model.OnlineActiveWindow
+	var lastActive *time.Time
+	if !runtime.LastActive.IsZero() {
+		lastActiveUTC := runtime.LastActive.UTC()
+		lastActive = &lastActiveUTC
+	}
+
+	// 响应统一以 UTC 序列化，避免窗口时间与库中读回的会话时间混用
+	// 本地偏移与 Z 两种写法（时段为值拷贝，不影响调用方）
+	utcPeriods := make([]model.OnlinePeriod, len(periods))
+	for i, p := range periods {
+		utcPeriods[i] = model.OnlinePeriod{Start: p.Start.UTC(), End: p.End.UTC(), Ongoing: p.Ongoing}
+	}
+
+	return &model.OnlineHistoryResponse{
+		ServerID:           serverID,
+		ServerName:         server.Name,
+		Days:               days,
+		MaxDays:            maxDays,
+		Timezone:           singleton.Loc.String(),
+		TzOffset:           tzOffsetSeconds(now, singleton.Loc),
+		WindowStart:        windowStart.UTC(),
+		WindowEnd:          windowEnd.UTC(),
+		TotalOnlineSeconds: totalSeconds,
+		OnlineRate:         onlineRate,
+		CurrentlyOnline:    currentlyOnline,
+		LastActive:         lastActive,
+		Periods:            utcPeriods,
+		Daily:              daily,
+	}, nil
+}
+
+// tzOffsetSeconds 返回指定时区在给定时刻相对 UTC 的偏移秒数。
+func tzOffsetSeconds(at time.Time, loc *time.Location) int {
+	_, offset := at.In(loc).Zone()
+	return offset
+}

cmd/dashboard/controller/online_history_test.go

@@ -0,0 +1,356 @@
+package controller
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/require"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+
+	"github.com/nezhahq/nezha/model"
+	"github.com/nezhahq/nezha/pkg/i18n"
+	"github.com/nezhahq/nezha/service/singleton"
+)
+
+func setupOnlineHistoryControllerTest(t *testing.T) (visibleServer, hiddenServer *model.Server) {
+	t.Helper()
+	originalDB := singleton.DB
+	originalLocalizer := singleton.Localizer
+	originalServer := singleton.ServerShared
+	originalConf := singleton.Conf
+	originalLoc := singleton.Loc
+	if singleton.Localizer == nil {
+		singleton.Localizer = i18n.NewLocalizer("en_US", "nezha", "translations", i18n.Translations)
+	}
+
+	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+	require.NoError(t, err)
+	sqlDB, err := db.DB()
+	require.NoError(t, err)
+	require.NoError(t, db.AutoMigrate(&model.User{}, &model.Server{}, &model.OnlineSession{}))
+
+	singleton.DB = db
+	singleton.ServerShared = singleton.NewEmptyServerClassForTest()
+	singleton.Conf = &singleton.ConfigClass{Config: &model.Config{}}
+	singleton.Conf.OnlineStatsDays = 30
+	singleton.Loc = time.FixedZone("TEST", 8*3600)
+
+	visibleServer = &model.Server{
+		Common:     model.Common{ID: 1},
+		Name:       "visible",
+		UUID:       "online-history-visible",
+		LastActive: time.Now(),
+	}
+	hiddenServer = &model.Server{
+		Common:       model.Common{ID: 2},
+		Name:         "hidden",
+		UUID:         "online-history-hidden",
+		HideForGuest: true,
+	}
+	singleton.ServerShared.InsertForTest(visibleServer)
+	singleton.ServerShared.InsertForTest(hiddenServer)
+
+	t.Cleanup(func() {
+		_ = sqlDB.Close()
+		singleton.DB = originalDB
+		singleton.Localizer = originalLocalizer
+		singleton.ServerShared = originalServer
+		singleton.Conf = originalConf
+		singleton.Loc = originalLoc
+	})
+	return visibleServer, hiddenServer
+}
+
+func onlineHistoryCtx(t *testing.T, serverID string, query string) *gin.Context {
+	t.Helper()
+	gin.SetMode(gin.TestMode)
+	c, _ := gin.CreateTestContext(httptest.NewRecorder())
+	c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/server/"+serverID+"/online-history?"+query, nil)
+	c.Params = gin.Params{{Key: "id", Value: serverID}}
+	return c
+}
+
+func onlineHistoryCtxAsMember(t *testing.T, serverID string, query string) *gin.Context {
+	t.Helper()
+	c := onlineHistoryCtx(t, serverID, query)
+	c.Set(model.CtxKeyAuthorizedUser, &model.User{Common: model.Common{ID: 100}, Role: model.RoleMember})
+	return c
+}
+
+func seedOnlineSessions(t *testing.T, serverID uint64, now time.Time) []model.OnlineSession {
+	t.Helper()
+	now = now.Truncate(time.Second)
+	sessions := []model.OnlineSession{
+		{ServerID: serverID, StartedAt: now.Add(-48 * time.Hour), EndedAt: now.Add(-47 * time.Hour), LastActiveAt: now.Add(-47 * time.Hour)},
+		{ServerID: serverID, StartedAt: now.Add(-25 * time.Hour), EndedAt: now.Add(-23*time.Hour - 30*time.Minute), LastActiveAt: now.Add(-23 * time.Hour)},
+		{ServerID: serverID, StartedAt: now.Add(-10 * time.Minute), Ongoing: true, LastActiveAt: now},
+		// 超出默认 7 天窗口、但在 30 天窗口内的会话
+		{ServerID: serverID, StartedAt: now.Add(-20 * time.Hour * 24), EndedAt: now.Add(-20*time.Hour*24 + time.Hour), LastActiveAt: now.Add(-20 * time.Hour * 24)},
+	}
+	for i := range sessions {
+		require.NoError(t, singleton.DB.Create(&sessions[i]).Error)
+	}
+	return sessions
+}
+
+func TestGetServerOnlineHistoryDefaultsToConfiguredWindow(t *testing.T) {
+	visible, _ := setupOnlineHistoryControllerTest(t)
+	now := time.Now()
+	seedOnlineSessions(t, visible.ID, now)
+
+	// 不带 days 参数：使用后台设置的统计范围（30 天）
+	c := onlineHistoryCtxAsMember(t, "1", "")
+	resp, err := getServerOnlineHistory(c)
+	require.NoError(t, err)
+	require.Equal(t, 30, resp.Days)
+	require.Equal(t, 30, resp.MaxDays)
+	// 4 个会话全部落在 30 天窗口内
+	require.Len(t, resp.Periods, 4)
+	require.True(t, resp.CurrentlyOnline)
+}
+
+func TestGetServerOnlineHistoryDefaultIsSevenDays(t *testing.T) {
+	visible, _ := setupOnlineHistoryControllerTest(t)
+	now := time.Now()
+	seedOnlineSessions(t, visible.ID, now)
+
+	// 出厂默认统计范围为 7 天
+	singleton.Conf.OnlineStatsDays = model.DefaultOnlineStatsDays
+	c := onlineHistoryCtxAsMember(t, "1", "")
+	resp, err := getServerOnlineHistory(c)
+	require.NoError(t, err)
+	require.Equal(t, 7, resp.Days)
+	// 20 天前的会话不进入 7 天窗口
+	require.Len(t, resp.Periods, 3)
+}
+
+func TestGetServerOnlineHistoryAggregatesAndClampsWindow(t *testing.T) {
+	visible, _ := setupOnlineHistoryControllerTest(t)
+	now := time.Now()
+	seedOnlineSessions(t, visible.ID, now)
+
+	c := onlineHistoryCtxAsMember(t, "1", "days=9999")
+	resp, err := getServerOnlineHistory(c)
+	require.NoError(t, err)
+	// 请求超出后台设置窗口时收敛到设置值
+	require.Equal(t, 30, resp.Days)
+
+	// 聚合正确性：三个已结束/进行中会话都计入，进行中会话按当前时间截断
+	var total uint64
+	for _, p := range resp.Periods {
+		total += uint64(p.End.Sub(p.Start).Seconds())
+	}
+	require.Equal(t, resp.TotalOnlineSeconds, total)
+	require.GreaterOrEqual(t, resp.TotalOnlineSeconds, uint64(3600+5400+599))
+
+	// 每日拆分合计等于总时长
+	var dailySum uint64
+	for _, d := range resp.Daily {
+		dailySum += d.OnlineSeconds
+	}
+	require.Equal(t, resp.TotalOnlineSeconds, dailySum)
+
+	// 在线率按窗口秒数计算
+	windowSeconds := resp.WindowEnd.Sub(resp.WindowStart).Seconds()
+	require.InDelta(t, float64(resp.TotalOnlineSeconds)/windowSeconds*100, resp.OnlineRate, 0.001)
+}
+
+func TestGetServerOnlineHistoryInvalidDays(t *testing.T) {
+	setupOnlineHistoryControllerTest(t)
+	for _, q := range []string{"days=abc", "days=0", "days=-1"} {
+		c := onlineHistoryCtxAsMember(t, "1", q)
+		_, err := getServerOnlineHistory(c)
+		require.EqualError(t, err, singleton.Localizer.ErrorT("invalid days parameter").Error(), "query %q should be rejected", q)
+	}
+}
+
+func TestGetServerOnlineHistoryUnknownServer(t *testing.T) {
+	setupOnlineHistoryControllerTest(t)
+	c := onlineHistoryCtx(t, "424242", "")
+	_, err := getServerOnlineHistory(c)
+	require.Error(t, err)
+}
+
+func TestGetServerOnlineHistoryGuestLimitedToOneDay(t *testing.T) {
+	visible, _ := setupOnlineHistoryControllerTest(t)
+	now := time.Now()
+	seedOnlineSessions(t, visible.ID, now)
+
+	// 游客不带 days 参数时默认为配置窗口（>1 天），与 metrics 的游客语义一致被拒绝
+	_, err := getServerOnlineHistory(onlineHistoryCtx(t, "1", ""))
+	require.Error(t, err)
+
+	// 游客显式请求多天被拒绝，1 天可用
+	_, err = getServerOnlineHistory(onlineHistoryCtx(t, "1", "days=3"))
+	require.Error(t, err)
+	resp, err := getServerOnlineHistory(onlineHistoryCtx(t, "1", "days=1"))
+	require.NoError(t, err)
+	require.Equal(t, 1, resp.Days)
+	// 与 1 天窗口相交的两个会话：一个被裁剪到窗口起点，一个进行中
+	require.Len(t, resp.Periods, 2)
+
+	// 登录用户不受游客窗口限制
+	resp, err = getServerOnlineHistory(onlineHistoryCtxAsMember(t, "1", "days=30"))
+	require.NoError(t, err)
+	require.Equal(t, 30, resp.Days)
+	require.Len(t, resp.Periods, 4)
+}
+
+func TestGetServerOnlineHistoryPermission(t *testing.T) {
+	visible, _ := setupOnlineHistoryControllerTest(t)
+
+	// 游客可以查看未隐藏服务器（窗口受限于 1 天）
+	guest := onlineHistoryCtx(t, "1", "days=1")
+	resp, err := getServerOnlineHistory(guest)
+	require.NoError(t, err)
+	require.Equal(t, visible.ID, resp.ServerID)
+
+	// 游客不能查看 HideForGuest 服务器
+	guestHidden := onlineHistoryCtx(t, "2", "days=1")
+	_, err = getServerOnlineHistory(guestHidden)
+	require.EqualError(t, err, singleton.Localizer.ErrorT("unauthorized").Error())
+
+	// 成员不能查看他人隐藏服务器
+	member := onlineHistoryCtx(t, "2", "days=7")
+	member.Set(model.CtxKeyAuthorizedUser, &model.User{Common: model.Common{ID: 100}, Role: model.RoleMember})
+	_, err = getServerOnlineHistory(member)
+	require.Error(t, err)
+}
+
+func patchOnlineStatsDays(t *testing.T, days any) error {
+	t.Helper()
+	gin.SetMode(gin.TestMode)
+	body, err := json.Marshal(map[string]any{"user_template": "user-dist", "online_stats_days": days})
+	require.NoError(t, err)
+	c, _ := gin.CreateTestContext(httptest.NewRecorder())
+	c.Request = httptest.NewRequest(http.MethodPatch, "/api/v1/setting", bytes.NewReader(body))
+	c.Request.Header.Set("Content-Type", "application/json")
+	_, err = updateConfig(c)
+	return err
+}
+
+func TestUpdateConfigOnlineStatsDaysRange(t *testing.T) {
+	originalLocalizer := singleton.Localizer
+	if singleton.Localizer == nil {
+		singleton.Localizer = i18n.NewLocalizer("en_US", "nezha", "translations", i18n.Translations)
+	}
+	t.Cleanup(func() { singleton.Localizer = originalLocalizer })
+
+	templates := []model.FrontendTemplate{{Path: "user-dist", IsAdmin: false}}
+	originalFrontendTemplates := singleton.FrontendTemplates
+	singleton.FrontendTemplates = templates
+	t.Cleanup(func() { singleton.FrontendTemplates = originalFrontendTemplates })
+	conf := &singleton.ConfigClass{Config: &model.Config{}}
+	configPath := filepath.Join(t.TempDir(), "config.yaml")
+	require.NoError(t, conf.Read(configPath, templates))
+	originalConf := singleton.Conf
+	singleton.Conf = conf
+	t.Cleanup(func() { singleton.Conf = originalConf })
+
+	require.Equal(t, model.DefaultOnlineStatsDays, conf.OnlineStatsDays)
+
+	// 合法边界值
+	for _, d := range []int{model.MinOnlineStatsDays, 30, model.MaxOnlineStatsDays} {
+		require.NoError(t, patchOnlineStatsDays(t, d), "days=%d should be accepted", d)
+		require.Equal(t, d, conf.OnlineStatsDays)
+	}
+
+	// 越界值整体拒绝且不修改现值
+	for _, d := range []int{6, model.MaxOnlineStatsDays + 1, 0, -5} {
+		require.Error(t, patchOnlineStatsDays(t, d), "days=%d should be rejected", d)
+		require.Equal(t, model.MaxOnlineStatsDays, conf.OnlineStatsDays)
+	}
+
+	// 非法值必须整体拒绝：校验先于任何对 Conf 的写入完成，其他字段不得
+	// 半提交（评审复现场景：错误返回后 SiteName 不能被请求内容覆盖）
+	conf.SiteName = "existing-site"
+	require.Error(t, patchOnlineStatsDays(t, 6))
+	require.Equal(t, "existing-site", conf.SiteName)
+	conf.SiteName = ""
+
+	// 字段缺省时保留现值（部分 PATCH 语义）
+	require.NoError(t, patchOnlineStatsDays(t, nil))
+	require.Equal(t, model.MaxOnlineStatsDays, conf.OnlineStatsDays)
+
+	// 重启持久化断言：updateConfig 显式 Save 后，重新从磁盘读取的配置
+	// 必须带上刚写入的值（持久化不再依赖 MCP 过渡块的副作用）
+	reloaded := &singleton.ConfigClass{Config: &model.Config{}}
+	require.NoError(t, reloaded.Read(configPath, templates))
+	require.Equal(t, model.MaxOnlineStatsDays, reloaded.OnlineStatsDays)
+}
+
+func TestSettingFormOnlineStatsDaysDecode(t *testing.T) {
+	var absent model.SettingForm
+	require.NoError(t, json.NewDecoder(bytes.NewReader([]byte(`{"site_name":"x"}`))).Decode(&absent))
+	require.Nil(t, absent.OnlineStatsDays, "omitted key must decode to nil")
+
+	var explicit model.SettingForm
+	require.NoError(t, json.NewDecoder(bytes.NewReader([]byte(`{"online_stats_days":3650}`))).Decode(&explicit))
+	require.NotNil(t, explicit.OnlineStatsDays)
+	require.Equal(t, 3650, *explicit.OnlineStatsDays)
+}
+
+func TestOnlineHistoryReadsPendingCloseBoundary(t *testing.T) {
+	setupOnlineHistoryControllerTest(t)
+	previous := singleton.OnlineHistoryShared
+	t.Cleanup(func() { singleton.OnlineHistoryShared = previous })
+	r := singleton.NewOnlineHistoryRecorder()
+	singleton.OnlineHistoryShared = r
+	start := time.Now().UTC().Add(-time.Hour).Truncate(time.Second)
+	r.RecordOnline(1, start)
+	require.NoError(t, singleton.DB.Exec(`CREATE TRIGGER reject_close BEFORE UPDATE ON online_sessions WHEN NEW.ongoing = 0 BEGIN SELECT RAISE(ABORT, 'close failure'); END`).Error)
+	r.RecordOffline(1, start.Add(time.Minute))
+	r.RecordOnline(1, start.Add(30*time.Minute))
+	resp, err := getServerOnlineHistory(onlineHistoryCtx(t, "1", "days=1"))
+	require.NoError(t, err)
+	require.Len(t, resp.Periods, 2)
+	require.False(t, resp.Periods[0].Ongoing)
+	require.True(t, resp.Periods[0].End.Equal(start.Add(time.Minute)))
+	require.Less(t, resp.TotalOnlineSeconds, uint64(1900))
+}
+
+func TestOnlineHistoryQueryWithNonUTCSession(t *testing.T) {
+	setupOnlineHistoryControllerTest(t)
+	now := time.Now()
+	offset := time.FixedZone("UTC+12", 12*3600)
+	session := model.OnlineSession{ServerID: 1, StartedAt: now.Add(-time.Hour).In(offset), LastActiveAt: now.Add(-time.Minute).In(offset), Ongoing: true}
+	require.NoError(t, singleton.DB.Create(&session).Error)
+	resp, err := getServerOnlineHistory(onlineHistoryCtx(t, "1", "days=1"))
+	require.NoError(t, err)
+	require.Len(t, resp.Periods, 1)
+	require.True(t, resp.Periods[0].Start.Equal(session.StartedAt))
+	require.InDelta(t, 3600, resp.TotalOnlineSeconds, 2)
+}
+
+func TestUpdateConfigFailedSavePreservesRetention(t *testing.T) {
+	server, _ := setupOnlineHistoryControllerTest(t)
+	require.NoError(t, singleton.DB.Create(server).Error)
+	templates := []model.FrontendTemplate{{Path: "user-dist"}}
+	previous := singleton.FrontendTemplates
+	singleton.FrontendTemplates = templates
+	t.Cleanup(func() { singleton.FrontendTemplates = previous })
+	path := filepath.Join(t.TempDir(), "config.yaml")
+	require.NoError(t, singleton.Conf.Read(path, templates))
+	singleton.Conf.OnlineStatsDays = 30
+	require.NoError(t, singleton.Conf.Save())
+	// Replacing the destination with a directory deterministically fails Save even as root.
+	require.NoError(t, os.Remove(path))
+	require.NoError(t, os.Mkdir(path, 0700))
+	now := time.Now().UTC()
+	old := model.OnlineSession{ServerID: 1, StartedAt: now.AddDate(0, 0, -15), EndedAt: now.AddDate(0, 0, -14)}
+	require.NoError(t, singleton.DB.Create(&old).Error)
+	require.Error(t, patchOnlineStatsDays(t, 7))
+	require.Equal(t, 30, singleton.Conf.OnlineStatsDays)
+	singleton.PruneOnlineSessions()
+	var retained model.OnlineSession
+	require.NoError(t, singleton.DB.First(&retained, old.ID).Error)
+	require.Equal(t, old.ID, retained.ID)
+}

cmd/dashboard/controller/server.go

@@ -236,10 +236,9 @@ func getServerConfig(c *gin.Context) (string, error) {
 	}
 
 	s, ok := singleton.ServerShared.Get(id)
-	if !ok {
-		return "", nil
-	}
-	if !s.HasPermission(c) {
+	// Foreign and unknown IDs deliberately share one response so sequential
+	// server IDs cannot be enumerated across tenants.
+	if !ok || !s.HasPermission(c) || !patAllowsServer(c, id) {
 		return "", singleton.Localizer.ErrorT("permission denied")
 	}
 	if s.GetTaskStream() == nil {
@@ -293,16 +292,16 @@ func setServerConfig(c *gin.Context) (*model.ServerTaskResponse, error) {
 	slist := singleton.ServerShared.GetList()
 	servers := make([]*model.Server, 0, len(configForm.Servers))
 	for _, sid := range configForm.Servers {
-		if s, ok := slist[sid]; ok {
-			if !s.HasPermission(c) {
-				return nil, singleton.Localizer.ErrorT("permission denied")
-			}
-			if s.GetTaskStream() == nil {
-				resp.Offline = append(resp.Offline, s.ID)
-				continue
-			}
-			servers = append(servers, s)
+		s, ok := slist[sid]
+		// Match getServerConfig: do not reveal whether a denied ID exists.
+		if !ok || !s.HasPermission(c) || !patAllowsServer(c, sid) {
+			return nil, singleton.Localizer.ErrorT("permission denied")
 		}
+		if s.GetTaskStream() == nil {
+			resp.Offline = append(resp.Offline, s.ID)
+			continue
+		}
+		servers = append(servers, s)
 	}
 
 	var wg sync.WaitGroup

cmd/dashboard/controller/server_config_oracle_test.go

@@ -0,0 +1,54 @@
+package controller
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/require"
+
+	"github.com/nezhahq/nezha/model"
+	"github.com/nezhahq/nezha/service/singleton"
+)
+
+func TestGetServerConfigForeignAndUnknownIDsAreIndistinguishable(t *testing.T) {
+	cleanup, _ := setupMCPTest(t)
+	defer cleanup()
+	require.NoError(t, singleton.DB.Create(&model.User{Common: model.Common{ID: 200}, Username: "bob", Role: model.RoleMember}).Error)
+	tok, _ := mkToken(t, 200, []string{model.ScopeServerRead}, nil)
+
+	probe := func(id string) error {
+		c, _ := patRequestCtx(t, tok, 200, http.MethodGet, "/api/v1/server/config/"+id, nil)
+		c.Params = gin.Params{{Key: "id", Value: id}}
+		_, err := getServerConfig(c)
+		return err
+	}
+
+	foreignErr := probe("7")
+	unknownErr := probe("999999")
+	require.Error(t, foreignErr)
+	require.Error(t, unknownErr)
+	require.Equal(t, foreignErr.Error(), unknownErr.Error())
+}
+
+func TestSetServerConfigForeignAndUnknownIDsAreIndistinguishable(t *testing.T) {
+	cleanup, _ := setupMCPTest(t)
+	defer cleanup()
+	require.NoError(t, singleton.DB.Create(&model.User{Common: model.Common{ID: 200}, Username: "bob", Role: model.RoleMember}).Error)
+	tok, _ := mkToken(t, 200, []string{model.ScopeServerWrite}, nil)
+
+	probe := func(id uint64) error {
+		c, _ := patRequestCtx(t, tok, 200, http.MethodPost, "/api/v1/server/config", model.ServerConfigForm{
+			Servers: []uint64{id},
+			Config:  "{}",
+		})
+		_, err := setServerConfig(c)
+		return err
+	}
+
+	foreignErr := probe(7)
+	unknownErr := probe(999999)
+	require.Error(t, foreignErr)
+	require.Error(t, unknownErr)
+	require.Equal(t, foreignErr.Error(), unknownErr.Error())
+}

cmd/dashboard/controller/setting.go

@@ -2,6 +2,7 @@ package controller
 
 import (
 	"errors"
+	"fmt"
 	"log"
 	"strings"
 
@@ -29,7 +30,7 @@ func listConfig(c *gin.Context) (*model.SettingResponse, error) {
 		isAdmin = user.Role.IsAdmin()
 	}
 
-	config := *singleton.Conf
+	config := singleton.Conf.SettingsSnapshot()
 	config.Language = strings.ReplaceAll(config.Language, "_", "-")
 
 	conf := model.SettingResponse{
@@ -92,11 +93,20 @@ func updateConfig(c *gin.Context) (any, error) {
 	if !userTemplateValid {
 		return nil, errors.New("invalid user template")
 	}
+	// 全部输入校验必须先于任何对 singleton.Conf 的写入：任一字段非法都应
+	// 整体拒绝，避免"请求以错误收场、先处理的字段却已生效"的半提交状态
+	if sf.OnlineStatsDays != nil &&
+		(*sf.OnlineStatsDays < model.MinOnlineStatsDays || *sf.OnlineStatsDays > model.MaxOnlineStatsDays) {
+		return nil, fmt.Errorf("online_stats_days must be between %d and %d", model.MinOnlineStatsDays, model.MaxOnlineStatsDays)
+	}
 
 	singleton.Conf.Language = strings.ReplaceAll(sf.Language, "-", "_")
 
 	singleton.Conf.EnableIPChangeNotification = sf.EnableIPChangeNotification
 	singleton.Conf.EnablePlainIPInNotification = sf.EnablePlainIPInNotification
+	if sf.AllowJWTIPChange != nil {
+		singleton.Conf.SetJWTIPChangeAllowed(*sf.AllowJWTIPChange)
+	}
 	singleton.Conf.Cover = sf.Cover
 	singleton.Conf.InstallHost = sf.InstallHost
 	singleton.Conf.DashboardHost = sf.DashboardHost
@@ -124,18 +134,22 @@ func updateConfig(c *gin.Context) (any, error) {
 	singleton.Conf.DomainExpiryNotificationDays = sf.DomainExpiryNotificationDays
 	singleton.Conf.ServerExpiryNotificationDays = sf.ServerExpiryNotificationDays
 
-	mcpWasEnabled := singleton.Conf.MCPEnabled()
-	mcpNext := resolveSettingEnableMCP(sf.EnableMCP, mcpWasEnabled)
-
-	if err := applyEnableMCPTransition(
-		mcpWasEnabled, mcpNext,
-		singleton.Conf.SetMCPEnabled,
-		singleton.Conf.Save,
-		fireMCPKillSwitch,
-	); err != nil {
+	if err := singleton.Conf.CommitOnlineHistoryRetention(sf.OnlineStatsDays, func() error {
+		mcpWasEnabled := singleton.Conf.MCPEnabled()
+		mcpNext := resolveSettingEnableMCP(sf.EnableMCP, mcpWasEnabled)
+		return applyEnableMCPTransition(
+			mcpWasEnabled, mcpNext,
+			singleton.Conf.SetMCPEnabled,
+			singleton.Conf.Save,
+			fireMCPKillSwitch,
+		)
+	}); err != nil {
 		return nil, newGormError("%v", err)
 	}
 
+	// applyEnableMCPTransition already persisted every setting, even when MCP
+	// is unchanged. A second save could fail after irreversible cleanup.
+
 	singleton.OnUpdateLang(singleton.Conf.Language)
 	singleton.InitTelegramBot()
 	return nil, nil
@@ -174,6 +188,12 @@ func fireMCPKillSwitch() {
 // the current config to avoid accidentally tripping the kill switch on
 // partial PATCH calls that omit enable_mcp.
 func resolveSettingEnableMCP(formValue *bool, current bool) bool {
+	return resolveOptionalBool(formValue, current)
+}
+
+// resolveOptionalBool preserves a setting for partial PATCH requests when its
+// boolean field is absent, while still allowing an explicit false value.
+func resolveOptionalBool(formValue *bool, current bool) bool {
 	if formValue == nil {
 		return current
 	}

cmd/dashboard/controller/setting_jwt_ip_change_test.go

@@ -0,0 +1,26 @@
+package controller
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/nezhahq/nezha/model"
+)
+
+func TestSettingForm_AllowJWTIPChangePreservesAbsentAndDecodesExplicitValues(t *testing.T) {
+	var omitted model.SettingForm
+	require.NoError(t, json.Unmarshal([]byte(`{"site_name":"X"}`), &omitted))
+	require.Nil(t, omitted.AllowJWTIPChange)
+
+	var enabled model.SettingForm
+	require.NoError(t, json.Unmarshal([]byte(`{"allow_jwt_ip_change":true}`), &enabled))
+	require.NotNil(t, enabled.AllowJWTIPChange)
+	require.True(t, *enabled.AllowJWTIPChange)
+
+	var disabled model.SettingForm
+	require.NoError(t, json.Unmarshal([]byte(`{"allow_jwt_ip_change":false}`), &disabled))
+	require.NotNil(t, disabled.AllowJWTIPChange)
+	require.False(t, *disabled.AllowJWTIPChange)
+}

cmd/dashboard/controller/setting_single_commit_test.go

@@ -0,0 +1,49 @@
+package controller
+
+import (
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// The transition helper owns Save (including unchanged MCP settings). A direct
+// second Save after it can fail after persistence and irreversible revocation.
+func TestUpdateConfigHasSinglePersistenceOwner(t *testing.T) {
+	file, err := parser.ParseFile(token.NewFileSet(), "setting.go", nil, 0)
+	require.NoError(t, err)
+	transitions, directSaves := 0, 0
+	found := false
+	for _, decl := range file.Decls {
+		fn, ok := decl.(*ast.FuncDecl)
+		if !ok || fn.Name.Name != "updateConfig" {
+			continue
+		}
+		found = true
+		ast.Inspect(fn.Body, func(n ast.Node) bool {
+			call, ok := n.(*ast.CallExpr)
+			if !ok {
+				return true
+			}
+			if ident, ok := call.Fun.(*ast.Ident); ok && ident.Name == "applyEnableMCPTransition" {
+				transitions++
+			}
+			if selector, ok := call.Fun.(*ast.SelectorExpr); ok && selector.Sel.Name == "Save" {
+				directSaves++
+			}
+			return true
+		})
+	}
+	require.True(t, found)
+	require.Equal(t, 1, transitions)
+	require.Zero(t, directSaves)
+}
+
+func TestUnchangedMCPTransitionStillSavesOnce(t *testing.T) {
+	saves := 0
+	err := applyEnableMCPTransition(true, true, func(bool) {}, func() error { saves++; return nil }, func() { t.Error("unchanged MCP must not revoke streams") })
+	require.NoError(t, err)
+	require.Equal(t, 1, saves)
+}

cmd/dashboard/controller/terminal_input_limit_test.go

@@ -0,0 +1,28 @@
+package controller
+
+import "testing"
+
+type readLimitRecorder struct {
+	limit int64
+}
+
+func (r *readLimitRecorder) SetReadLimit(limit int64) {
+	r.limit = limit
+}
+
+func TestTerminalWebSocketInputLimitAllowsBoundedPaste(t *testing.T) {
+	if terminalWebSocketInputLimit != 512*1024+64 {
+		t.Fatalf("terminal WebSocket input limit = %d, want 512 KiB plus control-byte allowance", terminalWebSocketInputLimit)
+	}
+}
+
+func TestFileManagerWebSocketInputLimitMatchesUploadChunk(t *testing.T) {
+	if fileManagerWebSocketInputLimit != 1024*1024 {
+		t.Fatalf("file-manager WebSocket input limit = %d, want official client's 1 MiB upload chunk", fileManagerWebSocketInputLimit)
+	}
+	recorder := new(readLimitRecorder)
+	limitFileManagerWebSocketInput(recorder)
+	if recorder.limit != fileManagerWebSocketInputLimit {
+		t.Fatalf("file-manager WebSocket configured limit = %d, want %d", recorder.limit, fileManagerWebSocketInputLimit)
+	}
+}

cmd/dashboard/controller/transfer.go

@@ -171,6 +171,12 @@ const transferStreamPingInterval = 30 * time.Second
 // @Success 200 {object} model.ServerTransfer
 // @Router /ws/transfer [get]
 func transferStream(c *gin.Context) (any, error) {
+	subID, ch, err := singleton.ServerTransferShared.Subscribe(getUid(c))
+	if err != nil {
+		return nil, err
+	}
+	defer singleton.ServerTransferShared.Unsubscribe(subID)
+
 	conn, err := upgrader.Upgrade(c.Writer, c.Request, nil)
 	if err != nil {
 		return nil, newWsError("%v", err)
@@ -180,9 +186,6 @@ func transferStream(c *gin.Context) (any, error) {
 	deregisterPAT := registerPATConnection(c, func() { _ = conn.Close() })
 	defer deregisterPAT()
 
-	subID, ch := singleton.ServerTransferShared.Subscribe()
-	defer singleton.ServerTransferShared.Unsubscribe(subID)
-
 	// Pings keep the socket warm even when the broker is quiet. Without this
 	// a long idle period followed by a transfer event would race against
 	// upstream proxy idle-timeouts that may have already closed the conn.

cmd/dashboard/dashboard_credential_wiring_test.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/nezhahq/nezha/cmd/dashboard/rpc"
+)
+
+// TestWireNATDashboardCredentialGate pins the NAT ingress wiring contract:
+// main() must connect the real batched dashboard-credential classifier
+// (controller.ClassifyDashboardCredentialValues)
+// through wireNATDashboardCredentialGate before any listener serves traffic.
+// That the wired functions really classify per channel is pinned end to end
+// by the controller package's real-classifier NAT flow test.
+func TestWireNATDashboardCredentialGate(t *testing.T) {
+	if rpc.NATDashboardCredentialGateWired() {
+		t.Fatal("NAT dashboard credential gate already wired before the test")
+	}
+
+	wireNATDashboardCredentialGate()
+
+	if !rpc.NATDashboardCredentialGateWired() {
+		t.Fatal("wireNATDashboardCredentialGate left the NAT dashboard credential gate unwired")
+	}
+	// Leave the process in the wired state main() establishes.
+	t.Cleanup(wireNATDashboardCredentialGate)
+}

cmd/dashboard/http_header_timeout.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"net/http"
+	"time"
+)
+
+const dashboardReadHeaderTimeout = 5 * time.Second
+
+func newDashboardHTTPServer(handler http.Handler) *http.Server {
+	// ReadHeaderTimeout protects incomplete HTTP/1 requests and is cleared after
+	// headers. Unlike ReadTimeout it does not impose a lifetime on HTTP/2 streams.
+	return &http.Server{Handler: handler, ReadHeaderTimeout: dashboardReadHeaderTimeout}
+}

cmd/dashboard/http_header_timeout_test.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"crypto/tls"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDashboardRejectsIncompleteHTTPHeaders(t *testing.T) {
+	for _, useTLS := range []bool{false, true} {
+		name := "http"
+		if useTLS {
+			name = "https"
+		}
+		t.Run(name, func(t *testing.T) {
+			server := httptest.NewUnstartedServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("incomplete headers reached handler") }))
+			server.Config = newDashboardHTTPServer(server.Config.Handler)
+			require.Positive(t, server.Config.ReadHeaderTimeout)
+			server.Config.ReadHeaderTimeout = 50 * time.Millisecond
+			if useTLS {
+				server.StartTLS()
+			} else {
+				server.Start()
+			}
+			defer server.Close()
+			var conn net.Conn
+			var err error
+			if useTLS {
+				conn, err = tls.Dial("tcp", server.Listener.Addr().String(), &tls.Config{InsecureSkipVerify: true})
+			} else {
+				conn, err = net.Dial("tcp", server.Listener.Addr().String())
+			}
+			require.NoError(t, err)
+			defer conn.Close()
+			require.NoError(t, conn.SetDeadline(time.Now().Add(2*time.Second)))
+			_, err = io.WriteString(conn, "GET / HTTP/1.1\r\nHost: dashboard\r\nX-Incomplete: ")
+			require.NoError(t, err)
+			_, err = io.ReadAll(conn)
+			if networkErr, ok := err.(net.Error); ok {
+				require.False(t, networkErr.Timeout(), "server did not close incomplete request")
+			}
+		})
+	}
+}
+
+func TestDashboardHeaderTimeoutDoesNotCancelHTTP2Stream(t *testing.T) {
+	for _, useTLS := range []bool{false, true} {
+		name := "h2c"
+		if useTLS {
+			name = "h2"
+		}
+		t.Run(name, func(t *testing.T) {
+			handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if r.ProtoMajor != 2 {
+					t.Errorf("expected HTTP/2, got %s", r.Proto)
+				}
+				require.NoError(t, http.NewResponseController(w).Flush())
+				body, err := io.ReadAll(r.Body)
+				if err != nil {
+					t.Errorf("long-lived request body: %v", err)
+					return
+				}
+				_, _ = w.Write(body)
+			})
+			server := httptest.NewUnstartedServer(handler)
+			server.Config = newDashboardHTTPServer(handler)
+			server.Config.ReadHeaderTimeout = 250 * time.Millisecond
+			server.Config.Protocols = new(http.Protocols)
+			server.Config.Protocols.SetHTTP1(true)
+			server.Config.Protocols.SetHTTP2(true)
+			server.Config.Protocols.SetUnencryptedHTTP2(true)
+			server.EnableHTTP2 = true
+			if useTLS {
+				server.StartTLS()
+			} else {
+				server.Start()
+			}
+			defer server.Close()
+			transport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, Protocols: new(http.Protocols)}
+			transport.Protocols.SetHTTP2(true)
+			transport.Protocols.SetUnencryptedHTTP2(true)
+			defer transport.CloseIdleConnections()
+			client := &http.Client{Transport: transport, Timeout: 10 * time.Second}
+			reader, writer := io.Pipe()
+			defer reader.Close()
+			defer writer.Close()
+			request, err := http.NewRequest(http.MethodPost, server.URL, reader)
+			require.NoError(t, err)
+			response, err := client.Do(request)
+			require.NoError(t, err)
+			defer response.Body.Close()
+			time.Sleep(4 * server.Config.ReadHeaderTimeout)
+			_, err = writer.Write([]byte("still streaming"))
+			require.NoError(t, err)
+			require.NoError(t, writer.Close())
+			body, err := io.ReadAll(response.Body)
+			require.NoError(t, err)
+			require.Equal(t, "still streaming", string(body))
+		})
+	}
+}

cmd/dashboard/main.go

@@ -12,7 +12,6 @@ import (
 	"os"
 	"runtime/debug"
 	"strings"
-	"time"
 	_ "time/tzdata"
 
 	"github.com/gin-gonic/gin"
@@ -86,6 +85,16 @@ func initSystem(bus chan<- *model.Service) error {
 		return err
 	}
 
+	// 在线会话兜底清扫：状态流异常断开（失联）时由这里关闭进行中的会话
+	if _, err := singleton.CronShared.AddFunc("@every 30s", singleton.SweepOnlineSessions); err != nil {
+		return err
+	}
+
+	// 在线会话保留清理：超出后台设置的统计窗口后删除
+	if _, err := singleton.CronShared.AddFunc("0 30 4 * * *", singleton.PruneOnlineSessions); err != nil {
+		return err
+	}
+
 	if err := singleton.StartJWTSessionGC(); err != nil {
 		return err
 	}
@@ -178,12 +187,11 @@ func main() {
 	grpcHandler := rpc.ServeRPC()
 	httpHandler := controller.ServeWeb(frontendDist)
 	controller.InitUpgrader()
+	wireNATDashboardCredentialGate()
 
 	muxHandler := newHTTPandGRPCMux(httpHandler, grpcHandler)
-	muxServerHTTP := &http.Server{
-		Handler:           muxHandler,
-		ReadHeaderTimeout: time.Second * 5,
-	}
+	muxServerHTTP := newDashboardHTTPServer(muxHandler)
+
 	muxServerHTTP.Protocols = new(http.Protocols)
 	muxServerHTTP.Protocols.SetHTTP1(true)
 	muxServerHTTP.Protocols.SetUnencryptedHTTP2(true)
@@ -191,9 +199,9 @@ func main() {
 	var muxServerHTTPS *http.Server
 	if singleton.Conf.HTTPS.ListenPort != 0 {
 		muxServerHTTPS = &http.Server{
+			ReadHeaderTimeout: dashboardReadHeaderTimeout,
 			Addr:              fmt.Sprintf("%s:%d", singleton.Conf.ListenHost, singleton.Conf.HTTPS.ListenPort),
 			Handler:           muxHandler,
-			ReadHeaderTimeout: time.Second * 5,
 			TLSConfig: &tls.Config{
 				InsecureSkipVerify: singleton.Conf.HTTPS.InsecureTLS,
 			},
@@ -237,6 +245,18 @@ func main() {
 	close(errChan)
 }
 
+// wireNATDashboardCredentialGate connects the NAT ingress to the real
+// dashboard-credential classifiers. The NAT ingress strips dashboard-issued
+// credentials (panel JWT / panel PAT) from every channel the panel's own
+// TokenLookup accepts — the Authorization header, the ?token= query parameter
+// and the nz-jwt cookie — before tunneling a request to the agent, and
+// forwards foreign values untouched. ServeWeb built the JWT parser the
+// classifier reuses, so the gate is wired right after it and before any
+// listener starts serving.
+func wireNATDashboardCredentialGate() {
+	rpc.SetNATDashboardCredentialGate(controller.ClassifyDashboardCredentialValues)
+}
+
 func newHTTPandGRPCMux(httpHandler http.Handler, grpcHandler http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		natConfig := singleton.NATShared.GetNATConfigByDomain(r.Host)

cmd/dashboard/rpc/nat.go

@@ -80,7 +80,15 @@ func ServeNAT(w http.ResponseWriter, r *http.Request, natConfig *model.NAT) {
 		return
 	}
 
-	// Authorization authenticates Dashboard access before NAT ingress; it must not become an origin credential for the configured NAT backend.
+	// Credential invariant at NAT ingress: credentials issued by this
+	// dashboard — the panel-signed JWT (verified by signature only, expiry
+	// deliberately ignored) and the panel API token — must never become origin
+	// credentials for the configured NAT backend. prepareNATCapability has
+	// already stripped them from every channel the panel's own TokenLookup
+	// accepts (Authorization header, ?token= query parameter, nz-jwt cookie);
+	// whatever remains on those channels is either absent or a foreign
+	// credential, which is forwarded untouched because the NAT backend may
+	// authenticate with credentials of its own.
 	wWrapped, err := utils.NewRequestWrapper(r, w)
 	if err != nil {
 		return

cmd/dashboard/rpc/nat_capability_agentcompat.go

@@ -22,16 +22,26 @@ type natCapabilityLease struct {
 func prepareNATCapability(request *http.Request, natConfig *model.NAT) (natCapabilityLease, error) {
 	values := request.Header.Values(agentcompatcontract.IOStreamCapabilityHeader)
 	if len(values) == 0 {
-		request.Header.Del("Authorization")
+		// No capability header: legacy NAT path. Strip dashboard-issued
+		// credentials (panel JWT or panel PAT) from the Authorization header,
+		// the ?token= query parameter and the nz-jwt cookie so they never
+		// become origin credentials; foreign values are ordinary request data.
+		stripDashboardCredentials(request)
 		return natCapabilityLease{}, nil
 	}
 	request.Header.Del(agentcompatcontract.IOStreamCapabilityHeader)
 	if len(values) != 1 || values[0] == "" {
-		request.Header.Del("Authorization")
+		// Invalid capability: nothing is forwarded, but dashboard-issued
+		// credentials still must not survive into errors or task data.
+		stripDashboardCredentials(request)
 		return natCapabilityLease{}, errors.New("invalid NAT capability")
 	}
 	access, handle, err := serviceRPC.NezhaHandlerSingleton.ConsumeAgentCompatNATCapabilityForProfile(values[0], natConfig.ServerID, natConfig.ID)
-	request.Header.Del("Authorization")
+	// From here the request bytes may reach the agent (active lease) or the
+	// request ends in a 503; either way dashboard-issued credentials are
+	// stripped from every TokenLookup channel, foreign ones are forwarded
+	// untouched.
+	stripDashboardCredentials(request)
 	if err != nil {
 		return natCapabilityLease{}, errors.New("invalid NAT capability")
 	}

cmd/dashboard/rpc/nat_capability_agentcompat_test.go

@@ -4,8 +4,10 @@ package rpc
 
 import (
 	"bytes"
+	"context"
 	"log"
 	"net/http"
+	"net/url"
 	"strings"
 	"testing"
 
@@ -59,7 +61,193 @@ func TestPrepareNATCapabilityRejectsDuplicateHeaderAfterRemovingAllValues(t *tes
 	}
 }
 
+func TestPrepareNATCapabilityAgentCompatStripsDashboardCredentials(t *testing.T) {
+	// Given — legacy path without a capability header, then the invalid
+	// capability path: both must drop dashboard-issued credentials from the
+	// Authorization header, the ?token= query parameter and the nz-jwt cookie.
+	installNATCredentialTestGate(t)
+	for name, credential := range map[string]string{
+		"dashboard jwt":         natTestDashboardJWTValue,
+		"expired dashboard jwt": natTestDashboardExpiredJWTValue,
+		"dashboard api token":   natTestDashboardPATValue,
+	} {
+		t.Run(name+"/legacy path", func(t *testing.T) {
+			request := &http.Request{
+				Header: make(http.Header),
+				URL:    &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + credential},
+			}
+			request.Header.Set("Authorization", "Bearer "+credential)
+			request.Header.Set("Cookie", "sid=abc; nz-jwt="+credential)
+
+			lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81})
+
+			if err != nil {
+				t.Fatalf("legacy NAT path returned error: %v", err)
+			}
+			if lease.active {
+				t.Fatal("legacy NAT path unexpectedly activated")
+			}
+			if got := request.Header.Get("Authorization"); got != "" {
+				t.Fatalf("legacy NAT path retained dashboard credential in Authorization %q", got)
+			}
+			if got := request.URL.RawQuery; got != "keep=1" {
+				t.Fatalf("legacy NAT path retained dashboard credential in query %q", got)
+			}
+			if got := request.Header.Get("Cookie"); got != "sid=abc" {
+				t.Fatalf("legacy NAT path retained dashboard credential in cookie %q", got)
+			}
+		})
+		t.Run(name+"/invalid capability path", func(t *testing.T) {
+			request := &http.Request{
+				Header: make(http.Header),
+				URL:    &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + credential},
+			}
+			request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, "malformed")
+			request.Header.Set("Authorization", "Bearer "+credential)
+			request.Header.Set("Cookie", "sid=abc; nz-jwt="+credential)
+
+			lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81})
+
+			if err == nil {
+				t.Fatal("malformed capability unexpectedly accepted")
+			}
+			if lease.active {
+				t.Fatal("malformed capability unexpectedly activated")
+			}
+			if got := request.Header.Get("Authorization"); got != "" {
+				t.Fatalf("invalid capability path retained dashboard credential in Authorization %q", got)
+			}
+			if got := request.URL.RawQuery; got != "keep=1" {
+				t.Fatalf("invalid capability path retained dashboard credential in query %q", got)
+			}
+			if got := request.Header.Get("Cookie"); got != "sid=abc" {
+				t.Fatalf("invalid capability path retained dashboard credential in cookie %q", got)
+			}
+		})
+	}
+}
+
+func TestPrepareNATCapabilityAgentCompatStripsDashboardCredentialAfterConsumingCapability(t *testing.T) {
+	// Given — a well-formed capability that consumes successfully: the request
+	// is about to be tunneled, so the dashboard credential must be gone.
+	installNATCredentialTestGate(t)
+	handler := serviceRPC.NewNezhaHandler()
+	original := serviceRPC.NezhaHandlerSingleton
+	serviceRPC.NezhaHandlerSingleton = handler
+	t.Cleanup(func() { serviceRPC.NezhaHandlerSingleton = original })
+
+	capability, err := handler.RegisterAgentCompatIOStreamCapability(context.Background(), serviceRPC.AgentCompatCapabilityRegistration{
+		Owner:               serviceRPC.AgentCompatCapabilityOwner{PATID: 1, UserID: 2},
+		Purpose:             serviceRPC.AgentCompatCapabilityNAT,
+		TargetServerID:      81,
+		ResourceID:          91,
+		ServerAccessAllowed: true,
+	})
+	if err != nil {
+		t.Fatalf("register capability: %v", err)
+	}
+	request := &http.Request{
+		Header: make(http.Header),
+		URL:    &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + natTestDashboardJWTValue},
+	}
+	request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, capability.String())
+	request.Header.Set("Authorization", natTestDashboardJWT)
+	request.Header.Set("Cookie", "sid=abc; nz-jwt="+natTestDashboardJWTValue)
+
+	// When
+	lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81})
+
+	// Then
+	if err != nil {
+		t.Fatalf("valid NAT capability rejected: %v", err)
+	}
+	if !lease.active {
+		t.Fatal("valid NAT capability did not activate")
+	}
+	if got := request.Header.Get("Authorization"); got != "" {
+		t.Fatalf("dashboard credential retained in Authorization after capability consumption: %q", got)
+	}
+	if got := request.URL.RawQuery; got != "keep=1" {
+		t.Fatalf("dashboard credential retained in query after capability consumption: %q", got)
+	}
+	if got := request.Header.Get("Cookie"); got != "sid=abc" {
+		t.Fatalf("dashboard credential retained in cookie after capability consumption: %q", got)
+	}
+}
+
+func TestPrepareNATCapabilityAgentCompatKeepsForeignAndMissingAuthorization(t *testing.T) {
+	// Given — legacy path: foreign credentials are ordinary request data and
+	// an absent header must stay untouched.
+	installNATCredentialTestGate(t)
+	for name, values := range map[string][]string{
+		"foreign jwt":    {natTestForeignJWT},
+		"foreign random": {natTestForeignRandom},
+		"empty value":    {""},
+		"missing header": nil,
+	} {
+		t.Run(name, func(t *testing.T) {
+			request := &http.Request{Header: make(http.Header)}
+			for _, value := range values {
+				request.Header.Set("Authorization", value)
+			}
+
+			lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81})
+
+			if err != nil {
+				t.Fatalf("legacy NAT path returned error: %v", err)
+			}
+			if lease.active {
+				t.Fatal("legacy NAT path unexpectedly activated")
+			}
+			got := request.Header.Values("Authorization")
+			if len(got) != len(values) {
+				t.Fatalf("legacy NAT path changed Authorization to %q, want %q", got, values)
+			}
+			for i := range values {
+				if got[i] != values[i] {
+					t.Fatalf("legacy NAT path changed Authorization to %q, want %q", got, values)
+				}
+			}
+		})
+	}
+}
+
+func TestPrepareNATCapabilityAgentCompatKeepsForeignAndMissingQueryAndCookie(t *testing.T) {
+	// Given — legacy path: foreign query tokens and cookies are ordinary
+	// request data, and absent channels must stay untouched.
+	installNATCredentialTestGate(t)
+	for name, tc := range map[string]struct{ rawQuery, cookie string }{
+		"foreign values":  {"keep=1&token=" + natTestForeignJWTValue + "&a=%2Fx", "sid=abc; nz-jwt=" + natTestForeignJWTValue},
+		"empty values":    {"keep=1&token=", "nz-jwt="},
+		"missing channel": {"keep=1", "sid=abc"},
+	} {
+		t.Run(name, func(t *testing.T) {
+			request := &http.Request{
+				Header: make(http.Header),
+				URL:    &url.URL{Path: "/nat", RawQuery: tc.rawQuery},
+			}
+			request.Header.Set("Cookie", tc.cookie)
+
+			lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81})
+
+			if err != nil {
+				t.Fatalf("legacy NAT path returned error: %v", err)
+			}
+			if lease.active {
+				t.Fatal("legacy NAT path unexpectedly activated")
+			}
+			if got := request.URL.RawQuery; got != tc.rawQuery {
+				t.Fatalf("legacy NAT path changed query to %q, want %q", got, tc.rawQuery)
+			}
+			if got := request.Header.Get("Cookie"); got != tc.cookie {
+				t.Fatalf("legacy NAT path changed cookie to %q, want %q", got, tc.cookie)
+			}
+		})
+	}
+}
+
 func TestServeNATAgentCompatSensitiveHeadersStayOutOfErrorsAndLogs(t *testing.T) {
+	installNATCredentialTestGate(t)
 	request := &http.Request{Header: make(http.Header)}
 	request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, "capability-secret")
 	request.Header.Set("Authorization", "Bearer deterministic-secret")

cmd/dashboard/rpc/nat_capability_default.go

@@ -18,7 +18,12 @@ type natCapabilityLease struct {
 }
 
 func prepareNATCapability(request *http.Request, _ *model.NAT) (natCapabilityLease, error) {
-	request.Header.Del("Authorization")
+	// The default build has no capability protocol: every request takes the
+	// legacy NAT path, so any dashboard-issued credential (panel JWT or panel
+	// PAT) carried by the Authorization header, the ?token= query parameter or
+	// the nz-jwt cookie is stripped before the request is tunneled to the
+	// agent. Foreign values are ordinary request data and stay put.
+	stripDashboardCredentials(request)
 	return natCapabilityLease{}, nil
 }
 

cmd/dashboard/rpc/nat_capability_default_flow_test.go

@@ -19,6 +19,7 @@ import (
 
 func TestServeNATDefaultForwardsCapabilityHeaderAsOrdinaryData(t *testing.T) {
 	fixture := newServeNATFixture(t)
+	installNATCredentialTestGate(t)
 	connection := newServeNATConn()
 	agent := &serveNATAgent{readErr: io.EOF, writeDone: make(chan struct{})}
 	request := &http.Request{
@@ -28,7 +29,7 @@ func TestServeNATDefaultForwardsCapabilityHeaderAsOrdinaryData(t *testing.T) {
 		Body:   &serveNATBody{},
 	}
 	request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, "ordinary-data")
-	request.Header.Set("Authorization", "Bearer deterministic-secret")
+	request.Header.Set("Authorization", natTestForeignRandom)
 	fixture.taskStream.onSend = func(task *proto.Task) error {
 		var nat model.TaskNAT
 		if err := json.Unmarshal([]byte(task.Data), &nat); err != nil {
@@ -58,7 +59,50 @@ func TestServeNATDefaultForwardsCapabilityHeaderAsOrdinaryData(t *testing.T) {
 	if !strings.Contains(forwarded, strings.ToLower(agentcompatcontract.IOStreamCapabilityHeader)+": ordinary-data") {
 		t.Fatal("default NAT flow did not forward capability header as ordinary request data")
 	}
-	if strings.Contains(forwarded, "Authorization:") || strings.Contains(forwarded, "deterministic-secret") {
-		t.Fatal("default NAT flow forwarded Authorization credentials")
+	if !strings.Contains(forwarded, strings.ToLower("Authorization: "+natTestForeignRandom)) {
+		t.Fatal("default NAT flow stripped a foreign credential that does not belong to the dashboard")
+	}
+}
+
+func TestServeNATDefaultDoesNotForwardDashboardCredentials(t *testing.T) {
+	fixture := newServeNATFixture(t)
+	installNATCredentialTestGate(t)
+	connection := newServeNATConn()
+	agent := &serveNATAgent{readErr: io.EOF, writeDone: make(chan struct{})}
+	request := &http.Request{
+		Method: http.MethodPost,
+		URL:    &url.URL{Scheme: "http", Host: "example.test", Path: "/nat"},
+		Header: make(http.Header),
+		Body:   &serveNATBody{},
+	}
+	request.Header.Set("Authorization", natTestDashboardJWT)
+	fixture.taskStream.onSend = func(task *proto.Task) error {
+		var nat model.TaskNAT
+		if err := json.Unmarshal([]byte(task.Data), &nat); err != nil {
+			return err
+		}
+		return fixture.handler.AgentConnected(nat.StreamID, agent)
+	}
+	done := make(chan struct{})
+	deadline, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	go func() {
+		ServeNAT(&serveNATResponseWriter{conn: connection}, request, &model.NAT{ServerID: fixture.server.ID, Host: "target.example"})
+		close(done)
+	}()
+	select {
+	case <-agent.writeDone:
+	case <-deadline.Done():
+		t.Fatal("agent did not receive ordinary NAT request")
+	}
+	_ = connection.Close()
+	select {
+	case <-done:
+	case <-deadline.Done():
+		t.Fatal("ServeNAT did not finish")
+	}
+	forwarded := strings.ToLower(string(agent.writtenBytes()))
+	if strings.Contains(forwarded, "authorization:") || strings.Contains(forwarded, "dashboard-signed-jwt") {
+		t.Fatal("default NAT flow forwarded dashboard credentials")
 	}
 }

cmd/dashboard/rpc/nat_capability_default_test.go

@@ -4,6 +4,7 @@ package rpc
 
 import (
 	"net/http"
+	"net/url"
 	"testing"
 
 	"github.com/nezhahq/nezha/model"
@@ -12,9 +13,10 @@ import (
 
 func TestPrepareNATCapabilityDefaultPreservesHeaderAsOrdinaryRequestData(t *testing.T) {
 	// Given
+	installNATCredentialTestGate(t)
 	request := &http.Request{Header: make(http.Header)}
 	request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, "ordinary-data")
-	request.Header.Set("Authorization", "Bearer deterministic-secret")
+	request.Header.Set("Authorization", natTestForeignRandom)
 
 	// When
 	lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81})
@@ -29,7 +31,119 @@ func TestPrepareNATCapabilityDefaultPreservesHeaderAsOrdinaryRequestData(t *test
 	if got := request.Header.Get(agentcompatcontract.IOStreamCapabilityHeader); got != "ordinary-data" {
 		t.Fatalf("default NAT capability hook changed header to %q", got)
 	}
-	if got := request.Header.Get("Authorization"); got != "" {
-		t.Fatalf("default NAT capability hook retained Authorization %q", got)
+	if got := request.Header.Get("Authorization"); got != natTestForeignRandom {
+		t.Fatalf("default NAT capability hook changed foreign Authorization to %q", got)
+	}
+}
+
+func TestPrepareNATCapabilityDefaultStripsDashboardCredentials(t *testing.T) {
+	// Given
+	installNATCredentialTestGate(t)
+	for name, credential := range map[string]string{
+		"dashboard jwt":         natTestDashboardJWTValue,
+		"expired dashboard jwt": natTestDashboardExpiredJWTValue,
+		"dashboard api token":   natTestDashboardPATValue,
+	} {
+		t.Run(name, func(t *testing.T) {
+			request := &http.Request{
+				Header: make(http.Header),
+				URL:    &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + credential},
+			}
+			request.Header.Set("Authorization", "Bearer "+credential)
+			request.Header.Set("Cookie", "sid=abc; nz-jwt="+credential)
+
+			// When
+			lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81})
+
+			// Then
+			if err != nil {
+				t.Fatalf("default NAT capability hook returned error: %v", err)
+			}
+			if lease.active {
+				t.Fatal("default NAT capability hook unexpectedly activated")
+			}
+			if got := request.Header.Get("Authorization"); got != "" {
+				t.Fatalf("default NAT capability hook retained dashboard credential in Authorization %q", got)
+			}
+			if got := request.URL.RawQuery; got != "keep=1" {
+				t.Fatalf("default NAT capability hook retained dashboard credential in query %q", got)
+			}
+			if got := request.Header.Get("Cookie"); got != "sid=abc" {
+				t.Fatalf("default NAT capability hook retained dashboard credential in cookie %q", got)
+			}
+		})
+	}
+}
+
+func TestPrepareNATCapabilityDefaultKeepsForeignAndMissingAuthorization(t *testing.T) {
+	// Given
+	installNATCredentialTestGate(t)
+	for name, values := range map[string][]string{
+		"foreign jwt":    {natTestForeignJWT},
+		"foreign random": {natTestForeignRandom},
+		"empty value":    {""},
+		"missing header": nil,
+	} {
+		t.Run(name, func(t *testing.T) {
+			request := &http.Request{Header: make(http.Header)}
+			for _, value := range values {
+				request.Header.Set("Authorization", value)
+			}
+
+			// When
+			lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81})
+
+			// Then
+			if err != nil {
+				t.Fatalf("default NAT capability hook returned error: %v", err)
+			}
+			if lease.active {
+				t.Fatal("default NAT capability hook unexpectedly activated")
+			}
+			got := request.Header.Values("Authorization")
+			if len(got) != len(values) {
+				t.Fatalf("default NAT capability hook changed Authorization to %q, want %q", got, values)
+			}
+			for i := range values {
+				if got[i] != values[i] {
+					t.Fatalf("default NAT capability hook changed Authorization to %q, want %q", got, values)
+				}
+			}
+		})
+	}
+}
+
+func TestPrepareNATCapabilityDefaultKeepsForeignAndMissingQueryAndCookie(t *testing.T) {
+	// Given
+	installNATCredentialTestGate(t)
+	for name, tc := range map[string]struct{ rawQuery, cookie string }{
+		"foreign values":  {"keep=1&token=" + natTestForeignJWTValue + "&a=%2Fx", "sid=abc; nz-jwt=" + natTestForeignJWTValue},
+		"empty values":    {"keep=1&token=", "nz-jwt="},
+		"missing channel": {"keep=1", "sid=abc"},
+	} {
+		t.Run(name, func(t *testing.T) {
+			request := &http.Request{
+				Header: make(http.Header),
+				URL:    &url.URL{Path: "/nat", RawQuery: tc.rawQuery},
+			}
+			request.Header.Set("Cookie", tc.cookie)
+
+			// When
+			lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81})
+
+			// Then
+			if err != nil {
+				t.Fatalf("default NAT capability hook returned error: %v", err)
+			}
+			if lease.active {
+				t.Fatal("default NAT capability hook unexpectedly activated")
+			}
+			if got := request.URL.RawQuery; got != tc.rawQuery {
+				t.Fatalf("default NAT capability hook changed query to %q, want %q", got, tc.rawQuery)
+			}
+			if got := request.Header.Get("Cookie"); got != tc.cookie {
+				t.Fatalf("default NAT capability hook changed cookie to %q, want %q", got, tc.cookie)
+			}
+		})
 	}
 }

cmd/dashboard/rpc/nat_capability_flow_agentcompat_test.go

@@ -23,6 +23,7 @@ import (
 
 func TestServeNATAgentCompatPublishesExactStreamAfterRequestTransfer(t *testing.T) {
 	fixture := newServeNATFixture(t)
+	installNATCredentialTestGate(t)
 	capability, err := fixture.handler.RegisterAgentCompatIOStreamCapability(context.Background(), serviceRPC.AgentCompatCapabilityRegistration{
 		Owner:               serviceRPC.AgentCompatCapabilityOwner{PATID: 1, UserID: 2},
 		Purpose:             serviceRPC.AgentCompatCapabilityNAT,
@@ -44,7 +45,7 @@ func TestServeNATAgentCompatPublishesExactStreamAfterRequestTransfer(t *testing.
 		Body:   io.NopCloser(strings.NewReader("payload")),
 	}
 	request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, capability.String())
-	request.Header.Set("Authorization", "Bearer deterministic-secret")
+	request.Header.Set("Authorization", natTestForeignJWT)
 	request.Header.Set("X-Ordinary-NAT", "ordinary")
 	fixture.taskStream.onSend = func(task *proto.Task) error {
 		var nat model.TaskNAT
@@ -106,8 +107,62 @@ func TestServeNATAgentCompatPublishesExactStreamAfterRequestTransfer(t *testing.
 		require.Contains(t, forwarded, expected)
 	}
 	require.NotContains(t, forwarded, agentcompatcontract.IOStreamCapabilityHeader)
-	require.NotContains(t, forwarded, "Authorization:")
-	require.NotContains(t, forwarded, "deterministic-secret")
+	// Foreign credentials are ordinary request data for the NAT backend and
+	// must survive the transfer; dashboard-issued values are stripped before
+	// any byte leaves the dashboard.
+	require.Contains(t, forwarded, "Authorization: "+natTestForeignJWT)
+}
+
+func TestServeNATAgentCompatDoesNotForwardDashboardCredentials(t *testing.T) {
+	fixture := newServeNATFixture(t)
+	installNATCredentialTestGate(t)
+	capability, err := fixture.handler.RegisterAgentCompatIOStreamCapability(context.Background(), serviceRPC.AgentCompatCapabilityRegistration{
+		Owner:               serviceRPC.AgentCompatCapabilityOwner{PATID: 1, UserID: 2},
+		Purpose:             serviceRPC.AgentCompatCapabilityNAT,
+		TargetServerID:      fixture.server.ID,
+		ResourceID:          91,
+		ServerAccessAllowed: true,
+	})
+	require.NoError(t, err)
+	connection := newServeNATConn()
+	writer := &serveNATResponseWriter{conn: connection}
+	agent := &serveNATAgent{readErr: io.EOF, writeDone: make(chan struct{})}
+	request := &http.Request{
+		Method: http.MethodPost,
+		URL:    &url.URL{Scheme: "http", Host: "example.test", Path: "/nat"},
+		Header: make(http.Header),
+		Body:   io.NopCloser(strings.NewReader("payload")),
+	}
+	request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, capability.String())
+	request.Header.Set("Authorization", natTestDashboardJWT)
+	fixture.taskStream.onSend = func(task *proto.Task) error {
+		var nat model.TaskNAT
+		if err := json.Unmarshal([]byte(task.Data), &nat); err != nil {
+			return err
+		}
+		return fixture.handler.AgentConnected(nat.StreamID, agent)
+	}
+	done := make(chan struct{})
+	go func() {
+		ServeNAT(writer, request, &model.NAT{Common: model.Common{ID: 91}, ServerID: fixture.server.ID, Host: "target.example"})
+		close(done)
+	}()
+	select {
+	case <-agent.writeDone:
+	case <-time.After(time.Second):
+		t.Fatal("agent did not receive the transferred NAT request")
+	}
+	require.NoError(t, connection.Close())
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("ServeNAT did not complete")
+	}
+	forwarded := strings.ToLower(string(agent.writtenBytes()))
+	require.NotContains(t, forwarded, "authorization:")
+	for _, dashboardCredential := range []string{natTestDashboardJWT, natTestDashboardExpiredJWT, natTestDashboardAPIToken} {
+		require.NotContains(t, forwarded, strings.ToLower(dashboardCredential))
+	}
 }
 
 type orderedNATAgent struct {

cmd/dashboard/rpc/nat_credential.go

@@ -0,0 +1,308 @@
+package rpc
+
+import (
+	"context"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+)
+
+// Channel names of the panel's own TokenLookup (controller/jwt.go
+// "header: Authorization, query: token, cookie: nz-jwt"). A dashboard JWT
+// can therefore reach the NAT ingress through the Authorization header, the
+// ?token= query parameter or the nz-jwt cookie, and all three must be
+// classified before the request is tunneled to the agent.
+const (
+	natDashboardJWTCookieName     = "nz-jwt"
+	natDashboardJWTQueryParameter = "token"
+	// Keep credential classification work constant even when an unauthenticated
+	// NAT request contains thousands of token parameters or cookies. Values
+	// beyond this per-request budget fail closed and are stripped.
+	maxNATDashboardCredentialValues = 16
+)
+
+var (
+	natDashboardCredentialGateMu sync.RWMutex
+	// natDashboardCredentialValueGate classifies a bounded batch of raw
+	// credential values. Batching lets the controller resolve all PAT hashes in
+	// one query instead of turning attacker-controlled query/cookie fan-out into
+	// one database query per value.
+	natDashboardCredentialValueGate func(ctx context.Context, values []string) []bool
+)
+
+// SetNATDashboardCredentialGate wires the dashboard-credential classifier
+// used by the NAT ingress. main() injects
+// controller.ClassifyDashboardCredentialValues;
+// the setter keeps the rpc package free of a dependency on the controller
+// package, the same pattern as SetReceiptGateListener and
+// SetMCPKillSwitchObserver. A nil function fails closed: every value on that
+// channel is then treated as dashboard-issued and stripped.
+func SetNATDashboardCredentialGate(valueClass func(ctx context.Context, values []string) []bool) {
+	natDashboardCredentialGateMu.Lock()
+	natDashboardCredentialValueGate = valueClass
+	natDashboardCredentialGateMu.Unlock()
+}
+
+// NATDashboardCredentialGateWired reports whether the dashboard-credential
+// classifier has been wired. It exists so the main() wiring contract can be
+// pinned by a test.
+func NATDashboardCredentialGateWired() bool {
+	natDashboardCredentialGateMu.RLock()
+	wired := natDashboardCredentialValueGate != nil
+	natDashboardCredentialGateMu.RUnlock()
+	return wired
+}
+
+// stripDashboardCredentials removes credentials issued by this dashboard from
+// every channel the panel's own TokenLookup accepts: the Authorization
+// header, the ?token= query parameter and the nz-jwt cookie. Such credentials
+// authenticate the dashboard itself and must never become origin credentials
+// for the NAT backend. Foreign values are forwarded untouched: the NAT
+// backend may authenticate with credentials of its own, and this dashboard
+// must not make policy for them.
+func stripDashboardCredentials(request *http.Request) {
+	classification := classifyNATDashboardCredentialValues(request)
+	stripDashboardAuthorizationHeader(request, classification)
+	stripDashboardCredentialQuery(request, classification)
+	stripDashboardCredentialCookie(request, classification)
+}
+
+type natDashboardCredentialClassification struct {
+	wired   bool
+	limited bool
+	issued  map[string]bool
+}
+
+func (c natDashboardCredentialClassification) isDashboardCredential(value string) bool {
+	if !c.wired {
+		return true
+	}
+	if issued, ok := c.issued[value]; ok {
+		return issued
+	}
+	// Only candidate values collected from this request reach this method. A
+	// missing result therefore means the distinct-value budget was exceeded;
+	// fail closed so a dashboard credential cannot be hidden after junk values.
+	return c.limited
+}
+
+func classifyNATDashboardCredentialValues(request *http.Request) natDashboardCredentialClassification {
+	natDashboardCredentialGateMu.RLock()
+	gate := natDashboardCredentialValueGate
+	natDashboardCredentialGateMu.RUnlock()
+	classification := natDashboardCredentialClassification{
+		wired:  gate != nil,
+		issued: make(map[string]bool),
+	}
+	if gate == nil {
+		return classification
+	}
+
+	values, limited := collectNATDashboardCredentialValues(request)
+	classification.limited = limited
+	results := gate(request.Context(), values)
+	if len(results) != len(values) {
+		// A broken classifier must not turn into credential forwarding.
+		for _, value := range values {
+			classification.issued[value] = true
+		}
+		classification.limited = true
+		return classification
+	}
+	for i, value := range values {
+		classification.issued[value] = results[i]
+	}
+	return classification
+}
+
+func collectNATDashboardCredentialValues(request *http.Request) ([]string, bool) {
+	values := make([]string, 0, maxNATDashboardCredentialValues)
+	seen := make(map[string]struct{}, maxNATDashboardCredentialValues)
+	limited := false
+	add := func(value string) {
+		if _, ok := seen[value]; ok {
+			return
+		}
+		if len(values) == maxNATDashboardCredentialValues {
+			limited = true
+			return
+		}
+		seen[value] = struct{}{}
+		values = append(values, value)
+	}
+
+	for _, authorization := range request.Header.Values("Authorization") {
+		raw := strings.TrimSpace(authorization)
+		if strings.HasPrefix(raw, "Bearer ") {
+			add(strings.TrimPrefix(raw, "Bearer "))
+		}
+	}
+	if request.URL != nil {
+		for _, element := range strings.Split(request.URL.RawQuery, "&") {
+			if value, ok := dashboardCredentialQueryElementValue(element); ok {
+				add(value)
+			}
+		}
+	}
+	for _, line := range request.Header.Values("Cookie") {
+		for _, pair := range strings.Split(line, ";") {
+			if value, ok := dashboardCredentialCookiePairValue(pair); ok {
+				add(value)
+			}
+		}
+	}
+	return values, limited
+}
+
+// stripDashboardAuthorizationHeader removes the Authorization header when it
+// carries a dashboard-issued credential.
+func stripDashboardAuthorizationHeader(request *http.Request, classification natDashboardCredentialClassification) {
+	if !classification.wired {
+		// Gate not wired (early startup or tests): fail closed and keep the
+		// historical behavior of stripping every Authorization header.
+		request.Header.Del("Authorization")
+		return
+	}
+	// A multi-valued Authorization header is malformed traffic, but any
+	// dashboard-issued value in it must still not survive into the tunnel, so
+	// the whole header is dropped when one is recognized.
+	for _, value := range request.Header.Values("Authorization") {
+		raw := strings.TrimSpace(value)
+		if strings.HasPrefix(raw, "Bearer ") && classification.isDashboardCredential(strings.TrimPrefix(raw, "Bearer ")) {
+			request.Header.Del("Authorization")
+			return
+		}
+	}
+}
+
+// stripDashboardCredentialQuery removes ?token= query parameters whose value
+// is a dashboard-issued credential. Every other element of the query string —
+// including foreign token values — is preserved byte-for-byte, so the
+// forwarded request keeps the exact query the client sent.
+func stripDashboardCredentialQuery(request *http.Request, classification natDashboardCredentialClassification) {
+	if request.URL == nil || request.URL.RawQuery == "" {
+		return
+	}
+	elements := strings.Split(request.URL.RawQuery, "&")
+	kept := elements[:0]
+	changed := false
+	for _, element := range elements {
+		if !classification.wired && dashboardCredentialQueryElementName(element) {
+			changed = true
+			continue
+		}
+		value, candidate := dashboardCredentialQueryElementValue(element)
+		if candidate && classification.isDashboardCredential(value) {
+			changed = true
+			continue
+		}
+		kept = append(kept, element)
+	}
+	if changed {
+		request.URL.RawQuery = strings.Join(kept, "&")
+	}
+}
+
+// dashboardCredentialQueryElementValue extracts the decoded value from a raw
+// query element carrying the panel's token parameter.
+func dashboardCredentialQueryElementValue(element string) (string, bool) {
+	rawName, rawValue, _ := strings.Cut(element, "=")
+	// The panel reads the parameter by its decoded name (gin-jwt c.Query),
+	// so an escaped key must be decoded before comparing.
+	name, err := url.QueryUnescape(rawName)
+	if err != nil || name != natDashboardJWTQueryParameter {
+		return "", false
+	}
+	// A value that fails to decode is one the panel's own query parsing drops
+	// as well, so it can never be a panel credential.
+	decoded, err := url.QueryUnescape(rawValue)
+	if err != nil {
+		return "", false
+	}
+	return decoded, true
+}
+
+func dashboardCredentialQueryElementName(element string) bool {
+	rawName, _, _ := strings.Cut(element, "=")
+	name, err := url.QueryUnescape(rawName)
+	return err == nil && name == natDashboardJWTQueryParameter
+}
+
+// stripDashboardCredentialCookie removes nz-jwt cookies whose value is a
+// dashboard-issued credential. All other cookie pairs — including foreign
+// nz-jwt values — are preserved verbatim.
+func stripDashboardCredentialCookie(request *http.Request, classification natDashboardCredentialClassification) {
+	lines := request.Header.Values("Cookie")
+	if len(lines) == 0 {
+		return
+	}
+	changed := false
+	kept := make([]string, 0, len(lines))
+	for _, line := range lines {
+		nextLine, lineChanged := stripDashboardCredentialCookieLine(line, classification)
+		changed = changed || lineChanged
+		if nextLine != "" {
+			kept = append(kept, nextLine)
+		}
+	}
+	if !changed {
+		return
+	}
+	request.Header.Del("Cookie")
+	for _, line := range kept {
+		request.Header.Add("Cookie", line)
+	}
+}
+
+// stripDashboardCredentialCookieLine rewrites a single Cookie header line,
+// dropping the dashboard-issued nz-jwt pairs. Kept pairs keep their raw form,
+// whitespace included — except that a pair following a dropped one no longer
+// carries the removed pair's share of the "; " separator.
+func stripDashboardCredentialCookieLine(line string, classification natDashboardCredentialClassification) (string, bool) {
+	pairs := strings.Split(line, ";")
+	kept := pairs[:0]
+	droppedBeforeFirstKept := true
+	changed := false
+	for _, pair := range pairs {
+		if !classification.wired && dashboardCredentialCookiePairName(pair) {
+			changed = true
+			continue
+		}
+		value, candidate := dashboardCredentialCookiePairValue(pair)
+		if candidate && classification.isDashboardCredential(value) {
+			changed = true
+			continue
+		}
+		if droppedBeforeFirstKept {
+			pair = strings.TrimLeft(pair, " ")
+		}
+		droppedBeforeFirstKept = false
+		kept = append(kept, pair)
+	}
+	if !changed {
+		return line, false
+	}
+	return strings.Join(kept, ";"), true
+}
+
+// dashboardCredentialCookiePairValue extracts a cookie value the same way the
+// panel hands it to the JWT verifier: surrounding whitespace/quotes are
+// presentation, then Gin applies QueryUnescape.
+func dashboardCredentialCookiePairValue(pair string) (string, bool) {
+	name, rawValue, found := strings.Cut(pair, "=")
+	if !found || strings.TrimSpace(name) != natDashboardJWTCookieName {
+		return "", false
+	}
+	value := strings.Trim(strings.TrimSpace(rawValue), `"`)
+	decoded, err := url.QueryUnescape(value)
+	if err != nil {
+		return "", false
+	}
+	return decoded, true
+}
+
+func dashboardCredentialCookiePairName(pair string) bool {
+	name, _, found := strings.Cut(pair, "=")
+	return found && strings.TrimSpace(name) == natDashboardJWTCookieName
+}

cmd/dashboard/rpc/nat_credential_test.go

@@ -0,0 +1,202 @@
+package rpc
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/nezhahq/nezha/model"
+)
+
+// Values standing in for the credentials this dashboard issues. The fake
+// gates below mirror the contract of controller.IsDashboardCredential and
+// controller.IsDashboardCredentialValue; the real classification semantics —
+// panel JWTs accepted regardless of expiry, foreign-signed JWTs and random
+// strings rejected — are pinned by the controller package tests
+// (dashboard_credential_test.go) and by the flow test that wires the real
+// classifiers into ServeNAT (dashboard_credential_nat_flow_test.go).
+const (
+	natTestDashboardJWTValue        = "dashboard-signed-jwt"
+	natTestDashboardExpiredJWTValue = "dashboard-signed-expired-jwt"
+	natTestDashboardPATValue        = model.APITokenPrefix + "dashboard-issued-pat"
+	natTestForeignJWTValue          = "foreign-signed-jwt"
+	natTestForeignRandomValue       = "foreign-random-string"
+
+	natTestDashboardJWT        = "Bearer " + natTestDashboardJWTValue
+	natTestDashboardExpiredJWT = "Bearer " + natTestDashboardExpiredJWTValue
+	natTestDashboardAPIToken   = "Bearer " + natTestDashboardPATValue
+	natTestForeignJWT          = "Bearer " + natTestForeignJWTValue
+	natTestForeignRandom       = "Bearer " + natTestForeignRandomValue
+)
+
+// installNATCredentialTestGate wires fake dashboard-credential classifiers
+// for the duration of the test: exactly the dashboard-issued test values above
+// are classified as dashboard credentials on every channel, everything else
+// is foreign.
+func installNATCredentialTestGate(t *testing.T) {
+	t.Helper()
+	natDashboardCredentialGateMu.RLock()
+	original := natDashboardCredentialValueGate
+	natDashboardCredentialGateMu.RUnlock()
+	SetNATDashboardCredentialGate(func(_ context.Context, values []string) []bool {
+		results := make([]bool, len(values))
+		for i, value := range values {
+			switch value {
+			case natTestDashboardJWTValue, natTestDashboardExpiredJWTValue, natTestDashboardPATValue:
+				results[i] = true
+			}
+		}
+		return results
+	})
+	t.Cleanup(func() {
+		SetNATDashboardCredentialGate(original)
+	})
+}
+
+func TestStripDashboardCredentialsQueryChannel(t *testing.T) {
+	// Given
+	installNATCredentialTestGate(t)
+	request := &http.Request{URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + natTestDashboardJWTValue + "&token=" + natTestForeignJWTValue + "&a=%2Fx"}}
+
+	// When
+	stripDashboardCredentials(request)
+
+	// Then
+	if got := request.URL.RawQuery; got != "keep=1&token="+natTestForeignJWTValue+"&a=%2Fx" {
+		t.Fatalf("query channel kept %q, want dashboard token dropped and foreign values byte-preserved", got)
+	}
+}
+
+func TestStripDashboardCredentialsQueryChannelEscapedValue(t *testing.T) {
+	// Given — a dashboard token percent-encoded in the query must still be
+	// recognized, while the escaped foreign element stays verbatim.
+	installNATCredentialTestGate(t)
+	request := &http.Request{URL: &url.URL{Path: "/nat", RawQuery: "token=" + url.QueryEscape(natTestDashboardJWTValue) + "&token=" + url.QueryEscape(natTestForeignJWTValue)}}
+
+	// When
+	stripDashboardCredentials(request)
+
+	// Then
+	if got := request.URL.RawQuery; got != "token="+url.QueryEscape(natTestForeignJWTValue) {
+		t.Fatalf("query channel kept %q, want escaped dashboard token dropped and foreign element verbatim", got)
+	}
+}
+
+func TestStripDashboardCredentialsCookieChannel(t *testing.T) {
+	// Given
+	installNATCredentialTestGate(t)
+	request := &http.Request{Header: make(http.Header)}
+	request.Header.Set("Cookie", "sid=abc; nz-jwt="+natTestDashboardJWTValue+"; nz-jwt="+natTestForeignJWTValue+"; theme=dark")
+
+	// When
+	stripDashboardCredentials(request)
+
+	// Then
+	if got := request.Header.Get("Cookie"); got != "sid=abc; nz-jwt="+natTestForeignJWTValue+"; theme=dark" {
+		t.Fatalf("cookie channel kept %q, want dashboard pair dropped and foreign pairs verbatim", got)
+	}
+}
+
+func TestStripDashboardCredentialsCookieChannelQuotedValue(t *testing.T) {
+	// Given — quoting is presentation the panel's cookie parsing strips before
+	// verifying, so a quoted dashboard token must be recognized.
+	installNATCredentialTestGate(t)
+	request := &http.Request{Header: make(http.Header)}
+	request.Header.Set("Cookie", `nz-jwt="`+natTestDashboardJWTValue+`"; sid=abc`)
+
+	// When
+	stripDashboardCredentials(request)
+
+	// Then
+	if got := request.Header.Get("Cookie"); got != "sid=abc" {
+		t.Fatalf("cookie channel kept %q, want quoted dashboard token dropped", got)
+	}
+}
+
+func TestStripDashboardCredentialsCookieChannelEscapedValue(t *testing.T) {
+	// Gin's cookie TokenLookup URL-decodes the value before JWT validation.
+	// NAT must classify the same decoded value or an encoded dashboard JWT can
+	// be accepted by the panel and then leak unchanged to the backend.
+	installNATCredentialTestGate(t)
+	request := &http.Request{Header: make(http.Header)}
+	escaped := strings.ReplaceAll(natTestDashboardJWTValue, "-", "%2D")
+	request.Header.Set("Cookie", "nz-jwt="+escaped+"; sid=abc")
+
+	stripDashboardCredentials(request)
+
+	if got := request.Header.Get("Cookie"); got != "sid=abc" {
+		t.Fatalf("cookie channel kept %q, want escaped dashboard token dropped", got)
+	}
+}
+
+func TestStripDashboardCredentialsDeduplicatesAndBoundsClassification(t *testing.T) {
+	natDashboardCredentialGateMu.RLock()
+	original := natDashboardCredentialValueGate
+	natDashboardCredentialGateMu.RUnlock()
+	t.Cleanup(func() { SetNATDashboardCredentialGate(original) })
+
+	classifiedValues := 0
+	gateCalls := 0
+	SetNATDashboardCredentialGate(func(_ context.Context, values []string) []bool {
+		gateCalls++
+		classifiedValues += len(values)
+		return make([]bool, len(values))
+	})
+
+	query := make(url.Values)
+	for range 100 {
+		query.Add("token", model.APITokenPrefix+"same-value")
+	}
+	for i := range 100 {
+		query.Add("token", fmt.Sprintf("%sunique-%d", model.APITokenPrefix, i))
+	}
+	request := &http.Request{Header: make(http.Header), URL: &url.URL{RawQuery: query.Encode()}}
+
+	stripDashboardCredentials(request)
+
+	if gateCalls != 1 {
+		t.Fatalf("classifier called %d times, want one batched call", gateCalls)
+	}
+	if classifiedValues > maxNATDashboardCredentialValues {
+		t.Fatalf("classified %d credential values, want at most %d per request", classifiedValues, maxNATDashboardCredentialValues)
+	}
+	if strings.Contains(request.URL.RawQuery, model.APITokenPrefix+"unique-99") {
+		t.Fatal("credential beyond the classification budget survived; overflow must fail closed")
+	}
+}
+
+func TestPrepareNATCapabilityFailsClosedWithoutGate(t *testing.T) {
+	// Given — unwired gates: every value on every channel is treated as
+	// dashboard-issued and stripped, foreign ones included.
+	SetNATDashboardCredentialGate(nil)
+	t.Cleanup(func() { SetNATDashboardCredentialGate(nil) })
+	request := &http.Request{
+		Header: make(http.Header),
+		URL:    &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + natTestForeignJWTValue + "&token=%ZZ"},
+	}
+	request.Header.Set("Authorization", natTestForeignRandom)
+	request.Header.Set("Cookie", "sid=abc; nz-jwt="+natTestForeignJWTValue+"; nz-jwt=%ZZ")
+
+	// When
+	lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81})
+
+	// Then
+	if err != nil {
+		t.Fatalf("fail-closed NAT capability hook returned error: %v", err)
+	}
+	if lease.active {
+		t.Fatal("fail-closed NAT capability hook unexpectedly activated")
+	}
+	if got := request.Header.Get("Authorization"); got != "" {
+		t.Fatalf("fail-closed hook retained Authorization %q", got)
+	}
+	if got := request.URL.RawQuery; got != "keep=1" {
+		t.Fatalf("fail-closed hook kept query %q, want token parameter stripped", got)
+	}
+	if got := request.Header.Get("Cookie"); got != "sid=abc" {
+		t.Fatalf("fail-closed hook kept cookie %q, want nz-jwt pair stripped", got)
+	}
+}

go.mod

@@ -1,9 +1,9 @@
 module github.com/nezhahq/nezha
 
-go 1.26.5
+go 1.26.10
 
 require (
-	github.com/VictoriaMetrics/VictoriaMetrics v1.148.0
+	github.com/VictoriaMetrics/VictoriaMetrics v1.151.0
 	github.com/appleboy/gin-jwt/v2 v2.10.3
 	github.com/dustinkirkland/golang-petname v0.0.0-20260215035315-f0c533e9ce9b
 	github.com/gin-contrib/pprof v1.5.4
@@ -14,18 +14,18 @@ require (
 	github.com/gorilla/websocket v1.5.3
 	github.com/hashicorp/go-uuid v1.0.3
 	github.com/jinzhu/copier v0.4.0
-	github.com/knadh/koanf/maps v0.1.2
+	github.com/knadh/koanf/maps v0.1.3
 	github.com/knadh/koanf/providers/env v1.1.0
 	github.com/knadh/koanf/providers/file v1.2.1
-	github.com/knadh/koanf/v2 v2.3.5
+	github.com/knadh/koanf/v2 v2.3.6
 	github.com/leonelquinteros/gotext v1.7.2
 	github.com/libdns/cloudflare v0.2.2
 	github.com/libdns/he v1.2.2
 	github.com/libdns/libdns v1.1.1
 	github.com/likexian/whois v1.15.7
 	github.com/likexian/whois-parser v1.24.21
-	github.com/mattn/go-sqlite3 v1.14.49
-	github.com/miekg/dns v1.1.72
+	github.com/mattn/go-sqlite3 v1.14.50
+	github.com/miekg/dns v1.1.73
 	github.com/modelcontextprotocol/go-sdk v1.7.0
 	github.com/nezhahq/libdns-tencentcloud v0.0.0-20260628095405-2ab294ec675b
 	github.com/ory/graceful v0.2.0
@@ -33,20 +33,20 @@ require (
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/sqids/sqids-go v0.4.1
-	github.com/stretchr/testify v1.11.1
+	github.com/stretchr/testify v1.12.1
 	github.com/swaggo/files v1.0.1
 	github.com/swaggo/gin-swagger v1.6.1
 	github.com/swaggo/swag v1.16.6
 	github.com/tidwall/gjson v1.19.0
-	golang.org/x/crypto v0.54.0
-	golang.org/x/exp v0.0.0-20260727155853-b88d891fe743
-	golang.org/x/mod v0.38.0
-	golang.org/x/net v0.57.0
+	golang.org/x/crypto v0.55.0
+	golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa
+	golang.org/x/mod v0.40.0
+	golang.org/x/net v0.58.0
 	golang.org/x/oauth2 v0.36.0
 	golang.org/x/sync v0.22.0
 	golang.org/x/sys v0.47.0
-	google.golang.org/grpc v1.83.0
-	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af
+	google.golang.org/grpc v1.83.2
+	google.golang.org/protobuf v1.36.12
 	gopkg.in/yaml.v3 v3.0.1
 	gorm.io/datatypes v1.2.7
 	gorm.io/driver/sqlite v1.6.0
@@ -66,7 +66,6 @@ require (
 	github.com/bytedance/sonic/loader v0.5.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudwego/base64x v0.1.7 // indirect
-	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/fsnotify/fsnotify v1.10.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.13 // indirect
 	github.com/gin-contrib/sse v1.1.1 // indirect
@@ -91,18 +90,17 @@ require (
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.18.6 // indirect
+	github.com/klauspost/compress v1.19.1 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/likexian/gokit v0.25.16 // indirect
-	github.com/mattn/go-isatty v0.0.22 // indirect
+	github.com/mattn/go-isatty v0.0.23 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/pelletier/go-toml/v2 v2.3.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/quic-go/qpack v0.6.0 // indirect
 	github.com/quic-go/quic-go v0.59.1 // indirect
 	github.com/rogpeppe/go-internal v1.14.1 // indirect
@@ -115,18 +113,18 @@ require (
 	github.com/ugorji/go/codec v1.3.1 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fastrand v1.1.0 // indirect
-	github.com/valyala/gozstd v1.24.0 // indirect
+	github.com/valyala/gozstd v1.25.0 // indirect
 	github.com/valyala/histogram v1.2.0 // indirect
 	github.com/valyala/quicktemplate v1.8.0 // indirect
 	github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
 	go.mongodb.org/mongo-driver/v2 v2.6.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
-	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	go.yaml.in/yaml/v3 v3.0.5 // indirect
 	golang.org/x/arch v0.27.0 // indirect
-	golang.org/x/text v0.40.0 // indirect
+	golang.org/x/text v0.41.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
-	golang.org/x/tools v0.48.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260615183401-62b3387ff324 // indirect
 	gorm.io/driver/mysql v1.5.6 // indirect
+	golang.org/x/tools v0.49.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a // indirect
 )

go.sum

@@ -2,8 +2,8 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
 github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
-github.com/VictoriaMetrics/VictoriaMetrics v1.148.0 h1:qfsMLjvkNnIIE3w3o7Sf1ejpYAE3pGwqAz4OqKNqZYI=
-github.com/VictoriaMetrics/VictoriaMetrics v1.148.0/go.mod h1:/grcppu6T0Y1k8z6e7ACb43sYUKGkvQ4xVVdCANK/d0=
+github.com/VictoriaMetrics/VictoriaMetrics v1.151.0 h1:Svzb/7gPYbzmQnvLjab8h1Jl9olB76mokDE7ykvF/ro=
+github.com/VictoriaMetrics/VictoriaMetrics v1.151.0/go.mod h1:Mp7qEBDOXwP50gC93cF8wJmaGjTFSJP2WA6dRtMI7qU=
 github.com/VictoriaMetrics/easyproto v1.2.0 h1:FJT9uNXA2isppFuJErbLqD306KoFlehl7Wn2dg/6oIE=
 github.com/VictoriaMetrics/easyproto v1.2.0/go.mod h1:QlGlzaJnDfFd8Lk6Ci/fuLxfTo3/GThPs2KH23mv710=
 github.com/VictoriaMetrics/fastcache v1.13.3 h1:rBabE0iIxcqKEMCwUmwHZ9dgEqXerg8FRbRDUvC7OVc=
@@ -46,8 +46,8 @@ github.com/gin-contrib/sse v1.1.1 h1:uGYpNwTacv5R68bSGMapo62iLTRa9l5zxGCps4hK6ko
 github.com/gin-contrib/sse v1.1.1/go.mod h1:QXzuVkA0YO7o/gun03UI1Q+FTI8ZV/n5t03kIQAI89s=
 github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8=
 github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc=
-github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
-github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
+github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-openapi/jsonpointer v0.23.1 h1:1HBACs7XIwR2RcmItfdSFlALhGbe6S92p0ry4d1GWg4=
@@ -133,18 +133,18 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
-github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
+github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
+github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
-github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo=
-github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI=
+github.com/knadh/koanf/maps v0.1.3 h1:P1z7EvTqdFBrPYbzSvorvrpib+sjkUMxf0FVvA5NKK4=
+github.com/knadh/koanf/maps v0.1.3/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI=
 github.com/knadh/koanf/providers/env v1.1.0 h1:U2VXPY0f+CsNDkvdsG8GcsnK4ah85WwWyJgef9oQMSc=
 github.com/knadh/koanf/providers/env v1.1.0/go.mod h1:QhHHHZ87h9JxJAn2czdEl6pdkNnDh/JS1Vtsyt65hTY=
 github.com/knadh/koanf/providers/file v1.2.1 h1:bEWbtQwYrA+W2DtdBrQWyXqJaJSG3KrP3AESOJYp9wM=
 github.com/knadh/koanf/providers/file v1.2.1/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA=
-github.com/knadh/koanf/v2 v2.3.5 h1:2dXJUYaKGm4SGYeoAtBviq9+02JZo/pxQ2ssOd60rJg=
-github.com/knadh/koanf/v2 v2.3.5/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28=
+github.com/knadh/koanf/v2 v2.3.6 h1:JoQPSJmvS4aP0xNc8xMDr5tcrkSEInL23/Il7pITAKo=
+github.com/knadh/koanf/v2 v2.3.6/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -167,12 +167,18 @@ github.com/likexian/whois-parser v1.24.21 h1:MxsrGRxDOiZIVp7q7N/yAIbKuN4QAkGjCpO
 github.com/likexian/whois-parser v1.24.21/go.mod h1:o3DUruO65Pb8WXCJCTlSVkTbwuYVrBCeoMTw2q0mxY4=
 github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
 github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
+github.com/mattn/go-isatty v0.0.23 h1:cYwCQTQf3HB6xUC+BtyCLZNr7IzbOmoZbmssVNzSyiQ=
+github.com/mattn/go-isatty v0.0.23/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
 github.com/mattn/go-sqlite3 v1.14.49 h1:B8jBHC3xhxZgxztrgruTuLucebnULQnx4W7cF7SAE9w=
 github.com/mattn/go-sqlite3 v1.14.49/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w=
+github.com/mattn/go-sqlite3 v1.14.50 h1:dmdFvo1XG4MPzA4IkAmE9upVz/Nj31uRoM5+jC8hYbY=
+github.com/mattn/go-sqlite3 v1.14.50/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w=
 github.com/microsoft/go-mssqldb v1.7.2 h1:CHkFJiObW7ItKTJfHo1QX7QBBD1iV+mn1eOyRP3b/PA=
 github.com/microsoft/go-mssqldb v1.7.2/go.mod h1:kOvZKUdrhhFQmxLZqbwUV0rHkNkZpthMITIb2Ko1IoA=
 github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
 github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
+github.com/miekg/dns v1.1.73 h1:uhT8nJxmTrPJYClxVxTCX+CVn6qnzSiybRk72Z6DgrE=
+github.com/miekg/dns v1.1.73/go.mod h1:RW2Obtfd5NZHvOFe3zYG0W8koWOQtAzyHaLo8vASBuQ=
 github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
 github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
@@ -198,8 +204,6 @@ github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
-github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
 github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
 github.com/quic-go/quic-go v0.59.1 h1:0Gmua0HW1Tv7ANR7hUYwRyD0MG5OJfgvYSZasGZzBic=
@@ -223,8 +227,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
-github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
+github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
 github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE=
 github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg=
 github.com/swaggo/gin-swagger v1.6.1 h1:Ri06G4gc9N4t4k8hekMigJ9zKTFSlqj/9paAQCQs7cY=
@@ -250,8 +254,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
 github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
-github.com/valyala/gozstd v1.24.0 h1:M/9L3h7bVwbj2gZwrmuoaxzwVrmBUvos2jG9cZtuhlc=
-github.com/valyala/gozstd v1.24.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
+github.com/valyala/gozstd v1.25.0 h1:7gS7+5zwidZT1BFQqGPAPII8ekZ3tvYTp5IvOEWC34Y=
+github.com/valyala/gozstd v1.25.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
 github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ=
 github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY=
 github.com/valyala/quicktemplate v1.8.0 h1:zU0tjbIqTRgKQzFY1L42zq0qR3eh4WoQQdIdqCysW5k=
@@ -279,25 +283,25 @@ go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
 go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
 go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
 go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
-go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
-go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
+go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
 golang.org/x/arch v0.27.0 h1:0WNVcR8u9yFz8j5FvdHpgwNp3FS5U4guYdzHwEiGjoU=
 golang.org/x/arch v0.27.0/go.mod h1:0X+GdSIP+kL5wPmpK7sdkEVTt2XoYP0cSjQSbZBwOi8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
-golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
-golang.org/x/exp v0.0.0-20260727155853-b88d891fe743 h1:ex206bKw+v3K0dm3andkrIF+ijyQKJG1pLgwQ2PYdQM=
-golang.org/x/exp v0.0.0-20260727155853-b88d891fe743/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q=
+golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
+golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
+golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk=
+golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
-golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
+golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
+golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
-golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
+golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
+golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
 golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
 golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -319,24 +323,24 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
-golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
+golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
+golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
 golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
 golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
-golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
+golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
+golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
 gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260615183401-62b3387ff324 h1:9HZDLIdYBJXAnaFOr9WHrKVycfpY+75s9HGadC0305A=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260615183401-62b3387ff324/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
-google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=
-google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
-google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
-google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a h1:qI/YMH1ep2qQtqcp00gMQyoU7mjvbhg88GJKCvfoLj0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
+google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
+google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
+google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

integration/agentcompat/internal/workflowpolicy/quality_workflow_test.go

@@ -169,7 +169,7 @@ func requireCheckoutAndSetupGo(t *testing.T, steps []qualityStep) {
 	require.False(t, *checkout.With.PersistCredentials)
 	setupGo := steps[1]
 	require.Equal(t, "actions/setup-go@v7", setupGo.Uses)
-	require.Equal(t, "1.26.x", setupGo.With.GoVersion)
+	require.Equal(t, "^1.26.10", setupGo.With.GoVersion)
 	require.NotNil(t, setupGo.With.Cache)
 	require.False(t, *setupGo.With.Cache)
 }

integration/agentcompat/internal/workflowpolicy/stress_workflow_test.go

@@ -57,7 +57,7 @@ func TestPolicy_NezhaStressWorkflowRunsCrossRepositoryTest(t *testing.T) {
 
 	setupGo := stressJob.stepNamed(t, "Set up Go")
 	require.Equal(t, "actions/setup-go@v7", setupGo.Uses)
-	require.Equal(t, "1.26.x", setupGo.With.GoVersion)
+	require.Equal(t, "^1.26.10", setupGo.With.GoVersion)
 	require.False(t, *setupGo.With.Cache)
 
 	prepareDashboardInputs := stressJob.stepNamed(t, "Prepare Dashboard build inputs")

model/alert_evaluation_test.go

@@ -0,0 +1,205 @@
+package model
+
+import (
+	"math"
+	"testing"
+	"time"
+
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+)
+
+func TestAlertEvaluationDistinguishesUnknownFromBreach(t *testing.T) {
+	server := &Server{Common: Common{ID: 1}}
+	rule := &Rule{Type: "cpu", Max: 80, Cover: RuleCoverAll}
+	if passed, known := rule.Evaluate(nil, server, nil); !passed || known {
+		t.Fatalf("missing state = (%v, %v), want unknown", passed, known)
+	}
+	lease := server.AttachStateStream(runtimeOwnershipStream{})
+	if !lease.UpdateState(&HostState{CPU: 99}, time.Now()) {
+		t.Fatal("state update rejected")
+	}
+	if passed, known := rule.Evaluate(nil, server, nil); passed || !known {
+		t.Fatalf("fresh high CPU = (%v, %v), want known breach", passed, known)
+	}
+	if !lease.Clear() {
+		t.Fatal("stream clear rejected")
+	}
+	// The raw metric remains available for historical display. The sentinel
+	// enforces freshness and must not count this value after disconnect.
+	if !server.RuntimeSnapshot().LastActive.IsZero() {
+		t.Fatal("disconnect did not clear online visibility")
+	}
+}
+
+func newCycleTestDB(t *testing.T, migrate bool) *gorm.DB {
+	t.Helper()
+	db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if migrate {
+		if err := db.AutoMigrate(&Transfer{}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return db
+}
+
+func newCycleTestStats() *CycleTransferStats {
+	return &CycleTransferStats{
+		ServerName: make(map[uint64]string),
+		Transfer:   make(map[uint64]uint64),
+		NextUpdate: make(map[uint64]time.Time),
+	}
+}
+
+func TestCycleRuleRejectsInvalidConfiguration(t *testing.T) {
+	start := time.Now().Add(-time.Hour)
+	rule := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Max: 100}
+	if !rule.HasSafeCycleConfiguration() {
+		t.Fatal("normal hourly cycle rejected")
+	}
+	for _, tc := range []struct {
+		name   string
+		mutate func(*Rule)
+	}{
+		{"unknown unit", func(r *Rule) { r.CycleUnit = "fortnight" }},
+		{"negative limit", func(r *Rule) { r.Max = -1 }},
+		{"NaN limit", func(r *Rule) { r.Max = math.NaN() }},
+		{"reversed limits", func(r *Rule) { r.Min, r.Max = 200, 100 }},
+		{"future start", func(r *Rule) { future := time.Now().Add(time.Hour); r.CycleStart = &future }},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			copy := *rule
+			tc.mutate(&copy)
+			if copy.HasSafeCycleConfiguration() {
+				t.Fatal("unsafe cycle accepted")
+			}
+		})
+	}
+}
+
+func TestCycleBoundsUseOneClockAtBoundary(t *testing.T) {
+	start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
+	rule := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Max: 100}
+	before := start.Add(59*time.Minute + 59*time.Second)
+	from, to := rule.transferDurationBounds(before)
+	if !from.Equal(start) || !to.Equal(start.Add(time.Hour)) {
+		t.Fatalf("before boundary = %v..%v", from, to)
+	}
+	from, to = rule.transferDurationBounds(start.Add(time.Hour))
+	if !from.Equal(start.Add(time.Hour)) || !to.Equal(start.Add(2*time.Hour)) {
+		t.Fatalf("at boundary = %v..%v", from, to)
+	}
+}
+
+func TestCycleQueryFailureIsUnknownAndDoesNotCache(t *testing.T) {
+	start := time.Now().Add(-time.Hour)
+	rule := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Max: 50}
+	server := &Server{Common: Common{ID: 1}, State: &HostState{}}
+	passed, known := rule.Evaluate(newCycleTestStats(), server, newCycleTestDB(t, false))
+	if !passed || known || len(rule.NextTransferAt) != 0 {
+		t.Fatalf("SQL error yielded passed=%v known=%v cache=%v", passed, known, rule.NextTransferAt)
+	}
+}
+
+func TestCycleQueryExcludesFutureRowsAndHandlesMinOnly(t *testing.T) {
+	start := time.Now().Add(-90 * time.Minute)
+	db := newCycleTestDB(t, true)
+	server := &Server{Common: Common{ID: 1}, State: &HostState{}}
+	for _, row := range []Transfer{
+		{Common: Common{CreatedAt: time.Now().Add(-time.Minute)}, ServerID: 1, Out: 10},
+		{Common: Common{CreatedAt: time.Now().Add(10 * time.Minute)}, ServerID: 1, Out: 100},
+	} {
+		if err := db.Create(&row).Error; err != nil {
+			t.Fatal(err)
+		}
+	}
+	rule := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Max: 50}
+	stats := newCycleTestStats()
+	passed, known := rule.Evaluate(stats, server, db)
+	if !passed || !known || stats.Transfer[1] != 10 {
+		t.Fatalf("future row included: passed=%v known=%v amount=%d", passed, known, stats.Transfer[1])
+	}
+	minOnly := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Min: 20}
+	passed, known = minOnly.Evaluate(newCycleTestStats(), server, db)
+	if passed || !known {
+		t.Fatalf("min-only cycle = (%v,%v), want known breach", passed, known)
+	}
+}
+
+func TestAlertEvaluationAllNetworkSpeedUsesBothDirections(t *testing.T) {
+	for _, tc := range []struct {
+		name    string
+		in, out uint64
+		breach  bool
+	}{
+		{"inbound only", 100, 0, true},
+		{"outbound below combined limit", 0, 30, false},
+		{"both directions", 25, 30, true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			server := &Server{Common: Common{ID: 1}, State: &HostState{NetInSpeed: tc.in, NetOutSpeed: tc.out}}
+			rule := &Rule{Type: "net_all_speed", Max: 50, Cover: RuleCoverAll}
+			passed, known := rule.Evaluate(nil, server, nil)
+			if !known || passed == tc.breach {
+				t.Fatalf("got passed=%v known=%v, want breach=%v", passed, known, tc.breach)
+			}
+		})
+	}
+}
+
+func TestAlertEvaluationUnavailableSensorsAreUnknown(t *testing.T) {
+	tests := []struct {
+		name  string
+		rule  string
+		state *HostState
+		host  *Host
+	}{
+		{"missing memory host", "memory", &HostState{MemUsed: 1}, nil},
+		{"zero memory total", "memory", &HostState{MemUsed: 1}, &Host{}},
+		{"missing GPU", "gpu_max", &HostState{}, &Host{}},
+		{"missing temperature", "temperature_max", &HostState{}, &Host{}},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			server := &Server{Common: Common{ID: 1}, State: tc.state, Host: tc.host}
+			passed, known := (&Rule{Type: tc.rule, Max: 50, Cover: RuleCoverAll}).Evaluate(nil, server, nil)
+			if !passed || known {
+				t.Fatalf("got passed=%v known=%v, want unknown", passed, known)
+			}
+		})
+	}
+}
+
+func TestOfflineRuleDoesNotRequireMetricState(t *testing.T) {
+	server := &Server{Common: Common{ID: 1}}
+	InitServer(server)
+	passed, known := (&Rule{Type: "offline", Cover: RuleCoverAll}).Evaluate(nil, server, nil)
+	if passed || !known {
+		t.Fatalf("offline without state = (%v, %v), want known breach", passed, known)
+	}
+}
+
+func TestCompoundAlertUsesOneImmutableAgentReport(t *testing.T) {
+	server := &Server{Common: Common{ID: 1}}
+	alert := &AlertRule{Rules: []*Rule{
+		{Type: "cpu", Max: 80, Cover: RuleCoverAll},
+		{Type: "memory", Max: 80, Cover: RuleCoverAll},
+	}}
+	oldReport := RuntimeSnapshot{
+		State: &HostState{CPU: 99, MemUsed: 0},
+		Host:  &Host{MemTotal: 100},
+	}
+	newReport := RuntimeSnapshot{
+		State: &HostState{CPU: 0, MemUsed: 99},
+		Host:  &Host{MemTotal: 100},
+	}
+	for _, report := range []RuntimeSnapshot{oldReport, newReport} {
+		point, known := alert.SnapshotStatusWithRuntime(nil, server, report, nil)
+		if !known || len(point) != 2 || point[0] == point[1] {
+			t.Fatalf("compound point combined incompatible reports: known=%v point=%v", known, point)
+		}
+	}
+}

model/alert_timed_test.go

@@ -0,0 +1,104 @@
+package model
+
+import (
+	"testing"
+	"time"
+)
+
+func timedRule(kind string, duration uint64) *AlertRule {
+	return &AlertRule{Rules: []*Rule{{Type: kind, Duration: duration, Max: 80, Cover: RuleCoverAll}}}
+}
+
+func timedPoints(start time.Time, values []bool, spacing time.Duration) []TimedAlertPoint {
+	points := make([]TimedAlertPoint, len(values))
+	for i, value := range values {
+		points[i] = TimedAlertPoint{At: start.Add(time.Duration(i) * spacing), Values: []bool{value}}
+	}
+	return points
+}
+
+func TestTimedAlertNeedsElapsedWindowAndDistinctReports(t *testing.T) {
+	start := time.Unix(1_700_000_000, 0)
+	rule := timedRule("cpu", 5)
+	points := timedPoints(start, []bool{false, false, false, false}, AlertSampleInterval)
+	if known, _ := rule.CheckTimed(points, points[len(points)-1].At); known {
+		t.Fatal("four reports decided a five-tick window")
+	}
+	points = append(points, TimedAlertPoint{At: start.Add(12 * time.Second), Values: []bool{false}})
+	if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || passed {
+		t.Fatalf("complete high-CPU window: known=%v passed=%v", known, passed)
+	}
+	// Repeating the same observation does not advance wall-clock coverage.
+	if known, _ := rule.CheckTimed(points, start.Add(30*time.Second)); known {
+		t.Fatal("stale sample created a new complete window")
+	}
+}
+
+func TestTimedAlertGapIsUnknownAndCannotResolve(t *testing.T) {
+	start := time.Unix(1_700_000_000, 0)
+	rule := timedRule("cpu", 5)
+	points := []TimedAlertPoint{
+		{At: start, Values: []bool{false}},
+		{At: start.Add(30 * time.Second), Values: []bool{true}},
+	}
+	if known, _ := rule.CheckTimed(points, start.Add(30*time.Second)); known {
+		t.Fatal("telemetry gap must not be interpreted as healthy")
+	}
+}
+
+func TestTimedAlertIrregularReportsUseElapsedTime(t *testing.T) {
+	start := time.Unix(1_700_000_000, 0)
+	rule := timedRule("cpu", 6)
+	points := timedPoints(start, []bool{false, false, false, false}, 5*time.Second)
+	if known, passed := rule.CheckTimed(points, start.Add(15*time.Second)); !known || passed {
+		t.Fatalf("irregular reports cover the wall-clock window: known=%v passed=%v", known, passed)
+	}
+}
+
+func TestTimedAlertExactSeventyPercentIsNotIncident(t *testing.T) {
+	start := time.Unix(1_700_000_000, 0)
+	rule := timedRule("cpu", 10)
+	values := []bool{false, false, false, false, false, false, false, true, true, true}
+	points := timedPoints(start, values, AlertSampleInterval)
+	if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || !passed {
+		t.Fatalf("70%% failed is below the >70%% trigger: known=%v passed=%v", known, passed)
+	}
+}
+
+func TestTimedAlertTenTickWindowDoesNotDecideAtNine(t *testing.T) {
+	start := time.Unix(1_700_000_000, 0)
+	rule := timedRule("cpu", 10)
+	points := timedPoints(start, []bool{false, false, false, false, false, false, false, false, false}, AlertSampleInterval)
+	if known, _ := rule.CheckTimed(points, points[len(points)-1].At); known {
+		t.Fatal("nine reports decided a ten-tick window one tick early")
+	}
+	points = append(points, TimedAlertPoint{At: start.Add(27 * time.Second), Values: []bool{false}})
+	if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || passed {
+		t.Fatalf("full ten-tick breach: known=%v passed=%v", known, passed)
+	}
+}
+
+func TestTimedOfflineRequiresSustainedAbsence(t *testing.T) {
+	start := time.Unix(1_700_000_000, 0)
+	rule := timedRule("offline", 5)
+	points := timedPoints(start, []bool{false, false, false, false, false}, AlertSampleInterval)
+	if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || passed {
+		t.Fatalf("continuous offline interval: known=%v passed=%v", known, passed)
+	}
+	points = append(points, TimedAlertPoint{At: start.Add(15 * time.Second), Values: []bool{true}})
+	if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || !passed {
+		t.Fatalf("online observation should resolve offline: known=%v passed=%v", known, passed)
+	}
+}
+
+func TestTimedCompoundRuleWaitsForAllKnown(t *testing.T) {
+	start := time.Unix(1_700_000_000, 0)
+	rule := &AlertRule{Rules: []*Rule{
+		{Type: "cpu", Duration: 5, Max: 80, Cover: RuleCoverAll},
+		{Type: "memory", Duration: 5, Max: 80, Cover: RuleCoverAll},
+	}}
+	points := timedPoints(start, []bool{false, false, false, false, false}, AlertSampleInterval)
+	if known, _ := rule.CheckTimed(points, points[len(points)-1].At); known {
+		t.Fatal("missing second condition treated as decisive")
+	}
+}

model/alertrule.go

@@ -2,12 +2,134 @@ package model
 
 import (
 	"slices"
+	"time"
 
 	"github.com/gin-gonic/gin"
 	"github.com/goccy/go-json"
 	"gorm.io/gorm"
 )
 
+const (
+	AlertSampleInterval = 3 * time.Second
+	AlertSampleMaxAge   = 6 * time.Second
+)
+
+// TimedAlertPoint records one distinct observation. Its timestamp is the
+// dashboard receipt time of the Agent report, or the sentinel tick for an
+// offline-only rule.
+type TimedAlertPoint struct {
+	At     time.Time
+	Values []bool
+}
+
+// CheckTimed evaluates the legacy Duration as Duration * 3 seconds, avoiding
+// both repeated stale samples and cadence-dependent early incidents. Unknown
+// coverage cannot create an incident or a recovery.
+func (r *AlertRule) CheckTimed(points []TimedAlertPoint, now time.Time) (known, passed bool) {
+	if r == nil || len(r.Rules) == 0 {
+		return false, false
+	}
+	allFailed := true
+	for index, rule := range r.Rules {
+		if rule == nil || !rule.IsSupportedType() {
+			return false, false
+		}
+		ruleKnown, rulePassed := rule.checkTimed(points, index, now)
+		if !ruleKnown {
+			return false, false
+		}
+		if rulePassed {
+			allFailed = false
+		}
+	}
+	return true, !allFailed
+}
+
+func (rule *Rule) checkTimed(points []TimedAlertPoint, index int, now time.Time) (known, passed bool) {
+	if len(points) == 0 {
+		return false, false
+	}
+	if rule.IsTransferDurationRule() {
+		last := points[len(points)-1]
+		if index >= len(last.Values) || now.Sub(last.At) > AlertSampleMaxAge {
+			return false, false
+		}
+		return true, last.Values[index]
+	}
+	window := time.Duration(rule.Duration) * AlertSampleInterval
+	if window <= 0 {
+		return false, false
+	}
+	end := now.Add(AlertSampleInterval) // the current observation represents one tick
+	start := end.Add(-window)
+	if points[0].At.After(start) {
+		return false, false // the configured wall-clock window has not elapsed
+	}
+	var covered, failed time.Duration
+	for i, point := range points {
+		if index >= len(point.Values) {
+			return false, false
+		}
+		segmentStart := point.At
+		if start.After(segmentStart) {
+			segmentStart = start
+		}
+		segmentEnd := point.At.Add(AlertSampleMaxAge)
+		if end.Before(segmentEnd) {
+			segmentEnd = end
+		}
+		if i+1 < len(points) {
+			if points[i+1].At.Before(segmentEnd) {
+				segmentEnd = points[i+1].At
+			}
+		} else {
+			if tickEnd := point.At.Add(AlertSampleInterval); tickEnd.Before(segmentEnd) {
+				segmentEnd = tickEnd
+			}
+		}
+		if !segmentEnd.After(segmentStart) {
+			continue
+		}
+		span := segmentEnd.Sub(segmentStart)
+		covered += span
+		if !point.Values[index] {
+			failed += span
+		}
+	}
+	if covered < window-window/10 {
+		return false, false
+	}
+	if rule.IsOfflineRule() {
+		if covered-failed >= AlertSampleInterval {
+			return true, true
+		}
+		if failed >= window-AlertSampleInterval && covered >= window-AlertSampleInterval {
+			return true, false
+		}
+		return false, false
+	}
+	if float64(failed) > float64(window)*0.70 {
+		return true, false
+	}
+	if float64(failed+window-covered) <= float64(window)*0.70 {
+		return true, true
+	}
+	return false, false
+}
+
+func (r *AlertRule) TimedRetention() time.Duration {
+	longest := AlertSampleMaxAge
+	for _, rule := range r.Rules {
+		if rule == nil || rule.IsTransferDurationRule() {
+			continue
+		}
+		if duration, ok := rule.DurationInt(); ok {
+			longest = max(longest, time.Duration(duration)*AlertSampleInterval+AlertSampleMaxAge)
+		}
+	}
+	return longest
+}
+
 const (
 	ModeAlwaysTrigger  = 0
 	ModeOnetimeTrigger = 1
@@ -64,6 +186,39 @@ func (r *AlertRule) Enabled() bool {
 	return r.Enable != nil && *r.Enable
 }
 
+// IsSafeToEvaluate validates persisted rule data before it reaches the alert
+// goroutine. API validation protects new writes; this second boundary protects
+// upgrades from malformed or deliberately poisoned rows already in the DB.
+func (r *AlertRule) IsSafeToEvaluate() bool {
+	if r == nil || len(r.Rules) == 0 {
+		return false
+	}
+	for _, rule := range r.Rules {
+		if rule == nil || !rule.IsSupportedType() {
+			return false
+		}
+		if !rule.HasSafeThresholds() {
+			return false
+		}
+		switch rule.Cover {
+		case RuleCoverAll, RuleCoverIgnoreAll:
+		default:
+			return false
+		}
+		if rule.IsTransferDurationRule() {
+			if !rule.HasSafeCycleConfiguration() {
+				return false
+			}
+			continue
+		}
+		duration, ok := rule.DurationInt()
+		if !ok || duration < 3 {
+			return false
+		}
+	}
+	return true
+}
+
 // HasPermission extends the default owner/admin check with PAT
 // server_ids whitelist enforcement. AlertRule.Snapshot fans out across
 // every owner-visible server filtered only by Rule.Ignore semantics
@@ -122,22 +277,52 @@ func (r *AlertRule) HasPermission(ctx *gin.Context) bool {
 
 // Snapshot 对传入的Server进行该报警规则下所有type的检查 返回每项检查结果
 func (r *AlertRule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) []bool {
+	point, _ := r.SnapshotStatus(cycleTransferStats, server, db)
+	return point
+}
+
+// SnapshotStatus preserves the distinction between a healthy measurement and
+// telemetry that is unavailable. A compound alert cannot be decided while any
+// of its applicable conditions is unknown.
+func (r *AlertRule) SnapshotStatus(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) ([]bool, bool) {
+	if server == nil {
+		return nil, false
+	}
+	return r.SnapshotStatusWithRuntime(cycleTransferStats, server, server.RuntimeSnapshot(), db)
+}
+
+// SnapshotStatusWithRuntime uses one immutable report for every condition.
+func (r *AlertRule) SnapshotStatusWithRuntime(cycleTransferStats *CycleTransferStats, server *Server, runtime RuntimeSnapshot, db *gorm.DB) ([]bool, bool) {
 	point := make([]bool, len(r.Rules))
+	known := true
 
 	for i, rule := range r.Rules {
-		point[i] = rule.Snapshot(cycleTransferStats, server, db)
+		if rule == nil || !rule.IsSupportedType() {
+			// Invalid persisted rules are ignored instead of being interpreted as
+			// a failed condition or allowed to panic the sentinel.
+			point[i] = true
+			known = false
+			continue
+		}
+		var valid bool
+		point[i], valid = rule.EvaluateRuntime(cycleTransferStats, server, runtime, db)
+		known = known && valid
 	}
-	return point
+	return point, known
 }
 
 // Check 传入包含当前报警规则下所有type检查结果 返回报警持续时间与是否通过报警检查(通过则返回true)
 func (r *AlertRule) Check(points [][]bool) (int, bool) {
 	var hasPassedRule bool
 	durations := make([]int, len(r.Rules))
+	validRules := 0
 
 	for ruleIndex, rule := range r.Rules {
-		duration := int(rule.Duration)
+		if rule == nil || !rule.IsSupportedType() {
+			continue
+		}
 		if rule.IsTransferDurationRule() {
+			validRules++
 			// 循环区间流量报警
 			if durations[ruleIndex] < 1 {
 				durations[ruleIndex] = 1
@@ -146,10 +331,21 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) {
 				continue
 			}
 			// 只要最后一次检查超出了规则范围 就认为检查未通过
-			if len(points) > 0 && points[len(points)-1][ruleIndex] {
-				hasPassedRule = true
+			if len(points) > 0 {
+				lastPoint := points[len(points)-1]
+				if ruleIndex >= len(lastPoint) || lastPoint[ruleIndex] {
+					hasPassedRule = true
+				}
 			}
-		} else if rule.IsOfflineRule() {
+			continue
+		}
+
+		duration, ok := rule.DurationInt()
+		if !ok || duration <= 0 {
+			continue
+		}
+		validRules++
+		if rule.IsOfflineRule() {
 			// 离线报警，检查直到最后一次在线的离线采样点是否大于 duration
 			if hasPassedRule = boundCheck(len(points), duration, hasPassedRule); hasPassedRule {
 				continue
@@ -157,7 +353,7 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) {
 			var fail int
 			for _, point := range slices.Backward(points[len(points)-duration:]) {
 				fail++
-				if point[ruleIndex] {
+				if ruleIndex >= len(point) || point[ruleIndex] {
 					hasPassedRule = true
 					break
 				}
@@ -168,11 +364,7 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) {
 			// 常规报警
 			// duration<=0 是无意义的规则（持续 0 秒）：直接跳过该规则，
 			// 既不污染 hasPassedRule（否则会连带跳过同一 alert 里其它有效
-			// 规则），也避免下方 fail*100/total 在 total=0 时整数除零 panic
-			// —— checkStatus 无 recover，一次 panic 会拖垮整个告警 goroutine。
-			if duration <= 0 {
-				continue
-			}
+			// 规则），也避免下方百分比计算在 total=0 时除零。
 			if hasPassedRule = boundCheck(len(points), duration, hasPassedRule); hasPassedRule {
 				continue
 			}
@@ -181,17 +373,20 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) {
 			}
 			total, fail := duration, 0
 			for timeTick := len(points) - duration; timeTick < len(points); timeTick++ {
-				if !points[timeTick][ruleIndex] {
+				if ruleIndex >= len(points[timeTick]) || !points[timeTick][ruleIndex] {
 					fail++
 				}
 			}
 			// 当70%以上的采样点未通过规则判断时 才认为当前检查未通过
-			if fail*100/total <= 70 {
+			if float64(fail)*100/float64(total) <= 70 {
 				hasPassedRule = true
 			}
 		}
 	}
 
+	if validRules == 0 {
+		return 0, true
+	}
 	// 仅当所有检查均未通过时 才触发告警
 	return slices.Max(durations), hasPassedRule
 }
@@ -205,10 +400,13 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) {
 func (r *AlertRule) RetentionWindow() int {
 	window := 0
 	for _, rule := range r.Rules {
+		if rule == nil || !rule.IsSupportedType() {
+			continue
+		}
 		var need int
 		if rule.IsTransferDurationRule() {
 			need = 1
-		} else if d := int(rule.Duration); d > 0 {
+		} else if d, ok := rule.DurationInt(); ok && d > 0 {
 			need = d
 		}
 		if need > window {

model/alertrule_security_test.go

@@ -0,0 +1,123 @@
+package model
+
+import (
+	"math"
+	"testing"
+	"time"
+)
+
+func TestRuleSnapshotIgnoresEmptyMetricSamples(t *testing.T) {
+	tests := []struct {
+		name      string
+		ruleType  string
+		hostState *HostState
+	}{
+		{
+			name:      "GPU list is empty",
+			ruleType:  "gpu_max",
+			hostState: &HostState{},
+		},
+		{
+			name:     "all temperature samples are filtered",
+			ruleType: "temperature_max",
+			hostState: &HostState{Temperatures: []SensorTemperature{
+				{Name: "coretemp", Temperature: 0},
+			}},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			server := &Server{Common: Common{ID: 1}, State: test.hostState, Host: &Host{}}
+			rule := &Rule{Type: test.ruleType, Max: 1, Duration: 3, Cover: RuleCoverAll}
+
+			defer func() {
+				if recovered := recover(); recovered != nil {
+					t.Fatalf("Snapshot panicked for missing metric data: %v", recovered)
+				}
+			}()
+			if passed := rule.Snapshot(nil, server, nil); !passed {
+				t.Fatal("missing metric data must be ignored instead of treated as a failed threshold")
+			}
+		})
+	}
+}
+
+func TestAlertRuleCheckRejectsOverflowingPersistedDuration(t *testing.T) {
+	rule := &AlertRule{Rules: []*Rule{{
+		Type:     "offline",
+		Duration: math.MaxUint64,
+		Cover:    RuleCoverAll,
+	}}}
+
+	defer func() {
+		if recovered := recover(); recovered != nil {
+			t.Fatalf("Check panicked after uint64-to-int overflow: %v", recovered)
+		}
+	}()
+	duration, passed := rule.Check([][]bool{{false}})
+	if duration != 0 || !passed {
+		t.Fatalf("invalid persisted duration must be ignored safely, got duration=%d passed=%v", duration, passed)
+	}
+	if window := rule.RetentionWindow(); window != 0 {
+		t.Fatalf("invalid persisted duration must not create a retention window, got %d", window)
+	}
+}
+
+func TestAlertRulePersistedDataSafety(t *testing.T) {
+	cycleStart := time.Now().Add(-time.Hour)
+	tests := []struct {
+		name string
+		rule *AlertRule
+		want bool
+	}{
+		{
+			name: "normal rule",
+			rule: &AlertRule{Rules: []*Rule{{
+				Type: "cpu", Duration: 3, Cover: RuleCoverAll,
+			}}},
+			want: true,
+		},
+		{
+			name: "normal cycle rule",
+			rule: &AlertRule{Rules: []*Rule{{
+				Type: "transfer_in_cycle", CycleStart: &cycleStart, CycleInterval: 1, Cover: RuleCoverAll,
+			}}},
+			want: true,
+		},
+		{
+			name: "unknown type",
+			rule: &AlertRule{Rules: []*Rule{{
+				Type: "attacker_controlled", Duration: 3, Cover: RuleCoverAll,
+			}}},
+		},
+		{
+			name: "overflowing duration",
+			rule: &AlertRule{Rules: []*Rule{{
+				Type: "offline", Duration: math.MaxUint64, Cover: RuleCoverAll,
+			}}},
+		},
+		{
+			name: "cycle without start",
+			rule: &AlertRule{Rules: []*Rule{{
+				Type: "transfer_in_cycle", CycleInterval: 1, Cover: RuleCoverAll,
+			}}},
+		},
+		{
+			name: "nil rule",
+			rule: &AlertRule{Rules: []*Rule{nil}},
+		},
+		{
+			name: "empty rule list",
+			rule: &AlertRule{},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := test.rule.IsSafeToEvaluate(); got != test.want {
+				t.Fatalf("IsSafeToEvaluate()=%v, want %v", got, test.want)
+			}
+		})
+	}
+}

model/alertrule_test.go

@@ -26,7 +26,7 @@ func testCycleRules(t *testing.T) {
 			rule: &AlertRule{
 				Rules: []*Rule{
 					{
-						Type: "_cycle",
+						Type: "transfer_in_cycle",
 					},
 				},
 			},
@@ -39,7 +39,7 @@ func testCycleRules(t *testing.T) {
 			rule: &AlertRule{
 				Rules: []*Rule{
 					{
-						Type: "_cycle",
+						Type: "transfer_in_cycle",
 					},
 				},
 			},
@@ -130,6 +130,7 @@ func testGeneralRules(t *testing.T) {
 			rule: &AlertRule{
 				Rules: []*Rule{
 					{
+						Type:     "cpu",
 						Duration: 10,
 					},
 				},
@@ -143,6 +144,7 @@ func testGeneralRules(t *testing.T) {
 			rule: &AlertRule{
 				Rules: []*Rule{
 					{
+						Type:     "cpu",
 						Duration: 10,
 					},
 				},
@@ -156,6 +158,7 @@ func testGeneralRules(t *testing.T) {
 			rule: &AlertRule{
 				Rules: []*Rule{
 					{
+						Type:     "cpu",
 						Duration: 10,
 					},
 				},
@@ -169,6 +172,7 @@ func testGeneralRules(t *testing.T) {
 			rule: &AlertRule{
 				Rules: []*Rule{
 					{
+						Type:     "cpu",
 						Duration: 10,
 					},
 				},
@@ -182,6 +186,7 @@ func testGeneralRules(t *testing.T) {
 			rule: &AlertRule{
 				Rules: []*Rule{
 					{
+						Type:     "cpu",
 						Duration: 10,
 					},
 				},
@@ -210,6 +215,7 @@ func testCombinedRules(t *testing.T) {
 						Duration: 10,
 					},
 					{
+						Type:     "cpu",
 						Duration: 10,
 					},
 				},
@@ -227,6 +233,7 @@ func testCombinedRules(t *testing.T) {
 						Duration: 10,
 					},
 					{
+						Type:     "cpu",
 						Duration: 10,
 					},
 				},
@@ -240,6 +247,7 @@ func testCombinedRules(t *testing.T) {
 			rule: &AlertRule{
 				Rules: []*Rule{
 					{
+						Type:     "cpu",
 						Duration: 10,
 					},
 					{
@@ -257,9 +265,11 @@ func testCombinedRules(t *testing.T) {
 			rule: &AlertRule{
 				Rules: []*Rule{
 					{
+						Type:     "cpu",
 						Duration: 10,
 					},
 					{
+						Type:     "memory",
 						Duration: 30,
 					},
 				},
@@ -273,9 +283,11 @@ func testCombinedRules(t *testing.T) {
 			rule: &AlertRule{
 				Rules: []*Rule{
 					{
+						Type:     "cpu",
 						Duration: 10,
 					},
 					{
+						Type:     "memory",
 						Duration: 30,
 					},
 				},
@@ -416,7 +428,7 @@ func TestAlertRule_RetentionWindow(t *testing.T) {
 		{"zero duration only", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 0}}}, 0},
 		{"mixed picks max", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 0}, {Type: "cpu", Duration: 7}}}, 7},
 		{"offline keeps Duration", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 30}}}, 30},
-		{"cycle looks back one", &AlertRule{Rules: []*Rule{{Type: "net_in_speed_cycle"}}}, 1},
+		{"cycle looks back one", &AlertRule{Rules: []*Rule{{Type: "transfer_in_cycle"}}}, 1},
 	}
 	for _, c := range cases {
 		if got := c.rule.RetentionWindow(); got != c.want {
@@ -469,7 +481,7 @@ func TestAlertRule_CombinedRuleAccumulatesSamples(t *testing.T) {
 	}{
 		{"general3+general10", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 3}, {Type: "memory", Duration: 10}}}, []bool{false, false}, 9, 10},
 		{"offline5+general10", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 5}, {Type: "cpu", Duration: 10}}}, []bool{false, false}, 9, 10},
-		{"transfer+general8", &AlertRule{Rules: []*Rule{{Type: "net_in_speed_cycle"}, {Type: "cpu", Duration: 8}}}, []bool{false, false}, 7, 8},
+		{"transfer+general8", &AlertRule{Rules: []*Rule{{Type: "transfer_in_cycle"}, {Type: "cpu", Duration: 8}}}, []bool{false, false}, 7, 8},
 		{"offline3+offline12", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 3}, {Type: "offline", Duration: 12}}}, []bool{false, false}, 11, 12},
 	}
 	for _, c := range cases {

model/config.go

@@ -66,6 +66,9 @@ type ConfigDashboard struct {
 	AdminTemplate     string `koanf:"admin_template" json:"admin_template,omitempty"`
 
 	EnablePlainIPInNotification bool `koanf:"enable_plain_ip_in_notification" json:"enable_plain_ip_in_notification,omitempty"` // 通知信息IP不打码
+	// AllowJWTIPChange disables the IP binding check for browser login sessions.
+	// Keep this disabled by default so a stolen JWT remains bound to its issuing IP.
+	AllowJWTIPChange bool `koanf:"allow_jwt_ip_change" json:"allow_jwt_ip_change,omitempty"`
 
 	EnableMCP bool `koanf:"enable_mcp" json:"enable_mcp,omitempty"` // 是否启用 MCP 入口（默认关闭；启用前请审视 PAT scope/whitelist）
 
@@ -91,6 +94,10 @@ type ConfigDashboard struct {
 	AdminEmail                   string `koanf:"admin_email" json:"admin_email,omitempty"`
 	DomainExpiryNotificationDays string `koanf:"domain_expiry_notification_days" json:"domain_expiry_notification_days,omitempty"`
 	ServerExpiryNotificationDays string `koanf:"server_expiry_notification_days" json:"server_expiry_notification_days,omitempty"`
+
+	// OnlineStatsDays 设备在线时间段统计的保留与查询窗口（天）。
+	// 取值范围 [MinOnlineStatsDays, MaxOnlineStatsDays]，默认 DefaultOnlineStatsDays。
+	OnlineStatsDays int `koanf:"online_stats_days" json:"online_stats_days,omitempty"`
 }
 
 type Config struct {
@@ -118,6 +125,10 @@ type Config struct {
 	// SettingResponse 按值拷贝 ConfigDashboard 触发 copylocks。
 	mcpEnabled atomic.Bool `koanf:"-" json:"-" yaml:"-"`
 
+	// jwtIPChangeAllowed mirrors AllowJWTIPChange for concurrent authentication
+	// and setting updates. Keep atomic fields outside ConfigDashboard.
+	jwtIPChangeAllowed atomic.Bool `koanf:"-" json:"-" yaml:"-"`
+
 	// oauth2 配置
 	Oauth2 map[string]*Oauth2Config `koanf:"oauth2" json:"oauth2,omitempty"`
 
@@ -240,6 +251,15 @@ func (c *Config) Read(path string, frontendTemplates []FrontendTemplate) error {
 		c.JWTTimeout = 1
 	}
 
+	if c.OnlineStatsDays == 0 {
+		c.OnlineStatsDays = DefaultOnlineStatsDays
+	} else if c.OnlineStatsDays < MinOnlineStatsDays || c.OnlineStatsDays > MaxOnlineStatsDays {
+		log.Printf("NEZHA>> online_stats_days %d out of range [%d, %d], reset to %d", c.OnlineStatsDays, MinOnlineStatsDays, MaxOnlineStatsDays, DefaultOnlineStatsDays)
+		c.OnlineStatsDays = DefaultOnlineStatsDays
+	}
+
+	c.jwtIPChangeAllowed.Store(c.AllowJWTIPChange)
+
 	if c.AgentSecretKey == "" {
 		c.AgentSecretKey, err = utils.GenerateRandomString(32)
 		if err != nil {
@@ -267,6 +287,17 @@ func (c *Config) SetMCPEnabled(v bool) {
 	c.mcpEnabled.Store(v)
 }
 
+// JWTIPChangeAllowed reports whether JWT sessions may change IP concurrently.
+func (c *Config) JWTIPChangeAllowed() bool {
+	return c.jwtIPChangeAllowed.Load()
+}
+
+// SetJWTIPChangeAllowed updates only the atomic mirror. save() synchronizes
+// the public field for persistence; authentication must use the accessor.
+func (c *Config) SetJWTIPChangeAllowed(v bool) {
+	c.jwtIPChangeAllowed.Store(v)
+}
+
 // Save 保存配置文件
 func (c *Config) Save() error {
 	return c.save()
@@ -335,6 +366,7 @@ func (c *Config) patchYAMLField(key string, value any) error {
 
 func (c *Config) save() error {
 	c.EnableMCP = c.mcpEnabled.Load()
+	c.AllowJWTIPChange = c.jwtIPChangeAllowed.Load()
 	data, err := yaml.Marshal(c)
 	if err != nil {
 		return err

model/config_test.go

@@ -24,6 +24,7 @@ func TestReadConfig(t *testing.T) {
 			{"user_template", c.UserTemplate, c.UserTemplate == "user-dist"},
 			{"admin_template", c.AdminTemplate, c.AdminTemplate == "admin-dist"},
 			{"agent_secret_key", c.AgentSecretKey, c.AgentSecretKey != ""},
+			{"allow_jwt_ip_change", c.AllowJWTIPChange, !c.AllowJWTIPChange},
 		}
 
 		for _, field := range testFields {
@@ -37,6 +38,7 @@ func TestReadConfig(t *testing.T) {
 
 	t.Run("ReadFile", func(t *testing.T) {
 		const testCfg = `
+allow_jwt_ip_change: true
 jwt_secret_key: test
 user_template: um
 admin_template: am
@@ -85,6 +87,7 @@ admin_email: admin@loohui.com
 			{"smtp_user", c.SMTPUser, c.SMTPUser == "test@loohui.com"},
 			{"smtp_password", c.SMTPPassword, c.SMTPPassword == "password123"},
 			{"admin_email", c.AdminEmail, c.AdminEmail == "admin@loohui.com"},
+			{"allow_jwt_ip_change", c.AllowJWTIPChange, c.AllowJWTIPChange},
 		}
 
 		for _, field := range testFields {

model/host.go

@@ -35,9 +35,28 @@ type HostState struct {
 	ProcessCount   uint64              `json:"process_count,omitempty"`
 	Temperatures   []SensorTemperature `json:"temperatures,omitempty"`
 	GPU            []float64           `json:"gpu,omitempty"`
+	GPUs           []GPUStat           `json:"gpus,omitempty"`
+}
+
+// GPUStat carries per-card figures, index-aligned with Host.GPU. Memory is in
+// MiB and stays zero for vendors that do not report it, so MemoryTotal == 0
+// means "unknown" rather than "no memory".
+type GPUStat struct {
+	Utilization float64 `json:"utilization"`
+	MemoryUsed  uint64  `json:"memory_used,omitempty"`
+	MemoryTotal uint64  `json:"memory_total,omitempty"`
 }
 
 func (s *HostState) PB() *pb.State {
+	gs := make([]*pb.State_GPU, 0, len(s.GPUs))
+	for _, g := range s.GPUs {
+		gs = append(gs, &pb.State_GPU{
+			Utilization: g.Utilization,
+			MemoryUsed:  g.MemoryUsed,
+			MemoryTotal: g.MemoryTotal,
+		})
+	}
+
 	var ts []*pb.State_SensorTemperature
 	for _, t := range s.Temperatures {
 		ts = append(ts, &pb.State_SensorTemperature{
@@ -64,10 +83,20 @@ func (s *HostState) PB() *pb.State {
 		ProcessCount:   s.ProcessCount,
 		Temperatures:   ts,
 		Gpu:            s.GPU,
+		Gpus:           gs,
 	}
 }
 
 func PB2State(s *pb.State) HostState {
+	var gs []GPUStat
+	for _, g := range s.GetGpus() {
+		gs = append(gs, GPUStat{
+			Utilization: g.GetUtilization(),
+			MemoryUsed:  g.GetMemoryUsed(),
+			MemoryTotal: g.GetMemoryTotal(),
+		})
+	}
+
 	var ts []SensorTemperature
 	for _, t := range s.GetTemperatures() {
 		ts = append(ts, SensorTemperature{
@@ -93,6 +122,7 @@ func PB2State(s *pb.State) HostState {
 		UdpConnCount:   s.GetUdpConnCount(),
 		ProcessCount:   s.GetProcessCount(),
 		Temperatures:   ts,
+		GPUs:           gs,
 		GPU:            s.GetGpu(),
 	}
 }

model/jwt_ip_change_atomic_test.go

@@ -0,0 +1,56 @@
+package model
+
+import (
+	"path/filepath"
+	"sync"
+	"testing"
+)
+
+func TestJWTIPChangeAllowedPersistence(t *testing.T) {
+	c := &Config{}
+	if c.JWTIPChangeAllowed() {
+		t.Fatal("zero-value Config must bind JWT sessions to their IP")
+	}
+	// An absent agent secret makes Read save the config during initialization.
+	path := newTempConfig(t, "jwt_secret_key: test\nallow_jwt_ip_change: true\n")
+	if err := c.Read(path, nil); err != nil {
+		t.Fatal(err)
+	}
+	if !c.JWTIPChangeAllowed() {
+		t.Fatal("Read must initialize the atomic mirror before saving")
+	}
+	for _, want := range []bool{true, false} {
+		c.SetJWTIPChangeAllowed(want)
+		if err := c.Save(); err != nil {
+			t.Fatal(err)
+		}
+		reloaded := &Config{}
+		if err := reloaded.Read(path, nil); err != nil {
+			t.Fatal(err)
+		}
+		if c.AllowJWTIPChange != want || reloaded.JWTIPChangeAllowed() != want {
+			t.Fatalf("saved and reloaded setting must be %v", want)
+		}
+	}
+}
+
+func TestJWTIPChangeAllowedConcurrentAccess(t *testing.T) {
+	c := &Config{filePath: filepath.Join(t.TempDir(), "config.yaml")}
+	var wg sync.WaitGroup
+	wg.Go(func() {
+		for i := 0; i < 10000; i++ {
+			c.SetJWTIPChangeAllowed(i%2 == 0)
+		}
+	})
+	wg.Go(func() {
+		for i := 0; i < 10000; i++ {
+			_ = c.JWTIPChangeAllowed()
+		}
+	})
+	for i := 0; i < 20; i++ {
+		if err := c.Save(); err != nil {
+			t.Error(err)
+		}
+	}
+	wg.Wait()
+}

model/notification.go

@@ -208,6 +208,7 @@ func (ns *NotificationServerBundle) replaceParamsInString(str string, message st
 	}
 
 	if ns.Server != nil {
+		runtime := ns.Server.RuntimeSnapshot()
 		var noteData struct {
 			BillingDataMod struct {
 				EndDate string `json:"endDate"`
@@ -238,8 +239,6 @@ func (ns *NotificationServerBundle) replaceParamsInString(str string, message st
 			"#SERVER.BILLING_AMOUNT#", mod(amountStr),
 			"#SERVER.BILLING_CYCLE#", mod(cycleStr),
 		)
-
-		runtime := ns.Server.RuntimeSnapshot()
 		if runtime.State != nil && runtime.Host != nil {
 			state := runtime.State
 			host := runtime.Host
@@ -321,13 +320,13 @@ func (ns *NotificationServerBundle) replaceParamsInString(str string, message st
 				ipv6 = ip.IPv6Addr
 				validIP = ipv6
 			}
-		}
 
-		replacements = append(replacements,
-			"#SERVER.IP#", mod(validIP),
-			"#SERVER.IPV4#", mod(ipv4),
-			"#SERVER.IPV6#", mod(ipv6),
-		)
+			replacements = append(replacements,
+				"#SERVER.IP#", mod(validIP),
+				"#SERVER.IPV4#", mod(ipv4),
+				"#SERVER.IPV6#", mod(ipv6),
+			)
+		}
 	}
 
 	replacer := strings.NewReplacer(replacements...)

model/online_history.go

@@ -0,0 +1,211 @@
+package model
+
+import (
+	"gorm.io/gorm"
+	"sort"
+	"time"
+)
+
+// 在线统计窗口的取值边界（天）。后台管理端可设置统计范围，查询窗口不允许
+// 超过该设置，历史数据按该设置做保留清理。
+const (
+	DefaultOnlineStatsDays = 7
+	MinOnlineStatsDays     = 7
+	MaxOnlineStatsDays     = 3650
+)
+
+// OnlineActiveWindow 服务器状态上报距该窗口内视为在线。与 MCP server.list
+// 的 onlineWindow 口径保持一致；超过该窗口未上报的进行中会话由定时清扫关闭。
+const OnlineActiveWindow = 30 * time.Second
+
+// OnlineSession 记录一台服务器的一段连续在线时间。Ongoing=true 表示会话
+// 仍在进行（此时 EndedAt 为零值）；会话结束时写入 EndedAt 并置 Ongoing=false。
+// LastActiveAt 由上报侧节流持久化，用于 dashboard 异常停机后把进行中的会话
+// 结转到最后一次已知活跃时刻。
+type OnlineSession struct {
+	ID        uint64    `gorm:"primaryKey" json:"id,omitempty"`
+	CreatedAt time.Time `gorm:"index;<-:create" json:"created_at,omitempty"`
+	UpdatedAt time.Time `gorm:"autoUpdateTime" json:"updated_at,omitempty"`
+	ServerID  uint64    `gorm:"index:idx_online_sessions_server_started;index:idx_online_sessions_server_closed,priority:1;index:idx_online_sessions_server_open,priority:1" json:"server_id,omitempty"`
+	StartedAt time.Time `gorm:"index:idx_online_sessions_server_started;index:idx_online_sessions_server_open,priority:3" json:"started_at,omitempty"`
+	EndedAt   time.Time `gorm:"index;index:idx_online_sessions_server_closed,priority:3" json:"ended_at,omitempty"` // 会话结束时刻，进行中为零值
+	// 上报侧节流持久化的最后活跃时刻（≥OnlineActiveWindow 间隔写一次）
+	LastActiveAt time.Time `json:"last_active_at,omitempty"`
+	Ongoing      bool      `gorm:"index:idx_online_sessions_server_closed,priority:2;index:idx_online_sessions_server_open,priority:2" json:"ongoing"`
+}
+
+// BeforeSave keeps SQLite's lexicographic timestamp comparisons chronological.
+// Query parameters and map-based updates must use UTC as well.
+func (s *OnlineSession) BeforeSave(_ *gorm.DB) error {
+	s.StartedAt = s.StartedAt.UTC()
+	s.EndedAt = s.EndedAt.UTC()
+	s.LastActiveAt = s.LastActiveAt.UTC()
+	return nil
+}
+
+// OnlinePeriod 是裁剪到查询窗口后的在线时间段，供前端时间轴直接渲染。
+type OnlinePeriod struct {
+	Start   time.Time `json:"start"`
+	End     time.Time `json:"end"`
+	Ongoing bool      `json:"ongoing"`
+}
+
+// DailyOnlineStat 是单个自然日（面板时区）在查询窗口内的在线秒数。
+// 窗口首尾为非整日时，只统计与窗口重叠的部分。
+type DailyOnlineStat struct {
+	Date          string `json:"date"`
+	OnlineSeconds uint64 `json:"online_seconds"`
+}
+
+// OnlineHistoryResponse 是单台服务器在线时间段统计的聚合结果。
+type OnlineHistoryResponse struct {
+	ServerID   uint64 `json:"server_id"`
+	ServerName string `json:"server_name"`
+	Days       int    `json:"days"`
+	MaxDays    int    `json:"max_days"`
+	// Daily 拆日所用面板时区：IANA 名与窗口末端相对 UTC 的偏移秒数，
+	// 前端按同一时区渲染，保证跨时区访客的图形与数字一致
+	Timezone           string    `json:"timezone"`
+	TzOffset           int       `json:"tz_offset"`
+	WindowStart        time.Time `json:"window_start"`
+	WindowEnd          time.Time `json:"window_end"`
+	TotalOnlineSeconds uint64    `json:"total_online_seconds"`
+	OnlineRate         float64   `json:"online_rate"`
+	CurrentlyOnline    bool      `json:"currently_online"`
+	// 从未上报过的服务器该值为 nil；指针避免把 Go 零值时间序列化成 0001-01-01
+	LastActive *time.Time        `json:"last_active,omitempty"`
+	Periods    []OnlinePeriod    `json:"periods"`
+	Daily      []DailyOnlineStat `json:"daily"`
+}
+
+// ClipOnlineSessions 把与窗口 [windowStart, windowEnd) 相交的会话裁剪成
+// OnlinePeriod 列表。进行中的会话以 windowEnd 为右边界（保留 Ongoing 标记）。
+// 输入需按 StartedAt 升序；输出与输入同序。
+func ClipOnlineSessions(sessions []OnlineSession, windowStart, windowEnd time.Time) []OnlinePeriod {
+	periods := make([]OnlinePeriod, 0, len(sessions))
+	for _, s := range sessions {
+		start := s.StartedAt
+		if start.Before(windowStart) {
+			start = windowStart
+		}
+		end := s.EndedAt
+		ongoing := s.Ongoing || end.IsZero()
+		if ongoing || end.After(windowEnd) {
+			end = windowEnd
+		}
+		if !start.Before(end) {
+			// 与窗口零交集（理论上不会由查询条件选出）或空会话
+			continue
+		}
+		periods = append(periods, OnlinePeriod{Start: start, End: end, Ongoing: ongoing})
+	}
+	return periods
+}
+
+// DailyOnlineBreakdown 把在线时间段按面板时区的自然日拆分，返回窗口覆盖的
+// 每一天的在线秒数（含在线为 0 秒的日期），按日期升序。
+// 取整口径：每天先累计原始时长（日界切分不损失精度），仅在每日汇总时向下
+// 取整一次；调用方以各日之和作为总时长，保证响应内部 total 与 daily[] 严格
+// 对账，不与"逐段截断再累加"的口径混用（亚秒会话会让后者系统性偏小）。
+// 时段二分定位起始日界后逐日推进，复杂度 O(P·log D + D)；单服务器的会话
+// 互不重叠，内层推进总量受窗口天数约束，长窗口（如 days=3650）下不会出现
+// 时段数×天数的双重循环。
+func DailyOnlineBreakdown(periods []OnlinePeriod, windowStart, windowEnd time.Time, loc *time.Location) []DailyOnlineStat {
+	first := windowStart.In(loc)
+	// Advance a timezone-independent calendar, not a normalized local midnight:
+	// midnight DST gaps can normalize into the previous day and repeat dates.
+	date := time.Date(first.Year(), first.Month(), first.Day(), 0, 0, 0, 0, time.UTC)
+	var days []DailyOnlineStat
+	var dayStarts, dayEnds []time.Time
+	for {
+		start := onlineCalendarBoundary(date, loc)
+		if !start.Before(windowEnd) {
+			break
+		}
+		nextDate := date.AddDate(0, 0, 1)
+		end := onlineCalendarBoundary(nextDate, loc)
+		if start.Before(end) {
+			dayStarts = append(dayStarts, start)
+			dayEnds = append(dayEnds, end)
+			days = append(days, DailyOnlineStat{Date: date.Format("2006-01-02")})
+		}
+		date = nextDate
+	}
+
+	if len(days) == 0 {
+		return days
+	}
+
+	dayDurations := make([]time.Duration, len(days))
+	for _, p := range periods {
+		// 该时段覆盖的第一天：不晚于时段开始的最后一个日界
+		i := sort.Search(len(dayStarts), func(i int) bool {
+			return dayStarts[i].After(p.Start)
+		}) - 1
+		if i < 0 {
+			i = 0
+		}
+		// 逐日推进，累加该时段与每一天 [dayStart, nextDay) 的交集
+		for ; i < len(dayStarts) && dayStarts[i].Before(p.End); i++ {
+			overlapStart := maxTime(p.Start, dayStarts[i])
+			overlapEnd := minTime(p.End, dayEnds[i])
+			if overlapStart.Before(overlapEnd) {
+				dayDurations[i] += overlapEnd.Sub(overlapStart)
+			}
+		}
+	}
+	for i := range days {
+		days[i].OnlineSeconds = uint64(dayDurations[i].Seconds())
+	}
+	return days
+}
+
+func maxTime(a, b time.Time) time.Time {
+	if a.After(b) {
+		return a
+	}
+	return b
+}
+
+func minTime(a, b time.Time) time.Time {
+	if a.Before(b) {
+		return a
+	}
+	return b
+}
+
+// A missing midnight (or an entirely skipped date) starts at the timezone
+// transition, not at the previous date returned by time.Date normalization.
+func onlineCalendarBoundary(date time.Time, loc *time.Location) time.Time {
+	boundary := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, loc)
+	if boundary.Format("2006-01-02") < date.Format("2006-01-02") {
+		_, end := boundary.ZoneBounds()
+		if !end.IsZero() {
+			return end
+		}
+	}
+	// time.Date may choose the second midnight in a rollback. Recover the
+	// earlier occurrence using the offset on the preceding side of the transition.
+	zoneStart, _ := boundary.ZoneBounds()
+	if !zoneStart.IsZero() {
+		_, currentOffset := boundary.Zone()
+		_, previousOffset := zoneStart.Add(-time.Nanosecond).Zone()
+		if previousOffset > currentOffset {
+			earlier := boundary.Add(-time.Duration(previousOffset-currentOffset) * time.Second)
+			if earlier.Format("2006-01-02 15:04:05") == boundary.Format("2006-01-02 15:04:05") {
+				boundary = earlier
+			}
+		}
+	}
+	return boundary
+}
+
+// OnlineHistoryWindowStart defines the shared query and retention calendar window.
+func OnlineHistoryWindowStart(now time.Time, days int, loc *time.Location) time.Time {
+	// Calendar days follow the dashboard timezone, not the host timezone;
+	// DST days can contain 23 or 25 hours. Queries and pruning must agree.
+	if loc == nil {
+		loc = time.UTC
+	}
+	return now.In(loc).AddDate(0, 0, -days).UTC()
+}

model/online_history_test.go

@@ -0,0 +1,220 @@
+package model
+
+import (
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+var testZone = time.FixedZone("TEST", 8*3600)
+
+func ts(y int, mo time.Month, d, h, mi, s int) time.Time {
+	return time.Date(y, mo, d, h, mi, s, 0, testZone)
+}
+
+func TestClipOnlineSessions(t *testing.T) {
+	windowStart := ts(2026, 10, 1, 0, 0, 0)
+	windowEnd := ts(2026, 10, 8, 0, 0, 0)
+
+	t.Run("session fully inside window", func(t *testing.T) {
+		sessions := []OnlineSession{
+			{ServerID: 1, StartedAt: ts(2026, 10, 2, 8, 0, 0), EndedAt: ts(2026, 10, 2, 9, 30, 0)},
+		}
+		periods := ClipOnlineSessions(sessions, windowStart, windowEnd)
+		require.Len(t, periods, 1)
+		require.Equal(t, ts(2026, 10, 2, 8, 0, 0), periods[0].Start)
+		require.Equal(t, ts(2026, 10, 2, 9, 30, 0), periods[0].End)
+		require.False(t, periods[0].Ongoing)
+	})
+
+	t.Run("session started before window is clipped", func(t *testing.T) {
+		sessions := []OnlineSession{
+			{ServerID: 1, StartedAt: ts(2026, 9, 28, 12, 0, 0), EndedAt: ts(2026, 10, 1, 5, 0, 0)},
+		}
+		periods := ClipOnlineSessions(sessions, windowStart, windowEnd)
+		require.Len(t, periods, 1)
+		require.Equal(t, windowStart, periods[0].Start)
+		require.Equal(t, ts(2026, 10, 1, 5, 0, 0), periods[0].End)
+	})
+
+	t.Run("ongoing session ends at window end", func(t *testing.T) {
+		sessions := []OnlineSession{
+			{ServerID: 1, StartedAt: ts(2026, 10, 7, 10, 0, 0), Ongoing: true},
+		}
+		periods := ClipOnlineSessions(sessions, windowStart, windowEnd)
+		require.Len(t, periods, 1)
+		require.Equal(t, windowEnd, periods[0].End)
+		require.True(t, periods[0].Ongoing)
+	})
+
+	t.Run("zero ended_at treated as ongoing", func(t *testing.T) {
+		sessions := []OnlineSession{
+			{ServerID: 1, StartedAt: ts(2026, 10, 7, 10, 0, 0)},
+		}
+		periods := ClipOnlineSessions(sessions, windowStart, windowEnd)
+		require.Len(t, periods, 1)
+		require.True(t, periods[0].Ongoing)
+		require.Equal(t, windowEnd, periods[0].End)
+	})
+
+	t.Run("session outside window yields no period", func(t *testing.T) {
+		sessions := []OnlineSession{
+			{ServerID: 1, StartedAt: ts(2026, 9, 1, 0, 0, 0), EndedAt: ts(2026, 9, 2, 0, 0, 0)},
+		}
+		require.Empty(t, ClipOnlineSessions(sessions, windowStart, windowEnd))
+	})
+}
+
+func TestDailyOnlineBreakdown(t *testing.T) {
+	loc := time.FixedZone("UTC+8", 8*3600)
+	windowStart := time.Date(2026, 10, 1, 10, 0, 0, 0, loc)
+	windowEnd := time.Date(2026, 10, 3, 14, 0, 0, 0, loc)
+
+	periods := []OnlinePeriod{
+		// 跨午夜：2 日 23:00 ~ 3 日 01:00
+		{Start: time.Date(2026, 10, 2, 23, 0, 0, 0, loc), End: time.Date(2026, 10, 3, 1, 0, 0, 0, loc)},
+		// 窗口前就开始的会话，裁剪后从窗口起点开始
+		{Start: windowStart, End: time.Date(2026, 10, 1, 12, 0, 0, 0, loc)},
+	}
+
+	daily := DailyOnlineBreakdown(periods, windowStart, windowEnd, loc)
+	require.Len(t, daily, 3)
+	require.Equal(t, "2026-10-01", daily[0].Date)
+	require.Equal(t, "2026-10-02", daily[1].Date)
+	require.Equal(t, "2026-10-03", daily[2].Date)
+
+	// 1 日：12:00-10:00 = 2h；2 日：23:00-24:00 = 1h；3 日：0:00-1:00 + 窗口外无 = 1h
+	require.Equal(t, uint64(7200), daily[0].OnlineSeconds)
+	require.Equal(t, uint64(3600), daily[1].OnlineSeconds)
+	require.Equal(t, uint64(3600), daily[2].OnlineSeconds)
+
+	// 每日合计即总时长（口径：每日累计原始时长后统一取整）
+	var sum uint64
+	for _, d := range daily {
+		sum += d.OnlineSeconds
+	}
+	require.Equal(t, uint64(7200+3600+3600), sum)
+}
+
+// 评审回归：取整必须按日累计原始时长后统一转换。逐段截断再累加会让亚秒
+// 会话系统性偏小——同一天两个 1.5s 的会话应得 3s，而不是 1+1=2s
+func TestDailyOnlineBreakdownSubSecond(t *testing.T) {
+	loc := testZone
+	windowStart := ts(2026, 10, 1, 0, 0, 0)
+	windowEnd := ts(2026, 10, 2, 0, 0, 0)
+	periods := []OnlinePeriod{
+		{Start: windowStart.Add(100 * time.Millisecond), End: windowStart.Add(1600 * time.Millisecond)},
+		{Start: windowStart.Add(2 * time.Second), End: windowStart.Add(3500 * time.Millisecond)},
+	}
+	daily := DailyOnlineBreakdown(periods, windowStart, windowEnd, loc)
+	require.Len(t, daily, 1)
+	require.Equal(t, uint64(3), daily[0].OnlineSeconds)
+}
+
+// 跨日时段按日界切分不损失精度：各日分别累计原始时长后各自取整
+func TestDailyOnlineBreakdownCrossDayFraction(t *testing.T) {
+	loc := testZone
+	windowStart := ts(2026, 10, 1, 0, 0, 0)
+	windowEnd := ts(2026, 10, 3, 0, 0, 0)
+	periods := []OnlinePeriod{{
+		Start: time.Date(2026, 10, 1, 23, 59, 59, 250000000, loc),
+		End:   time.Date(2026, 10, 2, 0, 0, 2, 750000000, loc),
+	}}
+	daily := DailyOnlineBreakdown(periods, windowStart, windowEnd, loc)
+	require.Len(t, daily, 2)
+	require.Equal(t, uint64(0), daily[0].OnlineSeconds) // 0.75s → 0
+	require.Equal(t, uint64(2), daily[1].OnlineSeconds) // 2.75s → 2
+}
+
+func TestDailyOnlineBreakdownZeroOnline(t *testing.T) {
+	loc := time.FixedZone("UTC+8", 8*3600)
+	windowStart := time.Date(2026, 10, 1, 0, 0, 0, 0, loc)
+	windowEnd := time.Date(2026, 10, 4, 0, 0, 0, 0, loc)
+	daily := DailyOnlineBreakdown(nil, windowStart, windowEnd, loc)
+	require.Len(t, daily, 3)
+	for _, d := range daily {
+		require.Zero(t, d.OnlineSeconds)
+	}
+}
+
+func TestOnlineStatsDaysBounds(t *testing.T) {
+	require.Equal(t, 7, DefaultOnlineStatsDays)
+	require.Equal(t, 7, MinOnlineStatsDays)
+	require.Equal(t, 3650, MaxOnlineStatsDays)
+}
+
+// F1 回归保护：指针推进实现必须与逐日朴素实现对任意不相交时段集合等价
+// （单服务器会话互不重叠的约束一致），长窗口大量时段下不再随时段数平方放大。
+func TestDailyOnlineBreakdownEquivalence(t *testing.T) {
+	loc := testZone
+	windowStart := ts(2026, 1, 1, 0, 0, 0)
+	windowEnd := windowStart.AddDate(0, 0, 90)
+
+	rng := rand.New(rand.NewSource(42))
+	var periods []OnlinePeriod
+	cursor := windowStart
+	for cursor.Before(windowEnd) {
+		// 起止带亚秒抖动：让逐段截断与按日累计两种口径可区分
+		start := cursor.Add(time.Duration(rng.Intn(4*3600))*time.Second + time.Duration(rng.Int63n(int64(time.Second))))
+		end := start.Add(time.Duration(1+rng.Intn(6*3600))*time.Second + time.Duration(rng.Int63n(int64(time.Second))))
+		if end.After(windowEnd) {
+			end = windowEnd
+		}
+		periods = append(periods, OnlinePeriod{Start: start, End: end})
+		cursor = end.Add(time.Duration(1+rng.Intn(3600)) * time.Second)
+	}
+
+	days := DailyOnlineBreakdown(periods, windowStart, windowEnd, loc)
+
+	// 参考实现与生产同一口径：每日累计原始时长、汇总时向下取整一次
+	reference := map[string]time.Duration{}
+	for _, p := range periods {
+		for d := windowStart; d.Before(windowEnd); d = d.AddDate(0, 0, 1) {
+			overlapStart := maxTime(p.Start, d)
+			overlapEnd := minTime(p.End, d.AddDate(0, 0, 1))
+			if overlapStart.Before(overlapEnd) {
+				reference[d.Format("2006-01-02")] += overlapEnd.Sub(overlapStart)
+			}
+		}
+	}
+	require.Len(t, days, 90)
+	for _, day := range days {
+		require.Equal(t, uint64(reference[day.Date].Seconds()), day.OnlineSeconds, "date %s", day.Date)
+	}
+}
+
+func TestDailyOnlineBreakdownMidnightDSTTransition(t *testing.T) {
+	loc, err := time.LoadLocation("America/Sao_Paulo")
+	require.NoError(t, err)
+	start := time.Date(2018, 11, 3, 0, 0, 0, 0, loc)
+	end := time.Date(2018, 11, 6, 0, 0, 0, 0, loc)
+	days := DailyOnlineBreakdown([]OnlinePeriod{{Start: start, End: end}}, start, end, loc)
+	require.Equal(t, []DailyOnlineStat{{Date: "2018-11-03", OnlineSeconds: 86400}, {Date: "2018-11-04", OnlineSeconds: 82800}, {Date: "2018-11-05", OnlineSeconds: 86400}}, days)
+}
+
+func TestDailyOnlineBreakdownRepeatedMidnight(t *testing.T) {
+	loc, err := time.LoadLocation("Asia/Amman")
+	require.NoError(t, err)
+	start := time.Date(2021, 10, 28, 21, 0, 0, 0, time.UTC)
+	end := time.Date(2021, 10, 29, 22, 0, 0, 0, time.UTC)
+	days := DailyOnlineBreakdown([]OnlinePeriod{{Start: start, End: end}}, start, end, loc)
+	require.Equal(t, []DailyOnlineStat{{Date: "2021-10-29", OnlineSeconds: 90000}}, days)
+}
+
+func TestOnlineHistoryWindowUsesDashboardCalendar(t *testing.T) {
+	loc, err := time.LoadLocation("America/New_York")
+	require.NoError(t, err)
+	for _, test := range []struct {
+		month      time.Month
+		day, hours int
+	}{{time.March, 9, 23}, {time.November, 2, 25}} {
+		end := time.Date(2026, test.month, test.day, 0, 30, 0, 0, loc)
+		start := OnlineHistoryWindowStart(end.UTC(), 1, loc)
+		require.True(t, start.Equal(time.Date(2026, test.month, test.day-1, 0, 30, 0, 0, loc)))
+		require.Equal(t, time.Duration(test.hours)*time.Hour, end.Sub(start))
+		require.Len(t, DailyOnlineBreakdown(nil, start, end, loc), 2)
+		require.True(t, OnlineHistoryWindowStart(end.UTC(), 7, loc).Equal(end.In(loc).AddDate(0, 0, -7)))
+	}
+}

model/rule.go

@@ -1,6 +1,8 @@
 package model
 
 import (
+	"log"
+	"math"
 	"slices"
 	"strings"
 	"time"
@@ -15,12 +17,23 @@ const (
 	RuleCoverIgnoreAll
 )
 
+// MaxAlertRuleDuration is the largest duration that can be converted to int
+// on every architecture supported by Go. Alert rules are persisted as uint64,
+// but their sampling windows use int indexes; keeping the public bound at
+// MaxInt32 prevents a crafted value from wrapping during that conversion.
+const MaxAlertRuleDuration uint64 = 1<<31 - 1
+
+// MaxAlertRuleCycleInterval also leaves room for the largest integer
+// multiplication performed by the calendar helpers (weeks become 7*interval)
+// on 32-bit targets.
+const MaxAlertRuleCycleInterval uint64 = (1<<31 - 1) / 7
+
 type NResult struct {
 	N uint64
 }
 
 type Rule struct {
-	// 指标类型，cpu、memory、swap、disk、net_in_speed、net_out_speed
+	// 指标类型，cpu、gpu/gpu_max、memory、swap、disk、net_in_speed、net_out_speed
 	// net_all_speed、transfer_in、transfer_out、transfer_all、offline
 	// transfer_in_cycle、transfer_out_cycle、transfer_all_cycle
 	Type          string          `json:"type"`
@@ -45,47 +58,147 @@ func percentage(used, total uint64) float64 {
 	return float64(used) * 100 / float64(total)
 }
 
-// Snapshot 未通过规则返回 false, 通过返回 true
+// IsSupportedType reports whether the rule type has an evaluator. Keep this
+// list in lockstep with Snapshot's switch so untrusted strings cannot enter the
+// persisted alert pipeline and silently acquire fallback semantics.
+func (u *Rule) IsSupportedType() bool {
+	if u == nil {
+		return false
+	}
+	switch u.Type {
+	case "cpu", "gpu", "gpu_max", "memory", "swap", "disk",
+		"net_in_speed", "net_out_speed", "net_all_speed",
+		"transfer_in", "transfer_out", "transfer_all", "offline",
+		"transfer_in_cycle", "transfer_out_cycle", "transfer_all_cycle",
+		"load1", "load5", "load15", "tcp_conn_count", "udp_conn_count",
+		"process_count", "temperature_max":
+		return true
+	default:
+		return false
+	}
+}
+
+// DurationInt converts the persisted duration without allowing uint64-to-int
+// truncation. Callers must handle ok=false as an invalid rule.
+func (u *Rule) DurationInt() (duration int, ok bool) {
+	if u == nil || u.Duration > MaxAlertRuleDuration {
+		return 0, false
+	}
+	return int(u.Duration), true
+}
+
+// HasSafeCycleConfiguration guards every value that the cycle evaluator later
+// converts or dereferences. It intentionally does not reject an empty unit,
+// which is the documented legacy spelling for hours.
+func (u *Rule) HasSafeCycleConfiguration() bool {
+	return u != nil && u.IsTransferDurationRule() && u.CycleStart != nil &&
+		!u.CycleStart.After(time.Now()) && u.CycleInterval > 0 && u.CycleInterval <= MaxAlertRuleCycleInterval &&
+		u.HasSafeThresholds() && u.HasSafeCycleUnit()
+}
+
+func (u *Rule) HasSafeCycleUnit() bool {
+	if u == nil {
+		return false
+	}
+	switch strings.ToLower(u.CycleUnit) {
+	case "", "hour", "day", "week", "month", "year":
+		return true
+	default:
+		return false
+	}
+}
+
+func (u *Rule) HasSafeThresholds() bool {
+	if u == nil || math.IsNaN(u.Min) || math.IsNaN(u.Max) || math.IsInf(u.Min, 0) || math.IsInf(u.Max, 0) || u.Min < 0 || u.Max < 0 {
+		return false
+	}
+	return u.Min == 0 || u.Max == 0 || u.Min <= u.Max
+}
+
+// Snapshot is retained for callers which only need a boolean result. The alert
+// sentinel uses Evaluate so missing telemetry is not mistaken for a breach.
 func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) bool {
+	passed, _ := u.Evaluate(cycleTransferStats, server, db)
+	return passed
+}
+
+// Evaluate returns known=false when a metric cannot be evaluated from the
+// current report. Unknown must neither trigger nor resolve an incident.
+func (u *Rule) Evaluate(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) (passed, known bool) {
+	if server == nil {
+		return true, false
+	}
+	return u.EvaluateRuntime(cycleTransferStats, server, server.RuntimeSnapshot(), db)
+}
+
+// EvaluateRuntime evaluates all conditions against the same immutable Agent
+// report. Callers evaluating a compound rule must share this snapshot.
+func (u *Rule) EvaluateRuntime(cycleTransferStats *CycleTransferStats, server *Server, runtime RuntimeSnapshot, db *gorm.DB) (passed, known bool) {
+	if u == nil || server == nil || !u.IsSupportedType() {
+		return true, false
+	}
+	if u.IsTransferDurationRule() && (!u.HasSafeCycleConfiguration() || cycleTransferStats == nil) {
+		return true, false
+	}
+
 	// 监控全部但是排除了此服务器
 	if u.Cover == RuleCoverAll && u.Ignore[server.ID] {
-		return true
+		return true, true
 	}
 	// 忽略全部但是指定监控了此服务器
 	if u.Cover == RuleCoverIgnoreAll && !u.Ignore[server.ID] {
-		return true
+		return true, true
 	}
 
+	evalNow := time.Now()
+	var cycleStart, cycleEnd time.Time
+	if u.IsTransferDurationRule() {
+		cycleStart, cycleEnd = u.transferDurationBounds(evalNow)
+	}
 	// 循环区间流量检测 · 短期无需重复检测
-	if u.IsTransferDurationRule() && u.NextTransferAt[server.ID].After(time.Now()) {
-		return u.LastCycleStatus[server.ID]
+	if u.IsTransferDurationRule() && u.NextTransferAt[server.ID].After(evalNow) {
+		return u.LastCycleStatus[server.ID], true
 	}
 
 	var src float64
-	runtime := server.RuntimeSnapshot()
+	if u.IsOfflineRule() {
+		return !runtime.LastActive.IsZero() && time.Since(runtime.LastActive) <= 6*time.Second, true
+	}
 	if runtime.State == nil {
-		return false
+		return true, false
 	}
 	state := runtime.State
 
 	switch u.Type {
 	case "cpu":
 		src = float64(state.CPU)
-	case "gpu_max":
+	case "gpu", "gpu_max":
+		if len(state.GPU) == 0 {
+			return true, false
+		}
 		src = slices.Max(state.GPU)
 	case "memory":
 		if runtime.Host == nil {
-			return false
+			return true, false
+		}
+		if runtime.Host.MemTotal == 0 {
+			return true, false
 		}
 		src = percentage(state.MemUsed, runtime.Host.MemTotal)
 	case "swap":
 		if runtime.Host == nil {
-			return false
+			return true, false
+		}
+		if runtime.Host.SwapTotal == 0 {
+			return true, false
 		}
 		src = percentage(state.SwapUsed, runtime.Host.SwapTotal)
 	case "disk":
 		if runtime.Host == nil {
-			return false
+			return true, false
+		}
+		if runtime.Host.DiskTotal == 0 {
+			return true, false
 		}
 		src = percentage(state.DiskUsed, runtime.Host.DiskTotal)
 	case "net_in_speed":
@@ -93,38 +206,50 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server,
 	case "net_out_speed":
 		src = float64(state.NetOutSpeed)
 	case "net_all_speed":
-		src = float64(state.NetOutSpeed + state.NetOutSpeed)
+		src = float64(state.NetInSpeed) + float64(state.NetOutSpeed)
 	case "transfer_in":
 		src = float64(state.NetInTransfer)
 	case "transfer_out":
 		src = float64(state.NetOutTransfer)
 	case "transfer_all":
 		src = float64(state.NetOutTransfer + state.NetInTransfer)
-	case "offline":
-		if runtime.LastActive.IsZero() {
-			src = 0
-		} else {
-			src = float64(runtime.LastActive.Unix())
-		}
 	case "transfer_in_cycle":
+		if db == nil {
+			return true, false
+		}
 		src = float64(utils.SubUintChecked(state.NetInTransfer, runtime.PrevTransferInSnapshot))
 		if u.CycleInterval != 0 {
 			var res NResult
-			db.Model(&Transfer{}).Select("SUM(`in`) AS n").Where("datetime(`created_at`) >= datetime(?) AND server_id = ?", u.GetTransferDurationStart().UTC(), server.ID).Scan(&res)
+			if err := db.Model(&Transfer{}).Select("SUM(`in`) AS n").Where("datetime(`created_at`) >= datetime(?) AND datetime(`created_at`) < datetime(?) AND datetime(`created_at`) <= datetime(?) AND server_id = ?", cycleStart.UTC(), cycleEnd.UTC(), evalNow.UTC(), server.ID).Scan(&res).Error; err != nil {
+				log.Printf("NEZHA>> Alert cycle transfer query failed for rule %s server %d: %v", u.Type, server.ID, err)
+				return true, false
+			}
 			src += float64(res.N)
 		}
 	case "transfer_out_cycle":
+		if db == nil {
+			return true, false
+		}
 		src = float64(utils.SubUintChecked(state.NetOutTransfer, runtime.PrevTransferOutSnapshot))
 		if u.CycleInterval != 0 {
 			var res NResult
-			db.Model(&Transfer{}).Select("SUM(`out`) AS n").Where("datetime(`created_at`) >= datetime(?) AND server_id = ?", u.GetTransferDurationStart().UTC(), server.ID).Scan(&res)
+			if err := db.Model(&Transfer{}).Select("SUM(`out`) AS n").Where("datetime(`created_at`) >= datetime(?) AND datetime(`created_at`) < datetime(?) AND datetime(`created_at`) <= datetime(?) AND server_id = ?", cycleStart.UTC(), cycleEnd.UTC(), evalNow.UTC(), server.ID).Scan(&res).Error; err != nil {
+				log.Printf("NEZHA>> Alert cycle transfer query failed for rule %s server %d: %v", u.Type, server.ID, err)
+				return true, false
+			}
 			src += float64(res.N)
 		}
 	case "transfer_all_cycle":
+		if db == nil {
+			return true, false
+		}
 		src = float64(utils.SubUintChecked(state.NetOutTransfer, runtime.PrevTransferOutSnapshot) + utils.SubUintChecked(state.NetInTransfer, runtime.PrevTransferInSnapshot))
 		if u.CycleInterval != 0 {
 			var res NResult
-			db.Model(&Transfer{}).Select("SUM(`in`+`out`) AS n").Where("datetime(`created_at`) >= datetime(?) AND server_id = ?", u.GetTransferDurationStart().UTC(), server.ID).Scan(&res)
+			if err := db.Model(&Transfer{}).Select("SUM(`in`+`out`) AS n").Where("datetime(`created_at`) >= datetime(?) AND datetime(`created_at`) < datetime(?) AND datetime(`created_at`) <= datetime(?) AND server_id = ?", cycleStart.UTC(), cycleEnd.UTC(), evalNow.UTC(), server.ID).Scan(&res).Error; err != nil {
+				log.Printf("NEZHA>> Alert cycle transfer query failed for rule %s server %d: %v", u.Type, server.ID, err)
+				return true, false
+			}
 			src += float64(res.N)
 		}
 	case "load1":
@@ -141,26 +266,38 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server,
 		src = float64(state.ProcessCount)
 	case "temperature_max":
 		var temp []float64
-		if state.Temperatures != nil {
-			for _, tempStat := range state.Temperatures {
-				if tempStat.Temperature != 0 {
-					temp = append(temp, tempStat.Temperature)
-				}
+		for _, tempStat := range state.Temperatures {
+			if tempStat.Temperature != 0 {
+				temp = append(temp, tempStat.Temperature)
 			}
-			src = slices.Max(temp)
 		}
+		if len(temp) == 0 {
+			return true, false
+		}
+		src = slices.Max(temp)
+	default:
+		return true, false
 	}
 
 	// 循环区间流量检测 · 更新下次需要检测时间
 	if u.IsTransferDurationRule() {
-		seconds := max(1800*((u.Max-src)/u.Max), 180)
+		seconds := float64(180)
+		if u.Max > 0 {
+			seconds = max(1800*((u.Max-src)/u.Max), 180)
+		} else if u.Min > 0 {
+			seconds = 1800
+		}
 		if u.NextTransferAt == nil {
 			u.NextTransferAt = make(map[uint64]time.Time)
 		}
 		if u.LastCycleStatus == nil {
 			u.LastCycleStatus = make(map[uint64]bool)
 		}
-		u.NextTransferAt[server.ID] = time.Now().Add(time.Second * time.Duration(seconds))
+		nextAt := evalNow.Add(time.Second * time.Duration(seconds))
+		if nextAt.After(cycleEnd) {
+			nextAt = cycleEnd
+		}
+		u.NextTransferAt[server.ID] = nextAt
 		if (u.Max > 0 && src > u.Max) || (u.Min > 0 && src < u.Min) {
 			u.LastCycleStatus[server.ID] = false
 		} else {
@@ -172,105 +309,77 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server,
 		cycleTransferStats.Transfer[server.ID] = uint64(src)
 		cycleTransferStats.NextUpdate[server.ID] = u.NextTransferAt[server.ID]
 		// 自动更新周期流量展示起止时间
-		cycleTransferStats.From = u.GetTransferDurationStart()
-		cycleTransferStats.To = u.GetTransferDurationEnd()
+		cycleTransferStats.From = cycleStart
+		cycleTransferStats.To = cycleEnd
 	}
 
-	if u.Type == "offline" && float64(time.Now().Unix())-src > 6 {
-		return false
-	} else if (u.Max > 0 && src > u.Max) || (u.Min > 0 && src < u.Min) {
-		return false
+	if (u.Max > 0 && src > u.Max) || (u.Min > 0 && src < u.Min) {
+		return false, true
 	}
 
-	return true
+	return true, true
 }
 
 // IsTransferDurationRule 判断该规则是否属于周期流量规则 属于则返回true
 func (u *Rule) IsTransferDurationRule() bool {
-	return strings.HasSuffix(u.Type, "_cycle")
+	if u == nil {
+		return false
+	}
+	switch u.Type {
+	case "transfer_in_cycle", "transfer_out_cycle", "transfer_all_cycle":
+		return true
+	default:
+		return false
+	}
 }
 
 func (u *Rule) IsOfflineRule() bool {
-	return u.Type == "offline"
+	return u != nil && u.Type == "offline"
+}
+
+// transferDurationBounds computes both edges from the same clock reading.
+// Separate time.Now calls can otherwise straddle a cycle boundary.
+func (u *Rule) transferDurationBounds(now time.Time) (time.Time, time.Time) {
+	if !u.HasSafeCycleConfiguration() {
+		return time.Time{}, time.Time{}
+	}
+	start := *u.CycleStart
+	var end time.Time
+	switch strings.ToLower(u.CycleUnit) {
+	case "year":
+		end = start.AddDate(int(u.CycleInterval), 0, 0)
+		for !now.Before(end) {
+			start, end = end, end.AddDate(int(u.CycleInterval), 0, 0)
+		}
+	case "month":
+		end = start.AddDate(0, int(u.CycleInterval), 0)
+		for !now.Before(end) {
+			start, end = end, end.AddDate(0, int(u.CycleInterval), 0)
+		}
+	case "week":
+		end = start.AddDate(0, 0, 7*int(u.CycleInterval))
+		for !now.Before(end) {
+			start, end = end, end.AddDate(0, 0, 7*int(u.CycleInterval))
+		}
+	case "day":
+		end = start.AddDate(0, 0, int(u.CycleInterval))
+		for !now.Before(end) {
+			start, end = end, end.AddDate(0, 0, int(u.CycleInterval))
+		}
+	default: // empty is the legacy spelling for hour
+		interval := 3600 * int64(u.CycleInterval)
+		start = time.Unix(u.CycleStart.Unix()+(now.Unix()-u.CycleStart.Unix())/interval*interval, 0)
+		end = time.Unix(start.Unix()+interval, 0)
+	}
+	return start, end
 }
 
-// GetTransferDurationStart 获取周期流量的起始时间
 func (u *Rule) GetTransferDurationStart() time.Time {
-	// Accept uppercase and lowercase
-	unit := strings.ToLower(u.CycleUnit)
-	startTime := *u.CycleStart
-	var nextTime time.Time
-	switch unit {
-	case "year":
-		nextTime = startTime.AddDate(int(u.CycleInterval), 0, 0)
-		for time.Now().After(nextTime) {
-			startTime = nextTime
-			nextTime = nextTime.AddDate(int(u.CycleInterval), 0, 0)
-		}
-	case "month":
-		nextTime = startTime.AddDate(0, int(u.CycleInterval), 0)
-		for time.Now().After(nextTime) {
-			startTime = nextTime
-			nextTime = nextTime.AddDate(0, int(u.CycleInterval), 0)
-		}
-	case "week":
-		nextTime = startTime.AddDate(0, 0, 7*int(u.CycleInterval))
-		for time.Now().After(nextTime) {
-			startTime = nextTime
-			nextTime = nextTime.AddDate(0, 0, 7*int(u.CycleInterval))
-		}
-	case "day":
-		nextTime = startTime.AddDate(0, 0, int(u.CycleInterval))
-		for time.Now().After(nextTime) {
-			startTime = nextTime
-			nextTime = nextTime.AddDate(0, 0, int(u.CycleInterval))
-		}
-	default:
-		// For hour unit or not set.
-		interval := 3600 * int64(u.CycleInterval)
-		startTime = time.Unix(u.CycleStart.Unix()+(time.Now().Unix()-u.CycleStart.Unix())/interval*interval, 0)
-	}
-
-	return startTime
+	start, _ := u.transferDurationBounds(time.Now())
+	return start
 }
 
-// GetTransferDurationEnd 获取周期流量结束时间
 func (u *Rule) GetTransferDurationEnd() time.Time {
-	// Accept uppercase and lowercase
-	unit := strings.ToLower(u.CycleUnit)
-	startTime := *u.CycleStart
-	var nextTime time.Time
-	switch unit {
-	case "year":
-		nextTime = startTime.AddDate(int(u.CycleInterval), 0, 0)
-		for time.Now().After(nextTime) {
-			startTime = nextTime
-			nextTime = nextTime.AddDate(int(u.CycleInterval), 0, 0)
-		}
-	case "month":
-		nextTime = startTime.AddDate(0, int(u.CycleInterval), 0)
-		for time.Now().After(nextTime) {
-			startTime = nextTime
-			nextTime = nextTime.AddDate(0, int(u.CycleInterval), 0)
-		}
-	case "week":
-		nextTime = startTime.AddDate(0, 0, 7*int(u.CycleInterval))
-		for time.Now().After(nextTime) {
-			startTime = nextTime
-			nextTime = nextTime.AddDate(0, 0, 7*int(u.CycleInterval))
-		}
-	case "day":
-		nextTime = startTime.AddDate(0, 0, int(u.CycleInterval))
-		for time.Now().After(nextTime) {
-			startTime = nextTime
-			nextTime = nextTime.AddDate(0, 0, int(u.CycleInterval))
-		}
-	default:
-		// For hour unit or not set.
-		interval := 3600 * int64(u.CycleInterval)
-		startTime = time.Unix(u.CycleStart.Unix()+(time.Now().Unix()-u.CycleStart.Unix())/interval*interval, 0)
-		nextTime = time.Unix(startTime.Unix()+interval, 0)
-	}
-
-	return nextTime
+	_, end := u.transferDurationBounds(time.Now())
+	return end
 }

model/server.go

@@ -16,6 +16,7 @@ import (
 )
 
 var runtimeHolderInitMu sync.Mutex
+var nextServerReportSequence atomic.Uint64
 
 type Server struct {
 	Common
@@ -75,15 +76,16 @@ type taskStreamHolder struct {
 }
 
 type serverRuntimeHolder struct {
-	mu         sync.Mutex
-	canonical  *Server
-	stream     pb.NezhaService_ReportSystemStateServer
-	generation uint64
-	state      *HostState
-	host       *Host
-	lastActive time.Time
-	prevIn     uint64
-	prevOut    uint64
+	mu             sync.Mutex
+	canonical      *Server
+	stream         pb.NezhaService_ReportSystemStateServer
+	generation     uint64
+	state          *HostState
+	host           *Host
+	lastActive     time.Time
+	reportSequence uint64
+	prevIn         uint64
+	prevOut        uint64
 }
 
 type StateStreamLease struct {
@@ -206,14 +208,21 @@ func (lease StateStreamLease) updateState(receiver *Server, state *HostState, la
 			return false
 		}
 	}
+	lease.holder.reportSequence = nextServerReportSequence.Add(1)
 	return true
 }
 
 func (lease StateStreamLease) Clear() bool {
-	return lease.clear(nil)
+	return lease.clear(nil, nil)
 }
 
-func (lease StateStreamLease) clear(receiver *Server) bool {
+// ClearWithSideEffect serializes disconnect persistence with replacement streams.
+// The callback must not acquire the runtime holder again.
+func (lease StateStreamLease) ClearWithSideEffect(sideEffect func()) bool {
+	return lease.clear(nil, sideEffect)
+}
+
+func (lease StateStreamLease) clear(receiver *Server, sideEffect func()) bool {
 	if lease.holder == nil {
 		return false
 	}
@@ -225,6 +234,9 @@ func (lease StateStreamLease) clear(receiver *Server) bool {
 	lease.holder.stream = nil
 	lease.holder.lastActive = time.Time{}
 	lease.holder.canonical.LastActive = time.Time{}
+	if sideEffect != nil {
+		sideEffect()
+	}
 	return true
 }
 
@@ -364,7 +376,7 @@ func (s *Server) UpdateStateIfCurrentWithSideEffect(lease StateStreamLease, stat
 }
 
 func (s *Server) ClearStateStreamIfCurrent(lease StateStreamLease) bool {
-	return lease.clear(s)
+	return lease.clear(s, nil)
 }
 
 // RuntimeSnapshot is a deep copy of the mutable runtime state.
@@ -372,6 +384,7 @@ type RuntimeSnapshot struct {
 	State                   *HostState
 	Host                    *Host
 	LastActive              time.Time
+	ReportSequence          uint64
 	PrevTransferInSnapshot  uint64
 	PrevTransferOutSnapshot uint64
 }
@@ -398,7 +411,7 @@ func (s *Server) RuntimeSnapshot() RuntimeSnapshot {
 			holder.host = cloneHost(s.Host)
 		}
 	}
-	return RuntimeSnapshot{State: cloneHostState(holder.state), Host: cloneHost(holder.host), LastActive: holder.lastActive, PrevTransferInSnapshot: holder.prevIn, PrevTransferOutSnapshot: holder.prevOut}
+	return RuntimeSnapshot{State: cloneHostState(holder.state), Host: cloneHost(holder.host), LastActive: holder.lastActive, ReportSequence: holder.reportSequence, PrevTransferInSnapshot: holder.prevIn, PrevTransferOutSnapshot: holder.prevOut}
 }
 
 func (s *Server) SetTransferSnapshots(inbound, outbound uint64) bool {

model/server_runtime_ownership_test.go

@@ -367,3 +367,30 @@ func TestServerRuntimeOwnership_failedRestartAllowsHourlyRecordThenRetry(t *test
 	require.Equal(t, uint64(0), result.Transfer.In)
 	require.Equal(t, uint64(0), result.Transfer.Out)
 }
+
+func TestStateStreamDisconnectSideEffectSerializesReplacement(t *testing.T) {
+	server := &Server{}
+	old := server.AttachStateStream(runtimeOwnershipStream{})
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	cleared := make(chan bool, 1)
+	go func() { cleared <- old.ClearWithSideEffect(func() { close(entered); <-release }) }()
+	<-entered
+	attached := make(chan StateStreamLease, 1)
+	go func() { attached <- server.AttachStateStream(runtimeOwnershipStream{}) }()
+	select {
+	case <-attached:
+		close(release)
+		t.Fatal("replacement attached before disconnect persistence finished")
+	case <-time.After(20 * time.Millisecond):
+	}
+	close(release)
+	require.True(t, <-cleared)
+	next := <-attached
+	require.True(t, next.UpdateState(&HostState{}, time.Now()))
+	called := false
+	require.False(t, old.ClearWithSideEffect(func() { called = true }))
+	require.False(t, called)
+	require.False(t, old.UpdateStateWithSideEffect(&HostState{}, time.Now(), func() error { called = true; return nil }))
+	require.False(t, called)
+}

model/setting_api.go

@@ -35,6 +35,8 @@ type SettingForm struct {
 	AdminEmail                   string `json:"admin_email,omitempty" validate:"optional"`
 	DomainExpiryNotificationDays string `json:"domain_expiry_notification_days,omitempty" validate:"optional"`
 	ServerExpiryNotificationDays string `json:"server_expiry_notification_days,omitempty" validate:"optional"`
+	AllowJWTIPChange             *bool  `json:"allow_jwt_ip_change,omitempty" validate:"optional"`
+	OnlineStatsDays              *int   `json:"online_stats_days,omitempty" validate:"optional"`
 }
 
 type Setting struct {

pkg/ddns/ddns.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log"
 	"net/netip"
+	"strings"
 	"time"
 
 	"github.com/libdns/libdns"
@@ -21,9 +22,6 @@ const (
 )
 
 type Provider struct {
-	prefix string
-	zone   string
-
 	DDNSProfile *model.DDNSProfile
 	IPAddrs     *model.IP
 	Setter      libdns.RecordSetter
@@ -34,61 +32,81 @@ func (provider *Provider) GetProfileID() uint64 {
 }
 
 func (provider *Provider) UpdateDomain(ctx context.Context, overrideDomains ...string) {
-	for _, domain := range utils.IfOr(len(overrideDomains) > 0, overrideDomains, provider.DDNSProfile.Domains) {
-		for retries := range int(provider.DDNSProfile.MaxRetries) {
-			log.Printf("NEZHA>> Updating DNS Record of domain %s: %d/%d", domain, retries+1, provider.DDNSProfile.MaxRetries)
-			if err := provider.updateDomain(ctx, domain); err != nil {
-				log.Printf("NEZHA>> Failed to update DNS record of domain %s: %v", domain, err)
-			} else {
-				log.Printf("NEZHA>> Update DNS record of domain %s succeeded", domain)
+	domains := utils.IfOr(len(overrideDomains) > 0, overrideDomains, provider.DDNSProfile.Domains)
+	maxRetries := int(provider.DDNSProfile.MaxRetries)
+	if maxRetries <= 0 {
+		maxRetries = 1
+	}
+
+	for _, domain := range domains {
+		var prefix, zone string
+		var soaErr error
+
+		for retries := 0; retries < maxRetries; retries++ {
+			prefix, zone, soaErr = provider.splitDomainSOA(ctx, domain)
+			if soaErr == nil {
 				break
 			}
+			log.Printf("NEZHA>> Failed to split domain SOA for %s (attempt %d/%d): %v", domain, retries+1, maxRetries, soaErr)
+		}
+
+		if soaErr != nil {
+			log.Printf("NEZHA>> Failed to split domain SOA for %s after %d retries, skipping domain", domain, maxRetries)
+			continue
+		}
+
+		// 独立处理 IPv4 更新或删除
+		if provider.DDNSProfile.EnableIPv4 != nil && *provider.DDNSProfile.EnableIPv4 {
+			for retries := 0; retries < maxRetries; retries++ {
+				log.Printf("NEZHA>> Updating IPv4 record of domain %s: %d/%d", domain, retries+1, maxRetries)
+				var ipv4Err error
+				if provider.IPAddrs.IPv4Addr == "" {
+					ipv4Err = provider.deleteDomainRecord(ctx, prefix, zone, "A")
+				} else {
+					ipv4Err = provider.addDomainRecord(ctx, prefix, zone, "A", provider.IPAddrs.IPv4Addr)
+				}
+
+				if ipv4Err != nil {
+					log.Printf("NEZHA>> Failed to update IPv4 record of domain %s: %v", domain, ipv4Err)
+				} else {
+					log.Printf("NEZHA>> Update IPv4 record of domain %s succeeded", domain)
+					break
+				}
+			}
+		}
+
+		// 独立处理 IPv6 更新或删除
+		if provider.DDNSProfile.EnableIPv6 != nil && *provider.DDNSProfile.EnableIPv6 {
+			for retries := 0; retries < maxRetries; retries++ {
+				log.Printf("NEZHA>> Updating IPv6 record of domain %s: %d/%d", domain, retries+1, maxRetries)
+				var ipv6Err error
+				if provider.IPAddrs.IPv6Addr == "" {
+					ipv6Err = provider.deleteDomainRecord(ctx, prefix, zone, "AAAA")
+				} else {
+					ipv6Err = provider.addDomainRecord(ctx, prefix, zone, "AAAA", provider.IPAddrs.IPv6Addr)
+				}
+
+				if ipv6Err != nil {
+					log.Printf("NEZHA>> Failed to update IPv6 record of domain %s: %v", domain, ipv6Err)
+				} else {
+					log.Printf("NEZHA>> Update IPv6 record of domain %s succeeded", domain)
+					break
+				}
+			}
 		}
 	}
 }
 
-func (provider *Provider) updateDomain(ctx context.Context, domain string) error {
-	var err error
-	provider.prefix, provider.zone, err = provider.splitDomainSOA(ctx, domain)
-	if err != nil {
-		return err
-	}
-
-	// 独立处理 IPv4 更新
-	if *provider.DDNSProfile.EnableIPv4 {
-		if provider.IPAddrs.IPv4Addr == "" {
-			log.Printf("NEZHA>> Skip IPv4 update for domain %s: IPv4 address is empty", domain)
-		} else {
-			if err = provider.addDomainRecord(ctx, "A", provider.IPAddrs.IPv4Addr); err != nil {
-				return err
-			}
-		}
-	}
-
-	// 独立处理 IPv6 更新
-	if *provider.DDNSProfile.EnableIPv6 {
-		if provider.IPAddrs.IPv6Addr == "" {
-			log.Printf("NEZHA>> Skip IPv6 update for domain %s: IPv6 address is empty", domain)
-		} else {
-			if err = provider.addDomainRecord(ctx, "AAAA", provider.IPAddrs.IPv6Addr); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (provider *Provider) addDomainRecord(ctx context.Context, recType, addr string) error {
+func (provider *Provider) addDomainRecord(ctx context.Context, prefix, zone, recType, addr string) error {
 	netipAddr, err := netip.ParseAddr(addr)
 	if err != nil {
 		return fmt.Errorf("parse error: %v", err)
 	}
 
-	_, err = provider.Setter.SetRecords(ctx, provider.zone,
+	_, err = provider.Setter.SetRecords(ctx, zone,
 		[]libdns.Record{
 			libdns.Address{
-				Name: provider.prefix,
+				Name: prefix,
 				IP:   netipAddr,
 				TTL:  time.Minute,
 			},
@@ -96,6 +114,28 @@ func (provider *Provider) addDomainRecord(ctx context.Context, recType, addr str
 	return err
 }
 
+func (provider *Provider) deleteDomainRecord(ctx context.Context, prefix, zone, recType string) error {
+	deleter, okDeleter := provider.Setter.(libdns.RecordDeleter)
+	if !okDeleter {
+		log.Printf("NEZHA>> DNS provider does not support RecordDeleter, safely skipping deletion for %s", recType)
+		return nil
+	}
+
+	targetRecType := strings.ToUpper(recType)
+	_, err := deleter.DeleteRecords(ctx, zone, []libdns.Record{
+		libdns.RR{
+			Name: prefix,
+			Type: targetRecType,
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("deleter.DeleteRecords failed: %w", err)
+	}
+
+	log.Printf("NEZHA>> Successfully deleted %s record for %s.%s", recType, prefix, zone)
+	return nil
+}
+
 func (provider *Provider) splitDomainSOA(ctx context.Context, domain string) (prefix string, zone string, err error) {
 	c := &dns.Client{Timeout: dnsTimeOut}
 
@@ -103,7 +143,6 @@ func (provider *Provider) splitDomainSOA(ctx context.Context, domain string) (pr
 	indexes := dns.Split(domain)
 
 	servers := utils.DNSServers
-
 	customDNSServers, _ := ctx.Value(DNSServerKey{}).([]string)
 	if len(customDNSServers) > 0 {
 		servers = customDNSServers
@@ -115,7 +154,7 @@ func (provider *Provider) splitDomainSOA(ctx context.Context, domain string) (pr
 			m.SetQuestion(domain[idx:], dns.TypeSOA)
 
 			r, _, err := c.Exchange(&m, server)
-			if err != nil {
+			if err != nil || r == nil {
 				continue
 			}
 

pkg/ddns/ddns_test.go

@@ -2,7 +2,15 @@ package ddns
 
 import (
 	"context"
+	"net"
+	"strings"
+	"sync/atomic"
 	"testing"
+
+	"github.com/libdns/libdns"
+	"github.com/miekg/dns"
+
+	"github.com/nezhahq/nezha/model"
 )
 
 type testSt struct {
@@ -12,6 +20,50 @@ type testSt struct {
 }
 
 func TestSplitDomainSOA(t *testing.T) {
+	handler := dns.NewServeMux()
+	handler.HandleFunc(".", func(w dns.ResponseWriter, r *dns.Msg) {
+		m := new(dns.Msg)
+		m.SetReply(r)
+		m.Authoritative = true
+		if len(r.Question) > 0 {
+			qname := r.Question[0].Name
+			var zoneName string
+			if strings.HasSuffix(qname, "example.co.uk.") {
+				zoneName = "example.co.uk."
+			} else if strings.HasSuffix(qname, "example.org.") {
+				zoneName = "example.org."
+			} else if strings.HasSuffix(qname, "example.com.") {
+				zoneName = "example.com."
+			}
+
+			if zoneName != "" {
+				soa := &dns.SOA{
+					Hdr:    dns.RR_Header{Name: zoneName, Rrtype: dns.TypeSOA, Class: dns.ClassINET, Ttl: 300},
+					Ns:     "ns.example.com.",
+					Mbox:   "admin.example.com.",
+					Serial: 1,
+				}
+				m.Answer = append(m.Answer, soa)
+				m.Ns = append(m.Ns, soa)
+			}
+		}
+		_ = w.WriteMsg(m)
+	})
+
+	pc, err := net.ListenPacket("udp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("failed to listen: %v", err)
+	}
+	defer pc.Close()
+
+	server := &dns.Server{PacketConn: pc, Net: "udp", Handler: handler}
+	go func() {
+		_ = server.ActivateAndServe()
+	}()
+	defer server.Shutdown()
+
+	addr := pc.LocalAddr().String()
+
 	cases := []testSt{
 		{
 			domain: "www.example.com",
@@ -30,13 +82,12 @@ func TestSplitDomainSOA(t *testing.T) {
 		},
 	}
 
-	ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{"1.1.1.1:53"})
+	ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{addr})
 	provider := &Provider{}
 	for _, c := range cases {
 		prefix, zone, err := provider.splitDomainSOA(ctx, c.domain)
 		if err != nil {
-			t.Skipf("Skipping live DNS test due to network: %v", err)
-			return
+			t.Fatalf("Error for domain %s: %s", c.domain, err)
 		}
 		if prefix != c.prefix {
 			t.Fatalf("Expected prefix %s, but got %s", c.prefix, prefix)
@@ -46,3 +97,191 @@ func TestSplitDomainSOA(t *testing.T) {
 		}
 	}
 }
+
+type MockSetter struct {
+	Records        []libdns.Record
+	SetCalled      bool
+	DelCalled      bool
+	DeletedRecords []libdns.Record
+}
+
+func (m *MockSetter) SetRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) {
+	m.SetCalled = true
+	m.Records = append(m.Records, records...)
+	return records, nil
+}
+
+func (m *MockSetter) DeleteRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) {
+	m.DelCalled = true
+	m.DeletedRecords = append(m.DeletedRecords, records...)
+	return records, nil
+}
+
+type MockUnsupportedSetter struct{}
+
+func (m *MockUnsupportedSetter) SetRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) {
+	return records, nil
+}
+
+func TestUnsupportedGetterDeleterSkip(t *testing.T) {
+	provider := &Provider{
+		DDNSProfile: &model.DDNSProfile{MaxRetries: 1},
+		IPAddrs:     &model.IP{IPv4Addr: ""},
+		Setter:      &MockUnsupportedSetter{},
+	}
+
+	err := provider.deleteDomainRecord(context.Background(), "sub", "example.com", "A")
+	if err != nil {
+		t.Fatalf("expected no error for unsupported deleter skip, got %v", err)
+	}
+}
+
+func TestIndependentDualStackProcessing(t *testing.T) {
+	enableIPv4 := true
+	enableIPv6 := true
+	mockSetter := &MockSetter{}
+
+	provider := &Provider{
+		DDNSProfile: &model.DDNSProfile{
+			MaxRetries: 1,
+			Domains:    []string{"example.com"},
+			EnableIPv4: &enableIPv4,
+			EnableIPv6: &enableIPv6,
+		},
+		IPAddrs: &model.IP{
+			IPv4Addr: "1.1.1.1",
+			IPv6Addr: "2001:db8::1",
+		},
+		Setter: mockSetter,
+	}
+
+	handler := dns.NewServeMux()
+	handler.HandleFunc(".", func(w dns.ResponseWriter, r *dns.Msg) {
+		m := new(dns.Msg)
+		m.SetReply(r)
+		m.Authoritative = true
+		if len(r.Question) > 0 {
+			soa := &dns.SOA{
+				Hdr:    dns.RR_Header{Name: r.Question[0].Name, Rrtype: dns.TypeSOA, Class: dns.ClassINET, Ttl: 300},
+				Ns:     "ns.example.com.",
+				Mbox:   "admin.example.com.",
+				Serial: 1,
+			}
+			m.Answer = append(m.Answer, soa)
+			m.Ns = append(m.Ns, soa)
+		}
+		_ = w.WriteMsg(m)
+	})
+
+	pc, err := net.ListenPacket("udp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("failed to listen: %v", err)
+	}
+	defer pc.Close()
+
+	server := &dns.Server{PacketConn: pc, Net: "udp", Handler: handler}
+	go func() {
+		_ = server.ActivateAndServe()
+	}()
+	defer server.Shutdown()
+
+	ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{pc.LocalAddr().String()})
+	provider.UpdateDomain(ctx)
+
+	if !mockSetter.SetCalled {
+		t.Fatalf("expected SetRecords to have been called for dual stack updates")
+	}
+
+	foundIPv4 := false
+	foundIPv6 := false
+	for _, rec := range mockSetter.Records {
+		if addrRec, ok := rec.(libdns.Address); ok {
+			if addrRec.IP.String() == "1.1.1.1" {
+				foundIPv4 = true
+			}
+			if addrRec.IP.String() == "2001:db8::1" {
+				foundIPv6 = true
+			}
+		}
+	}
+
+	if !foundIPv4 || !foundIPv6 {
+		t.Fatalf("expected both IPv4 and IPv6 records to be processed independently, got records: %v", mockSetter.Records)
+	}
+}
+
+func TestTransientSOAFailureAndRetry(t *testing.T) {
+	var attempts int64
+	handler := dns.NewServeMux()
+	handler.HandleFunc(".", func(w dns.ResponseWriter, r *dns.Msg) {
+		atomic.AddInt64(&attempts, 1)
+		m := new(dns.Msg)
+		m.SetReply(r)
+		_ = w.WriteMsg(m)
+	})
+
+	pc, err := net.ListenPacket("udp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("failed to listen: %v", err)
+	}
+	defer pc.Close()
+
+	server := &dns.Server{PacketConn: pc, Net: "udp", Handler: handler}
+	go func() {
+		_ = server.ActivateAndServe()
+	}()
+	defer server.Shutdown()
+
+	var maxRetries uint64 = 3
+	provider := &Provider{
+		DDNSProfile: &model.DDNSProfile{
+			MaxRetries: maxRetries,
+			Domains:    []string{"example.com"},
+			EnableIPv4: &[]bool{true}[0],
+			EnableIPv6: &[]bool{false}[0],
+		},
+		IPAddrs: &model.IP{IPv4Addr: "1.1.1.1"},
+		Setter:  &MockSetter{},
+	}
+
+	ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{pc.LocalAddr().String()})
+	provider.UpdateDomain(ctx)
+
+	actualAttempts := atomic.LoadInt64(&attempts)
+	expectedAttempts := int64(maxRetries) * 2
+	if actualAttempts != expectedAttempts {
+		t.Fatalf("expected exact attempt count of %d, got %d", expectedAttempts, actualAttempts)
+	}
+}
+
+func TestGenericRecordDeleterContract(t *testing.T) {
+	mockSetter := &MockSetter{}
+	provider := &Provider{
+		DDNSProfile: &model.DDNSProfile{MaxRetries: 1},
+		IPAddrs:     &model.IP{IPv4Addr: ""},
+		Setter:      mockSetter,
+	}
+
+	ctx := context.Background()
+	err := provider.deleteDomainRecord(ctx, "sub", "example.com", "A")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if !mockSetter.DelCalled {
+		t.Fatalf("expected DeleteRecords to be called")
+	}
+
+	if len(mockSetter.DeletedRecords) != 1 {
+		t.Fatalf("expected 1 record sent for deletion, got %d", len(mockSetter.DeletedRecords))
+	}
+
+	rr, ok := mockSetter.DeletedRecords[0].(libdns.RR)
+	if !ok {
+		t.Fatalf("expected deletion payload to be libdns.RR")
+	}
+
+	if rr.Name != "sub" || rr.Type != "A" {
+		t.Fatalf("unexpected RR fields: Name=%s, Type=%s", rr.Name, rr.Type)
+	}
+}

pkg/tsdb/disk_guard_test.go

@@ -0,0 +1,124 @@
+package tsdb
+
+import (
+	"path/filepath"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type testReadOnlyChecker struct {
+	value atomic.Bool
+}
+
+func (c *testReadOnlyChecker) IsReadOnly() bool {
+	return c.value.Load()
+}
+
+func newDiskGuardTestDB(t *testing.T) *TSDB {
+	t.Helper()
+	db, err := Open(&Config{
+		DataPath:           filepath.Join(t.TempDir(), "tsdb"),
+		RetentionDays:      1,
+		MinFreeDiskSpaceGB: 1,
+		DedupInterval:      time.Second,
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, db.Close()) })
+	return db
+}
+
+func bufferedRows(w *bufferedWriter) int {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return len(w.buffer)
+}
+
+func TestReadOnlyStorageDropsWritesAndKeepsQueriesAvailable(t *testing.T) {
+	db := newDiskGuardTestDB(t)
+	checker := &testReadOnlyChecker{}
+	db.readOnly = checker
+	originalAddRows := db.addRowsFn
+	var addRowsCalls atomic.Int32
+	db.addRowsFn = func(rows []storage.MetricRow, precisionBits uint8) {
+		addRowsCalls.Add(1)
+		originalAddRows(rows, precisionBits)
+	}
+
+	firstTimestamp := time.Now().Add(-time.Minute)
+	require.NoError(t, db.WriteServerMetrics(&ServerMetrics{
+		ServerID:  1,
+		Timestamp: firstTimestamp,
+		CPU:       10,
+	}))
+	db.Flush()
+	require.Equal(t, int32(1), addRowsCalls.Load())
+
+	before, err := db.QueryServerMetrics(1, MetricServerCPU, Period1Day)
+	require.NoError(t, err)
+	require.NotEmpty(t, before)
+
+	checker.value.Store(true)
+	require.NoError(t, db.WriteServerMetrics(&ServerMetrics{
+		ServerID:  1,
+		Timestamp: firstTimestamp.Add(2 * time.Second),
+		CPU:       20,
+	}))
+	assert.True(t, db.WritesPaused())
+	assert.Zero(t, bufferedRows(db.writer), "read-only writes must not accumulate in memory")
+	assert.Equal(t, int32(1), addRowsCalls.Load(), "read-only samples must not reach VictoriaMetrics")
+
+	afterDrop, err := db.QueryServerMetrics(1, MetricServerCPU, Period1Day)
+	require.NoError(t, err)
+	assert.Equal(t, before, afterDrop, "read-only mode must preserve existing query access")
+
+	checker.value.Store(false)
+	require.NoError(t, db.WriteServerMetrics(&ServerMetrics{
+		ServerID:  1,
+		Timestamp: firstTimestamp.Add(4 * time.Second),
+		CPU:       30,
+	}))
+	db.Flush()
+	assert.False(t, db.WritesPaused())
+	assert.Equal(t, int32(2), addRowsCalls.Load(), "writes must resume after storage leaves read-only mode")
+}
+
+func TestDiskFullAddRowsPanicPausesWrites(t *testing.T) {
+	db := newDiskGuardTestDB(t)
+	db.addRowsFn = func([]storage.MetricRow, uint8) {
+		panic("write data: no space left on device")
+	}
+
+	require.NoError(t, db.WriteServerMetrics(&ServerMetrics{
+		ServerID:  1,
+		Timestamp: time.Now(),
+		CPU:       10,
+	}))
+	assert.NotPanics(t, db.Flush)
+	assert.True(t, db.WritesPaused())
+
+	require.NoError(t, db.WriteServerMetrics(&ServerMetrics{
+		ServerID:  1,
+		Timestamp: time.Now().Add(time.Second),
+		CPU:       20,
+	}))
+	assert.Zero(t, bufferedRows(db.writer))
+}
+
+func TestNonDiskStoragePanicIsNotHidden(t *testing.T) {
+	db := newDiskGuardTestDB(t)
+	db.addRowsFn = func([]storage.MetricRow, uint8) {
+		panic("storage invariant failed")
+	}
+
+	require.NoError(t, db.WriteServerMetrics(&ServerMetrics{
+		ServerID:  1,
+		Timestamp: time.Now(),
+		CPU:       10,
+	}))
+	assert.PanicsWithValue(t, "storage invariant failed", db.Flush)
+}

pkg/tsdb/maintenance.go

@@ -10,8 +10,11 @@ func (db *TSDB) Maintenance() {
 	if db.closed {
 		return
 	}
+	if !db.acceptsWrites() {
+		return
+	}
 
 	log.Println("NEZHA>> TSDB starting maintenance (flush)...")
-	db.storage.DebugFlush()
+	db.debugFlushSafely()
 	log.Println("NEZHA>> TSDB maintenance completed")
 }

pkg/tsdb/tsdb.go

@@ -1,15 +1,26 @@
 package tsdb
 
 import (
+	"errors"
 	"fmt"
 	"log"
 	"path/filepath"
+	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
+// ErrDiskFull is returned when VictoriaMetrics cannot initialize because the
+// filesystem containing the TSDB has run out of space.
+var ErrDiskFull = errors.New("TSDB disk is full")
+
+type readOnlyChecker interface {
+	IsReadOnly() bool
+}
+
 // TSDB 封装 VictoriaMetrics 存储
 type TSDB struct {
 	storage *storage.Storage
@@ -18,6 +29,13 @@ type TSDB struct {
 	closed  bool
 
 	writer *bufferedWriter
+
+	readOnly     readOnlyChecker
+	addRowsFn    func([]storage.MetricRow, uint8)
+	debugFlushFn func()
+
+	readOnlyObserved atomic.Bool
+	diskFullPaused   atomic.Bool
 }
 
 // InitGlobalSettings 初始化 VictoriaMetrics 包级别的全局设置。
@@ -35,7 +53,18 @@ func InitGlobalSettings(config *Config) {
 }
 
 // Open 打开或创建 TSDB 存储
-func Open(config *Config) (*TSDB, error) {
+func Open(config *Config) (db *TSDB, err error) {
+	defer func() {
+		if recovered := recover(); recovered != nil {
+			if diskErr := diskFullPanicError(recovered); diskErr != nil {
+				db = nil
+				err = diskErr
+				return
+			}
+			panic(recovered)
+		}
+	}()
+
 	if config == nil {
 		config = DefaultConfig()
 	}
@@ -59,9 +88,12 @@ func Open(config *Config) (*TSDB, error) {
 
 	stor := storage.MustOpenStorage(dataPath, opts)
 
-	db := &TSDB{
-		storage: stor,
-		config:  config,
+	db = &TSDB{
+		storage:      stor,
+		config:       config,
+		readOnly:     stor,
+		addRowsFn:    stor.AddRows,
+		debugFlushFn: stor.DebugFlush,
 	}
 
 	db.writer = newBufferedWriter(db, config.WriteBufferSize, config.WriteBufferFlushInterval)
@@ -72,6 +104,86 @@ func Open(config *Config) (*TSDB, error) {
 	return db, nil
 }
 
+func diskFullPanicError(recovered any) error {
+	message := strings.ToLower(fmt.Sprint(recovered))
+	for _, marker := range []string{
+		"no space left on device",
+		"disk quota exceeded",
+		"not enough space on the disk",
+	} {
+		if strings.Contains(message, marker) {
+			return fmt.Errorf("%w: %v", ErrDiskFull, recovered)
+		}
+	}
+	return nil
+}
+
+// WritesPaused reports whether new samples are currently being discarded.
+// Queries remain available while VictoriaMetrics is in read-only mode.
+func (db *TSDB) WritesPaused() bool {
+	if db.diskFullPaused.Load() {
+		return true
+	}
+	return db.readOnly != nil && db.readOnly.IsReadOnly()
+}
+
+func (db *TSDB) acceptsWrites() bool {
+	if db.diskFullPaused.Load() {
+		return false
+	}
+
+	readOnly := db.readOnly != nil && db.readOnly.IsReadOnly()
+	if readOnly {
+		if db.readOnlyObserved.CompareAndSwap(false, true) {
+			log.Println("NEZHA>> TSDB writes paused because free disk space is below the configured limit; dashboard, queries, and alerts remain available")
+		}
+		return false
+	}
+
+	if db.readOnlyObserved.CompareAndSwap(true, false) {
+		log.Println("NEZHA>> TSDB writes resumed after free disk space recovered")
+	}
+	return true
+}
+
+func (db *TSDB) pauseWritesAfterDiskFull(recovered any) {
+	if db.diskFullPaused.CompareAndSwap(false, true) {
+		log.Printf("NEZHA>> TSDB disk is full; disabling TSDB writes until dashboard restart while keeping dashboard, queries, and alerts running: %v", recovered)
+	}
+}
+
+func (db *TSDB) addRowsSafely(rows []storage.MetricRow) {
+	if len(rows) == 0 || !db.acceptsWrites() {
+		return
+	}
+
+	defer func() {
+		if recovered := recover(); recovered != nil {
+			if diskFullPanicError(recovered) == nil {
+				panic(recovered)
+			}
+			db.pauseWritesAfterDiskFull(recovered)
+		}
+	}()
+	db.addRowsFn(rows, 64)
+}
+
+func (db *TSDB) debugFlushSafely() {
+	if !db.acceptsWrites() {
+		return
+	}
+
+	defer func() {
+		if recovered := recover(); recovered != nil {
+			if diskFullPanicError(recovered) == nil {
+				panic(recovered)
+			}
+			db.pauseWritesAfterDiskFull(recovered)
+		}
+	}()
+	db.debugFlushFn()
+}
+
 // Close 关闭 TSDB 存储
 func (db *TSDB) Close() error {
 	db.mu.Lock()
@@ -110,8 +222,14 @@ func (db *TSDB) IsClosed() bool {
 
 // Flush 强制刷盘（主要用于测试）
 func (db *TSDB) Flush() {
+	db.mu.RLock()
+	defer db.mu.RUnlock()
+	if db.closed {
+		return
+	}
+
 	if db.writer != nil {
 		db.writer.flush()
 	}
-	db.storage.DebugFlush()
+	db.debugFlushSafely()
 }

pkg/tsdb/writer.go

@@ -47,18 +47,29 @@ func (w *bufferedWriter) flushLoop() {
 }
 
 func (w *bufferedWriter) write(rows []storage.MetricRow) {
+	if !w.db.acceptsWrites() {
+		w.discard()
+		return
+	}
+
 	w.mu.Lock()
 	w.buffer = append(w.buffer, rows...)
 	if len(w.buffer) >= w.maxSize {
 		rows := w.buffer
 		w.buffer = make([]storage.MetricRow, 0, w.maxSize)
 		w.mu.Unlock()
-		w.db.storage.AddRows(rows, 64)
+		w.db.addRowsSafely(rows)
 		return
 	}
 	w.mu.Unlock()
 }
 
+func (w *bufferedWriter) discard() {
+	w.mu.Lock()
+	w.buffer = make([]storage.MetricRow, 0, w.maxSize)
+	w.mu.Unlock()
+}
+
 func (w *bufferedWriter) flush() {
 	w.mu.Lock()
 	if len(w.buffer) == 0 {
@@ -69,7 +80,7 @@ func (w *bufferedWriter) flush() {
 	w.buffer = make([]storage.MetricRow, 0, w.maxSize)
 	w.mu.Unlock()
 
-	w.db.storage.AddRows(rows, 64)
+	w.db.addRowsSafely(rows)
 }
 
 func (w *bufferedWriter) stop() {
@@ -171,7 +182,7 @@ func (db *TSDB) WriteServerMetrics(m *ServerMetrics) error {
 	if db.writer != nil {
 		db.writer.write(rows)
 	} else {
-		db.storage.AddRows(rows, 64)
+		db.addRowsSafely(rows)
 	}
 	return nil
 }
@@ -200,7 +211,7 @@ func (db *TSDB) WriteServiceMetrics(m *ServiceMetrics) error {
 	if db.writer != nil {
 		db.writer.write(rows)
 	} else {
-		db.storage.AddRows(rows, 64)
+		db.addRowsSafely(rows)
 	}
 	return nil
 }
@@ -265,7 +276,7 @@ func (db *TSDB) WriteBatchServerMetrics(metrics []*ServerMetrics) error {
 	if db.writer != nil {
 		db.writer.write(rows)
 	} else {
-		db.storage.AddRows(rows, 64)
+		db.addRowsSafely(rows)
 	}
 	return nil
 }
@@ -295,7 +306,7 @@ func (db *TSDB) WriteBatchServiceMetrics(metrics []*ServiceMetrics) error {
 	if db.writer != nil {
 		db.writer.write(rows)
 	} else {
-		db.storage.AddRows(rows, 64)
+		db.addRowsSafely(rows)
 	}
 	return nil
 }

proto/nezha.proto

@@ -44,6 +44,16 @@ message State {
   uint64 process_count = 15;
   repeated State_SensorTemperature temperatures = 16;
   repeated double gpu = 17;
+  repeated State_GPU gpus = 18;
+}
+
+// State_GPU carries per-card figures. Index-aligned with Host.gpu, matching
+// the existing `gpu` field it supersedes; that field stays populated so older
+// dashboards keep working.
+message State_GPU {
+  double utilization = 1;
+  uint64 memory_used = 2;
+  uint64 memory_total = 3;
 }
 
 message State_SensorTemperature {

service/rpc/nezha.go

@@ -204,7 +204,15 @@ func (s *NezhaHandler) ReportSystemState(stream pb.NezhaService_ReportSystemStat
 		return errors.New("server not found")
 	}
 	lease := server.AttachStateStream(stream)
-	defer lease.Clear()
+	defer func() {
+		// Persist while owning the lease; a replaced stream must never close a
+		// newer stream's session between Clear and RecordOffline.
+		lease.ClearWithSideEffect(func() {
+			if singleton.OnlineHistoryShared != nil {
+				singleton.OnlineHistoryShared.RecordOffline(clientID, time.Now())
+			}
+		})
+	}()
 	var state *pb.State
 	var stateCount uint64
 	for {
@@ -255,6 +263,9 @@ func (s *NezhaHandler) ReportSystemState(stream pb.NezhaService_ReportSystemStat
 					log.Printf("NEZHA>> Failed to write server metrics to TSDB: %v", err)
 				}
 			}
+			if singleton.OnlineHistoryShared != nil {
+				singleton.OnlineHistoryShared.RecordOnline(clientID, lastActive)
+			}
 			return nil
 		})
 		if !accepted {

service/singleton/alertsentinel.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/jinzhu/copier"
@@ -26,14 +27,125 @@ type NotificationHistory struct {
 var (
 	AlertsLock                    sync.RWMutex
 	Alerts                        []*model.AlertRule
-	alertsStore                   map[uint64]map[uint64][][]bool       // [alert_id][server_id] -> [timeTick][ruleId] 时间点对应的rule的检查结果
-	alertsPrevState               map[uint64]map[uint64]uint8          // [alert_id][server_id] -> 对应报警规则的上一次报警状态
-	AlertsCycleTransferStatsStore map[uint64]*model.CycleTransferStats // [alert_id] -> 对应报警规则的周期流量统计
+	alertsStore                   map[uint64]map[uint64][]model.TimedAlertPoint // distinct reports per alert/server
+	alertsPrevState               map[uint64]map[uint64]uint8                   // [alert_id][server_id] -> 对应报警规则的上一次报警状态
+	alertsLastMetricSeq           map[uint64]map[uint64]uint64                  // last distinct Agent metric report per alert/server
+	AlertsCycleTransferStatsStore map[uint64]*model.CycleTransferStats          // [alert_id] -> 对应报警规则的周期流量统计
+	alertDeliveryMu               sync.Mutex
+	alertDeliveries               map[uint64]map[uint64]*alertDeliveryEntry
+	alertEventSequence            atomic.Uint64
 )
 
+type alertDeliveryEntry struct {
+	cancel chan struct{}
+	done   chan struct{}
+	sendMu *sync.Mutex
+}
+
+type alertDeliveryPhase uint8
+
+const (
+	alertDeliveryIncident alertDeliveryPhase = iota
+	alertDeliveryRecovery
+)
+
+func cancelAlertDeliveries(alertID uint64) {
+	alertDeliveryMu.Lock()
+	defer alertDeliveryMu.Unlock()
+	for _, entry := range alertDeliveries[alertID] {
+		close(entry.cancel)
+	}
+	delete(alertDeliveries, alertID)
+}
+
+func cancelAlertDeliveriesForServers(serverIDs []uint64) {
+	alertDeliveryMu.Lock()
+	defer alertDeliveryMu.Unlock()
+	for alertID, byServer := range alertDeliveries {
+		for _, serverID := range serverIDs {
+			if entry := byServer[serverID]; entry != nil {
+				close(entry.cancel)
+				delete(byServer, serverID)
+			}
+		}
+		if len(byServer) == 0 {
+			delete(alertDeliveries, alertID)
+		}
+	}
+}
+
+func runAlertDelivery(cancel <-chan struct{}, send func() bool, always bool, retryDelay time.Duration) {
+	for {
+		select {
+		case <-cancel:
+			return
+		default:
+		}
+		delivered := send()
+		if delivered && !always {
+			return
+		}
+		delay := retryDelay
+		if delivered {
+			delay = firstNotificationDelay
+		}
+		timer := time.NewTimer(delay)
+		select {
+		case <-cancel:
+			timer.Stop()
+			return
+		case <-timer.C:
+		}
+	}
+}
+
+func startAlertDelivery(alert *model.AlertRule, server *model.Server, message, muteLabel string, phase alertDeliveryPhase) {
+	// A mute entry belongs to one transition, not all later incidents of the
+	// same phase. Concurrent in-flight sends may finish after a recovery.
+	muteLabel = fmt.Sprintf("%s:event-%d", muteLabel, alertEventSequence.Add(1))
+	alertDeliveryMu.Lock()
+	if alertDeliveries == nil {
+		alertDeliveries = make(map[uint64]map[uint64]*alertDeliveryEntry)
+	}
+	if alertDeliveries[alert.ID] == nil {
+		alertDeliveries[alert.ID] = make(map[uint64]*alertDeliveryEntry)
+	}
+	sendMu := new(sync.Mutex)
+	if old := alertDeliveries[alert.ID][server.ID]; old != nil {
+		close(old.cancel)
+		sendMu = old.sendMu
+	}
+	entry := &alertDeliveryEntry{cancel: make(chan struct{}), done: make(chan struct{}), sendMu: sendMu}
+	alertDeliveries[alert.ID][server.ID] = entry
+	alertDeliveryMu.Unlock()
+
+	groupID := alert.NotificationGroupID
+	always := phase == alertDeliveryIncident && alert.TriggerMode == model.ModeAlwaysTrigger
+	go func() {
+		defer close(entry.done)
+		runAlertDelivery(entry.cancel, func() bool {
+			// Serialize incident/recovery delivery for this alert/server pair.
+			// A recovery can cancel a pending incident, but never overtake an
+			// already-running network request.
+			entry.sendMu.Lock()
+			defer entry.sendMu.Unlock()
+			select {
+			case <-entry.cancel:
+				return true
+			default:
+			}
+			return NotificationShared.SendNotification(groupID, message, muteLabel, server)
+		}, always, 30*time.Second)
+	}()
+}
+
+func shouldRunAlertFailTasks(triggerMode uint8, newIncident bool) bool {
+	return newIncident || triggerMode == model.ModeAlwaysTrigger
+}
+
 // addCycleTransferStatsInfo 向AlertsCycleTransferStatsStore中添加周期流量报警统计信息
 func addCycleTransferStatsInfo(alert *model.AlertRule) {
-	if !alert.Enabled() {
+	if alert == nil || !alert.Enabled() || !alert.IsSafeToEvaluate() {
 		return
 	}
 	for _, rule := range alert.Rules {
@@ -59,16 +171,34 @@ func addCycleTransferStatsInfo(alert *model.AlertRule) {
 
 // AlertSentinelStart 报警器启动
 func AlertSentinelStart() {
-	alertsStore = make(map[uint64]map[uint64][][]bool)
+	alertDeliveryMu.Lock()
+	for _, byServer := range alertDeliveries {
+		for _, entry := range byServer {
+			close(entry.cancel)
+		}
+	}
+	alertDeliveries = make(map[uint64]map[uint64]*alertDeliveryEntry)
+	alertDeliveryMu.Unlock()
+	alertsStore = make(map[uint64]map[uint64][]model.TimedAlertPoint)
 	alertsPrevState = make(map[uint64]map[uint64]uint8)
+	alertsLastMetricSeq = make(map[uint64]map[uint64]uint64)
 	AlertsCycleTransferStatsStore = make(map[uint64]*model.CycleTransferStats)
 	AlertsLock.Lock()
 	if err := DB.Find(&Alerts).Error; err != nil {
 		panic(err)
 	}
 	for _, alert := range Alerts {
-		alertsStore[alert.ID] = make(map[uint64][][]bool)
+		if alert == nil {
+			log.Printf("NEZHA>> Skipping invalid nil alert rule loaded from database")
+			continue
+		}
+		alertsStore[alert.ID] = make(map[uint64][]model.TimedAlertPoint)
 		alertsPrevState[alert.ID] = make(map[uint64]uint8)
+		alertsLastMetricSeq[alert.ID] = make(map[uint64]uint64)
+		if !alert.IsSafeToEvaluate() {
+			log.Printf("NEZHA>> Skipping invalid alert rule %d loaded from database", alert.ID)
+			continue
+		}
 		addCycleTransferStatsInfo(alert)
 	}
 	AlertsLock.Unlock()
@@ -93,8 +223,10 @@ func AlertSentinelStart() {
 func OnRefreshOrAddAlert(alert *model.AlertRule) {
 	AlertsLock.Lock()
 	defer AlertsLock.Unlock()
+	cancelAlertDeliveries(alert.ID)
 	delete(alertsStore, alert.ID)
 	delete(alertsPrevState, alert.ID)
+	delete(alertsLastMetricSeq, alert.ID)
 	var isEdit bool
 	for i := range Alerts {
 		if Alerts[i].ID == alert.ID {
@@ -105,8 +237,9 @@ func OnRefreshOrAddAlert(alert *model.AlertRule) {
 	if !isEdit {
 		Alerts = append(Alerts, alert)
 	}
-	alertsStore[alert.ID] = make(map[uint64][][]bool)
+	alertsStore[alert.ID] = make(map[uint64][]model.TimedAlertPoint)
 	alertsPrevState[alert.ID] = make(map[uint64]uint8)
+	alertsLastMetricSeq[alert.ID] = make(map[uint64]uint64)
 	delete(AlertsCycleTransferStatsStore, alert.ID)
 	addCycleTransferStatsInfo(alert)
 }
@@ -115,8 +248,10 @@ func OnDeleteAlert(id []uint64) {
 	AlertsLock.Lock()
 	defer AlertsLock.Unlock()
 	for _, i := range id {
+		cancelAlertDeliveries(i)
 		delete(alertsStore, i)
 		delete(alertsPrevState, i)
+		delete(alertsLastMetricSeq, i)
 		currentAlerts := Alerts[:0]
 		for _, alert := range Alerts {
 			if alert.ID != i {
@@ -136,7 +271,7 @@ func checkStatus() {
 
 	for _, alert := range Alerts {
 		// 跳过未启用
-		if !alert.Enabled() {
+		if alert == nil || !alert.Enabled() || !alert.IsSafeToEvaluate() {
 			continue
 		}
 		for _, server := range m {
@@ -152,48 +287,92 @@ func checkStatus() {
 			if alert.UserID != server.GetUserID() && !role.IsAdmin() {
 				continue
 			}
-			alertsStore[alert.ID][server.ID] = append(alertsStore[alert.
-				ID][server.ID], alert.Snapshot(AlertsCycleTransferStatsStore[alert.ID], server, DB))
-			// 发送通知，分为触发报警和恢复通知
-			_, passed := alert.Check(alertsStore[alert.ID][server.ID])
-			// 保存当前服务器状态信息
-			curServer := model.Server{}
-			copier.Copy(&curServer, server)
-
-			// 本次未通过检查
-			if !passed {
-				// 始终触发模式或上次检查不为失败时触发报警（跳过单次触发+上次失败的情况）
-				if alert.TriggerMode == model.ModeAlwaysTrigger || alertsPrevState[alert.ID][server.ID] != _RuleCheckFail {
-					alertsPrevState[alert.ID][server.ID] = _RuleCheckFail
-					message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Incident"),
-						server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name)
-					go CronShared.SendTriggerTasks(alert.FailTriggerTasks, curServer.ID, alert.UserID)
-					go NotificationShared.SendNotification(alert.NotificationGroupID, message, NotificationMuteLabel.ServerIncident(server.ID, alert.ID), &curServer)
-					// 清除恢复通知的静音缓存
-					NotificationShared.UnMuteNotification(alert.NotificationGroupID, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID))
-				}
-			} else {
-				// 本次通过检查但上一次的状态为失败，则发送恢复通知
-				if alertsPrevState[alert.ID][server.ID] == _RuleCheckFail {
-					message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Resolved"),
-						server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name)
-					go CronShared.SendTriggerTasks(alert.RecoverTriggerTasks, curServer.ID, alert.UserID)
-					go NotificationShared.SendNotification(alert.NotificationGroupID, message, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), &curServer)
-					// 清除失败通知的静音缓存
-					NotificationShared.UnMuteNotification(alert.NotificationGroupID, NotificationMuteLabel.ServerIncident(server.ID, alert.ID))
-				}
-				alertsPrevState[alert.ID][server.ID] = _RuleCheckPass
-			}
-			// 清理旧数据：保留窗口由规则定义决定（各规则 Duration 的最大值），
-			// 而非 Check 的判定结果。window==0 表示没有任何有效规则需要回看历史
-			// （例如全部 Duration<=0），此时清空采样避免切片无限增长。
-			window := alert.RetentionWindow()
-			samples := alertsStore[alert.ID][server.ID]
-			if window <= 0 {
-				alertsStore[alert.ID][server.ID] = samples[:0]
-			} else if window < len(samples) {
-				alertsStore[alert.ID][server.ID] = samples[len(samples)-window:]
-			}
+			checkStatusForServer(alert, server)
 		}
 	}
 }
+
+// checkStatusForServer isolates each alert/server evaluation. Input validation
+// and model-level guards handle known malformed states; this recovery boundary
+// ensures an unexpected evaluator panic cannot terminate the dashboard process
+// or prevent unrelated servers from being checked on the same tick.
+func checkStatusForServer(alert *model.AlertRule, server *model.Server) {
+	defer func() {
+		if recovered := recover(); recovered != nil {
+			log.Printf("NEZHA>> Recovered panic evaluating alert rule %d for server %d: %v", alert.ID, server.ID, recovered)
+		}
+	}()
+
+	// Offline-only rules deliberately observe elapsed time on every tick. Metric
+	// rules, including compound rules, must never count the same Agent report
+	// repeatedly or treat a disconnected Agent's retained State as fresh data.
+	metricRule := false
+	runtime := server.RuntimeSnapshot()
+	for _, rule := range alert.Rules {
+		if !rule.IsOfflineRule() {
+			metricRule = true
+			break
+		}
+	}
+	if metricRule {
+		if runtime.LastActive.IsZero() || runtime.State == nil || time.Since(runtime.LastActive) > model.AlertSampleMaxAge || time.Since(runtime.LastActive) < 0 {
+			alertsStore[alert.ID][server.ID] = nil
+			return
+		}
+		if runtime.ReportSequence == 0 || runtime.ReportSequence == alertsLastMetricSeq[alert.ID][server.ID] {
+			return
+		}
+		alertsLastMetricSeq[alert.ID][server.ID] = runtime.ReportSequence
+	}
+	point, known := alert.SnapshotStatusWithRuntime(AlertsCycleTransferStatsStore[alert.ID], server, runtime, DB)
+	if !known {
+		alertsStore[alert.ID][server.ID] = nil
+		return
+	}
+	sampledAt := time.Now()
+	if metricRule {
+		sampledAt = runtime.LastActive
+	}
+	alertsStore[alert.ID][server.ID] = append(alertsStore[alert.ID][server.ID], model.TimedAlertPoint{At: sampledAt, Values: point})
+	// Bound memory even while coverage is insufficient to decide a state.
+	cutoff := time.Now().Add(-alert.TimedRetention())
+	samples := alertsStore[alert.ID][server.ID]
+	first := 0
+	for first < len(samples) && samples[first].At.Before(cutoff) {
+		first++
+	}
+	if first > 0 {
+		alertsStore[alert.ID][server.ID] = append([]model.TimedAlertPoint(nil), samples[first:]...)
+	}
+	// 发送通知，分为触发报警和恢复通知
+	known, passed := alert.CheckTimed(alertsStore[alert.ID][server.ID], time.Now())
+	if !known {
+		return
+	}
+	// 保存当前服务器状态信息
+	curServer := model.Server{}
+	copier.Copy(&curServer, server)
+
+	// 本次未通过检查
+	if !passed {
+		newIncident := alertsPrevState[alert.ID][server.ID] != _RuleCheckFail
+		if newIncident {
+			alertsPrevState[alert.ID][server.ID] = _RuleCheckFail
+			message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Incident"),
+				server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name)
+			startAlertDelivery(alert, &curServer, message, NotificationMuteLabel.ServerIncident(server.ID, alert.ID), alertDeliveryIncident)
+		}
+		if shouldRunAlertFailTasks(alert.TriggerMode, newIncident) {
+			go CronShared.SendTriggerTasks(alert.FailTriggerTasks, curServer.ID, alert.UserID)
+		}
+	} else {
+		// 本次通过检查但上一次的状态为失败，则发送恢复通知
+		if alertsPrevState[alert.ID][server.ID] == _RuleCheckFail {
+			message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Resolved"),
+				server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name)
+			go CronShared.SendTriggerTasks(alert.RecoverTriggerTasks, curServer.ID, alert.UserID)
+			startAlertDelivery(alert, &curServer, message, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), alertDeliveryRecovery)
+		}
+		alertsPrevState[alert.ID][server.ID] = _RuleCheckPass
+	}
+}

service/singleton/alertsentinel_freshness_test.go

@@ -0,0 +1,281 @@
+package singleton
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/patrickmn/go-cache"
+	"google.golang.org/grpc/metadata"
+
+	"github.com/nezhahq/nezha/model"
+	pb "github.com/nezhahq/nezha/proto"
+)
+
+type alertTestStateStream struct{}
+
+func (alertTestStateStream) Send(*pb.Receipt) error       { return nil }
+func (alertTestStateStream) Recv() (*pb.State, error)     { return nil, context.Canceled }
+func (alertTestStateStream) SetHeader(metadata.MD) error  { return nil }
+func (alertTestStateStream) SendHeader(metadata.MD) error { return nil }
+func (alertTestStateStream) SetTrailer(metadata.MD)       {}
+func (alertTestStateStream) Context() context.Context     { return context.Background() }
+func (alertTestStateStream) SendMsg(any) error            { return nil }
+func (alertTestStateStream) RecvMsg(any) error            { return nil }
+
+func TestAlertSentinelCountsOnlyFreshDistinctMetricReports(t *testing.T) {
+	oldStore, oldPrev, oldLast, oldCycle := alertsStore, alertsPrevState, alertsLastMetricSeq, AlertsCycleTransferStatsStore
+	alertsStore = map[uint64]map[uint64][]model.TimedAlertPoint{7: {1: nil}}
+	alertsPrevState = map[uint64]map[uint64]uint8{7: {}}
+	alertsLastMetricSeq = map[uint64]map[uint64]uint64{7: {}}
+	AlertsCycleTransferStatsStore = map[uint64]*model.CycleTransferStats{}
+	t.Cleanup(func() {
+		alertsStore, alertsPrevState, alertsLastMetricSeq, AlertsCycleTransferStatsStore = oldStore, oldPrev, oldLast, oldCycle
+	})
+
+	alert := &model.AlertRule{
+		Common: model.Common{ID: 7},
+		Rules:  []*model.Rule{{Type: "cpu", Max: 80, Duration: 100, Cover: model.RuleCoverAll}},
+	}
+	server := &model.Server{Common: model.Common{ID: 1}}
+	model.InitServer(server)
+	lease := server.AttachStateStream(alertTestStateStream{})
+	fixedReportTime := time.Now()
+	if !lease.UpdateState(&model.HostState{CPU: 99}, fixedReportTime) {
+		t.Fatal("state update rejected")
+	}
+	checkStatusForServer(alert, server)
+	if got := len(alertsStore[7][1]); got != 1 {
+		t.Fatalf("first report produced %d samples, want 1", got)
+	}
+	for i := 0; i < 100; i++ {
+		checkStatusForServer(alert, server)
+	}
+	if got := len(alertsStore[7][1]); got != 1 {
+		t.Fatalf("same report was counted %d times, want 1", got)
+	}
+	if !lease.Clear() {
+		t.Fatal("stream clear rejected")
+	}
+	checkStatusForServer(alert, server)
+	if got := len(alertsStore[7][1]); got != 0 {
+		t.Fatalf("disconnect retained %d high samples, want 0", got)
+	}
+	alertsPrevState[7][1] = _RuleCheckFail // incident was active before disconnect
+	newLease := server.AttachStateStream(alertTestStateStream{})
+	// Distinct reports may share a clock timestamp on coarse-resolution hosts.
+	if !newLease.UpdateState(&model.HostState{CPU: 99}, fixedReportTime) {
+		t.Fatal("reconnect state update rejected")
+	}
+	checkStatusForServer(alert, server)
+	if got := len(alertsStore[7][1]); got != 1 || alertsStore[7][1][0].Values[0] {
+		t.Fatalf("reconnect window = %v, want one high sample", alertsStore[7][1])
+	}
+	if alertsPrevState[7][1] != _RuleCheckFail {
+		t.Fatal("incomplete warm-up falsely resolved active incident")
+	}
+	replacement := &model.Server{Common: model.Common{ID: 1}}
+	model.InitServer(replacement)
+	replacementLease := replacement.AttachStateStream(alertTestStateStream{})
+	if !replacementLease.UpdateState(&model.HostState{CPU: 99}, fixedReportTime) {
+		t.Fatal("replacement server state update rejected")
+	}
+	checkStatusForServer(alert, replacement)
+	if got := len(alertsStore[7][1]); got != 2 {
+		t.Fatalf("replacement holder reused a report identity; samples=%d", got)
+	}
+}
+
+func TestAlertNewIncidentNotMutedByEarlierInflightIncident(t *testing.T) {
+	oldCache, oldNotifications := Cache, NotificationShared
+	Cache = cache.New(time.Minute, time.Minute)
+	n := &model.Notification{Common: model.Common{ID: 601}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 603}, Name: "alert-test"}, []uint64{n.ID})
+	NotificationShared = nc
+	t.Cleanup(func() {
+		cancelAlertDeliveries(602)
+		NotificationShared, Cache = oldNotifications, oldCache
+	})
+	started := make(chan struct{})
+	release := make(chan struct{})
+	delivered := make(chan string, 3)
+	nc.sendForTest = func(_ *model.Notification, desc string, _ *model.Server) error {
+		if desc == "first incident" {
+			close(started)
+			<-release
+		}
+		delivered <- desc
+		return nil
+	}
+	alert := &model.AlertRule{
+		Common:              model.Common{ID: 602},
+		NotificationGroupID: 603,
+		TriggerMode:         model.ModeOnetimeTrigger,
+	}
+	server := &model.Server{Common: model.Common{ID: 604}}
+	label := NotificationMuteLabel.ServerIncident(server.ID, alert.ID)
+	startAlertDelivery(alert, server, "first incident", label, alertDeliveryIncident)
+	<-started
+	startAlertDelivery(alert, server, "recovered", NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), alertDeliveryRecovery)
+	startAlertDelivery(alert, server, "second incident", label, alertDeliveryIncident)
+	close(release)
+	timer := time.NewTimer(time.Second)
+	defer timer.Stop()
+	seen := map[string]bool{}
+	for len(seen) < 2 {
+		select {
+		case message := <-delivered:
+			seen[message] = true
+		case <-timer.C:
+			t.Fatalf("new incident was suppressed after old send completed: %v", seen)
+		}
+	}
+	if !seen["first incident"] || !seen["second incident"] || seen["recovered"] {
+		t.Fatalf("unexpected transition deliveries: %v", seen)
+	}
+	alertDeliveryMu.Lock()
+	done := alertDeliveries[alert.ID][server.ID].done
+	alertDeliveryMu.Unlock()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("new incident delivery did not finish")
+	}
+}
+
+func TestAlertRecoveryDeliveryStopsAfterSuccessInAlwaysMode(t *testing.T) {
+	oldCache, oldNotifications := Cache, NotificationShared
+	Cache = cache.New(time.Minute, time.Minute)
+	n := &model.Notification{Common: model.Common{ID: 611}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 613}, Name: "recovery-test"}, []uint64{n.ID})
+	NotificationShared = nc
+	t.Cleanup(func() {
+		cancelAlertDeliveries(612)
+		NotificationShared, Cache = oldNotifications, oldCache
+	})
+
+	delivered := make(chan struct{}, 2)
+	nc.sendForTest = func(*model.Notification, string, *model.Server) error {
+		delivered <- struct{}{}
+		return nil
+	}
+	alert := &model.AlertRule{
+		Common:              model.Common{ID: 612},
+		NotificationGroupID: 613,
+		TriggerMode:         model.ModeAlwaysTrigger,
+	}
+	server := &model.Server{Common: model.Common{ID: 614}}
+	startAlertDelivery(alert, server, "recovered", "recovery-event", alertDeliveryRecovery)
+
+	select {
+	case <-delivered:
+	case <-time.After(time.Second):
+		t.Fatal("recovery notification was not delivered")
+	}
+	alertDeliveryMu.Lock()
+	done := alertDeliveries[alert.ID][server.ID].done
+	alertDeliveryMu.Unlock()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("successful recovery delivery kept repeating in always mode")
+	}
+	select {
+	case <-delivered:
+		t.Fatal("recovery notification was delivered more than once")
+	default:
+	}
+}
+
+func TestServerDeleteCancelsAlertDelivery(t *testing.T) {
+	alertDeliveryMu.Lock()
+	oldDeliveries := alertDeliveries
+	cancel := make(chan struct{})
+	alertDeliveries = map[uint64]map[uint64]*alertDeliveryEntry{
+		621: {622: {cancel: cancel, done: make(chan struct{})}},
+	}
+	alertDeliveryMu.Unlock()
+	t.Cleanup(func() {
+		alertDeliveryMu.Lock()
+		alertDeliveries = oldDeliveries
+		alertDeliveryMu.Unlock()
+	})
+
+	servers := &ServerClass{
+		class: class[uint64, *model.Server]{
+			list: map[uint64]*model.Server{
+				622: {Common: model.Common{ID: 622}, UUID: "delete-alert-worker"},
+			},
+		},
+		uuidToID: map[string]uint64{"delete-alert-worker": 622},
+	}
+	servers.Delete([]uint64{622})
+
+	select {
+	case <-cancel:
+	default:
+		t.Fatal("server deletion did not cancel its alert delivery")
+	}
+	alertDeliveryMu.Lock()
+	_, retained := alertDeliveries[621]
+	alertDeliveryMu.Unlock()
+	if retained {
+		t.Fatal("server deletion retained an empty alert delivery map")
+	}
+}
+
+func TestAlwaysAlertContinuesFailureTriggerTasks(t *testing.T) {
+	if !shouldRunAlertFailTasks(model.ModeAlwaysTrigger, false) {
+		t.Fatal("always mode stopped failure trigger tasks after the initial incident")
+	}
+	if shouldRunAlertFailTasks(model.ModeOnetimeTrigger, false) {
+		t.Fatal("one-time mode repeated failure trigger tasks")
+	}
+	if !shouldRunAlertFailTasks(model.ModeOnetimeTrigger, true) {
+		t.Fatal("one-time mode skipped failure trigger tasks for a new incident")
+	}
+}
+
+func TestAlertDeliveryRetriesFailureAndStopsAfterSuccess(t *testing.T) {
+	cancel := make(chan struct{})
+	done := make(chan struct{})
+	attempts := 0
+	go func() {
+		runAlertDelivery(cancel, func() bool {
+			attempts++
+			return attempts == 2
+		}, false, time.Millisecond)
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		close(cancel)
+		t.Fatal("delivery did not retry and finish")
+	}
+	if attempts != 2 {
+		t.Fatalf("got %d attempts, want 2", attempts)
+	}
+}
+
+func TestAlertDeliveryCancelsAfterRecovery(t *testing.T) {
+	cancel := make(chan struct{})
+	done := make(chan struct{})
+	started := make(chan struct{})
+	go func() {
+		runAlertDelivery(cancel, func() bool {
+			close(started)
+			return true
+		}, true, time.Millisecond)
+		close(done)
+	}()
+	<-started
+	close(cancel)
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("recovered incident delivery was not cancelled")
+	}
+}

service/singleton/alertsentinel_test.go

@@ -117,3 +117,53 @@ func TestCheckStatus_SampleMemoryBounded(t *testing.T) {
 		t.Fatalf("sample capacity grew unbounded: peak cap %d exceeds 4x window %d", peakCap, duration*4)
 	}
 }
+
+func TestCheckStatusForServerRecoversUnexpectedEvaluatorPanic(t *testing.T) {
+	originalStore := alertsStore
+	originalPrevState := alertsPrevState
+	originalCycleStore := AlertsCycleTransferStatsStore
+	alertsStore = map[uint64]map[uint64][]model.TimedAlertPoint{}
+	alertsPrevState = map[uint64]map[uint64]uint8{}
+	AlertsCycleTransferStatsStore = map[uint64]*model.CycleTransferStats{}
+	t.Cleanup(func() {
+		alertsStore = originalStore
+		alertsPrevState = originalPrevState
+		AlertsCycleTransferStatsStore = originalCycleStore
+	})
+
+	enabled := true
+	alert := &model.AlertRule{
+		Common: model.Common{ID: 77, UserID: 2},
+		Enable: &enabled,
+		Rules:  []*model.Rule{{Type: "cpu", Duration: 3, Cover: model.RuleCoverAll}},
+	}
+	server := &model.Server{Common: model.Common{ID: 1, UserID: 2}}
+	model.InitServer(server)
+
+	// The deliberately missing alertsStore[alert.ID] map panics at assignment
+	// after evaluation. The per-alert/server recovery boundary must contain it.
+	checkStatusForServer(alert, server)
+}
+
+func TestAddCycleTransferStatsSkipsPoisonedPersistedRule(t *testing.T) {
+	original := AlertsCycleTransferStatsStore
+	AlertsCycleTransferStatsStore = map[uint64]*model.CycleTransferStats{}
+	t.Cleanup(func() { AlertsCycleTransferStatsStore = original })
+
+	enabled := true
+	alert := &model.AlertRule{
+		Common: model.Common{ID: 88},
+		Enable: &enabled,
+		Rules: []*model.Rule{{
+			Type:          "transfer_in_cycle",
+			CycleInterval: 1,
+			Cover:         model.RuleCoverAll,
+			// CycleStart intentionally nil, as it may be in a poisoned legacy DB row.
+		}},
+	}
+
+	addCycleTransferStatsInfo(alert)
+	if _, exists := AlertsCycleTransferStatsStore[alert.ID]; exists {
+		t.Fatal("invalid persisted cycle rule must not be scheduled")
+	}
+}

service/singleton/config.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/nezhahq/nezha/model"
 	"github.com/nezhahq/nezha/pkg/utils"
@@ -14,6 +15,8 @@ var Conf *ConfigClass
 type ConfigClass struct {
 	*model.Config
 
+	onlineHistoryRetentionMu sync.RWMutex
+
 	IgnoredIPNotificationServerIDs map[uint64]bool `json:"ignored_ip_notification_server_ids,omitempty"`
 	Oauth2Providers                []string        `json:"oauth2_providers,omitempty"`
 }
@@ -59,3 +62,39 @@ func (c *ConfigClass) updateIgnoredIPNotificationID() {
 		}
 	}
 }
+
+// OnlineHistoryRetentionDays returns the retention policy visible to queries and pruning.
+func (c *ConfigClass) OnlineHistoryRetentionDays() int {
+	c.onlineHistoryRetentionMu.RLock()
+	defer c.onlineHistoryRetentionMu.RUnlock()
+	return c.OnlineStatsDays
+}
+
+// CommitOnlineHistoryRetention applies a retention change with the settings save.
+func (c *ConfigClass) CommitOnlineHistoryRetention(days *int, persist func() error) error {
+	// Readers must not capture a shorter cutoff while Save can still fail.
+	// Serialize assignment, persistence and rollback with the policy snapshot.
+	c.onlineHistoryRetentionMu.Lock()
+	defer c.onlineHistoryRetentionMu.Unlock()
+	previous := c.OnlineStatsDays
+	if days != nil {
+		c.OnlineStatsDays = *days
+	}
+	if err := persist(); err != nil {
+		c.OnlineStatsDays = previous
+		return err
+	}
+	return nil
+}
+
+// SettingsSnapshot copies response fields without copying the retention mutex.
+func (c *ConfigClass) SettingsSnapshot() model.Setting {
+	c.onlineHistoryRetentionMu.RLock()
+	defer c.onlineHistoryRetentionMu.RUnlock()
+	return model.Setting{
+		ConfigForGuests:                c.ConfigForGuests,
+		ConfigDashboard:                c.ConfigDashboard,
+		IgnoredIPNotificationServerIDs: c.IgnoredIPNotificationServerIDs,
+		Oauth2Providers:                c.Oauth2Providers,
+	}
+}

service/singleton/frontend-templates.yaml

@@ -2,24 +2,20 @@
   name: "OfficialAdmin"
   repository: "https://github.com/nezhahq/admin-frontend"
   author: "nezhahq"
-  version: "v2.3.4"
+  version: "v2.4.0"
   is_admin: true
   is_official: true
 - path: "user-dist"
   name: "Official"
   repository: "https://github.com/hamster1963/nezha-dash-v2"
   author: "hamster1963"
-  version: "v2.4.2"
+  version: "v2.5.1"
   is_official: true
 - path: "nezha-pixel-dist"
   name: "Nezha-Pixel"
   repository: "https://github.com/karllao/nezha-pixel"
   author: "karllao"
   version: "v1.6.0"
-# Third-party user themes consume the opaque Server.PublicNote field. Theme
-# maintainers must validate URL schemes (including after decoding) before using
-# values such as customData.orderLink in href or window.open; the Dashboard
-# backend and admin frontend do not execute those fields.
 - path: "nazhua-dist"
   name: "Nazhua"
   repository: "https://github.com/hi2shark/nazhua"
@@ -29,7 +25,7 @@
   name: "Aobobo"
   repository: "https://github.com/hi2shark/aobobo"
   author: "hi2shark"
-  version: "v1.5.1"
+  version: "v1.5.4"
 - path: "nezha-ascii-dist"
   name: "Nezha-ASCII"
   repository: "https://github.com/hamster1963/nezha-ascii"

service/singleton/notification.go

@@ -58,8 +58,228 @@ type NotificationClass struct {
 	groupToIDList map[uint64]map[uint64]*model.Notification
 	idToGroupList map[uint64]map[uint64]struct{}
 
-	groupList map[uint64]string
-	groupMu   sync.RWMutex
+	groupList       map[uint64]string
+	groupMu         sync.RWMutex
+	deliveryMu      sync.Mutex
+	deliveryKeys    map[string]*notificationKeyLock
+	asyncInFlight   map[string]struct{}
+	asyncRetryAfter map[string]time.Time
+	asyncRetryDelay map[string]time.Duration
+	muteGeneration  map[string]uint64
+	serviceDelivery map[uint64]*orderedServiceDelivery
+	sendForTest     func(*model.Notification, string, *model.Server) error
+}
+
+type serviceDeliveryRequest struct {
+	groupID uint64
+	desc    string
+	label   string
+}
+
+type orderedServiceDelivery struct {
+	current serviceDeliveryRequest
+	pending *serviceDeliveryRequest
+	update  chan struct{}
+	cancel  chan struct{}
+	done    chan struct{}
+}
+
+// SendServiceState preserves incident→recovery order without allowing a
+// slow webhook to accumulate one goroutine per probe result. Only the latest
+// pending phase is retained; a failed current phase retries with backoff.
+func (c *NotificationClass) SendServiceState(serviceID, groupID uint64, desc, label string) {
+	req := serviceDeliveryRequest{groupID: groupID, desc: desc, label: label}
+	c.deliveryMu.Lock()
+	if c.serviceDelivery == nil {
+		c.serviceDelivery = make(map[uint64]*orderedServiceDelivery)
+	}
+	state := c.serviceDelivery[serviceID]
+	if state == nil {
+		state = &orderedServiceDelivery{current: req, update: make(chan struct{}, 1), cancel: make(chan struct{}), done: make(chan struct{})}
+		c.serviceDelivery[serviceID] = state
+		c.deliveryMu.Unlock()
+		go c.runServiceDelivery(serviceID, state)
+		return
+	}
+	if state.pending == nil && state.current.label == label {
+		c.deliveryMu.Unlock()
+		return
+	}
+	if state.pending == nil || state.pending.label != label {
+		state.pending = &req
+		select {
+		case state.update <- struct{}{}:
+		default:
+		}
+	}
+	c.deliveryMu.Unlock()
+}
+
+func (c *NotificationClass) runServiceDelivery(serviceID uint64, state *orderedServiceDelivery) {
+	defer close(state.done)
+	delay := 30 * time.Second
+	for {
+		select {
+		case <-state.cancel:
+			return
+		default:
+		}
+		delivered := c.SendNotification(state.current.groupID, state.current.desc, state.current.label)
+		c.deliveryMu.Lock()
+		// Do not advance to a later phase until every recipient has received the
+		// current one. Otherwise a failed incident can be discarded as soon as a
+		// recovery is queued, leaving that recipient with a recovery but no incident.
+		if delivered && state.pending != nil {
+			next := state.pending
+			state.current = *next
+			state.pending = nil
+			delay = 30 * time.Second
+			select {
+			case <-state.update:
+			default:
+			}
+			c.deliveryMu.Unlock()
+			continue
+		}
+		if delivered {
+			if c.serviceDelivery[serviceID] == state {
+				delete(c.serviceDelivery, serviceID)
+			}
+			c.deliveryMu.Unlock()
+			return
+		}
+		c.deliveryMu.Unlock()
+		timer := time.NewTimer(delay)
+		select {
+		case <-state.cancel:
+			timer.Stop()
+			return
+		case <-state.update:
+			timer.Stop()
+		case <-timer.C:
+			delay = min(delay*2, 15*time.Minute)
+		}
+	}
+}
+
+func (c *NotificationClass) CancelServiceState(serviceID uint64) {
+	if c == nil {
+		return
+	}
+	c.deliveryMu.Lock()
+	if state := c.serviceDelivery[serviceID]; state != nil {
+		close(state.cancel)
+		delete(c.serviceDelivery, serviceID)
+	}
+	c.deliveryMu.Unlock()
+}
+
+// SendNotificationAsync coalesces concurrent service-probe reports for the
+// same event before allocating a webhook goroutine. Alert transitions use the
+// synchronous SendNotification path so they retain their retry semantics.
+func (c *NotificationClass) SendNotificationAsync(groupID uint64, desc, muteLabel string, ext ...*model.Server) {
+	c.sendNotificationAsync(groupID, desc, muteLabel, nil, ext...)
+}
+
+func (c *NotificationClass) sendNotificationAsync(groupID uint64, desc, muteLabel string, onSuccess func(), ext ...*model.Server) {
+	key := fmt.Sprintf("%d:%s", groupID, muteLabel)
+	c.deliveryMu.Lock()
+	if c.asyncInFlight == nil {
+		c.asyncInFlight = make(map[string]struct{})
+	}
+	if _, busy := c.asyncInFlight[key]; busy {
+		c.deliveryMu.Unlock()
+		return
+	}
+	if time.Now().Before(c.asyncRetryAfter[key]) {
+		c.deliveryMu.Unlock()
+		return
+	}
+	c.asyncInFlight[key] = struct{}{}
+	c.deliveryMu.Unlock()
+	go func() {
+		delivered := c.SendNotification(groupID, desc, muteLabel, ext...)
+		if delivered && onSuccess != nil {
+			onSuccess()
+		}
+		c.deliveryMu.Lock()
+		if delivered {
+			delete(c.asyncRetryAfter, key)
+			delete(c.asyncRetryDelay, key)
+		} else {
+			if c.asyncRetryAfter == nil {
+				c.asyncRetryAfter = make(map[string]time.Time)
+				c.asyncRetryDelay = make(map[string]time.Duration)
+			}
+			delay := c.asyncRetryDelay[key]
+			if delay == 0 {
+				delay = 30 * time.Second
+			} else {
+				delay = min(delay*2, 15*time.Minute)
+			}
+			c.asyncRetryDelay[key] = delay
+			c.asyncRetryAfter[key] = time.Now().Add(delay)
+			c.pruneAsyncBackoffLocked()
+		}
+		delete(c.asyncInFlight, key)
+		c.deliveryMu.Unlock()
+	}()
+}
+
+const maxAsyncBackoffKeys = 4096
+
+// Bound memory when a failing webhook sees many short-lived event labels.
+// Evicted old event keys are no longer active after their phase changes.
+func (c *NotificationClass) pruneAsyncBackoffLocked() {
+	if len(c.asyncRetryAfter) <= maxAsyncBackoffKeys {
+		return
+	}
+	now := time.Now()
+	for key, until := range c.asyncRetryAfter {
+		if !now.Before(until) {
+			delete(c.asyncRetryAfter, key)
+			delete(c.asyncRetryDelay, key)
+		}
+	}
+	for key := range c.asyncRetryAfter {
+		if len(c.asyncRetryAfter) <= maxAsyncBackoffKeys {
+			break
+		}
+		delete(c.asyncRetryAfter, key)
+		delete(c.asyncRetryDelay, key)
+	}
+}
+
+type notificationKeyLock struct {
+	mu   sync.Mutex
+	refs int
+}
+
+func (c *NotificationClass) lockDeliveryKey(key string) func() {
+	if key == "" {
+		return func() {}
+	}
+	c.deliveryMu.Lock()
+	if c.deliveryKeys == nil {
+		c.deliveryKeys = make(map[string]*notificationKeyLock)
+	}
+	entry := c.deliveryKeys[key]
+	if entry == nil {
+		entry = new(notificationKeyLock)
+		c.deliveryKeys[key] = entry
+	}
+	entry.refs++
+	c.deliveryMu.Unlock()
+	entry.mu.Lock()
+	return func() {
+		entry.mu.Unlock()
+		c.deliveryMu.Lock()
+		entry.refs--
+		if entry.refs == 0 {
+			delete(c.deliveryKeys, key)
+		}
+		c.deliveryMu.Unlock()
+	}
 }
 
 func NewNotificationClass() *NotificationClass {
@@ -117,20 +337,29 @@ func NewNotificationClass() *NotificationClass {
 }
 
 func (c *NotificationClass) Update(n *model.Notification) {
-	c.listMu.Lock()
+	func() {
+		c.listMu.Lock()
+		defer c.listMu.Unlock()
 
-	_, ok := c.list[n.ID]
-	c.list[n.ID] = n
-
-	if ok {
-		if gids, ok := c.idToGroupList[n.ID]; ok {
-			for gid := range gids {
-				c.groupToIDList[gid][n.ID] = n
-			}
+		_, ok := c.list[n.ID]
+		c.list[n.ID] = n
+		if !ok {
+			return
 		}
-	}
 
-	c.listMu.Unlock()
+		gids := c.idToGroupList[n.ID]
+		for gid := range gids {
+			group, exists := c.groupToIDList[gid]
+			if !exists {
+				delete(gids, gid)
+				continue
+			}
+			group[n.ID] = n
+		}
+		if len(gids) == 0 {
+			delete(c.idToGroupList, n.ID)
+		}
+	}()
 	c.sortList()
 }
 
@@ -138,71 +367,69 @@ func (c *NotificationClass) UpdateGroup(ng *model.NotificationGroup, ngn []uint6
 	c.groupMu.Lock()
 	defer c.groupMu.Unlock()
 
-	_, ok := c.groupList[ng.ID]
 	c.groupList[ng.ID] = ng.Name
 
 	c.listMu.Lock()
 	defer c.listMu.Unlock()
-	if !ok {
-		c.groupToIDList[ng.ID] = make(map[uint64]*model.Notification, len(ngn))
-		for _, n := range ngn {
-			if c.idToGroupList[n] == nil {
-				c.idToGroupList[n] = make(map[uint64]struct{})
-			}
-			c.idToGroupList[n][ng.ID] = struct{}{}
-			c.groupToIDList[ng.ID][n] = c.list[n]
-		}
-	} else {
-		oldList := make(map[uint64]struct{})
-		for nid := range c.groupToIDList[ng.ID] {
-			oldList[nid] = struct{}{}
-		}
 
-		c.groupToIDList[ng.ID] = make(map[uint64]*model.Notification)
-		for _, nid := range ngn {
-			c.groupToIDList[ng.ID][nid] = c.list[nid]
-			if c.idToGroupList[nid] == nil {
-				c.idToGroupList[nid] = make(map[uint64]struct{})
-			}
-			c.idToGroupList[nid][ng.ID] = struct{}{}
+	oldList := c.groupToIDList[ng.ID]
+	newList := make(map[uint64]*model.Notification, len(ngn))
+	for _, nid := range ngn {
+		n, ok := c.list[nid]
+		if !ok {
+			continue
 		}
+		newList[nid] = n
+		if c.idToGroupList[nid] == nil {
+			c.idToGroupList[nid] = make(map[uint64]struct{})
+		}
+		c.idToGroupList[nid][ng.ID] = struct{}{}
+	}
 
-		for oldID := range oldList {
-			if _, ok := c.groupToIDList[ng.ID][oldID]; !ok {
-				delete(c.groupToIDList[oldID], ng.ID)
-				if len(c.idToGroupList[oldID]) == 0 {
-					delete(c.idToGroupList, oldID)
-				}
-			}
+	for oldID := range oldList {
+		if _, ok := newList[oldID]; ok {
+			continue
+		}
+		delete(c.idToGroupList[oldID], ng.ID)
+		if len(c.idToGroupList[oldID]) == 0 {
+			delete(c.idToGroupList, oldID)
 		}
 	}
+	c.groupToIDList[ng.ID] = newList
 }
 
 func (c *NotificationClass) Delete(idList []uint64) {
-	c.listMu.Lock()
+	func() {
+		c.listMu.Lock()
+		defer c.listMu.Unlock()
 
-	for _, id := range idList {
-		delete(c.list, id)
-		// 如果绑定了通知组才删除
-		if gids, ok := c.idToGroupList[id]; ok {
-			for gid := range gids {
-				delete(c.groupToIDList[gid], id)
+		for _, id := range idList {
+			delete(c.list, id)
+			// 如果绑定了通知组才删除
+			if gids, ok := c.idToGroupList[id]; ok {
+				for gid := range gids {
+					delete(c.groupToIDList[gid], id)
+				}
 				delete(c.idToGroupList, id)
 			}
 		}
-	}
-
-	c.listMu.Unlock()
+	}()
 	c.sortList()
 }
 
 func (c *NotificationClass) DeleteGroup(gids []uint64) {
-	c.listMu.Lock()
-	defer c.listMu.Unlock()
 	c.groupMu.Lock()
 	defer c.groupMu.Unlock()
+	c.listMu.Lock()
+	defer c.listMu.Unlock()
 
 	for _, gid := range gids {
+		for nid := range c.groupToIDList[gid] {
+			delete(c.idToGroupList[nid], gid)
+			if len(c.idToGroupList[nid]) == 0 {
+				delete(c.idToGroupList, nid)
+			}
+		}
 		delete(c.groupList, gid)
 		delete(c.groupToIDList, gid)
 	}
@@ -231,66 +458,115 @@ func (c *NotificationClass) sortList() {
 
 func (c *NotificationClass) UnMuteNotification(notificationGroupID uint64, muteLabel string) {
 	fullMuteLabel := NotificationMuteLabel.AppendNotificationGroupName(muteLabel, c.GetGroupName(notificationGroupID))
-	Cache.Delete(fullMuteLabel)
+	c.listMu.RLock()
+	for id := range c.groupToIDList[notificationGroupID] {
+		key := notificationMethodMuteLabel(fullMuteLabel, id)
+		c.deliveryMu.Lock()
+		if c.muteGeneration == nil {
+			c.muteGeneration = make(map[string]uint64)
+		}
+		c.muteGeneration[key]++
+		Cache.Delete(key)
+		c.deliveryMu.Unlock()
+	}
+	c.listMu.RUnlock()
+}
+
+func notificationMethodMuteLabel(label string, id uint64) string {
+	return fmt.Sprintf("%s:method-%d", label, id)
 }
 
 // SendNotification 向指定的通知方式组的所有通知方式发送通知
-func (c *NotificationClass) SendNotification(notificationGroupID uint64, desc string, muteLabel string, ext ...*model.Server) {
+func (c *NotificationClass) SendNotification(notificationGroupID uint64, desc string, muteLabel string, ext ...*model.Server) bool {
+	fullMuteLabel := ""
 	if muteLabel != "" {
-		// 将通知方式组名称加入静音标志
-		muteLabel := NotificationMuteLabel.AppendNotificationGroupName(muteLabel, c.GetGroupName(notificationGroupID))
-		// 通知防骚扰策略
-		var flag bool
-		if cacheN, has := Cache.Get(muteLabel); has {
-			nHistory := cacheN.(NotificationHistory)
-			// 每次提醒都增加一倍等待时间，最后每天最多提醒一次
-			if time.Now().After(nHistory.Until) {
-				flag = true
-				nHistory.Duration *= 2
-				if nHistory.Duration > time.Hour*24 {
-					nHistory.Duration = time.Hour * 24
-				}
-				nHistory.Until = time.Now().Add(nHistory.Duration)
-				// 缓存有效期加 10 分钟
-				Cache.Set(muteLabel, nHistory, nHistory.Duration+time.Minute*10)
-			}
-		} else {
-			// 新提醒直接通知
-			flag = true
-			Cache.Set(muteLabel, NotificationHistory{
-				Duration: firstNotificationDelay,
-				Until:    time.Now().Add(firstNotificationDelay),
-			}, firstNotificationDelay+time.Minute*10)
-		}
-
-		if !flag {
-			if Conf.Debug {
-				log.Println("NEZHA>> Muted repeated notification", desc, muteLabel)
-			}
-			return
-		}
+		fullMuteLabel = NotificationMuteLabel.AppendNotificationGroupName(muteLabel, c.GetGroupName(notificationGroupID))
 	}
-	// 向该通知方式组的所有通知方式发出通知
+	// Copy the group under the lock. Webhook delivery can take minutes and must
+	// not block notification or notification-group updates while it is in flight.
 	c.listMu.RLock()
-	defer c.listMu.RUnlock()
+	notifications := make([]*model.Notification, 0, len(c.groupToIDList[notificationGroupID]))
 	for _, n := range c.groupToIDList[notificationGroupID] {
-		log.Printf("NEZHA>> Try to notify %s", n.Name)
+		notifications = append(notifications, n)
 	}
-	for _, n := range c.groupToIDList[notificationGroupID] {
-		ns := model.NotificationServerBundle{
-			Notification: n,
-			Server:       nil,
-			Loc:          Loc,
-		}
+	c.listMu.RUnlock()
+
+	if len(notifications) == 0 {
+		log.Printf("NEZHA>> Notification group %d has no recipients", notificationGroupID)
+		return false
+	}
+	allDelivered := true
+	for _, n := range notifications {
+		var server *model.Server
 		if len(ext) > 0 {
-			ns.Server = ext[0]
+			server = ext[0]
 		}
-		if err := ns.Send(desc); err != nil {
-			log.Printf("NEZHA>> Sending notification to %s failed: %v", n.Name, err)
-		} else {
-			log.Printf("NEZHA>> Sending notification to %s succeeded", n.Name)
+		if !c.sendRecipient(n, desc, fullMuteLabel, server) {
+			allDelivered = false
 		}
 	}
+	return allDelivered
+}
+
+func (c *NotificationClass) sendRecipient(n *model.Notification, desc, fullMuteLabel string, server *model.Server) bool {
+	if n == nil {
+		return false
+	}
+	cacheKey := ""
+	if fullMuteLabel != "" {
+		cacheKey = notificationMethodMuteLabel(fullMuteLabel, n.ID)
+	}
+	unlock := c.lockDeliveryKey(cacheKey)
+	defer unlock()
+	var generation uint64
+	if cacheKey != "" {
+		c.deliveryMu.Lock()
+		generation = c.muteGeneration[cacheKey]
+		c.deliveryMu.Unlock()
+	}
+	var history NotificationHistory
+	if cacheKey != "" {
+		if value, exists := Cache.Get(cacheKey); exists {
+			history = value.(NotificationHistory)
+			if time.Now().Before(history.Until) {
+				return true // this recipient already received this phase
+			}
+		}
+	}
+	log.Printf("NEZHA>> Try to notify %s", n.Name)
+	var err error
+	if c.sendForTest != nil {
+		err = c.sendForTest(n, desc, server)
+	} else {
+		err = (&model.NotificationServerBundle{Notification: n, Server: server, Loc: Loc}).Send(desc)
+	}
+	if err != nil {
+		// Webhook errors can contain a credential-bearing request URL. Keep
+		// failure type without ever writing the URL/token to dashboard logs.
+		log.Printf("NEZHA>> Sending notification to %s failed (%s)", n.Name, notificationFailureSummary(err))
+		return false
+	}
+	log.Printf("NEZHA>> Sending notification to %s succeeded", n.Name)
+	if cacheKey != "" {
+		if history.Duration == 0 {
+			history.Duration = firstNotificationDelay
+		} else {
+			history.Duration = min(history.Duration*2, 24*time.Hour)
+		}
+		history.Until = time.Now().Add(history.Duration)
+		c.deliveryMu.Lock()
+		if c.muteGeneration[cacheKey] == generation {
+			Cache.Set(cacheKey, history, history.Duration+10*time.Minute)
+		}
+		c.deliveryMu.Unlock()
+	}
+	return true
+}
+
+func notificationFailureSummary(err error) string {
+	// Even wrapped URL errors can include credentials in path/query. Log only
+	// the error type; the HTTP status path already returns a dedicated type.
+	return fmt.Sprintf("%T", err)
 }
 
 func sendSMTPAdminNotification(message string) error {

service/singleton/notification_test.go

@@ -0,0 +1,519 @@
+package singleton
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/patrickmn/go-cache"
+
+	"github.com/nezhahq/nezha/model"
+)
+
+func TestNotificationFailureDoesNotMuteRetryOrResendSuccessfulRecipient(t *testing.T) {
+	oldCache := Cache
+	Cache = cache.New(time.Minute, time.Minute)
+	t.Cleanup(func() { Cache = oldCache })
+	first := &model.Notification{Common: model.Common{ID: 301}, Name: "first"}
+	second := &model.Notification{Common: model.Common{ID: 302}, Name: "second"}
+	nc := newNotificationClassWithItems(first, second)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 303}, Name: "retry-test"}, []uint64{first.ID, second.ID})
+	counts := map[uint64]int{}
+	nc.sendForTest = func(n *model.Notification, _ string, _ *model.Server) error {
+		counts[n.ID]++
+		if n.ID == second.ID && counts[n.ID] == 1 {
+			return errors.New("temporary failure")
+		}
+		return nil
+	}
+	label := "alert-retry-test"
+	nc.UnMuteNotification(303, label)
+	if nc.SendNotification(303, "incident", label) {
+		t.Fatal("partial delivery must report failure")
+	}
+	if !nc.SendNotification(303, "incident", label) {
+		t.Fatal("failed recipient should be retried successfully")
+	}
+	if counts[first.ID] != 1 || counts[second.ID] != 2 {
+		t.Fatalf("delivery attempts = %v, want first=1 second=2", counts)
+	}
+	nc.UnMuteNotification(303, label)
+}
+
+func TestNotificationConcurrentSameRecipientSendsOnce(t *testing.T) {
+	oldCache := Cache
+	Cache = cache.New(time.Minute, time.Minute)
+	t.Cleanup(func() { Cache = oldCache })
+	n := &model.Notification{Common: model.Common{ID: 401}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 402}, Name: "concurrent-test"}, []uint64{n.ID})
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	var calls atomic.Int32
+	nc.sendForTest = func(*model.Notification, string, *model.Server) error {
+		if calls.Add(1) == 1 {
+			close(entered)
+			<-release
+		}
+		return nil
+	}
+	done := make(chan bool, 2)
+	go func() { done <- nc.SendNotification(402, "incident", "same-event") }()
+	<-entered
+	go func() { done <- nc.SendNotification(402, "incident", "same-event") }()
+	close(release)
+	if !<-done || !<-done {
+		t.Fatal("concurrent delivery returned failure")
+	}
+	if calls.Load() != 1 {
+		t.Fatalf("same event delivered %d times, want 1", calls.Load())
+	}
+}
+
+func TestNotificationAsyncCoalescesBlockedServiceReports(t *testing.T) {
+	oldCache := Cache
+	Cache = cache.New(time.Minute, time.Minute)
+	t.Cleanup(func() { Cache = oldCache })
+	n := &model.Notification{Common: model.Common{ID: 501}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 502}, Name: "service-test"}, []uint64{n.ID})
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	var calls atomic.Int32
+	nc.sendForTest = func(*model.Notification, string, *model.Server) error {
+		if calls.Add(1) == 1 {
+			close(entered)
+			<-release
+		}
+		return nil
+	}
+	nc.SendNotificationAsync(502, "down", "service-down")
+	<-entered
+	for i := 0; i < 1000; i++ {
+		nc.SendNotificationAsync(502, "down", "service-down")
+	}
+	nc.deliveryMu.Lock()
+	inFlight := len(nc.asyncInFlight)
+	nc.deliveryMu.Unlock()
+	if inFlight != 1 || calls.Load() != 1 {
+		t.Fatalf("in-flight sends=%d attempts=%d, want one each", inFlight, calls.Load())
+	}
+	close(release)
+	deadline := time.After(time.Second)
+	for {
+		nc.deliveryMu.Lock()
+		remaining := len(nc.asyncInFlight)
+		nc.deliveryMu.Unlock()
+		if remaining == 0 {
+			break
+		}
+		select {
+		case <-deadline:
+			t.Fatal("async notification did not finish")
+		case <-time.After(time.Millisecond):
+		}
+	}
+}
+
+func TestNotificationFailureSummaryOmitsCredentialURL(t *testing.T) {
+	err := &url.Error{Op: "Post", URL: "https://example.test/bot-secret-token/send?key=secret-query", Err: errors.New("timeout")}
+	if summary := notificationFailureSummary(err); strings.Contains(summary, "secret") || strings.Contains(summary, "example.test") {
+		t.Fatalf("credential-bearing URL leaked: %s", summary)
+	}
+}
+
+func TestNotificationAsyncBacksOffFastFailures(t *testing.T) {
+	oldCache := Cache
+	Cache = cache.New(time.Minute, time.Minute)
+	t.Cleanup(func() { Cache = oldCache })
+	n := &model.Notification{Common: model.Common{ID: 511}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 512}, Name: "backoff-test"}, []uint64{n.ID})
+	var calls atomic.Int32
+	nc.sendForTest = func(*model.Notification, string, *model.Server) error {
+		calls.Add(1)
+		return errors.New("fast failure")
+	}
+	key := "512:service-down"
+	nc.SendNotificationAsync(512, "down", "service-down")
+	deadline := time.After(time.Second)
+	for {
+		nc.deliveryMu.Lock()
+		busy := len(nc.asyncInFlight) != 0
+		after := nc.asyncRetryAfter[key]
+		nc.deliveryMu.Unlock()
+		if !busy && !after.IsZero() {
+			break
+		}
+		select {
+		case <-deadline:
+			t.Fatal("first failed send did not record backoff")
+		case <-time.After(time.Millisecond):
+		}
+	}
+	for i := 0; i < 1000; i++ {
+		nc.SendNotificationAsync(512, "down", "service-down")
+	}
+	if calls.Load() != 1 {
+		t.Fatalf("fast failures caused %d attempts before retry-after", calls.Load())
+	}
+}
+
+func TestUnMuteDuringInflightSendCannotRemuteRecovery(t *testing.T) {
+	oldCache := Cache
+	Cache = cache.New(time.Minute, time.Minute)
+	t.Cleanup(func() { Cache = oldCache })
+	n := &model.Notification{Common: model.Common{ID: 521}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 522}, Name: "generation-test"}, []uint64{n.ID})
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	var calls atomic.Int32
+	nc.sendForTest = func(*model.Notification, string, *model.Server) error {
+		if calls.Add(1) == 1 {
+			close(entered)
+			<-release
+		}
+		return nil
+	}
+	done := make(chan bool, 1)
+	go func() { done <- nc.SendNotification(522, "first failure", "static-label") }()
+	<-entered
+	nc.UnMuteNotification(522, "static-label")
+	close(release)
+	if !<-done {
+		t.Fatal("first send failed unexpectedly")
+	}
+	if !nc.SendNotification(522, "next failure", "static-label") || calls.Load() != 2 {
+		t.Fatalf("old in-flight send re-muted next failure: calls=%d", calls.Load())
+	}
+}
+
+func TestAsyncBackoffMemoryIsBounded(t *testing.T) {
+	nc := &NotificationClass{
+		asyncRetryAfter: make(map[string]time.Time),
+		asyncRetryDelay: make(map[string]time.Duration),
+	}
+	for i := 0; i < maxAsyncBackoffKeys+100; i++ {
+		key := fmt.Sprintf("old-event-%d", i)
+		nc.asyncRetryAfter[key] = time.Now().Add(time.Minute)
+		nc.asyncRetryDelay[key] = time.Minute
+	}
+	nc.pruneAsyncBackoffLocked()
+	if len(nc.asyncRetryAfter) > maxAsyncBackoffKeys || len(nc.asyncRetryDelay) != len(nc.asyncRetryAfter) {
+		t.Fatalf("unbounded backoff maps: after=%d delay=%d", len(nc.asyncRetryAfter), len(nc.asyncRetryDelay))
+	}
+}
+
+func TestServiceStateNotificationsKeepIncidentRecoveryOrder(t *testing.T) {
+	oldCache := Cache
+	Cache = cache.New(time.Minute, time.Minute)
+	t.Cleanup(func() { Cache = oldCache })
+	n := &model.Notification{Common: model.Common{ID: 531}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 532}, Name: "ordered-test"}, []uint64{n.ID})
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	delivered := make(chan string, 3)
+	nc.sendForTest = func(_ *model.Notification, desc string, _ *model.Server) error {
+		if desc == "down" {
+			close(entered)
+			<-release
+		}
+		delivered <- desc
+		return nil
+	}
+	nc.SendServiceState(533, 532, "down", "state-down-1")
+	<-entered
+	nc.SendServiceState(533, 532, "good", "state-good-2")
+	select {
+	case message := <-delivered:
+		t.Fatalf("service transition overtook blocked incident: %s", message)
+	default:
+	}
+	close(release)
+	for _, want := range []string{"down", "good"} {
+		select {
+		case got := <-delivered:
+			if got != want {
+				t.Fatalf("delivery order got %s, want %s", got, want)
+			}
+		case <-time.After(time.Second):
+			t.Fatalf("missing %s delivery", want)
+		}
+	}
+	deadline := time.After(time.Second)
+	for {
+		nc.deliveryMu.Lock()
+		remaining := len(nc.serviceDelivery)
+		nc.deliveryMu.Unlock()
+		if remaining == 0 {
+			break
+		}
+		select {
+		case <-deadline:
+			t.Fatal("ordered delivery worker did not exit")
+		case <-time.After(time.Millisecond):
+		}
+	}
+}
+
+func TestServiceStateRetriesFailedIncidentBeforePendingRecovery(t *testing.T) {
+	oldCache := Cache
+	Cache = cache.New(time.Minute, time.Minute)
+	t.Cleanup(func() { Cache = oldCache })
+	n := &model.Notification{Common: model.Common{ID: 535}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 536}, Name: "retry-order-test"}, []uint64{n.ID})
+	t.Cleanup(func() { nc.CancelServiceState(537) })
+
+	attempts := make(chan string, 3)
+	releaseFirst := make(chan struct{})
+	var downAttempts atomic.Int32
+	nc.sendForTest = func(_ *model.Notification, desc string, _ *model.Server) error {
+		attempts <- desc
+		if desc == "down" && downAttempts.Add(1) == 1 {
+			<-releaseFirst
+			return errors.New("temporary incident failure")
+		}
+		return nil
+	}
+
+	nc.SendServiceState(537, 536, "down", "state-down-1")
+	nc.deliveryMu.Lock()
+	state := nc.serviceDelivery[537]
+	nc.deliveryMu.Unlock()
+	select {
+	case got := <-attempts:
+		if got != "down" {
+			t.Fatalf("first delivery = %q, want down", got)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("incident delivery did not start")
+	}
+	nc.SendServiceState(537, 536, "good", "state-good-2")
+	close(releaseFirst)
+
+	for _, want := range []string{"down", "good"} {
+		select {
+		case got := <-attempts:
+			if got != want {
+				t.Fatalf("delivery after failed incident = %q, want %q", got, want)
+			}
+		case <-time.After(time.Second):
+			t.Fatalf("missing %s delivery", want)
+		}
+	}
+	select {
+	case <-state.done:
+	case <-time.After(time.Second):
+		t.Fatal("ordered delivery worker did not finish")
+	}
+}
+
+func TestServiceStateCancelStopsOldConfiguration(t *testing.T) {
+	oldCache := Cache
+	Cache = cache.New(time.Minute, time.Minute)
+	t.Cleanup(func() { Cache = oldCache })
+	n := &model.Notification{Common: model.Common{ID: 541}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 542}, Name: "old-group"}, []uint64{n.ID})
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	var calls atomic.Int32
+	nc.sendForTest = func(*model.Notification, string, *model.Server) error {
+		if calls.Add(1) == 1 {
+			close(entered)
+			<-release
+		}
+		return errors.New("old webhook failed")
+	}
+	nc.SendServiceState(543, 542, "down", "old-event")
+	<-entered
+	nc.deliveryMu.Lock()
+	state := nc.serviceDelivery[543]
+	nc.deliveryMu.Unlock()
+	nc.CancelServiceState(543)
+	nc.deliveryMu.Lock()
+	remaining := len(nc.serviceDelivery)
+	nc.deliveryMu.Unlock()
+	if remaining != 0 {
+		t.Fatal("old service delivery worker remained registered after cancellation")
+	}
+	close(release)
+	select {
+	case <-state.done:
+	case <-time.After(time.Second):
+		t.Fatal("canceled old delivery worker did not exit")
+	}
+}
+
+func TestServiceUpdateCancelsOldRecipientRetry(t *testing.T) {
+	ss := newServiceMonitorSecurityHarness(t)
+	service := &model.Service{
+		Common:              model.Common{ID: 551, UserID: 1},
+		Name:                "generic-probe",
+		Type:                model.TaskTypeTCPPing,
+		Target:              "example.invalid:443",
+		Duration:            3600,
+		Notify:              true,
+		NotificationGroupID: 552,
+		Cover:               model.ServiceCoverIgnoreAll,
+	}
+	addServiceMonitorSecurityService(t, ss, service)
+	n := &model.Notification{Common: model.Common{ID: 553}, Name: "old-hook"}
+	NotificationShared.InsertForTest(n)
+	NotificationShared.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 552}, Name: "old-group"}, []uint64{n.ID})
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	NotificationShared.sendForTest = func(*model.Notification, string, *model.Server) error {
+		close(entered)
+		<-release
+		return errors.New("failed old recipient")
+	}
+	NotificationShared.SendServiceState(service.ID, 552, "old down", "old-state")
+	<-entered
+	NotificationShared.deliveryMu.Lock()
+	state := NotificationShared.serviceDelivery[service.ID]
+	NotificationShared.deliveryMu.Unlock()
+	updated := *service
+	updated.Notify = false
+	updated.NotificationGroupID = 554
+	if err := ss.Update(&updated); err != nil {
+		t.Fatal(err)
+	}
+	close(release)
+	select {
+	case <-state.done:
+	case <-time.After(time.Second):
+		t.Fatal("old recipient retry did not stop after service update")
+	}
+	NotificationShared.deliveryMu.Lock()
+	remaining := len(NotificationShared.serviceDelivery)
+	NotificationShared.deliveryMu.Unlock()
+	if remaining != 0 {
+		t.Fatal("old recipient retained after service update")
+	}
+}
+
+func newNotificationClassWithItems(items ...*model.Notification) *NotificationClass {
+	nc := NewEmptyNotificationClassForTest()
+	for _, item := range items {
+		nc.InsertForTest(item)
+	}
+	return nc
+}
+
+func TestNotificationClassDeleteGroupCleansReverseIndex(t *testing.T) {
+	n := &model.Notification{Common: model.Common{ID: 1, UserID: 2}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	group := &model.NotificationGroup{Common: model.Common{ID: 10, UserID: 2}, Name: "group"}
+	nc.UpdateGroup(group, []uint64{n.ID})
+
+	nc.DeleteGroup([]uint64{group.ID})
+
+	if _, ok := nc.groupToIDList[group.ID]; ok {
+		t.Fatalf("forward index still contains deleted group %d", group.ID)
+	}
+	if groups, ok := nc.idToGroupList[n.ID]; ok {
+		if _, stale := groups[group.ID]; stale {
+			t.Fatalf("reverse index for notification %d still contains deleted group %d", n.ID, group.ID)
+		}
+	}
+}
+
+func TestNotificationClassUpdateGroupCleansRemovedMembership(t *testing.T) {
+	first := &model.Notification{Common: model.Common{ID: 1, UserID: 2}, Name: "first"}
+	second := &model.Notification{Common: model.Common{ID: 2, UserID: 2}, Name: "second"}
+	nc := newNotificationClassWithItems(first, second)
+	group := &model.NotificationGroup{Common: model.Common{ID: 10, UserID: 2}, Name: "group"}
+	nc.UpdateGroup(group, []uint64{first.ID, second.ID})
+
+	nc.UpdateGroup(group, []uint64{second.ID})
+
+	if groups, ok := nc.idToGroupList[first.ID]; ok {
+		if _, stale := groups[group.ID]; stale {
+			t.Fatalf("reverse index for notification %d still contains group %d", first.ID, group.ID)
+		}
+	}
+	if _, ok := nc.idToGroupList[second.ID][group.ID]; !ok {
+		t.Fatalf("reverse index lost retained membership of notification %d in group %d", second.ID, group.ID)
+	}
+}
+
+func TestNotificationClassUpdateRepairsOrphanedReverseIndex(t *testing.T) {
+	n := &model.Notification{Common: model.Common{ID: 1, UserID: 2}, Name: "hook"}
+	nc := newNotificationClassWithItems(n)
+	nc.idToGroupList[n.ID] = map[uint64]struct{}{99: {}}
+
+	n.Name = "updated"
+	nc.Update(n)
+
+	if groups, ok := nc.idToGroupList[n.ID]; ok && len(groups) != 0 {
+		t.Fatalf("orphaned reverse memberships were not removed: %v", groups)
+	}
+}
+
+func TestNotificationClassGroupMutationsDoNotDeadlock(t *testing.T) {
+	nc := NewEmptyNotificationClassForTest()
+	nc.listMu.RLock()
+	readLockHeld := true
+	defer func() {
+		if readLockHeld {
+			nc.listMu.RUnlock()
+		}
+	}()
+
+	deleteDone := make(chan struct{})
+	go func() {
+		nc.DeleteGroup([]uint64{10})
+		close(deleteDone)
+	}()
+
+	deadline := time.Now().Add(2 * time.Second)
+	for nc.listMu.TryRLock() {
+		nc.listMu.RUnlock()
+		if time.Now().After(deadline) {
+			t.Fatal("DeleteGroup did not queue for listMu")
+		}
+		time.Sleep(time.Millisecond)
+	}
+
+	updateDone := make(chan struct{})
+	go func() {
+		nc.UpdateGroup(&model.NotificationGroup{
+			Common: model.Common{ID: 10, UserID: 2},
+			Name:   "group",
+		}, nil)
+		close(updateDone)
+	}()
+
+	deadline = time.Now().Add(2 * time.Second)
+	for nc.groupMu.TryLock() {
+		nc.groupMu.Unlock()
+		if time.Now().After(deadline) {
+			t.Fatal("neither group mutation acquired groupMu")
+		}
+		time.Sleep(time.Millisecond)
+	}
+
+	nc.listMu.RUnlock()
+	readLockHeld = false
+
+	timer := time.NewTimer(2 * time.Second)
+	defer timer.Stop()
+	for deleteDone != nil || updateDone != nil {
+		select {
+		case <-deleteDone:
+			deleteDone = nil
+		case <-updateDone:
+			updateDone = nil
+		case <-timer.C:
+			t.Fatal("concurrent UpdateGroup and DeleteGroup deadlocked")
+		}
+	}
+}

service/singleton/online_history.go

@@ -0,0 +1,348 @@
+package singleton
+
+import (
+	"fmt"
+	"log"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/nezhahq/nezha/model"
+)
+
+// OnlineHistoryRecorder 维护每台服务器当前进行中的在线会话。会话的打开与
+// 关闭由 agent 状态流的建立/断开驱动（service/rpc），并由定时清扫兜底关闭
+// 失联（未正常断流）的会话；dashboard 重启时通过数据库中 Ongoing 会话恢复。
+type OnlineHistoryRecorder struct {
+	mu       sync.Mutex
+	restored bool
+	open     map[uint64]*onlineSessionRuntime
+	// closing 是关库失败后的待关闭状态：断线边界已被观察到，会话 ID 与
+	// 结束时间在此固定、不可被后续上报覆盖；重试只为旧会话完成持久化，
+	// 重连不得把它续接成活跃会话（同服务器可能同时有多条待关闭会话）。
+	closing map[uint64][]*onlineSessionClosing
+	// Preserve the first disconnect while startup reads are unavailable.
+	recoveryDisconnects map[uint64]time.Time
+}
+
+type onlineSessionRuntime struct {
+	id           uint64
+	lastObserved time.Time
+	revision     uint64
+	// lastPersist 是 LastActiveAt 最近一次落库时间，用于节流写库。
+	lastPersist time.Time
+}
+
+type onlineSessionClosing struct {
+	id uint64
+	// end 是已观察到的会话结束时刻，重试持久化时原样使用。
+	end time.Time
+}
+
+var OnlineHistoryShared *OnlineHistoryRecorder
+
+func NewOnlineHistoryRecorder() *OnlineHistoryRecorder {
+	r := &OnlineHistoryRecorder{
+		open:                make(map[uint64]*onlineSessionRuntime),
+		closing:             make(map[uint64][]*onlineSessionClosing),
+		recoveryDisconnects: make(map[uint64]time.Time),
+	}
+	r.mu.Lock()
+	r.restoreOngoingSessions()
+	r.mu.Unlock()
+	return r
+}
+
+// restoreOngoingSessions 每台服务器仅允许最新且仍在在线窗口内的会话续接，
+// 其余会话按最后持久化的活跃时间结转。崩溃会丢失内存中的待关闭边界，
+// 恢复时采用 LastActiveAt 保守结转，不推测未持久化的断线时间。
+func (r *OnlineHistoryRecorder) restoreOngoingSessions() bool {
+	if DB == nil {
+		return false
+	}
+	var ongoing []model.OnlineSession
+	if err := DB.Where("ongoing = ?", true).
+		Order("server_id ASC").Order("started_at DESC").Order("id DESC").Find(&ongoing).Error; err != nil {
+		log.Printf("NEZHA>> Failed to load ongoing online sessions: %v", err)
+		return false
+	}
+	now := time.Now()
+	seenServers := make(map[uint64]bool)
+	for _, s := range ongoing {
+		// 关闭失败后重连可留下同服务器多条 Ongoing 行；直接赋值 open 会覆盖
+		// 旧 ID，导致其永久脱管。按开始时间（同时间按 ID）选最新行，其余
+		// 必须关闭或进入 closing 重试，即使最新行本身已经过期也不能复活旧行。
+		duplicate := seenServers[s.ServerID]
+		seenServers[s.ServerID] = true
+		disconnect, observedDisconnect := r.recoveryDisconnects[s.ServerID]
+		observedDisconnect = observedDisconnect && !duplicate && !disconnect.Before(s.StartedAt)
+		if observedDisconnect || duplicate || s.LastActiveAt.Before(now.Add(-model.OnlineActiveWindow)) {
+			end := s.LastActiveAt
+			if observedDisconnect && disconnect.After(end) {
+				end = disconnect
+			}
+			if end.Before(s.StartedAt) {
+				end = s.StartedAt
+			}
+			if err := closeOnlineSessionRow(s.ID, end); err != nil {
+				log.Printf("NEZHA>> Failed to close recovered online session %d: %v", s.ID, err)
+				// 关库失败不能跳过，否则会留下永不闭合、也无人重试的
+				// Ongoing 行；进入待关闭状态交由清扫周期重试。过期或
+				// 被更新会话替代的行均不得被重连续接为活跃会话
+				r.closing[s.ServerID] = append(r.closing[s.ServerID],
+					&onlineSessionClosing{id: s.ID, end: end})
+			}
+			continue
+		}
+		r.open[s.ServerID] = &onlineSessionRuntime{id: s.ID, lastPersist: s.LastActiveAt, lastObserved: s.LastActiveAt}
+	}
+	clear(r.recoveryDisconnects)
+	r.restored = true
+	return true
+}
+
+// flushClosing 尽力重试关闭该服务器的待关闭会话：按入列时固定的结束时间
+// 持久化，成功才移除；失败的条目原样保留，结束时间不得被后续上报改写。
+func (r *OnlineHistoryRecorder) flushClosing(serverID uint64) {
+	pendings, ok := r.closing[serverID]
+	if !ok {
+		return
+	}
+	kept := pendings[:0]
+	for _, pc := range pendings {
+		if err := closeOnlineSessionRow(pc.id, pc.end); err != nil {
+			log.Printf("NEZHA>> Failed to retry closing online session %d: %v", pc.id, err)
+			kept = append(kept, pc)
+		}
+	}
+	if len(kept) == 0 {
+		delete(r.closing, serverID)
+	} else {
+		r.closing[serverID] = kept
+	}
+}
+
+// RecordOnline 记录一次状态上报：无进行中会话时开启新会话（重启遗留且仍在
+// 窗口内的 Ongoing 会话直接续接）；已有会话时节流落库 LastActiveAt。
+func (r *OnlineHistoryRecorder) RecordOnline(serverID uint64, at time.Time) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	// A failed startup read is not an empty history: retry before accepting reports.
+	if !r.restored && !r.restoreOngoingSessions() {
+		return
+	}
+	at = at.UTC()
+	r.flushClosing(serverID)
+
+	// A report can arrive after expiry but before the next sweep. Close at the
+	// previous observation before overwriting it, preserving failed-close retries.
+	if rt := r.open[serverID]; rt != nil && at.Sub(rt.lastObserved) > model.OnlineActiveWindow {
+		r.recordOfflineLocked(serverID, rt.lastObserved)
+	}
+	if rt, ok := r.open[serverID]; ok {
+		rt.lastObserved = at
+		rt.revision++
+		if at.Sub(rt.lastPersist) >= model.OnlineActiveWindow {
+			if err := DB.Model(&model.OnlineSession{}).Where("id = ?", rt.id).Update("last_active_at", at).Error; err != nil {
+				log.Printf("NEZHA>> Failed to persist online session last_active_at: %v", err)
+			} else {
+				// Only committed writes advance the throttle; failures retry on the next report.
+				rt.lastPersist = at
+			}
+		}
+		return
+	}
+
+	// All ongoing rows are owned by open or closing after successful recovery.
+
+	s := model.OnlineSession{ServerID: serverID, StartedAt: at, LastActiveAt: at, Ongoing: true}
+	if err := DB.Create(&s).Error; err != nil {
+		log.Printf("NEZHA>> Failed to create online session: %v", err)
+		return
+	}
+	r.open[serverID] = &onlineSessionRuntime{id: s.ID, lastPersist: at, lastObserved: at}
+}
+
+// RecordOffline 关闭该服务器当前进行中的会话。at 是会话结束时刻：正常断流
+// 为当前时间，清扫/结转为最后一次已知活跃时间。
+// 关库失败时把会话转入待关闭状态而非保留为活跃条目：会话 ID 与已观察到的
+// 结束时刻（at 与 lastPersist 的较大者）就此固定，后续上报不得覆盖；重试
+// 只为旧会话完成持久化，重连则开启新会话——断线后的离线间隔不得计作在线。
+func (r *OnlineHistoryRecorder) RecordOffline(serverID uint64, at time.Time) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	// Disconnect may be the first event after startup storage recovers.
+	if !r.restored && !r.restoreOngoingSessions() {
+		// Retrying SELECT alone loses confirmed offline gaps on a quick reconnect.
+		// Keep the first boundary; subsequent disconnected streams cannot extend it.
+		if r.recoveryDisconnects == nil {
+			r.recoveryDisconnects = make(map[uint64]time.Time)
+		}
+		if _, exists := r.recoveryDisconnects[serverID]; !exists {
+			r.recoveryDisconnects[serverID] = at.UTC()
+		}
+		return
+	}
+	r.recordOfflineLocked(serverID, at)
+}
+
+func (r *OnlineHistoryRecorder) recordOfflineLocked(serverID uint64, at time.Time) {
+	rt, ok := r.open[serverID]
+	if !ok {
+		return
+	}
+	end := at
+	if end.Before(rt.lastPersist) {
+		end = rt.lastPersist
+	}
+	if err := closeOnlineSessionRow(rt.id, end); err != nil {
+		log.Printf("NEZHA>> Failed to close online session %d: %v", rt.id, err)
+		// 结束时刻已观察到，不能因写库失败丢失，也不能把行继续当活跃
+		// 会话：转待关闭状态，由 RecordOnline 冲洗与清扫周期重试
+		r.closing[serverID] = append(r.closing[serverID], &onlineSessionClosing{id: rt.id, end: end})
+		delete(r.open, serverID)
+		return
+	}
+	delete(r.open, serverID)
+}
+
+// SweepStale 关闭超过在线窗口仍无状态上报的进行中会话。lastActiveFn 返回
+// 服务器的运行时最后活跃时间与是否存在。
+func (r *OnlineHistoryRecorder) SweepStale(now time.Time, lastActiveFn func(serverID uint64) (time.Time, bool)) {
+	// Never call the runtime observer while holding mu: reports acquire the
+	// runtime lease before this mutex. Snapshot identities, then revalidate them.
+	r.mu.Lock()
+	if !r.restored && !r.restoreOngoingSessions() {
+		r.mu.Unlock()
+		return
+	}
+	candidates := make(map[uint64]onlineSessionRuntime, len(r.open))
+	for id, rt := range r.open {
+		candidates[id] = *rt
+	}
+	for id := range r.closing {
+		r.flushClosing(id)
+	}
+	r.mu.Unlock()
+	for id, candidate := range candidates {
+		lastActive, ok := lastActiveFn(id)
+		r.mu.Lock()
+		rt := r.open[id]
+		if rt != nil && rt.id == candidate.id && rt.revision == candidate.revision {
+			if lastActive.Before(rt.lastObserved) {
+				lastActive = rt.lastObserved
+			}
+			if (!ok || lastActive.Before(now.Add(-model.OnlineActiveWindow))) && rt.lastObserved.Before(now.Add(-model.OnlineActiveWindow)) {
+				r.recordOfflineLocked(id, lastActive)
+			}
+		}
+		r.mu.Unlock()
+	}
+
+}
+
+// SweepOnlineSessions 是定时清扫入口，接上服务器运行时的最后活跃时间。
+func SweepOnlineSessions() {
+	if OnlineHistoryShared == nil || ServerShared == nil || DB == nil {
+		return
+	}
+	OnlineHistoryShared.SweepStale(time.Now(), onlineServerLastActive)
+}
+
+func onlineServerLastActive(serverID uint64) (time.Time, bool) {
+	server, ok := ServerShared.Get(serverID)
+	if !ok || server == nil {
+		return time.Time{}, false
+	}
+	lastActive := server.RuntimeSnapshot().LastActive
+	return lastActive, !lastActive.IsZero()
+}
+
+// PruneOnlineSessions 清理超出统计窗口的已结束会话与已删除服务器的残留记录。
+// 时间列直接比较（与写入时的绑定格式同构）以走 ended_at 索引，并用分批
+// DELETE 替换单条巨型删除，避免大表上长时间占用 sqlite 写锁、阻塞上报路径。
+func PruneOnlineSessions() {
+	if DB == nil {
+		return
+	}
+	keepDays := Conf.OnlineHistoryRetentionDays()
+	if keepDays < model.MinOnlineStatsDays || keepDays > model.MaxOnlineStatsDays {
+		keepDays = model.DefaultOnlineStatsDays
+	}
+	cutoff := model.OnlineHistoryWindowStart(time.Now(), keepDays, Loc)
+	pruneOnlineSessionsBatch("ongoing = ? AND ended_at < ?", false, cutoff)
+	pruneOnlineSessionsBatch("server_id NOT IN (SELECT `id` FROM servers)")
+}
+
+// pruneOnlineSessionsBatch 按主键分批删除匹配的行，每批之间释放写锁。
+func pruneOnlineSessionsBatch(where string, args ...any) {
+	const batchSize = 10000
+	for {
+		var ids []uint64
+		if err := DB.Model(&model.OnlineSession{}).Unscoped().
+			Where(where, args...).Limit(batchSize).Pluck("id", &ids).Error; err != nil {
+			log.Printf("NEZHA>> Failed to select online sessions for pruning: %v", err)
+			return
+		}
+		if len(ids) == 0 {
+			return
+		}
+		if err := DB.Unscoped().Delete(&model.OnlineSession{}, ids).Error; err != nil {
+			log.Printf("NEZHA>> Failed to prune online sessions: %v", err)
+			return
+		}
+		if len(ids) < batchSize {
+			return
+		}
+	}
+}
+
+func closeOnlineSessionRow(id uint64, end time.Time) error {
+	return DB.Model(&model.OnlineSession{}).Where("id = ?", id).Updates(map[string]any{
+		"ended_at": end.UTC(),
+		"ongoing":  false,
+	}).Error
+}
+
+// Sessions snapshots rows and volatile close boundaries under the same lock.
+// Otherwise a failed close can count confirmed offline time as ongoing, or a
+// successful retry can remove the boundary after the reader fetched a stale row.
+func (r *OnlineHistoryRecorder) Sessions(serverID uint64, start, end time.Time) ([]model.OnlineSession, error) {
+	if r != nil {
+		r.mu.Lock()
+		defer r.mu.Unlock()
+		if !r.restored && !r.restoreOngoingSessions() {
+			return nil, fmt.Errorf("online history recovery unavailable")
+		}
+	}
+	var sessions []model.OnlineSession
+	// Separate branches use (server_id, ongoing, ended_at) and
+	// (server_id, ongoing, started_at), avoiding a scan of retained closed rows.
+	for _, ongoing := range []bool{false, true} {
+		var rows []model.OnlineSession
+		q := DB.Where("server_id = ? AND ongoing = ?", serverID, ongoing)
+		if ongoing {
+			q = q.Where("started_at < ?", end.UTC())
+		} else {
+			q = q.Where("ended_at > ?", start.UTC())
+		}
+		if err := q.Find(&rows).Error; err != nil {
+			return nil, err
+		}
+		sessions = append(sessions, rows...)
+	}
+	if r != nil {
+		endings := make(map[uint64]time.Time)
+		for _, pending := range r.closing[serverID] {
+			endings[pending.id] = pending.end
+		}
+		for i := range sessions {
+			if end, ok := endings[sessions[i].ID]; ok {
+				sessions[i].Ongoing = false
+				sessions[i].EndedAt = end
+			}
+		}
+	}
+	sort.Slice(sessions, func(i, j int) bool { return sessions[i].StartedAt.Before(sessions[j].StartedAt) })
+	return sessions, nil
+}

service/singleton/online_history_failure_test.go

@@ -0,0 +1,224 @@
+package singleton
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/nezhahq/nezha/model"
+	"github.com/stretchr/testify/require"
+	"gorm.io/gorm"
+)
+
+func TestOnlineHistoryRetriesFailedHeartbeatImmediately(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	r := NewOnlineHistoryRecorder()
+	start := time.Now().UTC().Truncate(time.Second)
+	r.RecordOnline(1, start)
+	id := r.open[1].id
+	require.NoError(t, DB.Exec(`CREATE TRIGGER reject_heartbeat BEFORE UPDATE OF last_active_at ON online_sessions BEGIN SELECT RAISE(ABORT, 'heartbeat failure'); END`).Error)
+	r.RecordOnline(1, start.Add(30*time.Second))
+	require.NoError(t, DB.Exec("DROP TRIGGER reject_heartbeat").Error)
+	r.RecordOnline(1, start.Add(31*time.Second))
+	require.True(t, getOnlineSession(t, id).LastActiveAt.Equal(start.Add(31*time.Second)))
+}
+
+func TestOnlineHistoryRetriesStartupReadBeforeNewReport(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	start := time.Now().UTC().Add(-time.Hour).Truncate(time.Second)
+	old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true}
+	require.NoError(t, DB.Create(&old).Error)
+	require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("reject_history_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) }))
+	r := NewOnlineHistoryRecorder()
+	require.NoError(t, DB.Callback().Query().Remove("reject_history_restore"))
+	r.RecordOnline(1, time.Now())
+	stored := getOnlineSession(t, old.ID)
+	require.False(t, stored.Ongoing)
+	require.True(t, stored.EndedAt.Equal(start))
+	require.NotEqual(t, old.ID, r.open[1].id)
+	require.Equal(t, int64(1), countOnlineSessions(t, "ongoing = ?", true))
+}
+
+func TestOnlineHistorySweepKeepsSessionReconnectedDuringObservation(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	r := NewOnlineHistoryRecorder()
+	start := time.Now().Add(-time.Hour)
+	r.RecordOnline(1, start)
+	observed := make(chan struct{})
+	release := make(chan struct{})
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		r.SweepStale(time.Now(), func(uint64) (time.Time, bool) { close(observed); <-release; return start, true })
+	}()
+	<-observed
+	reconnected := make(chan struct{})
+	go func() { r.RecordOffline(1, start.Add(time.Second)); r.RecordOnline(1, time.Now()); close(reconnected) }()
+	// The observer must not hold the recorder lock: RPC updates take the runtime
+	// lock first and then enter the recorder. Holding both in reverse order deadlocks.
+	select {
+	case <-reconnected:
+	case <-time.After(time.Second):
+		close(release)
+		<-done
+		<-reconnected
+		t.Fatal("sweep holds recorder lock while consulting runtime state")
+	}
+	id := r.open[1].id
+	close(release)
+	<-done
+	require.Contains(t, r.open, uint64(1))
+	require.Equal(t, id, r.open[1].id)
+	require.True(t, getOnlineSession(t, id).Ongoing)
+}
+
+func TestOnlineHistoryQueryNormalizesOffsetsAndUsesWindowIndexes(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	east := time.FixedZone("UTC+8", 8*3600)
+	start := time.Date(2025, 1, 1, 12, 0, 0, 0, east)
+	r := NewOnlineHistoryRecorder()
+	r.RecordOnline(1, start)
+	r.RecordOffline(1, start.Add(time.Hour))
+	rows, err := r.Sessions(1, start.UTC().Add(-time.Minute), start.UTC().Add(2*time.Hour))
+	require.NoError(t, err)
+	require.Len(t, rows, 1)
+	require.True(t, rows[0].StartedAt.Equal(start))
+	require.True(t, rows[0].EndedAt.Equal(start.Add(time.Hour)))
+	for _, test := range []struct {
+		query, index string
+		args         []any
+	}{
+		{"EXPLAIN QUERY PLAN SELECT * FROM online_sessions WHERE server_id = ? AND ongoing = ? AND ended_at > ?", "idx_online_sessions_server_closed", []any{1, false, start.UTC()}},
+		{"EXPLAIN QUERY PLAN SELECT * FROM online_sessions WHERE server_id = ? AND ongoing = ? AND started_at < ?", "idx_online_sessions_server_open", []any{1, true, start.Add(2 * time.Hour).UTC()}},
+	} {
+		var plan []struct{ Detail string }
+		require.NoError(t, DB.Raw(test.query, test.args...).Scan(&plan).Error)
+		require.NotEmpty(t, plan)
+		require.Contains(t, plan[0].Detail, test.index)
+	}
+}
+
+func TestOnlineHistorySweepKeepsReportReceivedDuringObservation(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	r := NewOnlineHistoryRecorder()
+	start := time.Now().Add(-time.Hour)
+	r.RecordOnline(1, start)
+	id := r.open[1].id
+	r.SweepStale(time.Now(), func(uint64) (time.Time, bool) {
+		r.RecordOnline(1, time.Now())
+		return start, true
+	})
+	require.Contains(t, r.open, uint64(1))
+	// A report after an hour starts a new session; the stale sweep must not close it.
+	require.NotEqual(t, id, r.open[1].id)
+	require.False(t, getOnlineSession(t, id).Ongoing)
+	require.True(t, getOnlineSession(t, r.open[1].id).Ongoing)
+}
+
+func TestOnlineHistoryStartupReadFailureBlocksQueryAndSweepRetries(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	start := time.Now().Add(-time.Hour)
+	old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true}
+	require.NoError(t, DB.Create(&old).Error)
+	require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("reject_history_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) }))
+	r := NewOnlineHistoryRecorder()
+	_, err := r.Sessions(1, start, time.Now())
+	require.ErrorContains(t, err, "recovery unavailable")
+	r.RecordOnline(1, time.Now())
+	require.Empty(t, r.open)
+	require.NoError(t, DB.Callback().Query().Remove("reject_history_restore"))
+	r.SweepStale(time.Now(), func(uint64) (time.Time, bool) { return time.Time{}, false })
+	require.False(t, getOnlineSession(t, old.ID).Ongoing)
+	require.Equal(t, int64(0), countOnlineSessions(t, "ongoing = ?", true))
+}
+
+func TestOnlineHistoryDisconnectRetriesStartupRead(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	start := time.Now().Add(-10 * time.Second)
+	old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true}
+	require.NoError(t, DB.Create(&old).Error)
+	require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("reject_history_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) }))
+	r := NewOnlineHistoryRecorder()
+	require.NoError(t, DB.Callback().Query().Remove("reject_history_restore"))
+	end := start.Add(time.Second)
+	r.RecordOffline(1, end)
+	require.False(t, getOnlineSession(t, old.ID).Ongoing)
+	require.True(t, getOnlineSession(t, old.ID).EndedAt.Equal(end))
+	r.RecordOnline(1, time.Now())
+	require.NotEqual(t, old.ID, r.open[1].id)
+}
+
+func TestOnlineHistoryReportAfterTimeoutSplitsBeforeSweep(t *testing.T) {
+	for _, failClose := range []bool{false, true} {
+		t.Run(map[bool]string{false: "persisted", true: "retry_close"}[failClose], func(t *testing.T) {
+			setupOnlineHistoryTestDB(t)
+			r := NewOnlineHistoryRecorder()
+			start := time.Now().UTC().Truncate(time.Second)
+			r.RecordOnline(1, start)
+			r.RecordOnline(1, start.Add(time.Second))
+			first := r.open[1].id
+			r.SweepStale(start.Add(30*time.Second), func(uint64) (time.Time, bool) { return start.Add(time.Second), true })
+			require.Equal(t, first, r.open[1].id)
+			if failClose {
+				injectCloseUpdateFailure(t)
+			}
+			r.RecordOnline(1, start.Add(45*time.Second))
+			require.NotEqual(t, first, r.open[1].id)
+			second := r.open[1].id
+			rows, err := r.Sessions(1, start, start.Add(46*time.Second))
+			require.NoError(t, err)
+			require.Len(t, rows, 2)
+			require.False(t, rows[0].Ongoing)
+			require.True(t, rows[0].EndedAt.Equal(start.Add(time.Second)))
+			require.True(t, rows[1].StartedAt.Equal(start.Add(45*time.Second)))
+			if failClose {
+				require.Len(t, r.closing[1], 1)
+				dropCloseUpdateFailure(t)
+			}
+			r.SweepStale(start.Add(46*time.Second), func(uint64) (time.Time, bool) { return start.Add(45 * time.Second), true })
+			require.Empty(t, r.closing)
+			require.False(t, getOnlineSession(t, first).Ongoing)
+			require.True(t, getOnlineSession(t, first).EndedAt.Equal(start.Add(time.Second)))
+			require.True(t, getOnlineSession(t, second).Ongoing)
+		})
+	}
+}
+
+func TestOnlineHistoryDisconnectSurvivesUnavailableRecovery(t *testing.T) {
+	for _, failClose := range []bool{false, true} {
+		t.Run(map[bool]string{false: "close_success", true: "close_retry"}[failClose], func(t *testing.T) {
+			setupOnlineHistoryTestDB(t)
+			start := time.Now().UTC().Add(-10 * time.Second)
+			old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true}
+			require.NoError(t, DB.Create(&old).Error)
+			require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("unavailable_disconnect_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) }))
+			r := NewOnlineHistoryRecorder()
+			end := start.Add(time.Second)
+			r.RecordOffline(1, end)
+			// Later disconnects while recovery is unavailable must not extend the first session.
+			r.RecordOffline(1, end.Add(time.Second))
+			require.NoError(t, DB.Callback().Query().Remove("unavailable_disconnect_restore"))
+			if failClose {
+				injectCloseUpdateFailure(t)
+			}
+			reconnect := start.Add(5 * time.Second)
+			r.RecordOnline(1, reconnect)
+			require.NotEqual(t, old.ID, r.open[1].id)
+			rows, err := r.Sessions(1, start, reconnect.Add(time.Second))
+			require.NoError(t, err)
+			require.Len(t, rows, 2)
+			require.False(t, rows[0].Ongoing)
+			require.True(t, rows[0].EndedAt.Equal(end))
+			require.True(t, rows[1].StartedAt.Equal(reconnect))
+			if failClose {
+				require.Len(t, r.closing[1], 1)
+				dropCloseUpdateFailure(t)
+			}
+			r.SweepStale(reconnect.Add(time.Second), func(uint64) (time.Time, bool) { return reconnect, true })
+			require.Empty(t, r.closing)
+			require.False(t, getOnlineSession(t, old.ID).Ongoing)
+			require.True(t, getOnlineSession(t, old.ID).EndedAt.Equal(end))
+			require.True(t, getOnlineSession(t, r.open[1].id).Ongoing)
+		})
+	}
+}

service/singleton/online_history_restore_test.go

@@ -0,0 +1,73 @@
+package singleton
+
+import (
+	"testing"
+	"time"
+
+	"github.com/nezhahq/nezha/model"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRestoreOngoingSessionsReconcilesRecentDuplicates(t *testing.T) {
+	for _, failClose := range []bool{false, true} {
+		name := "successful_close"
+		if failClose {
+			name = "retry_failed_close"
+		}
+		t.Run(name, func(t *testing.T) {
+			setupOnlineHistoryTestDB(t)
+			base := time.Now().Add(-10 * time.Second).Truncate(time.Second)
+			// Insert out of chronological order: choosing by ID alone would keep the wrong session.
+			newest := model.OnlineSession{ServerID: 1, StartedAt: base.Add(4 * time.Second), LastActiveAt: base.Add(5 * time.Second), Ongoing: true}
+			oldest := model.OnlineSession{ServerID: 1, StartedAt: base, LastActiveAt: base.Add(time.Second), Ongoing: true}
+			middle := model.OnlineSession{ServerID: 1, StartedAt: base.Add(2 * time.Second), LastActiveAt: base.Add(3 * time.Second), Ongoing: true}
+			for _, session := range []*model.OnlineSession{&newest, &oldest, &middle} {
+				require.NoError(t, DB.Create(session).Error)
+			}
+			if failClose {
+				injectCloseUpdateFailure(t)
+			}
+			r := NewOnlineHistoryRecorder()
+			require.Len(t, r.open, 1)
+			require.Equal(t, newest.ID, r.open[1].id)
+			if failClose {
+				require.Len(t, r.closing[1], 2)
+				require.Equal(t, int64(3), countOnlineSessions(t, "ongoing = ?", true))
+				dropCloseUpdateFailure(t)
+			}
+			r.SweepStale(base.Add(6*time.Second), func(uint64) (time.Time, bool) { return newest.LastActiveAt, true })
+			require.Empty(t, r.closing)
+			require.True(t, getOnlineSession(t, newest.ID).Ongoing)
+			for _, previous := range []model.OnlineSession{oldest, middle} {
+				stored := getOnlineSession(t, previous.ID)
+				require.False(t, stored.Ongoing)
+				require.True(t, stored.EndedAt.Equal(previous.LastActiveAt))
+			}
+			r.RecordOnline(1, base.Add(7*time.Second))
+			require.Equal(t, newest.ID, r.open[1].id)
+			require.Equal(t, int64(3), countOnlineSessions(t, "server_id = ?", 1))
+			r.SweepStale(base.Add(2*model.OnlineActiveWindow), func(uint64) (time.Time, bool) { return time.Time{}, false })
+			require.Empty(t, r.open)
+			require.Empty(t, r.closing)
+			require.Equal(t, int64(0), countOnlineSessions(t, "ongoing = ?", true))
+		})
+	}
+}
+
+func TestRestoreOngoingSessionsUsesIDToBreakStartTimeTies(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	start := time.Now().Add(-5 * time.Second)
+	first := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true}
+	second := first
+	other := model.OnlineSession{ServerID: 2, StartedAt: start, LastActiveAt: start, Ongoing: true}
+	for _, session := range []*model.OnlineSession{&first, &second, &other} {
+		require.NoError(t, DB.Create(session).Error)
+	}
+	r := NewOnlineHistoryRecorder()
+	require.Len(t, r.open, 2)
+	require.Equal(t, second.ID, r.open[1].id)
+	require.Equal(t, other.ID, r.open[2].id)
+	require.False(t, getOnlineSession(t, first.ID).Ongoing)
+	require.True(t, getOnlineSession(t, first.ID).EndedAt.Equal(start))
+	require.Equal(t, int64(2), countOnlineSessions(t, "ongoing = ?", true))
+}

service/singleton/online_history_retention_test.go

@@ -0,0 +1,63 @@
+package singleton
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/nezhahq/nezha/model"
+	"github.com/stretchr/testify/require"
+)
+
+func TestOnlineHistoryPruneWaitsForRetentionCommit(t *testing.T) {
+	for _, saveFails := range []bool{true, false} {
+		t.Run(map[bool]string{true: "failed_save", false: "successful_save"}[saveFails], func(t *testing.T) {
+			setupOnlineHistoryTestDB(t)
+			previous := Conf
+			Conf = &ConfigClass{Config: &model.Config{}}
+			Conf.OnlineStatsDays = 30
+			t.Cleanup(func() { Conf = previous })
+			now := time.Now().UTC()
+			row := model.OnlineSession{ServerID: 1, StartedAt: now.AddDate(0, 0, -15), EndedAt: now.AddDate(0, 0, -14)}
+			require.NoError(t, DB.Create(&row).Error)
+			saveEntered := make(chan struct{})
+			releaseSave := make(chan struct{})
+			saved := make(chan error, 1)
+			days := 7
+			go func() {
+				saved <- Conf.CommitOnlineHistoryRetention(&days, func() error {
+					close(saveEntered)
+					<-releaseSave
+					if saveFails {
+						return errors.New("retention save unavailable")
+					}
+					return nil
+				})
+			}()
+			<-saveEntered
+			pruneStarted := make(chan struct{})
+			pruned := make(chan struct{})
+			go func() { close(pruneStarted); PruneOnlineSessions(); close(pruned) }()
+			<-pruneStarted
+			premature := false
+			select {
+			case <-pruned:
+				premature = true
+			case <-time.After(100 * time.Millisecond):
+			}
+			close(releaseSave)
+			err := <-saved
+			<-pruned
+			require.False(t, premature, "pruning must not read an uncommitted retention policy")
+			if saveFails {
+				require.ErrorContains(t, err, "retention save unavailable")
+				require.Equal(t, 30, Conf.OnlineHistoryRetentionDays())
+				require.Equal(t, int64(1), countOnlineSessions(t, "id = ?", row.ID))
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, 7, Conf.OnlineHistoryRetentionDays())
+				require.Equal(t, int64(0), countOnlineSessions(t, "id = ?", row.ID))
+			}
+		})
+	}
+}

service/singleton/online_history_test.go

@@ -0,0 +1,316 @@
+package singleton
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"gorm.io/gorm"
+
+	"github.com/nezhahq/nezha/model"
+)
+
+func setupOnlineHistoryTestDB(t *testing.T) {
+	t.Helper()
+
+	previousDB := DB
+	var err error
+	DB, err = gorm.Open(openSQLiteDialector(filepath.Join(t.TempDir(), "dashboard.sqlite")), &gorm.Config{})
+	require.NoError(t, err)
+	sqlDB, err := DB.DB()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		DB = previousDB
+		if err := sqlDB.Close(); err != nil {
+			t.Errorf("close online history test database: %v", err)
+		}
+	})
+
+	require.NoError(t, DB.AutoMigrate(&model.Server{}, &model.OnlineSession{}))
+	require.NoError(t, DB.Exec("INSERT INTO servers (id, name, uuid) VALUES (1, 'server-1', 'online-history-test-1')").Error)
+}
+
+func getOnlineSession(t *testing.T, id uint64) model.OnlineSession {
+	t.Helper()
+	var s model.OnlineSession
+	require.NoError(t, DB.First(&s, id).Error)
+	return s
+}
+
+func countOnlineSessions(t *testing.T, cond string, args ...any) int64 {
+	t.Helper()
+	var count int64
+	require.NoError(t, DB.Model(&model.OnlineSession{}).Where(cond, args...).Count(&count).Error)
+	return count
+}
+
+// injectCloseUpdateFailure 用 SQLite 触发器注入"SELECT 正常、仅关闭 UPDATE 失败"：
+// 只拦截把 ongoing 置 0 的更新，LastActiveAt 节流写与 INSERT 均不受影响，
+// 使后续 RecordOnline/恢复/清扫全部走真实执行路径。
+func injectCloseUpdateFailure(t *testing.T) {
+	t.Helper()
+	require.NoError(t, DB.Exec(`CREATE TRIGGER fail_online_session_close
+BEFORE UPDATE ON online_sessions
+WHEN NEW.ongoing = 0
+BEGIN
+    SELECT RAISE(ABORT, 'injected close failure');
+END`).Error)
+	t.Cleanup(func() { dropCloseUpdateFailure(t) })
+}
+
+func dropCloseUpdateFailure(t *testing.T) {
+	t.Helper()
+	require.NoError(t, DB.Exec("DROP TRIGGER IF EXISTS fail_online_session_close").Error)
+}
+
+// 评审回归（P2：关库失败丢边界/重连续接待关闭会话）：RecordOffline 关库失败时
+// 必须转入待关闭状态——会话 ID 与已观察到的断线时刻固定、后续上报不得覆盖；
+// 重连开启的是新会话而非续接旧行；失败恢复后由清扫周期按固定边界完成持久化，
+// 离线间隔不计入在线。时间自单一基准单调推进。
+func TestRecordOfflineCloseFailurePreservesBoundary(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	r := NewOnlineHistoryRecorder()
+
+	base := time.Now().Add(-time.Hour).Truncate(time.Second)
+	atStart := base
+	atLastActive := atStart.Add(model.OnlineActiveWindow) // 恰好达到节流窗口，触发落库
+	atOffline := atLastActive.Add(15 * time.Second)       // 观察到的断线时刻
+	atReconnect := atOffline.Add(10 * time.Minute)
+
+	r.RecordOnline(1, atStart)
+	r.RecordOnline(1, atLastActive)
+	require.Len(t, r.open, 1)
+	first := getOnlineSession(t, r.open[1].id)
+	require.WithinDuration(t, atLastActive, first.LastActiveAt, time.Second)
+
+	// 注入关闭失败：行保持 Ongoing 等待重试，会话转入待关闭而非继续当活跃会话
+	injectCloseUpdateFailure(t)
+	r.RecordOffline(1, atOffline)
+	require.NotContains(t, r.open, uint64(1))
+	stuck := getOnlineSession(t, first.ID)
+	require.True(t, stuck.Ongoing)
+	require.True(t, stuck.EndedAt.IsZero(), "关库失败时旧行必须保持未闭合等待重试")
+
+	// 失败未恢复时 agent 重连：旧行不得被续接，必须开启新会话
+	r.RecordOnline(1, atReconnect)
+	require.Contains(t, r.open, uint64(1))
+	require.NotEqual(t, first.ID, r.open[1].id, "待关闭会话不得被续接为活跃会话")
+	afterRetry := getOnlineSession(t, first.ID)
+	require.True(t, afterRetry.Ongoing, "失败未恢复时重试仍失败，旧行保持待关闭")
+	second := getOnlineSession(t, r.open[1].id)
+	require.True(t, second.Ongoing)
+	require.WithinDuration(t, atReconnect, second.StartedAt, time.Second)
+
+	// 失败恢复后清扫：旧行按观察到的断线时刻闭合，新会话仍活跃不受影响
+	dropCloseUpdateFailure(t)
+	r.SweepStale(atReconnect.Add(time.Second), func(uint64) (time.Time, bool) {
+		return atReconnect, true
+	})
+
+	closed := getOnlineSession(t, first.ID)
+	require.False(t, closed.Ongoing)
+	require.WithinDuration(t, atOffline, closed.EndedAt, time.Second, "旧行必须按观察到的断线时刻闭合")
+	require.WithinDuration(t, atStart, closed.StartedAt, time.Second)
+	stillOngoing := getOnlineSession(t, second.ID)
+	require.True(t, stillOngoing.Ongoing)
+
+	// 离线间隔不计入在线：两行在 [atStart, atReconnect) 内的在线合计止于断线时刻
+	var rows []model.OnlineSession
+	require.NoError(t, DB.Where("server_id = ?", 1).Order("id").Find(&rows).Error)
+	require.Len(t, rows, 2)
+	var online time.Duration
+	for _, s := range rows {
+		end := s.EndedAt
+		if s.Ongoing {
+			end = atReconnect
+		}
+		online += end.Sub(s.StartedAt)
+	}
+	require.Equal(t, atOffline.Sub(atStart), online, "离线间隔不得计作在线")
+}
+
+// 评审回归（P2 启动恢复路径）：重启恢复对超窗会话结转关库失败时，条目必须
+// 进入待关闭状态（经真实 NewOnlineHistoryRecorder 恢复路径验证，而非手工
+// 构造内存状态）；重连不得续接该行，清扫周期按固定边界时刻完成持久化。
+func TestRestoreStaleCloseFailureEntersPendingClose(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+
+	// 预置一条超窗 Ongoing 行（dashboard 停机前遗留），时间自单一基准单调推进
+	base := time.Now().Add(-3 * time.Hour).Truncate(time.Second)
+	staleAt := base.Add(time.Hour)
+	atReconnect := base.Add(2 * time.Hour)
+	legacy := model.OnlineSession{ServerID: 1, StartedAt: base, LastActiveAt: staleAt, Ongoing: true}
+	require.NoError(t, DB.Create(&legacy).Error)
+
+	injectCloseUpdateFailure(t)
+	r := NewOnlineHistoryRecorder()
+
+	stuck := getOnlineSession(t, legacy.ID)
+	require.True(t, stuck.Ongoing, "恢复结转关库失败时行必须保持 Ongoing 等待重试")
+
+	// 重连：待关闭的超窗会话不得被续接，开启新会话
+	r.RecordOnline(1, atReconnect)
+	require.Contains(t, r.open, uint64(1))
+	require.NotEqual(t, legacy.ID, r.open[1].id, "待关闭会话不得被续接为活跃会话")
+
+	dropCloseUpdateFailure(t)
+	r.SweepStale(atReconnect.Add(time.Second), func(uint64) (time.Time, bool) {
+		return atReconnect, true
+	})
+
+	closed := getOnlineSession(t, legacy.ID)
+	require.False(t, closed.Ongoing)
+	require.WithinDuration(t, staleAt, closed.EndedAt, time.Second, "旧行必须按恢复时观察到的边界闭合")
+	require.Equal(t, int64(2), countOnlineSessions(t, "server_id = ?", 1))
+	require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
+}
+
+func TestRecordOnlineOpensSingleSession(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	r := NewOnlineHistoryRecorder()
+
+	now := time.Now()
+	r.RecordOnline(1, now)
+	r.RecordOnline(1, now.Add(time.Second))
+
+	require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ?", 1))
+	require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
+}
+
+func TestRecordOnlinePersistsLastActiveThrottled(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	r := NewOnlineHistoryRecorder()
+
+	now := time.Now()
+	r.RecordOnline(1, now)
+	// 窗口内的重复上报不应触发落库
+	r.RecordOnline(1, now.Add(5*time.Second))
+	s := r.open[1]
+	stored := getOnlineSession(t, s.id)
+	require.WithinDuration(t, now, stored.LastActiveAt, time.Second)
+
+	// 超过窗口后落库
+	r.RecordOnline(1, now.Add(model.OnlineActiveWindow+time.Second))
+	stored = getOnlineSession(t, s.id)
+	require.WithinDuration(t, now.Add(model.OnlineActiveWindow+time.Second), stored.LastActiveAt, time.Second)
+}
+
+func TestRecordOfflineClosesSession(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	r := NewOnlineHistoryRecorder()
+
+	start := time.Now()
+	r.RecordOnline(1, start)
+	r.RecordOffline(1, start.Add(10*time.Minute))
+	// 重复关闭应为无操作
+	r.RecordOffline(1, start.Add(11*time.Minute))
+
+	require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ?", 1))
+	require.Equal(t, int64(0), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
+	var s model.OnlineSession
+	require.NoError(t, DB.First(&s).Error)
+	require.WithinDuration(t, start.Add(10*time.Minute), s.EndedAt, time.Second)
+	require.WithinDuration(t, start, s.StartedAt, time.Second)
+}
+
+func TestRecordOnlineContinuesSessionAfterRestart(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+
+	// 模拟重启前遗留的进行中会话（最后活跃时间在在线窗口内：dashboard 只是短暂重启）
+	now := time.Now()
+	legacy := model.OnlineSession{ServerID: 1, StartedAt: now.Add(-time.Minute), LastActiveAt: now.Add(-5 * time.Second), Ongoing: true}
+	require.NoError(t, DB.Create(&legacy).Error)
+
+	// 新 recorder（构造时从库恢复）：新近会话应保留并续接，不产生新会话
+	r := NewOnlineHistoryRecorder()
+	r.RecordOnline(1, now)
+
+	require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ?", 1))
+	continued := getOnlineSession(t, legacy.ID)
+	require.True(t, continued.Ongoing)
+	require.WithinDuration(t, now.Add(-time.Minute), continued.StartedAt, time.Second)
+}
+
+func TestRestoreOngoingSessionsClosesStaleSessions(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+
+	// 上次活跃时间远早于在线窗口：重启恢复时按最后已知活跃时刻结转
+	staleStart := time.Now().Add(-time.Hour)
+	stale := model.OnlineSession{ServerID: 1, StartedAt: staleStart, LastActiveAt: staleStart.Add(30 * time.Minute), Ongoing: true}
+	require.NoError(t, DB.Create(&stale).Error)
+
+	r := NewOnlineHistoryRecorder()
+	r.RecordOnline(1, time.Now())
+
+	closed := getOnlineSession(t, stale.ID)
+	require.False(t, closed.Ongoing)
+	require.WithinDuration(t, staleStart.Add(30*time.Minute), closed.EndedAt, time.Second)
+	// 结转后新的上报开启新会话
+	require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
+}
+
+func TestSweepStaleClosesOnlyStaleSessions(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	r := NewOnlineHistoryRecorder()
+
+	now := time.Now()
+	r.RecordOnline(1, now.Add(-time.Hour))
+	// 第二台服务器：会话在 DB 但没有 servers 记录（服务器已被删除），手工放进内存 map
+	require.NoError(t, DB.Exec("INSERT INTO online_sessions (server_id, started_at, last_active_at, ongoing, created_at, updated_at) VALUES (2, ?, ?, 1, ?, ?)", now.Add(-time.Hour), now.Add(-time.Hour), now, now).Error)
+	var second model.OnlineSession
+	require.NoError(t, DB.Where("server_id = ?", 2).First(&second).Error)
+	r.mu.Lock()
+	r.open[2] = &onlineSessionRuntime{id: second.ID, lastPersist: now.Add(-time.Hour)}
+	r.mu.Unlock()
+
+	// 服务器 1 活跃，服务器 2 不存在 → 仅关闭服务器 2 的会话
+	lastActive := func(serverID uint64) (time.Time, bool) {
+		if serverID == 1 {
+			return now, true
+		}
+		return time.Time{}, false
+	}
+	r.SweepStale(now, lastActive)
+
+	require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
+	require.Equal(t, int64(0), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 2, true))
+
+	// 服务器 1 失联后也会被关闭，结束时刻为最后已知活跃时间
+	r.SweepStale(now.Add(model.OnlineActiveWindow*2), func(uint64) (time.Time, bool) {
+		return now, true
+	})
+	require.Equal(t, int64(0), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
+	var closedFirst = getOnlineSession(t, func() uint64 {
+		var s model.OnlineSession
+		require.NoError(t, DB.Where("server_id = ?", 1).First(&s).Error)
+		return s.ID
+	}())
+	require.WithinDuration(t, now, closedFirst.EndedAt, time.Second)
+}
+
+func TestPruneOnlineSessions(t *testing.T) {
+	setupOnlineHistoryTestDB(t)
+	previousConf := Conf
+	Conf = &ConfigClass{Config: &model.Config{}}
+	Conf.OnlineStatsDays = 7
+	t.Cleanup(func() { Conf = previousConf })
+
+	now := time.Now()
+	old := now.AddDate(0, 0, -30)
+	rows := []model.OnlineSession{
+		{ServerID: 1, StartedAt: old, EndedAt: old.Add(time.Hour), LastActiveAt: old.Add(time.Hour)},   // 超窗口 → 清理
+		{ServerID: 1, StartedAt: old, EndedAt: now.Add(-time.Hour), LastActiveAt: now.Add(-time.Hour)}, // 跨窗口边界 → 保留
+		{ServerID: 1, StartedAt: now.Add(-time.Hour), EndedAt: now, LastActiveAt: now},                 // 窗口内 → 保留
+		{ServerID: 1, StartedAt: now, Ongoing: true, LastActiveAt: now},                                // 进行中 → 保留
+		{ServerID: 99, StartedAt: now.Add(-time.Hour), EndedAt: now, LastActiveAt: now},                // 服务器已删除 → 清理
+	}
+	for i := range rows {
+		require.NoError(t, DB.Create(&rows[i]).Error)
+	}
+
+	PruneOnlineSessions()
+
+	require.Equal(t, int64(3), countOnlineSessions(t, "1 = 1"))
+	require.Equal(t, int64(1), countOnlineSessions(t, "ongoing = ?", true))
+}

service/singleton/server.go

@@ -117,6 +117,7 @@ func (c *ServerClass) Update(s *model.Server, uuid string) {
 func (c *ServerClass) Delete(idList []uint64) {
 	c.lockLifecycleWrite()
 	defer c.unlockLifecycleWrite()
+	cancelAlertDeliveriesForServers(idList)
 
 	c.listMu.Lock()
 

service/singleton/server_transfer.go

@@ -152,6 +152,8 @@ type ServerTransferClass struct {
 
 	subMu     sync.Mutex
 	subs      map[uint64]chan *model.ServerTransfer
+	subOwners map[uint64]uint64
+	subCounts map[uint64]int
 	nextSubID uint64
 
 	timeout  time.Duration
@@ -174,6 +176,18 @@ var ErrServerAlreadyTransferring = errors.New("server already has an in-flight t
 // deferred to OnAgentReconnect / PushIfOnline.
 var ErrAgentTooOldForTransfer = fmt.Errorf("agent build older than %s does not support server transfer (TaskTypeServerTransferApply)", MinServerTransferAgentVersion)
 
+const (
+	// A normal dashboard needs one subscription per open transfer page. Keep a
+	// small per-user allowance for tabs/devices plus a process-wide hard stop.
+	maxServerTransferSubscribersPerUser = 8
+	maxServerTransferSubscribersGlobal  = 1024
+)
+
+var (
+	errTooManyServerTransferSubscribersForUser = errors.New("too many concurrent server transfer subscribers for this user")
+	errTooManyServerTransferSubscribersGlobal  = errors.New("too many concurrent server transfer subscribers")
+)
+
 // agentSupportsTransfer reports whether s has reported a build version >=
 // MinServerTransferAgentVersion. Returns true when version is unknown (agent
 // never reported) so callers can defer the decision; PushIfOnline re-checks
@@ -210,6 +224,8 @@ func NewServerTransferClass() *ServerTransferClass {
 		verifiedHandshakes:     make(map[uint64]string),
 		initiating:             make(map[uint64]bool),
 		subs:                   make(map[uint64]chan *model.ServerTransfer),
+		subOwners:              make(map[uint64]uint64),
+		subCounts:              make(map[uint64]int),
 		timeout:                defaultServerTransferTimeout,
 		stopCh:                 make(chan struct{}),
 	}
@@ -1410,20 +1426,37 @@ func (c *ServerTransferClass) sweepTimeouts() {
 // Subscribe registers a channel that will receive every transfer transition
 // event from this point forward. The caller MUST Unsubscribe when done or
 // the broker will block forever if the channel is unbuffered or full.
-func (c *ServerTransferClass) Subscribe() (uint64, <-chan *model.ServerTransfer) {
+func (c *ServerTransferClass) Subscribe(userID uint64) (uint64, <-chan *model.ServerTransfer, error) {
 	c.subMu.Lock()
 	defer c.subMu.Unlock()
+	if len(c.subs) >= maxServerTransferSubscribersGlobal {
+		return 0, nil, errTooManyServerTransferSubscribersGlobal
+	}
+	if c.subCounts[userID] >= maxServerTransferSubscribersPerUser {
+		return 0, nil, errTooManyServerTransferSubscribersForUser
+	}
 
 	id := atomic.AddUint64(&c.nextSubID, 1)
 	ch := make(chan *model.ServerTransfer, 16)
 	c.subs[id] = ch
-	return id, ch
+	c.subOwners[id] = userID
+	c.subCounts[userID]++
+	return id, ch, nil
 }
 
 func (c *ServerTransferClass) Unsubscribe(id uint64) {
 	c.subMu.Lock()
 	ch, ok := c.subs[id]
+	ownerID, ownerOK := c.subOwners[id]
 	delete(c.subs, id)
+	delete(c.subOwners, id)
+	if ownerOK {
+		if c.subCounts[ownerID] <= 1 {
+			delete(c.subCounts, ownerID)
+		} else {
+			c.subCounts[ownerID]--
+		}
+	}
 	c.subMu.Unlock()
 	if ok {
 		close(ch)

service/singleton/server_transfer_test.go

@@ -455,7 +455,8 @@ func TestServerTransferBroadcastReachesSubscribers(t *testing.T) {
 	defer cleanup()
 	seedServerForTransfer(t, 1, 100)
 
-	id, ch := c.Subscribe()
+	id, ch, err := c.Subscribe(100)
+	require.NoError(t, err)
 	defer c.Unsubscribe(id)
 
 	tr := initiateAndRegister(t, c, 1, 100, 200, 1)
@@ -469,7 +470,7 @@ func TestServerTransferBroadcastReachesSubscribers(t *testing.T) {
 		t.Fatal("expected Pending broadcast within 1s")
 	}
 
-	_, _, err := markPendingVerified(t, c, 1)
+	_, _, err = markPendingVerified(t, c, 1)
 	require.NoError(t, err)
 	select {
 	case ev := <-ch:
@@ -479,6 +480,57 @@ func TestServerTransferBroadcastReachesSubscribers(t *testing.T) {
 	}
 }
 
+func TestServerTransferSubscribeEnforcesPerUserLimitAndReleasesQuota(t *testing.T) {
+	c, cleanup := setupTransferFixture(t)
+	defer cleanup()
+
+	ids := make([]uint64, 0, maxServerTransferSubscribersPerUser)
+	for range maxServerTransferSubscribersPerUser {
+		id, ch, err := c.Subscribe(100)
+		require.NoError(t, err)
+		require.NotNil(t, ch)
+		ids = append(ids, id)
+	}
+
+	_, _, err := c.Subscribe(100)
+	require.ErrorIs(t, err, errTooManyServerTransferSubscribersForUser)
+
+	otherID, _, err := c.Subscribe(200)
+	require.NoError(t, err, "one user's quota must not block another user")
+	c.Unsubscribe(otherID)
+
+	c.Unsubscribe(ids[0])
+	replacementID, _, err := c.Subscribe(100)
+	require.NoError(t, err, "disconnecting must release the user's quota")
+	c.Unsubscribe(replacementID)
+	for _, id := range ids[1:] {
+		c.Unsubscribe(id)
+	}
+}
+
+func TestServerTransferSubscribeEnforcesGlobalLimit(t *testing.T) {
+	c, cleanup := setupTransferFixture(t)
+	defer cleanup()
+
+	ids := make([]uint64, 0, maxServerTransferSubscribersGlobal)
+	for userID := uint64(1); len(ids) < maxServerTransferSubscribersGlobal; userID++ {
+		for range maxServerTransferSubscribersPerUser {
+			if len(ids) == maxServerTransferSubscribersGlobal {
+				break
+			}
+			id, _, err := c.Subscribe(userID)
+			require.NoError(t, err)
+			ids = append(ids, id)
+		}
+	}
+
+	_, _, err := c.Subscribe(999999)
+	require.ErrorIs(t, err, errTooManyServerTransferSubscribersGlobal)
+	for _, id := range ids {
+		c.Unsubscribe(id)
+	}
+}
+
 // The "one active transfer per server" invariant must hold under concurrent
 // callers (two operators batch-moving the same server, or batch-move racing
 // retry). The old flow had a TOCTOU between HasPending() and Initiate() that
@@ -1626,7 +1678,8 @@ func TestOnServersDeletedTerminatesPendingTransfersAndClearsIndexes(t *testing.T
 	tr := initiateAndRegister(t, c, 1, 100, 200, 1)
 	require.True(t, c.HasPending(1))
 
-	subID, ch := c.Subscribe()
+	subID, ch, err := c.Subscribe(100)
+	require.NoError(t, err)
 	defer c.Unsubscribe(subID)
 
 	require.NoError(t, DB.Unscoped().Delete(&model.Server{}, tr.ServerID).Error)

service/singleton/servicesentinel.go

@@ -2,6 +2,7 @@ package singleton
 
 import (
 	"cmp"
+	"crypto/sha256"
 	"fmt"
 	"iter"
 	"log"
@@ -9,6 +10,7 @@ import (
 	"slices"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/gin-gonic/gin"
@@ -22,9 +24,11 @@ import (
 )
 
 const (
-	_CurrentStatusSize = 30 // 统计 15 分钟内的数据为当前状态
+	_CurrentStatusSize = 30 // initial capacity for the rolling service window
 )
 
+var serviceEventSequence atomic.Uint64
+
 type serviceResponseItem struct {
 	model.ServiceResponseItem
 
@@ -46,9 +50,101 @@ type _TodayStatsOfService struct {
 type serviceResponseData = _TodayStatsOfService
 
 type serviceTaskStatus struct {
-	lastStatus uint8
-	t          time.Time
-	result     []*pb.TaskResult
+	lastStatus    uint8
+	stateEpoch    uint64
+	lastHistoryAt time.Time
+	history       serviceResponseData
+	lastData      string
+	result        []serviceWindowBucket
+}
+
+type serviceProbeSample struct {
+	successful bool
+	failed     bool
+	delay      float64
+}
+
+type serviceWindowBucket struct {
+	at        time.Time
+	reporters map[uint64]serviceProbeSample
+}
+
+func (status *serviceTaskStatus) appendResult(now time.Time, reporter uint64, result *pb.TaskResult) {
+	bucketAt := now.Truncate(30 * time.Second)
+	if len(status.result) > 0 && bucketAt.Before(status.result[len(status.result)-1].at) {
+		status.result = nil // wall-clock moved backwards; old buckets are not comparable
+	}
+	if len(status.result) == 0 || !status.result[len(status.result)-1].at.Equal(bucketAt) {
+		status.result = append(status.result, serviceWindowBucket{at: bucketAt, reporters: make(map[uint64]serviceProbeSample)})
+	}
+	bucket := &status.result[len(status.result)-1]
+	sample := bucket.reporters[reporter]
+	if result.Successful {
+		sample.successful = true
+		sample.delay = float64(result.Delay)
+	} else {
+		sample.failed = true
+	}
+	bucket.reporters[reporter] = sample
+	cutoff := now.Add(-15 * time.Minute)
+	first := 0
+	for first < len(status.result) && status.result[first].at.Before(cutoff) {
+		first++
+	}
+	if first > 0 {
+		status.result = append([]serviceWindowBucket(nil), status.result[first:]...)
+	}
+	if result.Successful {
+		status.history.Up++
+		status.history.Delay = (status.history.Delay*float64(status.history.Up-1) + float64(result.Delay)) / float64(status.history.Up)
+	} else {
+		status.history.Down++
+	}
+	status.lastData = result.Data
+}
+
+// flushServiceHistory persists the previous non-overlapping reporting period
+// before the next observation is added, so long probe gaps cannot lose or
+// double-count samples in the 15-minute availability history.
+func (ss *ServiceSentinel) flushServiceHistory(serviceID uint64, status *serviceTaskStatus, now time.Time) {
+	if status.lastHistoryAt.IsZero() || now.Before(status.lastHistoryAt.Add(15*time.Minute)) {
+		return
+	}
+	if status.history.Up+status.history.Down == 0 {
+		status.lastHistoryAt = now
+		return
+	}
+	if !TSDBEnabled() {
+		if err := DB.Create(&model.ServiceHistory{
+			CreatedAt: status.lastHistoryAt,
+			ServiceID: serviceID,
+			AvgDelay:  status.history.Delay,
+			Data:      status.lastData,
+			Up:        status.history.Up,
+			Down:      status.history.Down,
+		}).Error; err != nil {
+			log.Printf("NEZHA>> Failed to save service monitor metrics: %v", err)
+			return
+		}
+	}
+	status.history = serviceResponseData{}
+	status.lastHistoryAt = now
+}
+
+func (status *serviceTaskStatus) responseData() serviceResponseData {
+	var data serviceResponseData
+	for _, bucket := range status.result {
+		for _, sample := range bucket.reporters {
+			if sample.successful {
+				data.Up++
+				data.Delay = (data.Delay*float64(data.Up-1) + sample.delay) / float64(data.Up)
+			}
+			if sample.failed {
+				data.Down++
+			}
+		}
+	}
+	return data
 }
 
 type pingStore struct {
@@ -228,7 +324,7 @@ func (ss *ServiceSentinel) loadServiceHistory() error {
 		}
 		ss.services[service.ID] = service
 		ss.serviceCurrentStatusData[service.ID] = new(serviceTaskStatus)
-		ss.serviceCurrentStatusData[service.ID].result = make([]*pb.TaskResult, 0, _CurrentStatusSize)
+		ss.serviceCurrentStatusData[service.ID].result = make([]serviceWindowBucket, 0, _CurrentStatusSize)
 		ss.serviceStatusToday[service.ID] = &_TodayStatsOfService{}
 		validServices = append(validServices, service)
 	}
@@ -335,10 +431,18 @@ func (ss *ServiceSentinel) loadTodayStats(today time.Time) {
 		totalDelay := make(map[uint64]float64)
 		totalDelayCount := make(map[uint64]int)
 		for _, mh := range mhs {
-			ss.serviceStatusToday[mh.ServiceID].Up += mh.Up
-			ss.monthlyStatus[mh.ServiceID].TotalUp += mh.Up
-			ss.serviceStatusToday[mh.ServiceID].Down += mh.Down
-			ss.monthlyStatus[mh.ServiceID].TotalDown += mh.Down
+			todayStats := ss.serviceStatusToday[mh.ServiceID]
+			monthlyStats := ss.monthlyStatus[mh.ServiceID]
+			// History can outlive a deleted service or belong to a quarantined
+			// service. Neither has startup status entries; skip its rows before
+			// accumulating counts or delays so retained history cannot panic startup.
+			if todayStats == nil || monthlyStats == nil {
+				continue
+			}
+			todayStats.Up += mh.Up
+			monthlyStats.TotalUp += mh.Up
+			todayStats.Down += mh.Down
+			monthlyStats.TotalDown += mh.Down
 			totalDelay[mh.ServiceID] += mh.AvgDelay
 			totalDelayCount[mh.ServiceID]++
 		}
@@ -371,6 +475,9 @@ func (ss *ServiceSentinel) Update(m *model.Service) error {
 	if err != nil {
 		return err
 	}
+	// A service edit may disable notifications or change their recipients.
+	// Cancel any retry worker holding the previous service configuration.
+	NotificationShared.CancelServiceState(m.ID)
 	if ss.services[m.ID] != nil {
 		// 停掉旧任务
 		CronShared.Remove(ss.services[m.ID].CronJobID)
@@ -387,7 +494,7 @@ func (ss *ServiceSentinel) Update(m *model.Service) error {
 		if ss.serviceCurrentStatusData[m.ID] == nil {
 			ss.serviceCurrentStatusData[m.ID] = new(serviceTaskStatus)
 		}
-		ss.serviceCurrentStatusData[m.ID].result = make([]*pb.TaskResult, 0, _CurrentStatusSize)
+		ss.serviceCurrentStatusData[m.ID].result = make([]serviceWindowBucket, 0, _CurrentStatusSize)
 		ss.serviceStatusToday[m.ID] = &_TodayStatsOfService{}
 	}
 	// 更新这个任务
@@ -404,6 +511,7 @@ func (ss *ServiceSentinel) Delete(ids []uint64) {
 	defer ss.servicesLock.Unlock()
 
 	for _, id := range ids {
+		NotificationShared.CancelServiceState(id)
 		delete(ss.serviceCurrentStatusData, id)
 		delete(ss.serviceResponseDataStore, id)
 		delete(ss.serviceResponsePing, id)
@@ -669,32 +777,16 @@ func (ss *ServiceSentinel) processReport(r ReportData, serverShared *ServerClass
 	}
 
 	currentTime := time.Now()
-	if serviceCurrentStatusData.t.IsZero() {
-		serviceCurrentStatusData.t = currentTime
-	}
-
-	// 写入当前数据
-	if serviceCurrentStatusData.t.Before(currentTime) {
-		serviceCurrentStatusData.t = currentTime.Add(30 * time.Second)
-		serviceCurrentStatusData.result = append(serviceCurrentStatusData.result, mh)
+	ss.flushServiceHistory(mh.GetId(), serviceCurrentStatusData, currentTime)
+	// Include every accepted reporter's result. A fixed count of samples is
+	// not a 15-minute window when probes have different intervals or reporters.
+	serviceCurrentStatusData.appendResult(currentTime, r.Reporter, mh)
+	if serviceCurrentStatusData.lastHistoryAt.IsZero() {
+		serviceCurrentStatusData.lastHistoryAt = currentTime
 	}
 
 	// 更新当前状态
-	ss.serviceResponseDataStore[mh.GetId()] = serviceResponseData{}
-
-	// 永远是最新的 30 个数据的状态 [01:00, 02:00, 03:00] -> [04:00, 02:00, 03: 00]
-	for _, cs := range serviceCurrentStatusData.result {
-		if cs.GetId() > 0 {
-			rd := ss.serviceResponseDataStore[mh.GetId()]
-			if cs.Successful {
-				rd.Up++
-				rd.Delay = (rd.Delay*float64(rd.Up-1) + float64(cs.Delay)) / float64(rd.Up)
-			} else {
-				rd.Down++
-			}
-			ss.serviceResponseDataStore[mh.GetId()] = rd
-		}
-	}
+	ss.serviceResponseDataStore[mh.GetId()] = serviceCurrentStatusData.responseData()
 
 	// 计算在线率，
 	var stateCode uint8
@@ -704,24 +796,11 @@ func (ss *ServiceSentinel) processReport(r ReportData, serverShared *ServerClass
 		if rd.Down+rd.Up > 0 {
 			upPercent = rd.Up * 100 / (rd.Down + rd.Up)
 		}
-		stateCode = GetStatusCode(upPercent)
-	}
-
-	if len(serviceCurrentStatusData.result) == _CurrentStatusSize {
-		serviceCurrentStatusData.t = currentTime
-		if !TSDBEnabled() {
-			rd := ss.serviceResponseDataStore[mh.GetId()]
-			if err := DB.Create(&model.ServiceHistory{
-				ServiceID: mh.GetId(),
-				AvgDelay:  rd.Delay,
-				Data:      mh.Data,
-				Up:        rd.Up,
-				Down:      rd.Down,
-			}).Error; err != nil {
-				log.Printf("NEZHA>> Failed to save service monitor metrics: %v", err)
-			}
+		if rd.Down+rd.Up == 0 {
+			stateCode = StatusNoData
+		} else {
+			stateCode = GetStatusCode(upPercent)
 		}
-		serviceCurrentStatusData.result = serviceCurrentStatusData.result[:0]
 	}
 
 	// 延迟报警
@@ -734,77 +813,100 @@ func (ss *ServiceSentinel) processReport(r ReportData, serverShared *ServerClass
 		lastStatus := serviceCurrentStatusData.lastStatus
 		// 存储新的状态值
 		serviceCurrentStatusData.lastStatus = stateCode
-
-		notifyCheck(&r, m, cs, mh, lastStatus, stateCode)
-	}
-
-	// TLS 证书报警
-	if ss.serviceReportBeforeTLSSideEffectsHook != nil {
-		ss.serviceReportBeforeTLSSideEffectsHook(mh.GetId())
-	}
-	var errMsg string
-	if strings.HasPrefix(mh.Data, "SSL证书错误：") {
-		// i/o timeout、connection timeout、EOF 错误
-		if !strings.HasSuffix(mh.Data, "timeout") &&
-			!strings.HasSuffix(mh.Data, "EOF") &&
-			!strings.HasSuffix(mh.Data, "timed out") {
-			errMsg = mh.Data
-			if cs.Notify {
-				muteLabel := NotificationMuteLabel.ServiceTLS(mh.GetId(), "network")
-				go NotificationShared.SendNotification(cs.NotificationGroupID, Localizer.Tf("[TLS] Fetch cert info failed, Reporter: %s, Error: %s", cs.Name, errMsg), muteLabel)
-			}
+		if stateCode != lastStatus {
+			serviceCurrentStatusData.stateEpoch = serviceEventSequence.Add(1)
 		}
-	} else {
-		// 清除网络错误静音缓存
-		NotificationShared.UnMuteNotification(cs.NotificationGroupID, NotificationMuteLabel.ServiceTLS(mh.GetId(), "network"))
 
-		var newCert = strings.Split(mh.Data, "|")
-		if len(newCert) > 1 {
-			enableNotify := cs.Notify
+		notifyCheck(&r, m, cs, mh, lastStatus, stateCode, serviceCurrentStatusData.stateEpoch)
+	}
 
-			// 首次获取证书信息时，缓存证书信息
-			if ss.tlsCertCache[mh.GetId()] == "" {
-				ss.tlsCertCache[mh.GetId()] = mh.Data
+	// TLS 证书报警仅适用于 HTTPS 探针。其它探针的任意响应文本
+	// 不得被当作证书资料解析。
+	if mh.Type == model.TaskTypeHTTPGet && strings.HasPrefix(strings.ToLower(cs.Target), "https://") {
+		if ss.serviceReportBeforeTLSSideEffectsHook != nil {
+			ss.serviceReportBeforeTLSSideEffectsHook(mh.GetId())
+		}
+		var errMsg string
+		if strings.HasPrefix(mh.Data, "SSL证书错误：") {
+			// i/o timeout、connection timeout、EOF 错误
+			if !strings.HasSuffix(mh.Data, "timeout") &&
+				!strings.HasSuffix(mh.Data, "EOF") &&
+				!strings.HasSuffix(mh.Data, "timed out") {
+				errMsg = mh.Data
+				if cs.Notify {
+					muteLabel := NotificationMuteLabel.ServiceTLS(mh.GetId(), "network")
+					NotificationShared.SendNotificationAsync(cs.NotificationGroupID, Localizer.Tf("[TLS] Fetch cert info failed, Reporter: %s, Error: %s", cs.Name, errMsg), muteLabel)
+				}
 			}
+		} else {
+			// 清除网络错误静音缓存
+			NotificationShared.UnMuteNotification(cs.NotificationGroupID, NotificationMuteLabel.ServiceTLS(mh.GetId(), "network"))
 
-			oldCert := strings.Split(ss.tlsCertCache[mh.GetId()], "|")
-			isCertChanged := false
-			expiresOld, _ := time.Parse("2006-01-02 15:04:05 -0700 MST", oldCert[1])
-			expiresNew, _ := time.Parse("2006-01-02 15:04:05 -0700 MST", newCert[1])
-
-			// 证书变更时，更新缓存
-			if oldCert[0] != newCert[0] && !expiresNew.Equal(expiresOld) {
-				isCertChanged = true
-				ss.tlsCertCache[mh.GetId()] = mh.Data
-			}
-
-			notificationGroupID := cs.NotificationGroupID
-			serviceName := cs.Name
-
-			// 需要发送提醒
-			if enableNotify {
-				// 证书过期提醒
-				if expiresNew.Before(time.Now().AddDate(0, 0, 7)) {
-					expiresTimeStr := expiresNew.Format("2006-01-02 15:04:05")
-					errMsg = Localizer.Tf(
-						"The TLS certificate will expire within seven days. Expiration time: %s",
-						expiresTimeStr,
-					)
-
-					// 静音规则： 服务id+证书过期时间
-					// 用于避免多个监测点对相同证书同时报警
-					muteLabel := NotificationMuteLabel.ServiceTLS(mh.GetId(), fmt.Sprintf("expire_%s", expiresTimeStr))
-					go NotificationShared.SendNotification(notificationGroupID, fmt.Sprintf("[TLS] %s %s", serviceName, errMsg), muteLabel)
+			var newCert = strings.Split(mh.Data, "|")
+			if len(newCert) > 1 {
+				enableNotify := cs.Notify
+				expiresNew, newErr := time.Parse("2006-01-02 15:04:05 -0700 MST", newCert[1])
+				if newErr != nil {
+					log.Printf("NEZHA>> Ignoring malformed TLS certificate expiry for service %d", mh.GetId())
+					return
 				}
 
-				// 证书变更提醒
-				if isCertChanged {
-					errMsg = Localizer.Tf(
-						"TLS certificate changed, old: issuer %s, expires at %s; new: issuer %s, expires at %s",
-						oldCert[0], expiresOld.Format("2006-01-02 15:04:05"), newCert[0], expiresNew.Format("2006-01-02 15:04:05"))
+				// 首次获取证书信息时，缓存证书信息
+				if ss.tlsCertCache[mh.GetId()] == "" {
+					ss.tlsCertCache[mh.GetId()] = mh.Data
+				}
 
-					// 证书变更后会自动更新缓存，所以不需要静音
-					go NotificationShared.SendNotification(notificationGroupID, fmt.Sprintf("[TLS] %s %s", serviceName, errMsg), "")
+				oldCert := strings.Split(ss.tlsCertCache[mh.GetId()], "|")
+				isCertChanged := false
+				expiresOld, oldErr := time.Parse("2006-01-02 15:04:05 -0700 MST", oldCert[1])
+				if oldErr != nil {
+					log.Printf("NEZHA>> Ignoring malformed TLS certificate expiry for service %d", mh.GetId())
+					return
+				}
+
+				// 只有通知成功后才推进基线；失败时下一次探针结果会重试。
+				if oldCert[0] != newCert[0] || !expiresNew.Equal(expiresOld) {
+					isCertChanged = true
+					if !enableNotify {
+						ss.tlsCertCache[mh.GetId()] = mh.Data
+					}
+				}
+
+				notificationGroupID := cs.NotificationGroupID
+				serviceName := cs.Name
+
+				// 需要发送提醒
+				if enableNotify {
+					// 证书过期提醒
+					if expiresNew.Before(time.Now().AddDate(0, 0, 7)) {
+						expiresTimeStr := expiresNew.Format("2006-01-02 15:04:05")
+						errMsg = Localizer.Tf(
+							"The TLS certificate will expire within seven days. Expiration time: %s",
+							expiresTimeStr,
+						)
+
+						// 静音规则： 服务id+证书过期时间
+						// 用于避免多个监测点对相同证书同时报警
+						muteLabel := NotificationMuteLabel.ServiceTLS(mh.GetId(), fmt.Sprintf("expire_%s", expiresTimeStr))
+						NotificationShared.SendNotificationAsync(notificationGroupID, fmt.Sprintf("[TLS] %s %s", serviceName, errMsg), muteLabel)
+					}
+
+					// 证书变更提醒
+					if isCertChanged {
+						errMsg = Localizer.Tf(
+							"TLS certificate changed, old: issuer %s, expires at %s; new: issuer %s, expires at %s",
+							oldCert[0], expiresOld.Format("2006-01-02 15:04:05"), newCert[0], expiresNew.Format("2006-01-02 15:04:05"))
+
+						oldRaw, newRaw, serviceID := ss.tlsCertCache[mh.GetId()], mh.Data, mh.GetId()
+						muteLabel := NotificationMuteLabel.ServiceTLS(serviceID, fmt.Sprintf("change_%x", sha256.Sum256([]byte(newRaw))))
+						NotificationShared.sendNotificationAsync(notificationGroupID, fmt.Sprintf("[TLS] %s %s", serviceName, errMsg), muteLabel, func() {
+							ss.serviceResponseDataStoreLock.Lock()
+							if ss.serviceCurrentStatusData[serviceID] != nil && ss.tlsCertCache[serviceID] == oldRaw {
+								ss.tlsCertCache[serviceID] = newRaw
+							}
+							ss.serviceResponseDataStoreLock.Unlock()
+						})
+					}
 				}
 			}
 		}
@@ -832,11 +934,11 @@ func delayCheck(r *ReportData, m map[uint64]*model.Server, ss *model.Service, mh
 	if mh.Delay > ss.MaxLatency {
 		// 延迟超过最大值
 		msg := Localizer.Tf("[Latency] %s %2f > %2f, Reporter: %s", ss.Name, mh.Delay, ss.MaxLatency, reporterServer.Name)
-		go NotificationShared.SendNotification(notificationGroupID, msg, minMuteLabel)
+		NotificationShared.SendNotificationAsync(notificationGroupID, msg, minMuteLabel)
 	} else if mh.Delay < ss.MinLatency {
 		// 延迟低于最小值
 		msg := Localizer.Tf("[Latency] %s %2f < %2f, Reporter: %s", ss.Name, mh.Delay, ss.MinLatency, reporterServer.Name)
-		go NotificationShared.SendNotification(notificationGroupID, msg, maxMuteLabel)
+		NotificationShared.SendNotificationAsync(notificationGroupID, msg, maxMuteLabel)
 	} else {
 		// 正常延迟， 清除静音缓存
 		NotificationShared.UnMuteNotification(notificationGroupID, minMuteLabel)
@@ -845,7 +947,7 @@ func delayCheck(r *ReportData, m map[uint64]*model.Server, ss *model.Service, mh
 }
 
 func notifyCheck(r *ReportData, m map[uint64]*model.Server,
-	ss *model.Service, mh *pb.TaskResult, lastStatus, stateCode uint8) {
+	ss *model.Service, mh *pb.TaskResult, lastStatus, stateCode uint8, stateEpoch uint64) {
 	// GHSA-jx78-55p5-rwv5: guard against concurrent server deletion (same TOCTOU
 	// class as the 2026-07-21 fix, a few dozen lines lower in the same worker).
 	// ServerShared has its own lock; m is a snapshot taken outside
@@ -858,14 +960,9 @@ func notifyCheck(r *ReportData, m map[uint64]*model.Server,
 	if isNeedSendNotification && reporterServer != nil {
 		notificationGroupID := ss.NotificationGroupID
 		notificationMsg := Localizer.Tf("[%s] %s Reporter: %s, Error: %s", StatusCodeToString(stateCode), ss.Name, reporterServer.Name, mh.Data)
-		muteLabel := NotificationMuteLabel.ServiceStateChanged(mh.GetId())
+		muteLabel := fmt.Sprintf("%s:%d:%d", NotificationMuteLabel.ServiceStateChanged(mh.GetId()), stateCode, stateEpoch)
 
-		// 状态变更时，清除静音缓存
-		if stateCode != lastStatus {
-			NotificationShared.UnMuteNotification(notificationGroupID, muteLabel)
-		}
-
-		go NotificationShared.SendNotification(notificationGroupID, notificationMsg, muteLabel)
+		NotificationShared.SendServiceState(mh.GetId(), notificationGroupID, notificationMsg, muteLabel)
 	}
 
 	// 判断是否需要触发任务
@@ -890,9 +987,6 @@ const (
 )
 
 func GetStatusCode[T constraints.Float | constraints.Integer](percent T) uint8 {
-	if percent == 0 {
-		return StatusNoData
-	}
 	if percent > 95 {
 		return StatusGood
 	}

service/singleton/servicesentinel_startup_history_test.go

@@ -0,0 +1,94 @@
+package singleton
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/nezhahq/nezha/model"
+)
+
+func TestServiceSentinelStartupWithRetainedHistory(t *testing.T) {
+	for _, serviceState := range []string{"retained", "deleted", "quarantined"} {
+		t.Run(serviceState, func(t *testing.T) {
+			initial := newServiceMonitorSecurityHarness(t)
+			initial.Close()
+			require.False(t, TSDBEnabled())
+
+			activeService := &model.Service{Name: "active monitor", Type: model.TaskTypeTCPPing}
+			historicalService := &model.Service{Name: "historical monitor", Type: model.TaskTypeHTTPGet}
+			require.NoError(t, DB.Create(activeService).Error)
+			require.NoError(t, DB.Create(historicalService).Error)
+
+			year, month, day := time.Now().Date()
+			today := time.Date(year, month, day, 0, 0, 0, 0, Loc)
+			// Use the production writer so retained rows have the same shape as
+			// ordinary aggregated history, including server_id = 0.
+			historicalStatus := &serviceTaskStatus{
+				lastHistoryAt: today,
+				history:       serviceResponseData{Up: 7, Down: 2, Delay: 90},
+			}
+			initial.flushServiceHistory(historicalService.ID, historicalStatus, today.Add(15*time.Minute))
+			var retainedHistory []model.ServiceHistory
+			require.NoError(t, DB.Where("service_id = ?", historicalService.ID).Find(&retainedHistory).Error)
+			require.Len(t, retainedHistory, 1)
+			require.Zero(t, retainedHistory[0].ServerID)
+			require.Equal(t, uint64(7), retainedHistory[0].Up)
+			require.Equal(t, uint64(2), retainedHistory[0].Down)
+			require.Equal(t, float64(90), retainedHistory[0].AvgDelay)
+
+			require.NoError(t, DB.Create(&[]model.ServiceHistory{
+				{ServiceID: activeService.ID, CreatedAt: today, Up: 5, Down: 1, AvgDelay: 10},
+				{ServiceID: activeService.ID, CreatedAt: today, Up: 3, Down: 2, AvgDelay: 30},
+				{ServiceID: activeService.ID, CreatedAt: today.Add(-time.Hour), Up: 4, Down: 1, AvgDelay: 15},
+				{ServiceID: activeService.ID, ServerID: 100, CreatedAt: today, Up: 100, AvgDelay: 500},
+			}).Error)
+
+			switch serviceState {
+			case "deleted":
+				// Match the controller's hard delete without cleaning history: old
+				// databases must remain loadable even if deletion cleanup changes.
+				result := DB.Unscoped().Delete(&model.Service{}, "id in (?)", []uint64{historicalService.ID})
+				require.NoError(t, result.Error)
+				require.EqualValues(t, 1, result.RowsAffected)
+			case "quarantined":
+				// Legacy invalid service types are deliberately excluded at startup.
+				result := DB.Model(historicalService).UpdateColumn("type", model.TaskTypeCommand)
+				require.NoError(t, result.Error)
+				require.EqualValues(t, 1, result.RowsAffected)
+			}
+			var historyCount int64
+			require.NoError(t, DB.Model(&model.ServiceHistory{}).Where("service_id = ?", historicalService.ID).Count(&historyCount).Error)
+			require.EqualValues(t, 1, historyCount)
+
+			var restarted *ServiceSentinel
+			var startupErr error
+			require.NotPanics(t, func() {
+				restarted, startupErr = NewServiceSentinel(make(chan *model.Service, 1))
+			})
+			if restarted != nil {
+				t.Cleanup(restarted.Close)
+			}
+			require.NoError(t, startupErr)
+			require.NotNil(t, restarted)
+			require.Equal(t, &_TodayStatsOfService{Up: 8, Down: 3, Delay: 20}, restarted.serviceStatusToday[activeService.ID])
+			monthly := restarted.monthlyStatus[activeService.ID]
+			require.NotNil(t, monthly)
+			require.Equal(t, uint64(12), monthly.TotalUp)
+			require.Equal(t, uint64(4), monthly.TotalDown)
+
+			if serviceState == "retained" {
+				require.Len(t, restarted.services, 2)
+				require.Equal(t, &_TodayStatsOfService{Up: 7, Down: 2, Delay: 90}, restarted.serviceStatusToday[historicalService.ID])
+				require.Equal(t, uint64(7), restarted.monthlyStatus[historicalService.ID].TotalUp)
+				require.Equal(t, uint64(2), restarted.monthlyStatus[historicalService.ID].TotalDown)
+			} else {
+				require.Len(t, restarted.services, 1)
+				require.NotContains(t, restarted.services, historicalService.ID)
+				require.NotContains(t, restarted.serviceStatusToday, historicalService.ID)
+				require.NotContains(t, restarted.monthlyStatus, historicalService.ID)
+			}
+		})
+	}
+}

service/singleton/servicesentinel_window_test.go

@@ -0,0 +1,63 @@
+package singleton
+
+import (
+	"testing"
+	"time"
+
+	pb "github.com/nezhahq/nezha/proto"
+)
+
+func TestServiceWindowCountsFirstFailureAsDown(t *testing.T) {
+	status := new(serviceTaskStatus)
+	status.appendResult(time.Unix(1000, 0), 1, &pb.TaskResult{Successful: false})
+	data := status.responseData()
+	if data.Down != 1 || data.Up != 0 || GetStatusCode(uint64(0)) != StatusDown {
+		t.Fatalf("first failed report = %+v, status %d; want Down", data, GetStatusCode(uint64(0)))
+	}
+}
+
+func TestServiceWindowExpiresOldReportsAndDoesNotResetAtThirty(t *testing.T) {
+	status := new(serviceTaskStatus)
+	start := time.Unix(1000, 0)
+	for i := 0; i < 30; i++ {
+		status.appendResult(start.Add(time.Duration(i)*30*time.Second), 1, &pb.TaskResult{Successful: true})
+	}
+	if data := status.responseData(); data.Up != 30 {
+		t.Fatalf("30th report reset window: %+v", data)
+	}
+	status.appendResult(start.Add(29*30*time.Second+time.Second), 2, &pb.TaskResult{Successful: false})
+	if data := status.responseData(); data.Up != 30 || data.Down != 1 {
+		t.Fatalf("31st report corrupted window: %+v", data)
+	}
+	status.appendResult(start.Add(31*time.Minute), 1, &pb.TaskResult{Successful: false})
+	if data := status.responseData(); data.Up != 0 || data.Down != 1 {
+		t.Fatalf("expired reports still influence status: %+v", data)
+	}
+}
+
+func TestServiceWindowDeduplicatesFloodWithinBucket(t *testing.T) {
+	status := new(serviceTaskStatus)
+	now := time.Unix(1000, 0)
+	for i := 0; i < 10000; i++ {
+		status.appendResult(now, 1, &pb.TaskResult{Successful: false, Data: "ignored payload"})
+	}
+	if len(status.result) != 1 || len(status.result[0].reporters) != 1 {
+		t.Fatalf("flood grew live window: buckets=%d", len(status.result))
+	}
+	if data := status.responseData(); data.Down != 1 {
+		t.Fatalf("duplicate reporter skewed status: %+v", data)
+	}
+	if status.history.Down != 10000 {
+		t.Fatalf("history lost raw reports: %+v", status.history)
+	}
+}
+
+func TestServiceWindowDoesNotEraseFailureWithLaterSuccess(t *testing.T) {
+	status := new(serviceTaskStatus)
+	now := time.Unix(1000, 0)
+	status.appendResult(now, 1, &pb.TaskResult{Successful: false})
+	status.appendResult(now.Add(time.Second), 1, &pb.TaskResult{Successful: true, Delay: 20})
+	if data := status.responseData(); data.Down != 1 || data.Up != 1 {
+		t.Fatalf("later success erased failure in same bucket: %+v", data)
+	}
+}

service/singleton/singleton.go

@@ -63,6 +63,7 @@ func LoadSingleton(bus chan<- *model.Service) (err error) {
 	ServerShared = NewServerClass()
 	CronShared = NewCronClass()
 	ServerTransferShared = NewServerTransferClass()
+	OnlineHistoryShared = NewOnlineHistoryRecorder()
 	// 最后初始化 ServiceSentinel
 	ServiceSentinelShared, err = NewServiceSentinel(bus)
 	if err == nil {
@@ -97,7 +98,7 @@ func InitDBFromPath(path string) error {
 		model.Cron{}, model.Transfer{}, model.ServerGroupServer{},
 		model.NAT{}, model.DDNSProfile{}, model.NotificationGroupNotification{},
 		model.WAF{}, model.Oauth2Bind{}, model.Domain{}, model.ServerTransfer{}, model.JWTSession{},
-		model.APIToken{}, model.MCPAuditLog{})
+		model.APIToken{}, model.MCPAuditLog{}, model.OnlineSession{})
 
 	if err != nil {
 		return err
@@ -171,7 +172,7 @@ func CleanMonitorHistory() {
 	for _, alert := range alerts {
 		for _, rule := range alert.Rules {
 			// 是不是流量记录规则
-			if !rule.IsTransferDurationRule() {
+			if !rule.HasSafeCycleConfiguration() {
 				continue
 			}
 			dataCouldRemoveBefore := rule.GetTransferDurationStart().UTC()

service/singleton/tsdb.go

@@ -1,6 +1,7 @@
 package singleton
 
 import (
+	"errors"
 	"log"
 	"time"
 
@@ -10,6 +11,8 @@ import (
 
 var TSDBShared *tsdb.TSDB
 
+var openTSDB = tsdb.Open
+
 func InitTSDB() error {
 	config := &tsdb.Config{
 		RetentionDays:      30,
@@ -44,11 +47,21 @@ func InitTSDB() error {
 		return nil
 	}
 
-	var err error
-	TSDBShared, err = tsdb.Open(config)
+	TSDBShared = nil
+	db, err := openTSDB(config)
 	if err != nil {
+		if errors.Is(err, tsdb.ErrDiskFull) {
+			log.Printf("NEZHA>> Warning: TSDB is unavailable because its disk is full; dashboard and alerts will continue without TSDB writes: %v", err)
+			if DB != nil {
+				if migrateErr := DB.AutoMigrate(model.ServiceHistory{}); migrateErr != nil {
+					log.Printf("NEZHA>> Warning: failed to prepare SQLite service history fallback: %v", migrateErr)
+				}
+			}
+			return nil
+		}
 		return err
 	}
+	TSDBShared = db
 
 	log.Println("NEZHA>> TSDB initialized successfully")
 

service/singleton/tsdb_disk_guard_test.go

@@ -0,0 +1,58 @@
+package singleton
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/nezhahq/nezha/model"
+	"github.com/nezhahq/nezha/pkg/tsdb"
+)
+
+func TestInitTSDBContinuesWhenDiskIsFull(t *testing.T) {
+	originalConf := Conf
+	originalDB := DB
+	originalShared := TSDBShared
+	originalOpen := openTSDB
+	t.Cleanup(func() {
+		Conf = originalConf
+		DB = originalDB
+		TSDBShared = originalShared
+		openTSDB = originalOpen
+	})
+
+	Conf = &ConfigClass{Config: &model.Config{
+		TSDB: model.TSDBConf{DataPath: "/full/tsdb"},
+	}}
+	DB = nil
+	TSDBShared = nil
+	openTSDB = func(*tsdb.Config) (*tsdb.TSDB, error) {
+		return nil, fmt.Errorf("open failed: %w", tsdb.ErrDiskFull)
+	}
+
+	require.NoError(t, InitTSDB())
+	require.Nil(t, TSDBShared)
+}
+
+func TestInitTSDBStillReturnsNonDiskErrors(t *testing.T) {
+	originalConf := Conf
+	originalShared := TSDBShared
+	originalOpen := openTSDB
+	t.Cleanup(func() {
+		Conf = originalConf
+		TSDBShared = originalShared
+		openTSDB = originalOpen
+	})
+
+	Conf = &ConfigClass{Config: &model.Config{
+		TSDB: model.TSDBConf{DataPath: "/broken/tsdb"},
+	}}
+	TSDBShared = nil
+	openTSDB = func(*tsdb.Config) (*tsdb.TSDB, error) {
+		return nil, fmt.Errorf("permission denied")
+	}
+
+	require.ErrorContains(t, InitTSDB(), "permission denied")
+	require.Nil(t, TSDBShared)
+}