diff --git a/.github/sponsor/feepay.jpg b/.github/sponsor/feepay.jpg new file mode 100644 index 00000000..cab5c154 Binary files /dev/null and b/.github/sponsor/feepay.jpg differ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 351f7552..d8f51eef 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,7 +52,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v7 with: - go-version: "1.26.x" + go-version: "^1.26.10" - name: generate swagger docs run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 69ead4b7..c55011cd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: persist-credentials: false - uses: actions/setup-go@v7 with: - go-version: "1.26.x" + go-version: "^1.26.10" cache: false - name: Generate Swagger docs @@ -61,7 +61,7 @@ jobs: persist-credentials: false - uses: actions/setup-go@v7 with: - go-version: "1.26.x" + go-version: "^1.26.10" cache: false - name: Generate Swagger docs @@ -113,7 +113,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v7 with: - go-version: "1.26.x" + go-version: "^1.26.10" cache: false - name: Prepare Dashboard build inputs working-directory: nezha diff --git a/SECURITY.md b/SECURITY.md index 9314b333..d932e53f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,4 +6,4 @@ Code in `master` branch. ## Reporting a Vulnerability -Thank you for your contribution to open source security, please submit on https://github.com/nezhahq/nezha/security +Thank you for your contribution to open source security. Please report vulnerabilities privately at https://github.com/nezhahq/nezha/security or by email to [hi@nai.ba](mailto:hi@nai.ba). diff --git a/cmd/dashboard/controller/alertrule.go b/cmd/dashboard/controller/alertrule.go index 3c54f4a2..5e5ce2d0 100644 --- a/cmd/dashboard/controller/alertrule.go +++ b/cmd/dashboard/controller/alertrule.go @@ -172,6 +172,15 @@ func validateRule(c *gin.Context, r *model.AlertRule) error { } if len(r.Rules) > 0 { for _, rule := range r.Rules { + if rule == nil { + return singleton.Localizer.ErrorT("rule is not set") + } + if !rule.IsSupportedType() { + return singleton.Localizer.ErrorT("unsupported rule type") + } + if !rule.HasSafeThresholds() { + return singleton.Localizer.ErrorT("invalid rule thresholds") + } switch rule.Cover { case model.RuleCoverAll, model.RuleCoverIgnoreAll: default: @@ -182,10 +191,19 @@ func validateRule(c *gin.Context, r *model.AlertRule) error { if rule.Duration < 3 { return singleton.Localizer.ErrorT("duration need to be at least 3") } + if rule.Duration > model.MaxAlertRuleDuration { + return singleton.Localizer.ErrorT("duration is too large") + } } else { + if !rule.HasSafeCycleUnit() { + return singleton.Localizer.ErrorT("invalid cycle unit") + } if rule.CycleInterval < 1 { return singleton.Localizer.ErrorT("cycle_interval need to be at least 1") } + if rule.CycleInterval > model.MaxAlertRuleCycleInterval { + return singleton.Localizer.ErrorT("cycle_interval is too large") + } if rule.CycleStart == nil { return singleton.Localizer.ErrorT("cycle_start is not set") } diff --git a/cmd/dashboard/controller/alertrule_security_test.go b/cmd/dashboard/controller/alertrule_security_test.go new file mode 100644 index 00000000..7d68d388 --- /dev/null +++ b/cmd/dashboard/controller/alertrule_security_test.go @@ -0,0 +1,67 @@ +package controller + +import ( + "math" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/nezhahq/nezha/model" +) + +func TestValidateRuleRejectsCrashPayloadsFromMember(t *testing.T) { + ctx := newMemberValidationContext(t) + cycleStart := time.Now().Add(-time.Hour) + tests := []struct { + name string + rule *model.Rule + }{ + { + name: "unknown rule type", + rule: &model.Rule{Type: "attacker_controlled", Duration: 3, Cover: model.RuleCoverAll}, + }, + { + name: "duration overflows int", + rule: &model.Rule{Type: "offline", Duration: math.MaxUint64, Cover: model.RuleCoverAll}, + }, + { + name: "cycle interval overflows int", + rule: &model.Rule{ + Type: "transfer_in_cycle", + CycleStart: &cycleStart, + CycleInterval: math.MaxUint64, + Cover: model.RuleCoverAll, + }, + }, + { + name: "nil rule", + rule: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + alert := &model.AlertRule{ + Common: model.Common{UserID: 200}, + Name: "security regression", + Rules: []*model.Rule{test.rule}, + } + require.Error(t, validateRule(ctx, alert)) + }) + } +} + +func TestValidateRuleAcceptsSafeDurationBoundary(t *testing.T) { + ctx := newMemberValidationContext(t) + alert := &model.AlertRule{ + Common: model.Common{UserID: 200}, + Name: "duration boundary", + Rules: []*model.Rule{{ + Type: "offline", + Duration: model.MaxAlertRuleDuration, + Cover: model.RuleCoverAll, + }}, + } + require.NoError(t, validateRule(ctx, alert)) +} diff --git a/cmd/dashboard/controller/controller.go b/cmd/dashboard/controller/controller.go index 9ea99633..45bca2dc 100644 --- a/cmd/dashboard/controller/controller.go +++ b/cmd/dashboard/controller/controller.go @@ -57,6 +57,9 @@ func routers(r *gin.Engine, frontendDist fs.FS) { if err := authMiddleware.MiddlewareInit(); err != nil { log.Fatal("authMiddleware.MiddlewareInit Error:" + err.Error()) } + // Signature-only view of the auth middleware for IsDashboardCredential: + // same key and pinned algorithm, claims validation deliberately skipped. + dashboardCredentialJWTParser = newDashboardCredentialJWTParser(authMiddleware) // /mcp — Model Context Protocol endpoint, authenticated by PAT only (闸 1 + 闸 2)。 // 不放在 /api/v1 下：MCP client 配置 URL 更短，且 MCP transport 协议演进与 REST API // 解耦。鉴权一律走 apiTokenAuthMiddleware；不接受 JWT 以避免浏览器误触。 @@ -101,6 +104,7 @@ func routers(r *gin.Engine, frontendDist fs.FS) { optionalAuth.GET("/service/:id/history", restScopeMiddleware(model.ScopeServiceRead), commonHandler(getServiceHistory)) optionalAuth.GET("/server/:id/service", restScopeMiddleware(model.ScopeServiceRead), commonHandler(listServerServices)) optionalAuth.GET("/server/:id/metrics", restScopeMiddleware(model.ScopeServerRead), commonHandler(getServerMetrics)) + optionalAuth.GET("/server/:id/online-history", restScopeMiddleware(model.ScopeServerRead), commonHandler(getServerOnlineHistory)) // CSRF middleware applies group-wide. Safe methods short-circuit and // PAT bearer requests bypass — so the only callers gated are diff --git a/cmd/dashboard/controller/credential_redaction_test.go b/cmd/dashboard/controller/credential_redaction_test.go index 14b289da..0daaa6fd 100644 --- a/cmd/dashboard/controller/credential_redaction_test.go +++ b/cmd/dashboard/controller/credential_redaction_test.go @@ -109,12 +109,14 @@ func TestListDDNS_RedactsCredentials(t *testing.T) { defer setupTenancyTest(t)() p := model.DDNSProfile{ - Common: model.Common{UserID: 10}, - Name: "cf", - Provider: "cloudflare", - AccessID: "id", - AccessSecret: "super-secret-token", - WebhookHeaders: `{"Authorization":"Bearer xxx"}`, + Common: model.Common{UserID: 10}, + Name: "cf", + Provider: "cloudflare", + AccessID: "id", + AccessSecret: "super-secret-token", + WebhookURL: "https://ddns.example/update?token=embedded-secret", + WebhookRequestBody: `{"token":"embedded-secret"}`, + WebhookHeaders: `{"Authorization":"Bearer xxx"}`, } require.NoError(t, singleton.DB.Create(&p).Error) singleton.DDNSShared.InsertForTest(&p) @@ -124,12 +126,18 @@ func TestListDDNS_RedactsCredentials(t *testing.T) { require.NoError(t, err) require.Len(t, out, 1) require.Empty(t, out[0].AccessSecret, "access_secret must be redacted in list response") + require.Empty(t, out[0].WebhookURL, "webhook_url must be redacted in list response") + require.Empty(t, out[0].WebhookRequestBody, "webhook_request_body must be redacted in list response") require.Empty(t, out[0].WebhookHeaders, "webhook_headers must be redacted in list response") require.Equal(t, "id", out[0].AccessID, "non-secret fields must be preserved") var stored model.DDNSProfile require.NoError(t, singleton.DB.First(&stored, p.ID).Error) require.Equal(t, "super-secret-token", stored.AccessSecret, "redaction must not mutate stored data") + require.Equal(t, "https://ddns.example/update?token=embedded-secret", stored.WebhookURL, + "redaction must not mutate stored webhook URL") + require.Equal(t, `{"token":"embedded-secret"}`, stored.WebhookRequestBody, + "redaction must not mutate stored webhook body") } func TestListNotification_RedactsCredentials(t *testing.T) { @@ -164,14 +172,15 @@ func TestUpdateDDNS_EmptySecretPreservesStored(t *testing.T) { defer setupTenancyTest(t)() existing := model.DDNSProfile{ - Common: model.Common{UserID: 10}, - Name: "cf", - Provider: "webhook", - AccessID: "id", - AccessSecret: "keep-me", - WebhookURL: "http://127.0.0.1/", - WebhookMethod: 1, - WebhookHeaders: `{"X-Token":"keep-header"}`, + Common: model.Common{UserID: 10}, + Name: "cf", + Provider: "webhook", + AccessID: "id", + AccessSecret: "keep-me", + WebhookURL: "http://127.0.0.1/", + WebhookRequestBody: `{"X-Token":"keep-body"}`, + WebhookMethod: 1, + WebhookHeaders: `{"X-Token":"keep-header"}`, } require.NoError(t, singleton.DB.Create(&existing).Error) singleton.DDNSShared.InsertForTest(&existing) @@ -181,10 +190,11 @@ func TestUpdateDDNS_EmptySecretPreservesStored(t *testing.T) { "provider": "webhook", "access_id": "id", "access_secret": "", - "webhook_url": "http://127.0.0.1/", + "webhook_url": "", "webhook_method": 1, "webhook_request_type": 1, "webhook_headers": "", + "webhook_request_body": "", "max_retries": 3, }) c.Params = gin.Params{{Key: "id", Value: itoa(existing.ID)}} @@ -197,6 +207,10 @@ func TestUpdateDDNS_EmptySecretPreservesStored(t *testing.T) { require.Equal(t, "keep-me", after.AccessSecret, "empty submitted secret must preserve stored value") require.Equal(t, `{"X-Token":"keep-header"}`, after.WebhookHeaders, "empty submitted headers must preserve stored value") + require.Equal(t, "http://127.0.0.1/", after.WebhookURL, + "empty submitted webhook URL must preserve stored value") + require.Equal(t, `{"X-Token":"keep-body"}`, after.WebhookRequestBody, + "empty submitted webhook body must preserve stored value") } func TestUpdateDDNS_NonEmptySecretOverwrites(t *testing.T) { diff --git a/cmd/dashboard/controller/dashboard_credential.go b/cmd/dashboard/controller/dashboard_credential.go new file mode 100644 index 00000000..f16e3e34 --- /dev/null +++ b/cmd/dashboard/controller/dashboard_credential.go @@ -0,0 +1,162 @@ +package controller + +import ( + "context" + "log" + "strings" + + jwt "github.com/golang-jwt/jwt/v4" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + ginjwt "github.com/appleboy/gin-jwt/v2" + "github.com/nezhahq/nezha/model" + "github.com/nezhahq/nezha/service/singleton" +) + +// dashboardCredentialJWTParser verifies that a JWT was signed by this +// dashboard. It is derived from the auth middleware created in routers() and +// exists only for dashboard-credential classification; it must never +// authenticate dashboard requests on its own. +var dashboardCredentialJWTParser *ginjwt.GinJWTMiddleware + +// newDashboardCredentialJWTParser derives a signature-only parser from the +// dashboard auth middleware. It keeps the exact key value and the pinned HS256 +// signing algorithm of initParams() — there is no second read of +// singleton.Conf.JWTSecretKey — and disables claims validation so expiry does +// not change the verdict: an expired token is still proof the dashboard issued +// it, and keeping expired dashboard tokens out of the NAT tunnel prevents a +// careless backend from treating "any JWT" as its own credential. The live +// auth middleware keeps full claims validation; dashboard authentication is +// not relaxed by this parser in any way. +func newDashboardCredentialJWTParser(authMiddleware *ginjwt.GinJWTMiddleware) *ginjwt.GinJWTMiddleware { + parser := *authMiddleware + parser.ParseOptions = []jwt.ParserOption{jwt.WithoutClaimsValidation()} + return &parser +} + +// IsDashboardCredential reports whether an Authorization header value was +// issued by this dashboard: a panel-signed JWT or a panel API token (PAT). +// +// NAT ingress uses this to enforce the tunnel credential invariant: values +// classified as dashboard-issued are stripped before the request is tunneled +// to the agent, while foreign credentials are forwarded untouched because the +// NAT backend may authenticate with credentials of its own. +// +// Decision policy mirrors the dashboard's own auth middlewares: only the +// case-sensitive "Bearer " scheme is considered; the value after the scheme is +// classified by IsDashboardCredentialValue, which is also the entry point for +// the panel's scheme-less TokenLookup channels. Only a deterministic proof +// that the value was not issued by this dashboard (scheme mismatch, unknown +// PAT hash, failed signature verification) returns false. +func IsDashboardCredential(authz string) bool { + raw := strings.TrimSpace(authz) + if !strings.HasPrefix(raw, "Bearer ") { + return false + } + return IsDashboardCredentialValue(strings.TrimPrefix(raw, "Bearer ")) +} + +// IsDashboardCredentialValue reports whether a raw credential value — the part +// after the "Bearer " scheme, exactly as the panel's other TokenLookup +// channels carry it (the ?token= query parameter and the nz-jwt cookie, see +// initParams) — was issued by this dashboard: a panel-signed JWT or a panel +// API token (PAT). +// +// Failure policy is identical to IsDashboardCredential: whenever the +// classification cannot be completed (parser or config not ready, database +// failure) the value is conservatively reported as a dashboard credential so +// the caller strips it; only a deterministic proof that the value was not +// issued by this dashboard (unknown PAT hash, failed signature verification) +// returns false. +func IsDashboardCredentialValue(value string) bool { + classified := ClassifyDashboardCredentialValues(context.Background(), []string{value}) + if len(classified) != 1 { + return true + } + return classified[0] +} + +// ClassifyDashboardCredentialValues classifies a bounded request batch. JWTs +// are checked independently, while all PAT-shaped values are hashed and +// resolved with one silent IN query. This prevents attacker-controlled NAT +// query parameters or cookies from amplifying one HTTP request into thousands +// of SQLite queries and record-not-found log entries. +func ClassifyDashboardCredentialValues(ctx context.Context, values []string) []bool { + if ctx == nil { + ctx = context.Background() + } + results := make([]bool, len(values)) + patIndexes := make(map[string][]int) + for i, value := range values { + plaintext := strings.TrimSpace(value) + if strings.HasPrefix(plaintext, model.APITokenPrefix) { + hash := model.HashAPIToken(plaintext) + patIndexes[hash] = append(patIndexes[hash], i) + continue + } + results[i] = isDashboardSignedJWT(plaintext) + } + if len(patIndexes) == 0 { + return results + } + if singleton.DB == nil { + logDashboardCredentialUnclassifiable("api token lookup unavailable") + for _, indexes := range patIndexes { + for _, i := range indexes { + results[i] = true + } + } + return results + } + + hashes := make([]string, 0, len(patIndexes)) + for hash := range patIndexes { + hashes = append(hashes, hash) + } + type tokenHashRow struct { + TokenHash string + } + var rows []tokenHashRow + err := singleton.DB.WithContext(ctx). + Session(&gorm.Session{Logger: logger.Discard}). + Model(&model.APIToken{}). + Select("token_hash"). + Where("token_hash IN ?", hashes). + Find(&rows).Error + if err != nil { + logDashboardCredentialUnclassifiable("api token lookup failed") + for _, indexes := range patIndexes { + for _, i := range indexes { + results[i] = true + } + } + return results + } + for _, row := range rows { + for _, i := range patIndexes[row.TokenHash] { + results[i] = true + } + } + return results +} + +// isDashboardSignedJWT verifies the token signature with the dashboard's +// pinned algorithm and key. Claims validation is disabled on purpose (see +// newDashboardCredentialJWTParser): expiry does not change who issued the +// token. +func isDashboardSignedJWT(token string) bool { + if dashboardCredentialJWTParser == nil { + // routers() has not run yet (startup incomplete): fail closed. + logDashboardCredentialUnclassifiable("dashboard credential parser unavailable") + return true + } + _, err := dashboardCredentialJWTParser.ParseTokenString(token) + return err == nil +} + +// logDashboardCredentialUnclassifiable records a classification failure. The +// credential value itself is never included in the message. +func logDashboardCredentialUnclassifiable(reason string) { + log.Printf("NEZHA>> NAT ingress: credential value could not be classified (%s); stripping it as a precaution", reason) +} diff --git a/cmd/dashboard/controller/dashboard_credential_nat_flow_test.go b/cmd/dashboard/controller/dashboard_credential_nat_flow_test.go new file mode 100644 index 00000000..770e6f5e --- /dev/null +++ b/cmd/dashboard/controller/dashboard_credential_nat_flow_test.go @@ -0,0 +1,250 @@ +package controller + +import ( + "bufio" + "bytes" + "context" + "io" + "net" + "net/http" + "net/url" + "sync" + "testing" + "time" + + "github.com/goccy/go-json" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/metadata" + + "github.com/nezhahq/nezha/cmd/dashboard/rpc" + "github.com/nezhahq/nezha/model" + "github.com/nezhahq/nezha/proto" + rpcService "github.com/nezhahq/nezha/service/rpc" + "github.com/nezhahq/nezha/service/singleton" +) + +// setupNATFlowTest extends setupDashboardCredentialTest with the minimal +// ServeNAT environment: a registered server with a task stream and a fresh +// handler singleton, mirroring the rpc package's serveNAT fixture. +func setupNATFlowTest(t *testing.T) (*rpcService.NezhaHandler, *model.Server, *natFlowTaskStream) { + t.Helper() + setupDashboardCredentialTest(t) + + require.NoError(t, singleton.DB.AutoMigrate(&model.Server{})) + originalServerShared, originalHandler := singleton.ServerShared, rpcService.NezhaHandlerSingleton + t.Cleanup(func() { + singleton.ServerShared, rpcService.NezhaHandlerSingleton = originalServerShared, originalHandler + }) + + serverRow := &model.Server{Common: model.Common{ID: 7}, UUID: "nat-credential-flow-test", Name: "nat-credential-flow-test"} + require.NoError(t, singleton.DB.Create(serverRow).Error) + singleton.ServerShared = singleton.NewServerClass() + handler := rpcService.NewNezhaHandler() + rpcService.NezhaHandlerSingleton = handler + + server, ok := singleton.ServerShared.Get(serverRow.ID) + require.True(t, ok) + taskStream := &natFlowTaskStream{} + server.SetTaskStream(taskStream) + + // Wire the gate exactly as cmd/dashboard/main.go does, with the real + // classifiers built by setupDashboardCredentialTest. + rpc.SetNATDashboardCredentialGate(ClassifyDashboardCredentialValues) + t.Cleanup(func() { rpc.SetNATDashboardCredentialGate(nil) }) + + return handler, server, taskStream +} + +// TestServeNATWithRealClassifierStripsDashboardCredentialsFromAllChannels pins +// the main() wiring contract end to end: with the real classifiers wired as +// main() does, ServeNAT forwards a request whose dashboard-issued credentials +// — Authorization header, ?token= query parameter, nz-jwt cookie — are all +// stripped, while foreign values and untouched query/cookie data survive +// byte-for-byte. +func TestServeNATWithRealClassifierStripsDashboardCredentialsFromAllChannels(t *testing.T) { + handler, server, taskStream := setupNATFlowTest(t) + + dashboardJWT := mintDashboardCredentialJWT(t, time.Hour) + foreignJWT := mintForeignJWT(t) + // No api_tokens row hashes to this PAT, so the classifier proves it was + // not issued by this dashboard and it must survive the tunnel. + foreignPAT := model.APITokenPrefix + "foreign-issued-pat" + dashboardPAT := model.APITokenPrefix + "dashboard-issued-pat" + require.NoError(t, singleton.DB.Create(&model.APIToken{ + UserID: 1, + Name: "nat-flow", + TokenHash: model.HashAPIToken(dashboardPAT), + }).Error) + + connection := &natFlowConn{closed: make(chan struct{})} + agent := &natFlowAgent{writeDone: make(chan struct{})} + writer := &natFlowWriter{conn: connection} + request := &http.Request{ + Method: http.MethodPost, + URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + dashboardJWT + "&token=" + url.QueryEscape(foreignJWT) + + "&token=" + foreignPAT + "&token=" + dashboardPAT}, + Header: make(http.Header), + Body: http.NoBody, + } + request.Header.Set("Authorization", "Bearer "+dashboardJWT) + request.Header.Set("Cookie", "sid=abc; nz-jwt="+dashboardJWT+"; nz-jwt="+foreignJWT+"; nz-jwt="+foreignPAT) + request.Header.Set("X-Ordinary-Nat", "ordinary") + taskStream.onSend = func(task *proto.Task) error { + var nat model.TaskNAT + if err := json.Unmarshal([]byte(task.Data), &nat); err != nil { + return err + } + return handler.AgentConnected(nat.StreamID, agent) + } + + done := make(chan struct{}) + go func() { + rpc.ServeNAT(writer, request, &model.NAT{ServerID: server.ID, Host: "target.example"}) + close(done) + }() + select { + case <-agent.writeDone: + case <-time.After(time.Second): + t.Fatal("agent did not receive the transferred NAT request") + } + require.NoError(t, connection.Close()) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("ServeNAT did not finish") + } + + forwarded := string(agent.writtenBytes()) + // Dashboard credentials are gone from every channel. + require.NotContains(t, forwarded, "Authorization:") + require.NotContains(t, forwarded, dashboardJWT) + require.NotContains(t, forwarded, dashboardPAT) + // Foreign credentials and untouched data survive byte-for-byte. + require.Contains(t, forwarded, "POST /nat?keep=1&token="+url.QueryEscape(foreignJWT)+"&token="+foreignPAT+" HTTP/1.1") + require.Contains(t, forwarded, "Cookie: sid=abc; nz-jwt="+foreignJWT+"; nz-jwt="+foreignPAT) + require.Contains(t, forwarded, "X-Ordinary-Nat: ordinary") +} + +// TestServeNATWithRealClassifierKeepsForeignCredentials pins that the real +// classifier deterministically proves foreign values were not issued by this +// dashboard and leaves every channel untouched. +func TestServeNATWithRealClassifierKeepsForeignCredentials(t *testing.T) { + handler, server, taskStream := setupNATFlowTest(t) + + foreignJWT := mintForeignJWT(t) + connection := &natFlowConn{closed: make(chan struct{})} + agent := &natFlowAgent{writeDone: make(chan struct{})} + writer := &natFlowWriter{conn: connection} + request := &http.Request{ + Method: http.MethodPost, + URL: &url.URL{Path: "/nat", RawQuery: "token=" + foreignJWT}, + Header: make(http.Header), + Body: http.NoBody, + } + request.Header.Set("Authorization", "Bearer "+foreignJWT) + request.Header.Set("Cookie", "nz-jwt="+foreignJWT) + taskStream.onSend = func(task *proto.Task) error { + var nat model.TaskNAT + if err := json.Unmarshal([]byte(task.Data), &nat); err != nil { + return err + } + return handler.AgentConnected(nat.StreamID, agent) + } + + done := make(chan struct{}) + go func() { + rpc.ServeNAT(writer, request, &model.NAT{ServerID: server.ID, Host: "target.example"}) + close(done) + }() + select { + case <-agent.writeDone: + case <-time.After(time.Second): + t.Fatal("agent did not receive the transferred NAT request") + } + require.NoError(t, connection.Close()) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("ServeNAT did not finish") + } + + forwarded := string(agent.writtenBytes()) + require.Contains(t, forwarded, "POST /nat?token="+foreignJWT+" HTTP/1.1") + require.Contains(t, forwarded, "Authorization: Bearer "+foreignJWT) + require.Contains(t, forwarded, "Cookie: nz-jwt="+foreignJWT) +} + +type natFlowTaskStream struct { + onSend func(*proto.Task) error +} + +func (stream *natFlowTaskStream) Send(task *proto.Task) error { + if stream.onSend != nil { + return stream.onSend(task) + } + return nil +} +func (*natFlowTaskStream) Recv() (*proto.TaskResult, error) { return nil, io.EOF } +func (*natFlowTaskStream) SetHeader(metadata.MD) error { return nil } +func (*natFlowTaskStream) SendHeader(metadata.MD) error { return nil } +func (*natFlowTaskStream) SetTrailer(metadata.MD) {} +func (*natFlowTaskStream) Context() context.Context { return context.Background() } +func (*natFlowTaskStream) SendMsg(any) error { return nil } +func (*natFlowTaskStream) RecvMsg(any) error { return io.EOF } + +type natFlowWriter struct{ conn *natFlowConn } + +func (writer *natFlowWriter) Header() http.Header { return http.Header{} } +func (writer *natFlowWriter) Write(data []byte) (int, error) { return len(data), nil } +func (writer *natFlowWriter) WriteHeader(int) {} +func (writer *natFlowWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + go func() { _, _ = writer.conn.Read(make([]byte, 1)) }() + return writer.conn, bufio.NewReadWriter(bufio.NewReader(writer.conn), bufio.NewWriter(writer.conn)), nil +} + +type natFlowConn struct { + closed chan struct{} + closeOnce sync.Once +} + +func (conn *natFlowConn) Read([]byte) (int, error) { + <-conn.closed + return 0, io.EOF +} +func (conn *natFlowConn) Write(data []byte) (int, error) { return len(data), nil } +func (conn *natFlowConn) Close() error { + conn.closeOnce.Do(func() { close(conn.closed) }) + return nil +} +func (conn *natFlowConn) LocalAddr() net.Addr { return natFlowAddr{} } +func (conn *natFlowConn) RemoteAddr() net.Addr { return natFlowAddr{} } +func (conn *natFlowConn) SetDeadline(time.Time) error { return nil } +func (conn *natFlowConn) SetReadDeadline(time.Time) error { return nil } +func (conn *natFlowConn) SetWriteDeadline(time.Time) error { return nil } + +type natFlowAddr struct{} + +func (natFlowAddr) Network() string { return "test" } +func (natFlowAddr) String() string { return "test" } + +type natFlowAgent struct { + mu sync.Mutex + written bytes.Buffer + writeDone chan struct{} + once sync.Once +} + +func (agent *natFlowAgent) Read([]byte) (int, error) { return 0, io.EOF } +func (agent *natFlowAgent) Write(data []byte) (int, error) { + agent.mu.Lock() + agent.written.Write(data) + agent.mu.Unlock() + agent.once.Do(func() { close(agent.writeDone) }) + return len(data), nil +} +func (*natFlowAgent) Close() error { return nil } +func (agent *natFlowAgent) writtenBytes() []byte { + agent.mu.Lock() + defer agent.mu.Unlock() + return append([]byte(nil), agent.written.Bytes()...) +} diff --git a/cmd/dashboard/controller/dashboard_credential_test.go b/cmd/dashboard/controller/dashboard_credential_test.go new file mode 100644 index 00000000..9c1cc81c --- /dev/null +++ b/cmd/dashboard/controller/dashboard_credential_test.go @@ -0,0 +1,226 @@ +package controller + +import ( + "bytes" + "context" + "log" + "testing" + "time" + + jwt "github.com/appleboy/gin-jwt/v2" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + + "github.com/nezhahq/nezha/model" + "github.com/nezhahq/nezha/service/singleton" +) + +func setupDashboardCredentialTest(t *testing.T) { + t.Helper() + originalConf, originalDB, originalParser := singleton.Conf, singleton.DB, dashboardCredentialJWTParser + t.Cleanup(func() { + singleton.Conf, singleton.DB, dashboardCredentialJWTParser = originalConf, originalDB, originalParser + }) + + singleton.Conf = &singleton.ConfigClass{Config: &model.Config{}} + singleton.Conf.JWTSecretKey = "dashboard-credential-classification-test-key" + singleton.Conf.JWTTimeout = 1 + + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + sqlDB, err := db.DB() + require.NoError(t, err) + t.Cleanup(func() { _ = sqlDB.Close() }) + require.NoError(t, db.AutoMigrate(&model.APIToken{})) + singleton.DB = db + + // Same construction path as routers(): jwt.New(initParams()). + authMiddleware, err := jwt.New(initParams()) + require.NoError(t, err) + dashboardCredentialJWTParser = newDashboardCredentialJWTParser(authMiddleware) +} + +// mintDashboardCredentialJWT signs a JWT with the dashboard key through +// initParams(), the same key source the production verifier uses. A negative +// timeout mints an already-expired token. +func mintDashboardCredentialJWT(t *testing.T, timeout time.Duration) string { + t.Helper() + params := initParams() + params.Timeout = timeout + mw, err := jwt.New(params) + require.NoError(t, err) + token, _, err := mw.TokenGenerator(nil) + require.NoError(t, err) + return token +} + +func mintForeignJWT(t *testing.T) string { + t.Helper() + params := initParams() + params.Key = []byte("foreign-signing-key-not-held-by-the-dashboard") + mw, err := jwt.New(params) + require.NoError(t, err) + token, _, err := mw.TokenGenerator(nil) + require.NoError(t, err) + return token +} + +func TestIsDashboardCredentialClassification(t *testing.T) { + setupDashboardCredentialTest(t) + + dashboardJWT := mintDashboardCredentialJWT(t, time.Hour) + expiredDashboardJWT := mintDashboardCredentialJWT(t, -time.Hour) + foreignJWT := mintForeignJWT(t) + + dashboardPAT := model.APITokenPrefix + "dashboard-credential-pat" + require.NoError(t, singleton.DB.Create(&model.APIToken{ + UserID: 1, + Name: "nat-classification", + TokenHash: model.HashAPIToken(dashboardPAT), + }).Error) + + t.Run("dashboard signed jwt is a dashboard credential", func(t *testing.T) { + require.True(t, IsDashboardCredential("Bearer "+dashboardJWT)) + }) + + t.Run("expired dashboard signed jwt is still a dashboard credential", func(t *testing.T) { + // Signature verification ignores exp on purpose: an expired token is + // still proof the dashboard issued it. + require.True(t, IsDashboardCredential("Bearer "+expiredDashboardJWT)) + }) + + t.Run("dashboard api token is a dashboard credential", func(t *testing.T) { + require.True(t, IsDashboardCredential("Bearer "+dashboardPAT)) + }) + + t.Run("foreign signed jwt is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredential("Bearer "+foreignJWT)) + }) + + t.Run("unknown api token is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredential("Bearer "+model.APITokenPrefix+"unknown-to-the-dashboard")) + }) + + t.Run("random string is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredential("Bearer random-not-a-token")) + }) + + t.Run("non bearer scheme is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredential("Basic "+dashboardJWT)) + // Scheme matching is case-sensitive, as in both dashboard auth middlewares. + require.False(t, IsDashboardCredential("bearer "+dashboardJWT)) + }) + + t.Run("empty or missing header is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredential("")) + require.False(t, IsDashboardCredential("Bearer ")) + }) +} + +// TestIsDashboardCredentialValueClassification pins the scheme-less classifier +// that backs IsDashboardCredential and drives the ?token= query parameter and +// nz-jwt cookie channels at NAT ingress (see IsDashboardCredentialValue). +func TestIsDashboardCredentialValueClassification(t *testing.T) { + setupDashboardCredentialTest(t) + + dashboardJWT := mintDashboardCredentialJWT(t, time.Hour) + expiredDashboardJWT := mintDashboardCredentialJWT(t, -time.Hour) + foreignJWT := mintForeignJWT(t) + + dashboardPAT := model.APITokenPrefix + "dashboard-credential-value-pat" + require.NoError(t, singleton.DB.Create(&model.APIToken{ + UserID: 1, + Name: "nat-value-classification", + TokenHash: model.HashAPIToken(dashboardPAT), + }).Error) + + t.Run("dashboard signed jwt is a dashboard credential", func(t *testing.T) { + require.True(t, IsDashboardCredentialValue(dashboardJWT)) + }) + t.Run("expired dashboard signed jwt is still a dashboard credential", func(t *testing.T) { + require.True(t, IsDashboardCredentialValue(expiredDashboardJWT)) + }) + t.Run("dashboard api token is a dashboard credential", func(t *testing.T) { + require.True(t, IsDashboardCredentialValue(dashboardPAT)) + }) + t.Run("foreign signed jwt is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredentialValue(foreignJWT)) + }) + t.Run("unknown api token is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredentialValue(model.APITokenPrefix+"unknown-to-the-dashboard")) + }) + t.Run("random string is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredentialValue("random-not-a-token")) + }) + t.Run("empty value is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredentialValue("")) + }) + t.Run("bearer scheme is not part of the value contract", func(t *testing.T) { + // The query and cookie channels carry the bare value; a value that + // itself starts with "Bearer " is not something the panel issues and + // fails signature verification. + require.False(t, IsDashboardCredentialValue("Bearer "+dashboardJWT)) + }) +} + +func TestClassifyDashboardCredentialValuesBatchesPATLookup(t *testing.T) { + setupDashboardCredentialTest(t) + + dashboardPAT := model.APITokenPrefix + "dashboard-batch-pat" + require.NoError(t, singleton.DB.Create(&model.APIToken{ + UserID: 1, + Name: "nat-batch-classification", + TokenHash: model.HashAPIToken(dashboardPAT), + }).Error) + + queryCount := 0 + const callbackName = "test:count-dashboard-credential-batch-query" + require.NoError(t, singleton.DB.Callback().Query().Before("gorm:query").Register(callbackName, func(*gorm.DB) { + queryCount++ + })) + t.Cleanup(func() { singleton.DB.Callback().Query().Remove(callbackName) }) + + values := []string{ + dashboardPAT, + model.APITokenPrefix + "unknown-one", + dashboardPAT, + model.APITokenPrefix + "unknown-two", + } + classified := ClassifyDashboardCredentialValues(context.Background(), values) + + require.Equal(t, []bool{true, false, true, false}, classified) + require.Equal(t, 1, queryCount, "all PAT hashes in one request must use one database query") +} + +func TestIsDashboardCredentialFailsClosedWhenParserUnavailable(t *testing.T) { + setupDashboardCredentialTest(t) + dashboardCredentialJWTParser = nil + + token := mintDashboardCredentialJWT(t, time.Hour) + require.True(t, IsDashboardCredential("Bearer "+token)) +} + +func TestIsDashboardCredentialFailsClosedWhenAPITokenLookupUnavailable(t *testing.T) { + setupDashboardCredentialTest(t) + singleton.DB = nil + + require.True(t, IsDashboardCredential("Bearer "+model.APITokenPrefix+"unresolvable")) +} + +func TestIsDashboardCredentialFailsClosedOnDatabaseError(t *testing.T) { + setupDashboardCredentialTest(t) + sqlDB, err := singleton.DB.DB() + require.NoError(t, err) + require.NoError(t, sqlDB.Close()) + + var logs bytes.Buffer + originalOutput := log.Writer() + log.SetOutput(&logs) + t.Cleanup(func() { log.SetOutput(originalOutput) }) + + plaintext := model.APITokenPrefix + "unresolvable-after-db-failure" + require.True(t, IsDashboardCredential("Bearer "+plaintext)) + // The credential content itself must never reach the log. + require.NotContains(t, logs.String(), plaintext) +} diff --git a/cmd/dashboard/controller/ddns.go b/cmd/dashboard/controller/ddns.go index a2a2c20e..b03ba5ca 100644 --- a/cmd/dashboard/controller/ddns.go +++ b/cmd/dashboard/controller/ddns.go @@ -34,6 +34,8 @@ func listDDNS(c *gin.Context) ([]*model.DDNSProfile, error) { // 不影响 singleton 内原始数据。 for _, p := range ddnsProfiles { p.AccessSecret = "" + p.WebhookURL = "" + p.WebhookRequestBody = "" p.WebhookHeaders = "" } @@ -144,10 +146,8 @@ func updateDDNS(c *gin.Context) (any, error) { p.Provider = df.Provider p.Domains = df.Domains p.AccessID = df.AccessID - p.WebhookURL = df.WebhookURL p.WebhookMethod = df.WebhookMethod p.WebhookRequestType = df.WebhookRequestType - p.WebhookRequestBody = df.WebhookRequestBody // 凭据在列表接口已脱敏，前端无法回填；空值视为"不修改"，保留旧值避免误清空。 if df.AccessSecret != "" { @@ -156,6 +156,12 @@ func updateDDNS(c *gin.Context) (any, error) { if df.WebhookHeaders != "" { p.WebhookHeaders = df.WebhookHeaders } + if df.WebhookURL != "" { + p.WebhookURL = df.WebhookURL + } + if df.WebhookRequestBody != "" { + p.WebhookRequestBody = df.WebhookRequestBody + } for n, domain := range p.Domains { // IDN to ASCII diff --git a/cmd/dashboard/controller/fm.go b/cmd/dashboard/controller/fm.go index bd9216fa..a010b479 100644 --- a/cmd/dashboard/controller/fm.go +++ b/cmd/dashboard/controller/fm.go @@ -15,6 +15,19 @@ import ( "github.com/nezhahq/nezha/service/singleton" ) +// The official file-manager client uploads in 1 MiB WebSocket messages. Cap +// each complete message at that protocol boundary so gorilla/websocket cannot +// buffer an attacker-sized frame before the IO stream relay sees it. +const fileManagerWebSocketInputLimit int64 = 1024 * 1024 + +type websocketReadLimiter interface { + SetReadLimit(limit int64) +} + +func limitFileManagerWebSocketInput(conn websocketReadLimiter) { + conn.SetReadLimit(fileManagerWebSocketInputLimit) +} + // Create FM session // @Summary Create FM session // @Description Create an "attached" FM. It is advised to only call this within a terminal session. @@ -103,6 +116,7 @@ func fmStream(c *gin.Context) (any, error) { if err != nil { return nil, newWsError("%v", err) } + limitFileManagerWebSocketInput(wsConn) conn := websocketx.NewConn(wsConn) pingTransport := newWebsocketPingTransport(conn, wsConn.Close) stopPing := startWebsocketPingTicker(c.Request.Context(), time.Second*10, pingTransport) diff --git a/cmd/dashboard/controller/jwt.go b/cmd/dashboard/controller/jwt.go index e7fad5d4..cf544c90 100644 --- a/cmd/dashboard/controller/jwt.go +++ b/cmd/dashboard/controller/jwt.go @@ -149,7 +149,7 @@ func identityHandler() func(c *gin.Context) any { } currentIP := c.GetString(model.CtxKeyRealIPStr) - if sess.IP != currentIP { + if !singleton.Conf.JWTIPChangeAllowed() && sess.IP != currentIP { c.Set(model.CtxKeyIsIPMismatch, true) return nil } diff --git a/cmd/dashboard/controller/jwt_session_test.go b/cmd/dashboard/controller/jwt_session_test.go index 29b5e15e..0f169cb3 100644 --- a/cmd/dashboard/controller/jwt_session_test.go +++ b/cmd/dashboard/controller/jwt_session_test.go @@ -193,6 +193,61 @@ func TestIdentityHandlerFlagsIPMismatch(t *testing.T) { assert.True(t, verify.GetBool(model.CtxKeyIsIPMismatch)) } +func TestIdentityHandlerAllowsIPMismatchWhenConfigured(t *testing.T) { + cleanup := setupJWTSessionTest(t) + defer cleanup() + singleton.Conf.SetJWTIPChangeAllowed(true) + + ctx := newCtxForUser(0, "1.2.3.4", "ua") + user := model.User{Common: model.Common{ID: 100}, TokenVersion: 7} + claims, err := issueJWTSession(ctx, &user, 1) + require.NoError(t, err) + + verify := newCtxForUser(0, "9.9.9.9", "ua") + verify.Set("JWT_PAYLOAD", jwt.MapClaims{ + jwtClaimUserID: claims[jwtClaimUserID], + jwtClaimKeyID: claims[jwtClaimKeyID], + }) + + identity := identityHandler()(verify) + require.NotNil(t, identity, "IP mismatch must be accepted when allow_jwt_ip_change is enabled") + assert.False(t, verify.GetBool(model.CtxKeyIsIPMismatch)) +} + +func TestIdentityHandlerConcurrentIPChangeSetting(t *testing.T) { + cleanup := setupJWTSessionTest(t) + defer cleanup() + + ctx := newCtxForUser(0, "1.2.3.4", "ua") + user := model.User{Common: model.Common{ID: 100}, TokenVersion: 7} + claims, err := issueJWTSession(ctx, &user, 1) + require.NoError(t, err) + + stop, done := make(chan struct{}), make(chan struct{}) + go func() { + defer close(done) + for { + select { + case <-stop: + return + default: + singleton.Conf.SetJWTIPChangeAllowed(true) + singleton.Conf.SetJWTIPChangeAllowed(false) + } + } + }() + defer func() { + close(stop) + <-done + }() + + for i := 0; i < 100; i++ { + verify := newCtxForUser(0, "1.2.3.4", "ua") + verify.Set("JWT_PAYLOAD", jwt.MapClaims(claims)) + require.NotNil(t, identityHandler()(verify), "matching IP must pass during setting updates") + } +} + func TestIdentityHandlerRejectsUnknownKeyID(t *testing.T) { cleanup := setupJWTSessionTest(t) defer cleanup() diff --git a/cmd/dashboard/controller/mcp_test.go b/cmd/dashboard/controller/mcp_test.go index 1b78f0d9..f90f4e23 100644 --- a/cmd/dashboard/controller/mcp_test.go +++ b/cmd/dashboard/controller/mcp_test.go @@ -39,7 +39,12 @@ func setupMCPTest(t *testing.T) (func(), uint64) { require.NoError(t, err) require.NoError(t, db.AutoMigrate(&model.User{}, &model.APIToken{}, &model.MCPAuditLog{}, &model.Server{}, &model.WAF{})) singleton.DB = db - singleton.Conf = &singleton.ConfigClass{Config: &model.Config{JWTTimeout: 1}} + singleton.Conf = &singleton.ConfigClass{Config: &model.Config{ + JWTTimeout: 1, + ConfigDashboard: model.ConfigDashboard{ + DashboardHost: "example.com", + }, + }} singleton.Conf.SetMCPEnabled(true) user := model.User{Common: model.Common{ID: 100}, Username: "alice", Role: model.RoleMember} diff --git a/cmd/dashboard/controller/mcp_transfer.go b/cmd/dashboard/controller/mcp_transfer.go index 6f204811..8aa09670 100644 --- a/cmd/dashboard/controller/mcp_transfer.go +++ b/cmd/dashboard/controller/mcp_transfer.go @@ -13,6 +13,7 @@ import ( "io" "math" "net/http" + "net/url" "strconv" "strings" "sync" @@ -91,6 +92,37 @@ var ( transferSecretVal string ) +var errMCPTransferHostNotConfigured = errors.New("MCP transfer URL host is not an operator-declared dashboard host") + +// mcpTransferURLBase never reflects an untrusted request Host into a bearer +// transfer URL. DashboardHost is the canonical public authority; deployments +// that intentionally serve multiple names must declare them as reserved hosts. +func mcpTransferURLBase(c *gin.Context) (string, error) { + if c == nil || c.Request == nil || singleton.Conf == nil { + return "", errMCPTransferHostNotConfigured + } + + requestHost := strings.TrimSpace(c.Request.Host) + host := strings.TrimSpace(singleton.Conf.DashboardHost) + if host == "" { + if !singleton.IsReservedDashboardHost(requestHost) { + return "", errMCPTransferHostNotConfigured + } + host = requestHost + } + + parsed, err := url.Parse("//" + host) + if err != nil || parsed.Host == "" || parsed.Host != host || parsed.User != nil || parsed.Path != "" || parsed.RawQuery != "" || parsed.Fragment != "" { + return "", errMCPTransferHostNotConfigured + } + + scheme := "https" + if c.Request.TLS == nil && c.Request.Header.Get("X-Forwarded-Proto") != "https" { + scheme = "http" + } + return scheme + "://" + host, nil +} + // transferHMACSecret 返回进程内随机生成的 HMAC key。 // 这是有意设计：transferEntries 本身也只活在内存 sync.Map 里，dashboard // 重启等价于全部 token 失效；让 secret 也随进程随机，可以避免“secret 来自 @@ -315,6 +347,10 @@ func mintTransferTool(c *gin.Context, serverID uint64, path string, ttlSeconds i if err := validateTransferPath(path); err != nil { return nil, err } + baseURL, err := mcpTransferURLBase(c) + if err != nil { + return nil, err + } ttl := time.Duration(ttlSeconds) * time.Second if ttl <= 0 { ttl = transferTokenTTLDefault @@ -348,14 +384,9 @@ func mintTransferTool(c *gin.Context, serverID uint64, path string, ttlSeconds i if err != nil { return nil, err } - scheme := "https" - if c.Request.TLS == nil && c.Request.Header.Get("X-Forwarded-Proto") != "https" { - scheme = "http" - } - host := c.Request.Host - url := fmt.Sprintf("%s://%s/mcp/%s/%s", scheme, host, dir, t) + transferURL := fmt.Sprintf("%s/mcp/%s/%s", baseURL, dir, t) return map[string]any{ - "url": url, + "url": transferURL, "method": map[transferDirection]string{transferDirDownload: "GET", transferDirUpload: "POST"}[dir], "expires_at": entry.ExpiresAt, }, nil diff --git a/cmd/dashboard/controller/mcp_transfer_host_test.go b/cmd/dashboard/controller/mcp_transfer_host_test.go new file mode 100644 index 00000000..58340fa0 --- /dev/null +++ b/cmd/dashboard/controller/mcp_transfer_host_test.go @@ -0,0 +1,80 @@ +package controller + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/nezhahq/nezha/model" + "github.com/nezhahq/nezha/service/singleton" +) + +func TestMCPTransferURLPinsUntrustedRequestHostToDashboardHost(t *testing.T) { + cleanup, uid := setupMCPTest(t) + defer cleanup() + PurgeTransferEntries() + t.Cleanup(func() { PurgeTransferEntries() }) + tok, _ := mkToken(t, uid, []string{model.ScopeServerRead}, nil) + + c, _ := mcpCallCtx(t, tok, uid, nil) + c.Request.Host = "attacker.example" + c.Request.Header.Set("X-Forwarded-Proto", "https") + + out, err := mintTransferTool(c, 7, "/srv/file", 60, transferDirDownload, transferEntry{}) + require.NoError(t, err) + result := out.(map[string]any) + require.Contains(t, result["url"], "https://example.com/mcp/download/") + require.NotContains(t, result["url"], "attacker.example") +} + +func TestMCPTransferURLRejectsUntrustedHostWithoutCanonicalHost(t *testing.T) { + cleanup, uid := setupMCPTest(t) + defer cleanup() + PurgeTransferEntries() + t.Cleanup(func() { PurgeTransferEntries() }) + tok, _ := mkToken(t, uid, []string{model.ScopeServerRead}, nil) + singleton.Conf.DashboardHost = "" + singleton.Conf.InstallHost = "" + singleton.Conf.ListenHost = "" + singleton.Conf.ReservedHosts = "" + + c, _ := mcpCallCtx(t, tok, uid, nil) + c.Request = c.Request.Clone(c.Request.Context()) + c.Request.Method = http.MethodPost + c.Request.Host = "attacker.example" + + _, err := mintTransferTool(c, 7, "/srv/file", 60, transferDirDownload, transferEntry{}) + require.ErrorIs(t, err, errMCPTransferHostNotConfigured) + entries := 0 + transferEntries.Range(func(_, _ any) bool { + entries++ + return true + }) + require.Zero(t, entries, "host validation must happen before a bearer transfer token is minted") +} + +func TestMCPTransferURLAllowsOperatorReservedHost(t *testing.T) { + cleanup, _ := setupMCPTest(t) + defer cleanup() + singleton.Conf.DashboardHost = "" + singleton.Conf.ReservedHosts = "mcp.example.com" + + c, _ := mcpCallCtx(t, nil, 0, nil) + c.Request.Host = "mcp.example.com" + c.Request.Header.Set("X-Forwarded-Proto", "https") + + base, err := mcpTransferURLBase(c) + require.NoError(t, err) + require.Equal(t, "https://mcp.example.com", base) +} + +func TestMCPTransferURLRejectsMalformedConfiguredHost(t *testing.T) { + cleanup, _ := setupMCPTest(t) + defer cleanup() + singleton.Conf.DashboardHost = "https://example.com/path" + + c, _ := mcpCallCtx(t, nil, 0, nil) + _, err := mcpTransferURLBase(c) + require.ErrorIs(t, err, errMCPTransferHostNotConfigured) +} diff --git a/cmd/dashboard/controller/notification.go b/cmd/dashboard/controller/notification.go index af983438..96eb3f6c 100644 --- a/cmd/dashboard/controller/notification.go +++ b/cmd/dashboard/controller/notification.go @@ -1,6 +1,7 @@ package controller import ( + "net/http" "slices" "strconv" @@ -12,6 +13,8 @@ import ( "github.com/nezhahq/nezha/service/singleton" ) +const notificationBatchDeleteMaxBodyBytes = 1 << 20 + // List notification // @Summary List notification // @Security BearerAuth @@ -176,6 +179,9 @@ func updateNotification(c *gin.Context) (any, error) { // @Router /batch-delete/notification [post] func batchDeleteNotification(c *gin.Context) (any, error) { var n []uint64 + if c.Request != nil && c.Request.Body != nil { + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, notificationBatchDeleteMaxBodyBytes) + } if err := c.ShouldBindJSON(&n); err != nil { return nil, err } diff --git a/cmd/dashboard/controller/notification_body_limit_test.go b/cmd/dashboard/controller/notification_body_limit_test.go new file mode 100644 index 00000000..4501cee9 --- /dev/null +++ b/cmd/dashboard/controller/notification_body_limit_test.go @@ -0,0 +1,25 @@ +package controller + +import ( + "bytes" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gin-gonic/gin" +) + +func TestBatchDeleteNotificationCapsRequestBody(t *testing.T) { + body := append(bytes.Repeat([]byte{' '}, notificationBatchDeleteMaxBodyBytes+1), '[', ']') + req := httptest.NewRequest(http.MethodPost, "/api/v1/batch-delete/notification", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = req + + _, err := batchDeleteNotification(c) + if err == nil || !strings.Contains(err.Error(), "request body too large") { + t.Fatalf("oversized request body was not rejected by MaxBytesReader: %v", err) + } +} diff --git a/cmd/dashboard/controller/online_history.go b/cmd/dashboard/controller/online_history.go new file mode 100644 index 00000000..302f1029 --- /dev/null +++ b/cmd/dashboard/controller/online_history.go @@ -0,0 +1,121 @@ +package controller + +import ( + "strconv" + "time" + + "github.com/gin-gonic/gin" + + "github.com/nezhahq/nezha/model" + "github.com/nezhahq/nezha/service/singleton" +) + +// Get server online history +// @Summary Get server online history +// @Security BearerAuth +// @Schemes +// @Description Get online periods and aggregated uptime statistics of a specific server +// @Tags common +// @Param id path uint true "Server ID" +// @Param days query int false "Query window in days (default: configured online_stats_days, clamped to it)" +// @Produce json +// @Success 200 {object} model.CommonResponse[model.OnlineHistoryResponse] +// @Router /server/{id}/online-history [get] +func getServerOnlineHistory(c *gin.Context) (*model.OnlineHistoryResponse, error) { + idStr := c.Param("id") + serverID, err := strconv.ParseUint(idStr, 10, 64) + if err != nil { + return nil, err + } + + server, ok := singleton.ServerShared.Get(serverID) + if !ok || server == nil { + return nil, singleton.Localizer.ErrorT("server not found") + } + if !userCanViewServer(c, server) { + return nil, singleton.Localizer.ErrorT("unauthorized") + } + _, isMember := c.Get(model.CtxKeyAuthorizedUser) + + maxDays := singleton.Conf.OnlineHistoryRetentionDays() + if maxDays < model.MinOnlineStatsDays || maxDays > model.MaxOnlineStatsDays { + maxDays = model.DefaultOnlineStatsDays + } + days := maxDays + if daysStr := c.Query("days"); daysStr != "" { + d, err := strconv.Atoi(daysStr) + if err != nil || d < 1 { + return nil, singleton.Localizer.ErrorT("invalid days parameter") + } + // 查询窗口收敛到后台设置的统计范围内 + if d > maxDays { + d = maxDays + } + days = d + } + + if !isMember && days > 1 { + return nil, singleton.Localizer.ErrorT("unauthorized: only 1d data available for guests") + } + + now := time.Now() + windowEnd := now + windowStart := model.OnlineHistoryWindowStart(now, days, singleton.Loc) + + sessions, err := singleton.OnlineHistoryShared.Sessions(serverID, windowStart, windowEnd) + if err != nil { + return nil, newGormError("%v", err) + } + + periods := model.ClipOnlineSessions(sessions, windowStart, windowEnd) + daily := model.DailyOnlineBreakdown(periods, windowStart, windowEnd, singleton.Loc) + // 总时长取各日之和，与 daily[] 严格对账（每日累计原始时长后统一取整， + // 口径见 DailyOnlineBreakdown） + var totalSeconds uint64 + for _, d := range daily { + totalSeconds += d.OnlineSeconds + } + windowSeconds := uint64(windowEnd.Sub(windowStart).Seconds()) + onlineRate := 0.0 + if windowSeconds > 0 { + onlineRate = float64(totalSeconds) / float64(windowSeconds) * 100 + } + + runtime := server.RuntimeSnapshot() + currentlyOnline := !runtime.LastActive.IsZero() && now.Sub(runtime.LastActive) <= model.OnlineActiveWindow + var lastActive *time.Time + if !runtime.LastActive.IsZero() { + lastActiveUTC := runtime.LastActive.UTC() + lastActive = &lastActiveUTC + } + + // 响应统一以 UTC 序列化，避免窗口时间与库中读回的会话时间混用 + // 本地偏移与 Z 两种写法（时段为值拷贝，不影响调用方） + utcPeriods := make([]model.OnlinePeriod, len(periods)) + for i, p := range periods { + utcPeriods[i] = model.OnlinePeriod{Start: p.Start.UTC(), End: p.End.UTC(), Ongoing: p.Ongoing} + } + + return &model.OnlineHistoryResponse{ + ServerID: serverID, + ServerName: server.Name, + Days: days, + MaxDays: maxDays, + Timezone: singleton.Loc.String(), + TzOffset: tzOffsetSeconds(now, singleton.Loc), + WindowStart: windowStart.UTC(), + WindowEnd: windowEnd.UTC(), + TotalOnlineSeconds: totalSeconds, + OnlineRate: onlineRate, + CurrentlyOnline: currentlyOnline, + LastActive: lastActive, + Periods: utcPeriods, + Daily: daily, + }, nil +} + +// tzOffsetSeconds 返回指定时区在给定时刻相对 UTC 的偏移秒数。 +func tzOffsetSeconds(at time.Time, loc *time.Location) int { + _, offset := at.In(loc).Zone() + return offset +} diff --git a/cmd/dashboard/controller/online_history_test.go b/cmd/dashboard/controller/online_history_test.go new file mode 100644 index 00000000..544703a4 --- /dev/null +++ b/cmd/dashboard/controller/online_history_test.go @@ -0,0 +1,356 @@ +package controller + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + + "github.com/nezhahq/nezha/model" + "github.com/nezhahq/nezha/pkg/i18n" + "github.com/nezhahq/nezha/service/singleton" +) + +func setupOnlineHistoryControllerTest(t *testing.T) (visibleServer, hiddenServer *model.Server) { + t.Helper() + originalDB := singleton.DB + originalLocalizer := singleton.Localizer + originalServer := singleton.ServerShared + originalConf := singleton.Conf + originalLoc := singleton.Loc + if singleton.Localizer == nil { + singleton.Localizer = i18n.NewLocalizer("en_US", "nezha", "translations", i18n.Translations) + } + + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + sqlDB, err := db.DB() + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&model.User{}, &model.Server{}, &model.OnlineSession{})) + + singleton.DB = db + singleton.ServerShared = singleton.NewEmptyServerClassForTest() + singleton.Conf = &singleton.ConfigClass{Config: &model.Config{}} + singleton.Conf.OnlineStatsDays = 30 + singleton.Loc = time.FixedZone("TEST", 8*3600) + + visibleServer = &model.Server{ + Common: model.Common{ID: 1}, + Name: "visible", + UUID: "online-history-visible", + LastActive: time.Now(), + } + hiddenServer = &model.Server{ + Common: model.Common{ID: 2}, + Name: "hidden", + UUID: "online-history-hidden", + HideForGuest: true, + } + singleton.ServerShared.InsertForTest(visibleServer) + singleton.ServerShared.InsertForTest(hiddenServer) + + t.Cleanup(func() { + _ = sqlDB.Close() + singleton.DB = originalDB + singleton.Localizer = originalLocalizer + singleton.ServerShared = originalServer + singleton.Conf = originalConf + singleton.Loc = originalLoc + }) + return visibleServer, hiddenServer +} + +func onlineHistoryCtx(t *testing.T, serverID string, query string) *gin.Context { + t.Helper() + gin.SetMode(gin.TestMode) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/server/"+serverID+"/online-history?"+query, nil) + c.Params = gin.Params{{Key: "id", Value: serverID}} + return c +} + +func onlineHistoryCtxAsMember(t *testing.T, serverID string, query string) *gin.Context { + t.Helper() + c := onlineHistoryCtx(t, serverID, query) + c.Set(model.CtxKeyAuthorizedUser, &model.User{Common: model.Common{ID: 100}, Role: model.RoleMember}) + return c +} + +func seedOnlineSessions(t *testing.T, serverID uint64, now time.Time) []model.OnlineSession { + t.Helper() + now = now.Truncate(time.Second) + sessions := []model.OnlineSession{ + {ServerID: serverID, StartedAt: now.Add(-48 * time.Hour), EndedAt: now.Add(-47 * time.Hour), LastActiveAt: now.Add(-47 * time.Hour)}, + {ServerID: serverID, StartedAt: now.Add(-25 * time.Hour), EndedAt: now.Add(-23*time.Hour - 30*time.Minute), LastActiveAt: now.Add(-23 * time.Hour)}, + {ServerID: serverID, StartedAt: now.Add(-10 * time.Minute), Ongoing: true, LastActiveAt: now}, + // 超出默认 7 天窗口、但在 30 天窗口内的会话 + {ServerID: serverID, StartedAt: now.Add(-20 * time.Hour * 24), EndedAt: now.Add(-20*time.Hour*24 + time.Hour), LastActiveAt: now.Add(-20 * time.Hour * 24)}, + } + for i := range sessions { + require.NoError(t, singleton.DB.Create(&sessions[i]).Error) + } + return sessions +} + +func TestGetServerOnlineHistoryDefaultsToConfiguredWindow(t *testing.T) { + visible, _ := setupOnlineHistoryControllerTest(t) + now := time.Now() + seedOnlineSessions(t, visible.ID, now) + + // 不带 days 参数：使用后台设置的统计范围（30 天） + c := onlineHistoryCtxAsMember(t, "1", "") + resp, err := getServerOnlineHistory(c) + require.NoError(t, err) + require.Equal(t, 30, resp.Days) + require.Equal(t, 30, resp.MaxDays) + // 4 个会话全部落在 30 天窗口内 + require.Len(t, resp.Periods, 4) + require.True(t, resp.CurrentlyOnline) +} + +func TestGetServerOnlineHistoryDefaultIsSevenDays(t *testing.T) { + visible, _ := setupOnlineHistoryControllerTest(t) + now := time.Now() + seedOnlineSessions(t, visible.ID, now) + + // 出厂默认统计范围为 7 天 + singleton.Conf.OnlineStatsDays = model.DefaultOnlineStatsDays + c := onlineHistoryCtxAsMember(t, "1", "") + resp, err := getServerOnlineHistory(c) + require.NoError(t, err) + require.Equal(t, 7, resp.Days) + // 20 天前的会话不进入 7 天窗口 + require.Len(t, resp.Periods, 3) +} + +func TestGetServerOnlineHistoryAggregatesAndClampsWindow(t *testing.T) { + visible, _ := setupOnlineHistoryControllerTest(t) + now := time.Now() + seedOnlineSessions(t, visible.ID, now) + + c := onlineHistoryCtxAsMember(t, "1", "days=9999") + resp, err := getServerOnlineHistory(c) + require.NoError(t, err) + // 请求超出后台设置窗口时收敛到设置值 + require.Equal(t, 30, resp.Days) + + // 聚合正确性：三个已结束/进行中会话都计入，进行中会话按当前时间截断 + var total uint64 + for _, p := range resp.Periods { + total += uint64(p.End.Sub(p.Start).Seconds()) + } + require.Equal(t, resp.TotalOnlineSeconds, total) + require.GreaterOrEqual(t, resp.TotalOnlineSeconds, uint64(3600+5400+599)) + + // 每日拆分合计等于总时长 + var dailySum uint64 + for _, d := range resp.Daily { + dailySum += d.OnlineSeconds + } + require.Equal(t, resp.TotalOnlineSeconds, dailySum) + + // 在线率按窗口秒数计算 + windowSeconds := resp.WindowEnd.Sub(resp.WindowStart).Seconds() + require.InDelta(t, float64(resp.TotalOnlineSeconds)/windowSeconds*100, resp.OnlineRate, 0.001) +} + +func TestGetServerOnlineHistoryInvalidDays(t *testing.T) { + setupOnlineHistoryControllerTest(t) + for _, q := range []string{"days=abc", "days=0", "days=-1"} { + c := onlineHistoryCtxAsMember(t, "1", q) + _, err := getServerOnlineHistory(c) + require.EqualError(t, err, singleton.Localizer.ErrorT("invalid days parameter").Error(), "query %q should be rejected", q) + } +} + +func TestGetServerOnlineHistoryUnknownServer(t *testing.T) { + setupOnlineHistoryControllerTest(t) + c := onlineHistoryCtx(t, "424242", "") + _, err := getServerOnlineHistory(c) + require.Error(t, err) +} + +func TestGetServerOnlineHistoryGuestLimitedToOneDay(t *testing.T) { + visible, _ := setupOnlineHistoryControllerTest(t) + now := time.Now() + seedOnlineSessions(t, visible.ID, now) + + // 游客不带 days 参数时默认为配置窗口（>1 天），与 metrics 的游客语义一致被拒绝 + _, err := getServerOnlineHistory(onlineHistoryCtx(t, "1", "")) + require.Error(t, err) + + // 游客显式请求多天被拒绝，1 天可用 + _, err = getServerOnlineHistory(onlineHistoryCtx(t, "1", "days=3")) + require.Error(t, err) + resp, err := getServerOnlineHistory(onlineHistoryCtx(t, "1", "days=1")) + require.NoError(t, err) + require.Equal(t, 1, resp.Days) + // 与 1 天窗口相交的两个会话：一个被裁剪到窗口起点，一个进行中 + require.Len(t, resp.Periods, 2) + + // 登录用户不受游客窗口限制 + resp, err = getServerOnlineHistory(onlineHistoryCtxAsMember(t, "1", "days=30")) + require.NoError(t, err) + require.Equal(t, 30, resp.Days) + require.Len(t, resp.Periods, 4) +} + +func TestGetServerOnlineHistoryPermission(t *testing.T) { + visible, _ := setupOnlineHistoryControllerTest(t) + + // 游客可以查看未隐藏服务器（窗口受限于 1 天） + guest := onlineHistoryCtx(t, "1", "days=1") + resp, err := getServerOnlineHistory(guest) + require.NoError(t, err) + require.Equal(t, visible.ID, resp.ServerID) + + // 游客不能查看 HideForGuest 服务器 + guestHidden := onlineHistoryCtx(t, "2", "days=1") + _, err = getServerOnlineHistory(guestHidden) + require.EqualError(t, err, singleton.Localizer.ErrorT("unauthorized").Error()) + + // 成员不能查看他人隐藏服务器 + member := onlineHistoryCtx(t, "2", "days=7") + member.Set(model.CtxKeyAuthorizedUser, &model.User{Common: model.Common{ID: 100}, Role: model.RoleMember}) + _, err = getServerOnlineHistory(member) + require.Error(t, err) +} + +func patchOnlineStatsDays(t *testing.T, days any) error { + t.Helper() + gin.SetMode(gin.TestMode) + body, err := json.Marshal(map[string]any{"user_template": "user-dist", "online_stats_days": days}) + require.NoError(t, err) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodPatch, "/api/v1/setting", bytes.NewReader(body)) + c.Request.Header.Set("Content-Type", "application/json") + _, err = updateConfig(c) + return err +} + +func TestUpdateConfigOnlineStatsDaysRange(t *testing.T) { + originalLocalizer := singleton.Localizer + if singleton.Localizer == nil { + singleton.Localizer = i18n.NewLocalizer("en_US", "nezha", "translations", i18n.Translations) + } + t.Cleanup(func() { singleton.Localizer = originalLocalizer }) + + templates := []model.FrontendTemplate{{Path: "user-dist", IsAdmin: false}} + originalFrontendTemplates := singleton.FrontendTemplates + singleton.FrontendTemplates = templates + t.Cleanup(func() { singleton.FrontendTemplates = originalFrontendTemplates }) + conf := &singleton.ConfigClass{Config: &model.Config{}} + configPath := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, conf.Read(configPath, templates)) + originalConf := singleton.Conf + singleton.Conf = conf + t.Cleanup(func() { singleton.Conf = originalConf }) + + require.Equal(t, model.DefaultOnlineStatsDays, conf.OnlineStatsDays) + + // 合法边界值 + for _, d := range []int{model.MinOnlineStatsDays, 30, model.MaxOnlineStatsDays} { + require.NoError(t, patchOnlineStatsDays(t, d), "days=%d should be accepted", d) + require.Equal(t, d, conf.OnlineStatsDays) + } + + // 越界值整体拒绝且不修改现值 + for _, d := range []int{6, model.MaxOnlineStatsDays + 1, 0, -5} { + require.Error(t, patchOnlineStatsDays(t, d), "days=%d should be rejected", d) + require.Equal(t, model.MaxOnlineStatsDays, conf.OnlineStatsDays) + } + + // 非法值必须整体拒绝：校验先于任何对 Conf 的写入完成，其他字段不得 + // 半提交（评审复现场景：错误返回后 SiteName 不能被请求内容覆盖） + conf.SiteName = "existing-site" + require.Error(t, patchOnlineStatsDays(t, 6)) + require.Equal(t, "existing-site", conf.SiteName) + conf.SiteName = "" + + // 字段缺省时保留现值（部分 PATCH 语义） + require.NoError(t, patchOnlineStatsDays(t, nil)) + require.Equal(t, model.MaxOnlineStatsDays, conf.OnlineStatsDays) + + // 重启持久化断言：updateConfig 显式 Save 后，重新从磁盘读取的配置 + // 必须带上刚写入的值（持久化不再依赖 MCP 过渡块的副作用） + reloaded := &singleton.ConfigClass{Config: &model.Config{}} + require.NoError(t, reloaded.Read(configPath, templates)) + require.Equal(t, model.MaxOnlineStatsDays, reloaded.OnlineStatsDays) +} + +func TestSettingFormOnlineStatsDaysDecode(t *testing.T) { + var absent model.SettingForm + require.NoError(t, json.NewDecoder(bytes.NewReader([]byte(`{"site_name":"x"}`))).Decode(&absent)) + require.Nil(t, absent.OnlineStatsDays, "omitted key must decode to nil") + + var explicit model.SettingForm + require.NoError(t, json.NewDecoder(bytes.NewReader([]byte(`{"online_stats_days":3650}`))).Decode(&explicit)) + require.NotNil(t, explicit.OnlineStatsDays) + require.Equal(t, 3650, *explicit.OnlineStatsDays) +} + +func TestOnlineHistoryReadsPendingCloseBoundary(t *testing.T) { + setupOnlineHistoryControllerTest(t) + previous := singleton.OnlineHistoryShared + t.Cleanup(func() { singleton.OnlineHistoryShared = previous }) + r := singleton.NewOnlineHistoryRecorder() + singleton.OnlineHistoryShared = r + start := time.Now().UTC().Add(-time.Hour).Truncate(time.Second) + r.RecordOnline(1, start) + require.NoError(t, singleton.DB.Exec(`CREATE TRIGGER reject_close BEFORE UPDATE ON online_sessions WHEN NEW.ongoing = 0 BEGIN SELECT RAISE(ABORT, 'close failure'); END`).Error) + r.RecordOffline(1, start.Add(time.Minute)) + r.RecordOnline(1, start.Add(30*time.Minute)) + resp, err := getServerOnlineHistory(onlineHistoryCtx(t, "1", "days=1")) + require.NoError(t, err) + require.Len(t, resp.Periods, 2) + require.False(t, resp.Periods[0].Ongoing) + require.True(t, resp.Periods[0].End.Equal(start.Add(time.Minute))) + require.Less(t, resp.TotalOnlineSeconds, uint64(1900)) +} + +func TestOnlineHistoryQueryWithNonUTCSession(t *testing.T) { + setupOnlineHistoryControllerTest(t) + now := time.Now() + offset := time.FixedZone("UTC+12", 12*3600) + session := model.OnlineSession{ServerID: 1, StartedAt: now.Add(-time.Hour).In(offset), LastActiveAt: now.Add(-time.Minute).In(offset), Ongoing: true} + require.NoError(t, singleton.DB.Create(&session).Error) + resp, err := getServerOnlineHistory(onlineHistoryCtx(t, "1", "days=1")) + require.NoError(t, err) + require.Len(t, resp.Periods, 1) + require.True(t, resp.Periods[0].Start.Equal(session.StartedAt)) + require.InDelta(t, 3600, resp.TotalOnlineSeconds, 2) +} + +func TestUpdateConfigFailedSavePreservesRetention(t *testing.T) { + server, _ := setupOnlineHistoryControllerTest(t) + require.NoError(t, singleton.DB.Create(server).Error) + templates := []model.FrontendTemplate{{Path: "user-dist"}} + previous := singleton.FrontendTemplates + singleton.FrontendTemplates = templates + t.Cleanup(func() { singleton.FrontendTemplates = previous }) + path := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, singleton.Conf.Read(path, templates)) + singleton.Conf.OnlineStatsDays = 30 + require.NoError(t, singleton.Conf.Save()) + // Replacing the destination with a directory deterministically fails Save even as root. + require.NoError(t, os.Remove(path)) + require.NoError(t, os.Mkdir(path, 0700)) + now := time.Now().UTC() + old := model.OnlineSession{ServerID: 1, StartedAt: now.AddDate(0, 0, -15), EndedAt: now.AddDate(0, 0, -14)} + require.NoError(t, singleton.DB.Create(&old).Error) + require.Error(t, patchOnlineStatsDays(t, 7)) + require.Equal(t, 30, singleton.Conf.OnlineStatsDays) + singleton.PruneOnlineSessions() + var retained model.OnlineSession + require.NoError(t, singleton.DB.First(&retained, old.ID).Error) + require.Equal(t, old.ID, retained.ID) +} diff --git a/cmd/dashboard/controller/server.go b/cmd/dashboard/controller/server.go index 62e8ee95..62f355d1 100644 --- a/cmd/dashboard/controller/server.go +++ b/cmd/dashboard/controller/server.go @@ -236,10 +236,9 @@ func getServerConfig(c *gin.Context) (string, error) { } s, ok := singleton.ServerShared.Get(id) - if !ok { - return "", nil - } - if !s.HasPermission(c) { + // Foreign and unknown IDs deliberately share one response so sequential + // server IDs cannot be enumerated across tenants. + if !ok || !s.HasPermission(c) || !patAllowsServer(c, id) { return "", singleton.Localizer.ErrorT("permission denied") } if s.GetTaskStream() == nil { @@ -293,16 +292,16 @@ func setServerConfig(c *gin.Context) (*model.ServerTaskResponse, error) { slist := singleton.ServerShared.GetList() servers := make([]*model.Server, 0, len(configForm.Servers)) for _, sid := range configForm.Servers { - if s, ok := slist[sid]; ok { - if !s.HasPermission(c) { - return nil, singleton.Localizer.ErrorT("permission denied") - } - if s.GetTaskStream() == nil { - resp.Offline = append(resp.Offline, s.ID) - continue - } - servers = append(servers, s) + s, ok := slist[sid] + // Match getServerConfig: do not reveal whether a denied ID exists. + if !ok || !s.HasPermission(c) || !patAllowsServer(c, sid) { + return nil, singleton.Localizer.ErrorT("permission denied") } + if s.GetTaskStream() == nil { + resp.Offline = append(resp.Offline, s.ID) + continue + } + servers = append(servers, s) } var wg sync.WaitGroup diff --git a/cmd/dashboard/controller/server_config_oracle_test.go b/cmd/dashboard/controller/server_config_oracle_test.go new file mode 100644 index 00000000..6ffe0542 --- /dev/null +++ b/cmd/dashboard/controller/server_config_oracle_test.go @@ -0,0 +1,54 @@ +package controller + +import ( + "net/http" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + + "github.com/nezhahq/nezha/model" + "github.com/nezhahq/nezha/service/singleton" +) + +func TestGetServerConfigForeignAndUnknownIDsAreIndistinguishable(t *testing.T) { + cleanup, _ := setupMCPTest(t) + defer cleanup() + require.NoError(t, singleton.DB.Create(&model.User{Common: model.Common{ID: 200}, Username: "bob", Role: model.RoleMember}).Error) + tok, _ := mkToken(t, 200, []string{model.ScopeServerRead}, nil) + + probe := func(id string) error { + c, _ := patRequestCtx(t, tok, 200, http.MethodGet, "/api/v1/server/config/"+id, nil) + c.Params = gin.Params{{Key: "id", Value: id}} + _, err := getServerConfig(c) + return err + } + + foreignErr := probe("7") + unknownErr := probe("999999") + require.Error(t, foreignErr) + require.Error(t, unknownErr) + require.Equal(t, foreignErr.Error(), unknownErr.Error()) +} + +func TestSetServerConfigForeignAndUnknownIDsAreIndistinguishable(t *testing.T) { + cleanup, _ := setupMCPTest(t) + defer cleanup() + require.NoError(t, singleton.DB.Create(&model.User{Common: model.Common{ID: 200}, Username: "bob", Role: model.RoleMember}).Error) + tok, _ := mkToken(t, 200, []string{model.ScopeServerWrite}, nil) + + probe := func(id uint64) error { + c, _ := patRequestCtx(t, tok, 200, http.MethodPost, "/api/v1/server/config", model.ServerConfigForm{ + Servers: []uint64{id}, + Config: "{}", + }) + _, err := setServerConfig(c) + return err + } + + foreignErr := probe(7) + unknownErr := probe(999999) + require.Error(t, foreignErr) + require.Error(t, unknownErr) + require.Equal(t, foreignErr.Error(), unknownErr.Error()) +} diff --git a/cmd/dashboard/controller/setting.go b/cmd/dashboard/controller/setting.go index b36d4616..d5a58c0f 100644 --- a/cmd/dashboard/controller/setting.go +++ b/cmd/dashboard/controller/setting.go @@ -2,6 +2,7 @@ package controller import ( "errors" + "fmt" "log" "strings" @@ -29,7 +30,7 @@ func listConfig(c *gin.Context) (*model.SettingResponse, error) { isAdmin = user.Role.IsAdmin() } - config := *singleton.Conf + config := singleton.Conf.SettingsSnapshot() config.Language = strings.ReplaceAll(config.Language, "_", "-") conf := model.SettingResponse{ @@ -92,11 +93,20 @@ func updateConfig(c *gin.Context) (any, error) { if !userTemplateValid { return nil, errors.New("invalid user template") } + // 全部输入校验必须先于任何对 singleton.Conf 的写入：任一字段非法都应 + // 整体拒绝，避免"请求以错误收场、先处理的字段却已生效"的半提交状态 + if sf.OnlineStatsDays != nil && + (*sf.OnlineStatsDays < model.MinOnlineStatsDays || *sf.OnlineStatsDays > model.MaxOnlineStatsDays) { + return nil, fmt.Errorf("online_stats_days must be between %d and %d", model.MinOnlineStatsDays, model.MaxOnlineStatsDays) + } singleton.Conf.Language = strings.ReplaceAll(sf.Language, "-", "_") singleton.Conf.EnableIPChangeNotification = sf.EnableIPChangeNotification singleton.Conf.EnablePlainIPInNotification = sf.EnablePlainIPInNotification + if sf.AllowJWTIPChange != nil { + singleton.Conf.SetJWTIPChangeAllowed(*sf.AllowJWTIPChange) + } singleton.Conf.Cover = sf.Cover singleton.Conf.InstallHost = sf.InstallHost singleton.Conf.DashboardHost = sf.DashboardHost @@ -124,18 +134,22 @@ func updateConfig(c *gin.Context) (any, error) { singleton.Conf.DomainExpiryNotificationDays = sf.DomainExpiryNotificationDays singleton.Conf.ServerExpiryNotificationDays = sf.ServerExpiryNotificationDays - mcpWasEnabled := singleton.Conf.MCPEnabled() - mcpNext := resolveSettingEnableMCP(sf.EnableMCP, mcpWasEnabled) - - if err := applyEnableMCPTransition( - mcpWasEnabled, mcpNext, - singleton.Conf.SetMCPEnabled, - singleton.Conf.Save, - fireMCPKillSwitch, - ); err != nil { + if err := singleton.Conf.CommitOnlineHistoryRetention(sf.OnlineStatsDays, func() error { + mcpWasEnabled := singleton.Conf.MCPEnabled() + mcpNext := resolveSettingEnableMCP(sf.EnableMCP, mcpWasEnabled) + return applyEnableMCPTransition( + mcpWasEnabled, mcpNext, + singleton.Conf.SetMCPEnabled, + singleton.Conf.Save, + fireMCPKillSwitch, + ) + }); err != nil { return nil, newGormError("%v", err) } + // applyEnableMCPTransition already persisted every setting, even when MCP + // is unchanged. A second save could fail after irreversible cleanup. + singleton.OnUpdateLang(singleton.Conf.Language) singleton.InitTelegramBot() return nil, nil @@ -174,6 +188,12 @@ func fireMCPKillSwitch() { // the current config to avoid accidentally tripping the kill switch on // partial PATCH calls that omit enable_mcp. func resolveSettingEnableMCP(formValue *bool, current bool) bool { + return resolveOptionalBool(formValue, current) +} + +// resolveOptionalBool preserves a setting for partial PATCH requests when its +// boolean field is absent, while still allowing an explicit false value. +func resolveOptionalBool(formValue *bool, current bool) bool { if formValue == nil { return current } diff --git a/cmd/dashboard/controller/setting_jwt_ip_change_test.go b/cmd/dashboard/controller/setting_jwt_ip_change_test.go new file mode 100644 index 00000000..a3f5229b --- /dev/null +++ b/cmd/dashboard/controller/setting_jwt_ip_change_test.go @@ -0,0 +1,26 @@ +package controller + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/nezhahq/nezha/model" +) + +func TestSettingForm_AllowJWTIPChangePreservesAbsentAndDecodesExplicitValues(t *testing.T) { + var omitted model.SettingForm + require.NoError(t, json.Unmarshal([]byte(`{"site_name":"X"}`), &omitted)) + require.Nil(t, omitted.AllowJWTIPChange) + + var enabled model.SettingForm + require.NoError(t, json.Unmarshal([]byte(`{"allow_jwt_ip_change":true}`), &enabled)) + require.NotNil(t, enabled.AllowJWTIPChange) + require.True(t, *enabled.AllowJWTIPChange) + + var disabled model.SettingForm + require.NoError(t, json.Unmarshal([]byte(`{"allow_jwt_ip_change":false}`), &disabled)) + require.NotNil(t, disabled.AllowJWTIPChange) + require.False(t, *disabled.AllowJWTIPChange) +} diff --git a/cmd/dashboard/controller/setting_single_commit_test.go b/cmd/dashboard/controller/setting_single_commit_test.go new file mode 100644 index 00000000..ee200526 --- /dev/null +++ b/cmd/dashboard/controller/setting_single_commit_test.go @@ -0,0 +1,49 @@ +package controller + +import ( + "go/ast" + "go/parser" + "go/token" + "testing" + + "github.com/stretchr/testify/require" +) + +// The transition helper owns Save (including unchanged MCP settings). A direct +// second Save after it can fail after persistence and irreversible revocation. +func TestUpdateConfigHasSinglePersistenceOwner(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), "setting.go", nil, 0) + require.NoError(t, err) + transitions, directSaves := 0, 0 + found := false + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Name.Name != "updateConfig" { + continue + } + found = true + ast.Inspect(fn.Body, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + if ident, ok := call.Fun.(*ast.Ident); ok && ident.Name == "applyEnableMCPTransition" { + transitions++ + } + if selector, ok := call.Fun.(*ast.SelectorExpr); ok && selector.Sel.Name == "Save" { + directSaves++ + } + return true + }) + } + require.True(t, found) + require.Equal(t, 1, transitions) + require.Zero(t, directSaves) +} + +func TestUnchangedMCPTransitionStillSavesOnce(t *testing.T) { + saves := 0 + err := applyEnableMCPTransition(true, true, func(bool) {}, func() error { saves++; return nil }, func() { t.Error("unchanged MCP must not revoke streams") }) + require.NoError(t, err) + require.Equal(t, 1, saves) +} diff --git a/cmd/dashboard/controller/terminal_input_limit_test.go b/cmd/dashboard/controller/terminal_input_limit_test.go new file mode 100644 index 00000000..8fb4cbab --- /dev/null +++ b/cmd/dashboard/controller/terminal_input_limit_test.go @@ -0,0 +1,28 @@ +package controller + +import "testing" + +type readLimitRecorder struct { + limit int64 +} + +func (r *readLimitRecorder) SetReadLimit(limit int64) { + r.limit = limit +} + +func TestTerminalWebSocketInputLimitAllowsBoundedPaste(t *testing.T) { + if terminalWebSocketInputLimit != 512*1024+64 { + t.Fatalf("terminal WebSocket input limit = %d, want 512 KiB plus control-byte allowance", terminalWebSocketInputLimit) + } +} + +func TestFileManagerWebSocketInputLimitMatchesUploadChunk(t *testing.T) { + if fileManagerWebSocketInputLimit != 1024*1024 { + t.Fatalf("file-manager WebSocket input limit = %d, want official client's 1 MiB upload chunk", fileManagerWebSocketInputLimit) + } + recorder := new(readLimitRecorder) + limitFileManagerWebSocketInput(recorder) + if recorder.limit != fileManagerWebSocketInputLimit { + t.Fatalf("file-manager WebSocket configured limit = %d, want %d", recorder.limit, fileManagerWebSocketInputLimit) + } +} diff --git a/cmd/dashboard/controller/transfer.go b/cmd/dashboard/controller/transfer.go index 0e8f3ca7..f3c76b59 100644 --- a/cmd/dashboard/controller/transfer.go +++ b/cmd/dashboard/controller/transfer.go @@ -171,6 +171,12 @@ const transferStreamPingInterval = 30 * time.Second // @Success 200 {object} model.ServerTransfer // @Router /ws/transfer [get] func transferStream(c *gin.Context) (any, error) { + subID, ch, err := singleton.ServerTransferShared.Subscribe(getUid(c)) + if err != nil { + return nil, err + } + defer singleton.ServerTransferShared.Unsubscribe(subID) + conn, err := upgrader.Upgrade(c.Writer, c.Request, nil) if err != nil { return nil, newWsError("%v", err) @@ -180,9 +186,6 @@ func transferStream(c *gin.Context) (any, error) { deregisterPAT := registerPATConnection(c, func() { _ = conn.Close() }) defer deregisterPAT() - subID, ch := singleton.ServerTransferShared.Subscribe() - defer singleton.ServerTransferShared.Unsubscribe(subID) - // Pings keep the socket warm even when the broker is quiet. Without this // a long idle period followed by a transfer event would race against // upstream proxy idle-timeouts that may have already closed the conn. diff --git a/cmd/dashboard/dashboard_credential_wiring_test.go b/cmd/dashboard/dashboard_credential_wiring_test.go new file mode 100644 index 00000000..d53e7c8f --- /dev/null +++ b/cmd/dashboard/dashboard_credential_wiring_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "testing" + + "github.com/nezhahq/nezha/cmd/dashboard/rpc" +) + +// TestWireNATDashboardCredentialGate pins the NAT ingress wiring contract: +// main() must connect the real batched dashboard-credential classifier +// (controller.ClassifyDashboardCredentialValues) +// through wireNATDashboardCredentialGate before any listener serves traffic. +// That the wired functions really classify per channel is pinned end to end +// by the controller package's real-classifier NAT flow test. +func TestWireNATDashboardCredentialGate(t *testing.T) { + if rpc.NATDashboardCredentialGateWired() { + t.Fatal("NAT dashboard credential gate already wired before the test") + } + + wireNATDashboardCredentialGate() + + if !rpc.NATDashboardCredentialGateWired() { + t.Fatal("wireNATDashboardCredentialGate left the NAT dashboard credential gate unwired") + } + // Leave the process in the wired state main() establishes. + t.Cleanup(wireNATDashboardCredentialGate) +} diff --git a/cmd/dashboard/http_header_timeout.go b/cmd/dashboard/http_header_timeout.go new file mode 100644 index 00000000..3db55188 --- /dev/null +++ b/cmd/dashboard/http_header_timeout.go @@ -0,0 +1,14 @@ +package main + +import ( + "net/http" + "time" +) + +const dashboardReadHeaderTimeout = 5 * time.Second + +func newDashboardHTTPServer(handler http.Handler) *http.Server { + // ReadHeaderTimeout protects incomplete HTTP/1 requests and is cleared after + // headers. Unlike ReadTimeout it does not impose a lifetime on HTTP/2 streams. + return &http.Server{Handler: handler, ReadHeaderTimeout: dashboardReadHeaderTimeout} +} diff --git a/cmd/dashboard/http_header_timeout_test.go b/cmd/dashboard/http_header_timeout_test.go new file mode 100644 index 00000000..d643afaa --- /dev/null +++ b/cmd/dashboard/http_header_timeout_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "crypto/tls" + "io" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestDashboardRejectsIncompleteHTTPHeaders(t *testing.T) { + for _, useTLS := range []bool{false, true} { + name := "http" + if useTLS { + name = "https" + } + t.Run(name, func(t *testing.T) { + server := httptest.NewUnstartedServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("incomplete headers reached handler") })) + server.Config = newDashboardHTTPServer(server.Config.Handler) + require.Positive(t, server.Config.ReadHeaderTimeout) + server.Config.ReadHeaderTimeout = 50 * time.Millisecond + if useTLS { + server.StartTLS() + } else { + server.Start() + } + defer server.Close() + var conn net.Conn + var err error + if useTLS { + conn, err = tls.Dial("tcp", server.Listener.Addr().String(), &tls.Config{InsecureSkipVerify: true}) + } else { + conn, err = net.Dial("tcp", server.Listener.Addr().String()) + } + require.NoError(t, err) + defer conn.Close() + require.NoError(t, conn.SetDeadline(time.Now().Add(2*time.Second))) + _, err = io.WriteString(conn, "GET / HTTP/1.1\r\nHost: dashboard\r\nX-Incomplete: ") + require.NoError(t, err) + _, err = io.ReadAll(conn) + if networkErr, ok := err.(net.Error); ok { + require.False(t, networkErr.Timeout(), "server did not close incomplete request") + } + }) + } +} + +func TestDashboardHeaderTimeoutDoesNotCancelHTTP2Stream(t *testing.T) { + for _, useTLS := range []bool{false, true} { + name := "h2c" + if useTLS { + name = "h2" + } + t.Run(name, func(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.ProtoMajor != 2 { + t.Errorf("expected HTTP/2, got %s", r.Proto) + } + require.NoError(t, http.NewResponseController(w).Flush()) + body, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("long-lived request body: %v", err) + return + } + _, _ = w.Write(body) + }) + server := httptest.NewUnstartedServer(handler) + server.Config = newDashboardHTTPServer(handler) + server.Config.ReadHeaderTimeout = 250 * time.Millisecond + server.Config.Protocols = new(http.Protocols) + server.Config.Protocols.SetHTTP1(true) + server.Config.Protocols.SetHTTP2(true) + server.Config.Protocols.SetUnencryptedHTTP2(true) + server.EnableHTTP2 = true + if useTLS { + server.StartTLS() + } else { + server.Start() + } + defer server.Close() + transport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, Protocols: new(http.Protocols)} + transport.Protocols.SetHTTP2(true) + transport.Protocols.SetUnencryptedHTTP2(true) + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport, Timeout: 10 * time.Second} + reader, writer := io.Pipe() + defer reader.Close() + defer writer.Close() + request, err := http.NewRequest(http.MethodPost, server.URL, reader) + require.NoError(t, err) + response, err := client.Do(request) + require.NoError(t, err) + defer response.Body.Close() + time.Sleep(4 * server.Config.ReadHeaderTimeout) + _, err = writer.Write([]byte("still streaming")) + require.NoError(t, err) + require.NoError(t, writer.Close()) + body, err := io.ReadAll(response.Body) + require.NoError(t, err) + require.Equal(t, "still streaming", string(body)) + }) + } +} diff --git a/cmd/dashboard/main.go b/cmd/dashboard/main.go index 066598af..37abbb3e 100644 --- a/cmd/dashboard/main.go +++ b/cmd/dashboard/main.go @@ -12,7 +12,6 @@ import ( "os" "runtime/debug" "strings" - "time" _ "time/tzdata" "github.com/gin-gonic/gin" @@ -86,6 +85,16 @@ func initSystem(bus chan<- *model.Service) error { return err } + // 在线会话兜底清扫：状态流异常断开（失联）时由这里关闭进行中的会话 + if _, err := singleton.CronShared.AddFunc("@every 30s", singleton.SweepOnlineSessions); err != nil { + return err + } + + // 在线会话保留清理：超出后台设置的统计窗口后删除 + if _, err := singleton.CronShared.AddFunc("0 30 4 * * *", singleton.PruneOnlineSessions); err != nil { + return err + } + if err := singleton.StartJWTSessionGC(); err != nil { return err } @@ -178,12 +187,11 @@ func main() { grpcHandler := rpc.ServeRPC() httpHandler := controller.ServeWeb(frontendDist) controller.InitUpgrader() + wireNATDashboardCredentialGate() muxHandler := newHTTPandGRPCMux(httpHandler, grpcHandler) - muxServerHTTP := &http.Server{ - Handler: muxHandler, - ReadHeaderTimeout: time.Second * 5, - } + muxServerHTTP := newDashboardHTTPServer(muxHandler) + muxServerHTTP.Protocols = new(http.Protocols) muxServerHTTP.Protocols.SetHTTP1(true) muxServerHTTP.Protocols.SetUnencryptedHTTP2(true) @@ -191,9 +199,9 @@ func main() { var muxServerHTTPS *http.Server if singleton.Conf.HTTPS.ListenPort != 0 { muxServerHTTPS = &http.Server{ + ReadHeaderTimeout: dashboardReadHeaderTimeout, Addr: fmt.Sprintf("%s:%d", singleton.Conf.ListenHost, singleton.Conf.HTTPS.ListenPort), Handler: muxHandler, - ReadHeaderTimeout: time.Second * 5, TLSConfig: &tls.Config{ InsecureSkipVerify: singleton.Conf.HTTPS.InsecureTLS, }, @@ -237,6 +245,18 @@ func main() { close(errChan) } +// wireNATDashboardCredentialGate connects the NAT ingress to the real +// dashboard-credential classifiers. The NAT ingress strips dashboard-issued +// credentials (panel JWT / panel PAT) from every channel the panel's own +// TokenLookup accepts — the Authorization header, the ?token= query parameter +// and the nz-jwt cookie — before tunneling a request to the agent, and +// forwards foreign values untouched. ServeWeb built the JWT parser the +// classifier reuses, so the gate is wired right after it and before any +// listener starts serving. +func wireNATDashboardCredentialGate() { + rpc.SetNATDashboardCredentialGate(controller.ClassifyDashboardCredentialValues) +} + func newHTTPandGRPCMux(httpHandler http.Handler, grpcHandler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { natConfig := singleton.NATShared.GetNATConfigByDomain(r.Host) diff --git a/cmd/dashboard/rpc/nat.go b/cmd/dashboard/rpc/nat.go index d5b3acb6..7b8be72b 100644 --- a/cmd/dashboard/rpc/nat.go +++ b/cmd/dashboard/rpc/nat.go @@ -80,7 +80,15 @@ func ServeNAT(w http.ResponseWriter, r *http.Request, natConfig *model.NAT) { return } - // Authorization authenticates Dashboard access before NAT ingress; it must not become an origin credential for the configured NAT backend. + // Credential invariant at NAT ingress: credentials issued by this + // dashboard — the panel-signed JWT (verified by signature only, expiry + // deliberately ignored) and the panel API token — must never become origin + // credentials for the configured NAT backend. prepareNATCapability has + // already stripped them from every channel the panel's own TokenLookup + // accepts (Authorization header, ?token= query parameter, nz-jwt cookie); + // whatever remains on those channels is either absent or a foreign + // credential, which is forwarded untouched because the NAT backend may + // authenticate with credentials of its own. wWrapped, err := utils.NewRequestWrapper(r, w) if err != nil { return diff --git a/cmd/dashboard/rpc/nat_capability_agentcompat.go b/cmd/dashboard/rpc/nat_capability_agentcompat.go index d346ae77..16fdf503 100644 --- a/cmd/dashboard/rpc/nat_capability_agentcompat.go +++ b/cmd/dashboard/rpc/nat_capability_agentcompat.go @@ -22,16 +22,26 @@ type natCapabilityLease struct { func prepareNATCapability(request *http.Request, natConfig *model.NAT) (natCapabilityLease, error) { values := request.Header.Values(agentcompatcontract.IOStreamCapabilityHeader) if len(values) == 0 { - request.Header.Del("Authorization") + // No capability header: legacy NAT path. Strip dashboard-issued + // credentials (panel JWT or panel PAT) from the Authorization header, + // the ?token= query parameter and the nz-jwt cookie so they never + // become origin credentials; foreign values are ordinary request data. + stripDashboardCredentials(request) return natCapabilityLease{}, nil } request.Header.Del(agentcompatcontract.IOStreamCapabilityHeader) if len(values) != 1 || values[0] == "" { - request.Header.Del("Authorization") + // Invalid capability: nothing is forwarded, but dashboard-issued + // credentials still must not survive into errors or task data. + stripDashboardCredentials(request) return natCapabilityLease{}, errors.New("invalid NAT capability") } access, handle, err := serviceRPC.NezhaHandlerSingleton.ConsumeAgentCompatNATCapabilityForProfile(values[0], natConfig.ServerID, natConfig.ID) - request.Header.Del("Authorization") + // From here the request bytes may reach the agent (active lease) or the + // request ends in a 503; either way dashboard-issued credentials are + // stripped from every TokenLookup channel, foreign ones are forwarded + // untouched. + stripDashboardCredentials(request) if err != nil { return natCapabilityLease{}, errors.New("invalid NAT capability") } diff --git a/cmd/dashboard/rpc/nat_capability_agentcompat_test.go b/cmd/dashboard/rpc/nat_capability_agentcompat_test.go index 94910c40..2bfae4a8 100644 --- a/cmd/dashboard/rpc/nat_capability_agentcompat_test.go +++ b/cmd/dashboard/rpc/nat_capability_agentcompat_test.go @@ -4,8 +4,10 @@ package rpc import ( "bytes" + "context" "log" "net/http" + "net/url" "strings" "testing" @@ -59,7 +61,193 @@ func TestPrepareNATCapabilityRejectsDuplicateHeaderAfterRemovingAllValues(t *tes } } +func TestPrepareNATCapabilityAgentCompatStripsDashboardCredentials(t *testing.T) { + // Given — legacy path without a capability header, then the invalid + // capability path: both must drop dashboard-issued credentials from the + // Authorization header, the ?token= query parameter and the nz-jwt cookie. + installNATCredentialTestGate(t) + for name, credential := range map[string]string{ + "dashboard jwt": natTestDashboardJWTValue, + "expired dashboard jwt": natTestDashboardExpiredJWTValue, + "dashboard api token": natTestDashboardPATValue, + } { + t.Run(name+"/legacy path", func(t *testing.T) { + request := &http.Request{ + Header: make(http.Header), + URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + credential}, + } + request.Header.Set("Authorization", "Bearer "+credential) + request.Header.Set("Cookie", "sid=abc; nz-jwt="+credential) + + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + if err != nil { + t.Fatalf("legacy NAT path returned error: %v", err) + } + if lease.active { + t.Fatal("legacy NAT path unexpectedly activated") + } + if got := request.Header.Get("Authorization"); got != "" { + t.Fatalf("legacy NAT path retained dashboard credential in Authorization %q", got) + } + if got := request.URL.RawQuery; got != "keep=1" { + t.Fatalf("legacy NAT path retained dashboard credential in query %q", got) + } + if got := request.Header.Get("Cookie"); got != "sid=abc" { + t.Fatalf("legacy NAT path retained dashboard credential in cookie %q", got) + } + }) + t.Run(name+"/invalid capability path", func(t *testing.T) { + request := &http.Request{ + Header: make(http.Header), + URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + credential}, + } + request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, "malformed") + request.Header.Set("Authorization", "Bearer "+credential) + request.Header.Set("Cookie", "sid=abc; nz-jwt="+credential) + + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + if err == nil { + t.Fatal("malformed capability unexpectedly accepted") + } + if lease.active { + t.Fatal("malformed capability unexpectedly activated") + } + if got := request.Header.Get("Authorization"); got != "" { + t.Fatalf("invalid capability path retained dashboard credential in Authorization %q", got) + } + if got := request.URL.RawQuery; got != "keep=1" { + t.Fatalf("invalid capability path retained dashboard credential in query %q", got) + } + if got := request.Header.Get("Cookie"); got != "sid=abc" { + t.Fatalf("invalid capability path retained dashboard credential in cookie %q", got) + } + }) + } +} + +func TestPrepareNATCapabilityAgentCompatStripsDashboardCredentialAfterConsumingCapability(t *testing.T) { + // Given — a well-formed capability that consumes successfully: the request + // is about to be tunneled, so the dashboard credential must be gone. + installNATCredentialTestGate(t) + handler := serviceRPC.NewNezhaHandler() + original := serviceRPC.NezhaHandlerSingleton + serviceRPC.NezhaHandlerSingleton = handler + t.Cleanup(func() { serviceRPC.NezhaHandlerSingleton = original }) + + capability, err := handler.RegisterAgentCompatIOStreamCapability(context.Background(), serviceRPC.AgentCompatCapabilityRegistration{ + Owner: serviceRPC.AgentCompatCapabilityOwner{PATID: 1, UserID: 2}, + Purpose: serviceRPC.AgentCompatCapabilityNAT, + TargetServerID: 81, + ResourceID: 91, + ServerAccessAllowed: true, + }) + if err != nil { + t.Fatalf("register capability: %v", err) + } + request := &http.Request{ + Header: make(http.Header), + URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + natTestDashboardJWTValue}, + } + request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, capability.String()) + request.Header.Set("Authorization", natTestDashboardJWT) + request.Header.Set("Cookie", "sid=abc; nz-jwt="+natTestDashboardJWTValue) + + // When + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + // Then + if err != nil { + t.Fatalf("valid NAT capability rejected: %v", err) + } + if !lease.active { + t.Fatal("valid NAT capability did not activate") + } + if got := request.Header.Get("Authorization"); got != "" { + t.Fatalf("dashboard credential retained in Authorization after capability consumption: %q", got) + } + if got := request.URL.RawQuery; got != "keep=1" { + t.Fatalf("dashboard credential retained in query after capability consumption: %q", got) + } + if got := request.Header.Get("Cookie"); got != "sid=abc" { + t.Fatalf("dashboard credential retained in cookie after capability consumption: %q", got) + } +} + +func TestPrepareNATCapabilityAgentCompatKeepsForeignAndMissingAuthorization(t *testing.T) { + // Given — legacy path: foreign credentials are ordinary request data and + // an absent header must stay untouched. + installNATCredentialTestGate(t) + for name, values := range map[string][]string{ + "foreign jwt": {natTestForeignJWT}, + "foreign random": {natTestForeignRandom}, + "empty value": {""}, + "missing header": nil, + } { + t.Run(name, func(t *testing.T) { + request := &http.Request{Header: make(http.Header)} + for _, value := range values { + request.Header.Set("Authorization", value) + } + + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + if err != nil { + t.Fatalf("legacy NAT path returned error: %v", err) + } + if lease.active { + t.Fatal("legacy NAT path unexpectedly activated") + } + got := request.Header.Values("Authorization") + if len(got) != len(values) { + t.Fatalf("legacy NAT path changed Authorization to %q, want %q", got, values) + } + for i := range values { + if got[i] != values[i] { + t.Fatalf("legacy NAT path changed Authorization to %q, want %q", got, values) + } + } + }) + } +} + +func TestPrepareNATCapabilityAgentCompatKeepsForeignAndMissingQueryAndCookie(t *testing.T) { + // Given — legacy path: foreign query tokens and cookies are ordinary + // request data, and absent channels must stay untouched. + installNATCredentialTestGate(t) + for name, tc := range map[string]struct{ rawQuery, cookie string }{ + "foreign values": {"keep=1&token=" + natTestForeignJWTValue + "&a=%2Fx", "sid=abc; nz-jwt=" + natTestForeignJWTValue}, + "empty values": {"keep=1&token=", "nz-jwt="}, + "missing channel": {"keep=1", "sid=abc"}, + } { + t.Run(name, func(t *testing.T) { + request := &http.Request{ + Header: make(http.Header), + URL: &url.URL{Path: "/nat", RawQuery: tc.rawQuery}, + } + request.Header.Set("Cookie", tc.cookie) + + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + if err != nil { + t.Fatalf("legacy NAT path returned error: %v", err) + } + if lease.active { + t.Fatal("legacy NAT path unexpectedly activated") + } + if got := request.URL.RawQuery; got != tc.rawQuery { + t.Fatalf("legacy NAT path changed query to %q, want %q", got, tc.rawQuery) + } + if got := request.Header.Get("Cookie"); got != tc.cookie { + t.Fatalf("legacy NAT path changed cookie to %q, want %q", got, tc.cookie) + } + }) + } +} + func TestServeNATAgentCompatSensitiveHeadersStayOutOfErrorsAndLogs(t *testing.T) { + installNATCredentialTestGate(t) request := &http.Request{Header: make(http.Header)} request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, "capability-secret") request.Header.Set("Authorization", "Bearer deterministic-secret") diff --git a/cmd/dashboard/rpc/nat_capability_default.go b/cmd/dashboard/rpc/nat_capability_default.go index 8773477b..b0480ce6 100644 --- a/cmd/dashboard/rpc/nat_capability_default.go +++ b/cmd/dashboard/rpc/nat_capability_default.go @@ -18,7 +18,12 @@ type natCapabilityLease struct { } func prepareNATCapability(request *http.Request, _ *model.NAT) (natCapabilityLease, error) { - request.Header.Del("Authorization") + // The default build has no capability protocol: every request takes the + // legacy NAT path, so any dashboard-issued credential (panel JWT or panel + // PAT) carried by the Authorization header, the ?token= query parameter or + // the nz-jwt cookie is stripped before the request is tunneled to the + // agent. Foreign values are ordinary request data and stay put. + stripDashboardCredentials(request) return natCapabilityLease{}, nil } diff --git a/cmd/dashboard/rpc/nat_capability_default_flow_test.go b/cmd/dashboard/rpc/nat_capability_default_flow_test.go index c3698afa..30c7d71b 100644 --- a/cmd/dashboard/rpc/nat_capability_default_flow_test.go +++ b/cmd/dashboard/rpc/nat_capability_default_flow_test.go @@ -19,6 +19,7 @@ import ( func TestServeNATDefaultForwardsCapabilityHeaderAsOrdinaryData(t *testing.T) { fixture := newServeNATFixture(t) + installNATCredentialTestGate(t) connection := newServeNATConn() agent := &serveNATAgent{readErr: io.EOF, writeDone: make(chan struct{})} request := &http.Request{ @@ -28,7 +29,7 @@ func TestServeNATDefaultForwardsCapabilityHeaderAsOrdinaryData(t *testing.T) { Body: &serveNATBody{}, } request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, "ordinary-data") - request.Header.Set("Authorization", "Bearer deterministic-secret") + request.Header.Set("Authorization", natTestForeignRandom) fixture.taskStream.onSend = func(task *proto.Task) error { var nat model.TaskNAT if err := json.Unmarshal([]byte(task.Data), &nat); err != nil { @@ -58,7 +59,50 @@ func TestServeNATDefaultForwardsCapabilityHeaderAsOrdinaryData(t *testing.T) { if !strings.Contains(forwarded, strings.ToLower(agentcompatcontract.IOStreamCapabilityHeader)+": ordinary-data") { t.Fatal("default NAT flow did not forward capability header as ordinary request data") } - if strings.Contains(forwarded, "Authorization:") || strings.Contains(forwarded, "deterministic-secret") { - t.Fatal("default NAT flow forwarded Authorization credentials") + if !strings.Contains(forwarded, strings.ToLower("Authorization: "+natTestForeignRandom)) { + t.Fatal("default NAT flow stripped a foreign credential that does not belong to the dashboard") + } +} + +func TestServeNATDefaultDoesNotForwardDashboardCredentials(t *testing.T) { + fixture := newServeNATFixture(t) + installNATCredentialTestGate(t) + connection := newServeNATConn() + agent := &serveNATAgent{readErr: io.EOF, writeDone: make(chan struct{})} + request := &http.Request{ + Method: http.MethodPost, + URL: &url.URL{Scheme: "http", Host: "example.test", Path: "/nat"}, + Header: make(http.Header), + Body: &serveNATBody{}, + } + request.Header.Set("Authorization", natTestDashboardJWT) + fixture.taskStream.onSend = func(task *proto.Task) error { + var nat model.TaskNAT + if err := json.Unmarshal([]byte(task.Data), &nat); err != nil { + return err + } + return fixture.handler.AgentConnected(nat.StreamID, agent) + } + done := make(chan struct{}) + deadline, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + go func() { + ServeNAT(&serveNATResponseWriter{conn: connection}, request, &model.NAT{ServerID: fixture.server.ID, Host: "target.example"}) + close(done) + }() + select { + case <-agent.writeDone: + case <-deadline.Done(): + t.Fatal("agent did not receive ordinary NAT request") + } + _ = connection.Close() + select { + case <-done: + case <-deadline.Done(): + t.Fatal("ServeNAT did not finish") + } + forwarded := strings.ToLower(string(agent.writtenBytes())) + if strings.Contains(forwarded, "authorization:") || strings.Contains(forwarded, "dashboard-signed-jwt") { + t.Fatal("default NAT flow forwarded dashboard credentials") } } diff --git a/cmd/dashboard/rpc/nat_capability_default_test.go b/cmd/dashboard/rpc/nat_capability_default_test.go index 7ff5e686..40827939 100644 --- a/cmd/dashboard/rpc/nat_capability_default_test.go +++ b/cmd/dashboard/rpc/nat_capability_default_test.go @@ -4,6 +4,7 @@ package rpc import ( "net/http" + "net/url" "testing" "github.com/nezhahq/nezha/model" @@ -12,9 +13,10 @@ import ( func TestPrepareNATCapabilityDefaultPreservesHeaderAsOrdinaryRequestData(t *testing.T) { // Given + installNATCredentialTestGate(t) request := &http.Request{Header: make(http.Header)} request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, "ordinary-data") - request.Header.Set("Authorization", "Bearer deterministic-secret") + request.Header.Set("Authorization", natTestForeignRandom) // When lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) @@ -29,7 +31,119 @@ func TestPrepareNATCapabilityDefaultPreservesHeaderAsOrdinaryRequestData(t *test if got := request.Header.Get(agentcompatcontract.IOStreamCapabilityHeader); got != "ordinary-data" { t.Fatalf("default NAT capability hook changed header to %q", got) } - if got := request.Header.Get("Authorization"); got != "" { - t.Fatalf("default NAT capability hook retained Authorization %q", got) + if got := request.Header.Get("Authorization"); got != natTestForeignRandom { + t.Fatalf("default NAT capability hook changed foreign Authorization to %q", got) + } +} + +func TestPrepareNATCapabilityDefaultStripsDashboardCredentials(t *testing.T) { + // Given + installNATCredentialTestGate(t) + for name, credential := range map[string]string{ + "dashboard jwt": natTestDashboardJWTValue, + "expired dashboard jwt": natTestDashboardExpiredJWTValue, + "dashboard api token": natTestDashboardPATValue, + } { + t.Run(name, func(t *testing.T) { + request := &http.Request{ + Header: make(http.Header), + URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + credential}, + } + request.Header.Set("Authorization", "Bearer "+credential) + request.Header.Set("Cookie", "sid=abc; nz-jwt="+credential) + + // When + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + // Then + if err != nil { + t.Fatalf("default NAT capability hook returned error: %v", err) + } + if lease.active { + t.Fatal("default NAT capability hook unexpectedly activated") + } + if got := request.Header.Get("Authorization"); got != "" { + t.Fatalf("default NAT capability hook retained dashboard credential in Authorization %q", got) + } + if got := request.URL.RawQuery; got != "keep=1" { + t.Fatalf("default NAT capability hook retained dashboard credential in query %q", got) + } + if got := request.Header.Get("Cookie"); got != "sid=abc" { + t.Fatalf("default NAT capability hook retained dashboard credential in cookie %q", got) + } + }) + } +} + +func TestPrepareNATCapabilityDefaultKeepsForeignAndMissingAuthorization(t *testing.T) { + // Given + installNATCredentialTestGate(t) + for name, values := range map[string][]string{ + "foreign jwt": {natTestForeignJWT}, + "foreign random": {natTestForeignRandom}, + "empty value": {""}, + "missing header": nil, + } { + t.Run(name, func(t *testing.T) { + request := &http.Request{Header: make(http.Header)} + for _, value := range values { + request.Header.Set("Authorization", value) + } + + // When + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + // Then + if err != nil { + t.Fatalf("default NAT capability hook returned error: %v", err) + } + if lease.active { + t.Fatal("default NAT capability hook unexpectedly activated") + } + got := request.Header.Values("Authorization") + if len(got) != len(values) { + t.Fatalf("default NAT capability hook changed Authorization to %q, want %q", got, values) + } + for i := range values { + if got[i] != values[i] { + t.Fatalf("default NAT capability hook changed Authorization to %q, want %q", got, values) + } + } + }) + } +} + +func TestPrepareNATCapabilityDefaultKeepsForeignAndMissingQueryAndCookie(t *testing.T) { + // Given + installNATCredentialTestGate(t) + for name, tc := range map[string]struct{ rawQuery, cookie string }{ + "foreign values": {"keep=1&token=" + natTestForeignJWTValue + "&a=%2Fx", "sid=abc; nz-jwt=" + natTestForeignJWTValue}, + "empty values": {"keep=1&token=", "nz-jwt="}, + "missing channel": {"keep=1", "sid=abc"}, + } { + t.Run(name, func(t *testing.T) { + request := &http.Request{ + Header: make(http.Header), + URL: &url.URL{Path: "/nat", RawQuery: tc.rawQuery}, + } + request.Header.Set("Cookie", tc.cookie) + + // When + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + // Then + if err != nil { + t.Fatalf("default NAT capability hook returned error: %v", err) + } + if lease.active { + t.Fatal("default NAT capability hook unexpectedly activated") + } + if got := request.URL.RawQuery; got != tc.rawQuery { + t.Fatalf("default NAT capability hook changed query to %q, want %q", got, tc.rawQuery) + } + if got := request.Header.Get("Cookie"); got != tc.cookie { + t.Fatalf("default NAT capability hook changed cookie to %q, want %q", got, tc.cookie) + } + }) } } diff --git a/cmd/dashboard/rpc/nat_capability_flow_agentcompat_test.go b/cmd/dashboard/rpc/nat_capability_flow_agentcompat_test.go index 2ed5a003..0bfe381a 100644 --- a/cmd/dashboard/rpc/nat_capability_flow_agentcompat_test.go +++ b/cmd/dashboard/rpc/nat_capability_flow_agentcompat_test.go @@ -23,6 +23,7 @@ import ( func TestServeNATAgentCompatPublishesExactStreamAfterRequestTransfer(t *testing.T) { fixture := newServeNATFixture(t) + installNATCredentialTestGate(t) capability, err := fixture.handler.RegisterAgentCompatIOStreamCapability(context.Background(), serviceRPC.AgentCompatCapabilityRegistration{ Owner: serviceRPC.AgentCompatCapabilityOwner{PATID: 1, UserID: 2}, Purpose: serviceRPC.AgentCompatCapabilityNAT, @@ -44,7 +45,7 @@ func TestServeNATAgentCompatPublishesExactStreamAfterRequestTransfer(t *testing. Body: io.NopCloser(strings.NewReader("payload")), } request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, capability.String()) - request.Header.Set("Authorization", "Bearer deterministic-secret") + request.Header.Set("Authorization", natTestForeignJWT) request.Header.Set("X-Ordinary-NAT", "ordinary") fixture.taskStream.onSend = func(task *proto.Task) error { var nat model.TaskNAT @@ -106,8 +107,62 @@ func TestServeNATAgentCompatPublishesExactStreamAfterRequestTransfer(t *testing. require.Contains(t, forwarded, expected) } require.NotContains(t, forwarded, agentcompatcontract.IOStreamCapabilityHeader) - require.NotContains(t, forwarded, "Authorization:") - require.NotContains(t, forwarded, "deterministic-secret") + // Foreign credentials are ordinary request data for the NAT backend and + // must survive the transfer; dashboard-issued values are stripped before + // any byte leaves the dashboard. + require.Contains(t, forwarded, "Authorization: "+natTestForeignJWT) +} + +func TestServeNATAgentCompatDoesNotForwardDashboardCredentials(t *testing.T) { + fixture := newServeNATFixture(t) + installNATCredentialTestGate(t) + capability, err := fixture.handler.RegisterAgentCompatIOStreamCapability(context.Background(), serviceRPC.AgentCompatCapabilityRegistration{ + Owner: serviceRPC.AgentCompatCapabilityOwner{PATID: 1, UserID: 2}, + Purpose: serviceRPC.AgentCompatCapabilityNAT, + TargetServerID: fixture.server.ID, + ResourceID: 91, + ServerAccessAllowed: true, + }) + require.NoError(t, err) + connection := newServeNATConn() + writer := &serveNATResponseWriter{conn: connection} + agent := &serveNATAgent{readErr: io.EOF, writeDone: make(chan struct{})} + request := &http.Request{ + Method: http.MethodPost, + URL: &url.URL{Scheme: "http", Host: "example.test", Path: "/nat"}, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader("payload")), + } + request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, capability.String()) + request.Header.Set("Authorization", natTestDashboardJWT) + fixture.taskStream.onSend = func(task *proto.Task) error { + var nat model.TaskNAT + if err := json.Unmarshal([]byte(task.Data), &nat); err != nil { + return err + } + return fixture.handler.AgentConnected(nat.StreamID, agent) + } + done := make(chan struct{}) + go func() { + ServeNAT(writer, request, &model.NAT{Common: model.Common{ID: 91}, ServerID: fixture.server.ID, Host: "target.example"}) + close(done) + }() + select { + case <-agent.writeDone: + case <-time.After(time.Second): + t.Fatal("agent did not receive the transferred NAT request") + } + require.NoError(t, connection.Close()) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("ServeNAT did not complete") + } + forwarded := strings.ToLower(string(agent.writtenBytes())) + require.NotContains(t, forwarded, "authorization:") + for _, dashboardCredential := range []string{natTestDashboardJWT, natTestDashboardExpiredJWT, natTestDashboardAPIToken} { + require.NotContains(t, forwarded, strings.ToLower(dashboardCredential)) + } } type orderedNATAgent struct { diff --git a/cmd/dashboard/rpc/nat_credential.go b/cmd/dashboard/rpc/nat_credential.go new file mode 100644 index 00000000..153f556e --- /dev/null +++ b/cmd/dashboard/rpc/nat_credential.go @@ -0,0 +1,308 @@ +package rpc + +import ( + "context" + "net/http" + "net/url" + "strings" + "sync" +) + +// Channel names of the panel's own TokenLookup (controller/jwt.go +// "header: Authorization, query: token, cookie: nz-jwt"). A dashboard JWT +// can therefore reach the NAT ingress through the Authorization header, the +// ?token= query parameter or the nz-jwt cookie, and all three must be +// classified before the request is tunneled to the agent. +const ( + natDashboardJWTCookieName = "nz-jwt" + natDashboardJWTQueryParameter = "token" + // Keep credential classification work constant even when an unauthenticated + // NAT request contains thousands of token parameters or cookies. Values + // beyond this per-request budget fail closed and are stripped. + maxNATDashboardCredentialValues = 16 +) + +var ( + natDashboardCredentialGateMu sync.RWMutex + // natDashboardCredentialValueGate classifies a bounded batch of raw + // credential values. Batching lets the controller resolve all PAT hashes in + // one query instead of turning attacker-controlled query/cookie fan-out into + // one database query per value. + natDashboardCredentialValueGate func(ctx context.Context, values []string) []bool +) + +// SetNATDashboardCredentialGate wires the dashboard-credential classifier +// used by the NAT ingress. main() injects +// controller.ClassifyDashboardCredentialValues; +// the setter keeps the rpc package free of a dependency on the controller +// package, the same pattern as SetReceiptGateListener and +// SetMCPKillSwitchObserver. A nil function fails closed: every value on that +// channel is then treated as dashboard-issued and stripped. +func SetNATDashboardCredentialGate(valueClass func(ctx context.Context, values []string) []bool) { + natDashboardCredentialGateMu.Lock() + natDashboardCredentialValueGate = valueClass + natDashboardCredentialGateMu.Unlock() +} + +// NATDashboardCredentialGateWired reports whether the dashboard-credential +// classifier has been wired. It exists so the main() wiring contract can be +// pinned by a test. +func NATDashboardCredentialGateWired() bool { + natDashboardCredentialGateMu.RLock() + wired := natDashboardCredentialValueGate != nil + natDashboardCredentialGateMu.RUnlock() + return wired +} + +// stripDashboardCredentials removes credentials issued by this dashboard from +// every channel the panel's own TokenLookup accepts: the Authorization +// header, the ?token= query parameter and the nz-jwt cookie. Such credentials +// authenticate the dashboard itself and must never become origin credentials +// for the NAT backend. Foreign values are forwarded untouched: the NAT +// backend may authenticate with credentials of its own, and this dashboard +// must not make policy for them. +func stripDashboardCredentials(request *http.Request) { + classification := classifyNATDashboardCredentialValues(request) + stripDashboardAuthorizationHeader(request, classification) + stripDashboardCredentialQuery(request, classification) + stripDashboardCredentialCookie(request, classification) +} + +type natDashboardCredentialClassification struct { + wired bool + limited bool + issued map[string]bool +} + +func (c natDashboardCredentialClassification) isDashboardCredential(value string) bool { + if !c.wired { + return true + } + if issued, ok := c.issued[value]; ok { + return issued + } + // Only candidate values collected from this request reach this method. A + // missing result therefore means the distinct-value budget was exceeded; + // fail closed so a dashboard credential cannot be hidden after junk values. + return c.limited +} + +func classifyNATDashboardCredentialValues(request *http.Request) natDashboardCredentialClassification { + natDashboardCredentialGateMu.RLock() + gate := natDashboardCredentialValueGate + natDashboardCredentialGateMu.RUnlock() + classification := natDashboardCredentialClassification{ + wired: gate != nil, + issued: make(map[string]bool), + } + if gate == nil { + return classification + } + + values, limited := collectNATDashboardCredentialValues(request) + classification.limited = limited + results := gate(request.Context(), values) + if len(results) != len(values) { + // A broken classifier must not turn into credential forwarding. + for _, value := range values { + classification.issued[value] = true + } + classification.limited = true + return classification + } + for i, value := range values { + classification.issued[value] = results[i] + } + return classification +} + +func collectNATDashboardCredentialValues(request *http.Request) ([]string, bool) { + values := make([]string, 0, maxNATDashboardCredentialValues) + seen := make(map[string]struct{}, maxNATDashboardCredentialValues) + limited := false + add := func(value string) { + if _, ok := seen[value]; ok { + return + } + if len(values) == maxNATDashboardCredentialValues { + limited = true + return + } + seen[value] = struct{}{} + values = append(values, value) + } + + for _, authorization := range request.Header.Values("Authorization") { + raw := strings.TrimSpace(authorization) + if strings.HasPrefix(raw, "Bearer ") { + add(strings.TrimPrefix(raw, "Bearer ")) + } + } + if request.URL != nil { + for _, element := range strings.Split(request.URL.RawQuery, "&") { + if value, ok := dashboardCredentialQueryElementValue(element); ok { + add(value) + } + } + } + for _, line := range request.Header.Values("Cookie") { + for _, pair := range strings.Split(line, ";") { + if value, ok := dashboardCredentialCookiePairValue(pair); ok { + add(value) + } + } + } + return values, limited +} + +// stripDashboardAuthorizationHeader removes the Authorization header when it +// carries a dashboard-issued credential. +func stripDashboardAuthorizationHeader(request *http.Request, classification natDashboardCredentialClassification) { + if !classification.wired { + // Gate not wired (early startup or tests): fail closed and keep the + // historical behavior of stripping every Authorization header. + request.Header.Del("Authorization") + return + } + // A multi-valued Authorization header is malformed traffic, but any + // dashboard-issued value in it must still not survive into the tunnel, so + // the whole header is dropped when one is recognized. + for _, value := range request.Header.Values("Authorization") { + raw := strings.TrimSpace(value) + if strings.HasPrefix(raw, "Bearer ") && classification.isDashboardCredential(strings.TrimPrefix(raw, "Bearer ")) { + request.Header.Del("Authorization") + return + } + } +} + +// stripDashboardCredentialQuery removes ?token= query parameters whose value +// is a dashboard-issued credential. Every other element of the query string — +// including foreign token values — is preserved byte-for-byte, so the +// forwarded request keeps the exact query the client sent. +func stripDashboardCredentialQuery(request *http.Request, classification natDashboardCredentialClassification) { + if request.URL == nil || request.URL.RawQuery == "" { + return + } + elements := strings.Split(request.URL.RawQuery, "&") + kept := elements[:0] + changed := false + for _, element := range elements { + if !classification.wired && dashboardCredentialQueryElementName(element) { + changed = true + continue + } + value, candidate := dashboardCredentialQueryElementValue(element) + if candidate && classification.isDashboardCredential(value) { + changed = true + continue + } + kept = append(kept, element) + } + if changed { + request.URL.RawQuery = strings.Join(kept, "&") + } +} + +// dashboardCredentialQueryElementValue extracts the decoded value from a raw +// query element carrying the panel's token parameter. +func dashboardCredentialQueryElementValue(element string) (string, bool) { + rawName, rawValue, _ := strings.Cut(element, "=") + // The panel reads the parameter by its decoded name (gin-jwt c.Query), + // so an escaped key must be decoded before comparing. + name, err := url.QueryUnescape(rawName) + if err != nil || name != natDashboardJWTQueryParameter { + return "", false + } + // A value that fails to decode is one the panel's own query parsing drops + // as well, so it can never be a panel credential. + decoded, err := url.QueryUnescape(rawValue) + if err != nil { + return "", false + } + return decoded, true +} + +func dashboardCredentialQueryElementName(element string) bool { + rawName, _, _ := strings.Cut(element, "=") + name, err := url.QueryUnescape(rawName) + return err == nil && name == natDashboardJWTQueryParameter +} + +// stripDashboardCredentialCookie removes nz-jwt cookies whose value is a +// dashboard-issued credential. All other cookie pairs — including foreign +// nz-jwt values — are preserved verbatim. +func stripDashboardCredentialCookie(request *http.Request, classification natDashboardCredentialClassification) { + lines := request.Header.Values("Cookie") + if len(lines) == 0 { + return + } + changed := false + kept := make([]string, 0, len(lines)) + for _, line := range lines { + nextLine, lineChanged := stripDashboardCredentialCookieLine(line, classification) + changed = changed || lineChanged + if nextLine != "" { + kept = append(kept, nextLine) + } + } + if !changed { + return + } + request.Header.Del("Cookie") + for _, line := range kept { + request.Header.Add("Cookie", line) + } +} + +// stripDashboardCredentialCookieLine rewrites a single Cookie header line, +// dropping the dashboard-issued nz-jwt pairs. Kept pairs keep their raw form, +// whitespace included — except that a pair following a dropped one no longer +// carries the removed pair's share of the "; " separator. +func stripDashboardCredentialCookieLine(line string, classification natDashboardCredentialClassification) (string, bool) { + pairs := strings.Split(line, ";") + kept := pairs[:0] + droppedBeforeFirstKept := true + changed := false + for _, pair := range pairs { + if !classification.wired && dashboardCredentialCookiePairName(pair) { + changed = true + continue + } + value, candidate := dashboardCredentialCookiePairValue(pair) + if candidate && classification.isDashboardCredential(value) { + changed = true + continue + } + if droppedBeforeFirstKept { + pair = strings.TrimLeft(pair, " ") + } + droppedBeforeFirstKept = false + kept = append(kept, pair) + } + if !changed { + return line, false + } + return strings.Join(kept, ";"), true +} + +// dashboardCredentialCookiePairValue extracts a cookie value the same way the +// panel hands it to the JWT verifier: surrounding whitespace/quotes are +// presentation, then Gin applies QueryUnescape. +func dashboardCredentialCookiePairValue(pair string) (string, bool) { + name, rawValue, found := strings.Cut(pair, "=") + if !found || strings.TrimSpace(name) != natDashboardJWTCookieName { + return "", false + } + value := strings.Trim(strings.TrimSpace(rawValue), `"`) + decoded, err := url.QueryUnescape(value) + if err != nil { + return "", false + } + return decoded, true +} + +func dashboardCredentialCookiePairName(pair string) bool { + name, _, found := strings.Cut(pair, "=") + return found && strings.TrimSpace(name) == natDashboardJWTCookieName +} diff --git a/cmd/dashboard/rpc/nat_credential_test.go b/cmd/dashboard/rpc/nat_credential_test.go new file mode 100644 index 00000000..234e78f2 --- /dev/null +++ b/cmd/dashboard/rpc/nat_credential_test.go @@ -0,0 +1,202 @@ +package rpc + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + "testing" + + "github.com/nezhahq/nezha/model" +) + +// Values standing in for the credentials this dashboard issues. The fake +// gates below mirror the contract of controller.IsDashboardCredential and +// controller.IsDashboardCredentialValue; the real classification semantics — +// panel JWTs accepted regardless of expiry, foreign-signed JWTs and random +// strings rejected — are pinned by the controller package tests +// (dashboard_credential_test.go) and by the flow test that wires the real +// classifiers into ServeNAT (dashboard_credential_nat_flow_test.go). +const ( + natTestDashboardJWTValue = "dashboard-signed-jwt" + natTestDashboardExpiredJWTValue = "dashboard-signed-expired-jwt" + natTestDashboardPATValue = model.APITokenPrefix + "dashboard-issued-pat" + natTestForeignJWTValue = "foreign-signed-jwt" + natTestForeignRandomValue = "foreign-random-string" + + natTestDashboardJWT = "Bearer " + natTestDashboardJWTValue + natTestDashboardExpiredJWT = "Bearer " + natTestDashboardExpiredJWTValue + natTestDashboardAPIToken = "Bearer " + natTestDashboardPATValue + natTestForeignJWT = "Bearer " + natTestForeignJWTValue + natTestForeignRandom = "Bearer " + natTestForeignRandomValue +) + +// installNATCredentialTestGate wires fake dashboard-credential classifiers +// for the duration of the test: exactly the dashboard-issued test values above +// are classified as dashboard credentials on every channel, everything else +// is foreign. +func installNATCredentialTestGate(t *testing.T) { + t.Helper() + natDashboardCredentialGateMu.RLock() + original := natDashboardCredentialValueGate + natDashboardCredentialGateMu.RUnlock() + SetNATDashboardCredentialGate(func(_ context.Context, values []string) []bool { + results := make([]bool, len(values)) + for i, value := range values { + switch value { + case natTestDashboardJWTValue, natTestDashboardExpiredJWTValue, natTestDashboardPATValue: + results[i] = true + } + } + return results + }) + t.Cleanup(func() { + SetNATDashboardCredentialGate(original) + }) +} + +func TestStripDashboardCredentialsQueryChannel(t *testing.T) { + // Given + installNATCredentialTestGate(t) + request := &http.Request{URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + natTestDashboardJWTValue + "&token=" + natTestForeignJWTValue + "&a=%2Fx"}} + + // When + stripDashboardCredentials(request) + + // Then + if got := request.URL.RawQuery; got != "keep=1&token="+natTestForeignJWTValue+"&a=%2Fx" { + t.Fatalf("query channel kept %q, want dashboard token dropped and foreign values byte-preserved", got) + } +} + +func TestStripDashboardCredentialsQueryChannelEscapedValue(t *testing.T) { + // Given — a dashboard token percent-encoded in the query must still be + // recognized, while the escaped foreign element stays verbatim. + installNATCredentialTestGate(t) + request := &http.Request{URL: &url.URL{Path: "/nat", RawQuery: "token=" + url.QueryEscape(natTestDashboardJWTValue) + "&token=" + url.QueryEscape(natTestForeignJWTValue)}} + + // When + stripDashboardCredentials(request) + + // Then + if got := request.URL.RawQuery; got != "token="+url.QueryEscape(natTestForeignJWTValue) { + t.Fatalf("query channel kept %q, want escaped dashboard token dropped and foreign element verbatim", got) + } +} + +func TestStripDashboardCredentialsCookieChannel(t *testing.T) { + // Given + installNATCredentialTestGate(t) + request := &http.Request{Header: make(http.Header)} + request.Header.Set("Cookie", "sid=abc; nz-jwt="+natTestDashboardJWTValue+"; nz-jwt="+natTestForeignJWTValue+"; theme=dark") + + // When + stripDashboardCredentials(request) + + // Then + if got := request.Header.Get("Cookie"); got != "sid=abc; nz-jwt="+natTestForeignJWTValue+"; theme=dark" { + t.Fatalf("cookie channel kept %q, want dashboard pair dropped and foreign pairs verbatim", got) + } +} + +func TestStripDashboardCredentialsCookieChannelQuotedValue(t *testing.T) { + // Given — quoting is presentation the panel's cookie parsing strips before + // verifying, so a quoted dashboard token must be recognized. + installNATCredentialTestGate(t) + request := &http.Request{Header: make(http.Header)} + request.Header.Set("Cookie", `nz-jwt="`+natTestDashboardJWTValue+`"; sid=abc`) + + // When + stripDashboardCredentials(request) + + // Then + if got := request.Header.Get("Cookie"); got != "sid=abc" { + t.Fatalf("cookie channel kept %q, want quoted dashboard token dropped", got) + } +} + +func TestStripDashboardCredentialsCookieChannelEscapedValue(t *testing.T) { + // Gin's cookie TokenLookup URL-decodes the value before JWT validation. + // NAT must classify the same decoded value or an encoded dashboard JWT can + // be accepted by the panel and then leak unchanged to the backend. + installNATCredentialTestGate(t) + request := &http.Request{Header: make(http.Header)} + escaped := strings.ReplaceAll(natTestDashboardJWTValue, "-", "%2D") + request.Header.Set("Cookie", "nz-jwt="+escaped+"; sid=abc") + + stripDashboardCredentials(request) + + if got := request.Header.Get("Cookie"); got != "sid=abc" { + t.Fatalf("cookie channel kept %q, want escaped dashboard token dropped", got) + } +} + +func TestStripDashboardCredentialsDeduplicatesAndBoundsClassification(t *testing.T) { + natDashboardCredentialGateMu.RLock() + original := natDashboardCredentialValueGate + natDashboardCredentialGateMu.RUnlock() + t.Cleanup(func() { SetNATDashboardCredentialGate(original) }) + + classifiedValues := 0 + gateCalls := 0 + SetNATDashboardCredentialGate(func(_ context.Context, values []string) []bool { + gateCalls++ + classifiedValues += len(values) + return make([]bool, len(values)) + }) + + query := make(url.Values) + for range 100 { + query.Add("token", model.APITokenPrefix+"same-value") + } + for i := range 100 { + query.Add("token", fmt.Sprintf("%sunique-%d", model.APITokenPrefix, i)) + } + request := &http.Request{Header: make(http.Header), URL: &url.URL{RawQuery: query.Encode()}} + + stripDashboardCredentials(request) + + if gateCalls != 1 { + t.Fatalf("classifier called %d times, want one batched call", gateCalls) + } + if classifiedValues > maxNATDashboardCredentialValues { + t.Fatalf("classified %d credential values, want at most %d per request", classifiedValues, maxNATDashboardCredentialValues) + } + if strings.Contains(request.URL.RawQuery, model.APITokenPrefix+"unique-99") { + t.Fatal("credential beyond the classification budget survived; overflow must fail closed") + } +} + +func TestPrepareNATCapabilityFailsClosedWithoutGate(t *testing.T) { + // Given — unwired gates: every value on every channel is treated as + // dashboard-issued and stripped, foreign ones included. + SetNATDashboardCredentialGate(nil) + t.Cleanup(func() { SetNATDashboardCredentialGate(nil) }) + request := &http.Request{ + Header: make(http.Header), + URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + natTestForeignJWTValue + "&token=%ZZ"}, + } + request.Header.Set("Authorization", natTestForeignRandom) + request.Header.Set("Cookie", "sid=abc; nz-jwt="+natTestForeignJWTValue+"; nz-jwt=%ZZ") + + // When + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + // Then + if err != nil { + t.Fatalf("fail-closed NAT capability hook returned error: %v", err) + } + if lease.active { + t.Fatal("fail-closed NAT capability hook unexpectedly activated") + } + if got := request.Header.Get("Authorization"); got != "" { + t.Fatalf("fail-closed hook retained Authorization %q", got) + } + if got := request.URL.RawQuery; got != "keep=1" { + t.Fatalf("fail-closed hook kept query %q, want token parameter stripped", got) + } + if got := request.Header.Get("Cookie"); got != "sid=abc" { + t.Fatalf("fail-closed hook kept cookie %q, want nz-jwt pair stripped", got) + } +} diff --git a/go.mod b/go.mod index a66f3768..55d8f7ed 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,9 @@ module github.com/nezhahq/nezha -go 1.26.5 +go 1.26.10 require ( - github.com/VictoriaMetrics/VictoriaMetrics v1.148.0 + github.com/VictoriaMetrics/VictoriaMetrics v1.151.0 github.com/appleboy/gin-jwt/v2 v2.10.3 github.com/dustinkirkland/golang-petname v0.0.0-20260215035315-f0c533e9ce9b github.com/gin-contrib/pprof v1.5.4 @@ -14,18 +14,18 @@ require ( github.com/gorilla/websocket v1.5.3 github.com/hashicorp/go-uuid v1.0.3 github.com/jinzhu/copier v0.4.0 - github.com/knadh/koanf/maps v0.1.2 + github.com/knadh/koanf/maps v0.1.3 github.com/knadh/koanf/providers/env v1.1.0 github.com/knadh/koanf/providers/file v1.2.1 - github.com/knadh/koanf/v2 v2.3.5 + github.com/knadh/koanf/v2 v2.3.6 github.com/leonelquinteros/gotext v1.7.2 github.com/libdns/cloudflare v0.2.2 github.com/libdns/he v1.2.2 github.com/libdns/libdns v1.1.1 github.com/likexian/whois v1.15.7 github.com/likexian/whois-parser v1.24.21 - github.com/mattn/go-sqlite3 v1.14.49 - github.com/miekg/dns v1.1.72 + github.com/mattn/go-sqlite3 v1.14.50 + github.com/miekg/dns v1.1.73 github.com/modelcontextprotocol/go-sdk v1.7.0 github.com/nezhahq/libdns-tencentcloud v0.0.0-20260628095405-2ab294ec675b github.com/ory/graceful v0.2.0 @@ -33,20 +33,20 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/robfig/cron/v3 v3.0.1 github.com/sqids/sqids-go v0.4.1 - github.com/stretchr/testify v1.11.1 + github.com/stretchr/testify v1.12.1 github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.1 github.com/swaggo/swag v1.16.6 github.com/tidwall/gjson v1.19.0 - golang.org/x/crypto v0.54.0 - golang.org/x/exp v0.0.0-20260727155853-b88d891fe743 - golang.org/x/mod v0.38.0 - golang.org/x/net v0.57.0 + golang.org/x/crypto v0.55.0 + golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa + golang.org/x/mod v0.40.0 + golang.org/x/net v0.58.0 golang.org/x/oauth2 v0.36.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 - google.golang.org/grpc v1.83.0 - google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af + google.golang.org/grpc v1.83.2 + google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 gorm.io/datatypes v1.2.7 gorm.io/driver/sqlite v1.6.0 @@ -66,7 +66,6 @@ require ( github.com/bytedance/sonic/loader v0.5.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.7 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gin-contrib/sse v1.1.1 // indirect @@ -91,18 +90,17 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/compress v1.19.1 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/likexian/gokit v0.25.16 // indirect - github.com/mattn/go-isatty v0.0.22 // indirect + github.com/mattn/go-isatty v0.0.23 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/pelletier/go-toml/v2 v2.3.1 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.1 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect @@ -115,18 +113,18 @@ require ( github.com/ugorji/go/codec v1.3.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fastrand v1.1.0 // indirect - github.com/valyala/gozstd v1.24.0 // indirect + github.com/valyala/gozstd v1.25.0 // indirect github.com/valyala/histogram v1.2.0 // indirect github.com/valyala/quicktemplate v1.8.0 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect go.mongodb.org/mongo-driver/v2 v2.6.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/arch v0.27.0 // indirect - golang.org/x/text v0.40.0 // indirect + golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.48.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260615183401-62b3387ff324 // indirect gorm.io/driver/mysql v1.5.6 // indirect + golang.org/x/tools v0.49.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a // indirect ) diff --git a/go.sum b/go.sum index 06725786..ed236cf9 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/VictoriaMetrics/VictoriaMetrics v1.148.0 h1:qfsMLjvkNnIIE3w3o7Sf1ejpYAE3pGwqAz4OqKNqZYI= -github.com/VictoriaMetrics/VictoriaMetrics v1.148.0/go.mod h1:/grcppu6T0Y1k8z6e7ACb43sYUKGkvQ4xVVdCANK/d0= +github.com/VictoriaMetrics/VictoriaMetrics v1.151.0 h1:Svzb/7gPYbzmQnvLjab8h1Jl9olB76mokDE7ykvF/ro= +github.com/VictoriaMetrics/VictoriaMetrics v1.151.0/go.mod h1:Mp7qEBDOXwP50gC93cF8wJmaGjTFSJP2WA6dRtMI7qU= github.com/VictoriaMetrics/easyproto v1.2.0 h1:FJT9uNXA2isppFuJErbLqD306KoFlehl7Wn2dg/6oIE= github.com/VictoriaMetrics/easyproto v1.2.0/go.mod h1:QlGlzaJnDfFd8Lk6Ci/fuLxfTo3/GThPs2KH23mv710= github.com/VictoriaMetrics/fastcache v1.13.3 h1:rBabE0iIxcqKEMCwUmwHZ9dgEqXerg8FRbRDUvC7OVc= @@ -46,8 +46,8 @@ github.com/gin-contrib/sse v1.1.1 h1:uGYpNwTacv5R68bSGMapo62iLTRa9l5zxGCps4hK6ko github.com/gin-contrib/sse v1.1.1/go.mod h1:QXzuVkA0YO7o/gun03UI1Q+FTI8ZV/n5t03kIQAI89s= github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.23.1 h1:1HBACs7XIwR2RcmItfdSFlALhGbe6S92p0ry4d1GWg4= @@ -133,18 +133,18 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= -github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= -github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/maps v0.1.3 h1:P1z7EvTqdFBrPYbzSvorvrpib+sjkUMxf0FVvA5NKK4= +github.com/knadh/koanf/maps v0.1.3/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= github.com/knadh/koanf/providers/env v1.1.0 h1:U2VXPY0f+CsNDkvdsG8GcsnK4ah85WwWyJgef9oQMSc= github.com/knadh/koanf/providers/env v1.1.0/go.mod h1:QhHHHZ87h9JxJAn2czdEl6pdkNnDh/JS1Vtsyt65hTY= github.com/knadh/koanf/providers/file v1.2.1 h1:bEWbtQwYrA+W2DtdBrQWyXqJaJSG3KrP3AESOJYp9wM= github.com/knadh/koanf/providers/file v1.2.1/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA= -github.com/knadh/koanf/v2 v2.3.5 h1:2dXJUYaKGm4SGYeoAtBviq9+02JZo/pxQ2ssOd60rJg= -github.com/knadh/koanf/v2 v2.3.5/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28= +github.com/knadh/koanf/v2 v2.3.6 h1:JoQPSJmvS4aP0xNc8xMDr5tcrkSEInL23/Il7pITAKo= +github.com/knadh/koanf/v2 v2.3.6/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -167,12 +167,18 @@ github.com/likexian/whois-parser v1.24.21 h1:MxsrGRxDOiZIVp7q7N/yAIbKuN4QAkGjCpO github.com/likexian/whois-parser v1.24.21/go.mod h1:o3DUruO65Pb8WXCJCTlSVkTbwuYVrBCeoMTw2q0mxY4= github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/mattn/go-isatty v0.0.23 h1:cYwCQTQf3HB6xUC+BtyCLZNr7IzbOmoZbmssVNzSyiQ= +github.com/mattn/go-isatty v0.0.23/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-sqlite3 v1.14.49 h1:B8jBHC3xhxZgxztrgruTuLucebnULQnx4W7cF7SAE9w= github.com/mattn/go-sqlite3 v1.14.49/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/mattn/go-sqlite3 v1.14.50 h1:dmdFvo1XG4MPzA4IkAmE9upVz/Nj31uRoM5+jC8hYbY= +github.com/mattn/go-sqlite3 v1.14.50/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/microsoft/go-mssqldb v1.7.2 h1:CHkFJiObW7ItKTJfHo1QX7QBBD1iV+mn1eOyRP3b/PA= github.com/microsoft/go-mssqldb v1.7.2/go.mod h1:kOvZKUdrhhFQmxLZqbwUV0rHkNkZpthMITIb2Ko1IoA= github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= +github.com/miekg/dns v1.1.73 h1:uhT8nJxmTrPJYClxVxTCX+CVn6qnzSiybRk72Z6DgrE= +github.com/miekg/dns v1.1.73/go.mod h1:RW2Obtfd5NZHvOFe3zYG0W8koWOQtAzyHaLo8vASBuQ= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -198,8 +204,6 @@ github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/quic-go/quic-go v0.59.1 h1:0Gmua0HW1Tv7ANR7hUYwRyD0MG5OJfgvYSZasGZzBic= @@ -223,8 +227,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg= github.com/swaggo/gin-swagger v1.6.1 h1:Ri06G4gc9N4t4k8hekMigJ9zKTFSlqj/9paAQCQs7cY= @@ -250,8 +254,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= -github.com/valyala/gozstd v1.24.0 h1:M/9L3h7bVwbj2gZwrmuoaxzwVrmBUvos2jG9cZtuhlc= -github.com/valyala/gozstd v1.24.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= +github.com/valyala/gozstd v1.25.0 h1:7gS7+5zwidZT1BFQqGPAPII8ekZ3tvYTp5IvOEWC34Y= +github.com/valyala/gozstd v1.25.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= github.com/valyala/quicktemplate v1.8.0 h1:zU0tjbIqTRgKQzFY1L42zq0qR3eh4WoQQdIdqCysW5k= @@ -279,25 +283,25 @@ go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/arch v0.27.0 h1:0WNVcR8u9yFz8j5FvdHpgwNp3FS5U4guYdzHwEiGjoU= golang.org/x/arch v0.27.0/go.mod h1:0X+GdSIP+kL5wPmpK7sdkEVTt2XoYP0cSjQSbZBwOi8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/exp v0.0.0-20260727155853-b88d891fe743 h1:ex206bKw+v3K0dm3andkrIF+ijyQKJG1pLgwQ2PYdQM= -golang.org/x/exp v0.0.0-20260727155853-b88d891fe743/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk= +golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= -golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -319,24 +323,24 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= -golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260615183401-62b3387ff324 h1:9HZDLIdYBJXAnaFOr9WHrKVycfpY+75s9HGadC0305A= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260615183401-62b3387ff324/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ= -google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a h1:qI/YMH1ep2qQtqcp00gMQyoU7mjvbhg88GJKCvfoLj0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/integration/agentcompat/internal/workflowpolicy/quality_workflow_test.go b/integration/agentcompat/internal/workflowpolicy/quality_workflow_test.go index 8192e58a..77965859 100644 --- a/integration/agentcompat/internal/workflowpolicy/quality_workflow_test.go +++ b/integration/agentcompat/internal/workflowpolicy/quality_workflow_test.go @@ -169,7 +169,7 @@ func requireCheckoutAndSetupGo(t *testing.T, steps []qualityStep) { require.False(t, *checkout.With.PersistCredentials) setupGo := steps[1] require.Equal(t, "actions/setup-go@v7", setupGo.Uses) - require.Equal(t, "1.26.x", setupGo.With.GoVersion) + require.Equal(t, "^1.26.10", setupGo.With.GoVersion) require.NotNil(t, setupGo.With.Cache) require.False(t, *setupGo.With.Cache) } diff --git a/integration/agentcompat/internal/workflowpolicy/stress_workflow_test.go b/integration/agentcompat/internal/workflowpolicy/stress_workflow_test.go index 6d28c0d9..fdd4c750 100644 --- a/integration/agentcompat/internal/workflowpolicy/stress_workflow_test.go +++ b/integration/agentcompat/internal/workflowpolicy/stress_workflow_test.go @@ -57,7 +57,7 @@ func TestPolicy_NezhaStressWorkflowRunsCrossRepositoryTest(t *testing.T) { setupGo := stressJob.stepNamed(t, "Set up Go") require.Equal(t, "actions/setup-go@v7", setupGo.Uses) - require.Equal(t, "1.26.x", setupGo.With.GoVersion) + require.Equal(t, "^1.26.10", setupGo.With.GoVersion) require.False(t, *setupGo.With.Cache) prepareDashboardInputs := stressJob.stepNamed(t, "Prepare Dashboard build inputs") diff --git a/model/alert_evaluation_test.go b/model/alert_evaluation_test.go new file mode 100644 index 00000000..3267685c --- /dev/null +++ b/model/alert_evaluation_test.go @@ -0,0 +1,205 @@ +package model + +import ( + "math" + "testing" + "time" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func TestAlertEvaluationDistinguishesUnknownFromBreach(t *testing.T) { + server := &Server{Common: Common{ID: 1}} + rule := &Rule{Type: "cpu", Max: 80, Cover: RuleCoverAll} + if passed, known := rule.Evaluate(nil, server, nil); !passed || known { + t.Fatalf("missing state = (%v, %v), want unknown", passed, known) + } + lease := server.AttachStateStream(runtimeOwnershipStream{}) + if !lease.UpdateState(&HostState{CPU: 99}, time.Now()) { + t.Fatal("state update rejected") + } + if passed, known := rule.Evaluate(nil, server, nil); passed || !known { + t.Fatalf("fresh high CPU = (%v, %v), want known breach", passed, known) + } + if !lease.Clear() { + t.Fatal("stream clear rejected") + } + // The raw metric remains available for historical display. The sentinel + // enforces freshness and must not count this value after disconnect. + if !server.RuntimeSnapshot().LastActive.IsZero() { + t.Fatal("disconnect did not clear online visibility") + } +} + +func newCycleTestDB(t *testing.T, migrate bool) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if migrate { + if err := db.AutoMigrate(&Transfer{}); err != nil { + t.Fatal(err) + } + } + return db +} + +func newCycleTestStats() *CycleTransferStats { + return &CycleTransferStats{ + ServerName: make(map[uint64]string), + Transfer: make(map[uint64]uint64), + NextUpdate: make(map[uint64]time.Time), + } +} + +func TestCycleRuleRejectsInvalidConfiguration(t *testing.T) { + start := time.Now().Add(-time.Hour) + rule := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Max: 100} + if !rule.HasSafeCycleConfiguration() { + t.Fatal("normal hourly cycle rejected") + } + for _, tc := range []struct { + name string + mutate func(*Rule) + }{ + {"unknown unit", func(r *Rule) { r.CycleUnit = "fortnight" }}, + {"negative limit", func(r *Rule) { r.Max = -1 }}, + {"NaN limit", func(r *Rule) { r.Max = math.NaN() }}, + {"reversed limits", func(r *Rule) { r.Min, r.Max = 200, 100 }}, + {"future start", func(r *Rule) { future := time.Now().Add(time.Hour); r.CycleStart = &future }}, + } { + t.Run(tc.name, func(t *testing.T) { + copy := *rule + tc.mutate(©) + if copy.HasSafeCycleConfiguration() { + t.Fatal("unsafe cycle accepted") + } + }) + } +} + +func TestCycleBoundsUseOneClockAtBoundary(t *testing.T) { + start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + rule := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Max: 100} + before := start.Add(59*time.Minute + 59*time.Second) + from, to := rule.transferDurationBounds(before) + if !from.Equal(start) || !to.Equal(start.Add(time.Hour)) { + t.Fatalf("before boundary = %v..%v", from, to) + } + from, to = rule.transferDurationBounds(start.Add(time.Hour)) + if !from.Equal(start.Add(time.Hour)) || !to.Equal(start.Add(2*time.Hour)) { + t.Fatalf("at boundary = %v..%v", from, to) + } +} + +func TestCycleQueryFailureIsUnknownAndDoesNotCache(t *testing.T) { + start := time.Now().Add(-time.Hour) + rule := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Max: 50} + server := &Server{Common: Common{ID: 1}, State: &HostState{}} + passed, known := rule.Evaluate(newCycleTestStats(), server, newCycleTestDB(t, false)) + if !passed || known || len(rule.NextTransferAt) != 0 { + t.Fatalf("SQL error yielded passed=%v known=%v cache=%v", passed, known, rule.NextTransferAt) + } +} + +func TestCycleQueryExcludesFutureRowsAndHandlesMinOnly(t *testing.T) { + start := time.Now().Add(-90 * time.Minute) + db := newCycleTestDB(t, true) + server := &Server{Common: Common{ID: 1}, State: &HostState{}} + for _, row := range []Transfer{ + {Common: Common{CreatedAt: time.Now().Add(-time.Minute)}, ServerID: 1, Out: 10}, + {Common: Common{CreatedAt: time.Now().Add(10 * time.Minute)}, ServerID: 1, Out: 100}, + } { + if err := db.Create(&row).Error; err != nil { + t.Fatal(err) + } + } + rule := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Max: 50} + stats := newCycleTestStats() + passed, known := rule.Evaluate(stats, server, db) + if !passed || !known || stats.Transfer[1] != 10 { + t.Fatalf("future row included: passed=%v known=%v amount=%d", passed, known, stats.Transfer[1]) + } + minOnly := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Min: 20} + passed, known = minOnly.Evaluate(newCycleTestStats(), server, db) + if passed || !known { + t.Fatalf("min-only cycle = (%v,%v), want known breach", passed, known) + } +} + +func TestAlertEvaluationAllNetworkSpeedUsesBothDirections(t *testing.T) { + for _, tc := range []struct { + name string + in, out uint64 + breach bool + }{ + {"inbound only", 100, 0, true}, + {"outbound below combined limit", 0, 30, false}, + {"both directions", 25, 30, true}, + } { + t.Run(tc.name, func(t *testing.T) { + server := &Server{Common: Common{ID: 1}, State: &HostState{NetInSpeed: tc.in, NetOutSpeed: tc.out}} + rule := &Rule{Type: "net_all_speed", Max: 50, Cover: RuleCoverAll} + passed, known := rule.Evaluate(nil, server, nil) + if !known || passed == tc.breach { + t.Fatalf("got passed=%v known=%v, want breach=%v", passed, known, tc.breach) + } + }) + } +} + +func TestAlertEvaluationUnavailableSensorsAreUnknown(t *testing.T) { + tests := []struct { + name string + rule string + state *HostState + host *Host + }{ + {"missing memory host", "memory", &HostState{MemUsed: 1}, nil}, + {"zero memory total", "memory", &HostState{MemUsed: 1}, &Host{}}, + {"missing GPU", "gpu_max", &HostState{}, &Host{}}, + {"missing temperature", "temperature_max", &HostState{}, &Host{}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := &Server{Common: Common{ID: 1}, State: tc.state, Host: tc.host} + passed, known := (&Rule{Type: tc.rule, Max: 50, Cover: RuleCoverAll}).Evaluate(nil, server, nil) + if !passed || known { + t.Fatalf("got passed=%v known=%v, want unknown", passed, known) + } + }) + } +} + +func TestOfflineRuleDoesNotRequireMetricState(t *testing.T) { + server := &Server{Common: Common{ID: 1}} + InitServer(server) + passed, known := (&Rule{Type: "offline", Cover: RuleCoverAll}).Evaluate(nil, server, nil) + if passed || !known { + t.Fatalf("offline without state = (%v, %v), want known breach", passed, known) + } +} + +func TestCompoundAlertUsesOneImmutableAgentReport(t *testing.T) { + server := &Server{Common: Common{ID: 1}} + alert := &AlertRule{Rules: []*Rule{ + {Type: "cpu", Max: 80, Cover: RuleCoverAll}, + {Type: "memory", Max: 80, Cover: RuleCoverAll}, + }} + oldReport := RuntimeSnapshot{ + State: &HostState{CPU: 99, MemUsed: 0}, + Host: &Host{MemTotal: 100}, + } + newReport := RuntimeSnapshot{ + State: &HostState{CPU: 0, MemUsed: 99}, + Host: &Host{MemTotal: 100}, + } + for _, report := range []RuntimeSnapshot{oldReport, newReport} { + point, known := alert.SnapshotStatusWithRuntime(nil, server, report, nil) + if !known || len(point) != 2 || point[0] == point[1] { + t.Fatalf("compound point combined incompatible reports: known=%v point=%v", known, point) + } + } +} diff --git a/model/alert_timed_test.go b/model/alert_timed_test.go new file mode 100644 index 00000000..9a3eb16e --- /dev/null +++ b/model/alert_timed_test.go @@ -0,0 +1,104 @@ +package model + +import ( + "testing" + "time" +) + +func timedRule(kind string, duration uint64) *AlertRule { + return &AlertRule{Rules: []*Rule{{Type: kind, Duration: duration, Max: 80, Cover: RuleCoverAll}}} +} + +func timedPoints(start time.Time, values []bool, spacing time.Duration) []TimedAlertPoint { + points := make([]TimedAlertPoint, len(values)) + for i, value := range values { + points[i] = TimedAlertPoint{At: start.Add(time.Duration(i) * spacing), Values: []bool{value}} + } + return points +} + +func TestTimedAlertNeedsElapsedWindowAndDistinctReports(t *testing.T) { + start := time.Unix(1_700_000_000, 0) + rule := timedRule("cpu", 5) + points := timedPoints(start, []bool{false, false, false, false}, AlertSampleInterval) + if known, _ := rule.CheckTimed(points, points[len(points)-1].At); known { + t.Fatal("four reports decided a five-tick window") + } + points = append(points, TimedAlertPoint{At: start.Add(12 * time.Second), Values: []bool{false}}) + if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || passed { + t.Fatalf("complete high-CPU window: known=%v passed=%v", known, passed) + } + // Repeating the same observation does not advance wall-clock coverage. + if known, _ := rule.CheckTimed(points, start.Add(30*time.Second)); known { + t.Fatal("stale sample created a new complete window") + } +} + +func TestTimedAlertGapIsUnknownAndCannotResolve(t *testing.T) { + start := time.Unix(1_700_000_000, 0) + rule := timedRule("cpu", 5) + points := []TimedAlertPoint{ + {At: start, Values: []bool{false}}, + {At: start.Add(30 * time.Second), Values: []bool{true}}, + } + if known, _ := rule.CheckTimed(points, start.Add(30*time.Second)); known { + t.Fatal("telemetry gap must not be interpreted as healthy") + } +} + +func TestTimedAlertIrregularReportsUseElapsedTime(t *testing.T) { + start := time.Unix(1_700_000_000, 0) + rule := timedRule("cpu", 6) + points := timedPoints(start, []bool{false, false, false, false}, 5*time.Second) + if known, passed := rule.CheckTimed(points, start.Add(15*time.Second)); !known || passed { + t.Fatalf("irregular reports cover the wall-clock window: known=%v passed=%v", known, passed) + } +} + +func TestTimedAlertExactSeventyPercentIsNotIncident(t *testing.T) { + start := time.Unix(1_700_000_000, 0) + rule := timedRule("cpu", 10) + values := []bool{false, false, false, false, false, false, false, true, true, true} + points := timedPoints(start, values, AlertSampleInterval) + if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || !passed { + t.Fatalf("70%% failed is below the >70%% trigger: known=%v passed=%v", known, passed) + } +} + +func TestTimedAlertTenTickWindowDoesNotDecideAtNine(t *testing.T) { + start := time.Unix(1_700_000_000, 0) + rule := timedRule("cpu", 10) + points := timedPoints(start, []bool{false, false, false, false, false, false, false, false, false}, AlertSampleInterval) + if known, _ := rule.CheckTimed(points, points[len(points)-1].At); known { + t.Fatal("nine reports decided a ten-tick window one tick early") + } + points = append(points, TimedAlertPoint{At: start.Add(27 * time.Second), Values: []bool{false}}) + if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || passed { + t.Fatalf("full ten-tick breach: known=%v passed=%v", known, passed) + } +} + +func TestTimedOfflineRequiresSustainedAbsence(t *testing.T) { + start := time.Unix(1_700_000_000, 0) + rule := timedRule("offline", 5) + points := timedPoints(start, []bool{false, false, false, false, false}, AlertSampleInterval) + if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || passed { + t.Fatalf("continuous offline interval: known=%v passed=%v", known, passed) + } + points = append(points, TimedAlertPoint{At: start.Add(15 * time.Second), Values: []bool{true}}) + if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || !passed { + t.Fatalf("online observation should resolve offline: known=%v passed=%v", known, passed) + } +} + +func TestTimedCompoundRuleWaitsForAllKnown(t *testing.T) { + start := time.Unix(1_700_000_000, 0) + rule := &AlertRule{Rules: []*Rule{ + {Type: "cpu", Duration: 5, Max: 80, Cover: RuleCoverAll}, + {Type: "memory", Duration: 5, Max: 80, Cover: RuleCoverAll}, + }} + points := timedPoints(start, []bool{false, false, false, false, false}, AlertSampleInterval) + if known, _ := rule.CheckTimed(points, points[len(points)-1].At); known { + t.Fatal("missing second condition treated as decisive") + } +} diff --git a/model/alertrule.go b/model/alertrule.go index 37e734c6..c4f65dae 100644 --- a/model/alertrule.go +++ b/model/alertrule.go @@ -2,12 +2,134 @@ package model import ( "slices" + "time" "github.com/gin-gonic/gin" "github.com/goccy/go-json" "gorm.io/gorm" ) +const ( + AlertSampleInterval = 3 * time.Second + AlertSampleMaxAge = 6 * time.Second +) + +// TimedAlertPoint records one distinct observation. Its timestamp is the +// dashboard receipt time of the Agent report, or the sentinel tick for an +// offline-only rule. +type TimedAlertPoint struct { + At time.Time + Values []bool +} + +// CheckTimed evaluates the legacy Duration as Duration * 3 seconds, avoiding +// both repeated stale samples and cadence-dependent early incidents. Unknown +// coverage cannot create an incident or a recovery. +func (r *AlertRule) CheckTimed(points []TimedAlertPoint, now time.Time) (known, passed bool) { + if r == nil || len(r.Rules) == 0 { + return false, false + } + allFailed := true + for index, rule := range r.Rules { + if rule == nil || !rule.IsSupportedType() { + return false, false + } + ruleKnown, rulePassed := rule.checkTimed(points, index, now) + if !ruleKnown { + return false, false + } + if rulePassed { + allFailed = false + } + } + return true, !allFailed +} + +func (rule *Rule) checkTimed(points []TimedAlertPoint, index int, now time.Time) (known, passed bool) { + if len(points) == 0 { + return false, false + } + if rule.IsTransferDurationRule() { + last := points[len(points)-1] + if index >= len(last.Values) || now.Sub(last.At) > AlertSampleMaxAge { + return false, false + } + return true, last.Values[index] + } + window := time.Duration(rule.Duration) * AlertSampleInterval + if window <= 0 { + return false, false + } + end := now.Add(AlertSampleInterval) // the current observation represents one tick + start := end.Add(-window) + if points[0].At.After(start) { + return false, false // the configured wall-clock window has not elapsed + } + var covered, failed time.Duration + for i, point := range points { + if index >= len(point.Values) { + return false, false + } + segmentStart := point.At + if start.After(segmentStart) { + segmentStart = start + } + segmentEnd := point.At.Add(AlertSampleMaxAge) + if end.Before(segmentEnd) { + segmentEnd = end + } + if i+1 < len(points) { + if points[i+1].At.Before(segmentEnd) { + segmentEnd = points[i+1].At + } + } else { + if tickEnd := point.At.Add(AlertSampleInterval); tickEnd.Before(segmentEnd) { + segmentEnd = tickEnd + } + } + if !segmentEnd.After(segmentStart) { + continue + } + span := segmentEnd.Sub(segmentStart) + covered += span + if !point.Values[index] { + failed += span + } + } + if covered < window-window/10 { + return false, false + } + if rule.IsOfflineRule() { + if covered-failed >= AlertSampleInterval { + return true, true + } + if failed >= window-AlertSampleInterval && covered >= window-AlertSampleInterval { + return true, false + } + return false, false + } + if float64(failed) > float64(window)*0.70 { + return true, false + } + if float64(failed+window-covered) <= float64(window)*0.70 { + return true, true + } + return false, false +} + +func (r *AlertRule) TimedRetention() time.Duration { + longest := AlertSampleMaxAge + for _, rule := range r.Rules { + if rule == nil || rule.IsTransferDurationRule() { + continue + } + if duration, ok := rule.DurationInt(); ok { + longest = max(longest, time.Duration(duration)*AlertSampleInterval+AlertSampleMaxAge) + } + } + return longest +} + const ( ModeAlwaysTrigger = 0 ModeOnetimeTrigger = 1 @@ -64,6 +186,39 @@ func (r *AlertRule) Enabled() bool { return r.Enable != nil && *r.Enable } +// IsSafeToEvaluate validates persisted rule data before it reaches the alert +// goroutine. API validation protects new writes; this second boundary protects +// upgrades from malformed or deliberately poisoned rows already in the DB. +func (r *AlertRule) IsSafeToEvaluate() bool { + if r == nil || len(r.Rules) == 0 { + return false + } + for _, rule := range r.Rules { + if rule == nil || !rule.IsSupportedType() { + return false + } + if !rule.HasSafeThresholds() { + return false + } + switch rule.Cover { + case RuleCoverAll, RuleCoverIgnoreAll: + default: + return false + } + if rule.IsTransferDurationRule() { + if !rule.HasSafeCycleConfiguration() { + return false + } + continue + } + duration, ok := rule.DurationInt() + if !ok || duration < 3 { + return false + } + } + return true +} + // HasPermission extends the default owner/admin check with PAT // server_ids whitelist enforcement. AlertRule.Snapshot fans out across // every owner-visible server filtered only by Rule.Ignore semantics @@ -122,22 +277,52 @@ func (r *AlertRule) HasPermission(ctx *gin.Context) bool { // Snapshot 对传入的Server进行该报警规则下所有type的检查 返回每项检查结果 func (r *AlertRule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) []bool { + point, _ := r.SnapshotStatus(cycleTransferStats, server, db) + return point +} + +// SnapshotStatus preserves the distinction between a healthy measurement and +// telemetry that is unavailable. A compound alert cannot be decided while any +// of its applicable conditions is unknown. +func (r *AlertRule) SnapshotStatus(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) ([]bool, bool) { + if server == nil { + return nil, false + } + return r.SnapshotStatusWithRuntime(cycleTransferStats, server, server.RuntimeSnapshot(), db) +} + +// SnapshotStatusWithRuntime uses one immutable report for every condition. +func (r *AlertRule) SnapshotStatusWithRuntime(cycleTransferStats *CycleTransferStats, server *Server, runtime RuntimeSnapshot, db *gorm.DB) ([]bool, bool) { point := make([]bool, len(r.Rules)) + known := true for i, rule := range r.Rules { - point[i] = rule.Snapshot(cycleTransferStats, server, db) + if rule == nil || !rule.IsSupportedType() { + // Invalid persisted rules are ignored instead of being interpreted as + // a failed condition or allowed to panic the sentinel. + point[i] = true + known = false + continue + } + var valid bool + point[i], valid = rule.EvaluateRuntime(cycleTransferStats, server, runtime, db) + known = known && valid } - return point + return point, known } // Check 传入包含当前报警规则下所有type检查结果 返回报警持续时间与是否通过报警检查(通过则返回true) func (r *AlertRule) Check(points [][]bool) (int, bool) { var hasPassedRule bool durations := make([]int, len(r.Rules)) + validRules := 0 for ruleIndex, rule := range r.Rules { - duration := int(rule.Duration) + if rule == nil || !rule.IsSupportedType() { + continue + } if rule.IsTransferDurationRule() { + validRules++ // 循环区间流量报警 if durations[ruleIndex] < 1 { durations[ruleIndex] = 1 @@ -146,10 +331,21 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) { continue } // 只要最后一次检查超出了规则范围 就认为检查未通过 - if len(points) > 0 && points[len(points)-1][ruleIndex] { - hasPassedRule = true + if len(points) > 0 { + lastPoint := points[len(points)-1] + if ruleIndex >= len(lastPoint) || lastPoint[ruleIndex] { + hasPassedRule = true + } } - } else if rule.IsOfflineRule() { + continue + } + + duration, ok := rule.DurationInt() + if !ok || duration <= 0 { + continue + } + validRules++ + if rule.IsOfflineRule() { // 离线报警，检查直到最后一次在线的离线采样点是否大于 duration if hasPassedRule = boundCheck(len(points), duration, hasPassedRule); hasPassedRule { continue @@ -157,7 +353,7 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) { var fail int for _, point := range slices.Backward(points[len(points)-duration:]) { fail++ - if point[ruleIndex] { + if ruleIndex >= len(point) || point[ruleIndex] { hasPassedRule = true break } @@ -168,11 +364,7 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) { // 常规报警 // duration<=0 是无意义的规则（持续 0 秒）：直接跳过该规则， // 既不污染 hasPassedRule（否则会连带跳过同一 alert 里其它有效 - // 规则），也避免下方 fail*100/total 在 total=0 时整数除零 panic - // —— checkStatus 无 recover，一次 panic 会拖垮整个告警 goroutine。 - if duration <= 0 { - continue - } + // 规则），也避免下方百分比计算在 total=0 时除零。 if hasPassedRule = boundCheck(len(points), duration, hasPassedRule); hasPassedRule { continue } @@ -181,17 +373,20 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) { } total, fail := duration, 0 for timeTick := len(points) - duration; timeTick < len(points); timeTick++ { - if !points[timeTick][ruleIndex] { + if ruleIndex >= len(points[timeTick]) || !points[timeTick][ruleIndex] { fail++ } } // 当70%以上的采样点未通过规则判断时 才认为当前检查未通过 - if fail*100/total <= 70 { + if float64(fail)*100/float64(total) <= 70 { hasPassedRule = true } } } + if validRules == 0 { + return 0, true + } // 仅当所有检查均未通过时 才触发告警 return slices.Max(durations), hasPassedRule } @@ -205,10 +400,13 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) { func (r *AlertRule) RetentionWindow() int { window := 0 for _, rule := range r.Rules { + if rule == nil || !rule.IsSupportedType() { + continue + } var need int if rule.IsTransferDurationRule() { need = 1 - } else if d := int(rule.Duration); d > 0 { + } else if d, ok := rule.DurationInt(); ok && d > 0 { need = d } if need > window { diff --git a/model/alertrule_security_test.go b/model/alertrule_security_test.go new file mode 100644 index 00000000..8ded87b5 --- /dev/null +++ b/model/alertrule_security_test.go @@ -0,0 +1,123 @@ +package model + +import ( + "math" + "testing" + "time" +) + +func TestRuleSnapshotIgnoresEmptyMetricSamples(t *testing.T) { + tests := []struct { + name string + ruleType string + hostState *HostState + }{ + { + name: "GPU list is empty", + ruleType: "gpu_max", + hostState: &HostState{}, + }, + { + name: "all temperature samples are filtered", + ruleType: "temperature_max", + hostState: &HostState{Temperatures: []SensorTemperature{ + {Name: "coretemp", Temperature: 0}, + }}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := &Server{Common: Common{ID: 1}, State: test.hostState, Host: &Host{}} + rule := &Rule{Type: test.ruleType, Max: 1, Duration: 3, Cover: RuleCoverAll} + + defer func() { + if recovered := recover(); recovered != nil { + t.Fatalf("Snapshot panicked for missing metric data: %v", recovered) + } + }() + if passed := rule.Snapshot(nil, server, nil); !passed { + t.Fatal("missing metric data must be ignored instead of treated as a failed threshold") + } + }) + } +} + +func TestAlertRuleCheckRejectsOverflowingPersistedDuration(t *testing.T) { + rule := &AlertRule{Rules: []*Rule{{ + Type: "offline", + Duration: math.MaxUint64, + Cover: RuleCoverAll, + }}} + + defer func() { + if recovered := recover(); recovered != nil { + t.Fatalf("Check panicked after uint64-to-int overflow: %v", recovered) + } + }() + duration, passed := rule.Check([][]bool{{false}}) + if duration != 0 || !passed { + t.Fatalf("invalid persisted duration must be ignored safely, got duration=%d passed=%v", duration, passed) + } + if window := rule.RetentionWindow(); window != 0 { + t.Fatalf("invalid persisted duration must not create a retention window, got %d", window) + } +} + +func TestAlertRulePersistedDataSafety(t *testing.T) { + cycleStart := time.Now().Add(-time.Hour) + tests := []struct { + name string + rule *AlertRule + want bool + }{ + { + name: "normal rule", + rule: &AlertRule{Rules: []*Rule{{ + Type: "cpu", Duration: 3, Cover: RuleCoverAll, + }}}, + want: true, + }, + { + name: "normal cycle rule", + rule: &AlertRule{Rules: []*Rule{{ + Type: "transfer_in_cycle", CycleStart: &cycleStart, CycleInterval: 1, Cover: RuleCoverAll, + }}}, + want: true, + }, + { + name: "unknown type", + rule: &AlertRule{Rules: []*Rule{{ + Type: "attacker_controlled", Duration: 3, Cover: RuleCoverAll, + }}}, + }, + { + name: "overflowing duration", + rule: &AlertRule{Rules: []*Rule{{ + Type: "offline", Duration: math.MaxUint64, Cover: RuleCoverAll, + }}}, + }, + { + name: "cycle without start", + rule: &AlertRule{Rules: []*Rule{{ + Type: "transfer_in_cycle", CycleInterval: 1, Cover: RuleCoverAll, + }}}, + }, + { + name: "nil rule", + rule: &AlertRule{Rules: []*Rule{nil}}, + }, + { + name: "empty rule list", + rule: &AlertRule{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := test.rule.IsSafeToEvaluate(); got != test.want { + t.Fatalf("IsSafeToEvaluate()=%v, want %v", got, test.want) + } + }) + } +} diff --git a/model/alertrule_test.go b/model/alertrule_test.go index 72cd18df..81ae7b5b 100644 --- a/model/alertrule_test.go +++ b/model/alertrule_test.go @@ -26,7 +26,7 @@ func testCycleRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { - Type: "_cycle", + Type: "transfer_in_cycle", }, }, }, @@ -39,7 +39,7 @@ func testCycleRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { - Type: "_cycle", + Type: "transfer_in_cycle", }, }, }, @@ -130,6 +130,7 @@ func testGeneralRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, }, @@ -143,6 +144,7 @@ func testGeneralRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, }, @@ -156,6 +158,7 @@ func testGeneralRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, }, @@ -169,6 +172,7 @@ func testGeneralRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, }, @@ -182,6 +186,7 @@ func testGeneralRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, }, @@ -210,6 +215,7 @@ func testCombinedRules(t *testing.T) { Duration: 10, }, { + Type: "cpu", Duration: 10, }, }, @@ -227,6 +233,7 @@ func testCombinedRules(t *testing.T) { Duration: 10, }, { + Type: "cpu", Duration: 10, }, }, @@ -240,6 +247,7 @@ func testCombinedRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, { @@ -257,9 +265,11 @@ func testCombinedRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, { + Type: "memory", Duration: 30, }, }, @@ -273,9 +283,11 @@ func testCombinedRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, { + Type: "memory", Duration: 30, }, }, @@ -416,7 +428,7 @@ func TestAlertRule_RetentionWindow(t *testing.T) { {"zero duration only", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 0}}}, 0}, {"mixed picks max", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 0}, {Type: "cpu", Duration: 7}}}, 7}, {"offline keeps Duration", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 30}}}, 30}, - {"cycle looks back one", &AlertRule{Rules: []*Rule{{Type: "net_in_speed_cycle"}}}, 1}, + {"cycle looks back one", &AlertRule{Rules: []*Rule{{Type: "transfer_in_cycle"}}}, 1}, } for _, c := range cases { if got := c.rule.RetentionWindow(); got != c.want { @@ -469,7 +481,7 @@ func TestAlertRule_CombinedRuleAccumulatesSamples(t *testing.T) { }{ {"general3+general10", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 3}, {Type: "memory", Duration: 10}}}, []bool{false, false}, 9, 10}, {"offline5+general10", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 5}, {Type: "cpu", Duration: 10}}}, []bool{false, false}, 9, 10}, - {"transfer+general8", &AlertRule{Rules: []*Rule{{Type: "net_in_speed_cycle"}, {Type: "cpu", Duration: 8}}}, []bool{false, false}, 7, 8}, + {"transfer+general8", &AlertRule{Rules: []*Rule{{Type: "transfer_in_cycle"}, {Type: "cpu", Duration: 8}}}, []bool{false, false}, 7, 8}, {"offline3+offline12", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 3}, {Type: "offline", Duration: 12}}}, []bool{false, false}, 11, 12}, } for _, c := range cases { diff --git a/model/config.go b/model/config.go index 3f936e46..41783765 100644 --- a/model/config.go +++ b/model/config.go @@ -66,6 +66,9 @@ type ConfigDashboard struct { AdminTemplate string `koanf:"admin_template" json:"admin_template,omitempty"` EnablePlainIPInNotification bool `koanf:"enable_plain_ip_in_notification" json:"enable_plain_ip_in_notification,omitempty"` // 通知信息IP不打码 + // AllowJWTIPChange disables the IP binding check for browser login sessions. + // Keep this disabled by default so a stolen JWT remains bound to its issuing IP. + AllowJWTIPChange bool `koanf:"allow_jwt_ip_change" json:"allow_jwt_ip_change,omitempty"` EnableMCP bool `koanf:"enable_mcp" json:"enable_mcp,omitempty"` // 是否启用 MCP 入口（默认关闭；启用前请审视 PAT scope/whitelist） @@ -91,6 +94,10 @@ type ConfigDashboard struct { AdminEmail string `koanf:"admin_email" json:"admin_email,omitempty"` DomainExpiryNotificationDays string `koanf:"domain_expiry_notification_days" json:"domain_expiry_notification_days,omitempty"` ServerExpiryNotificationDays string `koanf:"server_expiry_notification_days" json:"server_expiry_notification_days,omitempty"` + + // OnlineStatsDays 设备在线时间段统计的保留与查询窗口（天）。 + // 取值范围 [MinOnlineStatsDays, MaxOnlineStatsDays]，默认 DefaultOnlineStatsDays。 + OnlineStatsDays int `koanf:"online_stats_days" json:"online_stats_days,omitempty"` } type Config struct { @@ -118,6 +125,10 @@ type Config struct { // SettingResponse 按值拷贝 ConfigDashboard 触发 copylocks。 mcpEnabled atomic.Bool `koanf:"-" json:"-" yaml:"-"` + // jwtIPChangeAllowed mirrors AllowJWTIPChange for concurrent authentication + // and setting updates. Keep atomic fields outside ConfigDashboard. + jwtIPChangeAllowed atomic.Bool `koanf:"-" json:"-" yaml:"-"` + // oauth2 配置 Oauth2 map[string]*Oauth2Config `koanf:"oauth2" json:"oauth2,omitempty"` @@ -240,6 +251,15 @@ func (c *Config) Read(path string, frontendTemplates []FrontendTemplate) error { c.JWTTimeout = 1 } + if c.OnlineStatsDays == 0 { + c.OnlineStatsDays = DefaultOnlineStatsDays + } else if c.OnlineStatsDays < MinOnlineStatsDays || c.OnlineStatsDays > MaxOnlineStatsDays { + log.Printf("NEZHA>> online_stats_days %d out of range [%d, %d], reset to %d", c.OnlineStatsDays, MinOnlineStatsDays, MaxOnlineStatsDays, DefaultOnlineStatsDays) + c.OnlineStatsDays = DefaultOnlineStatsDays + } + + c.jwtIPChangeAllowed.Store(c.AllowJWTIPChange) + if c.AgentSecretKey == "" { c.AgentSecretKey, err = utils.GenerateRandomString(32) if err != nil { @@ -267,6 +287,17 @@ func (c *Config) SetMCPEnabled(v bool) { c.mcpEnabled.Store(v) } +// JWTIPChangeAllowed reports whether JWT sessions may change IP concurrently. +func (c *Config) JWTIPChangeAllowed() bool { + return c.jwtIPChangeAllowed.Load() +} + +// SetJWTIPChangeAllowed updates only the atomic mirror. save() synchronizes +// the public field for persistence; authentication must use the accessor. +func (c *Config) SetJWTIPChangeAllowed(v bool) { + c.jwtIPChangeAllowed.Store(v) +} + // Save 保存配置文件 func (c *Config) Save() error { return c.save() @@ -335,6 +366,7 @@ func (c *Config) patchYAMLField(key string, value any) error { func (c *Config) save() error { c.EnableMCP = c.mcpEnabled.Load() + c.AllowJWTIPChange = c.jwtIPChangeAllowed.Load() data, err := yaml.Marshal(c) if err != nil { return err diff --git a/model/config_test.go b/model/config_test.go index 9c00eb26..bd737b81 100644 --- a/model/config_test.go +++ b/model/config_test.go @@ -24,6 +24,7 @@ func TestReadConfig(t *testing.T) { {"user_template", c.UserTemplate, c.UserTemplate == "user-dist"}, {"admin_template", c.AdminTemplate, c.AdminTemplate == "admin-dist"}, {"agent_secret_key", c.AgentSecretKey, c.AgentSecretKey != ""}, + {"allow_jwt_ip_change", c.AllowJWTIPChange, !c.AllowJWTIPChange}, } for _, field := range testFields { @@ -37,6 +38,7 @@ func TestReadConfig(t *testing.T) { t.Run("ReadFile", func(t *testing.T) { const testCfg = ` +allow_jwt_ip_change: true jwt_secret_key: test user_template: um admin_template: am @@ -85,6 +87,7 @@ admin_email: admin@loohui.com {"smtp_user", c.SMTPUser, c.SMTPUser == "test@loohui.com"}, {"smtp_password", c.SMTPPassword, c.SMTPPassword == "password123"}, {"admin_email", c.AdminEmail, c.AdminEmail == "admin@loohui.com"}, + {"allow_jwt_ip_change", c.AllowJWTIPChange, c.AllowJWTIPChange}, } for _, field := range testFields { diff --git a/model/host.go b/model/host.go index f054798b..18f4e3da 100644 --- a/model/host.go +++ b/model/host.go @@ -35,9 +35,28 @@ type HostState struct { ProcessCount uint64 `json:"process_count,omitempty"` Temperatures []SensorTemperature `json:"temperatures,omitempty"` GPU []float64 `json:"gpu,omitempty"` + GPUs []GPUStat `json:"gpus,omitempty"` +} + +// GPUStat carries per-card figures, index-aligned with Host.GPU. Memory is in +// MiB and stays zero for vendors that do not report it, so MemoryTotal == 0 +// means "unknown" rather than "no memory". +type GPUStat struct { + Utilization float64 `json:"utilization"` + MemoryUsed uint64 `json:"memory_used,omitempty"` + MemoryTotal uint64 `json:"memory_total,omitempty"` } func (s *HostState) PB() *pb.State { + gs := make([]*pb.State_GPU, 0, len(s.GPUs)) + for _, g := range s.GPUs { + gs = append(gs, &pb.State_GPU{ + Utilization: g.Utilization, + MemoryUsed: g.MemoryUsed, + MemoryTotal: g.MemoryTotal, + }) + } + var ts []*pb.State_SensorTemperature for _, t := range s.Temperatures { ts = append(ts, &pb.State_SensorTemperature{ @@ -64,10 +83,20 @@ func (s *HostState) PB() *pb.State { ProcessCount: s.ProcessCount, Temperatures: ts, Gpu: s.GPU, + Gpus: gs, } } func PB2State(s *pb.State) HostState { + var gs []GPUStat + for _, g := range s.GetGpus() { + gs = append(gs, GPUStat{ + Utilization: g.GetUtilization(), + MemoryUsed: g.GetMemoryUsed(), + MemoryTotal: g.GetMemoryTotal(), + }) + } + var ts []SensorTemperature for _, t := range s.GetTemperatures() { ts = append(ts, SensorTemperature{ @@ -93,6 +122,7 @@ func PB2State(s *pb.State) HostState { UdpConnCount: s.GetUdpConnCount(), ProcessCount: s.GetProcessCount(), Temperatures: ts, + GPUs: gs, GPU: s.GetGpu(), } } diff --git a/model/jwt_ip_change_atomic_test.go b/model/jwt_ip_change_atomic_test.go new file mode 100644 index 00000000..bbb2b343 --- /dev/null +++ b/model/jwt_ip_change_atomic_test.go @@ -0,0 +1,56 @@ +package model + +import ( + "path/filepath" + "sync" + "testing" +) + +func TestJWTIPChangeAllowedPersistence(t *testing.T) { + c := &Config{} + if c.JWTIPChangeAllowed() { + t.Fatal("zero-value Config must bind JWT sessions to their IP") + } + // An absent agent secret makes Read save the config during initialization. + path := newTempConfig(t, "jwt_secret_key: test\nallow_jwt_ip_change: true\n") + if err := c.Read(path, nil); err != nil { + t.Fatal(err) + } + if !c.JWTIPChangeAllowed() { + t.Fatal("Read must initialize the atomic mirror before saving") + } + for _, want := range []bool{true, false} { + c.SetJWTIPChangeAllowed(want) + if err := c.Save(); err != nil { + t.Fatal(err) + } + reloaded := &Config{} + if err := reloaded.Read(path, nil); err != nil { + t.Fatal(err) + } + if c.AllowJWTIPChange != want || reloaded.JWTIPChangeAllowed() != want { + t.Fatalf("saved and reloaded setting must be %v", want) + } + } +} + +func TestJWTIPChangeAllowedConcurrentAccess(t *testing.T) { + c := &Config{filePath: filepath.Join(t.TempDir(), "config.yaml")} + var wg sync.WaitGroup + wg.Go(func() { + for i := 0; i < 10000; i++ { + c.SetJWTIPChangeAllowed(i%2 == 0) + } + }) + wg.Go(func() { + for i := 0; i < 10000; i++ { + _ = c.JWTIPChangeAllowed() + } + }) + for i := 0; i < 20; i++ { + if err := c.Save(); err != nil { + t.Error(err) + } + } + wg.Wait() +} diff --git a/model/notification.go b/model/notification.go index 86a6d186..e03fad8a 100644 --- a/model/notification.go +++ b/model/notification.go @@ -208,6 +208,7 @@ func (ns *NotificationServerBundle) replaceParamsInString(str string, message st } if ns.Server != nil { + runtime := ns.Server.RuntimeSnapshot() var noteData struct { BillingDataMod struct { EndDate string `json:"endDate"` @@ -238,8 +239,6 @@ func (ns *NotificationServerBundle) replaceParamsInString(str string, message st "#SERVER.BILLING_AMOUNT#", mod(amountStr), "#SERVER.BILLING_CYCLE#", mod(cycleStr), ) - - runtime := ns.Server.RuntimeSnapshot() if runtime.State != nil && runtime.Host != nil { state := runtime.State host := runtime.Host @@ -321,13 +320,13 @@ func (ns *NotificationServerBundle) replaceParamsInString(str string, message st ipv6 = ip.IPv6Addr validIP = ipv6 } - } - replacements = append(replacements, - "#SERVER.IP#", mod(validIP), - "#SERVER.IPV4#", mod(ipv4), - "#SERVER.IPV6#", mod(ipv6), - ) + replacements = append(replacements, + "#SERVER.IP#", mod(validIP), + "#SERVER.IPV4#", mod(ipv4), + "#SERVER.IPV6#", mod(ipv6), + ) + } } replacer := strings.NewReplacer(replacements...) diff --git a/model/online_history.go b/model/online_history.go new file mode 100644 index 00000000..79ba766c --- /dev/null +++ b/model/online_history.go @@ -0,0 +1,211 @@ +package model + +import ( + "gorm.io/gorm" + "sort" + "time" +) + +// 在线统计窗口的取值边界（天）。后台管理端可设置统计范围，查询窗口不允许 +// 超过该设置，历史数据按该设置做保留清理。 +const ( + DefaultOnlineStatsDays = 7 + MinOnlineStatsDays = 7 + MaxOnlineStatsDays = 3650 +) + +// OnlineActiveWindow 服务器状态上报距该窗口内视为在线。与 MCP server.list +// 的 onlineWindow 口径保持一致；超过该窗口未上报的进行中会话由定时清扫关闭。 +const OnlineActiveWindow = 30 * time.Second + +// OnlineSession 记录一台服务器的一段连续在线时间。Ongoing=true 表示会话 +// 仍在进行（此时 EndedAt 为零值）；会话结束时写入 EndedAt 并置 Ongoing=false。 +// LastActiveAt 由上报侧节流持久化，用于 dashboard 异常停机后把进行中的会话 +// 结转到最后一次已知活跃时刻。 +type OnlineSession struct { + ID uint64 `gorm:"primaryKey" json:"id,omitempty"` + CreatedAt time.Time `gorm:"index;<-:create" json:"created_at,omitempty"` + UpdatedAt time.Time `gorm:"autoUpdateTime" json:"updated_at,omitempty"` + ServerID uint64 `gorm:"index:idx_online_sessions_server_started;index:idx_online_sessions_server_closed,priority:1;index:idx_online_sessions_server_open,priority:1" json:"server_id,omitempty"` + StartedAt time.Time `gorm:"index:idx_online_sessions_server_started;index:idx_online_sessions_server_open,priority:3" json:"started_at,omitempty"` + EndedAt time.Time `gorm:"index;index:idx_online_sessions_server_closed,priority:3" json:"ended_at,omitempty"` // 会话结束时刻，进行中为零值 + // 上报侧节流持久化的最后活跃时刻（≥OnlineActiveWindow 间隔写一次） + LastActiveAt time.Time `json:"last_active_at,omitempty"` + Ongoing bool `gorm:"index:idx_online_sessions_server_closed,priority:2;index:idx_online_sessions_server_open,priority:2" json:"ongoing"` +} + +// BeforeSave keeps SQLite's lexicographic timestamp comparisons chronological. +// Query parameters and map-based updates must use UTC as well. +func (s *OnlineSession) BeforeSave(_ *gorm.DB) error { + s.StartedAt = s.StartedAt.UTC() + s.EndedAt = s.EndedAt.UTC() + s.LastActiveAt = s.LastActiveAt.UTC() + return nil +} + +// OnlinePeriod 是裁剪到查询窗口后的在线时间段，供前端时间轴直接渲染。 +type OnlinePeriod struct { + Start time.Time `json:"start"` + End time.Time `json:"end"` + Ongoing bool `json:"ongoing"` +} + +// DailyOnlineStat 是单个自然日（面板时区）在查询窗口内的在线秒数。 +// 窗口首尾为非整日时，只统计与窗口重叠的部分。 +type DailyOnlineStat struct { + Date string `json:"date"` + OnlineSeconds uint64 `json:"online_seconds"` +} + +// OnlineHistoryResponse 是单台服务器在线时间段统计的聚合结果。 +type OnlineHistoryResponse struct { + ServerID uint64 `json:"server_id"` + ServerName string `json:"server_name"` + Days int `json:"days"` + MaxDays int `json:"max_days"` + // Daily 拆日所用面板时区：IANA 名与窗口末端相对 UTC 的偏移秒数， + // 前端按同一时区渲染，保证跨时区访客的图形与数字一致 + Timezone string `json:"timezone"` + TzOffset int `json:"tz_offset"` + WindowStart time.Time `json:"window_start"` + WindowEnd time.Time `json:"window_end"` + TotalOnlineSeconds uint64 `json:"total_online_seconds"` + OnlineRate float64 `json:"online_rate"` + CurrentlyOnline bool `json:"currently_online"` + // 从未上报过的服务器该值为 nil；指针避免把 Go 零值时间序列化成 0001-01-01 + LastActive *time.Time `json:"last_active,omitempty"` + Periods []OnlinePeriod `json:"periods"` + Daily []DailyOnlineStat `json:"daily"` +} + +// ClipOnlineSessions 把与窗口 [windowStart, windowEnd) 相交的会话裁剪成 +// OnlinePeriod 列表。进行中的会话以 windowEnd 为右边界（保留 Ongoing 标记）。 +// 输入需按 StartedAt 升序；输出与输入同序。 +func ClipOnlineSessions(sessions []OnlineSession, windowStart, windowEnd time.Time) []OnlinePeriod { + periods := make([]OnlinePeriod, 0, len(sessions)) + for _, s := range sessions { + start := s.StartedAt + if start.Before(windowStart) { + start = windowStart + } + end := s.EndedAt + ongoing := s.Ongoing || end.IsZero() + if ongoing || end.After(windowEnd) { + end = windowEnd + } + if !start.Before(end) { + // 与窗口零交集（理论上不会由查询条件选出）或空会话 + continue + } + periods = append(periods, OnlinePeriod{Start: start, End: end, Ongoing: ongoing}) + } + return periods +} + +// DailyOnlineBreakdown 把在线时间段按面板时区的自然日拆分，返回窗口覆盖的 +// 每一天的在线秒数（含在线为 0 秒的日期），按日期升序。 +// 取整口径：每天先累计原始时长（日界切分不损失精度），仅在每日汇总时向下 +// 取整一次；调用方以各日之和作为总时长，保证响应内部 total 与 daily[] 严格 +// 对账，不与"逐段截断再累加"的口径混用（亚秒会话会让后者系统性偏小）。 +// 时段二分定位起始日界后逐日推进，复杂度 O(P·log D + D)；单服务器的会话 +// 互不重叠，内层推进总量受窗口天数约束，长窗口（如 days=3650）下不会出现 +// 时段数×天数的双重循环。 +func DailyOnlineBreakdown(periods []OnlinePeriod, windowStart, windowEnd time.Time, loc *time.Location) []DailyOnlineStat { + first := windowStart.In(loc) + // Advance a timezone-independent calendar, not a normalized local midnight: + // midnight DST gaps can normalize into the previous day and repeat dates. + date := time.Date(first.Year(), first.Month(), first.Day(), 0, 0, 0, 0, time.UTC) + var days []DailyOnlineStat + var dayStarts, dayEnds []time.Time + for { + start := onlineCalendarBoundary(date, loc) + if !start.Before(windowEnd) { + break + } + nextDate := date.AddDate(0, 0, 1) + end := onlineCalendarBoundary(nextDate, loc) + if start.Before(end) { + dayStarts = append(dayStarts, start) + dayEnds = append(dayEnds, end) + days = append(days, DailyOnlineStat{Date: date.Format("2006-01-02")}) + } + date = nextDate + } + + if len(days) == 0 { + return days + } + + dayDurations := make([]time.Duration, len(days)) + for _, p := range periods { + // 该时段覆盖的第一天：不晚于时段开始的最后一个日界 + i := sort.Search(len(dayStarts), func(i int) bool { + return dayStarts[i].After(p.Start) + }) - 1 + if i < 0 { + i = 0 + } + // 逐日推进，累加该时段与每一天 [dayStart, nextDay) 的交集 + for ; i < len(dayStarts) && dayStarts[i].Before(p.End); i++ { + overlapStart := maxTime(p.Start, dayStarts[i]) + overlapEnd := minTime(p.End, dayEnds[i]) + if overlapStart.Before(overlapEnd) { + dayDurations[i] += overlapEnd.Sub(overlapStart) + } + } + } + for i := range days { + days[i].OnlineSeconds = uint64(dayDurations[i].Seconds()) + } + return days +} + +func maxTime(a, b time.Time) time.Time { + if a.After(b) { + return a + } + return b +} + +func minTime(a, b time.Time) time.Time { + if a.Before(b) { + return a + } + return b +} + +// A missing midnight (or an entirely skipped date) starts at the timezone +// transition, not at the previous date returned by time.Date normalization. +func onlineCalendarBoundary(date time.Time, loc *time.Location) time.Time { + boundary := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, loc) + if boundary.Format("2006-01-02") < date.Format("2006-01-02") { + _, end := boundary.ZoneBounds() + if !end.IsZero() { + return end + } + } + // time.Date may choose the second midnight in a rollback. Recover the + // earlier occurrence using the offset on the preceding side of the transition. + zoneStart, _ := boundary.ZoneBounds() + if !zoneStart.IsZero() { + _, currentOffset := boundary.Zone() + _, previousOffset := zoneStart.Add(-time.Nanosecond).Zone() + if previousOffset > currentOffset { + earlier := boundary.Add(-time.Duration(previousOffset-currentOffset) * time.Second) + if earlier.Format("2006-01-02 15:04:05") == boundary.Format("2006-01-02 15:04:05") { + boundary = earlier + } + } + } + return boundary +} + +// OnlineHistoryWindowStart defines the shared query and retention calendar window. +func OnlineHistoryWindowStart(now time.Time, days int, loc *time.Location) time.Time { + // Calendar days follow the dashboard timezone, not the host timezone; + // DST days can contain 23 or 25 hours. Queries and pruning must agree. + if loc == nil { + loc = time.UTC + } + return now.In(loc).AddDate(0, 0, -days).UTC() +} diff --git a/model/online_history_test.go b/model/online_history_test.go new file mode 100644 index 00000000..5403ef98 --- /dev/null +++ b/model/online_history_test.go @@ -0,0 +1,220 @@ +package model + +import ( + "math/rand" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +var testZone = time.FixedZone("TEST", 8*3600) + +func ts(y int, mo time.Month, d, h, mi, s int) time.Time { + return time.Date(y, mo, d, h, mi, s, 0, testZone) +} + +func TestClipOnlineSessions(t *testing.T) { + windowStart := ts(2026, 10, 1, 0, 0, 0) + windowEnd := ts(2026, 10, 8, 0, 0, 0) + + t.Run("session fully inside window", func(t *testing.T) { + sessions := []OnlineSession{ + {ServerID: 1, StartedAt: ts(2026, 10, 2, 8, 0, 0), EndedAt: ts(2026, 10, 2, 9, 30, 0)}, + } + periods := ClipOnlineSessions(sessions, windowStart, windowEnd) + require.Len(t, periods, 1) + require.Equal(t, ts(2026, 10, 2, 8, 0, 0), periods[0].Start) + require.Equal(t, ts(2026, 10, 2, 9, 30, 0), periods[0].End) + require.False(t, periods[0].Ongoing) + }) + + t.Run("session started before window is clipped", func(t *testing.T) { + sessions := []OnlineSession{ + {ServerID: 1, StartedAt: ts(2026, 9, 28, 12, 0, 0), EndedAt: ts(2026, 10, 1, 5, 0, 0)}, + } + periods := ClipOnlineSessions(sessions, windowStart, windowEnd) + require.Len(t, periods, 1) + require.Equal(t, windowStart, periods[0].Start) + require.Equal(t, ts(2026, 10, 1, 5, 0, 0), periods[0].End) + }) + + t.Run("ongoing session ends at window end", func(t *testing.T) { + sessions := []OnlineSession{ + {ServerID: 1, StartedAt: ts(2026, 10, 7, 10, 0, 0), Ongoing: true}, + } + periods := ClipOnlineSessions(sessions, windowStart, windowEnd) + require.Len(t, periods, 1) + require.Equal(t, windowEnd, periods[0].End) + require.True(t, periods[0].Ongoing) + }) + + t.Run("zero ended_at treated as ongoing", func(t *testing.T) { + sessions := []OnlineSession{ + {ServerID: 1, StartedAt: ts(2026, 10, 7, 10, 0, 0)}, + } + periods := ClipOnlineSessions(sessions, windowStart, windowEnd) + require.Len(t, periods, 1) + require.True(t, periods[0].Ongoing) + require.Equal(t, windowEnd, periods[0].End) + }) + + t.Run("session outside window yields no period", func(t *testing.T) { + sessions := []OnlineSession{ + {ServerID: 1, StartedAt: ts(2026, 9, 1, 0, 0, 0), EndedAt: ts(2026, 9, 2, 0, 0, 0)}, + } + require.Empty(t, ClipOnlineSessions(sessions, windowStart, windowEnd)) + }) +} + +func TestDailyOnlineBreakdown(t *testing.T) { + loc := time.FixedZone("UTC+8", 8*3600) + windowStart := time.Date(2026, 10, 1, 10, 0, 0, 0, loc) + windowEnd := time.Date(2026, 10, 3, 14, 0, 0, 0, loc) + + periods := []OnlinePeriod{ + // 跨午夜：2 日 23:00 ~ 3 日 01:00 + {Start: time.Date(2026, 10, 2, 23, 0, 0, 0, loc), End: time.Date(2026, 10, 3, 1, 0, 0, 0, loc)}, + // 窗口前就开始的会话，裁剪后从窗口起点开始 + {Start: windowStart, End: time.Date(2026, 10, 1, 12, 0, 0, 0, loc)}, + } + + daily := DailyOnlineBreakdown(periods, windowStart, windowEnd, loc) + require.Len(t, daily, 3) + require.Equal(t, "2026-10-01", daily[0].Date) + require.Equal(t, "2026-10-02", daily[1].Date) + require.Equal(t, "2026-10-03", daily[2].Date) + + // 1 日：12:00-10:00 = 2h；2 日：23:00-24:00 = 1h；3 日：0:00-1:00 + 窗口外无 = 1h + require.Equal(t, uint64(7200), daily[0].OnlineSeconds) + require.Equal(t, uint64(3600), daily[1].OnlineSeconds) + require.Equal(t, uint64(3600), daily[2].OnlineSeconds) + + // 每日合计即总时长（口径：每日累计原始时长后统一取整） + var sum uint64 + for _, d := range daily { + sum += d.OnlineSeconds + } + require.Equal(t, uint64(7200+3600+3600), sum) +} + +// 评审回归：取整必须按日累计原始时长后统一转换。逐段截断再累加会让亚秒 +// 会话系统性偏小——同一天两个 1.5s 的会话应得 3s，而不是 1+1=2s +func TestDailyOnlineBreakdownSubSecond(t *testing.T) { + loc := testZone + windowStart := ts(2026, 10, 1, 0, 0, 0) + windowEnd := ts(2026, 10, 2, 0, 0, 0) + periods := []OnlinePeriod{ + {Start: windowStart.Add(100 * time.Millisecond), End: windowStart.Add(1600 * time.Millisecond)}, + {Start: windowStart.Add(2 * time.Second), End: windowStart.Add(3500 * time.Millisecond)}, + } + daily := DailyOnlineBreakdown(periods, windowStart, windowEnd, loc) + require.Len(t, daily, 1) + require.Equal(t, uint64(3), daily[0].OnlineSeconds) +} + +// 跨日时段按日界切分不损失精度：各日分别累计原始时长后各自取整 +func TestDailyOnlineBreakdownCrossDayFraction(t *testing.T) { + loc := testZone + windowStart := ts(2026, 10, 1, 0, 0, 0) + windowEnd := ts(2026, 10, 3, 0, 0, 0) + periods := []OnlinePeriod{{ + Start: time.Date(2026, 10, 1, 23, 59, 59, 250000000, loc), + End: time.Date(2026, 10, 2, 0, 0, 2, 750000000, loc), + }} + daily := DailyOnlineBreakdown(periods, windowStart, windowEnd, loc) + require.Len(t, daily, 2) + require.Equal(t, uint64(0), daily[0].OnlineSeconds) // 0.75s → 0 + require.Equal(t, uint64(2), daily[1].OnlineSeconds) // 2.75s → 2 +} + +func TestDailyOnlineBreakdownZeroOnline(t *testing.T) { + loc := time.FixedZone("UTC+8", 8*3600) + windowStart := time.Date(2026, 10, 1, 0, 0, 0, 0, loc) + windowEnd := time.Date(2026, 10, 4, 0, 0, 0, 0, loc) + daily := DailyOnlineBreakdown(nil, windowStart, windowEnd, loc) + require.Len(t, daily, 3) + for _, d := range daily { + require.Zero(t, d.OnlineSeconds) + } +} + +func TestOnlineStatsDaysBounds(t *testing.T) { + require.Equal(t, 7, DefaultOnlineStatsDays) + require.Equal(t, 7, MinOnlineStatsDays) + require.Equal(t, 3650, MaxOnlineStatsDays) +} + +// F1 回归保护：指针推进实现必须与逐日朴素实现对任意不相交时段集合等价 +// （单服务器会话互不重叠的约束一致），长窗口大量时段下不再随时段数平方放大。 +func TestDailyOnlineBreakdownEquivalence(t *testing.T) { + loc := testZone + windowStart := ts(2026, 1, 1, 0, 0, 0) + windowEnd := windowStart.AddDate(0, 0, 90) + + rng := rand.New(rand.NewSource(42)) + var periods []OnlinePeriod + cursor := windowStart + for cursor.Before(windowEnd) { + // 起止带亚秒抖动：让逐段截断与按日累计两种口径可区分 + start := cursor.Add(time.Duration(rng.Intn(4*3600))*time.Second + time.Duration(rng.Int63n(int64(time.Second)))) + end := start.Add(time.Duration(1+rng.Intn(6*3600))*time.Second + time.Duration(rng.Int63n(int64(time.Second)))) + if end.After(windowEnd) { + end = windowEnd + } + periods = append(periods, OnlinePeriod{Start: start, End: end}) + cursor = end.Add(time.Duration(1+rng.Intn(3600)) * time.Second) + } + + days := DailyOnlineBreakdown(periods, windowStart, windowEnd, loc) + + // 参考实现与生产同一口径：每日累计原始时长、汇总时向下取整一次 + reference := map[string]time.Duration{} + for _, p := range periods { + for d := windowStart; d.Before(windowEnd); d = d.AddDate(0, 0, 1) { + overlapStart := maxTime(p.Start, d) + overlapEnd := minTime(p.End, d.AddDate(0, 0, 1)) + if overlapStart.Before(overlapEnd) { + reference[d.Format("2006-01-02")] += overlapEnd.Sub(overlapStart) + } + } + } + require.Len(t, days, 90) + for _, day := range days { + require.Equal(t, uint64(reference[day.Date].Seconds()), day.OnlineSeconds, "date %s", day.Date) + } +} + +func TestDailyOnlineBreakdownMidnightDSTTransition(t *testing.T) { + loc, err := time.LoadLocation("America/Sao_Paulo") + require.NoError(t, err) + start := time.Date(2018, 11, 3, 0, 0, 0, 0, loc) + end := time.Date(2018, 11, 6, 0, 0, 0, 0, loc) + days := DailyOnlineBreakdown([]OnlinePeriod{{Start: start, End: end}}, start, end, loc) + require.Equal(t, []DailyOnlineStat{{Date: "2018-11-03", OnlineSeconds: 86400}, {Date: "2018-11-04", OnlineSeconds: 82800}, {Date: "2018-11-05", OnlineSeconds: 86400}}, days) +} + +func TestDailyOnlineBreakdownRepeatedMidnight(t *testing.T) { + loc, err := time.LoadLocation("Asia/Amman") + require.NoError(t, err) + start := time.Date(2021, 10, 28, 21, 0, 0, 0, time.UTC) + end := time.Date(2021, 10, 29, 22, 0, 0, 0, time.UTC) + days := DailyOnlineBreakdown([]OnlinePeriod{{Start: start, End: end}}, start, end, loc) + require.Equal(t, []DailyOnlineStat{{Date: "2021-10-29", OnlineSeconds: 90000}}, days) +} + +func TestOnlineHistoryWindowUsesDashboardCalendar(t *testing.T) { + loc, err := time.LoadLocation("America/New_York") + require.NoError(t, err) + for _, test := range []struct { + month time.Month + day, hours int + }{{time.March, 9, 23}, {time.November, 2, 25}} { + end := time.Date(2026, test.month, test.day, 0, 30, 0, 0, loc) + start := OnlineHistoryWindowStart(end.UTC(), 1, loc) + require.True(t, start.Equal(time.Date(2026, test.month, test.day-1, 0, 30, 0, 0, loc))) + require.Equal(t, time.Duration(test.hours)*time.Hour, end.Sub(start)) + require.Len(t, DailyOnlineBreakdown(nil, start, end, loc), 2) + require.True(t, OnlineHistoryWindowStart(end.UTC(), 7, loc).Equal(end.In(loc).AddDate(0, 0, -7))) + } +} diff --git a/model/rule.go b/model/rule.go index e069eef5..1155ce71 100644 --- a/model/rule.go +++ b/model/rule.go @@ -1,6 +1,8 @@ package model import ( + "log" + "math" "slices" "strings" "time" @@ -15,12 +17,23 @@ const ( RuleCoverIgnoreAll ) +// MaxAlertRuleDuration is the largest duration that can be converted to int +// on every architecture supported by Go. Alert rules are persisted as uint64, +// but their sampling windows use int indexes; keeping the public bound at +// MaxInt32 prevents a crafted value from wrapping during that conversion. +const MaxAlertRuleDuration uint64 = 1<<31 - 1 + +// MaxAlertRuleCycleInterval also leaves room for the largest integer +// multiplication performed by the calendar helpers (weeks become 7*interval) +// on 32-bit targets. +const MaxAlertRuleCycleInterval uint64 = (1<<31 - 1) / 7 + type NResult struct { N uint64 } type Rule struct { - // 指标类型，cpu、memory、swap、disk、net_in_speed、net_out_speed + // 指标类型，cpu、gpu/gpu_max、memory、swap、disk、net_in_speed、net_out_speed // net_all_speed、transfer_in、transfer_out、transfer_all、offline // transfer_in_cycle、transfer_out_cycle、transfer_all_cycle Type string `json:"type"` @@ -45,47 +58,147 @@ func percentage(used, total uint64) float64 { return float64(used) * 100 / float64(total) } -// Snapshot 未通过规则返回 false, 通过返回 true +// IsSupportedType reports whether the rule type has an evaluator. Keep this +// list in lockstep with Snapshot's switch so untrusted strings cannot enter the +// persisted alert pipeline and silently acquire fallback semantics. +func (u *Rule) IsSupportedType() bool { + if u == nil { + return false + } + switch u.Type { + case "cpu", "gpu", "gpu_max", "memory", "swap", "disk", + "net_in_speed", "net_out_speed", "net_all_speed", + "transfer_in", "transfer_out", "transfer_all", "offline", + "transfer_in_cycle", "transfer_out_cycle", "transfer_all_cycle", + "load1", "load5", "load15", "tcp_conn_count", "udp_conn_count", + "process_count", "temperature_max": + return true + default: + return false + } +} + +// DurationInt converts the persisted duration without allowing uint64-to-int +// truncation. Callers must handle ok=false as an invalid rule. +func (u *Rule) DurationInt() (duration int, ok bool) { + if u == nil || u.Duration > MaxAlertRuleDuration { + return 0, false + } + return int(u.Duration), true +} + +// HasSafeCycleConfiguration guards every value that the cycle evaluator later +// converts or dereferences. It intentionally does not reject an empty unit, +// which is the documented legacy spelling for hours. +func (u *Rule) HasSafeCycleConfiguration() bool { + return u != nil && u.IsTransferDurationRule() && u.CycleStart != nil && + !u.CycleStart.After(time.Now()) && u.CycleInterval > 0 && u.CycleInterval <= MaxAlertRuleCycleInterval && + u.HasSafeThresholds() && u.HasSafeCycleUnit() +} + +func (u *Rule) HasSafeCycleUnit() bool { + if u == nil { + return false + } + switch strings.ToLower(u.CycleUnit) { + case "", "hour", "day", "week", "month", "year": + return true + default: + return false + } +} + +func (u *Rule) HasSafeThresholds() bool { + if u == nil || math.IsNaN(u.Min) || math.IsNaN(u.Max) || math.IsInf(u.Min, 0) || math.IsInf(u.Max, 0) || u.Min < 0 || u.Max < 0 { + return false + } + return u.Min == 0 || u.Max == 0 || u.Min <= u.Max +} + +// Snapshot is retained for callers which only need a boolean result. The alert +// sentinel uses Evaluate so missing telemetry is not mistaken for a breach. func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) bool { + passed, _ := u.Evaluate(cycleTransferStats, server, db) + return passed +} + +// Evaluate returns known=false when a metric cannot be evaluated from the +// current report. Unknown must neither trigger nor resolve an incident. +func (u *Rule) Evaluate(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) (passed, known bool) { + if server == nil { + return true, false + } + return u.EvaluateRuntime(cycleTransferStats, server, server.RuntimeSnapshot(), db) +} + +// EvaluateRuntime evaluates all conditions against the same immutable Agent +// report. Callers evaluating a compound rule must share this snapshot. +func (u *Rule) EvaluateRuntime(cycleTransferStats *CycleTransferStats, server *Server, runtime RuntimeSnapshot, db *gorm.DB) (passed, known bool) { + if u == nil || server == nil || !u.IsSupportedType() { + return true, false + } + if u.IsTransferDurationRule() && (!u.HasSafeCycleConfiguration() || cycleTransferStats == nil) { + return true, false + } + // 监控全部但是排除了此服务器 if u.Cover == RuleCoverAll && u.Ignore[server.ID] { - return true + return true, true } // 忽略全部但是指定监控了此服务器 if u.Cover == RuleCoverIgnoreAll && !u.Ignore[server.ID] { - return true + return true, true } + evalNow := time.Now() + var cycleStart, cycleEnd time.Time + if u.IsTransferDurationRule() { + cycleStart, cycleEnd = u.transferDurationBounds(evalNow) + } // 循环区间流量检测 · 短期无需重复检测 - if u.IsTransferDurationRule() && u.NextTransferAt[server.ID].After(time.Now()) { - return u.LastCycleStatus[server.ID] + if u.IsTransferDurationRule() && u.NextTransferAt[server.ID].After(evalNow) { + return u.LastCycleStatus[server.ID], true } var src float64 - runtime := server.RuntimeSnapshot() + if u.IsOfflineRule() { + return !runtime.LastActive.IsZero() && time.Since(runtime.LastActive) <= 6*time.Second, true + } if runtime.State == nil { - return false + return true, false } state := runtime.State switch u.Type { case "cpu": src = float64(state.CPU) - case "gpu_max": + case "gpu", "gpu_max": + if len(state.GPU) == 0 { + return true, false + } src = slices.Max(state.GPU) case "memory": if runtime.Host == nil { - return false + return true, false + } + if runtime.Host.MemTotal == 0 { + return true, false } src = percentage(state.MemUsed, runtime.Host.MemTotal) case "swap": if runtime.Host == nil { - return false + return true, false + } + if runtime.Host.SwapTotal == 0 { + return true, false } src = percentage(state.SwapUsed, runtime.Host.SwapTotal) case "disk": if runtime.Host == nil { - return false + return true, false + } + if runtime.Host.DiskTotal == 0 { + return true, false } src = percentage(state.DiskUsed, runtime.Host.DiskTotal) case "net_in_speed": @@ -93,38 +206,50 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, case "net_out_speed": src = float64(state.NetOutSpeed) case "net_all_speed": - src = float64(state.NetOutSpeed + state.NetOutSpeed) + src = float64(state.NetInSpeed) + float64(state.NetOutSpeed) case "transfer_in": src = float64(state.NetInTransfer) case "transfer_out": src = float64(state.NetOutTransfer) case "transfer_all": src = float64(state.NetOutTransfer + state.NetInTransfer) - case "offline": - if runtime.LastActive.IsZero() { - src = 0 - } else { - src = float64(runtime.LastActive.Unix()) - } case "transfer_in_cycle": + if db == nil { + return true, false + } src = float64(utils.SubUintChecked(state.NetInTransfer, runtime.PrevTransferInSnapshot)) if u.CycleInterval != 0 { var res NResult - db.Model(&Transfer{}).Select("SUM(`in`) AS n").Where("datetime(`created_at`) >= datetime(?) AND server_id = ?", u.GetTransferDurationStart().UTC(), server.ID).Scan(&res) + if err := db.Model(&Transfer{}).Select("SUM(`in`) AS n").Where("datetime(`created_at`) >= datetime(?) AND datetime(`created_at`) < datetime(?) AND datetime(`created_at`) <= datetime(?) AND server_id = ?", cycleStart.UTC(), cycleEnd.UTC(), evalNow.UTC(), server.ID).Scan(&res).Error; err != nil { + log.Printf("NEZHA>> Alert cycle transfer query failed for rule %s server %d: %v", u.Type, server.ID, err) + return true, false + } src += float64(res.N) } case "transfer_out_cycle": + if db == nil { + return true, false + } src = float64(utils.SubUintChecked(state.NetOutTransfer, runtime.PrevTransferOutSnapshot)) if u.CycleInterval != 0 { var res NResult - db.Model(&Transfer{}).Select("SUM(`out`) AS n").Where("datetime(`created_at`) >= datetime(?) AND server_id = ?", u.GetTransferDurationStart().UTC(), server.ID).Scan(&res) + if err := db.Model(&Transfer{}).Select("SUM(`out`) AS n").Where("datetime(`created_at`) >= datetime(?) AND datetime(`created_at`) < datetime(?) AND datetime(`created_at`) <= datetime(?) AND server_id = ?", cycleStart.UTC(), cycleEnd.UTC(), evalNow.UTC(), server.ID).Scan(&res).Error; err != nil { + log.Printf("NEZHA>> Alert cycle transfer query failed for rule %s server %d: %v", u.Type, server.ID, err) + return true, false + } src += float64(res.N) } case "transfer_all_cycle": + if db == nil { + return true, false + } src = float64(utils.SubUintChecked(state.NetOutTransfer, runtime.PrevTransferOutSnapshot) + utils.SubUintChecked(state.NetInTransfer, runtime.PrevTransferInSnapshot)) if u.CycleInterval != 0 { var res NResult - db.Model(&Transfer{}).Select("SUM(`in`+`out`) AS n").Where("datetime(`created_at`) >= datetime(?) AND server_id = ?", u.GetTransferDurationStart().UTC(), server.ID).Scan(&res) + if err := db.Model(&Transfer{}).Select("SUM(`in`+`out`) AS n").Where("datetime(`created_at`) >= datetime(?) AND datetime(`created_at`) < datetime(?) AND datetime(`created_at`) <= datetime(?) AND server_id = ?", cycleStart.UTC(), cycleEnd.UTC(), evalNow.UTC(), server.ID).Scan(&res).Error; err != nil { + log.Printf("NEZHA>> Alert cycle transfer query failed for rule %s server %d: %v", u.Type, server.ID, err) + return true, false + } src += float64(res.N) } case "load1": @@ -141,26 +266,38 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, src = float64(state.ProcessCount) case "temperature_max": var temp []float64 - if state.Temperatures != nil { - for _, tempStat := range state.Temperatures { - if tempStat.Temperature != 0 { - temp = append(temp, tempStat.Temperature) - } + for _, tempStat := range state.Temperatures { + if tempStat.Temperature != 0 { + temp = append(temp, tempStat.Temperature) } - src = slices.Max(temp) } + if len(temp) == 0 { + return true, false + } + src = slices.Max(temp) + default: + return true, false } // 循环区间流量检测 · 更新下次需要检测时间 if u.IsTransferDurationRule() { - seconds := max(1800*((u.Max-src)/u.Max), 180) + seconds := float64(180) + if u.Max > 0 { + seconds = max(1800*((u.Max-src)/u.Max), 180) + } else if u.Min > 0 { + seconds = 1800 + } if u.NextTransferAt == nil { u.NextTransferAt = make(map[uint64]time.Time) } if u.LastCycleStatus == nil { u.LastCycleStatus = make(map[uint64]bool) } - u.NextTransferAt[server.ID] = time.Now().Add(time.Second * time.Duration(seconds)) + nextAt := evalNow.Add(time.Second * time.Duration(seconds)) + if nextAt.After(cycleEnd) { + nextAt = cycleEnd + } + u.NextTransferAt[server.ID] = nextAt if (u.Max > 0 && src > u.Max) || (u.Min > 0 && src < u.Min) { u.LastCycleStatus[server.ID] = false } else { @@ -172,105 +309,77 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, cycleTransferStats.Transfer[server.ID] = uint64(src) cycleTransferStats.NextUpdate[server.ID] = u.NextTransferAt[server.ID] // 自动更新周期流量展示起止时间 - cycleTransferStats.From = u.GetTransferDurationStart() - cycleTransferStats.To = u.GetTransferDurationEnd() + cycleTransferStats.From = cycleStart + cycleTransferStats.To = cycleEnd } - if u.Type == "offline" && float64(time.Now().Unix())-src > 6 { - return false - } else if (u.Max > 0 && src > u.Max) || (u.Min > 0 && src < u.Min) { - return false + if (u.Max > 0 && src > u.Max) || (u.Min > 0 && src < u.Min) { + return false, true } - return true + return true, true } // IsTransferDurationRule 判断该规则是否属于周期流量规则 属于则返回true func (u *Rule) IsTransferDurationRule() bool { - return strings.HasSuffix(u.Type, "_cycle") + if u == nil { + return false + } + switch u.Type { + case "transfer_in_cycle", "transfer_out_cycle", "transfer_all_cycle": + return true + default: + return false + } } func (u *Rule) IsOfflineRule() bool { - return u.Type == "offline" + return u != nil && u.Type == "offline" +} + +// transferDurationBounds computes both edges from the same clock reading. +// Separate time.Now calls can otherwise straddle a cycle boundary. +func (u *Rule) transferDurationBounds(now time.Time) (time.Time, time.Time) { + if !u.HasSafeCycleConfiguration() { + return time.Time{}, time.Time{} + } + start := *u.CycleStart + var end time.Time + switch strings.ToLower(u.CycleUnit) { + case "year": + end = start.AddDate(int(u.CycleInterval), 0, 0) + for !now.Before(end) { + start, end = end, end.AddDate(int(u.CycleInterval), 0, 0) + } + case "month": + end = start.AddDate(0, int(u.CycleInterval), 0) + for !now.Before(end) { + start, end = end, end.AddDate(0, int(u.CycleInterval), 0) + } + case "week": + end = start.AddDate(0, 0, 7*int(u.CycleInterval)) + for !now.Before(end) { + start, end = end, end.AddDate(0, 0, 7*int(u.CycleInterval)) + } + case "day": + end = start.AddDate(0, 0, int(u.CycleInterval)) + for !now.Before(end) { + start, end = end, end.AddDate(0, 0, int(u.CycleInterval)) + } + default: // empty is the legacy spelling for hour + interval := 3600 * int64(u.CycleInterval) + start = time.Unix(u.CycleStart.Unix()+(now.Unix()-u.CycleStart.Unix())/interval*interval, 0) + end = time.Unix(start.Unix()+interval, 0) + } + return start, end } -// GetTransferDurationStart 获取周期流量的起始时间 func (u *Rule) GetTransferDurationStart() time.Time { - // Accept uppercase and lowercase - unit := strings.ToLower(u.CycleUnit) - startTime := *u.CycleStart - var nextTime time.Time - switch unit { - case "year": - nextTime = startTime.AddDate(int(u.CycleInterval), 0, 0) - for time.Now().After(nextTime) { - startTime = nextTime - nextTime = nextTime.AddDate(int(u.CycleInterval), 0, 0) - } - case "month": - nextTime = startTime.AddDate(0, int(u.CycleInterval), 0) - for time.Now().After(nextTime) { - startTime = nextTime - nextTime = nextTime.AddDate(0, int(u.CycleInterval), 0) - } - case "week": - nextTime = startTime.AddDate(0, 0, 7*int(u.CycleInterval)) - for time.Now().After(nextTime) { - startTime = nextTime - nextTime = nextTime.AddDate(0, 0, 7*int(u.CycleInterval)) - } - case "day": - nextTime = startTime.AddDate(0, 0, int(u.CycleInterval)) - for time.Now().After(nextTime) { - startTime = nextTime - nextTime = nextTime.AddDate(0, 0, int(u.CycleInterval)) - } - default: - // For hour unit or not set. - interval := 3600 * int64(u.CycleInterval) - startTime = time.Unix(u.CycleStart.Unix()+(time.Now().Unix()-u.CycleStart.Unix())/interval*interval, 0) - } - - return startTime + start, _ := u.transferDurationBounds(time.Now()) + return start } -// GetTransferDurationEnd 获取周期流量结束时间 func (u *Rule) GetTransferDurationEnd() time.Time { - // Accept uppercase and lowercase - unit := strings.ToLower(u.CycleUnit) - startTime := *u.CycleStart - var nextTime time.Time - switch unit { - case "year": - nextTime = startTime.AddDate(int(u.CycleInterval), 0, 0) - for time.Now().After(nextTime) { - startTime = nextTime - nextTime = nextTime.AddDate(int(u.CycleInterval), 0, 0) - } - case "month": - nextTime = startTime.AddDate(0, int(u.CycleInterval), 0) - for time.Now().After(nextTime) { - startTime = nextTime - nextTime = nextTime.AddDate(0, int(u.CycleInterval), 0) - } - case "week": - nextTime = startTime.AddDate(0, 0, 7*int(u.CycleInterval)) - for time.Now().After(nextTime) { - startTime = nextTime - nextTime = nextTime.AddDate(0, 0, 7*int(u.CycleInterval)) - } - case "day": - nextTime = startTime.AddDate(0, 0, int(u.CycleInterval)) - for time.Now().After(nextTime) { - startTime = nextTime - nextTime = nextTime.AddDate(0, 0, int(u.CycleInterval)) - } - default: - // For hour unit or not set. - interval := 3600 * int64(u.CycleInterval) - startTime = time.Unix(u.CycleStart.Unix()+(time.Now().Unix()-u.CycleStart.Unix())/interval*interval, 0) - nextTime = time.Unix(startTime.Unix()+interval, 0) - } - - return nextTime + _, end := u.transferDurationBounds(time.Now()) + return end } diff --git a/model/server.go b/model/server.go index b3777afb..7f67af06 100644 --- a/model/server.go +++ b/model/server.go @@ -16,6 +16,7 @@ import ( ) var runtimeHolderInitMu sync.Mutex +var nextServerReportSequence atomic.Uint64 type Server struct { Common @@ -75,15 +76,16 @@ type taskStreamHolder struct { } type serverRuntimeHolder struct { - mu sync.Mutex - canonical *Server - stream pb.NezhaService_ReportSystemStateServer - generation uint64 - state *HostState - host *Host - lastActive time.Time - prevIn uint64 - prevOut uint64 + mu sync.Mutex + canonical *Server + stream pb.NezhaService_ReportSystemStateServer + generation uint64 + state *HostState + host *Host + lastActive time.Time + reportSequence uint64 + prevIn uint64 + prevOut uint64 } type StateStreamLease struct { @@ -206,14 +208,21 @@ func (lease StateStreamLease) updateState(receiver *Server, state *HostState, la return false } } + lease.holder.reportSequence = nextServerReportSequence.Add(1) return true } func (lease StateStreamLease) Clear() bool { - return lease.clear(nil) + return lease.clear(nil, nil) } -func (lease StateStreamLease) clear(receiver *Server) bool { +// ClearWithSideEffect serializes disconnect persistence with replacement streams. +// The callback must not acquire the runtime holder again. +func (lease StateStreamLease) ClearWithSideEffect(sideEffect func()) bool { + return lease.clear(nil, sideEffect) +} + +func (lease StateStreamLease) clear(receiver *Server, sideEffect func()) bool { if lease.holder == nil { return false } @@ -225,6 +234,9 @@ func (lease StateStreamLease) clear(receiver *Server) bool { lease.holder.stream = nil lease.holder.lastActive = time.Time{} lease.holder.canonical.LastActive = time.Time{} + if sideEffect != nil { + sideEffect() + } return true } @@ -364,7 +376,7 @@ func (s *Server) UpdateStateIfCurrentWithSideEffect(lease StateStreamLease, stat } func (s *Server) ClearStateStreamIfCurrent(lease StateStreamLease) bool { - return lease.clear(s) + return lease.clear(s, nil) } // RuntimeSnapshot is a deep copy of the mutable runtime state. @@ -372,6 +384,7 @@ type RuntimeSnapshot struct { State *HostState Host *Host LastActive time.Time + ReportSequence uint64 PrevTransferInSnapshot uint64 PrevTransferOutSnapshot uint64 } @@ -398,7 +411,7 @@ func (s *Server) RuntimeSnapshot() RuntimeSnapshot { holder.host = cloneHost(s.Host) } } - return RuntimeSnapshot{State: cloneHostState(holder.state), Host: cloneHost(holder.host), LastActive: holder.lastActive, PrevTransferInSnapshot: holder.prevIn, PrevTransferOutSnapshot: holder.prevOut} + return RuntimeSnapshot{State: cloneHostState(holder.state), Host: cloneHost(holder.host), LastActive: holder.lastActive, ReportSequence: holder.reportSequence, PrevTransferInSnapshot: holder.prevIn, PrevTransferOutSnapshot: holder.prevOut} } func (s *Server) SetTransferSnapshots(inbound, outbound uint64) bool { diff --git a/model/server_runtime_ownership_test.go b/model/server_runtime_ownership_test.go index 970ac7df..b29f5146 100644 --- a/model/server_runtime_ownership_test.go +++ b/model/server_runtime_ownership_test.go @@ -367,3 +367,30 @@ func TestServerRuntimeOwnership_failedRestartAllowsHourlyRecordThenRetry(t *test require.Equal(t, uint64(0), result.Transfer.In) require.Equal(t, uint64(0), result.Transfer.Out) } + +func TestStateStreamDisconnectSideEffectSerializesReplacement(t *testing.T) { + server := &Server{} + old := server.AttachStateStream(runtimeOwnershipStream{}) + entered := make(chan struct{}) + release := make(chan struct{}) + cleared := make(chan bool, 1) + go func() { cleared <- old.ClearWithSideEffect(func() { close(entered); <-release }) }() + <-entered + attached := make(chan StateStreamLease, 1) + go func() { attached <- server.AttachStateStream(runtimeOwnershipStream{}) }() + select { + case <-attached: + close(release) + t.Fatal("replacement attached before disconnect persistence finished") + case <-time.After(20 * time.Millisecond): + } + close(release) + require.True(t, <-cleared) + next := <-attached + require.True(t, next.UpdateState(&HostState{}, time.Now())) + called := false + require.False(t, old.ClearWithSideEffect(func() { called = true })) + require.False(t, called) + require.False(t, old.UpdateStateWithSideEffect(&HostState{}, time.Now(), func() error { called = true; return nil })) + require.False(t, called) +} diff --git a/model/setting_api.go b/model/setting_api.go index c85a28ef..02d7b7c4 100644 --- a/model/setting_api.go +++ b/model/setting_api.go @@ -35,6 +35,8 @@ type SettingForm struct { AdminEmail string `json:"admin_email,omitempty" validate:"optional"` DomainExpiryNotificationDays string `json:"domain_expiry_notification_days,omitempty" validate:"optional"` ServerExpiryNotificationDays string `json:"server_expiry_notification_days,omitempty" validate:"optional"` + AllowJWTIPChange *bool `json:"allow_jwt_ip_change,omitempty" validate:"optional"` + OnlineStatsDays *int `json:"online_stats_days,omitempty" validate:"optional"` } type Setting struct { diff --git a/pkg/ddns/ddns.go b/pkg/ddns/ddns.go index 62d68b07..b3cc7b4a 100644 --- a/pkg/ddns/ddns.go +++ b/pkg/ddns/ddns.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "net/netip" + "strings" "time" "github.com/libdns/libdns" @@ -21,9 +22,6 @@ const ( ) type Provider struct { - prefix string - zone string - DDNSProfile *model.DDNSProfile IPAddrs *model.IP Setter libdns.RecordSetter @@ -34,61 +32,81 @@ func (provider *Provider) GetProfileID() uint64 { } func (provider *Provider) UpdateDomain(ctx context.Context, overrideDomains ...string) { - for _, domain := range utils.IfOr(len(overrideDomains) > 0, overrideDomains, provider.DDNSProfile.Domains) { - for retries := range int(provider.DDNSProfile.MaxRetries) { - log.Printf("NEZHA>> Updating DNS Record of domain %s: %d/%d", domain, retries+1, provider.DDNSProfile.MaxRetries) - if err := provider.updateDomain(ctx, domain); err != nil { - log.Printf("NEZHA>> Failed to update DNS record of domain %s: %v", domain, err) - } else { - log.Printf("NEZHA>> Update DNS record of domain %s succeeded", domain) + domains := utils.IfOr(len(overrideDomains) > 0, overrideDomains, provider.DDNSProfile.Domains) + maxRetries := int(provider.DDNSProfile.MaxRetries) + if maxRetries <= 0 { + maxRetries = 1 + } + + for _, domain := range domains { + var prefix, zone string + var soaErr error + + for retries := 0; retries < maxRetries; retries++ { + prefix, zone, soaErr = provider.splitDomainSOA(ctx, domain) + if soaErr == nil { break } + log.Printf("NEZHA>> Failed to split domain SOA for %s (attempt %d/%d): %v", domain, retries+1, maxRetries, soaErr) + } + + if soaErr != nil { + log.Printf("NEZHA>> Failed to split domain SOA for %s after %d retries, skipping domain", domain, maxRetries) + continue + } + + // 独立处理 IPv4 更新或删除 + if provider.DDNSProfile.EnableIPv4 != nil && *provider.DDNSProfile.EnableIPv4 { + for retries := 0; retries < maxRetries; retries++ { + log.Printf("NEZHA>> Updating IPv4 record of domain %s: %d/%d", domain, retries+1, maxRetries) + var ipv4Err error + if provider.IPAddrs.IPv4Addr == "" { + ipv4Err = provider.deleteDomainRecord(ctx, prefix, zone, "A") + } else { + ipv4Err = provider.addDomainRecord(ctx, prefix, zone, "A", provider.IPAddrs.IPv4Addr) + } + + if ipv4Err != nil { + log.Printf("NEZHA>> Failed to update IPv4 record of domain %s: %v", domain, ipv4Err) + } else { + log.Printf("NEZHA>> Update IPv4 record of domain %s succeeded", domain) + break + } + } + } + + // 独立处理 IPv6 更新或删除 + if provider.DDNSProfile.EnableIPv6 != nil && *provider.DDNSProfile.EnableIPv6 { + for retries := 0; retries < maxRetries; retries++ { + log.Printf("NEZHA>> Updating IPv6 record of domain %s: %d/%d", domain, retries+1, maxRetries) + var ipv6Err error + if provider.IPAddrs.IPv6Addr == "" { + ipv6Err = provider.deleteDomainRecord(ctx, prefix, zone, "AAAA") + } else { + ipv6Err = provider.addDomainRecord(ctx, prefix, zone, "AAAA", provider.IPAddrs.IPv6Addr) + } + + if ipv6Err != nil { + log.Printf("NEZHA>> Failed to update IPv6 record of domain %s: %v", domain, ipv6Err) + } else { + log.Printf("NEZHA>> Update IPv6 record of domain %s succeeded", domain) + break + } + } } } } -func (provider *Provider) updateDomain(ctx context.Context, domain string) error { - var err error - provider.prefix, provider.zone, err = provider.splitDomainSOA(ctx, domain) - if err != nil { - return err - } - - // 独立处理 IPv4 更新 - if *provider.DDNSProfile.EnableIPv4 { - if provider.IPAddrs.IPv4Addr == "" { - log.Printf("NEZHA>> Skip IPv4 update for domain %s: IPv4 address is empty", domain) - } else { - if err = provider.addDomainRecord(ctx, "A", provider.IPAddrs.IPv4Addr); err != nil { - return err - } - } - } - - // 独立处理 IPv6 更新 - if *provider.DDNSProfile.EnableIPv6 { - if provider.IPAddrs.IPv6Addr == "" { - log.Printf("NEZHA>> Skip IPv6 update for domain %s: IPv6 address is empty", domain) - } else { - if err = provider.addDomainRecord(ctx, "AAAA", provider.IPAddrs.IPv6Addr); err != nil { - return err - } - } - } - - return nil -} - -func (provider *Provider) addDomainRecord(ctx context.Context, recType, addr string) error { +func (provider *Provider) addDomainRecord(ctx context.Context, prefix, zone, recType, addr string) error { netipAddr, err := netip.ParseAddr(addr) if err != nil { return fmt.Errorf("parse error: %v", err) } - _, err = provider.Setter.SetRecords(ctx, provider.zone, + _, err = provider.Setter.SetRecords(ctx, zone, []libdns.Record{ libdns.Address{ - Name: provider.prefix, + Name: prefix, IP: netipAddr, TTL: time.Minute, }, @@ -96,6 +114,28 @@ func (provider *Provider) addDomainRecord(ctx context.Context, recType, addr str return err } +func (provider *Provider) deleteDomainRecord(ctx context.Context, prefix, zone, recType string) error { + deleter, okDeleter := provider.Setter.(libdns.RecordDeleter) + if !okDeleter { + log.Printf("NEZHA>> DNS provider does not support RecordDeleter, safely skipping deletion for %s", recType) + return nil + } + + targetRecType := strings.ToUpper(recType) + _, err := deleter.DeleteRecords(ctx, zone, []libdns.Record{ + libdns.RR{ + Name: prefix, + Type: targetRecType, + }, + }) + if err != nil { + return fmt.Errorf("deleter.DeleteRecords failed: %w", err) + } + + log.Printf("NEZHA>> Successfully deleted %s record for %s.%s", recType, prefix, zone) + return nil +} + func (provider *Provider) splitDomainSOA(ctx context.Context, domain string) (prefix string, zone string, err error) { c := &dns.Client{Timeout: dnsTimeOut} @@ -103,7 +143,6 @@ func (provider *Provider) splitDomainSOA(ctx context.Context, domain string) (pr indexes := dns.Split(domain) servers := utils.DNSServers - customDNSServers, _ := ctx.Value(DNSServerKey{}).([]string) if len(customDNSServers) > 0 { servers = customDNSServers @@ -115,7 +154,7 @@ func (provider *Provider) splitDomainSOA(ctx context.Context, domain string) (pr m.SetQuestion(domain[idx:], dns.TypeSOA) r, _, err := c.Exchange(&m, server) - if err != nil { + if err != nil || r == nil { continue } diff --git a/pkg/ddns/ddns_test.go b/pkg/ddns/ddns_test.go index 552fefd7..4058e751 100644 --- a/pkg/ddns/ddns_test.go +++ b/pkg/ddns/ddns_test.go @@ -2,7 +2,15 @@ package ddns import ( "context" + "net" + "strings" + "sync/atomic" "testing" + + "github.com/libdns/libdns" + "github.com/miekg/dns" + + "github.com/nezhahq/nezha/model" ) type testSt struct { @@ -12,6 +20,50 @@ type testSt struct { } func TestSplitDomainSOA(t *testing.T) { + handler := dns.NewServeMux() + handler.HandleFunc(".", func(w dns.ResponseWriter, r *dns.Msg) { + m := new(dns.Msg) + m.SetReply(r) + m.Authoritative = true + if len(r.Question) > 0 { + qname := r.Question[0].Name + var zoneName string + if strings.HasSuffix(qname, "example.co.uk.") { + zoneName = "example.co.uk." + } else if strings.HasSuffix(qname, "example.org.") { + zoneName = "example.org." + } else if strings.HasSuffix(qname, "example.com.") { + zoneName = "example.com." + } + + if zoneName != "" { + soa := &dns.SOA{ + Hdr: dns.RR_Header{Name: zoneName, Rrtype: dns.TypeSOA, Class: dns.ClassINET, Ttl: 300}, + Ns: "ns.example.com.", + Mbox: "admin.example.com.", + Serial: 1, + } + m.Answer = append(m.Answer, soa) + m.Ns = append(m.Ns, soa) + } + } + _ = w.WriteMsg(m) + }) + + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + defer pc.Close() + + server := &dns.Server{PacketConn: pc, Net: "udp", Handler: handler} + go func() { + _ = server.ActivateAndServe() + }() + defer server.Shutdown() + + addr := pc.LocalAddr().String() + cases := []testSt{ { domain: "www.example.com", @@ -30,13 +82,12 @@ func TestSplitDomainSOA(t *testing.T) { }, } - ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{"1.1.1.1:53"}) + ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{addr}) provider := &Provider{} for _, c := range cases { prefix, zone, err := provider.splitDomainSOA(ctx, c.domain) if err != nil { - t.Skipf("Skipping live DNS test due to network: %v", err) - return + t.Fatalf("Error for domain %s: %s", c.domain, err) } if prefix != c.prefix { t.Fatalf("Expected prefix %s, but got %s", c.prefix, prefix) @@ -46,3 +97,191 @@ func TestSplitDomainSOA(t *testing.T) { } } } + +type MockSetter struct { + Records []libdns.Record + SetCalled bool + DelCalled bool + DeletedRecords []libdns.Record +} + +func (m *MockSetter) SetRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) { + m.SetCalled = true + m.Records = append(m.Records, records...) + return records, nil +} + +func (m *MockSetter) DeleteRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) { + m.DelCalled = true + m.DeletedRecords = append(m.DeletedRecords, records...) + return records, nil +} + +type MockUnsupportedSetter struct{} + +func (m *MockUnsupportedSetter) SetRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) { + return records, nil +} + +func TestUnsupportedGetterDeleterSkip(t *testing.T) { + provider := &Provider{ + DDNSProfile: &model.DDNSProfile{MaxRetries: 1}, + IPAddrs: &model.IP{IPv4Addr: ""}, + Setter: &MockUnsupportedSetter{}, + } + + err := provider.deleteDomainRecord(context.Background(), "sub", "example.com", "A") + if err != nil { + t.Fatalf("expected no error for unsupported deleter skip, got %v", err) + } +} + +func TestIndependentDualStackProcessing(t *testing.T) { + enableIPv4 := true + enableIPv6 := true + mockSetter := &MockSetter{} + + provider := &Provider{ + DDNSProfile: &model.DDNSProfile{ + MaxRetries: 1, + Domains: []string{"example.com"}, + EnableIPv4: &enableIPv4, + EnableIPv6: &enableIPv6, + }, + IPAddrs: &model.IP{ + IPv4Addr: "1.1.1.1", + IPv6Addr: "2001:db8::1", + }, + Setter: mockSetter, + } + + handler := dns.NewServeMux() + handler.HandleFunc(".", func(w dns.ResponseWriter, r *dns.Msg) { + m := new(dns.Msg) + m.SetReply(r) + m.Authoritative = true + if len(r.Question) > 0 { + soa := &dns.SOA{ + Hdr: dns.RR_Header{Name: r.Question[0].Name, Rrtype: dns.TypeSOA, Class: dns.ClassINET, Ttl: 300}, + Ns: "ns.example.com.", + Mbox: "admin.example.com.", + Serial: 1, + } + m.Answer = append(m.Answer, soa) + m.Ns = append(m.Ns, soa) + } + _ = w.WriteMsg(m) + }) + + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + defer pc.Close() + + server := &dns.Server{PacketConn: pc, Net: "udp", Handler: handler} + go func() { + _ = server.ActivateAndServe() + }() + defer server.Shutdown() + + ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{pc.LocalAddr().String()}) + provider.UpdateDomain(ctx) + + if !mockSetter.SetCalled { + t.Fatalf("expected SetRecords to have been called for dual stack updates") + } + + foundIPv4 := false + foundIPv6 := false + for _, rec := range mockSetter.Records { + if addrRec, ok := rec.(libdns.Address); ok { + if addrRec.IP.String() == "1.1.1.1" { + foundIPv4 = true + } + if addrRec.IP.String() == "2001:db8::1" { + foundIPv6 = true + } + } + } + + if !foundIPv4 || !foundIPv6 { + t.Fatalf("expected both IPv4 and IPv6 records to be processed independently, got records: %v", mockSetter.Records) + } +} + +func TestTransientSOAFailureAndRetry(t *testing.T) { + var attempts int64 + handler := dns.NewServeMux() + handler.HandleFunc(".", func(w dns.ResponseWriter, r *dns.Msg) { + atomic.AddInt64(&attempts, 1) + m := new(dns.Msg) + m.SetReply(r) + _ = w.WriteMsg(m) + }) + + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + defer pc.Close() + + server := &dns.Server{PacketConn: pc, Net: "udp", Handler: handler} + go func() { + _ = server.ActivateAndServe() + }() + defer server.Shutdown() + + var maxRetries uint64 = 3 + provider := &Provider{ + DDNSProfile: &model.DDNSProfile{ + MaxRetries: maxRetries, + Domains: []string{"example.com"}, + EnableIPv4: &[]bool{true}[0], + EnableIPv6: &[]bool{false}[0], + }, + IPAddrs: &model.IP{IPv4Addr: "1.1.1.1"}, + Setter: &MockSetter{}, + } + + ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{pc.LocalAddr().String()}) + provider.UpdateDomain(ctx) + + actualAttempts := atomic.LoadInt64(&attempts) + expectedAttempts := int64(maxRetries) * 2 + if actualAttempts != expectedAttempts { + t.Fatalf("expected exact attempt count of %d, got %d", expectedAttempts, actualAttempts) + } +} + +func TestGenericRecordDeleterContract(t *testing.T) { + mockSetter := &MockSetter{} + provider := &Provider{ + DDNSProfile: &model.DDNSProfile{MaxRetries: 1}, + IPAddrs: &model.IP{IPv4Addr: ""}, + Setter: mockSetter, + } + + ctx := context.Background() + err := provider.deleteDomainRecord(ctx, "sub", "example.com", "A") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !mockSetter.DelCalled { + t.Fatalf("expected DeleteRecords to be called") + } + + if len(mockSetter.DeletedRecords) != 1 { + t.Fatalf("expected 1 record sent for deletion, got %d", len(mockSetter.DeletedRecords)) + } + + rr, ok := mockSetter.DeletedRecords[0].(libdns.RR) + if !ok { + t.Fatalf("expected deletion payload to be libdns.RR") + } + + if rr.Name != "sub" || rr.Type != "A" { + t.Fatalf("unexpected RR fields: Name=%s, Type=%s", rr.Name, rr.Type) + } +} diff --git a/pkg/tsdb/disk_guard_test.go b/pkg/tsdb/disk_guard_test.go new file mode 100644 index 00000000..2fa2109a --- /dev/null +++ b/pkg/tsdb/disk_guard_test.go @@ -0,0 +1,124 @@ +package tsdb + +import ( + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type testReadOnlyChecker struct { + value atomic.Bool +} + +func (c *testReadOnlyChecker) IsReadOnly() bool { + return c.value.Load() +} + +func newDiskGuardTestDB(t *testing.T) *TSDB { + t.Helper() + db, err := Open(&Config{ + DataPath: filepath.Join(t.TempDir(), "tsdb"), + RetentionDays: 1, + MinFreeDiskSpaceGB: 1, + DedupInterval: time.Second, + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + return db +} + +func bufferedRows(w *bufferedWriter) int { + w.mu.Lock() + defer w.mu.Unlock() + return len(w.buffer) +} + +func TestReadOnlyStorageDropsWritesAndKeepsQueriesAvailable(t *testing.T) { + db := newDiskGuardTestDB(t) + checker := &testReadOnlyChecker{} + db.readOnly = checker + originalAddRows := db.addRowsFn + var addRowsCalls atomic.Int32 + db.addRowsFn = func(rows []storage.MetricRow, precisionBits uint8) { + addRowsCalls.Add(1) + originalAddRows(rows, precisionBits) + } + + firstTimestamp := time.Now().Add(-time.Minute) + require.NoError(t, db.WriteServerMetrics(&ServerMetrics{ + ServerID: 1, + Timestamp: firstTimestamp, + CPU: 10, + })) + db.Flush() + require.Equal(t, int32(1), addRowsCalls.Load()) + + before, err := db.QueryServerMetrics(1, MetricServerCPU, Period1Day) + require.NoError(t, err) + require.NotEmpty(t, before) + + checker.value.Store(true) + require.NoError(t, db.WriteServerMetrics(&ServerMetrics{ + ServerID: 1, + Timestamp: firstTimestamp.Add(2 * time.Second), + CPU: 20, + })) + assert.True(t, db.WritesPaused()) + assert.Zero(t, bufferedRows(db.writer), "read-only writes must not accumulate in memory") + assert.Equal(t, int32(1), addRowsCalls.Load(), "read-only samples must not reach VictoriaMetrics") + + afterDrop, err := db.QueryServerMetrics(1, MetricServerCPU, Period1Day) + require.NoError(t, err) + assert.Equal(t, before, afterDrop, "read-only mode must preserve existing query access") + + checker.value.Store(false) + require.NoError(t, db.WriteServerMetrics(&ServerMetrics{ + ServerID: 1, + Timestamp: firstTimestamp.Add(4 * time.Second), + CPU: 30, + })) + db.Flush() + assert.False(t, db.WritesPaused()) + assert.Equal(t, int32(2), addRowsCalls.Load(), "writes must resume after storage leaves read-only mode") +} + +func TestDiskFullAddRowsPanicPausesWrites(t *testing.T) { + db := newDiskGuardTestDB(t) + db.addRowsFn = func([]storage.MetricRow, uint8) { + panic("write data: no space left on device") + } + + require.NoError(t, db.WriteServerMetrics(&ServerMetrics{ + ServerID: 1, + Timestamp: time.Now(), + CPU: 10, + })) + assert.NotPanics(t, db.Flush) + assert.True(t, db.WritesPaused()) + + require.NoError(t, db.WriteServerMetrics(&ServerMetrics{ + ServerID: 1, + Timestamp: time.Now().Add(time.Second), + CPU: 20, + })) + assert.Zero(t, bufferedRows(db.writer)) +} + +func TestNonDiskStoragePanicIsNotHidden(t *testing.T) { + db := newDiskGuardTestDB(t) + db.addRowsFn = func([]storage.MetricRow, uint8) { + panic("storage invariant failed") + } + + require.NoError(t, db.WriteServerMetrics(&ServerMetrics{ + ServerID: 1, + Timestamp: time.Now(), + CPU: 10, + })) + assert.PanicsWithValue(t, "storage invariant failed", db.Flush) +} diff --git a/pkg/tsdb/maintenance.go b/pkg/tsdb/maintenance.go index fe9c73e1..4f2c4a66 100644 --- a/pkg/tsdb/maintenance.go +++ b/pkg/tsdb/maintenance.go @@ -10,8 +10,11 @@ func (db *TSDB) Maintenance() { if db.closed { return } + if !db.acceptsWrites() { + return + } log.Println("NEZHA>> TSDB starting maintenance (flush)...") - db.storage.DebugFlush() + db.debugFlushSafely() log.Println("NEZHA>> TSDB maintenance completed") } diff --git a/pkg/tsdb/tsdb.go b/pkg/tsdb/tsdb.go index d3627b61..9ae4a16b 100644 --- a/pkg/tsdb/tsdb.go +++ b/pkg/tsdb/tsdb.go @@ -1,15 +1,26 @@ package tsdb import ( + "errors" "fmt" "log" "path/filepath" + "strings" "sync" + "sync/atomic" "time" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" ) +// ErrDiskFull is returned when VictoriaMetrics cannot initialize because the +// filesystem containing the TSDB has run out of space. +var ErrDiskFull = errors.New("TSDB disk is full") + +type readOnlyChecker interface { + IsReadOnly() bool +} + // TSDB 封装 VictoriaMetrics 存储 type TSDB struct { storage *storage.Storage @@ -18,6 +29,13 @@ type TSDB struct { closed bool writer *bufferedWriter + + readOnly readOnlyChecker + addRowsFn func([]storage.MetricRow, uint8) + debugFlushFn func() + + readOnlyObserved atomic.Bool + diskFullPaused atomic.Bool } // InitGlobalSettings 初始化 VictoriaMetrics 包级别的全局设置。 @@ -35,7 +53,18 @@ func InitGlobalSettings(config *Config) { } // Open 打开或创建 TSDB 存储 -func Open(config *Config) (*TSDB, error) { +func Open(config *Config) (db *TSDB, err error) { + defer func() { + if recovered := recover(); recovered != nil { + if diskErr := diskFullPanicError(recovered); diskErr != nil { + db = nil + err = diskErr + return + } + panic(recovered) + } + }() + if config == nil { config = DefaultConfig() } @@ -59,9 +88,12 @@ func Open(config *Config) (*TSDB, error) { stor := storage.MustOpenStorage(dataPath, opts) - db := &TSDB{ - storage: stor, - config: config, + db = &TSDB{ + storage: stor, + config: config, + readOnly: stor, + addRowsFn: stor.AddRows, + debugFlushFn: stor.DebugFlush, } db.writer = newBufferedWriter(db, config.WriteBufferSize, config.WriteBufferFlushInterval) @@ -72,6 +104,86 @@ func Open(config *Config) (*TSDB, error) { return db, nil } +func diskFullPanicError(recovered any) error { + message := strings.ToLower(fmt.Sprint(recovered)) + for _, marker := range []string{ + "no space left on device", + "disk quota exceeded", + "not enough space on the disk", + } { + if strings.Contains(message, marker) { + return fmt.Errorf("%w: %v", ErrDiskFull, recovered) + } + } + return nil +} + +// WritesPaused reports whether new samples are currently being discarded. +// Queries remain available while VictoriaMetrics is in read-only mode. +func (db *TSDB) WritesPaused() bool { + if db.diskFullPaused.Load() { + return true + } + return db.readOnly != nil && db.readOnly.IsReadOnly() +} + +func (db *TSDB) acceptsWrites() bool { + if db.diskFullPaused.Load() { + return false + } + + readOnly := db.readOnly != nil && db.readOnly.IsReadOnly() + if readOnly { + if db.readOnlyObserved.CompareAndSwap(false, true) { + log.Println("NEZHA>> TSDB writes paused because free disk space is below the configured limit; dashboard, queries, and alerts remain available") + } + return false + } + + if db.readOnlyObserved.CompareAndSwap(true, false) { + log.Println("NEZHA>> TSDB writes resumed after free disk space recovered") + } + return true +} + +func (db *TSDB) pauseWritesAfterDiskFull(recovered any) { + if db.diskFullPaused.CompareAndSwap(false, true) { + log.Printf("NEZHA>> TSDB disk is full; disabling TSDB writes until dashboard restart while keeping dashboard, queries, and alerts running: %v", recovered) + } +} + +func (db *TSDB) addRowsSafely(rows []storage.MetricRow) { + if len(rows) == 0 || !db.acceptsWrites() { + return + } + + defer func() { + if recovered := recover(); recovered != nil { + if diskFullPanicError(recovered) == nil { + panic(recovered) + } + db.pauseWritesAfterDiskFull(recovered) + } + }() + db.addRowsFn(rows, 64) +} + +func (db *TSDB) debugFlushSafely() { + if !db.acceptsWrites() { + return + } + + defer func() { + if recovered := recover(); recovered != nil { + if diskFullPanicError(recovered) == nil { + panic(recovered) + } + db.pauseWritesAfterDiskFull(recovered) + } + }() + db.debugFlushFn() +} + // Close 关闭 TSDB 存储 func (db *TSDB) Close() error { db.mu.Lock() @@ -110,8 +222,14 @@ func (db *TSDB) IsClosed() bool { // Flush 强制刷盘（主要用于测试） func (db *TSDB) Flush() { + db.mu.RLock() + defer db.mu.RUnlock() + if db.closed { + return + } + if db.writer != nil { db.writer.flush() } - db.storage.DebugFlush() + db.debugFlushSafely() } diff --git a/pkg/tsdb/writer.go b/pkg/tsdb/writer.go index 7896c094..d6471e2e 100644 --- a/pkg/tsdb/writer.go +++ b/pkg/tsdb/writer.go @@ -47,18 +47,29 @@ func (w *bufferedWriter) flushLoop() { } func (w *bufferedWriter) write(rows []storage.MetricRow) { + if !w.db.acceptsWrites() { + w.discard() + return + } + w.mu.Lock() w.buffer = append(w.buffer, rows...) if len(w.buffer) >= w.maxSize { rows := w.buffer w.buffer = make([]storage.MetricRow, 0, w.maxSize) w.mu.Unlock() - w.db.storage.AddRows(rows, 64) + w.db.addRowsSafely(rows) return } w.mu.Unlock() } +func (w *bufferedWriter) discard() { + w.mu.Lock() + w.buffer = make([]storage.MetricRow, 0, w.maxSize) + w.mu.Unlock() +} + func (w *bufferedWriter) flush() { w.mu.Lock() if len(w.buffer) == 0 { @@ -69,7 +80,7 @@ func (w *bufferedWriter) flush() { w.buffer = make([]storage.MetricRow, 0, w.maxSize) w.mu.Unlock() - w.db.storage.AddRows(rows, 64) + w.db.addRowsSafely(rows) } func (w *bufferedWriter) stop() { @@ -171,7 +182,7 @@ func (db *TSDB) WriteServerMetrics(m *ServerMetrics) error { if db.writer != nil { db.writer.write(rows) } else { - db.storage.AddRows(rows, 64) + db.addRowsSafely(rows) } return nil } @@ -200,7 +211,7 @@ func (db *TSDB) WriteServiceMetrics(m *ServiceMetrics) error { if db.writer != nil { db.writer.write(rows) } else { - db.storage.AddRows(rows, 64) + db.addRowsSafely(rows) } return nil } @@ -265,7 +276,7 @@ func (db *TSDB) WriteBatchServerMetrics(metrics []*ServerMetrics) error { if db.writer != nil { db.writer.write(rows) } else { - db.storage.AddRows(rows, 64) + db.addRowsSafely(rows) } return nil } @@ -295,7 +306,7 @@ func (db *TSDB) WriteBatchServiceMetrics(metrics []*ServiceMetrics) error { if db.writer != nil { db.writer.write(rows) } else { - db.storage.AddRows(rows, 64) + db.addRowsSafely(rows) } return nil } diff --git a/proto/nezha.pb.go b/proto/nezha.pb.go index 9aa9910b..16c73f50 100644 --- a/proto/nezha.pb.go +++ b/proto/nezha.pb.go @@ -1,18 +1,16 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.11 -// protoc v3.21.12 +// protoc-gen-go v1.34.2 +// protoc v5.29.3 // source: proto/nezha.proto package proto import ( - reflect "reflect" - sync "sync" - unsafe "unsafe" - protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) const ( @@ -23,27 +21,30 @@ const ( ) type Host struct { - state protoimpl.MessageState `protogen:"open.v1"` - Platform string `protobuf:"bytes,1,opt,name=platform,proto3" json:"platform,omitempty"` - PlatformVersion string `protobuf:"bytes,2,opt,name=platform_version,json=platformVersion,proto3" json:"platform_version,omitempty"` - Cpu []string `protobuf:"bytes,3,rep,name=cpu,proto3" json:"cpu,omitempty"` - MemTotal uint64 `protobuf:"varint,4,opt,name=mem_total,json=memTotal,proto3" json:"mem_total,omitempty"` - DiskTotal uint64 `protobuf:"varint,5,opt,name=disk_total,json=diskTotal,proto3" json:"disk_total,omitempty"` - SwapTotal uint64 `protobuf:"varint,6,opt,name=swap_total,json=swapTotal,proto3" json:"swap_total,omitempty"` - Arch string `protobuf:"bytes,7,opt,name=arch,proto3" json:"arch,omitempty"` - Virtualization string `protobuf:"bytes,8,opt,name=virtualization,proto3" json:"virtualization,omitempty"` - BootTime uint64 `protobuf:"varint,9,opt,name=boot_time,json=bootTime,proto3" json:"boot_time,omitempty"` - Version string `protobuf:"bytes,10,opt,name=version,proto3" json:"version,omitempty"` - Gpu []string `protobuf:"bytes,11,rep,name=gpu,proto3" json:"gpu,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Platform string `protobuf:"bytes,1,opt,name=platform,proto3" json:"platform,omitempty"` + PlatformVersion string `protobuf:"bytes,2,opt,name=platform_version,json=platformVersion,proto3" json:"platform_version,omitempty"` + Cpu []string `protobuf:"bytes,3,rep,name=cpu,proto3" json:"cpu,omitempty"` + MemTotal uint64 `protobuf:"varint,4,opt,name=mem_total,json=memTotal,proto3" json:"mem_total,omitempty"` + DiskTotal uint64 `protobuf:"varint,5,opt,name=disk_total,json=diskTotal,proto3" json:"disk_total,omitempty"` + SwapTotal uint64 `protobuf:"varint,6,opt,name=swap_total,json=swapTotal,proto3" json:"swap_total,omitempty"` + Arch string `protobuf:"bytes,7,opt,name=arch,proto3" json:"arch,omitempty"` + Virtualization string `protobuf:"bytes,8,opt,name=virtualization,proto3" json:"virtualization,omitempty"` + BootTime uint64 `protobuf:"varint,9,opt,name=boot_time,json=bootTime,proto3" json:"boot_time,omitempty"` + Version string `protobuf:"bytes,10,opt,name=version,proto3" json:"version,omitempty"` + Gpu []string `protobuf:"bytes,11,rep,name=gpu,proto3" json:"gpu,omitempty"` } func (x *Host) Reset() { *x = Host{} - mi := &file_proto_nezha_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) + if protoimpl.UnsafeEnabled { + mi := &file_proto_nezha_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } func (x *Host) String() string { @@ -54,7 +55,7 @@ func (*Host) ProtoMessage() {} func (x *Host) ProtoReflect() protoreflect.Message { mi := &file_proto_nezha_proto_msgTypes[0] - if x != nil { + if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -147,7 +148,10 @@ func (x *Host) GetGpu() []string { } type State struct { - state protoimpl.MessageState `protogen:"open.v1"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + Cpu float64 `protobuf:"fixed64,1,opt,name=cpu,proto3" json:"cpu,omitempty"` MemUsed uint64 `protobuf:"varint,2,opt,name=mem_used,json=memUsed,proto3" json:"mem_used,omitempty"` SwapUsed uint64 `protobuf:"varint,3,opt,name=swap_used,json=swapUsed,proto3" json:"swap_used,omitempty"` @@ -165,15 +169,16 @@ type State struct { ProcessCount uint64 `protobuf:"varint,15,opt,name=process_count,json=processCount,proto3" json:"process_count,omitempty"` Temperatures []*State_SensorTemperature `protobuf:"bytes,16,rep,name=temperatures,proto3" json:"temperatures,omitempty"` Gpu []float64 `protobuf:"fixed64,17,rep,packed,name=gpu,proto3" json:"gpu,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Gpus []*State_GPU `protobuf:"bytes,18,rep,name=gpus,proto3" json:"gpus,omitempty"` } func (x *State) Reset() { *x = State{} - mi := &file_proto_nezha_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) + if protoimpl.UnsafeEnabled { + mi := &file_proto_nezha_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } func (x *State) String() string { @@ -184,7 +189,7 @@ func (*State) ProtoMessage() {} func (x *State) ProtoReflect() protoreflect.Message { mi := &file_proto_nezha_proto_msgTypes[1] - if x != nil { + if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -318,19 +323,95 @@ func (x *State) GetGpu() []float64 { return nil } -type State_SensorTemperature struct { - state protoimpl.MessageState `protogen:"open.v1"` - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Temperature float64 `protobuf:"fixed64,2,opt,name=temperature,proto3" json:"temperature,omitempty"` - unknownFields protoimpl.UnknownFields +func (x *State) GetGpus() []*State_GPU { + if x != nil { + return x.Gpus + } + return nil +} + +// State_GPU carries per-card figures. Index-aligned with Host.gpu, matching +// the existing `gpu` field it supersedes; that field stays populated so older +// dashboards keep working. +type State_GPU struct { + state protoimpl.MessageState sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Utilization float64 `protobuf:"fixed64,1,opt,name=utilization,proto3" json:"utilization,omitempty"` + MemoryUsed uint64 `protobuf:"varint,2,opt,name=memory_used,json=memoryUsed,proto3" json:"memory_used,omitempty"` + MemoryTotal uint64 `protobuf:"varint,3,opt,name=memory_total,json=memoryTotal,proto3" json:"memory_total,omitempty"` +} + +func (x *State_GPU) Reset() { + *x = State_GPU{} + if protoimpl.UnsafeEnabled { + mi := &file_proto_nezha_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *State_GPU) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*State_GPU) ProtoMessage() {} + +func (x *State_GPU) ProtoReflect() protoreflect.Message { + mi := &file_proto_nezha_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use State_GPU.ProtoReflect.Descriptor instead. +func (*State_GPU) Descriptor() ([]byte, []int) { + return file_proto_nezha_proto_rawDescGZIP(), []int{2} +} + +func (x *State_GPU) GetUtilization() float64 { + if x != nil { + return x.Utilization + } + return 0 +} + +func (x *State_GPU) GetMemoryUsed() uint64 { + if x != nil { + return x.MemoryUsed + } + return 0 +} + +func (x *State_GPU) GetMemoryTotal() uint64 { + if x != nil { + return x.MemoryTotal + } + return 0 +} + +type State_SensorTemperature struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Temperature float64 `protobuf:"fixed64,2,opt,name=temperature,proto3" json:"temperature,omitempty"` } func (x *State_SensorTemperature) Reset() { *x = State_SensorTemperature{} - mi := &file_proto_nezha_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) + if protoimpl.UnsafeEnabled { + mi := &file_proto_nezha_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } func (x *State_SensorTemperature) String() string { @@ -340,8 +421,8 @@ func (x *State_SensorTemperature) String() string { func (*State_SensorTemperature) ProtoMessage() {} func (x *State_SensorTemperature) ProtoReflect() protoreflect.Message { - mi := &file_proto_nezha_proto_msgTypes[2] - if x != nil { + mi := &file_proto_nezha_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -353,7 +434,7 @@ func (x *State_SensorTemperature) ProtoReflect() protoreflect.Message { // Deprecated: Use State_SensorTemperature.ProtoReflect.Descriptor instead. func (*State_SensorTemperature) Descriptor() ([]byte, []int) { - return file_proto_nezha_proto_rawDescGZIP(), []int{2} + return file_proto_nezha_proto_rawDescGZIP(), []int{3} } func (x *State_SensorTemperature) GetName() string { @@ -371,19 +452,22 @@ func (x *State_SensorTemperature) GetTemperature() float64 { } type Task struct { - state protoimpl.MessageState `protogen:"open.v1"` - Id uint64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` - Type uint64 `protobuf:"varint,2,opt,name=type,proto3" json:"type,omitempty"` - Data string `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"` - unknownFields protoimpl.UnknownFields + state protoimpl.MessageState sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id uint64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Type uint64 `protobuf:"varint,2,opt,name=type,proto3" json:"type,omitempty"` + Data string `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"` } func (x *Task) Reset() { *x = Task{} - mi := &file_proto_nezha_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) + if protoimpl.UnsafeEnabled { + mi := &file_proto_nezha_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } func (x *Task) String() string { @@ -393,8 +477,8 @@ func (x *Task) String() string { func (*Task) ProtoMessage() {} func (x *Task) ProtoReflect() protoreflect.Message { - mi := &file_proto_nezha_proto_msgTypes[3] - if x != nil { + mi := &file_proto_nezha_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -406,7 +490,7 @@ func (x *Task) ProtoReflect() protoreflect.Message { // Deprecated: Use Task.ProtoReflect.Descriptor instead. func (*Task) Descriptor() ([]byte, []int) { - return file_proto_nezha_proto_rawDescGZIP(), []int{3} + return file_proto_nezha_proto_rawDescGZIP(), []int{4} } func (x *Task) GetId() uint64 { @@ -431,21 +515,24 @@ func (x *Task) GetData() string { } type TaskResult struct { - state protoimpl.MessageState `protogen:"open.v1"` - Id uint64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` - Type uint64 `protobuf:"varint,2,opt,name=type,proto3" json:"type,omitempty"` - Delay float32 `protobuf:"fixed32,3,opt,name=delay,proto3" json:"delay,omitempty"` - Data string `protobuf:"bytes,4,opt,name=data,proto3" json:"data,omitempty"` - Successful bool `protobuf:"varint,5,opt,name=successful,proto3" json:"successful,omitempty"` - unknownFields protoimpl.UnknownFields + state protoimpl.MessageState sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id uint64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Type uint64 `protobuf:"varint,2,opt,name=type,proto3" json:"type,omitempty"` + Delay float32 `protobuf:"fixed32,3,opt,name=delay,proto3" json:"delay,omitempty"` + Data string `protobuf:"bytes,4,opt,name=data,proto3" json:"data,omitempty"` + Successful bool `protobuf:"varint,5,opt,name=successful,proto3" json:"successful,omitempty"` } func (x *TaskResult) Reset() { *x = TaskResult{} - mi := &file_proto_nezha_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) + if protoimpl.UnsafeEnabled { + mi := &file_proto_nezha_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } func (x *TaskResult) String() string { @@ -455,8 +542,8 @@ func (x *TaskResult) String() string { func (*TaskResult) ProtoMessage() {} func (x *TaskResult) ProtoReflect() protoreflect.Message { - mi := &file_proto_nezha_proto_msgTypes[4] - if x != nil { + mi := &file_proto_nezha_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -468,7 +555,7 @@ func (x *TaskResult) ProtoReflect() protoreflect.Message { // Deprecated: Use TaskResult.ProtoReflect.Descriptor instead. func (*TaskResult) Descriptor() ([]byte, []int) { - return file_proto_nezha_proto_rawDescGZIP(), []int{4} + return file_proto_nezha_proto_rawDescGZIP(), []int{5} } func (x *TaskResult) GetId() uint64 { @@ -507,17 +594,20 @@ func (x *TaskResult) GetSuccessful() bool { } type Receipt struct { - state protoimpl.MessageState `protogen:"open.v1"` - Proced bool `protobuf:"varint,1,opt,name=proced,proto3" json:"proced,omitempty"` - unknownFields protoimpl.UnknownFields + state protoimpl.MessageState sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Proced bool `protobuf:"varint,1,opt,name=proced,proto3" json:"proced,omitempty"` } func (x *Receipt) Reset() { *x = Receipt{} - mi := &file_proto_nezha_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) + if protoimpl.UnsafeEnabled { + mi := &file_proto_nezha_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } func (x *Receipt) String() string { @@ -527,8 +617,8 @@ func (x *Receipt) String() string { func (*Receipt) ProtoMessage() {} func (x *Receipt) ProtoReflect() protoreflect.Message { - mi := &file_proto_nezha_proto_msgTypes[5] - if x != nil { + mi := &file_proto_nezha_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -540,7 +630,7 @@ func (x *Receipt) ProtoReflect() protoreflect.Message { // Deprecated: Use Receipt.ProtoReflect.Descriptor instead. func (*Receipt) Descriptor() ([]byte, []int) { - return file_proto_nezha_proto_rawDescGZIP(), []int{5} + return file_proto_nezha_proto_rawDescGZIP(), []int{6} } func (x *Receipt) GetProced() bool { @@ -551,17 +641,20 @@ func (x *Receipt) GetProced() bool { } type Uint64Receipt struct { - state protoimpl.MessageState `protogen:"open.v1"` - Data uint64 `protobuf:"varint,1,opt,name=data,proto3" json:"data,omitempty"` - unknownFields protoimpl.UnknownFields + state protoimpl.MessageState sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Data uint64 `protobuf:"varint,1,opt,name=data,proto3" json:"data,omitempty"` } func (x *Uint64Receipt) Reset() { *x = Uint64Receipt{} - mi := &file_proto_nezha_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) + if protoimpl.UnsafeEnabled { + mi := &file_proto_nezha_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } func (x *Uint64Receipt) String() string { @@ -571,8 +664,8 @@ func (x *Uint64Receipt) String() string { func (*Uint64Receipt) ProtoMessage() {} func (x *Uint64Receipt) ProtoReflect() protoreflect.Message { - mi := &file_proto_nezha_proto_msgTypes[6] - if x != nil { + mi := &file_proto_nezha_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -584,7 +677,7 @@ func (x *Uint64Receipt) ProtoReflect() protoreflect.Message { // Deprecated: Use Uint64Receipt.ProtoReflect.Descriptor instead. func (*Uint64Receipt) Descriptor() ([]byte, []int) { - return file_proto_nezha_proto_rawDescGZIP(), []int{6} + return file_proto_nezha_proto_rawDescGZIP(), []int{7} } func (x *Uint64Receipt) GetData() uint64 { @@ -595,17 +688,20 @@ func (x *Uint64Receipt) GetData() uint64 { } type IOStreamData struct { - state protoimpl.MessageState `protogen:"open.v1"` - Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` - unknownFields protoimpl.UnknownFields + state protoimpl.MessageState sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` } func (x *IOStreamData) Reset() { *x = IOStreamData{} - mi := &file_proto_nezha_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) + if protoimpl.UnsafeEnabled { + mi := &file_proto_nezha_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } func (x *IOStreamData) String() string { @@ -615,8 +711,8 @@ func (x *IOStreamData) String() string { func (*IOStreamData) ProtoMessage() {} func (x *IOStreamData) ProtoReflect() protoreflect.Message { - mi := &file_proto_nezha_proto_msgTypes[7] - if x != nil { + mi := &file_proto_nezha_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -628,7 +724,7 @@ func (x *IOStreamData) ProtoReflect() protoreflect.Message { // Deprecated: Use IOStreamData.ProtoReflect.Descriptor instead. func (*IOStreamData) Descriptor() ([]byte, []int) { - return file_proto_nezha_proto_rawDescGZIP(), []int{7} + return file_proto_nezha_proto_rawDescGZIP(), []int{8} } func (x *IOStreamData) GetData() []byte { @@ -639,20 +735,23 @@ func (x *IOStreamData) GetData() []byte { } type GeoIP struct { - state protoimpl.MessageState `protogen:"open.v1"` - Use6 bool `protobuf:"varint,1,opt,name=use6,proto3" json:"use6,omitempty"` - Ip *IP `protobuf:"bytes,2,opt,name=ip,proto3" json:"ip,omitempty"` - CountryCode string `protobuf:"bytes,3,opt,name=country_code,json=countryCode,proto3" json:"country_code,omitempty"` - DashboardBootTime uint64 `protobuf:"varint,4,opt,name=dashboard_boot_time,json=dashboardBootTime,proto3" json:"dashboard_boot_time,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Use6 bool `protobuf:"varint,1,opt,name=use6,proto3" json:"use6,omitempty"` + Ip *IP `protobuf:"bytes,2,opt,name=ip,proto3" json:"ip,omitempty"` + CountryCode string `protobuf:"bytes,3,opt,name=country_code,json=countryCode,proto3" json:"country_code,omitempty"` + DashboardBootTime uint64 `protobuf:"varint,4,opt,name=dashboard_boot_time,json=dashboardBootTime,proto3" json:"dashboard_boot_time,omitempty"` } func (x *GeoIP) Reset() { *x = GeoIP{} - mi := &file_proto_nezha_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) + if protoimpl.UnsafeEnabled { + mi := &file_proto_nezha_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } func (x *GeoIP) String() string { @@ -662,8 +761,8 @@ func (x *GeoIP) String() string { func (*GeoIP) ProtoMessage() {} func (x *GeoIP) ProtoReflect() protoreflect.Message { - mi := &file_proto_nezha_proto_msgTypes[8] - if x != nil { + mi := &file_proto_nezha_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -675,7 +774,7 @@ func (x *GeoIP) ProtoReflect() protoreflect.Message { // Deprecated: Use GeoIP.ProtoReflect.Descriptor instead. func (*GeoIP) Descriptor() ([]byte, []int) { - return file_proto_nezha_proto_rawDescGZIP(), []int{8} + return file_proto_nezha_proto_rawDescGZIP(), []int{9} } func (x *GeoIP) GetUse6() bool { @@ -707,18 +806,21 @@ func (x *GeoIP) GetDashboardBootTime() uint64 { } type IP struct { - state protoimpl.MessageState `protogen:"open.v1"` - Ipv4 string `protobuf:"bytes,1,opt,name=ipv4,proto3" json:"ipv4,omitempty"` - Ipv6 string `protobuf:"bytes,2,opt,name=ipv6,proto3" json:"ipv6,omitempty"` - unknownFields protoimpl.UnknownFields + state protoimpl.MessageState sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Ipv4 string `protobuf:"bytes,1,opt,name=ipv4,proto3" json:"ipv4,omitempty"` + Ipv6 string `protobuf:"bytes,2,opt,name=ipv6,proto3" json:"ipv6,omitempty"` } func (x *IP) Reset() { *x = IP{} - mi := &file_proto_nezha_proto_msgTypes[9] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) + if protoimpl.UnsafeEnabled { + mi := &file_proto_nezha_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } func (x *IP) String() string { @@ -728,8 +830,8 @@ func (x *IP) String() string { func (*IP) ProtoMessage() {} func (x *IP) ProtoReflect() protoreflect.Message { - mi := &file_proto_nezha_proto_msgTypes[9] - if x != nil { + mi := &file_proto_nezha_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -741,7 +843,7 @@ func (x *IP) ProtoReflect() protoreflect.Message { // Deprecated: Use IP.ProtoReflect.Descriptor instead. func (*IP) Descriptor() ([]byte, []int) { - return file_proto_nezha_proto_rawDescGZIP(), []int{9} + return file_proto_nezha_proto_rawDescGZIP(), []int{10} } func (x *IP) GetIpv4() string { @@ -760,128 +862,180 @@ func (x *IP) GetIpv6() string { var File_proto_nezha_proto protoreflect.FileDescriptor -const file_proto_nezha_proto_rawDesc = "" + - "\n" + - "\x11proto/nezha.proto\x12\x05proto\"\xbf\x02\n" + - "\x04Host\x12\x1a\n" + - "\bplatform\x18\x01 \x01(\tR\bplatform\x12)\n" + - "\x10platform_version\x18\x02 \x01(\tR\x0fplatformVersion\x12\x10\n" + - "\x03cpu\x18\x03 \x03(\tR\x03cpu\x12\x1b\n" + - "\tmem_total\x18\x04 \x01(\x04R\bmemTotal\x12\x1d\n" + - "\n" + - "disk_total\x18\x05 \x01(\x04R\tdiskTotal\x12\x1d\n" + - "\n" + - "swap_total\x18\x06 \x01(\x04R\tswapTotal\x12\x12\n" + - "\x04arch\x18\a \x01(\tR\x04arch\x12&\n" + - "\x0evirtualization\x18\b \x01(\tR\x0evirtualization\x12\x1b\n" + - "\tboot_time\x18\t \x01(\x04R\bbootTime\x12\x18\n" + - "\aversion\x18\n" + - " \x01(\tR\aversion\x12\x10\n" + - "\x03gpu\x18\v \x03(\tR\x03gpu\"\xa9\x04\n" + - "\x05State\x12\x10\n" + - "\x03cpu\x18\x01 \x01(\x01R\x03cpu\x12\x19\n" + - "\bmem_used\x18\x02 \x01(\x04R\amemUsed\x12\x1b\n" + - "\tswap_used\x18\x03 \x01(\x04R\bswapUsed\x12\x1b\n" + - "\tdisk_used\x18\x04 \x01(\x04R\bdiskUsed\x12&\n" + - "\x0fnet_in_transfer\x18\x05 \x01(\x04R\rnetInTransfer\x12(\n" + - "\x10net_out_transfer\x18\x06 \x01(\x04R\x0enetOutTransfer\x12 \n" + - "\fnet_in_speed\x18\a \x01(\x04R\n" + - "netInSpeed\x12\"\n" + - "\rnet_out_speed\x18\b \x01(\x04R\vnetOutSpeed\x12\x16\n" + - "\x06uptime\x18\t \x01(\x04R\x06uptime\x12\x14\n" + - "\x05load1\x18\n" + - " \x01(\x01R\x05load1\x12\x14\n" + - "\x05load5\x18\v \x01(\x01R\x05load5\x12\x16\n" + - "\x06load15\x18\f \x01(\x01R\x06load15\x12$\n" + - "\x0etcp_conn_count\x18\r \x01(\x04R\ftcpConnCount\x12$\n" + - "\x0eudp_conn_count\x18\x0e \x01(\x04R\fudpConnCount\x12#\n" + - "\rprocess_count\x18\x0f \x01(\x04R\fprocessCount\x12B\n" + - "\ftemperatures\x18\x10 \x03(\v2\x1e.proto.State_SensorTemperatureR\ftemperatures\x12\x10\n" + - "\x03gpu\x18\x11 \x03(\x01R\x03gpu\"O\n" + - "\x17State_SensorTemperature\x12\x12\n" + - "\x04name\x18\x01 \x01(\tR\x04name\x12 \n" + - "\vtemperature\x18\x02 \x01(\x01R\vtemperature\">\n" + - "\x04Task\x12\x0e\n" + - "\x02id\x18\x01 \x01(\x04R\x02id\x12\x12\n" + - "\x04type\x18\x02 \x01(\x04R\x04type\x12\x12\n" + - "\x04data\x18\x03 \x01(\tR\x04data\"z\n" + - "\n" + - "TaskResult\x12\x0e\n" + - "\x02id\x18\x01 \x01(\x04R\x02id\x12\x12\n" + - "\x04type\x18\x02 \x01(\x04R\x04type\x12\x14\n" + - "\x05delay\x18\x03 \x01(\x02R\x05delay\x12\x12\n" + - "\x04data\x18\x04 \x01(\tR\x04data\x12\x1e\n" + - "\n" + - "successful\x18\x05 \x01(\bR\n" + - "successful\"!\n" + - "\aReceipt\x12\x16\n" + - "\x06proced\x18\x01 \x01(\bR\x06proced\"#\n" + - "\rUint64Receipt\x12\x12\n" + - "\x04data\x18\x01 \x01(\x04R\x04data\"\"\n" + - "\fIOStreamData\x12\x12\n" + - "\x04data\x18\x01 \x01(\fR\x04data\"\x89\x01\n" + - "\x05GeoIP\x12\x12\n" + - "\x04use6\x18\x01 \x01(\bR\x04use6\x12\x19\n" + - "\x02ip\x18\x02 \x01(\v2\t.proto.IPR\x02ip\x12!\n" + - "\fcountry_code\x18\x03 \x01(\tR\vcountryCode\x12.\n" + - "\x13dashboard_boot_time\x18\x04 \x01(\x04R\x11dashboardBootTime\",\n" + - "\x02IP\x12\x12\n" + - "\x04ipv4\x18\x01 \x01(\tR\x04ipv4\x12\x12\n" + - "\x04ipv6\x18\x02 \x01(\tR\x04ipv62\xd2\x02\n" + - "\fNezhaService\x127\n" + - "\x11ReportSystemState\x12\f.proto.State\x1a\x0e.proto.Receipt\"\x00(\x010\x01\x121\n" + - "\x10ReportSystemInfo\x12\v.proto.Host\x1a\x0e.proto.Receipt\"\x00\x123\n" + - "\vRequestTask\x12\x11.proto.TaskResult\x1a\v.proto.Task\"\x00(\x010\x01\x12:\n" + - "\bIOStream\x12\x13.proto.IOStreamData\x1a\x13.proto.IOStreamData\"\x00(\x010\x01\x12+\n" + - "\vReportGeoIP\x12\f.proto.GeoIP\x1a\f.proto.GeoIP\"\x00\x128\n" + - "\x11ReportSystemInfo2\x12\v.proto.Host\x1a\x14.proto.Uint64Receipt\"\x00B\tZ\a./protob\x06proto3" +var file_proto_nezha_proto_rawDesc = []byte{ + 0x0a, 0x11, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6e, 0x65, 0x7a, 0x68, 0x61, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xbf, 0x02, 0x0a, 0x04, 0x48, + 0x6f, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, + 0x29, 0x0a, 0x10, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x5f, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x70, 0x6c, 0x61, 0x74, 0x66, + 0x6f, 0x72, 0x6d, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x70, + 0x75, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x63, 0x70, 0x75, 0x12, 0x1b, 0x0a, 0x09, + 0x6d, 0x65, 0x6d, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x08, 0x6d, 0x65, 0x6d, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x64, 0x69, 0x73, + 0x6b, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x64, + 0x69, 0x73, 0x6b, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x77, 0x61, 0x70, + 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x73, 0x77, + 0x61, 0x70, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x61, 0x72, 0x63, 0x68, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x61, 0x72, 0x63, 0x68, 0x12, 0x26, 0x0a, 0x0e, 0x76, + 0x69, 0x72, 0x74, 0x75, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0e, 0x76, 0x69, 0x72, 0x74, 0x75, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x62, 0x6f, 0x6f, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, + 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x62, 0x6f, 0x6f, 0x74, 0x54, 0x69, 0x6d, 0x65, + 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x70, + 0x75, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x67, 0x70, 0x75, 0x22, 0xcf, 0x04, 0x0a, + 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x70, 0x75, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x01, 0x52, 0x03, 0x63, 0x70, 0x75, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x65, 0x6d, 0x5f, + 0x75, 0x73, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6d, 0x65, 0x6d, 0x55, + 0x73, 0x65, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x77, 0x61, 0x70, 0x5f, 0x75, 0x73, 0x65, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x73, 0x77, 0x61, 0x70, 0x55, 0x73, 0x65, 0x64, + 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x75, 0x73, 0x65, 0x64, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x55, 0x73, 0x65, 0x64, 0x12, 0x26, 0x0a, + 0x0f, 0x6e, 0x65, 0x74, 0x5f, 0x69, 0x6e, 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x6e, 0x65, 0x74, 0x49, 0x6e, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x66, 0x65, 0x72, 0x12, 0x28, 0x0a, 0x10, 0x6e, 0x65, 0x74, 0x5f, 0x6f, 0x75, 0x74, + 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x0e, 0x6e, 0x65, 0x74, 0x4f, 0x75, 0x74, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x12, + 0x20, 0x0a, 0x0c, 0x6e, 0x65, 0x74, 0x5f, 0x69, 0x6e, 0x5f, 0x73, 0x70, 0x65, 0x65, 0x64, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x6e, 0x65, 0x74, 0x49, 0x6e, 0x53, 0x70, 0x65, 0x65, + 0x64, 0x12, 0x22, 0x0a, 0x0d, 0x6e, 0x65, 0x74, 0x5f, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x70, 0x65, + 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x6e, 0x65, 0x74, 0x4f, 0x75, 0x74, + 0x53, 0x70, 0x65, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x18, + 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x14, 0x0a, + 0x05, 0x6c, 0x6f, 0x61, 0x64, 0x31, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x6c, 0x6f, + 0x61, 0x64, 0x31, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x6f, 0x61, 0x64, 0x35, 0x18, 0x0b, 0x20, 0x01, + 0x28, 0x01, 0x52, 0x05, 0x6c, 0x6f, 0x61, 0x64, 0x35, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x6f, 0x61, + 0x64, 0x31, 0x35, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x01, 0x52, 0x06, 0x6c, 0x6f, 0x61, 0x64, 0x31, + 0x35, 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x63, 0x70, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x5f, 0x63, 0x6f, + 0x75, 0x6e, 0x74, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x74, 0x63, 0x70, 0x43, 0x6f, + 0x6e, 0x6e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x24, 0x0a, 0x0e, 0x75, 0x64, 0x70, 0x5f, 0x63, + 0x6f, 0x6e, 0x6e, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x0c, 0x75, 0x64, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x23, 0x0a, + 0x0d, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x0f, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x43, 0x6f, 0x75, + 0x6e, 0x74, 0x12, 0x42, 0x0a, 0x0c, 0x74, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x73, 0x18, 0x10, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x53, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x54, 0x65, 0x6d, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x0c, 0x74, 0x65, 0x6d, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x70, 0x75, 0x18, 0x11, 0x20, + 0x03, 0x28, 0x01, 0x52, 0x03, 0x67, 0x70, 0x75, 0x12, 0x24, 0x0a, 0x04, 0x67, 0x70, 0x75, 0x73, + 0x18, 0x12, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x5f, 0x47, 0x50, 0x55, 0x52, 0x04, 0x67, 0x70, 0x75, 0x73, 0x22, 0x71, + 0x0a, 0x09, 0x53, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x47, 0x50, 0x55, 0x12, 0x20, 0x0a, 0x0b, 0x75, + 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, + 0x52, 0x0b, 0x75, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, + 0x0b, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x75, 0x73, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x0a, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x55, 0x73, 0x65, 0x64, 0x12, 0x21, + 0x0a, 0x0c, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x54, 0x6f, 0x74, 0x61, + 0x6c, 0x22, 0x4f, 0x0a, 0x17, 0x53, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x53, 0x65, 0x6e, 0x73, 0x6f, + 0x72, 0x54, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x20, 0x0a, 0x0b, 0x74, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x74, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x22, 0x3e, 0x0a, 0x04, 0x54, 0x61, 0x73, 0x6b, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, + 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x64, 0x61, + 0x74, 0x61, 0x22, 0x7a, 0x0a, 0x0a, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x02, 0x69, 0x64, + 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x02, 0x52, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, + 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1e, + 0x0a, 0x0a, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x66, 0x75, 0x6c, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0a, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x66, 0x75, 0x6c, 0x22, 0x21, + 0x0a, 0x07, 0x52, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x6f, + 0x63, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x70, 0x72, 0x6f, 0x63, 0x65, + 0x64, 0x22, 0x23, 0x0a, 0x0d, 0x55, 0x69, 0x6e, 0x74, 0x36, 0x34, 0x52, 0x65, 0x63, 0x65, 0x69, + 0x70, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x22, 0x0a, 0x0c, 0x49, 0x4f, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x89, 0x01, 0x0a, 0x05, 0x47, + 0x65, 0x6f, 0x49, 0x50, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x73, 0x65, 0x36, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x04, 0x75, 0x73, 0x65, 0x36, 0x12, 0x19, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x50, 0x52, + 0x02, 0x69, 0x70, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x79, 0x5f, 0x63, + 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x72, 0x79, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x2e, 0x0a, 0x13, 0x64, 0x61, 0x73, 0x68, 0x62, 0x6f, + 0x61, 0x72, 0x64, 0x5f, 0x62, 0x6f, 0x6f, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x11, 0x64, 0x61, 0x73, 0x68, 0x62, 0x6f, 0x61, 0x72, 0x64, 0x42, 0x6f, + 0x6f, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x2c, 0x0a, 0x02, 0x49, 0x50, 0x12, 0x12, 0x0a, 0x04, + 0x69, 0x70, 0x76, 0x34, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x69, 0x70, 0x76, 0x34, + 0x12, 0x12, 0x0a, 0x04, 0x69, 0x70, 0x76, 0x36, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x69, 0x70, 0x76, 0x36, 0x32, 0xd2, 0x02, 0x0a, 0x0c, 0x4e, 0x65, 0x7a, 0x68, 0x61, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x37, 0x0a, 0x11, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x53, + 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0c, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x1a, 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x52, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x31, + 0x0a, 0x10, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, + 0x66, 0x6f, 0x12, 0x0b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x1a, + 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x22, + 0x00, 0x12, 0x33, 0x0a, 0x0b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x61, 0x73, 0x6b, + 0x12, 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x1a, 0x0b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x54, 0x61, 0x73, 0x6b, + 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x3a, 0x0a, 0x08, 0x49, 0x4f, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x12, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x4f, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x49, 0x4f, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x22, 0x00, 0x28, 0x01, + 0x30, 0x01, 0x12, 0x2b, 0x0a, 0x0b, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x47, 0x65, 0x6f, 0x49, + 0x50, 0x12, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x6f, 0x49, 0x50, 0x1a, + 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x6f, 0x49, 0x50, 0x22, 0x00, 0x12, + 0x38, 0x0a, 0x11, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, + 0x6e, 0x66, 0x6f, 0x32, 0x12, 0x0b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x48, 0x6f, 0x73, + 0x74, 0x1a, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x69, 0x6e, 0x74, 0x36, 0x34, + 0x52, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x22, 0x00, 0x42, 0x09, 0x5a, 0x07, 0x2e, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} var ( file_proto_nezha_proto_rawDescOnce sync.Once - file_proto_nezha_proto_rawDescData []byte + file_proto_nezha_proto_rawDescData = file_proto_nezha_proto_rawDesc ) func file_proto_nezha_proto_rawDescGZIP() []byte { file_proto_nezha_proto_rawDescOnce.Do(func() { - // #nosec G103 - file_proto_nezha_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_proto_nezha_proto_rawDesc), len(file_proto_nezha_proto_rawDesc))) + file_proto_nezha_proto_rawDescData = protoimpl.X.CompressGZIP(file_proto_nezha_proto_rawDescData) }) return file_proto_nezha_proto_rawDescData } -var file_proto_nezha_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_proto_nezha_proto_msgTypes = make([]protoimpl.MessageInfo, 11) var file_proto_nezha_proto_goTypes = []any{ (*Host)(nil), // 0: proto.Host (*State)(nil), // 1: proto.State - (*State_SensorTemperature)(nil), // 2: proto.State_SensorTemperature - (*Task)(nil), // 3: proto.Task - (*TaskResult)(nil), // 4: proto.TaskResult - (*Receipt)(nil), // 5: proto.Receipt - (*Uint64Receipt)(nil), // 6: proto.Uint64Receipt - (*IOStreamData)(nil), // 7: proto.IOStreamData - (*GeoIP)(nil), // 8: proto.GeoIP - (*IP)(nil), // 9: proto.IP + (*State_GPU)(nil), // 2: proto.State_GPU + (*State_SensorTemperature)(nil), // 3: proto.State_SensorTemperature + (*Task)(nil), // 4: proto.Task + (*TaskResult)(nil), // 5: proto.TaskResult + (*Receipt)(nil), // 6: proto.Receipt + (*Uint64Receipt)(nil), // 7: proto.Uint64Receipt + (*IOStreamData)(nil), // 8: proto.IOStreamData + (*GeoIP)(nil), // 9: proto.GeoIP + (*IP)(nil), // 10: proto.IP } var file_proto_nezha_proto_depIdxs = []int32{ - 2, // 0: proto.State.temperatures:type_name -> proto.State_SensorTemperature - 9, // 1: proto.GeoIP.ip:type_name -> proto.IP - 1, // 2: proto.NezhaService.ReportSystemState:input_type -> proto.State - 0, // 3: proto.NezhaService.ReportSystemInfo:input_type -> proto.Host - 4, // 4: proto.NezhaService.RequestTask:input_type -> proto.TaskResult - 7, // 5: proto.NezhaService.IOStream:input_type -> proto.IOStreamData - 8, // 6: proto.NezhaService.ReportGeoIP:input_type -> proto.GeoIP - 0, // 7: proto.NezhaService.ReportSystemInfo2:input_type -> proto.Host - 5, // 8: proto.NezhaService.ReportSystemState:output_type -> proto.Receipt - 5, // 9: proto.NezhaService.ReportSystemInfo:output_type -> proto.Receipt - 3, // 10: proto.NezhaService.RequestTask:output_type -> proto.Task - 7, // 11: proto.NezhaService.IOStream:output_type -> proto.IOStreamData - 8, // 12: proto.NezhaService.ReportGeoIP:output_type -> proto.GeoIP - 6, // 13: proto.NezhaService.ReportSystemInfo2:output_type -> proto.Uint64Receipt - 8, // [8:14] is the sub-list for method output_type - 2, // [2:8] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 3, // 0: proto.State.temperatures:type_name -> proto.State_SensorTemperature + 2, // 1: proto.State.gpus:type_name -> proto.State_GPU + 10, // 2: proto.GeoIP.ip:type_name -> proto.IP + 1, // 3: proto.NezhaService.ReportSystemState:input_type -> proto.State + 0, // 4: proto.NezhaService.ReportSystemInfo:input_type -> proto.Host + 5, // 5: proto.NezhaService.RequestTask:input_type -> proto.TaskResult + 8, // 6: proto.NezhaService.IOStream:input_type -> proto.IOStreamData + 9, // 7: proto.NezhaService.ReportGeoIP:input_type -> proto.GeoIP + 0, // 8: proto.NezhaService.ReportSystemInfo2:input_type -> proto.Host + 6, // 9: proto.NezhaService.ReportSystemState:output_type -> proto.Receipt + 6, // 10: proto.NezhaService.ReportSystemInfo:output_type -> proto.Receipt + 4, // 11: proto.NezhaService.RequestTask:output_type -> proto.Task + 8, // 12: proto.NezhaService.IOStream:output_type -> proto.IOStreamData + 9, // 13: proto.NezhaService.ReportGeoIP:output_type -> proto.GeoIP + 7, // 14: proto.NezhaService.ReportSystemInfo2:output_type -> proto.Uint64Receipt + 9, // [9:15] is the sub-list for method output_type + 3, // [3:9] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_proto_nezha_proto_init() } @@ -889,23 +1043,156 @@ func file_proto_nezha_proto_init() { if File_proto_nezha_proto != nil { return } + if !protoimpl.UnsafeEnabled { + file_proto_nezha_proto_msgTypes[0].Exporter = func(v any, i int) any { + switch v := v.(*Host); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_nezha_proto_msgTypes[1].Exporter = func(v any, i int) any { + switch v := v.(*State); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_nezha_proto_msgTypes[2].Exporter = func(v any, i int) any { + switch v := v.(*State_GPU); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_nezha_proto_msgTypes[3].Exporter = func(v any, i int) any { + switch v := v.(*State_SensorTemperature); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_nezha_proto_msgTypes[4].Exporter = func(v any, i int) any { + switch v := v.(*Task); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_nezha_proto_msgTypes[5].Exporter = func(v any, i int) any { + switch v := v.(*TaskResult); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_nezha_proto_msgTypes[6].Exporter = func(v any, i int) any { + switch v := v.(*Receipt); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_nezha_proto_msgTypes[7].Exporter = func(v any, i int) any { + switch v := v.(*Uint64Receipt); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_nezha_proto_msgTypes[8].Exporter = func(v any, i int) any { + switch v := v.(*IOStreamData); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_nezha_proto_msgTypes[9].Exporter = func(v any, i int) any { + switch v := v.(*GeoIP); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_nezha_proto_msgTypes[10].Exporter = func(v any, i int) any { + switch v := v.(*IP); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - // #nosec G103 - RawDescriptor: unsafe.Slice(unsafe.StringData(file_proto_nezha_proto_rawDesc), len(file_proto_nezha_proto_rawDesc)), + RawDescriptor: file_proto_nezha_proto_rawDesc, NumEnums: 0, - NumMessages: 10, + NumMessages: 11, NumExtensions: 0, NumServices: 1, }, - GoTypes: file_proto_nezha_proto_goTypes, DependencyIndexes: file_proto_nezha_proto_depIdxs, MessageInfos: file_proto_nezha_proto_msgTypes, }.Build() File_proto_nezha_proto = out.File + file_proto_nezha_proto_rawDesc = nil file_proto_nezha_proto_goTypes = nil file_proto_nezha_proto_depIdxs = nil } diff --git a/proto/nezha.proto b/proto/nezha.proto index 75debccd..89383b32 100644 --- a/proto/nezha.proto +++ b/proto/nezha.proto @@ -44,6 +44,16 @@ message State { uint64 process_count = 15; repeated State_SensorTemperature temperatures = 16; repeated double gpu = 17; + repeated State_GPU gpus = 18; +} + +// State_GPU carries per-card figures. Index-aligned with Host.gpu, matching +// the existing `gpu` field it supersedes; that field stays populated so older +// dashboards keep working. +message State_GPU { + double utilization = 1; + uint64 memory_used = 2; + uint64 memory_total = 3; } message State_SensorTemperature { diff --git a/service/rpc/nezha.go b/service/rpc/nezha.go index 80d16191..8d8c891a 100644 --- a/service/rpc/nezha.go +++ b/service/rpc/nezha.go @@ -204,7 +204,15 @@ func (s *NezhaHandler) ReportSystemState(stream pb.NezhaService_ReportSystemStat return errors.New("server not found") } lease := server.AttachStateStream(stream) - defer lease.Clear() + defer func() { + // Persist while owning the lease; a replaced stream must never close a + // newer stream's session between Clear and RecordOffline. + lease.ClearWithSideEffect(func() { + if singleton.OnlineHistoryShared != nil { + singleton.OnlineHistoryShared.RecordOffline(clientID, time.Now()) + } + }) + }() var state *pb.State var stateCount uint64 for { @@ -255,6 +263,9 @@ func (s *NezhaHandler) ReportSystemState(stream pb.NezhaService_ReportSystemStat log.Printf("NEZHA>> Failed to write server metrics to TSDB: %v", err) } } + if singleton.OnlineHistoryShared != nil { + singleton.OnlineHistoryShared.RecordOnline(clientID, lastActive) + } return nil }) if !accepted { diff --git a/service/singleton/alertsentinel.go b/service/singleton/alertsentinel.go index b17aeed8..838f936c 100644 --- a/service/singleton/alertsentinel.go +++ b/service/singleton/alertsentinel.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "sync" + "sync/atomic" "time" "github.com/jinzhu/copier" @@ -26,14 +27,125 @@ type NotificationHistory struct { var ( AlertsLock sync.RWMutex Alerts []*model.AlertRule - alertsStore map[uint64]map[uint64][][]bool // [alert_id][server_id] -> [timeTick][ruleId] 时间点对应的rule的检查结果 - alertsPrevState map[uint64]map[uint64]uint8 // [alert_id][server_id] -> 对应报警规则的上一次报警状态 - AlertsCycleTransferStatsStore map[uint64]*model.CycleTransferStats // [alert_id] -> 对应报警规则的周期流量统计 + alertsStore map[uint64]map[uint64][]model.TimedAlertPoint // distinct reports per alert/server + alertsPrevState map[uint64]map[uint64]uint8 // [alert_id][server_id] -> 对应报警规则的上一次报警状态 + alertsLastMetricSeq map[uint64]map[uint64]uint64 // last distinct Agent metric report per alert/server + AlertsCycleTransferStatsStore map[uint64]*model.CycleTransferStats // [alert_id] -> 对应报警规则的周期流量统计 + alertDeliveryMu sync.Mutex + alertDeliveries map[uint64]map[uint64]*alertDeliveryEntry + alertEventSequence atomic.Uint64 ) +type alertDeliveryEntry struct { + cancel chan struct{} + done chan struct{} + sendMu *sync.Mutex +} + +type alertDeliveryPhase uint8 + +const ( + alertDeliveryIncident alertDeliveryPhase = iota + alertDeliveryRecovery +) + +func cancelAlertDeliveries(alertID uint64) { + alertDeliveryMu.Lock() + defer alertDeliveryMu.Unlock() + for _, entry := range alertDeliveries[alertID] { + close(entry.cancel) + } + delete(alertDeliveries, alertID) +} + +func cancelAlertDeliveriesForServers(serverIDs []uint64) { + alertDeliveryMu.Lock() + defer alertDeliveryMu.Unlock() + for alertID, byServer := range alertDeliveries { + for _, serverID := range serverIDs { + if entry := byServer[serverID]; entry != nil { + close(entry.cancel) + delete(byServer, serverID) + } + } + if len(byServer) == 0 { + delete(alertDeliveries, alertID) + } + } +} + +func runAlertDelivery(cancel <-chan struct{}, send func() bool, always bool, retryDelay time.Duration) { + for { + select { + case <-cancel: + return + default: + } + delivered := send() + if delivered && !always { + return + } + delay := retryDelay + if delivered { + delay = firstNotificationDelay + } + timer := time.NewTimer(delay) + select { + case <-cancel: + timer.Stop() + return + case <-timer.C: + } + } +} + +func startAlertDelivery(alert *model.AlertRule, server *model.Server, message, muteLabel string, phase alertDeliveryPhase) { + // A mute entry belongs to one transition, not all later incidents of the + // same phase. Concurrent in-flight sends may finish after a recovery. + muteLabel = fmt.Sprintf("%s:event-%d", muteLabel, alertEventSequence.Add(1)) + alertDeliveryMu.Lock() + if alertDeliveries == nil { + alertDeliveries = make(map[uint64]map[uint64]*alertDeliveryEntry) + } + if alertDeliveries[alert.ID] == nil { + alertDeliveries[alert.ID] = make(map[uint64]*alertDeliveryEntry) + } + sendMu := new(sync.Mutex) + if old := alertDeliveries[alert.ID][server.ID]; old != nil { + close(old.cancel) + sendMu = old.sendMu + } + entry := &alertDeliveryEntry{cancel: make(chan struct{}), done: make(chan struct{}), sendMu: sendMu} + alertDeliveries[alert.ID][server.ID] = entry + alertDeliveryMu.Unlock() + + groupID := alert.NotificationGroupID + always := phase == alertDeliveryIncident && alert.TriggerMode == model.ModeAlwaysTrigger + go func() { + defer close(entry.done) + runAlertDelivery(entry.cancel, func() bool { + // Serialize incident/recovery delivery for this alert/server pair. + // A recovery can cancel a pending incident, but never overtake an + // already-running network request. + entry.sendMu.Lock() + defer entry.sendMu.Unlock() + select { + case <-entry.cancel: + return true + default: + } + return NotificationShared.SendNotification(groupID, message, muteLabel, server) + }, always, 30*time.Second) + }() +} + +func shouldRunAlertFailTasks(triggerMode uint8, newIncident bool) bool { + return newIncident || triggerMode == model.ModeAlwaysTrigger +} + // addCycleTransferStatsInfo 向AlertsCycleTransferStatsStore中添加周期流量报警统计信息 func addCycleTransferStatsInfo(alert *model.AlertRule) { - if !alert.Enabled() { + if alert == nil || !alert.Enabled() || !alert.IsSafeToEvaluate() { return } for _, rule := range alert.Rules { @@ -59,16 +171,34 @@ func addCycleTransferStatsInfo(alert *model.AlertRule) { // AlertSentinelStart 报警器启动 func AlertSentinelStart() { - alertsStore = make(map[uint64]map[uint64][][]bool) + alertDeliveryMu.Lock() + for _, byServer := range alertDeliveries { + for _, entry := range byServer { + close(entry.cancel) + } + } + alertDeliveries = make(map[uint64]map[uint64]*alertDeliveryEntry) + alertDeliveryMu.Unlock() + alertsStore = make(map[uint64]map[uint64][]model.TimedAlertPoint) alertsPrevState = make(map[uint64]map[uint64]uint8) + alertsLastMetricSeq = make(map[uint64]map[uint64]uint64) AlertsCycleTransferStatsStore = make(map[uint64]*model.CycleTransferStats) AlertsLock.Lock() if err := DB.Find(&Alerts).Error; err != nil { panic(err) } for _, alert := range Alerts { - alertsStore[alert.ID] = make(map[uint64][][]bool) + if alert == nil { + log.Printf("NEZHA>> Skipping invalid nil alert rule loaded from database") + continue + } + alertsStore[alert.ID] = make(map[uint64][]model.TimedAlertPoint) alertsPrevState[alert.ID] = make(map[uint64]uint8) + alertsLastMetricSeq[alert.ID] = make(map[uint64]uint64) + if !alert.IsSafeToEvaluate() { + log.Printf("NEZHA>> Skipping invalid alert rule %d loaded from database", alert.ID) + continue + } addCycleTransferStatsInfo(alert) } AlertsLock.Unlock() @@ -93,8 +223,10 @@ func AlertSentinelStart() { func OnRefreshOrAddAlert(alert *model.AlertRule) { AlertsLock.Lock() defer AlertsLock.Unlock() + cancelAlertDeliveries(alert.ID) delete(alertsStore, alert.ID) delete(alertsPrevState, alert.ID) + delete(alertsLastMetricSeq, alert.ID) var isEdit bool for i := range Alerts { if Alerts[i].ID == alert.ID { @@ -105,8 +237,9 @@ func OnRefreshOrAddAlert(alert *model.AlertRule) { if !isEdit { Alerts = append(Alerts, alert) } - alertsStore[alert.ID] = make(map[uint64][][]bool) + alertsStore[alert.ID] = make(map[uint64][]model.TimedAlertPoint) alertsPrevState[alert.ID] = make(map[uint64]uint8) + alertsLastMetricSeq[alert.ID] = make(map[uint64]uint64) delete(AlertsCycleTransferStatsStore, alert.ID) addCycleTransferStatsInfo(alert) } @@ -115,8 +248,10 @@ func OnDeleteAlert(id []uint64) { AlertsLock.Lock() defer AlertsLock.Unlock() for _, i := range id { + cancelAlertDeliveries(i) delete(alertsStore, i) delete(alertsPrevState, i) + delete(alertsLastMetricSeq, i) currentAlerts := Alerts[:0] for _, alert := range Alerts { if alert.ID != i { @@ -136,7 +271,7 @@ func checkStatus() { for _, alert := range Alerts { // 跳过未启用 - if !alert.Enabled() { + if alert == nil || !alert.Enabled() || !alert.IsSafeToEvaluate() { continue } for _, server := range m { @@ -152,48 +287,92 @@ func checkStatus() { if alert.UserID != server.GetUserID() && !role.IsAdmin() { continue } - alertsStore[alert.ID][server.ID] = append(alertsStore[alert. - ID][server.ID], alert.Snapshot(AlertsCycleTransferStatsStore[alert.ID], server, DB)) - // 发送通知，分为触发报警和恢复通知 - _, passed := alert.Check(alertsStore[alert.ID][server.ID]) - // 保存当前服务器状态信息 - curServer := model.Server{} - copier.Copy(&curServer, server) - - // 本次未通过检查 - if !passed { - // 始终触发模式或上次检查不为失败时触发报警（跳过单次触发+上次失败的情况） - if alert.TriggerMode == model.ModeAlwaysTrigger || alertsPrevState[alert.ID][server.ID] != _RuleCheckFail { - alertsPrevState[alert.ID][server.ID] = _RuleCheckFail - message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Incident"), - server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name) - go CronShared.SendTriggerTasks(alert.FailTriggerTasks, curServer.ID, alert.UserID) - go NotificationShared.SendNotification(alert.NotificationGroupID, message, NotificationMuteLabel.ServerIncident(server.ID, alert.ID), &curServer) - // 清除恢复通知的静音缓存 - NotificationShared.UnMuteNotification(alert.NotificationGroupID, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID)) - } - } else { - // 本次通过检查但上一次的状态为失败，则发送恢复通知 - if alertsPrevState[alert.ID][server.ID] == _RuleCheckFail { - message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Resolved"), - server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name) - go CronShared.SendTriggerTasks(alert.RecoverTriggerTasks, curServer.ID, alert.UserID) - go NotificationShared.SendNotification(alert.NotificationGroupID, message, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), &curServer) - // 清除失败通知的静音缓存 - NotificationShared.UnMuteNotification(alert.NotificationGroupID, NotificationMuteLabel.ServerIncident(server.ID, alert.ID)) - } - alertsPrevState[alert.ID][server.ID] = _RuleCheckPass - } - // 清理旧数据：保留窗口由规则定义决定（各规则 Duration 的最大值）， - // 而非 Check 的判定结果。window==0 表示没有任何有效规则需要回看历史 - // （例如全部 Duration<=0），此时清空采样避免切片无限增长。 - window := alert.RetentionWindow() - samples := alertsStore[alert.ID][server.ID] - if window <= 0 { - alertsStore[alert.ID][server.ID] = samples[:0] - } else if window < len(samples) { - alertsStore[alert.ID][server.ID] = samples[len(samples)-window:] - } + checkStatusForServer(alert, server) } } } + +// checkStatusForServer isolates each alert/server evaluation. Input validation +// and model-level guards handle known malformed states; this recovery boundary +// ensures an unexpected evaluator panic cannot terminate the dashboard process +// or prevent unrelated servers from being checked on the same tick. +func checkStatusForServer(alert *model.AlertRule, server *model.Server) { + defer func() { + if recovered := recover(); recovered != nil { + log.Printf("NEZHA>> Recovered panic evaluating alert rule %d for server %d: %v", alert.ID, server.ID, recovered) + } + }() + + // Offline-only rules deliberately observe elapsed time on every tick. Metric + // rules, including compound rules, must never count the same Agent report + // repeatedly or treat a disconnected Agent's retained State as fresh data. + metricRule := false + runtime := server.RuntimeSnapshot() + for _, rule := range alert.Rules { + if !rule.IsOfflineRule() { + metricRule = true + break + } + } + if metricRule { + if runtime.LastActive.IsZero() || runtime.State == nil || time.Since(runtime.LastActive) > model.AlertSampleMaxAge || time.Since(runtime.LastActive) < 0 { + alertsStore[alert.ID][server.ID] = nil + return + } + if runtime.ReportSequence == 0 || runtime.ReportSequence == alertsLastMetricSeq[alert.ID][server.ID] { + return + } + alertsLastMetricSeq[alert.ID][server.ID] = runtime.ReportSequence + } + point, known := alert.SnapshotStatusWithRuntime(AlertsCycleTransferStatsStore[alert.ID], server, runtime, DB) + if !known { + alertsStore[alert.ID][server.ID] = nil + return + } + sampledAt := time.Now() + if metricRule { + sampledAt = runtime.LastActive + } + alertsStore[alert.ID][server.ID] = append(alertsStore[alert.ID][server.ID], model.TimedAlertPoint{At: sampledAt, Values: point}) + // Bound memory even while coverage is insufficient to decide a state. + cutoff := time.Now().Add(-alert.TimedRetention()) + samples := alertsStore[alert.ID][server.ID] + first := 0 + for first < len(samples) && samples[first].At.Before(cutoff) { + first++ + } + if first > 0 { + alertsStore[alert.ID][server.ID] = append([]model.TimedAlertPoint(nil), samples[first:]...) + } + // 发送通知，分为触发报警和恢复通知 + known, passed := alert.CheckTimed(alertsStore[alert.ID][server.ID], time.Now()) + if !known { + return + } + // 保存当前服务器状态信息 + curServer := model.Server{} + copier.Copy(&curServer, server) + + // 本次未通过检查 + if !passed { + newIncident := alertsPrevState[alert.ID][server.ID] != _RuleCheckFail + if newIncident { + alertsPrevState[alert.ID][server.ID] = _RuleCheckFail + message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Incident"), + server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name) + startAlertDelivery(alert, &curServer, message, NotificationMuteLabel.ServerIncident(server.ID, alert.ID), alertDeliveryIncident) + } + if shouldRunAlertFailTasks(alert.TriggerMode, newIncident) { + go CronShared.SendTriggerTasks(alert.FailTriggerTasks, curServer.ID, alert.UserID) + } + } else { + // 本次通过检查但上一次的状态为失败，则发送恢复通知 + if alertsPrevState[alert.ID][server.ID] == _RuleCheckFail { + message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Resolved"), + server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name) + go CronShared.SendTriggerTasks(alert.RecoverTriggerTasks, curServer.ID, alert.UserID) + startAlertDelivery(alert, &curServer, message, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), alertDeliveryRecovery) + } + alertsPrevState[alert.ID][server.ID] = _RuleCheckPass + } +} diff --git a/service/singleton/alertsentinel_freshness_test.go b/service/singleton/alertsentinel_freshness_test.go new file mode 100644 index 00000000..b68aeae5 --- /dev/null +++ b/service/singleton/alertsentinel_freshness_test.go @@ -0,0 +1,281 @@ +package singleton + +import ( + "context" + "testing" + "time" + + "github.com/patrickmn/go-cache" + "google.golang.org/grpc/metadata" + + "github.com/nezhahq/nezha/model" + pb "github.com/nezhahq/nezha/proto" +) + +type alertTestStateStream struct{} + +func (alertTestStateStream) Send(*pb.Receipt) error { return nil } +func (alertTestStateStream) Recv() (*pb.State, error) { return nil, context.Canceled } +func (alertTestStateStream) SetHeader(metadata.MD) error { return nil } +func (alertTestStateStream) SendHeader(metadata.MD) error { return nil } +func (alertTestStateStream) SetTrailer(metadata.MD) {} +func (alertTestStateStream) Context() context.Context { return context.Background() } +func (alertTestStateStream) SendMsg(any) error { return nil } +func (alertTestStateStream) RecvMsg(any) error { return nil } + +func TestAlertSentinelCountsOnlyFreshDistinctMetricReports(t *testing.T) { + oldStore, oldPrev, oldLast, oldCycle := alertsStore, alertsPrevState, alertsLastMetricSeq, AlertsCycleTransferStatsStore + alertsStore = map[uint64]map[uint64][]model.TimedAlertPoint{7: {1: nil}} + alertsPrevState = map[uint64]map[uint64]uint8{7: {}} + alertsLastMetricSeq = map[uint64]map[uint64]uint64{7: {}} + AlertsCycleTransferStatsStore = map[uint64]*model.CycleTransferStats{} + t.Cleanup(func() { + alertsStore, alertsPrevState, alertsLastMetricSeq, AlertsCycleTransferStatsStore = oldStore, oldPrev, oldLast, oldCycle + }) + + alert := &model.AlertRule{ + Common: model.Common{ID: 7}, + Rules: []*model.Rule{{Type: "cpu", Max: 80, Duration: 100, Cover: model.RuleCoverAll}}, + } + server := &model.Server{Common: model.Common{ID: 1}} + model.InitServer(server) + lease := server.AttachStateStream(alertTestStateStream{}) + fixedReportTime := time.Now() + if !lease.UpdateState(&model.HostState{CPU: 99}, fixedReportTime) { + t.Fatal("state update rejected") + } + checkStatusForServer(alert, server) + if got := len(alertsStore[7][1]); got != 1 { + t.Fatalf("first report produced %d samples, want 1", got) + } + for i := 0; i < 100; i++ { + checkStatusForServer(alert, server) + } + if got := len(alertsStore[7][1]); got != 1 { + t.Fatalf("same report was counted %d times, want 1", got) + } + if !lease.Clear() { + t.Fatal("stream clear rejected") + } + checkStatusForServer(alert, server) + if got := len(alertsStore[7][1]); got != 0 { + t.Fatalf("disconnect retained %d high samples, want 0", got) + } + alertsPrevState[7][1] = _RuleCheckFail // incident was active before disconnect + newLease := server.AttachStateStream(alertTestStateStream{}) + // Distinct reports may share a clock timestamp on coarse-resolution hosts. + if !newLease.UpdateState(&model.HostState{CPU: 99}, fixedReportTime) { + t.Fatal("reconnect state update rejected") + } + checkStatusForServer(alert, server) + if got := len(alertsStore[7][1]); got != 1 || alertsStore[7][1][0].Values[0] { + t.Fatalf("reconnect window = %v, want one high sample", alertsStore[7][1]) + } + if alertsPrevState[7][1] != _RuleCheckFail { + t.Fatal("incomplete warm-up falsely resolved active incident") + } + replacement := &model.Server{Common: model.Common{ID: 1}} + model.InitServer(replacement) + replacementLease := replacement.AttachStateStream(alertTestStateStream{}) + if !replacementLease.UpdateState(&model.HostState{CPU: 99}, fixedReportTime) { + t.Fatal("replacement server state update rejected") + } + checkStatusForServer(alert, replacement) + if got := len(alertsStore[7][1]); got != 2 { + t.Fatalf("replacement holder reused a report identity; samples=%d", got) + } +} + +func TestAlertNewIncidentNotMutedByEarlierInflightIncident(t *testing.T) { + oldCache, oldNotifications := Cache, NotificationShared + Cache = cache.New(time.Minute, time.Minute) + n := &model.Notification{Common: model.Common{ID: 601}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 603}, Name: "alert-test"}, []uint64{n.ID}) + NotificationShared = nc + t.Cleanup(func() { + cancelAlertDeliveries(602) + NotificationShared, Cache = oldNotifications, oldCache + }) + started := make(chan struct{}) + release := make(chan struct{}) + delivered := make(chan string, 3) + nc.sendForTest = func(_ *model.Notification, desc string, _ *model.Server) error { + if desc == "first incident" { + close(started) + <-release + } + delivered <- desc + return nil + } + alert := &model.AlertRule{ + Common: model.Common{ID: 602}, + NotificationGroupID: 603, + TriggerMode: model.ModeOnetimeTrigger, + } + server := &model.Server{Common: model.Common{ID: 604}} + label := NotificationMuteLabel.ServerIncident(server.ID, alert.ID) + startAlertDelivery(alert, server, "first incident", label, alertDeliveryIncident) + <-started + startAlertDelivery(alert, server, "recovered", NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), alertDeliveryRecovery) + startAlertDelivery(alert, server, "second incident", label, alertDeliveryIncident) + close(release) + timer := time.NewTimer(time.Second) + defer timer.Stop() + seen := map[string]bool{} + for len(seen) < 2 { + select { + case message := <-delivered: + seen[message] = true + case <-timer.C: + t.Fatalf("new incident was suppressed after old send completed: %v", seen) + } + } + if !seen["first incident"] || !seen["second incident"] || seen["recovered"] { + t.Fatalf("unexpected transition deliveries: %v", seen) + } + alertDeliveryMu.Lock() + done := alertDeliveries[alert.ID][server.ID].done + alertDeliveryMu.Unlock() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("new incident delivery did not finish") + } +} + +func TestAlertRecoveryDeliveryStopsAfterSuccessInAlwaysMode(t *testing.T) { + oldCache, oldNotifications := Cache, NotificationShared + Cache = cache.New(time.Minute, time.Minute) + n := &model.Notification{Common: model.Common{ID: 611}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 613}, Name: "recovery-test"}, []uint64{n.ID}) + NotificationShared = nc + t.Cleanup(func() { + cancelAlertDeliveries(612) + NotificationShared, Cache = oldNotifications, oldCache + }) + + delivered := make(chan struct{}, 2) + nc.sendForTest = func(*model.Notification, string, *model.Server) error { + delivered <- struct{}{} + return nil + } + alert := &model.AlertRule{ + Common: model.Common{ID: 612}, + NotificationGroupID: 613, + TriggerMode: model.ModeAlwaysTrigger, + } + server := &model.Server{Common: model.Common{ID: 614}} + startAlertDelivery(alert, server, "recovered", "recovery-event", alertDeliveryRecovery) + + select { + case <-delivered: + case <-time.After(time.Second): + t.Fatal("recovery notification was not delivered") + } + alertDeliveryMu.Lock() + done := alertDeliveries[alert.ID][server.ID].done + alertDeliveryMu.Unlock() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("successful recovery delivery kept repeating in always mode") + } + select { + case <-delivered: + t.Fatal("recovery notification was delivered more than once") + default: + } +} + +func TestServerDeleteCancelsAlertDelivery(t *testing.T) { + alertDeliveryMu.Lock() + oldDeliveries := alertDeliveries + cancel := make(chan struct{}) + alertDeliveries = map[uint64]map[uint64]*alertDeliveryEntry{ + 621: {622: {cancel: cancel, done: make(chan struct{})}}, + } + alertDeliveryMu.Unlock() + t.Cleanup(func() { + alertDeliveryMu.Lock() + alertDeliveries = oldDeliveries + alertDeliveryMu.Unlock() + }) + + servers := &ServerClass{ + class: class[uint64, *model.Server]{ + list: map[uint64]*model.Server{ + 622: {Common: model.Common{ID: 622}, UUID: "delete-alert-worker"}, + }, + }, + uuidToID: map[string]uint64{"delete-alert-worker": 622}, + } + servers.Delete([]uint64{622}) + + select { + case <-cancel: + default: + t.Fatal("server deletion did not cancel its alert delivery") + } + alertDeliveryMu.Lock() + _, retained := alertDeliveries[621] + alertDeliveryMu.Unlock() + if retained { + t.Fatal("server deletion retained an empty alert delivery map") + } +} + +func TestAlwaysAlertContinuesFailureTriggerTasks(t *testing.T) { + if !shouldRunAlertFailTasks(model.ModeAlwaysTrigger, false) { + t.Fatal("always mode stopped failure trigger tasks after the initial incident") + } + if shouldRunAlertFailTasks(model.ModeOnetimeTrigger, false) { + t.Fatal("one-time mode repeated failure trigger tasks") + } + if !shouldRunAlertFailTasks(model.ModeOnetimeTrigger, true) { + t.Fatal("one-time mode skipped failure trigger tasks for a new incident") + } +} + +func TestAlertDeliveryRetriesFailureAndStopsAfterSuccess(t *testing.T) { + cancel := make(chan struct{}) + done := make(chan struct{}) + attempts := 0 + go func() { + runAlertDelivery(cancel, func() bool { + attempts++ + return attempts == 2 + }, false, time.Millisecond) + close(done) + }() + select { + case <-done: + case <-time.After(time.Second): + close(cancel) + t.Fatal("delivery did not retry and finish") + } + if attempts != 2 { + t.Fatalf("got %d attempts, want 2", attempts) + } +} + +func TestAlertDeliveryCancelsAfterRecovery(t *testing.T) { + cancel := make(chan struct{}) + done := make(chan struct{}) + started := make(chan struct{}) + go func() { + runAlertDelivery(cancel, func() bool { + close(started) + return true + }, true, time.Millisecond) + close(done) + }() + <-started + close(cancel) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("recovered incident delivery was not cancelled") + } +} diff --git a/service/singleton/alertsentinel_test.go b/service/singleton/alertsentinel_test.go index 1d290523..a78bf334 100644 --- a/service/singleton/alertsentinel_test.go +++ b/service/singleton/alertsentinel_test.go @@ -117,3 +117,53 @@ func TestCheckStatus_SampleMemoryBounded(t *testing.T) { t.Fatalf("sample capacity grew unbounded: peak cap %d exceeds 4x window %d", peakCap, duration*4) } } + +func TestCheckStatusForServerRecoversUnexpectedEvaluatorPanic(t *testing.T) { + originalStore := alertsStore + originalPrevState := alertsPrevState + originalCycleStore := AlertsCycleTransferStatsStore + alertsStore = map[uint64]map[uint64][]model.TimedAlertPoint{} + alertsPrevState = map[uint64]map[uint64]uint8{} + AlertsCycleTransferStatsStore = map[uint64]*model.CycleTransferStats{} + t.Cleanup(func() { + alertsStore = originalStore + alertsPrevState = originalPrevState + AlertsCycleTransferStatsStore = originalCycleStore + }) + + enabled := true + alert := &model.AlertRule{ + Common: model.Common{ID: 77, UserID: 2}, + Enable: &enabled, + Rules: []*model.Rule{{Type: "cpu", Duration: 3, Cover: model.RuleCoverAll}}, + } + server := &model.Server{Common: model.Common{ID: 1, UserID: 2}} + model.InitServer(server) + + // The deliberately missing alertsStore[alert.ID] map panics at assignment + // after evaluation. The per-alert/server recovery boundary must contain it. + checkStatusForServer(alert, server) +} + +func TestAddCycleTransferStatsSkipsPoisonedPersistedRule(t *testing.T) { + original := AlertsCycleTransferStatsStore + AlertsCycleTransferStatsStore = map[uint64]*model.CycleTransferStats{} + t.Cleanup(func() { AlertsCycleTransferStatsStore = original }) + + enabled := true + alert := &model.AlertRule{ + Common: model.Common{ID: 88}, + Enable: &enabled, + Rules: []*model.Rule{{ + Type: "transfer_in_cycle", + CycleInterval: 1, + Cover: model.RuleCoverAll, + // CycleStart intentionally nil, as it may be in a poisoned legacy DB row. + }}, + } + + addCycleTransferStatsInfo(alert) + if _, exists := AlertsCycleTransferStatsStore[alert.ID]; exists { + t.Fatal("invalid persisted cycle rule must not be scheduled") + } +} diff --git a/service/singleton/config.go b/service/singleton/config.go index 65f19687..6dcd1072 100644 --- a/service/singleton/config.go +++ b/service/singleton/config.go @@ -4,6 +4,7 @@ import ( "log" "strconv" "strings" + "sync" "github.com/nezhahq/nezha/model" "github.com/nezhahq/nezha/pkg/utils" @@ -14,6 +15,8 @@ var Conf *ConfigClass type ConfigClass struct { *model.Config + onlineHistoryRetentionMu sync.RWMutex + IgnoredIPNotificationServerIDs map[uint64]bool `json:"ignored_ip_notification_server_ids,omitempty"` Oauth2Providers []string `json:"oauth2_providers,omitempty"` } @@ -59,3 +62,39 @@ func (c *ConfigClass) updateIgnoredIPNotificationID() { } } } + +// OnlineHistoryRetentionDays returns the retention policy visible to queries and pruning. +func (c *ConfigClass) OnlineHistoryRetentionDays() int { + c.onlineHistoryRetentionMu.RLock() + defer c.onlineHistoryRetentionMu.RUnlock() + return c.OnlineStatsDays +} + +// CommitOnlineHistoryRetention applies a retention change with the settings save. +func (c *ConfigClass) CommitOnlineHistoryRetention(days *int, persist func() error) error { + // Readers must not capture a shorter cutoff while Save can still fail. + // Serialize assignment, persistence and rollback with the policy snapshot. + c.onlineHistoryRetentionMu.Lock() + defer c.onlineHistoryRetentionMu.Unlock() + previous := c.OnlineStatsDays + if days != nil { + c.OnlineStatsDays = *days + } + if err := persist(); err != nil { + c.OnlineStatsDays = previous + return err + } + return nil +} + +// SettingsSnapshot copies response fields without copying the retention mutex. +func (c *ConfigClass) SettingsSnapshot() model.Setting { + c.onlineHistoryRetentionMu.RLock() + defer c.onlineHistoryRetentionMu.RUnlock() + return model.Setting{ + ConfigForGuests: c.ConfigForGuests, + ConfigDashboard: c.ConfigDashboard, + IgnoredIPNotificationServerIDs: c.IgnoredIPNotificationServerIDs, + Oauth2Providers: c.Oauth2Providers, + } +} diff --git a/service/singleton/frontend-templates.yaml b/service/singleton/frontend-templates.yaml index 050c0f1b..d31147c3 100644 --- a/service/singleton/frontend-templates.yaml +++ b/service/singleton/frontend-templates.yaml @@ -2,24 +2,20 @@ name: "OfficialAdmin" repository: "https://github.com/nezhahq/admin-frontend" author: "nezhahq" - version: "v2.3.4" + version: "v2.4.0" is_admin: true is_official: true - path: "user-dist" name: "Official" repository: "https://github.com/hamster1963/nezha-dash-v2" author: "hamster1963" - version: "v2.4.2" + version: "v2.5.1" is_official: true - path: "nezha-pixel-dist" name: "Nezha-Pixel" repository: "https://github.com/karllao/nezha-pixel" author: "karllao" version: "v1.6.0" -# Third-party user themes consume the opaque Server.PublicNote field. Theme -# maintainers must validate URL schemes (including after decoding) before using -# values such as customData.orderLink in href or window.open; the Dashboard -# backend and admin frontend do not execute those fields. - path: "nazhua-dist" name: "Nazhua" repository: "https://github.com/hi2shark/nazhua" @@ -29,7 +25,7 @@ name: "Aobobo" repository: "https://github.com/hi2shark/aobobo" author: "hi2shark" - version: "v1.5.1" + version: "v1.5.4" - path: "nezha-ascii-dist" name: "Nezha-ASCII" repository: "https://github.com/hamster1963/nezha-ascii" diff --git a/service/singleton/notification.go b/service/singleton/notification.go index 0e63be78..4cf7e661 100644 --- a/service/singleton/notification.go +++ b/service/singleton/notification.go @@ -58,8 +58,228 @@ type NotificationClass struct { groupToIDList map[uint64]map[uint64]*model.Notification idToGroupList map[uint64]map[uint64]struct{} - groupList map[uint64]string - groupMu sync.RWMutex + groupList map[uint64]string + groupMu sync.RWMutex + deliveryMu sync.Mutex + deliveryKeys map[string]*notificationKeyLock + asyncInFlight map[string]struct{} + asyncRetryAfter map[string]time.Time + asyncRetryDelay map[string]time.Duration + muteGeneration map[string]uint64 + serviceDelivery map[uint64]*orderedServiceDelivery + sendForTest func(*model.Notification, string, *model.Server) error +} + +type serviceDeliveryRequest struct { + groupID uint64 + desc string + label string +} + +type orderedServiceDelivery struct { + current serviceDeliveryRequest + pending *serviceDeliveryRequest + update chan struct{} + cancel chan struct{} + done chan struct{} +} + +// SendServiceState preserves incident→recovery order without allowing a +// slow webhook to accumulate one goroutine per probe result. Only the latest +// pending phase is retained; a failed current phase retries with backoff. +func (c *NotificationClass) SendServiceState(serviceID, groupID uint64, desc, label string) { + req := serviceDeliveryRequest{groupID: groupID, desc: desc, label: label} + c.deliveryMu.Lock() + if c.serviceDelivery == nil { + c.serviceDelivery = make(map[uint64]*orderedServiceDelivery) + } + state := c.serviceDelivery[serviceID] + if state == nil { + state = &orderedServiceDelivery{current: req, update: make(chan struct{}, 1), cancel: make(chan struct{}), done: make(chan struct{})} + c.serviceDelivery[serviceID] = state + c.deliveryMu.Unlock() + go c.runServiceDelivery(serviceID, state) + return + } + if state.pending == nil && state.current.label == label { + c.deliveryMu.Unlock() + return + } + if state.pending == nil || state.pending.label != label { + state.pending = &req + select { + case state.update <- struct{}{}: + default: + } + } + c.deliveryMu.Unlock() +} + +func (c *NotificationClass) runServiceDelivery(serviceID uint64, state *orderedServiceDelivery) { + defer close(state.done) + delay := 30 * time.Second + for { + select { + case <-state.cancel: + return + default: + } + delivered := c.SendNotification(state.current.groupID, state.current.desc, state.current.label) + c.deliveryMu.Lock() + // Do not advance to a later phase until every recipient has received the + // current one. Otherwise a failed incident can be discarded as soon as a + // recovery is queued, leaving that recipient with a recovery but no incident. + if delivered && state.pending != nil { + next := state.pending + state.current = *next + state.pending = nil + delay = 30 * time.Second + select { + case <-state.update: + default: + } + c.deliveryMu.Unlock() + continue + } + if delivered { + if c.serviceDelivery[serviceID] == state { + delete(c.serviceDelivery, serviceID) + } + c.deliveryMu.Unlock() + return + } + c.deliveryMu.Unlock() + timer := time.NewTimer(delay) + select { + case <-state.cancel: + timer.Stop() + return + case <-state.update: + timer.Stop() + case <-timer.C: + delay = min(delay*2, 15*time.Minute) + } + } +} + +func (c *NotificationClass) CancelServiceState(serviceID uint64) { + if c == nil { + return + } + c.deliveryMu.Lock() + if state := c.serviceDelivery[serviceID]; state != nil { + close(state.cancel) + delete(c.serviceDelivery, serviceID) + } + c.deliveryMu.Unlock() +} + +// SendNotificationAsync coalesces concurrent service-probe reports for the +// same event before allocating a webhook goroutine. Alert transitions use the +// synchronous SendNotification path so they retain their retry semantics. +func (c *NotificationClass) SendNotificationAsync(groupID uint64, desc, muteLabel string, ext ...*model.Server) { + c.sendNotificationAsync(groupID, desc, muteLabel, nil, ext...) +} + +func (c *NotificationClass) sendNotificationAsync(groupID uint64, desc, muteLabel string, onSuccess func(), ext ...*model.Server) { + key := fmt.Sprintf("%d:%s", groupID, muteLabel) + c.deliveryMu.Lock() + if c.asyncInFlight == nil { + c.asyncInFlight = make(map[string]struct{}) + } + if _, busy := c.asyncInFlight[key]; busy { + c.deliveryMu.Unlock() + return + } + if time.Now().Before(c.asyncRetryAfter[key]) { + c.deliveryMu.Unlock() + return + } + c.asyncInFlight[key] = struct{}{} + c.deliveryMu.Unlock() + go func() { + delivered := c.SendNotification(groupID, desc, muteLabel, ext...) + if delivered && onSuccess != nil { + onSuccess() + } + c.deliveryMu.Lock() + if delivered { + delete(c.asyncRetryAfter, key) + delete(c.asyncRetryDelay, key) + } else { + if c.asyncRetryAfter == nil { + c.asyncRetryAfter = make(map[string]time.Time) + c.asyncRetryDelay = make(map[string]time.Duration) + } + delay := c.asyncRetryDelay[key] + if delay == 0 { + delay = 30 * time.Second + } else { + delay = min(delay*2, 15*time.Minute) + } + c.asyncRetryDelay[key] = delay + c.asyncRetryAfter[key] = time.Now().Add(delay) + c.pruneAsyncBackoffLocked() + } + delete(c.asyncInFlight, key) + c.deliveryMu.Unlock() + }() +} + +const maxAsyncBackoffKeys = 4096 + +// Bound memory when a failing webhook sees many short-lived event labels. +// Evicted old event keys are no longer active after their phase changes. +func (c *NotificationClass) pruneAsyncBackoffLocked() { + if len(c.asyncRetryAfter) <= maxAsyncBackoffKeys { + return + } + now := time.Now() + for key, until := range c.asyncRetryAfter { + if !now.Before(until) { + delete(c.asyncRetryAfter, key) + delete(c.asyncRetryDelay, key) + } + } + for key := range c.asyncRetryAfter { + if len(c.asyncRetryAfter) <= maxAsyncBackoffKeys { + break + } + delete(c.asyncRetryAfter, key) + delete(c.asyncRetryDelay, key) + } +} + +type notificationKeyLock struct { + mu sync.Mutex + refs int +} + +func (c *NotificationClass) lockDeliveryKey(key string) func() { + if key == "" { + return func() {} + } + c.deliveryMu.Lock() + if c.deliveryKeys == nil { + c.deliveryKeys = make(map[string]*notificationKeyLock) + } + entry := c.deliveryKeys[key] + if entry == nil { + entry = new(notificationKeyLock) + c.deliveryKeys[key] = entry + } + entry.refs++ + c.deliveryMu.Unlock() + entry.mu.Lock() + return func() { + entry.mu.Unlock() + c.deliveryMu.Lock() + entry.refs-- + if entry.refs == 0 { + delete(c.deliveryKeys, key) + } + c.deliveryMu.Unlock() + } } func NewNotificationClass() *NotificationClass { @@ -117,20 +337,29 @@ func NewNotificationClass() *NotificationClass { } func (c *NotificationClass) Update(n *model.Notification) { - c.listMu.Lock() + func() { + c.listMu.Lock() + defer c.listMu.Unlock() - _, ok := c.list[n.ID] - c.list[n.ID] = n - - if ok { - if gids, ok := c.idToGroupList[n.ID]; ok { - for gid := range gids { - c.groupToIDList[gid][n.ID] = n - } + _, ok := c.list[n.ID] + c.list[n.ID] = n + if !ok { + return } - } - c.listMu.Unlock() + gids := c.idToGroupList[n.ID] + for gid := range gids { + group, exists := c.groupToIDList[gid] + if !exists { + delete(gids, gid) + continue + } + group[n.ID] = n + } + if len(gids) == 0 { + delete(c.idToGroupList, n.ID) + } + }() c.sortList() } @@ -138,71 +367,69 @@ func (c *NotificationClass) UpdateGroup(ng *model.NotificationGroup, ngn []uint6 c.groupMu.Lock() defer c.groupMu.Unlock() - _, ok := c.groupList[ng.ID] c.groupList[ng.ID] = ng.Name c.listMu.Lock() defer c.listMu.Unlock() - if !ok { - c.groupToIDList[ng.ID] = make(map[uint64]*model.Notification, len(ngn)) - for _, n := range ngn { - if c.idToGroupList[n] == nil { - c.idToGroupList[n] = make(map[uint64]struct{}) - } - c.idToGroupList[n][ng.ID] = struct{}{} - c.groupToIDList[ng.ID][n] = c.list[n] - } - } else { - oldList := make(map[uint64]struct{}) - for nid := range c.groupToIDList[ng.ID] { - oldList[nid] = struct{}{} - } - c.groupToIDList[ng.ID] = make(map[uint64]*model.Notification) - for _, nid := range ngn { - c.groupToIDList[ng.ID][nid] = c.list[nid] - if c.idToGroupList[nid] == nil { - c.idToGroupList[nid] = make(map[uint64]struct{}) - } - c.idToGroupList[nid][ng.ID] = struct{}{} + oldList := c.groupToIDList[ng.ID] + newList := make(map[uint64]*model.Notification, len(ngn)) + for _, nid := range ngn { + n, ok := c.list[nid] + if !ok { + continue } + newList[nid] = n + if c.idToGroupList[nid] == nil { + c.idToGroupList[nid] = make(map[uint64]struct{}) + } + c.idToGroupList[nid][ng.ID] = struct{}{} + } - for oldID := range oldList { - if _, ok := c.groupToIDList[ng.ID][oldID]; !ok { - delete(c.groupToIDList[oldID], ng.ID) - if len(c.idToGroupList[oldID]) == 0 { - delete(c.idToGroupList, oldID) - } - } + for oldID := range oldList { + if _, ok := newList[oldID]; ok { + continue + } + delete(c.idToGroupList[oldID], ng.ID) + if len(c.idToGroupList[oldID]) == 0 { + delete(c.idToGroupList, oldID) } } + c.groupToIDList[ng.ID] = newList } func (c *NotificationClass) Delete(idList []uint64) { - c.listMu.Lock() + func() { + c.listMu.Lock() + defer c.listMu.Unlock() - for _, id := range idList { - delete(c.list, id) - // 如果绑定了通知组才删除 - if gids, ok := c.idToGroupList[id]; ok { - for gid := range gids { - delete(c.groupToIDList[gid], id) + for _, id := range idList { + delete(c.list, id) + // 如果绑定了通知组才删除 + if gids, ok := c.idToGroupList[id]; ok { + for gid := range gids { + delete(c.groupToIDList[gid], id) + } delete(c.idToGroupList, id) } } - } - - c.listMu.Unlock() + }() c.sortList() } func (c *NotificationClass) DeleteGroup(gids []uint64) { - c.listMu.Lock() - defer c.listMu.Unlock() c.groupMu.Lock() defer c.groupMu.Unlock() + c.listMu.Lock() + defer c.listMu.Unlock() for _, gid := range gids { + for nid := range c.groupToIDList[gid] { + delete(c.idToGroupList[nid], gid) + if len(c.idToGroupList[nid]) == 0 { + delete(c.idToGroupList, nid) + } + } delete(c.groupList, gid) delete(c.groupToIDList, gid) } @@ -231,66 +458,115 @@ func (c *NotificationClass) sortList() { func (c *NotificationClass) UnMuteNotification(notificationGroupID uint64, muteLabel string) { fullMuteLabel := NotificationMuteLabel.AppendNotificationGroupName(muteLabel, c.GetGroupName(notificationGroupID)) - Cache.Delete(fullMuteLabel) + c.listMu.RLock() + for id := range c.groupToIDList[notificationGroupID] { + key := notificationMethodMuteLabel(fullMuteLabel, id) + c.deliveryMu.Lock() + if c.muteGeneration == nil { + c.muteGeneration = make(map[string]uint64) + } + c.muteGeneration[key]++ + Cache.Delete(key) + c.deliveryMu.Unlock() + } + c.listMu.RUnlock() +} + +func notificationMethodMuteLabel(label string, id uint64) string { + return fmt.Sprintf("%s:method-%d", label, id) } // SendNotification 向指定的通知方式组的所有通知方式发送通知 -func (c *NotificationClass) SendNotification(notificationGroupID uint64, desc string, muteLabel string, ext ...*model.Server) { +func (c *NotificationClass) SendNotification(notificationGroupID uint64, desc string, muteLabel string, ext ...*model.Server) bool { + fullMuteLabel := "" if muteLabel != "" { - // 将通知方式组名称加入静音标志 - muteLabel := NotificationMuteLabel.AppendNotificationGroupName(muteLabel, c.GetGroupName(notificationGroupID)) - // 通知防骚扰策略 - var flag bool - if cacheN, has := Cache.Get(muteLabel); has { - nHistory := cacheN.(NotificationHistory) - // 每次提醒都增加一倍等待时间，最后每天最多提醒一次 - if time.Now().After(nHistory.Until) { - flag = true - nHistory.Duration *= 2 - if nHistory.Duration > time.Hour*24 { - nHistory.Duration = time.Hour * 24 - } - nHistory.Until = time.Now().Add(nHistory.Duration) - // 缓存有效期加 10 分钟 - Cache.Set(muteLabel, nHistory, nHistory.Duration+time.Minute*10) - } - } else { - // 新提醒直接通知 - flag = true - Cache.Set(muteLabel, NotificationHistory{ - Duration: firstNotificationDelay, - Until: time.Now().Add(firstNotificationDelay), - }, firstNotificationDelay+time.Minute*10) - } - - if !flag { - if Conf.Debug { - log.Println("NEZHA>> Muted repeated notification", desc, muteLabel) - } - return - } + fullMuteLabel = NotificationMuteLabel.AppendNotificationGroupName(muteLabel, c.GetGroupName(notificationGroupID)) } - // 向该通知方式组的所有通知方式发出通知 + // Copy the group under the lock. Webhook delivery can take minutes and must + // not block notification or notification-group updates while it is in flight. c.listMu.RLock() - defer c.listMu.RUnlock() + notifications := make([]*model.Notification, 0, len(c.groupToIDList[notificationGroupID])) for _, n := range c.groupToIDList[notificationGroupID] { - log.Printf("NEZHA>> Try to notify %s", n.Name) + notifications = append(notifications, n) } - for _, n := range c.groupToIDList[notificationGroupID] { - ns := model.NotificationServerBundle{ - Notification: n, - Server: nil, - Loc: Loc, - } + c.listMu.RUnlock() + + if len(notifications) == 0 { + log.Printf("NEZHA>> Notification group %d has no recipients", notificationGroupID) + return false + } + allDelivered := true + for _, n := range notifications { + var server *model.Server if len(ext) > 0 { - ns.Server = ext[0] + server = ext[0] } - if err := ns.Send(desc); err != nil { - log.Printf("NEZHA>> Sending notification to %s failed: %v", n.Name, err) - } else { - log.Printf("NEZHA>> Sending notification to %s succeeded", n.Name) + if !c.sendRecipient(n, desc, fullMuteLabel, server) { + allDelivered = false } } + return allDelivered +} + +func (c *NotificationClass) sendRecipient(n *model.Notification, desc, fullMuteLabel string, server *model.Server) bool { + if n == nil { + return false + } + cacheKey := "" + if fullMuteLabel != "" { + cacheKey = notificationMethodMuteLabel(fullMuteLabel, n.ID) + } + unlock := c.lockDeliveryKey(cacheKey) + defer unlock() + var generation uint64 + if cacheKey != "" { + c.deliveryMu.Lock() + generation = c.muteGeneration[cacheKey] + c.deliveryMu.Unlock() + } + var history NotificationHistory + if cacheKey != "" { + if value, exists := Cache.Get(cacheKey); exists { + history = value.(NotificationHistory) + if time.Now().Before(history.Until) { + return true // this recipient already received this phase + } + } + } + log.Printf("NEZHA>> Try to notify %s", n.Name) + var err error + if c.sendForTest != nil { + err = c.sendForTest(n, desc, server) + } else { + err = (&model.NotificationServerBundle{Notification: n, Server: server, Loc: Loc}).Send(desc) + } + if err != nil { + // Webhook errors can contain a credential-bearing request URL. Keep + // failure type without ever writing the URL/token to dashboard logs. + log.Printf("NEZHA>> Sending notification to %s failed (%s)", n.Name, notificationFailureSummary(err)) + return false + } + log.Printf("NEZHA>> Sending notification to %s succeeded", n.Name) + if cacheKey != "" { + if history.Duration == 0 { + history.Duration = firstNotificationDelay + } else { + history.Duration = min(history.Duration*2, 24*time.Hour) + } + history.Until = time.Now().Add(history.Duration) + c.deliveryMu.Lock() + if c.muteGeneration[cacheKey] == generation { + Cache.Set(cacheKey, history, history.Duration+10*time.Minute) + } + c.deliveryMu.Unlock() + } + return true +} + +func notificationFailureSummary(err error) string { + // Even wrapped URL errors can include credentials in path/query. Log only + // the error type; the HTTP status path already returns a dedicated type. + return fmt.Sprintf("%T", err) } func sendSMTPAdminNotification(message string) error { diff --git a/service/singleton/notification_test.go b/service/singleton/notification_test.go new file mode 100644 index 00000000..39ea4102 --- /dev/null +++ b/service/singleton/notification_test.go @@ -0,0 +1,519 @@ +package singleton + +import ( + "errors" + "fmt" + "net/url" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/patrickmn/go-cache" + + "github.com/nezhahq/nezha/model" +) + +func TestNotificationFailureDoesNotMuteRetryOrResendSuccessfulRecipient(t *testing.T) { + oldCache := Cache + Cache = cache.New(time.Minute, time.Minute) + t.Cleanup(func() { Cache = oldCache }) + first := &model.Notification{Common: model.Common{ID: 301}, Name: "first"} + second := &model.Notification{Common: model.Common{ID: 302}, Name: "second"} + nc := newNotificationClassWithItems(first, second) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 303}, Name: "retry-test"}, []uint64{first.ID, second.ID}) + counts := map[uint64]int{} + nc.sendForTest = func(n *model.Notification, _ string, _ *model.Server) error { + counts[n.ID]++ + if n.ID == second.ID && counts[n.ID] == 1 { + return errors.New("temporary failure") + } + return nil + } + label := "alert-retry-test" + nc.UnMuteNotification(303, label) + if nc.SendNotification(303, "incident", label) { + t.Fatal("partial delivery must report failure") + } + if !nc.SendNotification(303, "incident", label) { + t.Fatal("failed recipient should be retried successfully") + } + if counts[first.ID] != 1 || counts[second.ID] != 2 { + t.Fatalf("delivery attempts = %v, want first=1 second=2", counts) + } + nc.UnMuteNotification(303, label) +} + +func TestNotificationConcurrentSameRecipientSendsOnce(t *testing.T) { + oldCache := Cache + Cache = cache.New(time.Minute, time.Minute) + t.Cleanup(func() { Cache = oldCache }) + n := &model.Notification{Common: model.Common{ID: 401}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 402}, Name: "concurrent-test"}, []uint64{n.ID}) + entered := make(chan struct{}) + release := make(chan struct{}) + var calls atomic.Int32 + nc.sendForTest = func(*model.Notification, string, *model.Server) error { + if calls.Add(1) == 1 { + close(entered) + <-release + } + return nil + } + done := make(chan bool, 2) + go func() { done <- nc.SendNotification(402, "incident", "same-event") }() + <-entered + go func() { done <- nc.SendNotification(402, "incident", "same-event") }() + close(release) + if !<-done || !<-done { + t.Fatal("concurrent delivery returned failure") + } + if calls.Load() != 1 { + t.Fatalf("same event delivered %d times, want 1", calls.Load()) + } +} + +func TestNotificationAsyncCoalescesBlockedServiceReports(t *testing.T) { + oldCache := Cache + Cache = cache.New(time.Minute, time.Minute) + t.Cleanup(func() { Cache = oldCache }) + n := &model.Notification{Common: model.Common{ID: 501}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 502}, Name: "service-test"}, []uint64{n.ID}) + entered := make(chan struct{}) + release := make(chan struct{}) + var calls atomic.Int32 + nc.sendForTest = func(*model.Notification, string, *model.Server) error { + if calls.Add(1) == 1 { + close(entered) + <-release + } + return nil + } + nc.SendNotificationAsync(502, "down", "service-down") + <-entered + for i := 0; i < 1000; i++ { + nc.SendNotificationAsync(502, "down", "service-down") + } + nc.deliveryMu.Lock() + inFlight := len(nc.asyncInFlight) + nc.deliveryMu.Unlock() + if inFlight != 1 || calls.Load() != 1 { + t.Fatalf("in-flight sends=%d attempts=%d, want one each", inFlight, calls.Load()) + } + close(release) + deadline := time.After(time.Second) + for { + nc.deliveryMu.Lock() + remaining := len(nc.asyncInFlight) + nc.deliveryMu.Unlock() + if remaining == 0 { + break + } + select { + case <-deadline: + t.Fatal("async notification did not finish") + case <-time.After(time.Millisecond): + } + } +} + +func TestNotificationFailureSummaryOmitsCredentialURL(t *testing.T) { + err := &url.Error{Op: "Post", URL: "https://example.test/bot-secret-token/send?key=secret-query", Err: errors.New("timeout")} + if summary := notificationFailureSummary(err); strings.Contains(summary, "secret") || strings.Contains(summary, "example.test") { + t.Fatalf("credential-bearing URL leaked: %s", summary) + } +} + +func TestNotificationAsyncBacksOffFastFailures(t *testing.T) { + oldCache := Cache + Cache = cache.New(time.Minute, time.Minute) + t.Cleanup(func() { Cache = oldCache }) + n := &model.Notification{Common: model.Common{ID: 511}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 512}, Name: "backoff-test"}, []uint64{n.ID}) + var calls atomic.Int32 + nc.sendForTest = func(*model.Notification, string, *model.Server) error { + calls.Add(1) + return errors.New("fast failure") + } + key := "512:service-down" + nc.SendNotificationAsync(512, "down", "service-down") + deadline := time.After(time.Second) + for { + nc.deliveryMu.Lock() + busy := len(nc.asyncInFlight) != 0 + after := nc.asyncRetryAfter[key] + nc.deliveryMu.Unlock() + if !busy && !after.IsZero() { + break + } + select { + case <-deadline: + t.Fatal("first failed send did not record backoff") + case <-time.After(time.Millisecond): + } + } + for i := 0; i < 1000; i++ { + nc.SendNotificationAsync(512, "down", "service-down") + } + if calls.Load() != 1 { + t.Fatalf("fast failures caused %d attempts before retry-after", calls.Load()) + } +} + +func TestUnMuteDuringInflightSendCannotRemuteRecovery(t *testing.T) { + oldCache := Cache + Cache = cache.New(time.Minute, time.Minute) + t.Cleanup(func() { Cache = oldCache }) + n := &model.Notification{Common: model.Common{ID: 521}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 522}, Name: "generation-test"}, []uint64{n.ID}) + entered := make(chan struct{}) + release := make(chan struct{}) + var calls atomic.Int32 + nc.sendForTest = func(*model.Notification, string, *model.Server) error { + if calls.Add(1) == 1 { + close(entered) + <-release + } + return nil + } + done := make(chan bool, 1) + go func() { done <- nc.SendNotification(522, "first failure", "static-label") }() + <-entered + nc.UnMuteNotification(522, "static-label") + close(release) + if !<-done { + t.Fatal("first send failed unexpectedly") + } + if !nc.SendNotification(522, "next failure", "static-label") || calls.Load() != 2 { + t.Fatalf("old in-flight send re-muted next failure: calls=%d", calls.Load()) + } +} + +func TestAsyncBackoffMemoryIsBounded(t *testing.T) { + nc := &NotificationClass{ + asyncRetryAfter: make(map[string]time.Time), + asyncRetryDelay: make(map[string]time.Duration), + } + for i := 0; i < maxAsyncBackoffKeys+100; i++ { + key := fmt.Sprintf("old-event-%d", i) + nc.asyncRetryAfter[key] = time.Now().Add(time.Minute) + nc.asyncRetryDelay[key] = time.Minute + } + nc.pruneAsyncBackoffLocked() + if len(nc.asyncRetryAfter) > maxAsyncBackoffKeys || len(nc.asyncRetryDelay) != len(nc.asyncRetryAfter) { + t.Fatalf("unbounded backoff maps: after=%d delay=%d", len(nc.asyncRetryAfter), len(nc.asyncRetryDelay)) + } +} + +func TestServiceStateNotificationsKeepIncidentRecoveryOrder(t *testing.T) { + oldCache := Cache + Cache = cache.New(time.Minute, time.Minute) + t.Cleanup(func() { Cache = oldCache }) + n := &model.Notification{Common: model.Common{ID: 531}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 532}, Name: "ordered-test"}, []uint64{n.ID}) + entered := make(chan struct{}) + release := make(chan struct{}) + delivered := make(chan string, 3) + nc.sendForTest = func(_ *model.Notification, desc string, _ *model.Server) error { + if desc == "down" { + close(entered) + <-release + } + delivered <- desc + return nil + } + nc.SendServiceState(533, 532, "down", "state-down-1") + <-entered + nc.SendServiceState(533, 532, "good", "state-good-2") + select { + case message := <-delivered: + t.Fatalf("service transition overtook blocked incident: %s", message) + default: + } + close(release) + for _, want := range []string{"down", "good"} { + select { + case got := <-delivered: + if got != want { + t.Fatalf("delivery order got %s, want %s", got, want) + } + case <-time.After(time.Second): + t.Fatalf("missing %s delivery", want) + } + } + deadline := time.After(time.Second) + for { + nc.deliveryMu.Lock() + remaining := len(nc.serviceDelivery) + nc.deliveryMu.Unlock() + if remaining == 0 { + break + } + select { + case <-deadline: + t.Fatal("ordered delivery worker did not exit") + case <-time.After(time.Millisecond): + } + } +} + +func TestServiceStateRetriesFailedIncidentBeforePendingRecovery(t *testing.T) { + oldCache := Cache + Cache = cache.New(time.Minute, time.Minute) + t.Cleanup(func() { Cache = oldCache }) + n := &model.Notification{Common: model.Common{ID: 535}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 536}, Name: "retry-order-test"}, []uint64{n.ID}) + t.Cleanup(func() { nc.CancelServiceState(537) }) + + attempts := make(chan string, 3) + releaseFirst := make(chan struct{}) + var downAttempts atomic.Int32 + nc.sendForTest = func(_ *model.Notification, desc string, _ *model.Server) error { + attempts <- desc + if desc == "down" && downAttempts.Add(1) == 1 { + <-releaseFirst + return errors.New("temporary incident failure") + } + return nil + } + + nc.SendServiceState(537, 536, "down", "state-down-1") + nc.deliveryMu.Lock() + state := nc.serviceDelivery[537] + nc.deliveryMu.Unlock() + select { + case got := <-attempts: + if got != "down" { + t.Fatalf("first delivery = %q, want down", got) + } + case <-time.After(time.Second): + t.Fatal("incident delivery did not start") + } + nc.SendServiceState(537, 536, "good", "state-good-2") + close(releaseFirst) + + for _, want := range []string{"down", "good"} { + select { + case got := <-attempts: + if got != want { + t.Fatalf("delivery after failed incident = %q, want %q", got, want) + } + case <-time.After(time.Second): + t.Fatalf("missing %s delivery", want) + } + } + select { + case <-state.done: + case <-time.After(time.Second): + t.Fatal("ordered delivery worker did not finish") + } +} + +func TestServiceStateCancelStopsOldConfiguration(t *testing.T) { + oldCache := Cache + Cache = cache.New(time.Minute, time.Minute) + t.Cleanup(func() { Cache = oldCache }) + n := &model.Notification{Common: model.Common{ID: 541}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 542}, Name: "old-group"}, []uint64{n.ID}) + entered := make(chan struct{}) + release := make(chan struct{}) + var calls atomic.Int32 + nc.sendForTest = func(*model.Notification, string, *model.Server) error { + if calls.Add(1) == 1 { + close(entered) + <-release + } + return errors.New("old webhook failed") + } + nc.SendServiceState(543, 542, "down", "old-event") + <-entered + nc.deliveryMu.Lock() + state := nc.serviceDelivery[543] + nc.deliveryMu.Unlock() + nc.CancelServiceState(543) + nc.deliveryMu.Lock() + remaining := len(nc.serviceDelivery) + nc.deliveryMu.Unlock() + if remaining != 0 { + t.Fatal("old service delivery worker remained registered after cancellation") + } + close(release) + select { + case <-state.done: + case <-time.After(time.Second): + t.Fatal("canceled old delivery worker did not exit") + } +} + +func TestServiceUpdateCancelsOldRecipientRetry(t *testing.T) { + ss := newServiceMonitorSecurityHarness(t) + service := &model.Service{ + Common: model.Common{ID: 551, UserID: 1}, + Name: "generic-probe", + Type: model.TaskTypeTCPPing, + Target: "example.invalid:443", + Duration: 3600, + Notify: true, + NotificationGroupID: 552, + Cover: model.ServiceCoverIgnoreAll, + } + addServiceMonitorSecurityService(t, ss, service) + n := &model.Notification{Common: model.Common{ID: 553}, Name: "old-hook"} + NotificationShared.InsertForTest(n) + NotificationShared.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 552}, Name: "old-group"}, []uint64{n.ID}) + entered := make(chan struct{}) + release := make(chan struct{}) + NotificationShared.sendForTest = func(*model.Notification, string, *model.Server) error { + close(entered) + <-release + return errors.New("failed old recipient") + } + NotificationShared.SendServiceState(service.ID, 552, "old down", "old-state") + <-entered + NotificationShared.deliveryMu.Lock() + state := NotificationShared.serviceDelivery[service.ID] + NotificationShared.deliveryMu.Unlock() + updated := *service + updated.Notify = false + updated.NotificationGroupID = 554 + if err := ss.Update(&updated); err != nil { + t.Fatal(err) + } + close(release) + select { + case <-state.done: + case <-time.After(time.Second): + t.Fatal("old recipient retry did not stop after service update") + } + NotificationShared.deliveryMu.Lock() + remaining := len(NotificationShared.serviceDelivery) + NotificationShared.deliveryMu.Unlock() + if remaining != 0 { + t.Fatal("old recipient retained after service update") + } +} + +func newNotificationClassWithItems(items ...*model.Notification) *NotificationClass { + nc := NewEmptyNotificationClassForTest() + for _, item := range items { + nc.InsertForTest(item) + } + return nc +} + +func TestNotificationClassDeleteGroupCleansReverseIndex(t *testing.T) { + n := &model.Notification{Common: model.Common{ID: 1, UserID: 2}, Name: "hook"} + nc := newNotificationClassWithItems(n) + group := &model.NotificationGroup{Common: model.Common{ID: 10, UserID: 2}, Name: "group"} + nc.UpdateGroup(group, []uint64{n.ID}) + + nc.DeleteGroup([]uint64{group.ID}) + + if _, ok := nc.groupToIDList[group.ID]; ok { + t.Fatalf("forward index still contains deleted group %d", group.ID) + } + if groups, ok := nc.idToGroupList[n.ID]; ok { + if _, stale := groups[group.ID]; stale { + t.Fatalf("reverse index for notification %d still contains deleted group %d", n.ID, group.ID) + } + } +} + +func TestNotificationClassUpdateGroupCleansRemovedMembership(t *testing.T) { + first := &model.Notification{Common: model.Common{ID: 1, UserID: 2}, Name: "first"} + second := &model.Notification{Common: model.Common{ID: 2, UserID: 2}, Name: "second"} + nc := newNotificationClassWithItems(first, second) + group := &model.NotificationGroup{Common: model.Common{ID: 10, UserID: 2}, Name: "group"} + nc.UpdateGroup(group, []uint64{first.ID, second.ID}) + + nc.UpdateGroup(group, []uint64{second.ID}) + + if groups, ok := nc.idToGroupList[first.ID]; ok { + if _, stale := groups[group.ID]; stale { + t.Fatalf("reverse index for notification %d still contains group %d", first.ID, group.ID) + } + } + if _, ok := nc.idToGroupList[second.ID][group.ID]; !ok { + t.Fatalf("reverse index lost retained membership of notification %d in group %d", second.ID, group.ID) + } +} + +func TestNotificationClassUpdateRepairsOrphanedReverseIndex(t *testing.T) { + n := &model.Notification{Common: model.Common{ID: 1, UserID: 2}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.idToGroupList[n.ID] = map[uint64]struct{}{99: {}} + + n.Name = "updated" + nc.Update(n) + + if groups, ok := nc.idToGroupList[n.ID]; ok && len(groups) != 0 { + t.Fatalf("orphaned reverse memberships were not removed: %v", groups) + } +} + +func TestNotificationClassGroupMutationsDoNotDeadlock(t *testing.T) { + nc := NewEmptyNotificationClassForTest() + nc.listMu.RLock() + readLockHeld := true + defer func() { + if readLockHeld { + nc.listMu.RUnlock() + } + }() + + deleteDone := make(chan struct{}) + go func() { + nc.DeleteGroup([]uint64{10}) + close(deleteDone) + }() + + deadline := time.Now().Add(2 * time.Second) + for nc.listMu.TryRLock() { + nc.listMu.RUnlock() + if time.Now().After(deadline) { + t.Fatal("DeleteGroup did not queue for listMu") + } + time.Sleep(time.Millisecond) + } + + updateDone := make(chan struct{}) + go func() { + nc.UpdateGroup(&model.NotificationGroup{ + Common: model.Common{ID: 10, UserID: 2}, + Name: "group", + }, nil) + close(updateDone) + }() + + deadline = time.Now().Add(2 * time.Second) + for nc.groupMu.TryLock() { + nc.groupMu.Unlock() + if time.Now().After(deadline) { + t.Fatal("neither group mutation acquired groupMu") + } + time.Sleep(time.Millisecond) + } + + nc.listMu.RUnlock() + readLockHeld = false + + timer := time.NewTimer(2 * time.Second) + defer timer.Stop() + for deleteDone != nil || updateDone != nil { + select { + case <-deleteDone: + deleteDone = nil + case <-updateDone: + updateDone = nil + case <-timer.C: + t.Fatal("concurrent UpdateGroup and DeleteGroup deadlocked") + } + } +} diff --git a/service/singleton/online_history.go b/service/singleton/online_history.go new file mode 100644 index 00000000..9f61a864 --- /dev/null +++ b/service/singleton/online_history.go @@ -0,0 +1,348 @@ +package singleton + +import ( + "fmt" + "log" + "sort" + "sync" + "time" + + "github.com/nezhahq/nezha/model" +) + +// OnlineHistoryRecorder 维护每台服务器当前进行中的在线会话。会话的打开与 +// 关闭由 agent 状态流的建立/断开驱动（service/rpc），并由定时清扫兜底关闭 +// 失联（未正常断流）的会话；dashboard 重启时通过数据库中 Ongoing 会话恢复。 +type OnlineHistoryRecorder struct { + mu sync.Mutex + restored bool + open map[uint64]*onlineSessionRuntime + // closing 是关库失败后的待关闭状态：断线边界已被观察到，会话 ID 与 + // 结束时间在此固定、不可被后续上报覆盖；重试只为旧会话完成持久化， + // 重连不得把它续接成活跃会话（同服务器可能同时有多条待关闭会话）。 + closing map[uint64][]*onlineSessionClosing + // Preserve the first disconnect while startup reads are unavailable. + recoveryDisconnects map[uint64]time.Time +} + +type onlineSessionRuntime struct { + id uint64 + lastObserved time.Time + revision uint64 + // lastPersist 是 LastActiveAt 最近一次落库时间，用于节流写库。 + lastPersist time.Time +} + +type onlineSessionClosing struct { + id uint64 + // end 是已观察到的会话结束时刻，重试持久化时原样使用。 + end time.Time +} + +var OnlineHistoryShared *OnlineHistoryRecorder + +func NewOnlineHistoryRecorder() *OnlineHistoryRecorder { + r := &OnlineHistoryRecorder{ + open: make(map[uint64]*onlineSessionRuntime), + closing: make(map[uint64][]*onlineSessionClosing), + recoveryDisconnects: make(map[uint64]time.Time), + } + r.mu.Lock() + r.restoreOngoingSessions() + r.mu.Unlock() + return r +} + +// restoreOngoingSessions 每台服务器仅允许最新且仍在在线窗口内的会话续接， +// 其余会话按最后持久化的活跃时间结转。崩溃会丢失内存中的待关闭边界， +// 恢复时采用 LastActiveAt 保守结转，不推测未持久化的断线时间。 +func (r *OnlineHistoryRecorder) restoreOngoingSessions() bool { + if DB == nil { + return false + } + var ongoing []model.OnlineSession + if err := DB.Where("ongoing = ?", true). + Order("server_id ASC").Order("started_at DESC").Order("id DESC").Find(&ongoing).Error; err != nil { + log.Printf("NEZHA>> Failed to load ongoing online sessions: %v", err) + return false + } + now := time.Now() + seenServers := make(map[uint64]bool) + for _, s := range ongoing { + // 关闭失败后重连可留下同服务器多条 Ongoing 行；直接赋值 open 会覆盖 + // 旧 ID，导致其永久脱管。按开始时间（同时间按 ID）选最新行，其余 + // 必须关闭或进入 closing 重试，即使最新行本身已经过期也不能复活旧行。 + duplicate := seenServers[s.ServerID] + seenServers[s.ServerID] = true + disconnect, observedDisconnect := r.recoveryDisconnects[s.ServerID] + observedDisconnect = observedDisconnect && !duplicate && !disconnect.Before(s.StartedAt) + if observedDisconnect || duplicate || s.LastActiveAt.Before(now.Add(-model.OnlineActiveWindow)) { + end := s.LastActiveAt + if observedDisconnect && disconnect.After(end) { + end = disconnect + } + if end.Before(s.StartedAt) { + end = s.StartedAt + } + if err := closeOnlineSessionRow(s.ID, end); err != nil { + log.Printf("NEZHA>> Failed to close recovered online session %d: %v", s.ID, err) + // 关库失败不能跳过，否则会留下永不闭合、也无人重试的 + // Ongoing 行；进入待关闭状态交由清扫周期重试。过期或 + // 被更新会话替代的行均不得被重连续接为活跃会话 + r.closing[s.ServerID] = append(r.closing[s.ServerID], + &onlineSessionClosing{id: s.ID, end: end}) + } + continue + } + r.open[s.ServerID] = &onlineSessionRuntime{id: s.ID, lastPersist: s.LastActiveAt, lastObserved: s.LastActiveAt} + } + clear(r.recoveryDisconnects) + r.restored = true + return true +} + +// flushClosing 尽力重试关闭该服务器的待关闭会话：按入列时固定的结束时间 +// 持久化，成功才移除；失败的条目原样保留，结束时间不得被后续上报改写。 +func (r *OnlineHistoryRecorder) flushClosing(serverID uint64) { + pendings, ok := r.closing[serverID] + if !ok { + return + } + kept := pendings[:0] + for _, pc := range pendings { + if err := closeOnlineSessionRow(pc.id, pc.end); err != nil { + log.Printf("NEZHA>> Failed to retry closing online session %d: %v", pc.id, err) + kept = append(kept, pc) + } + } + if len(kept) == 0 { + delete(r.closing, serverID) + } else { + r.closing[serverID] = kept + } +} + +// RecordOnline 记录一次状态上报：无进行中会话时开启新会话（重启遗留且仍在 +// 窗口内的 Ongoing 会话直接续接）；已有会话时节流落库 LastActiveAt。 +func (r *OnlineHistoryRecorder) RecordOnline(serverID uint64, at time.Time) { + r.mu.Lock() + defer r.mu.Unlock() + // A failed startup read is not an empty history: retry before accepting reports. + if !r.restored && !r.restoreOngoingSessions() { + return + } + at = at.UTC() + r.flushClosing(serverID) + + // A report can arrive after expiry but before the next sweep. Close at the + // previous observation before overwriting it, preserving failed-close retries. + if rt := r.open[serverID]; rt != nil && at.Sub(rt.lastObserved) > model.OnlineActiveWindow { + r.recordOfflineLocked(serverID, rt.lastObserved) + } + if rt, ok := r.open[serverID]; ok { + rt.lastObserved = at + rt.revision++ + if at.Sub(rt.lastPersist) >= model.OnlineActiveWindow { + if err := DB.Model(&model.OnlineSession{}).Where("id = ?", rt.id).Update("last_active_at", at).Error; err != nil { + log.Printf("NEZHA>> Failed to persist online session last_active_at: %v", err) + } else { + // Only committed writes advance the throttle; failures retry on the next report. + rt.lastPersist = at + } + } + return + } + + // All ongoing rows are owned by open or closing after successful recovery. + + s := model.OnlineSession{ServerID: serverID, StartedAt: at, LastActiveAt: at, Ongoing: true} + if err := DB.Create(&s).Error; err != nil { + log.Printf("NEZHA>> Failed to create online session: %v", err) + return + } + r.open[serverID] = &onlineSessionRuntime{id: s.ID, lastPersist: at, lastObserved: at} +} + +// RecordOffline 关闭该服务器当前进行中的会话。at 是会话结束时刻：正常断流 +// 为当前时间，清扫/结转为最后一次已知活跃时间。 +// 关库失败时把会话转入待关闭状态而非保留为活跃条目：会话 ID 与已观察到的 +// 结束时刻（at 与 lastPersist 的较大者）就此固定，后续上报不得覆盖；重试 +// 只为旧会话完成持久化，重连则开启新会话——断线后的离线间隔不得计作在线。 +func (r *OnlineHistoryRecorder) RecordOffline(serverID uint64, at time.Time) { + r.mu.Lock() + defer r.mu.Unlock() + // Disconnect may be the first event after startup storage recovers. + if !r.restored && !r.restoreOngoingSessions() { + // Retrying SELECT alone loses confirmed offline gaps on a quick reconnect. + // Keep the first boundary; subsequent disconnected streams cannot extend it. + if r.recoveryDisconnects == nil { + r.recoveryDisconnects = make(map[uint64]time.Time) + } + if _, exists := r.recoveryDisconnects[serverID]; !exists { + r.recoveryDisconnects[serverID] = at.UTC() + } + return + } + r.recordOfflineLocked(serverID, at) +} + +func (r *OnlineHistoryRecorder) recordOfflineLocked(serverID uint64, at time.Time) { + rt, ok := r.open[serverID] + if !ok { + return + } + end := at + if end.Before(rt.lastPersist) { + end = rt.lastPersist + } + if err := closeOnlineSessionRow(rt.id, end); err != nil { + log.Printf("NEZHA>> Failed to close online session %d: %v", rt.id, err) + // 结束时刻已观察到，不能因写库失败丢失，也不能把行继续当活跃 + // 会话：转待关闭状态，由 RecordOnline 冲洗与清扫周期重试 + r.closing[serverID] = append(r.closing[serverID], &onlineSessionClosing{id: rt.id, end: end}) + delete(r.open, serverID) + return + } + delete(r.open, serverID) +} + +// SweepStale 关闭超过在线窗口仍无状态上报的进行中会话。lastActiveFn 返回 +// 服务器的运行时最后活跃时间与是否存在。 +func (r *OnlineHistoryRecorder) SweepStale(now time.Time, lastActiveFn func(serverID uint64) (time.Time, bool)) { + // Never call the runtime observer while holding mu: reports acquire the + // runtime lease before this mutex. Snapshot identities, then revalidate them. + r.mu.Lock() + if !r.restored && !r.restoreOngoingSessions() { + r.mu.Unlock() + return + } + candidates := make(map[uint64]onlineSessionRuntime, len(r.open)) + for id, rt := range r.open { + candidates[id] = *rt + } + for id := range r.closing { + r.flushClosing(id) + } + r.mu.Unlock() + for id, candidate := range candidates { + lastActive, ok := lastActiveFn(id) + r.mu.Lock() + rt := r.open[id] + if rt != nil && rt.id == candidate.id && rt.revision == candidate.revision { + if lastActive.Before(rt.lastObserved) { + lastActive = rt.lastObserved + } + if (!ok || lastActive.Before(now.Add(-model.OnlineActiveWindow))) && rt.lastObserved.Before(now.Add(-model.OnlineActiveWindow)) { + r.recordOfflineLocked(id, lastActive) + } + } + r.mu.Unlock() + } + +} + +// SweepOnlineSessions 是定时清扫入口，接上服务器运行时的最后活跃时间。 +func SweepOnlineSessions() { + if OnlineHistoryShared == nil || ServerShared == nil || DB == nil { + return + } + OnlineHistoryShared.SweepStale(time.Now(), onlineServerLastActive) +} + +func onlineServerLastActive(serverID uint64) (time.Time, bool) { + server, ok := ServerShared.Get(serverID) + if !ok || server == nil { + return time.Time{}, false + } + lastActive := server.RuntimeSnapshot().LastActive + return lastActive, !lastActive.IsZero() +} + +// PruneOnlineSessions 清理超出统计窗口的已结束会话与已删除服务器的残留记录。 +// 时间列直接比较（与写入时的绑定格式同构）以走 ended_at 索引，并用分批 +// DELETE 替换单条巨型删除，避免大表上长时间占用 sqlite 写锁、阻塞上报路径。 +func PruneOnlineSessions() { + if DB == nil { + return + } + keepDays := Conf.OnlineHistoryRetentionDays() + if keepDays < model.MinOnlineStatsDays || keepDays > model.MaxOnlineStatsDays { + keepDays = model.DefaultOnlineStatsDays + } + cutoff := model.OnlineHistoryWindowStart(time.Now(), keepDays, Loc) + pruneOnlineSessionsBatch("ongoing = ? AND ended_at < ?", false, cutoff) + pruneOnlineSessionsBatch("server_id NOT IN (SELECT `id` FROM servers)") +} + +// pruneOnlineSessionsBatch 按主键分批删除匹配的行，每批之间释放写锁。 +func pruneOnlineSessionsBatch(where string, args ...any) { + const batchSize = 10000 + for { + var ids []uint64 + if err := DB.Model(&model.OnlineSession{}).Unscoped(). + Where(where, args...).Limit(batchSize).Pluck("id", &ids).Error; err != nil { + log.Printf("NEZHA>> Failed to select online sessions for pruning: %v", err) + return + } + if len(ids) == 0 { + return + } + if err := DB.Unscoped().Delete(&model.OnlineSession{}, ids).Error; err != nil { + log.Printf("NEZHA>> Failed to prune online sessions: %v", err) + return + } + if len(ids) < batchSize { + return + } + } +} + +func closeOnlineSessionRow(id uint64, end time.Time) error { + return DB.Model(&model.OnlineSession{}).Where("id = ?", id).Updates(map[string]any{ + "ended_at": end.UTC(), + "ongoing": false, + }).Error +} + +// Sessions snapshots rows and volatile close boundaries under the same lock. +// Otherwise a failed close can count confirmed offline time as ongoing, or a +// successful retry can remove the boundary after the reader fetched a stale row. +func (r *OnlineHistoryRecorder) Sessions(serverID uint64, start, end time.Time) ([]model.OnlineSession, error) { + if r != nil { + r.mu.Lock() + defer r.mu.Unlock() + if !r.restored && !r.restoreOngoingSessions() { + return nil, fmt.Errorf("online history recovery unavailable") + } + } + var sessions []model.OnlineSession + // Separate branches use (server_id, ongoing, ended_at) and + // (server_id, ongoing, started_at), avoiding a scan of retained closed rows. + for _, ongoing := range []bool{false, true} { + var rows []model.OnlineSession + q := DB.Where("server_id = ? AND ongoing = ?", serverID, ongoing) + if ongoing { + q = q.Where("started_at < ?", end.UTC()) + } else { + q = q.Where("ended_at > ?", start.UTC()) + } + if err := q.Find(&rows).Error; err != nil { + return nil, err + } + sessions = append(sessions, rows...) + } + if r != nil { + endings := make(map[uint64]time.Time) + for _, pending := range r.closing[serverID] { + endings[pending.id] = pending.end + } + for i := range sessions { + if end, ok := endings[sessions[i].ID]; ok { + sessions[i].Ongoing = false + sessions[i].EndedAt = end + } + } + } + sort.Slice(sessions, func(i, j int) bool { return sessions[i].StartedAt.Before(sessions[j].StartedAt) }) + return sessions, nil +} diff --git a/service/singleton/online_history_failure_test.go b/service/singleton/online_history_failure_test.go new file mode 100644 index 00000000..8f22a7e8 --- /dev/null +++ b/service/singleton/online_history_failure_test.go @@ -0,0 +1,224 @@ +package singleton + +import ( + "errors" + "testing" + "time" + + "github.com/nezhahq/nezha/model" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +func TestOnlineHistoryRetriesFailedHeartbeatImmediately(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + start := time.Now().UTC().Truncate(time.Second) + r.RecordOnline(1, start) + id := r.open[1].id + require.NoError(t, DB.Exec(`CREATE TRIGGER reject_heartbeat BEFORE UPDATE OF last_active_at ON online_sessions BEGIN SELECT RAISE(ABORT, 'heartbeat failure'); END`).Error) + r.RecordOnline(1, start.Add(30*time.Second)) + require.NoError(t, DB.Exec("DROP TRIGGER reject_heartbeat").Error) + r.RecordOnline(1, start.Add(31*time.Second)) + require.True(t, getOnlineSession(t, id).LastActiveAt.Equal(start.Add(31*time.Second))) +} + +func TestOnlineHistoryRetriesStartupReadBeforeNewReport(t *testing.T) { + setupOnlineHistoryTestDB(t) + start := time.Now().UTC().Add(-time.Hour).Truncate(time.Second) + old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true} + require.NoError(t, DB.Create(&old).Error) + require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("reject_history_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) })) + r := NewOnlineHistoryRecorder() + require.NoError(t, DB.Callback().Query().Remove("reject_history_restore")) + r.RecordOnline(1, time.Now()) + stored := getOnlineSession(t, old.ID) + require.False(t, stored.Ongoing) + require.True(t, stored.EndedAt.Equal(start)) + require.NotEqual(t, old.ID, r.open[1].id) + require.Equal(t, int64(1), countOnlineSessions(t, "ongoing = ?", true)) +} + +func TestOnlineHistorySweepKeepsSessionReconnectedDuringObservation(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + start := time.Now().Add(-time.Hour) + r.RecordOnline(1, start) + observed := make(chan struct{}) + release := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + r.SweepStale(time.Now(), func(uint64) (time.Time, bool) { close(observed); <-release; return start, true }) + }() + <-observed + reconnected := make(chan struct{}) + go func() { r.RecordOffline(1, start.Add(time.Second)); r.RecordOnline(1, time.Now()); close(reconnected) }() + // The observer must not hold the recorder lock: RPC updates take the runtime + // lock first and then enter the recorder. Holding both in reverse order deadlocks. + select { + case <-reconnected: + case <-time.After(time.Second): + close(release) + <-done + <-reconnected + t.Fatal("sweep holds recorder lock while consulting runtime state") + } + id := r.open[1].id + close(release) + <-done + require.Contains(t, r.open, uint64(1)) + require.Equal(t, id, r.open[1].id) + require.True(t, getOnlineSession(t, id).Ongoing) +} + +func TestOnlineHistoryQueryNormalizesOffsetsAndUsesWindowIndexes(t *testing.T) { + setupOnlineHistoryTestDB(t) + east := time.FixedZone("UTC+8", 8*3600) + start := time.Date(2025, 1, 1, 12, 0, 0, 0, east) + r := NewOnlineHistoryRecorder() + r.RecordOnline(1, start) + r.RecordOffline(1, start.Add(time.Hour)) + rows, err := r.Sessions(1, start.UTC().Add(-time.Minute), start.UTC().Add(2*time.Hour)) + require.NoError(t, err) + require.Len(t, rows, 1) + require.True(t, rows[0].StartedAt.Equal(start)) + require.True(t, rows[0].EndedAt.Equal(start.Add(time.Hour))) + for _, test := range []struct { + query, index string + args []any + }{ + {"EXPLAIN QUERY PLAN SELECT * FROM online_sessions WHERE server_id = ? AND ongoing = ? AND ended_at > ?", "idx_online_sessions_server_closed", []any{1, false, start.UTC()}}, + {"EXPLAIN QUERY PLAN SELECT * FROM online_sessions WHERE server_id = ? AND ongoing = ? AND started_at < ?", "idx_online_sessions_server_open", []any{1, true, start.Add(2 * time.Hour).UTC()}}, + } { + var plan []struct{ Detail string } + require.NoError(t, DB.Raw(test.query, test.args...).Scan(&plan).Error) + require.NotEmpty(t, plan) + require.Contains(t, plan[0].Detail, test.index) + } +} + +func TestOnlineHistorySweepKeepsReportReceivedDuringObservation(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + start := time.Now().Add(-time.Hour) + r.RecordOnline(1, start) + id := r.open[1].id + r.SweepStale(time.Now(), func(uint64) (time.Time, bool) { + r.RecordOnline(1, time.Now()) + return start, true + }) + require.Contains(t, r.open, uint64(1)) + // A report after an hour starts a new session; the stale sweep must not close it. + require.NotEqual(t, id, r.open[1].id) + require.False(t, getOnlineSession(t, id).Ongoing) + require.True(t, getOnlineSession(t, r.open[1].id).Ongoing) +} + +func TestOnlineHistoryStartupReadFailureBlocksQueryAndSweepRetries(t *testing.T) { + setupOnlineHistoryTestDB(t) + start := time.Now().Add(-time.Hour) + old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true} + require.NoError(t, DB.Create(&old).Error) + require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("reject_history_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) })) + r := NewOnlineHistoryRecorder() + _, err := r.Sessions(1, start, time.Now()) + require.ErrorContains(t, err, "recovery unavailable") + r.RecordOnline(1, time.Now()) + require.Empty(t, r.open) + require.NoError(t, DB.Callback().Query().Remove("reject_history_restore")) + r.SweepStale(time.Now(), func(uint64) (time.Time, bool) { return time.Time{}, false }) + require.False(t, getOnlineSession(t, old.ID).Ongoing) + require.Equal(t, int64(0), countOnlineSessions(t, "ongoing = ?", true)) +} + +func TestOnlineHistoryDisconnectRetriesStartupRead(t *testing.T) { + setupOnlineHistoryTestDB(t) + start := time.Now().Add(-10 * time.Second) + old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true} + require.NoError(t, DB.Create(&old).Error) + require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("reject_history_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) })) + r := NewOnlineHistoryRecorder() + require.NoError(t, DB.Callback().Query().Remove("reject_history_restore")) + end := start.Add(time.Second) + r.RecordOffline(1, end) + require.False(t, getOnlineSession(t, old.ID).Ongoing) + require.True(t, getOnlineSession(t, old.ID).EndedAt.Equal(end)) + r.RecordOnline(1, time.Now()) + require.NotEqual(t, old.ID, r.open[1].id) +} + +func TestOnlineHistoryReportAfterTimeoutSplitsBeforeSweep(t *testing.T) { + for _, failClose := range []bool{false, true} { + t.Run(map[bool]string{false: "persisted", true: "retry_close"}[failClose], func(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + start := time.Now().UTC().Truncate(time.Second) + r.RecordOnline(1, start) + r.RecordOnline(1, start.Add(time.Second)) + first := r.open[1].id + r.SweepStale(start.Add(30*time.Second), func(uint64) (time.Time, bool) { return start.Add(time.Second), true }) + require.Equal(t, first, r.open[1].id) + if failClose { + injectCloseUpdateFailure(t) + } + r.RecordOnline(1, start.Add(45*time.Second)) + require.NotEqual(t, first, r.open[1].id) + second := r.open[1].id + rows, err := r.Sessions(1, start, start.Add(46*time.Second)) + require.NoError(t, err) + require.Len(t, rows, 2) + require.False(t, rows[0].Ongoing) + require.True(t, rows[0].EndedAt.Equal(start.Add(time.Second))) + require.True(t, rows[1].StartedAt.Equal(start.Add(45*time.Second))) + if failClose { + require.Len(t, r.closing[1], 1) + dropCloseUpdateFailure(t) + } + r.SweepStale(start.Add(46*time.Second), func(uint64) (time.Time, bool) { return start.Add(45 * time.Second), true }) + require.Empty(t, r.closing) + require.False(t, getOnlineSession(t, first).Ongoing) + require.True(t, getOnlineSession(t, first).EndedAt.Equal(start.Add(time.Second))) + require.True(t, getOnlineSession(t, second).Ongoing) + }) + } +} + +func TestOnlineHistoryDisconnectSurvivesUnavailableRecovery(t *testing.T) { + for _, failClose := range []bool{false, true} { + t.Run(map[bool]string{false: "close_success", true: "close_retry"}[failClose], func(t *testing.T) { + setupOnlineHistoryTestDB(t) + start := time.Now().UTC().Add(-10 * time.Second) + old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true} + require.NoError(t, DB.Create(&old).Error) + require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("unavailable_disconnect_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) })) + r := NewOnlineHistoryRecorder() + end := start.Add(time.Second) + r.RecordOffline(1, end) + // Later disconnects while recovery is unavailable must not extend the first session. + r.RecordOffline(1, end.Add(time.Second)) + require.NoError(t, DB.Callback().Query().Remove("unavailable_disconnect_restore")) + if failClose { + injectCloseUpdateFailure(t) + } + reconnect := start.Add(5 * time.Second) + r.RecordOnline(1, reconnect) + require.NotEqual(t, old.ID, r.open[1].id) + rows, err := r.Sessions(1, start, reconnect.Add(time.Second)) + require.NoError(t, err) + require.Len(t, rows, 2) + require.False(t, rows[0].Ongoing) + require.True(t, rows[0].EndedAt.Equal(end)) + require.True(t, rows[1].StartedAt.Equal(reconnect)) + if failClose { + require.Len(t, r.closing[1], 1) + dropCloseUpdateFailure(t) + } + r.SweepStale(reconnect.Add(time.Second), func(uint64) (time.Time, bool) { return reconnect, true }) + require.Empty(t, r.closing) + require.False(t, getOnlineSession(t, old.ID).Ongoing) + require.True(t, getOnlineSession(t, old.ID).EndedAt.Equal(end)) + require.True(t, getOnlineSession(t, r.open[1].id).Ongoing) + }) + } +} diff --git a/service/singleton/online_history_restore_test.go b/service/singleton/online_history_restore_test.go new file mode 100644 index 00000000..ebafdaaf --- /dev/null +++ b/service/singleton/online_history_restore_test.go @@ -0,0 +1,73 @@ +package singleton + +import ( + "testing" + "time" + + "github.com/nezhahq/nezha/model" + "github.com/stretchr/testify/require" +) + +func TestRestoreOngoingSessionsReconcilesRecentDuplicates(t *testing.T) { + for _, failClose := range []bool{false, true} { + name := "successful_close" + if failClose { + name = "retry_failed_close" + } + t.Run(name, func(t *testing.T) { + setupOnlineHistoryTestDB(t) + base := time.Now().Add(-10 * time.Second).Truncate(time.Second) + // Insert out of chronological order: choosing by ID alone would keep the wrong session. + newest := model.OnlineSession{ServerID: 1, StartedAt: base.Add(4 * time.Second), LastActiveAt: base.Add(5 * time.Second), Ongoing: true} + oldest := model.OnlineSession{ServerID: 1, StartedAt: base, LastActiveAt: base.Add(time.Second), Ongoing: true} + middle := model.OnlineSession{ServerID: 1, StartedAt: base.Add(2 * time.Second), LastActiveAt: base.Add(3 * time.Second), Ongoing: true} + for _, session := range []*model.OnlineSession{&newest, &oldest, &middle} { + require.NoError(t, DB.Create(session).Error) + } + if failClose { + injectCloseUpdateFailure(t) + } + r := NewOnlineHistoryRecorder() + require.Len(t, r.open, 1) + require.Equal(t, newest.ID, r.open[1].id) + if failClose { + require.Len(t, r.closing[1], 2) + require.Equal(t, int64(3), countOnlineSessions(t, "ongoing = ?", true)) + dropCloseUpdateFailure(t) + } + r.SweepStale(base.Add(6*time.Second), func(uint64) (time.Time, bool) { return newest.LastActiveAt, true }) + require.Empty(t, r.closing) + require.True(t, getOnlineSession(t, newest.ID).Ongoing) + for _, previous := range []model.OnlineSession{oldest, middle} { + stored := getOnlineSession(t, previous.ID) + require.False(t, stored.Ongoing) + require.True(t, stored.EndedAt.Equal(previous.LastActiveAt)) + } + r.RecordOnline(1, base.Add(7*time.Second)) + require.Equal(t, newest.ID, r.open[1].id) + require.Equal(t, int64(3), countOnlineSessions(t, "server_id = ?", 1)) + r.SweepStale(base.Add(2*model.OnlineActiveWindow), func(uint64) (time.Time, bool) { return time.Time{}, false }) + require.Empty(t, r.open) + require.Empty(t, r.closing) + require.Equal(t, int64(0), countOnlineSessions(t, "ongoing = ?", true)) + }) + } +} + +func TestRestoreOngoingSessionsUsesIDToBreakStartTimeTies(t *testing.T) { + setupOnlineHistoryTestDB(t) + start := time.Now().Add(-5 * time.Second) + first := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true} + second := first + other := model.OnlineSession{ServerID: 2, StartedAt: start, LastActiveAt: start, Ongoing: true} + for _, session := range []*model.OnlineSession{&first, &second, &other} { + require.NoError(t, DB.Create(session).Error) + } + r := NewOnlineHistoryRecorder() + require.Len(t, r.open, 2) + require.Equal(t, second.ID, r.open[1].id) + require.Equal(t, other.ID, r.open[2].id) + require.False(t, getOnlineSession(t, first.ID).Ongoing) + require.True(t, getOnlineSession(t, first.ID).EndedAt.Equal(start)) + require.Equal(t, int64(2), countOnlineSessions(t, "ongoing = ?", true)) +} diff --git a/service/singleton/online_history_retention_test.go b/service/singleton/online_history_retention_test.go new file mode 100644 index 00000000..f505ba2f --- /dev/null +++ b/service/singleton/online_history_retention_test.go @@ -0,0 +1,63 @@ +package singleton + +import ( + "errors" + "testing" + "time" + + "github.com/nezhahq/nezha/model" + "github.com/stretchr/testify/require" +) + +func TestOnlineHistoryPruneWaitsForRetentionCommit(t *testing.T) { + for _, saveFails := range []bool{true, false} { + t.Run(map[bool]string{true: "failed_save", false: "successful_save"}[saveFails], func(t *testing.T) { + setupOnlineHistoryTestDB(t) + previous := Conf + Conf = &ConfigClass{Config: &model.Config{}} + Conf.OnlineStatsDays = 30 + t.Cleanup(func() { Conf = previous }) + now := time.Now().UTC() + row := model.OnlineSession{ServerID: 1, StartedAt: now.AddDate(0, 0, -15), EndedAt: now.AddDate(0, 0, -14)} + require.NoError(t, DB.Create(&row).Error) + saveEntered := make(chan struct{}) + releaseSave := make(chan struct{}) + saved := make(chan error, 1) + days := 7 + go func() { + saved <- Conf.CommitOnlineHistoryRetention(&days, func() error { + close(saveEntered) + <-releaseSave + if saveFails { + return errors.New("retention save unavailable") + } + return nil + }) + }() + <-saveEntered + pruneStarted := make(chan struct{}) + pruned := make(chan struct{}) + go func() { close(pruneStarted); PruneOnlineSessions(); close(pruned) }() + <-pruneStarted + premature := false + select { + case <-pruned: + premature = true + case <-time.After(100 * time.Millisecond): + } + close(releaseSave) + err := <-saved + <-pruned + require.False(t, premature, "pruning must not read an uncommitted retention policy") + if saveFails { + require.ErrorContains(t, err, "retention save unavailable") + require.Equal(t, 30, Conf.OnlineHistoryRetentionDays()) + require.Equal(t, int64(1), countOnlineSessions(t, "id = ?", row.ID)) + } else { + require.NoError(t, err) + require.Equal(t, 7, Conf.OnlineHistoryRetentionDays()) + require.Equal(t, int64(0), countOnlineSessions(t, "id = ?", row.ID)) + } + }) + } +} diff --git a/service/singleton/online_history_test.go b/service/singleton/online_history_test.go new file mode 100644 index 00000000..285f8f31 --- /dev/null +++ b/service/singleton/online_history_test.go @@ -0,0 +1,316 @@ +package singleton + +import ( + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "gorm.io/gorm" + + "github.com/nezhahq/nezha/model" +) + +func setupOnlineHistoryTestDB(t *testing.T) { + t.Helper() + + previousDB := DB + var err error + DB, err = gorm.Open(openSQLiteDialector(filepath.Join(t.TempDir(), "dashboard.sqlite")), &gorm.Config{}) + require.NoError(t, err) + sqlDB, err := DB.DB() + require.NoError(t, err) + t.Cleanup(func() { + DB = previousDB + if err := sqlDB.Close(); err != nil { + t.Errorf("close online history test database: %v", err) + } + }) + + require.NoError(t, DB.AutoMigrate(&model.Server{}, &model.OnlineSession{})) + require.NoError(t, DB.Exec("INSERT INTO servers (id, name, uuid) VALUES (1, 'server-1', 'online-history-test-1')").Error) +} + +func getOnlineSession(t *testing.T, id uint64) model.OnlineSession { + t.Helper() + var s model.OnlineSession + require.NoError(t, DB.First(&s, id).Error) + return s +} + +func countOnlineSessions(t *testing.T, cond string, args ...any) int64 { + t.Helper() + var count int64 + require.NoError(t, DB.Model(&model.OnlineSession{}).Where(cond, args...).Count(&count).Error) + return count +} + +// injectCloseUpdateFailure 用 SQLite 触发器注入"SELECT 正常、仅关闭 UPDATE 失败"： +// 只拦截把 ongoing 置 0 的更新，LastActiveAt 节流写与 INSERT 均不受影响， +// 使后续 RecordOnline/恢复/清扫全部走真实执行路径。 +func injectCloseUpdateFailure(t *testing.T) { + t.Helper() + require.NoError(t, DB.Exec(`CREATE TRIGGER fail_online_session_close +BEFORE UPDATE ON online_sessions +WHEN NEW.ongoing = 0 +BEGIN + SELECT RAISE(ABORT, 'injected close failure'); +END`).Error) + t.Cleanup(func() { dropCloseUpdateFailure(t) }) +} + +func dropCloseUpdateFailure(t *testing.T) { + t.Helper() + require.NoError(t, DB.Exec("DROP TRIGGER IF EXISTS fail_online_session_close").Error) +} + +// 评审回归（P2：关库失败丢边界/重连续接待关闭会话）：RecordOffline 关库失败时 +// 必须转入待关闭状态——会话 ID 与已观察到的断线时刻固定、后续上报不得覆盖； +// 重连开启的是新会话而非续接旧行；失败恢复后由清扫周期按固定边界完成持久化， +// 离线间隔不计入在线。时间自单一基准单调推进。 +func TestRecordOfflineCloseFailurePreservesBoundary(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + + base := time.Now().Add(-time.Hour).Truncate(time.Second) + atStart := base + atLastActive := atStart.Add(model.OnlineActiveWindow) // 恰好达到节流窗口，触发落库 + atOffline := atLastActive.Add(15 * time.Second) // 观察到的断线时刻 + atReconnect := atOffline.Add(10 * time.Minute) + + r.RecordOnline(1, atStart) + r.RecordOnline(1, atLastActive) + require.Len(t, r.open, 1) + first := getOnlineSession(t, r.open[1].id) + require.WithinDuration(t, atLastActive, first.LastActiveAt, time.Second) + + // 注入关闭失败：行保持 Ongoing 等待重试，会话转入待关闭而非继续当活跃会话 + injectCloseUpdateFailure(t) + r.RecordOffline(1, atOffline) + require.NotContains(t, r.open, uint64(1)) + stuck := getOnlineSession(t, first.ID) + require.True(t, stuck.Ongoing) + require.True(t, stuck.EndedAt.IsZero(), "关库失败时旧行必须保持未闭合等待重试") + + // 失败未恢复时 agent 重连：旧行不得被续接，必须开启新会话 + r.RecordOnline(1, atReconnect) + require.Contains(t, r.open, uint64(1)) + require.NotEqual(t, first.ID, r.open[1].id, "待关闭会话不得被续接为活跃会话") + afterRetry := getOnlineSession(t, first.ID) + require.True(t, afterRetry.Ongoing, "失败未恢复时重试仍失败，旧行保持待关闭") + second := getOnlineSession(t, r.open[1].id) + require.True(t, second.Ongoing) + require.WithinDuration(t, atReconnect, second.StartedAt, time.Second) + + // 失败恢复后清扫：旧行按观察到的断线时刻闭合，新会话仍活跃不受影响 + dropCloseUpdateFailure(t) + r.SweepStale(atReconnect.Add(time.Second), func(uint64) (time.Time, bool) { + return atReconnect, true + }) + + closed := getOnlineSession(t, first.ID) + require.False(t, closed.Ongoing) + require.WithinDuration(t, atOffline, closed.EndedAt, time.Second, "旧行必须按观察到的断线时刻闭合") + require.WithinDuration(t, atStart, closed.StartedAt, time.Second) + stillOngoing := getOnlineSession(t, second.ID) + require.True(t, stillOngoing.Ongoing) + + // 离线间隔不计入在线：两行在 [atStart, atReconnect) 内的在线合计止于断线时刻 + var rows []model.OnlineSession + require.NoError(t, DB.Where("server_id = ?", 1).Order("id").Find(&rows).Error) + require.Len(t, rows, 2) + var online time.Duration + for _, s := range rows { + end := s.EndedAt + if s.Ongoing { + end = atReconnect + } + online += end.Sub(s.StartedAt) + } + require.Equal(t, atOffline.Sub(atStart), online, "离线间隔不得计作在线") +} + +// 评审回归（P2 启动恢复路径）：重启恢复对超窗会话结转关库失败时，条目必须 +// 进入待关闭状态（经真实 NewOnlineHistoryRecorder 恢复路径验证，而非手工 +// 构造内存状态）；重连不得续接该行，清扫周期按固定边界时刻完成持久化。 +func TestRestoreStaleCloseFailureEntersPendingClose(t *testing.T) { + setupOnlineHistoryTestDB(t) + + // 预置一条超窗 Ongoing 行（dashboard 停机前遗留），时间自单一基准单调推进 + base := time.Now().Add(-3 * time.Hour).Truncate(time.Second) + staleAt := base.Add(time.Hour) + atReconnect := base.Add(2 * time.Hour) + legacy := model.OnlineSession{ServerID: 1, StartedAt: base, LastActiveAt: staleAt, Ongoing: true} + require.NoError(t, DB.Create(&legacy).Error) + + injectCloseUpdateFailure(t) + r := NewOnlineHistoryRecorder() + + stuck := getOnlineSession(t, legacy.ID) + require.True(t, stuck.Ongoing, "恢复结转关库失败时行必须保持 Ongoing 等待重试") + + // 重连：待关闭的超窗会话不得被续接，开启新会话 + r.RecordOnline(1, atReconnect) + require.Contains(t, r.open, uint64(1)) + require.NotEqual(t, legacy.ID, r.open[1].id, "待关闭会话不得被续接为活跃会话") + + dropCloseUpdateFailure(t) + r.SweepStale(atReconnect.Add(time.Second), func(uint64) (time.Time, bool) { + return atReconnect, true + }) + + closed := getOnlineSession(t, legacy.ID) + require.False(t, closed.Ongoing) + require.WithinDuration(t, staleAt, closed.EndedAt, time.Second, "旧行必须按恢复时观察到的边界闭合") + require.Equal(t, int64(2), countOnlineSessions(t, "server_id = ?", 1)) + require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true)) +} + +func TestRecordOnlineOpensSingleSession(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + + now := time.Now() + r.RecordOnline(1, now) + r.RecordOnline(1, now.Add(time.Second)) + + require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ?", 1)) + require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true)) +} + +func TestRecordOnlinePersistsLastActiveThrottled(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + + now := time.Now() + r.RecordOnline(1, now) + // 窗口内的重复上报不应触发落库 + r.RecordOnline(1, now.Add(5*time.Second)) + s := r.open[1] + stored := getOnlineSession(t, s.id) + require.WithinDuration(t, now, stored.LastActiveAt, time.Second) + + // 超过窗口后落库 + r.RecordOnline(1, now.Add(model.OnlineActiveWindow+time.Second)) + stored = getOnlineSession(t, s.id) + require.WithinDuration(t, now.Add(model.OnlineActiveWindow+time.Second), stored.LastActiveAt, time.Second) +} + +func TestRecordOfflineClosesSession(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + + start := time.Now() + r.RecordOnline(1, start) + r.RecordOffline(1, start.Add(10*time.Minute)) + // 重复关闭应为无操作 + r.RecordOffline(1, start.Add(11*time.Minute)) + + require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ?", 1)) + require.Equal(t, int64(0), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true)) + var s model.OnlineSession + require.NoError(t, DB.First(&s).Error) + require.WithinDuration(t, start.Add(10*time.Minute), s.EndedAt, time.Second) + require.WithinDuration(t, start, s.StartedAt, time.Second) +} + +func TestRecordOnlineContinuesSessionAfterRestart(t *testing.T) { + setupOnlineHistoryTestDB(t) + + // 模拟重启前遗留的进行中会话（最后活跃时间在在线窗口内：dashboard 只是短暂重启） + now := time.Now() + legacy := model.OnlineSession{ServerID: 1, StartedAt: now.Add(-time.Minute), LastActiveAt: now.Add(-5 * time.Second), Ongoing: true} + require.NoError(t, DB.Create(&legacy).Error) + + // 新 recorder（构造时从库恢复）：新近会话应保留并续接，不产生新会话 + r := NewOnlineHistoryRecorder() + r.RecordOnline(1, now) + + require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ?", 1)) + continued := getOnlineSession(t, legacy.ID) + require.True(t, continued.Ongoing) + require.WithinDuration(t, now.Add(-time.Minute), continued.StartedAt, time.Second) +} + +func TestRestoreOngoingSessionsClosesStaleSessions(t *testing.T) { + setupOnlineHistoryTestDB(t) + + // 上次活跃时间远早于在线窗口：重启恢复时按最后已知活跃时刻结转 + staleStart := time.Now().Add(-time.Hour) + stale := model.OnlineSession{ServerID: 1, StartedAt: staleStart, LastActiveAt: staleStart.Add(30 * time.Minute), Ongoing: true} + require.NoError(t, DB.Create(&stale).Error) + + r := NewOnlineHistoryRecorder() + r.RecordOnline(1, time.Now()) + + closed := getOnlineSession(t, stale.ID) + require.False(t, closed.Ongoing) + require.WithinDuration(t, staleStart.Add(30*time.Minute), closed.EndedAt, time.Second) + // 结转后新的上报开启新会话 + require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true)) +} + +func TestSweepStaleClosesOnlyStaleSessions(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + + now := time.Now() + r.RecordOnline(1, now.Add(-time.Hour)) + // 第二台服务器：会话在 DB 但没有 servers 记录（服务器已被删除），手工放进内存 map + require.NoError(t, DB.Exec("INSERT INTO online_sessions (server_id, started_at, last_active_at, ongoing, created_at, updated_at) VALUES (2, ?, ?, 1, ?, ?)", now.Add(-time.Hour), now.Add(-time.Hour), now, now).Error) + var second model.OnlineSession + require.NoError(t, DB.Where("server_id = ?", 2).First(&second).Error) + r.mu.Lock() + r.open[2] = &onlineSessionRuntime{id: second.ID, lastPersist: now.Add(-time.Hour)} + r.mu.Unlock() + + // 服务器 1 活跃，服务器 2 不存在 → 仅关闭服务器 2 的会话 + lastActive := func(serverID uint64) (time.Time, bool) { + if serverID == 1 { + return now, true + } + return time.Time{}, false + } + r.SweepStale(now, lastActive) + + require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true)) + require.Equal(t, int64(0), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 2, true)) + + // 服务器 1 失联后也会被关闭，结束时刻为最后已知活跃时间 + r.SweepStale(now.Add(model.OnlineActiveWindow*2), func(uint64) (time.Time, bool) { + return now, true + }) + require.Equal(t, int64(0), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true)) + var closedFirst = getOnlineSession(t, func() uint64 { + var s model.OnlineSession + require.NoError(t, DB.Where("server_id = ?", 1).First(&s).Error) + return s.ID + }()) + require.WithinDuration(t, now, closedFirst.EndedAt, time.Second) +} + +func TestPruneOnlineSessions(t *testing.T) { + setupOnlineHistoryTestDB(t) + previousConf := Conf + Conf = &ConfigClass{Config: &model.Config{}} + Conf.OnlineStatsDays = 7 + t.Cleanup(func() { Conf = previousConf }) + + now := time.Now() + old := now.AddDate(0, 0, -30) + rows := []model.OnlineSession{ + {ServerID: 1, StartedAt: old, EndedAt: old.Add(time.Hour), LastActiveAt: old.Add(time.Hour)}, // 超窗口 → 清理 + {ServerID: 1, StartedAt: old, EndedAt: now.Add(-time.Hour), LastActiveAt: now.Add(-time.Hour)}, // 跨窗口边界 → 保留 + {ServerID: 1, StartedAt: now.Add(-time.Hour), EndedAt: now, LastActiveAt: now}, // 窗口内 → 保留 + {ServerID: 1, StartedAt: now, Ongoing: true, LastActiveAt: now}, // 进行中 → 保留 + {ServerID: 99, StartedAt: now.Add(-time.Hour), EndedAt: now, LastActiveAt: now}, // 服务器已删除 → 清理 + } + for i := range rows { + require.NoError(t, DB.Create(&rows[i]).Error) + } + + PruneOnlineSessions() + + require.Equal(t, int64(3), countOnlineSessions(t, "1 = 1")) + require.Equal(t, int64(1), countOnlineSessions(t, "ongoing = ?", true)) +} diff --git a/service/singleton/server.go b/service/singleton/server.go index a74a7f62..64ffd645 100644 --- a/service/singleton/server.go +++ b/service/singleton/server.go @@ -117,6 +117,7 @@ func (c *ServerClass) Update(s *model.Server, uuid string) { func (c *ServerClass) Delete(idList []uint64) { c.lockLifecycleWrite() defer c.unlockLifecycleWrite() + cancelAlertDeliveriesForServers(idList) c.listMu.Lock() diff --git a/service/singleton/server_transfer.go b/service/singleton/server_transfer.go index cef41476..ff3f93ef 100644 --- a/service/singleton/server_transfer.go +++ b/service/singleton/server_transfer.go @@ -152,6 +152,8 @@ type ServerTransferClass struct { subMu sync.Mutex subs map[uint64]chan *model.ServerTransfer + subOwners map[uint64]uint64 + subCounts map[uint64]int nextSubID uint64 timeout time.Duration @@ -174,6 +176,18 @@ var ErrServerAlreadyTransferring = errors.New("server already has an in-flight t // deferred to OnAgentReconnect / PushIfOnline. var ErrAgentTooOldForTransfer = fmt.Errorf("agent build older than %s does not support server transfer (TaskTypeServerTransferApply)", MinServerTransferAgentVersion) +const ( + // A normal dashboard needs one subscription per open transfer page. Keep a + // small per-user allowance for tabs/devices plus a process-wide hard stop. + maxServerTransferSubscribersPerUser = 8 + maxServerTransferSubscribersGlobal = 1024 +) + +var ( + errTooManyServerTransferSubscribersForUser = errors.New("too many concurrent server transfer subscribers for this user") + errTooManyServerTransferSubscribersGlobal = errors.New("too many concurrent server transfer subscribers") +) + // agentSupportsTransfer reports whether s has reported a build version >= // MinServerTransferAgentVersion. Returns true when version is unknown (agent // never reported) so callers can defer the decision; PushIfOnline re-checks @@ -210,6 +224,8 @@ func NewServerTransferClass() *ServerTransferClass { verifiedHandshakes: make(map[uint64]string), initiating: make(map[uint64]bool), subs: make(map[uint64]chan *model.ServerTransfer), + subOwners: make(map[uint64]uint64), + subCounts: make(map[uint64]int), timeout: defaultServerTransferTimeout, stopCh: make(chan struct{}), } @@ -1410,20 +1426,37 @@ func (c *ServerTransferClass) sweepTimeouts() { // Subscribe registers a channel that will receive every transfer transition // event from this point forward. The caller MUST Unsubscribe when done or // the broker will block forever if the channel is unbuffered or full. -func (c *ServerTransferClass) Subscribe() (uint64, <-chan *model.ServerTransfer) { +func (c *ServerTransferClass) Subscribe(userID uint64) (uint64, <-chan *model.ServerTransfer, error) { c.subMu.Lock() defer c.subMu.Unlock() + if len(c.subs) >= maxServerTransferSubscribersGlobal { + return 0, nil, errTooManyServerTransferSubscribersGlobal + } + if c.subCounts[userID] >= maxServerTransferSubscribersPerUser { + return 0, nil, errTooManyServerTransferSubscribersForUser + } id := atomic.AddUint64(&c.nextSubID, 1) ch := make(chan *model.ServerTransfer, 16) c.subs[id] = ch - return id, ch + c.subOwners[id] = userID + c.subCounts[userID]++ + return id, ch, nil } func (c *ServerTransferClass) Unsubscribe(id uint64) { c.subMu.Lock() ch, ok := c.subs[id] + ownerID, ownerOK := c.subOwners[id] delete(c.subs, id) + delete(c.subOwners, id) + if ownerOK { + if c.subCounts[ownerID] <= 1 { + delete(c.subCounts, ownerID) + } else { + c.subCounts[ownerID]-- + } + } c.subMu.Unlock() if ok { close(ch) diff --git a/service/singleton/server_transfer_test.go b/service/singleton/server_transfer_test.go index 2be8eb05..fa6fecf2 100644 --- a/service/singleton/server_transfer_test.go +++ b/service/singleton/server_transfer_test.go @@ -455,7 +455,8 @@ func TestServerTransferBroadcastReachesSubscribers(t *testing.T) { defer cleanup() seedServerForTransfer(t, 1, 100) - id, ch := c.Subscribe() + id, ch, err := c.Subscribe(100) + require.NoError(t, err) defer c.Unsubscribe(id) tr := initiateAndRegister(t, c, 1, 100, 200, 1) @@ -469,7 +470,7 @@ func TestServerTransferBroadcastReachesSubscribers(t *testing.T) { t.Fatal("expected Pending broadcast within 1s") } - _, _, err := markPendingVerified(t, c, 1) + _, _, err = markPendingVerified(t, c, 1) require.NoError(t, err) select { case ev := <-ch: @@ -479,6 +480,57 @@ func TestServerTransferBroadcastReachesSubscribers(t *testing.T) { } } +func TestServerTransferSubscribeEnforcesPerUserLimitAndReleasesQuota(t *testing.T) { + c, cleanup := setupTransferFixture(t) + defer cleanup() + + ids := make([]uint64, 0, maxServerTransferSubscribersPerUser) + for range maxServerTransferSubscribersPerUser { + id, ch, err := c.Subscribe(100) + require.NoError(t, err) + require.NotNil(t, ch) + ids = append(ids, id) + } + + _, _, err := c.Subscribe(100) + require.ErrorIs(t, err, errTooManyServerTransferSubscribersForUser) + + otherID, _, err := c.Subscribe(200) + require.NoError(t, err, "one user's quota must not block another user") + c.Unsubscribe(otherID) + + c.Unsubscribe(ids[0]) + replacementID, _, err := c.Subscribe(100) + require.NoError(t, err, "disconnecting must release the user's quota") + c.Unsubscribe(replacementID) + for _, id := range ids[1:] { + c.Unsubscribe(id) + } +} + +func TestServerTransferSubscribeEnforcesGlobalLimit(t *testing.T) { + c, cleanup := setupTransferFixture(t) + defer cleanup() + + ids := make([]uint64, 0, maxServerTransferSubscribersGlobal) + for userID := uint64(1); len(ids) < maxServerTransferSubscribersGlobal; userID++ { + for range maxServerTransferSubscribersPerUser { + if len(ids) == maxServerTransferSubscribersGlobal { + break + } + id, _, err := c.Subscribe(userID) + require.NoError(t, err) + ids = append(ids, id) + } + } + + _, _, err := c.Subscribe(999999) + require.ErrorIs(t, err, errTooManyServerTransferSubscribersGlobal) + for _, id := range ids { + c.Unsubscribe(id) + } +} + // The "one active transfer per server" invariant must hold under concurrent // callers (two operators batch-moving the same server, or batch-move racing // retry). The old flow had a TOCTOU between HasPending() and Initiate() that @@ -1626,7 +1678,8 @@ func TestOnServersDeletedTerminatesPendingTransfersAndClearsIndexes(t *testing.T tr := initiateAndRegister(t, c, 1, 100, 200, 1) require.True(t, c.HasPending(1)) - subID, ch := c.Subscribe() + subID, ch, err := c.Subscribe(100) + require.NoError(t, err) defer c.Unsubscribe(subID) require.NoError(t, DB.Unscoped().Delete(&model.Server{}, tr.ServerID).Error) diff --git a/service/singleton/servicesentinel.go b/service/singleton/servicesentinel.go index a2c6ef8d..9d6c66fa 100644 --- a/service/singleton/servicesentinel.go +++ b/service/singleton/servicesentinel.go @@ -2,6 +2,7 @@ package singleton import ( "cmp" + "crypto/sha256" "fmt" "iter" "log" @@ -9,6 +10,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "time" "github.com/gin-gonic/gin" @@ -22,9 +24,11 @@ import ( ) const ( - _CurrentStatusSize = 30 // 统计 15 分钟内的数据为当前状态 + _CurrentStatusSize = 30 // initial capacity for the rolling service window ) +var serviceEventSequence atomic.Uint64 + type serviceResponseItem struct { model.ServiceResponseItem @@ -46,9 +50,101 @@ type _TodayStatsOfService struct { type serviceResponseData = _TodayStatsOfService type serviceTaskStatus struct { - lastStatus uint8 - t time.Time - result []*pb.TaskResult + lastStatus uint8 + stateEpoch uint64 + lastHistoryAt time.Time + history serviceResponseData + lastData string + result []serviceWindowBucket +} + +type serviceProbeSample struct { + successful bool + failed bool + delay float64 +} + +type serviceWindowBucket struct { + at time.Time + reporters map[uint64]serviceProbeSample +} + +func (status *serviceTaskStatus) appendResult(now time.Time, reporter uint64, result *pb.TaskResult) { + bucketAt := now.Truncate(30 * time.Second) + if len(status.result) > 0 && bucketAt.Before(status.result[len(status.result)-1].at) { + status.result = nil // wall-clock moved backwards; old buckets are not comparable + } + if len(status.result) == 0 || !status.result[len(status.result)-1].at.Equal(bucketAt) { + status.result = append(status.result, serviceWindowBucket{at: bucketAt, reporters: make(map[uint64]serviceProbeSample)}) + } + bucket := &status.result[len(status.result)-1] + sample := bucket.reporters[reporter] + if result.Successful { + sample.successful = true + sample.delay = float64(result.Delay) + } else { + sample.failed = true + } + bucket.reporters[reporter] = sample + cutoff := now.Add(-15 * time.Minute) + first := 0 + for first < len(status.result) && status.result[first].at.Before(cutoff) { + first++ + } + if first > 0 { + status.result = append([]serviceWindowBucket(nil), status.result[first:]...) + } + if result.Successful { + status.history.Up++ + status.history.Delay = (status.history.Delay*float64(status.history.Up-1) + float64(result.Delay)) / float64(status.history.Up) + } else { + status.history.Down++ + } + status.lastData = result.Data +} + +// flushServiceHistory persists the previous non-overlapping reporting period +// before the next observation is added, so long probe gaps cannot lose or +// double-count samples in the 15-minute availability history. +func (ss *ServiceSentinel) flushServiceHistory(serviceID uint64, status *serviceTaskStatus, now time.Time) { + if status.lastHistoryAt.IsZero() || now.Before(status.lastHistoryAt.Add(15*time.Minute)) { + return + } + if status.history.Up+status.history.Down == 0 { + status.lastHistoryAt = now + return + } + if !TSDBEnabled() { + if err := DB.Create(&model.ServiceHistory{ + CreatedAt: status.lastHistoryAt, + ServiceID: serviceID, + AvgDelay: status.history.Delay, + Data: status.lastData, + Up: status.history.Up, + Down: status.history.Down, + }).Error; err != nil { + log.Printf("NEZHA>> Failed to save service monitor metrics: %v", err) + return + } + } + status.history = serviceResponseData{} + status.lastHistoryAt = now +} + +func (status *serviceTaskStatus) responseData() serviceResponseData { + var data serviceResponseData + for _, bucket := range status.result { + for _, sample := range bucket.reporters { + if sample.successful { + data.Up++ + data.Delay = (data.Delay*float64(data.Up-1) + sample.delay) / float64(data.Up) + } + if sample.failed { + data.Down++ + } + } + } + return data } type pingStore struct { @@ -228,7 +324,7 @@ func (ss *ServiceSentinel) loadServiceHistory() error { } ss.services[service.ID] = service ss.serviceCurrentStatusData[service.ID] = new(serviceTaskStatus) - ss.serviceCurrentStatusData[service.ID].result = make([]*pb.TaskResult, 0, _CurrentStatusSize) + ss.serviceCurrentStatusData[service.ID].result = make([]serviceWindowBucket, 0, _CurrentStatusSize) ss.serviceStatusToday[service.ID] = &_TodayStatsOfService{} validServices = append(validServices, service) } @@ -335,10 +431,18 @@ func (ss *ServiceSentinel) loadTodayStats(today time.Time) { totalDelay := make(map[uint64]float64) totalDelayCount := make(map[uint64]int) for _, mh := range mhs { - ss.serviceStatusToday[mh.ServiceID].Up += mh.Up - ss.monthlyStatus[mh.ServiceID].TotalUp += mh.Up - ss.serviceStatusToday[mh.ServiceID].Down += mh.Down - ss.monthlyStatus[mh.ServiceID].TotalDown += mh.Down + todayStats := ss.serviceStatusToday[mh.ServiceID] + monthlyStats := ss.monthlyStatus[mh.ServiceID] + // History can outlive a deleted service or belong to a quarantined + // service. Neither has startup status entries; skip its rows before + // accumulating counts or delays so retained history cannot panic startup. + if todayStats == nil || monthlyStats == nil { + continue + } + todayStats.Up += mh.Up + monthlyStats.TotalUp += mh.Up + todayStats.Down += mh.Down + monthlyStats.TotalDown += mh.Down totalDelay[mh.ServiceID] += mh.AvgDelay totalDelayCount[mh.ServiceID]++ } @@ -371,6 +475,9 @@ func (ss *ServiceSentinel) Update(m *model.Service) error { if err != nil { return err } + // A service edit may disable notifications or change their recipients. + // Cancel any retry worker holding the previous service configuration. + NotificationShared.CancelServiceState(m.ID) if ss.services[m.ID] != nil { // 停掉旧任务 CronShared.Remove(ss.services[m.ID].CronJobID) @@ -387,7 +494,7 @@ func (ss *ServiceSentinel) Update(m *model.Service) error { if ss.serviceCurrentStatusData[m.ID] == nil { ss.serviceCurrentStatusData[m.ID] = new(serviceTaskStatus) } - ss.serviceCurrentStatusData[m.ID].result = make([]*pb.TaskResult, 0, _CurrentStatusSize) + ss.serviceCurrentStatusData[m.ID].result = make([]serviceWindowBucket, 0, _CurrentStatusSize) ss.serviceStatusToday[m.ID] = &_TodayStatsOfService{} } // 更新这个任务 @@ -404,6 +511,7 @@ func (ss *ServiceSentinel) Delete(ids []uint64) { defer ss.servicesLock.Unlock() for _, id := range ids { + NotificationShared.CancelServiceState(id) delete(ss.serviceCurrentStatusData, id) delete(ss.serviceResponseDataStore, id) delete(ss.serviceResponsePing, id) @@ -669,32 +777,16 @@ func (ss *ServiceSentinel) processReport(r ReportData, serverShared *ServerClass } currentTime := time.Now() - if serviceCurrentStatusData.t.IsZero() { - serviceCurrentStatusData.t = currentTime - } - - // 写入当前数据 - if serviceCurrentStatusData.t.Before(currentTime) { - serviceCurrentStatusData.t = currentTime.Add(30 * time.Second) - serviceCurrentStatusData.result = append(serviceCurrentStatusData.result, mh) + ss.flushServiceHistory(mh.GetId(), serviceCurrentStatusData, currentTime) + // Include every accepted reporter's result. A fixed count of samples is + // not a 15-minute window when probes have different intervals or reporters. + serviceCurrentStatusData.appendResult(currentTime, r.Reporter, mh) + if serviceCurrentStatusData.lastHistoryAt.IsZero() { + serviceCurrentStatusData.lastHistoryAt = currentTime } // 更新当前状态 - ss.serviceResponseDataStore[mh.GetId()] = serviceResponseData{} - - // 永远是最新的 30 个数据的状态 [01:00, 02:00, 03:00] -> [04:00, 02:00, 03: 00] - for _, cs := range serviceCurrentStatusData.result { - if cs.GetId() > 0 { - rd := ss.serviceResponseDataStore[mh.GetId()] - if cs.Successful { - rd.Up++ - rd.Delay = (rd.Delay*float64(rd.Up-1) + float64(cs.Delay)) / float64(rd.Up) - } else { - rd.Down++ - } - ss.serviceResponseDataStore[mh.GetId()] = rd - } - } + ss.serviceResponseDataStore[mh.GetId()] = serviceCurrentStatusData.responseData() // 计算在线率， var stateCode uint8 @@ -704,24 +796,11 @@ func (ss *ServiceSentinel) processReport(r ReportData, serverShared *ServerClass if rd.Down+rd.Up > 0 { upPercent = rd.Up * 100 / (rd.Down + rd.Up) } - stateCode = GetStatusCode(upPercent) - } - - if len(serviceCurrentStatusData.result) == _CurrentStatusSize { - serviceCurrentStatusData.t = currentTime - if !TSDBEnabled() { - rd := ss.serviceResponseDataStore[mh.GetId()] - if err := DB.Create(&model.ServiceHistory{ - ServiceID: mh.GetId(), - AvgDelay: rd.Delay, - Data: mh.Data, - Up: rd.Up, - Down: rd.Down, - }).Error; err != nil { - log.Printf("NEZHA>> Failed to save service monitor metrics: %v", err) - } + if rd.Down+rd.Up == 0 { + stateCode = StatusNoData + } else { + stateCode = GetStatusCode(upPercent) } - serviceCurrentStatusData.result = serviceCurrentStatusData.result[:0] } // 延迟报警 @@ -734,77 +813,100 @@ func (ss *ServiceSentinel) processReport(r ReportData, serverShared *ServerClass lastStatus := serviceCurrentStatusData.lastStatus // 存储新的状态值 serviceCurrentStatusData.lastStatus = stateCode - - notifyCheck(&r, m, cs, mh, lastStatus, stateCode) - } - - // TLS 证书报警 - if ss.serviceReportBeforeTLSSideEffectsHook != nil { - ss.serviceReportBeforeTLSSideEffectsHook(mh.GetId()) - } - var errMsg string - if strings.HasPrefix(mh.Data, "SSL证书错误：") { - // i/o timeout、connection timeout、EOF 错误 - if !strings.HasSuffix(mh.Data, "timeout") && - !strings.HasSuffix(mh.Data, "EOF") && - !strings.HasSuffix(mh.Data, "timed out") { - errMsg = mh.Data - if cs.Notify { - muteLabel := NotificationMuteLabel.ServiceTLS(mh.GetId(), "network") - go NotificationShared.SendNotification(cs.NotificationGroupID, Localizer.Tf("[TLS] Fetch cert info failed, Reporter: %s, Error: %s", cs.Name, errMsg), muteLabel) - } + if stateCode != lastStatus { + serviceCurrentStatusData.stateEpoch = serviceEventSequence.Add(1) } - } else { - // 清除网络错误静音缓存 - NotificationShared.UnMuteNotification(cs.NotificationGroupID, NotificationMuteLabel.ServiceTLS(mh.GetId(), "network")) - var newCert = strings.Split(mh.Data, "|") - if len(newCert) > 1 { - enableNotify := cs.Notify + notifyCheck(&r, m, cs, mh, lastStatus, stateCode, serviceCurrentStatusData.stateEpoch) + } - // 首次获取证书信息时，缓存证书信息 - if ss.tlsCertCache[mh.GetId()] == "" { - ss.tlsCertCache[mh.GetId()] = mh.Data + // TLS 证书报警仅适用于 HTTPS 探针。其它探针的任意响应文本 + // 不得被当作证书资料解析。 + if mh.Type == model.TaskTypeHTTPGet && strings.HasPrefix(strings.ToLower(cs.Target), "https://") { + if ss.serviceReportBeforeTLSSideEffectsHook != nil { + ss.serviceReportBeforeTLSSideEffectsHook(mh.GetId()) + } + var errMsg string + if strings.HasPrefix(mh.Data, "SSL证书错误：") { + // i/o timeout、connection timeout、EOF 错误 + if !strings.HasSuffix(mh.Data, "timeout") && + !strings.HasSuffix(mh.Data, "EOF") && + !strings.HasSuffix(mh.Data, "timed out") { + errMsg = mh.Data + if cs.Notify { + muteLabel := NotificationMuteLabel.ServiceTLS(mh.GetId(), "network") + NotificationShared.SendNotificationAsync(cs.NotificationGroupID, Localizer.Tf("[TLS] Fetch cert info failed, Reporter: %s, Error: %s", cs.Name, errMsg), muteLabel) + } } + } else { + // 清除网络错误静音缓存 + NotificationShared.UnMuteNotification(cs.NotificationGroupID, NotificationMuteLabel.ServiceTLS(mh.GetId(), "network")) - oldCert := strings.Split(ss.tlsCertCache[mh.GetId()], "|") - isCertChanged := false - expiresOld, _ := time.Parse("2006-01-02 15:04:05 -0700 MST", oldCert[1]) - expiresNew, _ := time.Parse("2006-01-02 15:04:05 -0700 MST", newCert[1]) - - // 证书变更时，更新缓存 - if oldCert[0] != newCert[0] && !expiresNew.Equal(expiresOld) { - isCertChanged = true - ss.tlsCertCache[mh.GetId()] = mh.Data - } - - notificationGroupID := cs.NotificationGroupID - serviceName := cs.Name - - // 需要发送提醒 - if enableNotify { - // 证书过期提醒 - if expiresNew.Before(time.Now().AddDate(0, 0, 7)) { - expiresTimeStr := expiresNew.Format("2006-01-02 15:04:05") - errMsg = Localizer.Tf( - "The TLS certificate will expire within seven days. Expiration time: %s", - expiresTimeStr, - ) - - // 静音规则： 服务id+证书过期时间 - // 用于避免多个监测点对相同证书同时报警 - muteLabel := NotificationMuteLabel.ServiceTLS(mh.GetId(), fmt.Sprintf("expire_%s", expiresTimeStr)) - go NotificationShared.SendNotification(notificationGroupID, fmt.Sprintf("[TLS] %s %s", serviceName, errMsg), muteLabel) + var newCert = strings.Split(mh.Data, "|") + if len(newCert) > 1 { + enableNotify := cs.Notify + expiresNew, newErr := time.Parse("2006-01-02 15:04:05 -0700 MST", newCert[1]) + if newErr != nil { + log.Printf("NEZHA>> Ignoring malformed TLS certificate expiry for service %d", mh.GetId()) + return } - // 证书变更提醒 - if isCertChanged { - errMsg = Localizer.Tf( - "TLS certificate changed, old: issuer %s, expires at %s; new: issuer %s, expires at %s", - oldCert[0], expiresOld.Format("2006-01-02 15:04:05"), newCert[0], expiresNew.Format("2006-01-02 15:04:05")) + // 首次获取证书信息时，缓存证书信息 + if ss.tlsCertCache[mh.GetId()] == "" { + ss.tlsCertCache[mh.GetId()] = mh.Data + } - // 证书变更后会自动更新缓存，所以不需要静音 - go NotificationShared.SendNotification(notificationGroupID, fmt.Sprintf("[TLS] %s %s", serviceName, errMsg), "") + oldCert := strings.Split(ss.tlsCertCache[mh.GetId()], "|") + isCertChanged := false + expiresOld, oldErr := time.Parse("2006-01-02 15:04:05 -0700 MST", oldCert[1]) + if oldErr != nil { + log.Printf("NEZHA>> Ignoring malformed TLS certificate expiry for service %d", mh.GetId()) + return + } + + // 只有通知成功后才推进基线；失败时下一次探针结果会重试。 + if oldCert[0] != newCert[0] || !expiresNew.Equal(expiresOld) { + isCertChanged = true + if !enableNotify { + ss.tlsCertCache[mh.GetId()] = mh.Data + } + } + + notificationGroupID := cs.NotificationGroupID + serviceName := cs.Name + + // 需要发送提醒 + if enableNotify { + // 证书过期提醒 + if expiresNew.Before(time.Now().AddDate(0, 0, 7)) { + expiresTimeStr := expiresNew.Format("2006-01-02 15:04:05") + errMsg = Localizer.Tf( + "The TLS certificate will expire within seven days. Expiration time: %s", + expiresTimeStr, + ) + + // 静音规则： 服务id+证书过期时间 + // 用于避免多个监测点对相同证书同时报警 + muteLabel := NotificationMuteLabel.ServiceTLS(mh.GetId(), fmt.Sprintf("expire_%s", expiresTimeStr)) + NotificationShared.SendNotificationAsync(notificationGroupID, fmt.Sprintf("[TLS] %s %s", serviceName, errMsg), muteLabel) + } + + // 证书变更提醒 + if isCertChanged { + errMsg = Localizer.Tf( + "TLS certificate changed, old: issuer %s, expires at %s; new: issuer %s, expires at %s", + oldCert[0], expiresOld.Format("2006-01-02 15:04:05"), newCert[0], expiresNew.Format("2006-01-02 15:04:05")) + + oldRaw, newRaw, serviceID := ss.tlsCertCache[mh.GetId()], mh.Data, mh.GetId() + muteLabel := NotificationMuteLabel.ServiceTLS(serviceID, fmt.Sprintf("change_%x", sha256.Sum256([]byte(newRaw)))) + NotificationShared.sendNotificationAsync(notificationGroupID, fmt.Sprintf("[TLS] %s %s", serviceName, errMsg), muteLabel, func() { + ss.serviceResponseDataStoreLock.Lock() + if ss.serviceCurrentStatusData[serviceID] != nil && ss.tlsCertCache[serviceID] == oldRaw { + ss.tlsCertCache[serviceID] = newRaw + } + ss.serviceResponseDataStoreLock.Unlock() + }) + } } } } @@ -832,11 +934,11 @@ func delayCheck(r *ReportData, m map[uint64]*model.Server, ss *model.Service, mh if mh.Delay > ss.MaxLatency { // 延迟超过最大值 msg := Localizer.Tf("[Latency] %s %2f > %2f, Reporter: %s", ss.Name, mh.Delay, ss.MaxLatency, reporterServer.Name) - go NotificationShared.SendNotification(notificationGroupID, msg, minMuteLabel) + NotificationShared.SendNotificationAsync(notificationGroupID, msg, minMuteLabel) } else if mh.Delay < ss.MinLatency { // 延迟低于最小值 msg := Localizer.Tf("[Latency] %s %2f < %2f, Reporter: %s", ss.Name, mh.Delay, ss.MinLatency, reporterServer.Name) - go NotificationShared.SendNotification(notificationGroupID, msg, maxMuteLabel) + NotificationShared.SendNotificationAsync(notificationGroupID, msg, maxMuteLabel) } else { // 正常延迟， 清除静音缓存 NotificationShared.UnMuteNotification(notificationGroupID, minMuteLabel) @@ -845,7 +947,7 @@ func delayCheck(r *ReportData, m map[uint64]*model.Server, ss *model.Service, mh } func notifyCheck(r *ReportData, m map[uint64]*model.Server, - ss *model.Service, mh *pb.TaskResult, lastStatus, stateCode uint8) { + ss *model.Service, mh *pb.TaskResult, lastStatus, stateCode uint8, stateEpoch uint64) { // GHSA-jx78-55p5-rwv5: guard against concurrent server deletion (same TOCTOU // class as the 2026-07-21 fix, a few dozen lines lower in the same worker). // ServerShared has its own lock; m is a snapshot taken outside @@ -858,14 +960,9 @@ func notifyCheck(r *ReportData, m map[uint64]*model.Server, if isNeedSendNotification && reporterServer != nil { notificationGroupID := ss.NotificationGroupID notificationMsg := Localizer.Tf("[%s] %s Reporter: %s, Error: %s", StatusCodeToString(stateCode), ss.Name, reporterServer.Name, mh.Data) - muteLabel := NotificationMuteLabel.ServiceStateChanged(mh.GetId()) + muteLabel := fmt.Sprintf("%s:%d:%d", NotificationMuteLabel.ServiceStateChanged(mh.GetId()), stateCode, stateEpoch) - // 状态变更时，清除静音缓存 - if stateCode != lastStatus { - NotificationShared.UnMuteNotification(notificationGroupID, muteLabel) - } - - go NotificationShared.SendNotification(notificationGroupID, notificationMsg, muteLabel) + NotificationShared.SendServiceState(mh.GetId(), notificationGroupID, notificationMsg, muteLabel) } // 判断是否需要触发任务 @@ -890,9 +987,6 @@ const ( ) func GetStatusCode[T constraints.Float | constraints.Integer](percent T) uint8 { - if percent == 0 { - return StatusNoData - } if percent > 95 { return StatusGood } diff --git a/service/singleton/servicesentinel_startup_history_test.go b/service/singleton/servicesentinel_startup_history_test.go new file mode 100644 index 00000000..8656c880 --- /dev/null +++ b/service/singleton/servicesentinel_startup_history_test.go @@ -0,0 +1,94 @@ +package singleton + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/nezhahq/nezha/model" +) + +func TestServiceSentinelStartupWithRetainedHistory(t *testing.T) { + for _, serviceState := range []string{"retained", "deleted", "quarantined"} { + t.Run(serviceState, func(t *testing.T) { + initial := newServiceMonitorSecurityHarness(t) + initial.Close() + require.False(t, TSDBEnabled()) + + activeService := &model.Service{Name: "active monitor", Type: model.TaskTypeTCPPing} + historicalService := &model.Service{Name: "historical monitor", Type: model.TaskTypeHTTPGet} + require.NoError(t, DB.Create(activeService).Error) + require.NoError(t, DB.Create(historicalService).Error) + + year, month, day := time.Now().Date() + today := time.Date(year, month, day, 0, 0, 0, 0, Loc) + // Use the production writer so retained rows have the same shape as + // ordinary aggregated history, including server_id = 0. + historicalStatus := &serviceTaskStatus{ + lastHistoryAt: today, + history: serviceResponseData{Up: 7, Down: 2, Delay: 90}, + } + initial.flushServiceHistory(historicalService.ID, historicalStatus, today.Add(15*time.Minute)) + var retainedHistory []model.ServiceHistory + require.NoError(t, DB.Where("service_id = ?", historicalService.ID).Find(&retainedHistory).Error) + require.Len(t, retainedHistory, 1) + require.Zero(t, retainedHistory[0].ServerID) + require.Equal(t, uint64(7), retainedHistory[0].Up) + require.Equal(t, uint64(2), retainedHistory[0].Down) + require.Equal(t, float64(90), retainedHistory[0].AvgDelay) + + require.NoError(t, DB.Create(&[]model.ServiceHistory{ + {ServiceID: activeService.ID, CreatedAt: today, Up: 5, Down: 1, AvgDelay: 10}, + {ServiceID: activeService.ID, CreatedAt: today, Up: 3, Down: 2, AvgDelay: 30}, + {ServiceID: activeService.ID, CreatedAt: today.Add(-time.Hour), Up: 4, Down: 1, AvgDelay: 15}, + {ServiceID: activeService.ID, ServerID: 100, CreatedAt: today, Up: 100, AvgDelay: 500}, + }).Error) + + switch serviceState { + case "deleted": + // Match the controller's hard delete without cleaning history: old + // databases must remain loadable even if deletion cleanup changes. + result := DB.Unscoped().Delete(&model.Service{}, "id in (?)", []uint64{historicalService.ID}) + require.NoError(t, result.Error) + require.EqualValues(t, 1, result.RowsAffected) + case "quarantined": + // Legacy invalid service types are deliberately excluded at startup. + result := DB.Model(historicalService).UpdateColumn("type", model.TaskTypeCommand) + require.NoError(t, result.Error) + require.EqualValues(t, 1, result.RowsAffected) + } + var historyCount int64 + require.NoError(t, DB.Model(&model.ServiceHistory{}).Where("service_id = ?", historicalService.ID).Count(&historyCount).Error) + require.EqualValues(t, 1, historyCount) + + var restarted *ServiceSentinel + var startupErr error + require.NotPanics(t, func() { + restarted, startupErr = NewServiceSentinel(make(chan *model.Service, 1)) + }) + if restarted != nil { + t.Cleanup(restarted.Close) + } + require.NoError(t, startupErr) + require.NotNil(t, restarted) + require.Equal(t, &_TodayStatsOfService{Up: 8, Down: 3, Delay: 20}, restarted.serviceStatusToday[activeService.ID]) + monthly := restarted.monthlyStatus[activeService.ID] + require.NotNil(t, monthly) + require.Equal(t, uint64(12), monthly.TotalUp) + require.Equal(t, uint64(4), monthly.TotalDown) + + if serviceState == "retained" { + require.Len(t, restarted.services, 2) + require.Equal(t, &_TodayStatsOfService{Up: 7, Down: 2, Delay: 90}, restarted.serviceStatusToday[historicalService.ID]) + require.Equal(t, uint64(7), restarted.monthlyStatus[historicalService.ID].TotalUp) + require.Equal(t, uint64(2), restarted.monthlyStatus[historicalService.ID].TotalDown) + } else { + require.Len(t, restarted.services, 1) + require.NotContains(t, restarted.services, historicalService.ID) + require.NotContains(t, restarted.serviceStatusToday, historicalService.ID) + require.NotContains(t, restarted.monthlyStatus, historicalService.ID) + } + }) + } +} diff --git a/service/singleton/servicesentinel_window_test.go b/service/singleton/servicesentinel_window_test.go new file mode 100644 index 00000000..e5c85147 --- /dev/null +++ b/service/singleton/servicesentinel_window_test.go @@ -0,0 +1,63 @@ +package singleton + +import ( + "testing" + "time" + + pb "github.com/nezhahq/nezha/proto" +) + +func TestServiceWindowCountsFirstFailureAsDown(t *testing.T) { + status := new(serviceTaskStatus) + status.appendResult(time.Unix(1000, 0), 1, &pb.TaskResult{Successful: false}) + data := status.responseData() + if data.Down != 1 || data.Up != 0 || GetStatusCode(uint64(0)) != StatusDown { + t.Fatalf("first failed report = %+v, status %d; want Down", data, GetStatusCode(uint64(0))) + } +} + +func TestServiceWindowExpiresOldReportsAndDoesNotResetAtThirty(t *testing.T) { + status := new(serviceTaskStatus) + start := time.Unix(1000, 0) + for i := 0; i < 30; i++ { + status.appendResult(start.Add(time.Duration(i)*30*time.Second), 1, &pb.TaskResult{Successful: true}) + } + if data := status.responseData(); data.Up != 30 { + t.Fatalf("30th report reset window: %+v", data) + } + status.appendResult(start.Add(29*30*time.Second+time.Second), 2, &pb.TaskResult{Successful: false}) + if data := status.responseData(); data.Up != 30 || data.Down != 1 { + t.Fatalf("31st report corrupted window: %+v", data) + } + status.appendResult(start.Add(31*time.Minute), 1, &pb.TaskResult{Successful: false}) + if data := status.responseData(); data.Up != 0 || data.Down != 1 { + t.Fatalf("expired reports still influence status: %+v", data) + } +} + +func TestServiceWindowDeduplicatesFloodWithinBucket(t *testing.T) { + status := new(serviceTaskStatus) + now := time.Unix(1000, 0) + for i := 0; i < 10000; i++ { + status.appendResult(now, 1, &pb.TaskResult{Successful: false, Data: "ignored payload"}) + } + if len(status.result) != 1 || len(status.result[0].reporters) != 1 { + t.Fatalf("flood grew live window: buckets=%d", len(status.result)) + } + if data := status.responseData(); data.Down != 1 { + t.Fatalf("duplicate reporter skewed status: %+v", data) + } + if status.history.Down != 10000 { + t.Fatalf("history lost raw reports: %+v", status.history) + } +} + +func TestServiceWindowDoesNotEraseFailureWithLaterSuccess(t *testing.T) { + status := new(serviceTaskStatus) + now := time.Unix(1000, 0) + status.appendResult(now, 1, &pb.TaskResult{Successful: false}) + status.appendResult(now.Add(time.Second), 1, &pb.TaskResult{Successful: true, Delay: 20}) + if data := status.responseData(); data.Down != 1 || data.Up != 1 { + t.Fatalf("later success erased failure in same bucket: %+v", data) + } +} diff --git a/service/singleton/singleton.go b/service/singleton/singleton.go index 03cc29a5..a65214a7 100644 --- a/service/singleton/singleton.go +++ b/service/singleton/singleton.go @@ -63,6 +63,7 @@ func LoadSingleton(bus chan<- *model.Service) (err error) { ServerShared = NewServerClass() CronShared = NewCronClass() ServerTransferShared = NewServerTransferClass() + OnlineHistoryShared = NewOnlineHistoryRecorder() // 最后初始化 ServiceSentinel ServiceSentinelShared, err = NewServiceSentinel(bus) if err == nil { @@ -97,7 +98,7 @@ func InitDBFromPath(path string) error { model.Cron{}, model.Transfer{}, model.ServerGroupServer{}, model.NAT{}, model.DDNSProfile{}, model.NotificationGroupNotification{}, model.WAF{}, model.Oauth2Bind{}, model.Domain{}, model.ServerTransfer{}, model.JWTSession{}, - model.APIToken{}, model.MCPAuditLog{}) + model.APIToken{}, model.MCPAuditLog{}, model.OnlineSession{}) if err != nil { return err @@ -171,7 +172,7 @@ func CleanMonitorHistory() { for _, alert := range alerts { for _, rule := range alert.Rules { // 是不是流量记录规则 - if !rule.IsTransferDurationRule() { + if !rule.HasSafeCycleConfiguration() { continue } dataCouldRemoveBefore := rule.GetTransferDurationStart().UTC() diff --git a/service/singleton/tsdb.go b/service/singleton/tsdb.go index 745e83d6..e5311bb1 100644 --- a/service/singleton/tsdb.go +++ b/service/singleton/tsdb.go @@ -1,6 +1,7 @@ package singleton import ( + "errors" "log" "time" @@ -10,6 +11,8 @@ import ( var TSDBShared *tsdb.TSDB +var openTSDB = tsdb.Open + func InitTSDB() error { config := &tsdb.Config{ RetentionDays: 30, @@ -44,11 +47,21 @@ func InitTSDB() error { return nil } - var err error - TSDBShared, err = tsdb.Open(config) + TSDBShared = nil + db, err := openTSDB(config) if err != nil { + if errors.Is(err, tsdb.ErrDiskFull) { + log.Printf("NEZHA>> Warning: TSDB is unavailable because its disk is full; dashboard and alerts will continue without TSDB writes: %v", err) + if DB != nil { + if migrateErr := DB.AutoMigrate(model.ServiceHistory{}); migrateErr != nil { + log.Printf("NEZHA>> Warning: failed to prepare SQLite service history fallback: %v", migrateErr) + } + } + return nil + } return err } + TSDBShared = db log.Println("NEZHA>> TSDB initialized successfully") diff --git a/service/singleton/tsdb_disk_guard_test.go b/service/singleton/tsdb_disk_guard_test.go new file mode 100644 index 00000000..50f71f3f --- /dev/null +++ b/service/singleton/tsdb_disk_guard_test.go @@ -0,0 +1,58 @@ +package singleton + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/nezhahq/nezha/model" + "github.com/nezhahq/nezha/pkg/tsdb" +) + +func TestInitTSDBContinuesWhenDiskIsFull(t *testing.T) { + originalConf := Conf + originalDB := DB + originalShared := TSDBShared + originalOpen := openTSDB + t.Cleanup(func() { + Conf = originalConf + DB = originalDB + TSDBShared = originalShared + openTSDB = originalOpen + }) + + Conf = &ConfigClass{Config: &model.Config{ + TSDB: model.TSDBConf{DataPath: "/full/tsdb"}, + }} + DB = nil + TSDBShared = nil + openTSDB = func(*tsdb.Config) (*tsdb.TSDB, error) { + return nil, fmt.Errorf("open failed: %w", tsdb.ErrDiskFull) + } + + require.NoError(t, InitTSDB()) + require.Nil(t, TSDBShared) +} + +func TestInitTSDBStillReturnsNonDiskErrors(t *testing.T) { + originalConf := Conf + originalShared := TSDBShared + originalOpen := openTSDB + t.Cleanup(func() { + Conf = originalConf + TSDBShared = originalShared + openTSDB = originalOpen + }) + + Conf = &ConfigClass{Config: &model.Config{ + TSDB: model.TSDBConf{DataPath: "/broken/tsdb"}, + }} + TSDBShared = nil + openTSDB = func(*tsdb.Config) (*tsdb.TSDB, error) { + return nil, fmt.Errorf("permission denied") + } + + require.ErrorContains(t, InitTSDB(), "permission denied") + require.Nil(t, TSDBShared) +}