Merge upstream monitoring features preserving domain extensions and require fixed Go toolchain

pkg/ddns/ddns.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log"
 	"net/netip"
+	"strings"
 	"time"
 
 	"github.com/libdns/libdns"
@@ -21,9 +22,6 @@ const (
 )
 
 type Provider struct {
-	prefix string
-	zone   string
-
 	DDNSProfile *model.DDNSProfile
 	IPAddrs     *model.IP
 	Setter      libdns.RecordSetter
@@ -34,61 +32,81 @@ func (provider *Provider) GetProfileID() uint64 {
 }
 
 func (provider *Provider) UpdateDomain(ctx context.Context, overrideDomains ...string) {
-	for _, domain := range utils.IfOr(len(overrideDomains) > 0, overrideDomains, provider.DDNSProfile.Domains) {
-		for retries := range int(provider.DDNSProfile.MaxRetries) {
-			log.Printf("NEZHA>> Updating DNS Record of domain %s: %d/%d", domain, retries+1, provider.DDNSProfile.MaxRetries)
-			if err := provider.updateDomain(ctx, domain); err != nil {
-				log.Printf("NEZHA>> Failed to update DNS record of domain %s: %v", domain, err)
-			} else {
-				log.Printf("NEZHA>> Update DNS record of domain %s succeeded", domain)
+	domains := utils.IfOr(len(overrideDomains) > 0, overrideDomains, provider.DDNSProfile.Domains)
+	maxRetries := int(provider.DDNSProfile.MaxRetries)
+	if maxRetries <= 0 {
+		maxRetries = 1
+	}
+
+	for _, domain := range domains {
+		var prefix, zone string
+		var soaErr error
+
+		for retries := 0; retries < maxRetries; retries++ {
+			prefix, zone, soaErr = provider.splitDomainSOA(ctx, domain)
+			if soaErr == nil {
 				break
 			}
+			log.Printf("NEZHA>> Failed to split domain SOA for %s (attempt %d/%d): %v", domain, retries+1, maxRetries, soaErr)
+		}
+
+		if soaErr != nil {
+			log.Printf("NEZHA>> Failed to split domain SOA for %s after %d retries, skipping domain", domain, maxRetries)
+			continue
+		}
+
+		// 独立处理 IPv4 更新或删除
+		if provider.DDNSProfile.EnableIPv4 != nil && *provider.DDNSProfile.EnableIPv4 {
+			for retries := 0; retries < maxRetries; retries++ {
+				log.Printf("NEZHA>> Updating IPv4 record of domain %s: %d/%d", domain, retries+1, maxRetries)
+				var ipv4Err error
+				if provider.IPAddrs.IPv4Addr == "" {
+					ipv4Err = provider.deleteDomainRecord(ctx, prefix, zone, "A")
+				} else {
+					ipv4Err = provider.addDomainRecord(ctx, prefix, zone, "A", provider.IPAddrs.IPv4Addr)
+				}
+
+				if ipv4Err != nil {
+					log.Printf("NEZHA>> Failed to update IPv4 record of domain %s: %v", domain, ipv4Err)
+				} else {
+					log.Printf("NEZHA>> Update IPv4 record of domain %s succeeded", domain)
+					break
+				}
+			}
+		}
+
+		// 独立处理 IPv6 更新或删除
+		if provider.DDNSProfile.EnableIPv6 != nil && *provider.DDNSProfile.EnableIPv6 {
+			for retries := 0; retries < maxRetries; retries++ {
+				log.Printf("NEZHA>> Updating IPv6 record of domain %s: %d/%d", domain, retries+1, maxRetries)
+				var ipv6Err error
+				if provider.IPAddrs.IPv6Addr == "" {
+					ipv6Err = provider.deleteDomainRecord(ctx, prefix, zone, "AAAA")
+				} else {
+					ipv6Err = provider.addDomainRecord(ctx, prefix, zone, "AAAA", provider.IPAddrs.IPv6Addr)
+				}
+
+				if ipv6Err != nil {
+					log.Printf("NEZHA>> Failed to update IPv6 record of domain %s: %v", domain, ipv6Err)
+				} else {
+					log.Printf("NEZHA>> Update IPv6 record of domain %s succeeded", domain)
+					break
+				}
+			}
 		}
 	}
 }
 
-func (provider *Provider) updateDomain(ctx context.Context, domain string) error {
-	var err error
-	provider.prefix, provider.zone, err = provider.splitDomainSOA(ctx, domain)
-	if err != nil {
-		return err
-	}
-
-	// 独立处理 IPv4 更新
-	if *provider.DDNSProfile.EnableIPv4 {
-		if provider.IPAddrs.IPv4Addr == "" {
-			log.Printf("NEZHA>> Skip IPv4 update for domain %s: IPv4 address is empty", domain)
-		} else {
-			if err = provider.addDomainRecord(ctx, "A", provider.IPAddrs.IPv4Addr); err != nil {
-				return err
-			}
-		}
-	}
-
-	// 独立处理 IPv6 更新
-	if *provider.DDNSProfile.EnableIPv6 {
-		if provider.IPAddrs.IPv6Addr == "" {
-			log.Printf("NEZHA>> Skip IPv6 update for domain %s: IPv6 address is empty", domain)
-		} else {
-			if err = provider.addDomainRecord(ctx, "AAAA", provider.IPAddrs.IPv6Addr); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (provider *Provider) addDomainRecord(ctx context.Context, recType, addr string) error {
+func (provider *Provider) addDomainRecord(ctx context.Context, prefix, zone, recType, addr string) error {
 	netipAddr, err := netip.ParseAddr(addr)
 	if err != nil {
 		return fmt.Errorf("parse error: %v", err)
 	}
 
-	_, err = provider.Setter.SetRecords(ctx, provider.zone,
+	_, err = provider.Setter.SetRecords(ctx, zone,
 		[]libdns.Record{
 			libdns.Address{
-				Name: provider.prefix,
+				Name: prefix,
 				IP:   netipAddr,
 				TTL:  time.Minute,
 			},
@@ -96,6 +114,28 @@ func (provider *Provider) addDomainRecord(ctx context.Context, recType, addr str
 	return err
 }
 
+func (provider *Provider) deleteDomainRecord(ctx context.Context, prefix, zone, recType string) error {
+	deleter, okDeleter := provider.Setter.(libdns.RecordDeleter)
+	if !okDeleter {
+		log.Printf("NEZHA>> DNS provider does not support RecordDeleter, safely skipping deletion for %s", recType)
+		return nil
+	}
+
+	targetRecType := strings.ToUpper(recType)
+	_, err := deleter.DeleteRecords(ctx, zone, []libdns.Record{
+		libdns.RR{
+			Name: prefix,
+			Type: targetRecType,
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("deleter.DeleteRecords failed: %w", err)
+	}
+
+	log.Printf("NEZHA>> Successfully deleted %s record for %s.%s", recType, prefix, zone)
+	return nil
+}
+
 func (provider *Provider) splitDomainSOA(ctx context.Context, domain string) (prefix string, zone string, err error) {
 	c := &dns.Client{Timeout: dnsTimeOut}
 
@@ -103,7 +143,6 @@ func (provider *Provider) splitDomainSOA(ctx context.Context, domain string) (pr
 	indexes := dns.Split(domain)
 
 	servers := utils.DNSServers
-
 	customDNSServers, _ := ctx.Value(DNSServerKey{}).([]string)
 	if len(customDNSServers) > 0 {
 		servers = customDNSServers
@@ -115,7 +154,7 @@ func (provider *Provider) splitDomainSOA(ctx context.Context, domain string) (pr
 			m.SetQuestion(domain[idx:], dns.TypeSOA)
 
 			r, _, err := c.Exchange(&m, server)
-			if err != nil {
+			if err != nil || r == nil {
 				continue
 			}
 

pkg/ddns/ddns_test.go

@@ -2,7 +2,15 @@ package ddns
 
 import (
 	"context"
+	"net"
+	"strings"
+	"sync/atomic"
 	"testing"
+
+	"github.com/libdns/libdns"
+	"github.com/miekg/dns"
+
+	"github.com/nezhahq/nezha/model"
 )
 
 type testSt struct {
@@ -12,6 +20,50 @@ type testSt struct {
 }
 
 func TestSplitDomainSOA(t *testing.T) {
+	handler := dns.NewServeMux()
+	handler.HandleFunc(".", func(w dns.ResponseWriter, r *dns.Msg) {
+		m := new(dns.Msg)
+		m.SetReply(r)
+		m.Authoritative = true
+		if len(r.Question) > 0 {
+			qname := r.Question[0].Name
+			var zoneName string
+			if strings.HasSuffix(qname, "example.co.uk.") {
+				zoneName = "example.co.uk."
+			} else if strings.HasSuffix(qname, "example.org.") {
+				zoneName = "example.org."
+			} else if strings.HasSuffix(qname, "example.com.") {
+				zoneName = "example.com."
+			}
+
+			if zoneName != "" {
+				soa := &dns.SOA{
+					Hdr:    dns.RR_Header{Name: zoneName, Rrtype: dns.TypeSOA, Class: dns.ClassINET, Ttl: 300},
+					Ns:     "ns.example.com.",
+					Mbox:   "admin.example.com.",
+					Serial: 1,
+				}
+				m.Answer = append(m.Answer, soa)
+				m.Ns = append(m.Ns, soa)
+			}
+		}
+		_ = w.WriteMsg(m)
+	})
+
+	pc, err := net.ListenPacket("udp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("failed to listen: %v", err)
+	}
+	defer pc.Close()
+
+	server := &dns.Server{PacketConn: pc, Net: "udp", Handler: handler}
+	go func() {
+		_ = server.ActivateAndServe()
+	}()
+	defer server.Shutdown()
+
+	addr := pc.LocalAddr().String()
+
 	cases := []testSt{
 		{
 			domain: "www.example.com",
@@ -30,13 +82,12 @@ func TestSplitDomainSOA(t *testing.T) {
 		},
 	}
 
-	ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{"1.1.1.1:53"})
+	ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{addr})
 	provider := &Provider{}
 	for _, c := range cases {
 		prefix, zone, err := provider.splitDomainSOA(ctx, c.domain)
 		if err != nil {
-			t.Skipf("Skipping live DNS test due to network: %v", err)
-			return
+			t.Fatalf("Error for domain %s: %s", c.domain, err)
 		}
 		if prefix != c.prefix {
 			t.Fatalf("Expected prefix %s, but got %s", c.prefix, prefix)
@@ -46,3 +97,191 @@ func TestSplitDomainSOA(t *testing.T) {
 		}
 	}
 }
+
+type MockSetter struct {
+	Records        []libdns.Record
+	SetCalled      bool
+	DelCalled      bool
+	DeletedRecords []libdns.Record
+}
+
+func (m *MockSetter) SetRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) {
+	m.SetCalled = true
+	m.Records = append(m.Records, records...)
+	return records, nil
+}
+
+func (m *MockSetter) DeleteRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) {
+	m.DelCalled = true
+	m.DeletedRecords = append(m.DeletedRecords, records...)
+	return records, nil
+}
+
+type MockUnsupportedSetter struct{}
+
+func (m *MockUnsupportedSetter) SetRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) {
+	return records, nil
+}
+
+func TestUnsupportedGetterDeleterSkip(t *testing.T) {
+	provider := &Provider{
+		DDNSProfile: &model.DDNSProfile{MaxRetries: 1},
+		IPAddrs:     &model.IP{IPv4Addr: ""},
+		Setter:      &MockUnsupportedSetter{},
+	}
+
+	err := provider.deleteDomainRecord(context.Background(), "sub", "example.com", "A")
+	if err != nil {
+		t.Fatalf("expected no error for unsupported deleter skip, got %v", err)
+	}
+}
+
+func TestIndependentDualStackProcessing(t *testing.T) {
+	enableIPv4 := true
+	enableIPv6 := true
+	mockSetter := &MockSetter{}
+
+	provider := &Provider{
+		DDNSProfile: &model.DDNSProfile{
+			MaxRetries: 1,
+			Domains:    []string{"example.com"},
+			EnableIPv4: &enableIPv4,
+			EnableIPv6: &enableIPv6,
+		},
+		IPAddrs: &model.IP{
+			IPv4Addr: "1.1.1.1",
+			IPv6Addr: "2001:db8::1",
+		},
+		Setter: mockSetter,
+	}
+
+	handler := dns.NewServeMux()
+	handler.HandleFunc(".", func(w dns.ResponseWriter, r *dns.Msg) {
+		m := new(dns.Msg)
+		m.SetReply(r)
+		m.Authoritative = true
+		if len(r.Question) > 0 {
+			soa := &dns.SOA{
+				Hdr:    dns.RR_Header{Name: r.Question[0].Name, Rrtype: dns.TypeSOA, Class: dns.ClassINET, Ttl: 300},
+				Ns:     "ns.example.com.",
+				Mbox:   "admin.example.com.",
+				Serial: 1,
+			}
+			m.Answer = append(m.Answer, soa)
+			m.Ns = append(m.Ns, soa)
+		}
+		_ = w.WriteMsg(m)
+	})
+
+	pc, err := net.ListenPacket("udp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("failed to listen: %v", err)
+	}
+	defer pc.Close()
+
+	server := &dns.Server{PacketConn: pc, Net: "udp", Handler: handler}
+	go func() {
+		_ = server.ActivateAndServe()
+	}()
+	defer server.Shutdown()
+
+	ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{pc.LocalAddr().String()})
+	provider.UpdateDomain(ctx)
+
+	if !mockSetter.SetCalled {
+		t.Fatalf("expected SetRecords to have been called for dual stack updates")
+	}
+
+	foundIPv4 := false
+	foundIPv6 := false
+	for _, rec := range mockSetter.Records {
+		if addrRec, ok := rec.(libdns.Address); ok {
+			if addrRec.IP.String() == "1.1.1.1" {
+				foundIPv4 = true
+			}
+			if addrRec.IP.String() == "2001:db8::1" {
+				foundIPv6 = true
+			}
+		}
+	}
+
+	if !foundIPv4 || !foundIPv6 {
+		t.Fatalf("expected both IPv4 and IPv6 records to be processed independently, got records: %v", mockSetter.Records)
+	}
+}
+
+func TestTransientSOAFailureAndRetry(t *testing.T) {
+	var attempts int64
+	handler := dns.NewServeMux()
+	handler.HandleFunc(".", func(w dns.ResponseWriter, r *dns.Msg) {
+		atomic.AddInt64(&attempts, 1)
+		m := new(dns.Msg)
+		m.SetReply(r)
+		_ = w.WriteMsg(m)
+	})
+
+	pc, err := net.ListenPacket("udp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("failed to listen: %v", err)
+	}
+	defer pc.Close()
+
+	server := &dns.Server{PacketConn: pc, Net: "udp", Handler: handler}
+	go func() {
+		_ = server.ActivateAndServe()
+	}()
+	defer server.Shutdown()
+
+	var maxRetries uint64 = 3
+	provider := &Provider{
+		DDNSProfile: &model.DDNSProfile{
+			MaxRetries: maxRetries,
+			Domains:    []string{"example.com"},
+			EnableIPv4: &[]bool{true}[0],
+			EnableIPv6: &[]bool{false}[0],
+		},
+		IPAddrs: &model.IP{IPv4Addr: "1.1.1.1"},
+		Setter:  &MockSetter{},
+	}
+
+	ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{pc.LocalAddr().String()})
+	provider.UpdateDomain(ctx)
+
+	actualAttempts := atomic.LoadInt64(&attempts)
+	expectedAttempts := int64(maxRetries) * 2
+	if actualAttempts != expectedAttempts {
+		t.Fatalf("expected exact attempt count of %d, got %d", expectedAttempts, actualAttempts)
+	}
+}
+
+func TestGenericRecordDeleterContract(t *testing.T) {
+	mockSetter := &MockSetter{}
+	provider := &Provider{
+		DDNSProfile: &model.DDNSProfile{MaxRetries: 1},
+		IPAddrs:     &model.IP{IPv4Addr: ""},
+		Setter:      mockSetter,
+	}
+
+	ctx := context.Background()
+	err := provider.deleteDomainRecord(ctx, "sub", "example.com", "A")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if !mockSetter.DelCalled {
+		t.Fatalf("expected DeleteRecords to be called")
+	}
+
+	if len(mockSetter.DeletedRecords) != 1 {
+		t.Fatalf("expected 1 record sent for deletion, got %d", len(mockSetter.DeletedRecords))
+	}
+
+	rr, ok := mockSetter.DeletedRecords[0].(libdns.RR)
+	if !ok {
+		t.Fatalf("expected deletion payload to be libdns.RR")
+	}
+
+	if rr.Name != "sub" || rr.Type != "A" {
+		t.Fatalf("unexpected RR fields: Name=%s, Type=%s", rr.Name, rr.Type)
+	}
+}

pkg/tsdb/disk_guard_test.go

@@ -0,0 +1,124 @@
+package tsdb
+
+import (
+	"path/filepath"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type testReadOnlyChecker struct {
+	value atomic.Bool
+}
+
+func (c *testReadOnlyChecker) IsReadOnly() bool {
+	return c.value.Load()
+}
+
+func newDiskGuardTestDB(t *testing.T) *TSDB {
+	t.Helper()
+	db, err := Open(&Config{
+		DataPath:           filepath.Join(t.TempDir(), "tsdb"),
+		RetentionDays:      1,
+		MinFreeDiskSpaceGB: 1,
+		DedupInterval:      time.Second,
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, db.Close()) })
+	return db
+}
+
+func bufferedRows(w *bufferedWriter) int {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return len(w.buffer)
+}
+
+func TestReadOnlyStorageDropsWritesAndKeepsQueriesAvailable(t *testing.T) {
+	db := newDiskGuardTestDB(t)
+	checker := &testReadOnlyChecker{}
+	db.readOnly = checker
+	originalAddRows := db.addRowsFn
+	var addRowsCalls atomic.Int32
+	db.addRowsFn = func(rows []storage.MetricRow, precisionBits uint8) {
+		addRowsCalls.Add(1)
+		originalAddRows(rows, precisionBits)
+	}
+
+	firstTimestamp := time.Now().Add(-time.Minute)
+	require.NoError(t, db.WriteServerMetrics(&ServerMetrics{
+		ServerID:  1,
+		Timestamp: firstTimestamp,
+		CPU:       10,
+	}))
+	db.Flush()
+	require.Equal(t, int32(1), addRowsCalls.Load())
+
+	before, err := db.QueryServerMetrics(1, MetricServerCPU, Period1Day)
+	require.NoError(t, err)
+	require.NotEmpty(t, before)
+
+	checker.value.Store(true)
+	require.NoError(t, db.WriteServerMetrics(&ServerMetrics{
+		ServerID:  1,
+		Timestamp: firstTimestamp.Add(2 * time.Second),
+		CPU:       20,
+	}))
+	assert.True(t, db.WritesPaused())
+	assert.Zero(t, bufferedRows(db.writer), "read-only writes must not accumulate in memory")
+	assert.Equal(t, int32(1), addRowsCalls.Load(), "read-only samples must not reach VictoriaMetrics")
+
+	afterDrop, err := db.QueryServerMetrics(1, MetricServerCPU, Period1Day)
+	require.NoError(t, err)
+	assert.Equal(t, before, afterDrop, "read-only mode must preserve existing query access")
+
+	checker.value.Store(false)
+	require.NoError(t, db.WriteServerMetrics(&ServerMetrics{
+		ServerID:  1,
+		Timestamp: firstTimestamp.Add(4 * time.Second),
+		CPU:       30,
+	}))
+	db.Flush()
+	assert.False(t, db.WritesPaused())
+	assert.Equal(t, int32(2), addRowsCalls.Load(), "writes must resume after storage leaves read-only mode")
+}
+
+func TestDiskFullAddRowsPanicPausesWrites(t *testing.T) {
+	db := newDiskGuardTestDB(t)
+	db.addRowsFn = func([]storage.MetricRow, uint8) {
+		panic("write data: no space left on device")
+	}
+
+	require.NoError(t, db.WriteServerMetrics(&ServerMetrics{
+		ServerID:  1,
+		Timestamp: time.Now(),
+		CPU:       10,
+	}))
+	assert.NotPanics(t, db.Flush)
+	assert.True(t, db.WritesPaused())
+
+	require.NoError(t, db.WriteServerMetrics(&ServerMetrics{
+		ServerID:  1,
+		Timestamp: time.Now().Add(time.Second),
+		CPU:       20,
+	}))
+	assert.Zero(t, bufferedRows(db.writer))
+}
+
+func TestNonDiskStoragePanicIsNotHidden(t *testing.T) {
+	db := newDiskGuardTestDB(t)
+	db.addRowsFn = func([]storage.MetricRow, uint8) {
+		panic("storage invariant failed")
+	}
+
+	require.NoError(t, db.WriteServerMetrics(&ServerMetrics{
+		ServerID:  1,
+		Timestamp: time.Now(),
+		CPU:       10,
+	}))
+	assert.PanicsWithValue(t, "storage invariant failed", db.Flush)
+}

pkg/tsdb/maintenance.go

@@ -10,8 +10,11 @@ func (db *TSDB) Maintenance() {
 	if db.closed {
 		return
 	}
+	if !db.acceptsWrites() {
+		return
+	}
 
 	log.Println("NEZHA>> TSDB starting maintenance (flush)...")
-	db.storage.DebugFlush()
+	db.debugFlushSafely()
 	log.Println("NEZHA>> TSDB maintenance completed")
 }

pkg/tsdb/tsdb.go

@@ -1,15 +1,26 @@
 package tsdb
 
 import (
+	"errors"
 	"fmt"
 	"log"
 	"path/filepath"
+	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
+// ErrDiskFull is returned when VictoriaMetrics cannot initialize because the
+// filesystem containing the TSDB has run out of space.
+var ErrDiskFull = errors.New("TSDB disk is full")
+
+type readOnlyChecker interface {
+	IsReadOnly() bool
+}
+
 // TSDB 封装 VictoriaMetrics 存储
 type TSDB struct {
 	storage *storage.Storage
@@ -18,6 +29,13 @@ type TSDB struct {
 	closed  bool
 
 	writer *bufferedWriter
+
+	readOnly     readOnlyChecker
+	addRowsFn    func([]storage.MetricRow, uint8)
+	debugFlushFn func()
+
+	readOnlyObserved atomic.Bool
+	diskFullPaused   atomic.Bool
 }
 
 // InitGlobalSettings 初始化 VictoriaMetrics 包级别的全局设置。
@@ -35,7 +53,18 @@ func InitGlobalSettings(config *Config) {
 }
 
 // Open 打开或创建 TSDB 存储
-func Open(config *Config) (*TSDB, error) {
+func Open(config *Config) (db *TSDB, err error) {
+	defer func() {
+		if recovered := recover(); recovered != nil {
+			if diskErr := diskFullPanicError(recovered); diskErr != nil {
+				db = nil
+				err = diskErr
+				return
+			}
+			panic(recovered)
+		}
+	}()
+
 	if config == nil {
 		config = DefaultConfig()
 	}
@@ -59,9 +88,12 @@ func Open(config *Config) (*TSDB, error) {
 
 	stor := storage.MustOpenStorage(dataPath, opts)
 
-	db := &TSDB{
-		storage: stor,
-		config:  config,
+	db = &TSDB{
+		storage:      stor,
+		config:       config,
+		readOnly:     stor,
+		addRowsFn:    stor.AddRows,
+		debugFlushFn: stor.DebugFlush,
 	}
 
 	db.writer = newBufferedWriter(db, config.WriteBufferSize, config.WriteBufferFlushInterval)
@@ -72,6 +104,86 @@ func Open(config *Config) (*TSDB, error) {
 	return db, nil
 }
 
+func diskFullPanicError(recovered any) error {
+	message := strings.ToLower(fmt.Sprint(recovered))
+	for _, marker := range []string{
+		"no space left on device",
+		"disk quota exceeded",
+		"not enough space on the disk",
+	} {
+		if strings.Contains(message, marker) {
+			return fmt.Errorf("%w: %v", ErrDiskFull, recovered)
+		}
+	}
+	return nil
+}
+
+// WritesPaused reports whether new samples are currently being discarded.
+// Queries remain available while VictoriaMetrics is in read-only mode.
+func (db *TSDB) WritesPaused() bool {
+	if db.diskFullPaused.Load() {
+		return true
+	}
+	return db.readOnly != nil && db.readOnly.IsReadOnly()
+}
+
+func (db *TSDB) acceptsWrites() bool {
+	if db.diskFullPaused.Load() {
+		return false
+	}
+
+	readOnly := db.readOnly != nil && db.readOnly.IsReadOnly()
+	if readOnly {
+		if db.readOnlyObserved.CompareAndSwap(false, true) {
+			log.Println("NEZHA>> TSDB writes paused because free disk space is below the configured limit; dashboard, queries, and alerts remain available")
+		}
+		return false
+	}
+
+	if db.readOnlyObserved.CompareAndSwap(true, false) {
+		log.Println("NEZHA>> TSDB writes resumed after free disk space recovered")
+	}
+	return true
+}
+
+func (db *TSDB) pauseWritesAfterDiskFull(recovered any) {
+	if db.diskFullPaused.CompareAndSwap(false, true) {
+		log.Printf("NEZHA>> TSDB disk is full; disabling TSDB writes until dashboard restart while keeping dashboard, queries, and alerts running: %v", recovered)
+	}
+}
+
+func (db *TSDB) addRowsSafely(rows []storage.MetricRow) {
+	if len(rows) == 0 || !db.acceptsWrites() {
+		return
+	}
+
+	defer func() {
+		if recovered := recover(); recovered != nil {
+			if diskFullPanicError(recovered) == nil {
+				panic(recovered)
+			}
+			db.pauseWritesAfterDiskFull(recovered)
+		}
+	}()
+	db.addRowsFn(rows, 64)
+}
+
+func (db *TSDB) debugFlushSafely() {
+	if !db.acceptsWrites() {
+		return
+	}
+
+	defer func() {
+		if recovered := recover(); recovered != nil {
+			if diskFullPanicError(recovered) == nil {
+				panic(recovered)
+			}
+			db.pauseWritesAfterDiskFull(recovered)
+		}
+	}()
+	db.debugFlushFn()
+}
+
 // Close 关闭 TSDB 存储
 func (db *TSDB) Close() error {
 	db.mu.Lock()
@@ -110,8 +222,14 @@ func (db *TSDB) IsClosed() bool {
 
 // Flush 强制刷盘（主要用于测试）
 func (db *TSDB) Flush() {
+	db.mu.RLock()
+	defer db.mu.RUnlock()
+	if db.closed {
+		return
+	}
+
 	if db.writer != nil {
 		db.writer.flush()
 	}
-	db.storage.DebugFlush()
+	db.debugFlushSafely()
 }

pkg/tsdb/writer.go

@@ -47,18 +47,29 @@ func (w *bufferedWriter) flushLoop() {
 }
 
 func (w *bufferedWriter) write(rows []storage.MetricRow) {
+	if !w.db.acceptsWrites() {
+		w.discard()
+		return
+	}
+
 	w.mu.Lock()
 	w.buffer = append(w.buffer, rows...)
 	if len(w.buffer) >= w.maxSize {
 		rows := w.buffer
 		w.buffer = make([]storage.MetricRow, 0, w.maxSize)
 		w.mu.Unlock()
-		w.db.storage.AddRows(rows, 64)
+		w.db.addRowsSafely(rows)
 		return
 	}
 	w.mu.Unlock()
 }
 
+func (w *bufferedWriter) discard() {
+	w.mu.Lock()
+	w.buffer = make([]storage.MetricRow, 0, w.maxSize)
+	w.mu.Unlock()
+}
+
 func (w *bufferedWriter) flush() {
 	w.mu.Lock()
 	if len(w.buffer) == 0 {
@@ -69,7 +80,7 @@ func (w *bufferedWriter) flush() {
 	w.buffer = make([]storage.MetricRow, 0, w.maxSize)
 	w.mu.Unlock()
 
-	w.db.storage.AddRows(rows, 64)
+	w.db.addRowsSafely(rows)
 }
 
 func (w *bufferedWriter) stop() {
@@ -171,7 +182,7 @@ func (db *TSDB) WriteServerMetrics(m *ServerMetrics) error {
 	if db.writer != nil {
 		db.writer.write(rows)
 	} else {
-		db.storage.AddRows(rows, 64)
+		db.addRowsSafely(rows)
 	}
 	return nil
 }
@@ -200,7 +211,7 @@ func (db *TSDB) WriteServiceMetrics(m *ServiceMetrics) error {
 	if db.writer != nil {
 		db.writer.write(rows)
 	} else {
-		db.storage.AddRows(rows, 64)
+		db.addRowsSafely(rows)
 	}
 	return nil
 }
@@ -265,7 +276,7 @@ func (db *TSDB) WriteBatchServerMetrics(metrics []*ServerMetrics) error {
 	if db.writer != nil {
 		db.writer.write(rows)
 	} else {
-		db.storage.AddRows(rows, 64)
+		db.addRowsSafely(rows)
 	}
 	return nil
 }
@@ -295,7 +306,7 @@ func (db *TSDB) WriteBatchServiceMetrics(metrics []*ServiceMetrics) error {
 	if db.writer != nil {
 		db.writer.write(rows)
 	} else {
-		db.storage.AddRows(rows, 64)
+		db.addRowsSafely(rows)
 	}
 	return nil
 }