fix(tsdb): survive low disk space

service/singleton/tsdb.go

@@ -1,6 +1,7 @@
 package singleton
 
 import (
+	"errors"
 	"log"
 	"time"
 
@@ -10,6 +11,8 @@ import (
 
 var TSDBShared *tsdb.TSDB
 
+var openTSDB = tsdb.Open
+
 func InitTSDB() error {
 	config := &tsdb.Config{
 		RetentionDays:      30,
@@ -44,11 +47,21 @@ func InitTSDB() error {
 		return nil
 	}
 
-	var err error
-	TSDBShared, err = tsdb.Open(config)
+	TSDBShared = nil
+	db, err := openTSDB(config)
 	if err != nil {
+		if errors.Is(err, tsdb.ErrDiskFull) {
+			log.Printf("NEZHA>> Warning: TSDB is unavailable because its disk is full; dashboard and alerts will continue without TSDB writes: %v", err)
+			if DB != nil {
+				if migrateErr := DB.AutoMigrate(model.ServiceHistory{}); migrateErr != nil {
+					log.Printf("NEZHA>> Warning: failed to prepare SQLite service history fallback: %v", migrateErr)
+				}
+			}
+			return nil
+		}
 		return err
 	}
+	TSDBShared = db
 
 	log.Println("NEZHA>> TSDB initialized successfully")
 

service/singleton/tsdb_disk_guard_test.go

@@ -0,0 +1,58 @@
+package singleton
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/nezhahq/nezha/model"
+	"github.com/nezhahq/nezha/pkg/tsdb"
+)
+
+func TestInitTSDBContinuesWhenDiskIsFull(t *testing.T) {
+	originalConf := Conf
+	originalDB := DB
+	originalShared := TSDBShared
+	originalOpen := openTSDB
+	t.Cleanup(func() {
+		Conf = originalConf
+		DB = originalDB
+		TSDBShared = originalShared
+		openTSDB = originalOpen
+	})
+
+	Conf = &ConfigClass{Config: &model.Config{
+		TSDB: model.TSDBConf{DataPath: "/full/tsdb"},
+	}}
+	DB = nil
+	TSDBShared = nil
+	openTSDB = func(*tsdb.Config) (*tsdb.TSDB, error) {
+		return nil, fmt.Errorf("open failed: %w", tsdb.ErrDiskFull)
+	}
+
+	require.NoError(t, InitTSDB())
+	require.Nil(t, TSDBShared)
+}
+
+func TestInitTSDBStillReturnsNonDiskErrors(t *testing.T) {
+	originalConf := Conf
+	originalShared := TSDBShared
+	originalOpen := openTSDB
+	t.Cleanup(func() {
+		Conf = originalConf
+		TSDBShared = originalShared
+		openTSDB = originalOpen
+	})
+
+	Conf = &ConfigClass{Config: &model.Config{
+		TSDB: model.TSDBConf{DataPath: "/broken/tsdb"},
+	}}
+	TSDBShared = nil
+	openTSDB = func(*tsdb.Config) (*tsdb.TSDB, error) {
+		return nil, fmt.Errorf("permission denied")
+	}
+
+	require.ErrorContains(t, InitTSDB(), "permission denied")
+	require.Nil(t, TSDBShared)
+}