mirror of
https://github.com/Buriburizaem0n/nezha_domains.git
synced 2026-11-04 16:40:21 +00:00
feat(server): online session history with configurable stats window (#1243)
Add persisted online sessions, configurable retention and online history statistics with lifecycle, recovery, timezone and concurrency regression coverage. Co-authored-by: naiba/CloudCode <hi+cloudcode@nai.ba>
This commit is contained in:
1 parent
9881c47f0c
commit
8721a15b12
22 files changed
+2242
-20
No files matched your search
@@ -4,6 +4,7 @@ import (
|
||||
"log"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/nezhahq/nezha/model"
|
||||
"github.com/nezhahq/nezha/pkg/utils"
|
||||
@@ -14,6 +15,8 @@ var Conf *ConfigClass
|
||||
type ConfigClass struct {
|
||||
*model.Config
|
||||
|
||||
onlineHistoryRetentionMu sync.RWMutex
|
||||
|
||||
IgnoredIPNotificationServerIDs map[uint64]bool `json:"ignored_ip_notification_server_ids,omitempty"`
|
||||
Oauth2Providers []string `json:"oauth2_providers,omitempty"`
|
||||
}
|
||||
@@ -59,3 +62,39 @@ func (c *ConfigClass) updateIgnoredIPNotificationID() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// OnlineHistoryRetentionDays returns the retention policy visible to queries and pruning.
|
||||
func (c *ConfigClass) OnlineHistoryRetentionDays() int {
|
||||
c.onlineHistoryRetentionMu.RLock()
|
||||
defer c.onlineHistoryRetentionMu.RUnlock()
|
||||
return c.OnlineStatsDays
|
||||
}
|
||||
|
||||
// CommitOnlineHistoryRetention applies a retention change with the settings save.
|
||||
func (c *ConfigClass) CommitOnlineHistoryRetention(days *int, persist func() error) error {
|
||||
// Readers must not capture a shorter cutoff while Save can still fail.
|
||||
// Serialize assignment, persistence and rollback with the policy snapshot.
|
||||
c.onlineHistoryRetentionMu.Lock()
|
||||
defer c.onlineHistoryRetentionMu.Unlock()
|
||||
previous := c.OnlineStatsDays
|
||||
if days != nil {
|
||||
c.OnlineStatsDays = *days
|
||||
}
|
||||
if err := persist(); err != nil {
|
||||
c.OnlineStatsDays = previous
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SettingsSnapshot copies response fields without copying the retention mutex.
|
||||
func (c *ConfigClass) SettingsSnapshot() model.Setting {
|
||||
c.onlineHistoryRetentionMu.RLock()
|
||||
defer c.onlineHistoryRetentionMu.RUnlock()
|
||||
return model.Setting{
|
||||
ConfigForGuests: c.ConfigForGuests,
|
||||
ConfigDashboard: c.ConfigDashboard,
|
||||
IgnoredIPNotificationServerIDs: c.IgnoredIPNotificationServerIDs,
|
||||
Oauth2Providers: c.Oauth2Providers,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,348 @@
|
||||
package singleton
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/model"
|
||||
)
|
||||
|
||||
// OnlineHistoryRecorder 维护每台服务器当前进行中的在线会话。会话的打开与
|
||||
// 关闭由 agent 状态流的建立/断开驱动(service/rpc),并由定时清扫兜底关闭
|
||||
// 失联(未正常断流)的会话;dashboard 重启时通过数据库中 Ongoing 会话恢复。
|
||||
type OnlineHistoryRecorder struct {
|
||||
mu sync.Mutex
|
||||
restored bool
|
||||
open map[uint64]*onlineSessionRuntime
|
||||
// closing 是关库失败后的待关闭状态:断线边界已被观察到,会话 ID 与
|
||||
// 结束时间在此固定、不可被后续上报覆盖;重试只为旧会话完成持久化,
|
||||
// 重连不得把它续接成活跃会话(同服务器可能同时有多条待关闭会话)。
|
||||
closing map[uint64][]*onlineSessionClosing
|
||||
// Preserve the first disconnect while startup reads are unavailable.
|
||||
recoveryDisconnects map[uint64]time.Time
|
||||
}
|
||||
|
||||
type onlineSessionRuntime struct {
|
||||
id uint64
|
||||
lastObserved time.Time
|
||||
revision uint64
|
||||
// lastPersist 是 LastActiveAt 最近一次落库时间,用于节流写库。
|
||||
lastPersist time.Time
|
||||
}
|
||||
|
||||
type onlineSessionClosing struct {
|
||||
id uint64
|
||||
// end 是已观察到的会话结束时刻,重试持久化时原样使用。
|
||||
end time.Time
|
||||
}
|
||||
|
||||
var OnlineHistoryShared *OnlineHistoryRecorder
|
||||
|
||||
func NewOnlineHistoryRecorder() *OnlineHistoryRecorder {
|
||||
r := &OnlineHistoryRecorder{
|
||||
open: make(map[uint64]*onlineSessionRuntime),
|
||||
closing: make(map[uint64][]*onlineSessionClosing),
|
||||
recoveryDisconnects: make(map[uint64]time.Time),
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.restoreOngoingSessions()
|
||||
r.mu.Unlock()
|
||||
return r
|
||||
}
|
||||
|
||||
// restoreOngoingSessions 每台服务器仅允许最新且仍在在线窗口内的会话续接,
|
||||
// 其余会话按最后持久化的活跃时间结转。崩溃会丢失内存中的待关闭边界,
|
||||
// 恢复时采用 LastActiveAt 保守结转,不推测未持久化的断线时间。
|
||||
func (r *OnlineHistoryRecorder) restoreOngoingSessions() bool {
|
||||
if DB == nil {
|
||||
return false
|
||||
}
|
||||
var ongoing []model.OnlineSession
|
||||
if err := DB.Where("ongoing = ?", true).
|
||||
Order("server_id ASC").Order("started_at DESC").Order("id DESC").Find(&ongoing).Error; err != nil {
|
||||
log.Printf("NEZHA>> Failed to load ongoing online sessions: %v", err)
|
||||
return false
|
||||
}
|
||||
now := time.Now()
|
||||
seenServers := make(map[uint64]bool)
|
||||
for _, s := range ongoing {
|
||||
// 关闭失败后重连可留下同服务器多条 Ongoing 行;直接赋值 open 会覆盖
|
||||
// 旧 ID,导致其永久脱管。按开始时间(同时间按 ID)选最新行,其余
|
||||
// 必须关闭或进入 closing 重试,即使最新行本身已经过期也不能复活旧行。
|
||||
duplicate := seenServers[s.ServerID]
|
||||
seenServers[s.ServerID] = true
|
||||
disconnect, observedDisconnect := r.recoveryDisconnects[s.ServerID]
|
||||
observedDisconnect = observedDisconnect && !duplicate && !disconnect.Before(s.StartedAt)
|
||||
if observedDisconnect || duplicate || s.LastActiveAt.Before(now.Add(-model.OnlineActiveWindow)) {
|
||||
end := s.LastActiveAt
|
||||
if observedDisconnect && disconnect.After(end) {
|
||||
end = disconnect
|
||||
}
|
||||
if end.Before(s.StartedAt) {
|
||||
end = s.StartedAt
|
||||
}
|
||||
if err := closeOnlineSessionRow(s.ID, end); err != nil {
|
||||
log.Printf("NEZHA>> Failed to close recovered online session %d: %v", s.ID, err)
|
||||
// 关库失败不能跳过,否则会留下永不闭合、也无人重试的
|
||||
// Ongoing 行;进入待关闭状态交由清扫周期重试。过期或
|
||||
// 被更新会话替代的行均不得被重连续接为活跃会话
|
||||
r.closing[s.ServerID] = append(r.closing[s.ServerID],
|
||||
&onlineSessionClosing{id: s.ID, end: end})
|
||||
}
|
||||
continue
|
||||
}
|
||||
r.open[s.ServerID] = &onlineSessionRuntime{id: s.ID, lastPersist: s.LastActiveAt, lastObserved: s.LastActiveAt}
|
||||
}
|
||||
clear(r.recoveryDisconnects)
|
||||
r.restored = true
|
||||
return true
|
||||
}
|
||||
|
||||
// flushClosing 尽力重试关闭该服务器的待关闭会话:按入列时固定的结束时间
|
||||
// 持久化,成功才移除;失败的条目原样保留,结束时间不得被后续上报改写。
|
||||
func (r *OnlineHistoryRecorder) flushClosing(serverID uint64) {
|
||||
pendings, ok := r.closing[serverID]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
kept := pendings[:0]
|
||||
for _, pc := range pendings {
|
||||
if err := closeOnlineSessionRow(pc.id, pc.end); err != nil {
|
||||
log.Printf("NEZHA>> Failed to retry closing online session %d: %v", pc.id, err)
|
||||
kept = append(kept, pc)
|
||||
}
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
delete(r.closing, serverID)
|
||||
} else {
|
||||
r.closing[serverID] = kept
|
||||
}
|
||||
}
|
||||
|
||||
// RecordOnline 记录一次状态上报:无进行中会话时开启新会话(重启遗留且仍在
|
||||
// 窗口内的 Ongoing 会话直接续接);已有会话时节流落库 LastActiveAt。
|
||||
func (r *OnlineHistoryRecorder) RecordOnline(serverID uint64, at time.Time) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
// A failed startup read is not an empty history: retry before accepting reports.
|
||||
if !r.restored && !r.restoreOngoingSessions() {
|
||||
return
|
||||
}
|
||||
at = at.UTC()
|
||||
r.flushClosing(serverID)
|
||||
|
||||
// A report can arrive after expiry but before the next sweep. Close at the
|
||||
// previous observation before overwriting it, preserving failed-close retries.
|
||||
if rt := r.open[serverID]; rt != nil && at.Sub(rt.lastObserved) > model.OnlineActiveWindow {
|
||||
r.recordOfflineLocked(serverID, rt.lastObserved)
|
||||
}
|
||||
if rt, ok := r.open[serverID]; ok {
|
||||
rt.lastObserved = at
|
||||
rt.revision++
|
||||
if at.Sub(rt.lastPersist) >= model.OnlineActiveWindow {
|
||||
if err := DB.Model(&model.OnlineSession{}).Where("id = ?", rt.id).Update("last_active_at", at).Error; err != nil {
|
||||
log.Printf("NEZHA>> Failed to persist online session last_active_at: %v", err)
|
||||
} else {
|
||||
// Only committed writes advance the throttle; failures retry on the next report.
|
||||
rt.lastPersist = at
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// All ongoing rows are owned by open or closing after successful recovery.
|
||||
|
||||
s := model.OnlineSession{ServerID: serverID, StartedAt: at, LastActiveAt: at, Ongoing: true}
|
||||
if err := DB.Create(&s).Error; err != nil {
|
||||
log.Printf("NEZHA>> Failed to create online session: %v", err)
|
||||
return
|
||||
}
|
||||
r.open[serverID] = &onlineSessionRuntime{id: s.ID, lastPersist: at, lastObserved: at}
|
||||
}
|
||||
|
||||
// RecordOffline 关闭该服务器当前进行中的会话。at 是会话结束时刻:正常断流
|
||||
// 为当前时间,清扫/结转为最后一次已知活跃时间。
|
||||
// 关库失败时把会话转入待关闭状态而非保留为活跃条目:会话 ID 与已观察到的
|
||||
// 结束时刻(at 与 lastPersist 的较大者)就此固定,后续上报不得覆盖;重试
|
||||
// 只为旧会话完成持久化,重连则开启新会话——断线后的离线间隔不得计作在线。
|
||||
func (r *OnlineHistoryRecorder) RecordOffline(serverID uint64, at time.Time) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
// Disconnect may be the first event after startup storage recovers.
|
||||
if !r.restored && !r.restoreOngoingSessions() {
|
||||
// Retrying SELECT alone loses confirmed offline gaps on a quick reconnect.
|
||||
// Keep the first boundary; subsequent disconnected streams cannot extend it.
|
||||
if r.recoveryDisconnects == nil {
|
||||
r.recoveryDisconnects = make(map[uint64]time.Time)
|
||||
}
|
||||
if _, exists := r.recoveryDisconnects[serverID]; !exists {
|
||||
r.recoveryDisconnects[serverID] = at.UTC()
|
||||
}
|
||||
return
|
||||
}
|
||||
r.recordOfflineLocked(serverID, at)
|
||||
}
|
||||
|
||||
func (r *OnlineHistoryRecorder) recordOfflineLocked(serverID uint64, at time.Time) {
|
||||
rt, ok := r.open[serverID]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
end := at
|
||||
if end.Before(rt.lastPersist) {
|
||||
end = rt.lastPersist
|
||||
}
|
||||
if err := closeOnlineSessionRow(rt.id, end); err != nil {
|
||||
log.Printf("NEZHA>> Failed to close online session %d: %v", rt.id, err)
|
||||
// 结束时刻已观察到,不能因写库失败丢失,也不能把行继续当活跃
|
||||
// 会话:转待关闭状态,由 RecordOnline 冲洗与清扫周期重试
|
||||
r.closing[serverID] = append(r.closing[serverID], &onlineSessionClosing{id: rt.id, end: end})
|
||||
delete(r.open, serverID)
|
||||
return
|
||||
}
|
||||
delete(r.open, serverID)
|
||||
}
|
||||
|
||||
// SweepStale 关闭超过在线窗口仍无状态上报的进行中会话。lastActiveFn 返回
|
||||
// 服务器的运行时最后活跃时间与是否存在。
|
||||
func (r *OnlineHistoryRecorder) SweepStale(now time.Time, lastActiveFn func(serverID uint64) (time.Time, bool)) {
|
||||
// Never call the runtime observer while holding mu: reports acquire the
|
||||
// runtime lease before this mutex. Snapshot identities, then revalidate them.
|
||||
r.mu.Lock()
|
||||
if !r.restored && !r.restoreOngoingSessions() {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
candidates := make(map[uint64]onlineSessionRuntime, len(r.open))
|
||||
for id, rt := range r.open {
|
||||
candidates[id] = *rt
|
||||
}
|
||||
for id := range r.closing {
|
||||
r.flushClosing(id)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
for id, candidate := range candidates {
|
||||
lastActive, ok := lastActiveFn(id)
|
||||
r.mu.Lock()
|
||||
rt := r.open[id]
|
||||
if rt != nil && rt.id == candidate.id && rt.revision == candidate.revision {
|
||||
if lastActive.Before(rt.lastObserved) {
|
||||
lastActive = rt.lastObserved
|
||||
}
|
||||
if (!ok || lastActive.Before(now.Add(-model.OnlineActiveWindow))) && rt.lastObserved.Before(now.Add(-model.OnlineActiveWindow)) {
|
||||
r.recordOfflineLocked(id, lastActive)
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// SweepOnlineSessions 是定时清扫入口,接上服务器运行时的最后活跃时间。
|
||||
func SweepOnlineSessions() {
|
||||
if OnlineHistoryShared == nil || ServerShared == nil || DB == nil {
|
||||
return
|
||||
}
|
||||
OnlineHistoryShared.SweepStale(time.Now(), onlineServerLastActive)
|
||||
}
|
||||
|
||||
func onlineServerLastActive(serverID uint64) (time.Time, bool) {
|
||||
server, ok := ServerShared.Get(serverID)
|
||||
if !ok || server == nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
lastActive := server.RuntimeSnapshot().LastActive
|
||||
return lastActive, !lastActive.IsZero()
|
||||
}
|
||||
|
||||
// PruneOnlineSessions 清理超出统计窗口的已结束会话与已删除服务器的残留记录。
|
||||
// 时间列直接比较(与写入时的绑定格式同构)以走 ended_at 索引,并用分批
|
||||
// DELETE 替换单条巨型删除,避免大表上长时间占用 sqlite 写锁、阻塞上报路径。
|
||||
func PruneOnlineSessions() {
|
||||
if DB == nil {
|
||||
return
|
||||
}
|
||||
keepDays := Conf.OnlineHistoryRetentionDays()
|
||||
if keepDays < model.MinOnlineStatsDays || keepDays > model.MaxOnlineStatsDays {
|
||||
keepDays = model.DefaultOnlineStatsDays
|
||||
}
|
||||
cutoff := model.OnlineHistoryWindowStart(time.Now(), keepDays, Loc)
|
||||
pruneOnlineSessionsBatch("ongoing = ? AND ended_at < ?", false, cutoff)
|
||||
pruneOnlineSessionsBatch("server_id NOT IN (SELECT `id` FROM servers)")
|
||||
}
|
||||
|
||||
// pruneOnlineSessionsBatch 按主键分批删除匹配的行,每批之间释放写锁。
|
||||
func pruneOnlineSessionsBatch(where string, args ...any) {
|
||||
const batchSize = 10000
|
||||
for {
|
||||
var ids []uint64
|
||||
if err := DB.Model(&model.OnlineSession{}).Unscoped().
|
||||
Where(where, args...).Limit(batchSize).Pluck("id", &ids).Error; err != nil {
|
||||
log.Printf("NEZHA>> Failed to select online sessions for pruning: %v", err)
|
||||
return
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return
|
||||
}
|
||||
if err := DB.Unscoped().Delete(&model.OnlineSession{}, ids).Error; err != nil {
|
||||
log.Printf("NEZHA>> Failed to prune online sessions: %v", err)
|
||||
return
|
||||
}
|
||||
if len(ids) < batchSize {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func closeOnlineSessionRow(id uint64, end time.Time) error {
|
||||
return DB.Model(&model.OnlineSession{}).Where("id = ?", id).Updates(map[string]any{
|
||||
"ended_at": end.UTC(),
|
||||
"ongoing": false,
|
||||
}).Error
|
||||
}
|
||||
|
||||
// Sessions snapshots rows and volatile close boundaries under the same lock.
|
||||
// Otherwise a failed close can count confirmed offline time as ongoing, or a
|
||||
// successful retry can remove the boundary after the reader fetched a stale row.
|
||||
func (r *OnlineHistoryRecorder) Sessions(serverID uint64, start, end time.Time) ([]model.OnlineSession, error) {
|
||||
if r != nil {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if !r.restored && !r.restoreOngoingSessions() {
|
||||
return nil, fmt.Errorf("online history recovery unavailable")
|
||||
}
|
||||
}
|
||||
var sessions []model.OnlineSession
|
||||
// Separate branches use (server_id, ongoing, ended_at) and
|
||||
// (server_id, ongoing, started_at), avoiding a scan of retained closed rows.
|
||||
for _, ongoing := range []bool{false, true} {
|
||||
var rows []model.OnlineSession
|
||||
q := DB.Where("server_id = ? AND ongoing = ?", serverID, ongoing)
|
||||
if ongoing {
|
||||
q = q.Where("started_at < ?", end.UTC())
|
||||
} else {
|
||||
q = q.Where("ended_at > ?", start.UTC())
|
||||
}
|
||||
if err := q.Find(&rows).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sessions = append(sessions, rows...)
|
||||
}
|
||||
if r != nil {
|
||||
endings := make(map[uint64]time.Time)
|
||||
for _, pending := range r.closing[serverID] {
|
||||
endings[pending.id] = pending.end
|
||||
}
|
||||
for i := range sessions {
|
||||
if end, ok := endings[sessions[i].ID]; ok {
|
||||
sessions[i].Ongoing = false
|
||||
sessions[i].EndedAt = end
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(sessions, func(i, j int) bool { return sessions[i].StartedAt.Before(sessions[j].StartedAt) })
|
||||
return sessions, nil
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
package singleton
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/model"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestOnlineHistoryRetriesFailedHeartbeatImmediately(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
r := NewOnlineHistoryRecorder()
|
||||
start := time.Now().UTC().Truncate(time.Second)
|
||||
r.RecordOnline(1, start)
|
||||
id := r.open[1].id
|
||||
require.NoError(t, DB.Exec(`CREATE TRIGGER reject_heartbeat BEFORE UPDATE OF last_active_at ON online_sessions BEGIN SELECT RAISE(ABORT, 'heartbeat failure'); END`).Error)
|
||||
r.RecordOnline(1, start.Add(30*time.Second))
|
||||
require.NoError(t, DB.Exec("DROP TRIGGER reject_heartbeat").Error)
|
||||
r.RecordOnline(1, start.Add(31*time.Second))
|
||||
require.True(t, getOnlineSession(t, id).LastActiveAt.Equal(start.Add(31*time.Second)))
|
||||
}
|
||||
|
||||
func TestOnlineHistoryRetriesStartupReadBeforeNewReport(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
start := time.Now().UTC().Add(-time.Hour).Truncate(time.Second)
|
||||
old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true}
|
||||
require.NoError(t, DB.Create(&old).Error)
|
||||
require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("reject_history_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) }))
|
||||
r := NewOnlineHistoryRecorder()
|
||||
require.NoError(t, DB.Callback().Query().Remove("reject_history_restore"))
|
||||
r.RecordOnline(1, time.Now())
|
||||
stored := getOnlineSession(t, old.ID)
|
||||
require.False(t, stored.Ongoing)
|
||||
require.True(t, stored.EndedAt.Equal(start))
|
||||
require.NotEqual(t, old.ID, r.open[1].id)
|
||||
require.Equal(t, int64(1), countOnlineSessions(t, "ongoing = ?", true))
|
||||
}
|
||||
|
||||
func TestOnlineHistorySweepKeepsSessionReconnectedDuringObservation(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
r := NewOnlineHistoryRecorder()
|
||||
start := time.Now().Add(-time.Hour)
|
||||
r.RecordOnline(1, start)
|
||||
observed := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
r.SweepStale(time.Now(), func(uint64) (time.Time, bool) { close(observed); <-release; return start, true })
|
||||
}()
|
||||
<-observed
|
||||
reconnected := make(chan struct{})
|
||||
go func() { r.RecordOffline(1, start.Add(time.Second)); r.RecordOnline(1, time.Now()); close(reconnected) }()
|
||||
// The observer must not hold the recorder lock: RPC updates take the runtime
|
||||
// lock first and then enter the recorder. Holding both in reverse order deadlocks.
|
||||
select {
|
||||
case <-reconnected:
|
||||
case <-time.After(time.Second):
|
||||
close(release)
|
||||
<-done
|
||||
<-reconnected
|
||||
t.Fatal("sweep holds recorder lock while consulting runtime state")
|
||||
}
|
||||
id := r.open[1].id
|
||||
close(release)
|
||||
<-done
|
||||
require.Contains(t, r.open, uint64(1))
|
||||
require.Equal(t, id, r.open[1].id)
|
||||
require.True(t, getOnlineSession(t, id).Ongoing)
|
||||
}
|
||||
|
||||
func TestOnlineHistoryQueryNormalizesOffsetsAndUsesWindowIndexes(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
east := time.FixedZone("UTC+8", 8*3600)
|
||||
start := time.Date(2025, 1, 1, 12, 0, 0, 0, east)
|
||||
r := NewOnlineHistoryRecorder()
|
||||
r.RecordOnline(1, start)
|
||||
r.RecordOffline(1, start.Add(time.Hour))
|
||||
rows, err := r.Sessions(1, start.UTC().Add(-time.Minute), start.UTC().Add(2*time.Hour))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
require.True(t, rows[0].StartedAt.Equal(start))
|
||||
require.True(t, rows[0].EndedAt.Equal(start.Add(time.Hour)))
|
||||
for _, test := range []struct {
|
||||
query, index string
|
||||
args []any
|
||||
}{
|
||||
{"EXPLAIN QUERY PLAN SELECT * FROM online_sessions WHERE server_id = ? AND ongoing = ? AND ended_at > ?", "idx_online_sessions_server_closed", []any{1, false, start.UTC()}},
|
||||
{"EXPLAIN QUERY PLAN SELECT * FROM online_sessions WHERE server_id = ? AND ongoing = ? AND started_at < ?", "idx_online_sessions_server_open", []any{1, true, start.Add(2 * time.Hour).UTC()}},
|
||||
} {
|
||||
var plan []struct{ Detail string }
|
||||
require.NoError(t, DB.Raw(test.query, test.args...).Scan(&plan).Error)
|
||||
require.NotEmpty(t, plan)
|
||||
require.Contains(t, plan[0].Detail, test.index)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlineHistorySweepKeepsReportReceivedDuringObservation(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
r := NewOnlineHistoryRecorder()
|
||||
start := time.Now().Add(-time.Hour)
|
||||
r.RecordOnline(1, start)
|
||||
id := r.open[1].id
|
||||
r.SweepStale(time.Now(), func(uint64) (time.Time, bool) {
|
||||
r.RecordOnline(1, time.Now())
|
||||
return start, true
|
||||
})
|
||||
require.Contains(t, r.open, uint64(1))
|
||||
// A report after an hour starts a new session; the stale sweep must not close it.
|
||||
require.NotEqual(t, id, r.open[1].id)
|
||||
require.False(t, getOnlineSession(t, id).Ongoing)
|
||||
require.True(t, getOnlineSession(t, r.open[1].id).Ongoing)
|
||||
}
|
||||
|
||||
func TestOnlineHistoryStartupReadFailureBlocksQueryAndSweepRetries(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
start := time.Now().Add(-time.Hour)
|
||||
old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true}
|
||||
require.NoError(t, DB.Create(&old).Error)
|
||||
require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("reject_history_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) }))
|
||||
r := NewOnlineHistoryRecorder()
|
||||
_, err := r.Sessions(1, start, time.Now())
|
||||
require.ErrorContains(t, err, "recovery unavailable")
|
||||
r.RecordOnline(1, time.Now())
|
||||
require.Empty(t, r.open)
|
||||
require.NoError(t, DB.Callback().Query().Remove("reject_history_restore"))
|
||||
r.SweepStale(time.Now(), func(uint64) (time.Time, bool) { return time.Time{}, false })
|
||||
require.False(t, getOnlineSession(t, old.ID).Ongoing)
|
||||
require.Equal(t, int64(0), countOnlineSessions(t, "ongoing = ?", true))
|
||||
}
|
||||
|
||||
func TestOnlineHistoryDisconnectRetriesStartupRead(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
start := time.Now().Add(-10 * time.Second)
|
||||
old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true}
|
||||
require.NoError(t, DB.Create(&old).Error)
|
||||
require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("reject_history_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) }))
|
||||
r := NewOnlineHistoryRecorder()
|
||||
require.NoError(t, DB.Callback().Query().Remove("reject_history_restore"))
|
||||
end := start.Add(time.Second)
|
||||
r.RecordOffline(1, end)
|
||||
require.False(t, getOnlineSession(t, old.ID).Ongoing)
|
||||
require.True(t, getOnlineSession(t, old.ID).EndedAt.Equal(end))
|
||||
r.RecordOnline(1, time.Now())
|
||||
require.NotEqual(t, old.ID, r.open[1].id)
|
||||
}
|
||||
|
||||
func TestOnlineHistoryReportAfterTimeoutSplitsBeforeSweep(t *testing.T) {
|
||||
for _, failClose := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "persisted", true: "retry_close"}[failClose], func(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
r := NewOnlineHistoryRecorder()
|
||||
start := time.Now().UTC().Truncate(time.Second)
|
||||
r.RecordOnline(1, start)
|
||||
r.RecordOnline(1, start.Add(time.Second))
|
||||
first := r.open[1].id
|
||||
r.SweepStale(start.Add(30*time.Second), func(uint64) (time.Time, bool) { return start.Add(time.Second), true })
|
||||
require.Equal(t, first, r.open[1].id)
|
||||
if failClose {
|
||||
injectCloseUpdateFailure(t)
|
||||
}
|
||||
r.RecordOnline(1, start.Add(45*time.Second))
|
||||
require.NotEqual(t, first, r.open[1].id)
|
||||
second := r.open[1].id
|
||||
rows, err := r.Sessions(1, start, start.Add(46*time.Second))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 2)
|
||||
require.False(t, rows[0].Ongoing)
|
||||
require.True(t, rows[0].EndedAt.Equal(start.Add(time.Second)))
|
||||
require.True(t, rows[1].StartedAt.Equal(start.Add(45*time.Second)))
|
||||
if failClose {
|
||||
require.Len(t, r.closing[1], 1)
|
||||
dropCloseUpdateFailure(t)
|
||||
}
|
||||
r.SweepStale(start.Add(46*time.Second), func(uint64) (time.Time, bool) { return start.Add(45 * time.Second), true })
|
||||
require.Empty(t, r.closing)
|
||||
require.False(t, getOnlineSession(t, first).Ongoing)
|
||||
require.True(t, getOnlineSession(t, first).EndedAt.Equal(start.Add(time.Second)))
|
||||
require.True(t, getOnlineSession(t, second).Ongoing)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlineHistoryDisconnectSurvivesUnavailableRecovery(t *testing.T) {
|
||||
for _, failClose := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "close_success", true: "close_retry"}[failClose], func(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
start := time.Now().UTC().Add(-10 * time.Second)
|
||||
old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true}
|
||||
require.NoError(t, DB.Create(&old).Error)
|
||||
require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("unavailable_disconnect_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) }))
|
||||
r := NewOnlineHistoryRecorder()
|
||||
end := start.Add(time.Second)
|
||||
r.RecordOffline(1, end)
|
||||
// Later disconnects while recovery is unavailable must not extend the first session.
|
||||
r.RecordOffline(1, end.Add(time.Second))
|
||||
require.NoError(t, DB.Callback().Query().Remove("unavailable_disconnect_restore"))
|
||||
if failClose {
|
||||
injectCloseUpdateFailure(t)
|
||||
}
|
||||
reconnect := start.Add(5 * time.Second)
|
||||
r.RecordOnline(1, reconnect)
|
||||
require.NotEqual(t, old.ID, r.open[1].id)
|
||||
rows, err := r.Sessions(1, start, reconnect.Add(time.Second))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 2)
|
||||
require.False(t, rows[0].Ongoing)
|
||||
require.True(t, rows[0].EndedAt.Equal(end))
|
||||
require.True(t, rows[1].StartedAt.Equal(reconnect))
|
||||
if failClose {
|
||||
require.Len(t, r.closing[1], 1)
|
||||
dropCloseUpdateFailure(t)
|
||||
}
|
||||
r.SweepStale(reconnect.Add(time.Second), func(uint64) (time.Time, bool) { return reconnect, true })
|
||||
require.Empty(t, r.closing)
|
||||
require.False(t, getOnlineSession(t, old.ID).Ongoing)
|
||||
require.True(t, getOnlineSession(t, old.ID).EndedAt.Equal(end))
|
||||
require.True(t, getOnlineSession(t, r.open[1].id).Ongoing)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package singleton
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/model"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRestoreOngoingSessionsReconcilesRecentDuplicates(t *testing.T) {
|
||||
for _, failClose := range []bool{false, true} {
|
||||
name := "successful_close"
|
||||
if failClose {
|
||||
name = "retry_failed_close"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
base := time.Now().Add(-10 * time.Second).Truncate(time.Second)
|
||||
// Insert out of chronological order: choosing by ID alone would keep the wrong session.
|
||||
newest := model.OnlineSession{ServerID: 1, StartedAt: base.Add(4 * time.Second), LastActiveAt: base.Add(5 * time.Second), Ongoing: true}
|
||||
oldest := model.OnlineSession{ServerID: 1, StartedAt: base, LastActiveAt: base.Add(time.Second), Ongoing: true}
|
||||
middle := model.OnlineSession{ServerID: 1, StartedAt: base.Add(2 * time.Second), LastActiveAt: base.Add(3 * time.Second), Ongoing: true}
|
||||
for _, session := range []*model.OnlineSession{&newest, &oldest, &middle} {
|
||||
require.NoError(t, DB.Create(session).Error)
|
||||
}
|
||||
if failClose {
|
||||
injectCloseUpdateFailure(t)
|
||||
}
|
||||
r := NewOnlineHistoryRecorder()
|
||||
require.Len(t, r.open, 1)
|
||||
require.Equal(t, newest.ID, r.open[1].id)
|
||||
if failClose {
|
||||
require.Len(t, r.closing[1], 2)
|
||||
require.Equal(t, int64(3), countOnlineSessions(t, "ongoing = ?", true))
|
||||
dropCloseUpdateFailure(t)
|
||||
}
|
||||
r.SweepStale(base.Add(6*time.Second), func(uint64) (time.Time, bool) { return newest.LastActiveAt, true })
|
||||
require.Empty(t, r.closing)
|
||||
require.True(t, getOnlineSession(t, newest.ID).Ongoing)
|
||||
for _, previous := range []model.OnlineSession{oldest, middle} {
|
||||
stored := getOnlineSession(t, previous.ID)
|
||||
require.False(t, stored.Ongoing)
|
||||
require.True(t, stored.EndedAt.Equal(previous.LastActiveAt))
|
||||
}
|
||||
r.RecordOnline(1, base.Add(7*time.Second))
|
||||
require.Equal(t, newest.ID, r.open[1].id)
|
||||
require.Equal(t, int64(3), countOnlineSessions(t, "server_id = ?", 1))
|
||||
r.SweepStale(base.Add(2*model.OnlineActiveWindow), func(uint64) (time.Time, bool) { return time.Time{}, false })
|
||||
require.Empty(t, r.open)
|
||||
require.Empty(t, r.closing)
|
||||
require.Equal(t, int64(0), countOnlineSessions(t, "ongoing = ?", true))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreOngoingSessionsUsesIDToBreakStartTimeTies(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
start := time.Now().Add(-5 * time.Second)
|
||||
first := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true}
|
||||
second := first
|
||||
other := model.OnlineSession{ServerID: 2, StartedAt: start, LastActiveAt: start, Ongoing: true}
|
||||
for _, session := range []*model.OnlineSession{&first, &second, &other} {
|
||||
require.NoError(t, DB.Create(session).Error)
|
||||
}
|
||||
r := NewOnlineHistoryRecorder()
|
||||
require.Len(t, r.open, 2)
|
||||
require.Equal(t, second.ID, r.open[1].id)
|
||||
require.Equal(t, other.ID, r.open[2].id)
|
||||
require.False(t, getOnlineSession(t, first.ID).Ongoing)
|
||||
require.True(t, getOnlineSession(t, first.ID).EndedAt.Equal(start))
|
||||
require.Equal(t, int64(2), countOnlineSessions(t, "ongoing = ?", true))
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package singleton
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/model"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestOnlineHistoryPruneWaitsForRetentionCommit(t *testing.T) {
|
||||
for _, saveFails := range []bool{true, false} {
|
||||
t.Run(map[bool]string{true: "failed_save", false: "successful_save"}[saveFails], func(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
previous := Conf
|
||||
Conf = &ConfigClass{Config: &model.Config{}}
|
||||
Conf.OnlineStatsDays = 30
|
||||
t.Cleanup(func() { Conf = previous })
|
||||
now := time.Now().UTC()
|
||||
row := model.OnlineSession{ServerID: 1, StartedAt: now.AddDate(0, 0, -15), EndedAt: now.AddDate(0, 0, -14)}
|
||||
require.NoError(t, DB.Create(&row).Error)
|
||||
saveEntered := make(chan struct{})
|
||||
releaseSave := make(chan struct{})
|
||||
saved := make(chan error, 1)
|
||||
days := 7
|
||||
go func() {
|
||||
saved <- Conf.CommitOnlineHistoryRetention(&days, func() error {
|
||||
close(saveEntered)
|
||||
<-releaseSave
|
||||
if saveFails {
|
||||
return errors.New("retention save unavailable")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}()
|
||||
<-saveEntered
|
||||
pruneStarted := make(chan struct{})
|
||||
pruned := make(chan struct{})
|
||||
go func() { close(pruneStarted); PruneOnlineSessions(); close(pruned) }()
|
||||
<-pruneStarted
|
||||
premature := false
|
||||
select {
|
||||
case <-pruned:
|
||||
premature = true
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
close(releaseSave)
|
||||
err := <-saved
|
||||
<-pruned
|
||||
require.False(t, premature, "pruning must not read an uncommitted retention policy")
|
||||
if saveFails {
|
||||
require.ErrorContains(t, err, "retention save unavailable")
|
||||
require.Equal(t, 30, Conf.OnlineHistoryRetentionDays())
|
||||
require.Equal(t, int64(1), countOnlineSessions(t, "id = ?", row.ID))
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 7, Conf.OnlineHistoryRetentionDays())
|
||||
require.Equal(t, int64(0), countOnlineSessions(t, "id = ?", row.ID))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
package singleton
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/nezhahq/nezha/model"
|
||||
)
|
||||
|
||||
func setupOnlineHistoryTestDB(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
previousDB := DB
|
||||
var err error
|
||||
DB, err = gorm.Open(openSQLiteDialector(filepath.Join(t.TempDir(), "dashboard.sqlite")), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
sqlDB, err := DB.DB()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
DB = previousDB
|
||||
if err := sqlDB.Close(); err != nil {
|
||||
t.Errorf("close online history test database: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
require.NoError(t, DB.AutoMigrate(&model.Server{}, &model.OnlineSession{}))
|
||||
require.NoError(t, DB.Exec("INSERT INTO servers (id, name, uuid) VALUES (1, 'server-1', 'online-history-test-1')").Error)
|
||||
}
|
||||
|
||||
func getOnlineSession(t *testing.T, id uint64) model.OnlineSession {
|
||||
t.Helper()
|
||||
var s model.OnlineSession
|
||||
require.NoError(t, DB.First(&s, id).Error)
|
||||
return s
|
||||
}
|
||||
|
||||
func countOnlineSessions(t *testing.T, cond string, args ...any) int64 {
|
||||
t.Helper()
|
||||
var count int64
|
||||
require.NoError(t, DB.Model(&model.OnlineSession{}).Where(cond, args...).Count(&count).Error)
|
||||
return count
|
||||
}
|
||||
|
||||
// injectCloseUpdateFailure 用 SQLite 触发器注入"SELECT 正常、仅关闭 UPDATE 失败":
|
||||
// 只拦截把 ongoing 置 0 的更新,LastActiveAt 节流写与 INSERT 均不受影响,
|
||||
// 使后续 RecordOnline/恢复/清扫全部走真实执行路径。
|
||||
func injectCloseUpdateFailure(t *testing.T) {
|
||||
t.Helper()
|
||||
require.NoError(t, DB.Exec(`CREATE TRIGGER fail_online_session_close
|
||||
BEFORE UPDATE ON online_sessions
|
||||
WHEN NEW.ongoing = 0
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'injected close failure');
|
||||
END`).Error)
|
||||
t.Cleanup(func() { dropCloseUpdateFailure(t) })
|
||||
}
|
||||
|
||||
func dropCloseUpdateFailure(t *testing.T) {
|
||||
t.Helper()
|
||||
require.NoError(t, DB.Exec("DROP TRIGGER IF EXISTS fail_online_session_close").Error)
|
||||
}
|
||||
|
||||
// 评审回归(P2:关库失败丢边界/重连续接待关闭会话):RecordOffline 关库失败时
|
||||
// 必须转入待关闭状态——会话 ID 与已观察到的断线时刻固定、后续上报不得覆盖;
|
||||
// 重连开启的是新会话而非续接旧行;失败恢复后由清扫周期按固定边界完成持久化,
|
||||
// 离线间隔不计入在线。时间自单一基准单调推进。
|
||||
func TestRecordOfflineCloseFailurePreservesBoundary(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
r := NewOnlineHistoryRecorder()
|
||||
|
||||
base := time.Now().Add(-time.Hour).Truncate(time.Second)
|
||||
atStart := base
|
||||
atLastActive := atStart.Add(model.OnlineActiveWindow) // 恰好达到节流窗口,触发落库
|
||||
atOffline := atLastActive.Add(15 * time.Second) // 观察到的断线时刻
|
||||
atReconnect := atOffline.Add(10 * time.Minute)
|
||||
|
||||
r.RecordOnline(1, atStart)
|
||||
r.RecordOnline(1, atLastActive)
|
||||
require.Len(t, r.open, 1)
|
||||
first := getOnlineSession(t, r.open[1].id)
|
||||
require.WithinDuration(t, atLastActive, first.LastActiveAt, time.Second)
|
||||
|
||||
// 注入关闭失败:行保持 Ongoing 等待重试,会话转入待关闭而非继续当活跃会话
|
||||
injectCloseUpdateFailure(t)
|
||||
r.RecordOffline(1, atOffline)
|
||||
require.NotContains(t, r.open, uint64(1))
|
||||
stuck := getOnlineSession(t, first.ID)
|
||||
require.True(t, stuck.Ongoing)
|
||||
require.True(t, stuck.EndedAt.IsZero(), "关库失败时旧行必须保持未闭合等待重试")
|
||||
|
||||
// 失败未恢复时 agent 重连:旧行不得被续接,必须开启新会话
|
||||
r.RecordOnline(1, atReconnect)
|
||||
require.Contains(t, r.open, uint64(1))
|
||||
require.NotEqual(t, first.ID, r.open[1].id, "待关闭会话不得被续接为活跃会话")
|
||||
afterRetry := getOnlineSession(t, first.ID)
|
||||
require.True(t, afterRetry.Ongoing, "失败未恢复时重试仍失败,旧行保持待关闭")
|
||||
second := getOnlineSession(t, r.open[1].id)
|
||||
require.True(t, second.Ongoing)
|
||||
require.WithinDuration(t, atReconnect, second.StartedAt, time.Second)
|
||||
|
||||
// 失败恢复后清扫:旧行按观察到的断线时刻闭合,新会话仍活跃不受影响
|
||||
dropCloseUpdateFailure(t)
|
||||
r.SweepStale(atReconnect.Add(time.Second), func(uint64) (time.Time, bool) {
|
||||
return atReconnect, true
|
||||
})
|
||||
|
||||
closed := getOnlineSession(t, first.ID)
|
||||
require.False(t, closed.Ongoing)
|
||||
require.WithinDuration(t, atOffline, closed.EndedAt, time.Second, "旧行必须按观察到的断线时刻闭合")
|
||||
require.WithinDuration(t, atStart, closed.StartedAt, time.Second)
|
||||
stillOngoing := getOnlineSession(t, second.ID)
|
||||
require.True(t, stillOngoing.Ongoing)
|
||||
|
||||
// 离线间隔不计入在线:两行在 [atStart, atReconnect) 内的在线合计止于断线时刻
|
||||
var rows []model.OnlineSession
|
||||
require.NoError(t, DB.Where("server_id = ?", 1).Order("id").Find(&rows).Error)
|
||||
require.Len(t, rows, 2)
|
||||
var online time.Duration
|
||||
for _, s := range rows {
|
||||
end := s.EndedAt
|
||||
if s.Ongoing {
|
||||
end = atReconnect
|
||||
}
|
||||
online += end.Sub(s.StartedAt)
|
||||
}
|
||||
require.Equal(t, atOffline.Sub(atStart), online, "离线间隔不得计作在线")
|
||||
}
|
||||
|
||||
// 评审回归(P2 启动恢复路径):重启恢复对超窗会话结转关库失败时,条目必须
|
||||
// 进入待关闭状态(经真实 NewOnlineHistoryRecorder 恢复路径验证,而非手工
|
||||
// 构造内存状态);重连不得续接该行,清扫周期按固定边界时刻完成持久化。
|
||||
func TestRestoreStaleCloseFailureEntersPendingClose(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
|
||||
// 预置一条超窗 Ongoing 行(dashboard 停机前遗留),时间自单一基准单调推进
|
||||
base := time.Now().Add(-3 * time.Hour).Truncate(time.Second)
|
||||
staleAt := base.Add(time.Hour)
|
||||
atReconnect := base.Add(2 * time.Hour)
|
||||
legacy := model.OnlineSession{ServerID: 1, StartedAt: base, LastActiveAt: staleAt, Ongoing: true}
|
||||
require.NoError(t, DB.Create(&legacy).Error)
|
||||
|
||||
injectCloseUpdateFailure(t)
|
||||
r := NewOnlineHistoryRecorder()
|
||||
|
||||
stuck := getOnlineSession(t, legacy.ID)
|
||||
require.True(t, stuck.Ongoing, "恢复结转关库失败时行必须保持 Ongoing 等待重试")
|
||||
|
||||
// 重连:待关闭的超窗会话不得被续接,开启新会话
|
||||
r.RecordOnline(1, atReconnect)
|
||||
require.Contains(t, r.open, uint64(1))
|
||||
require.NotEqual(t, legacy.ID, r.open[1].id, "待关闭会话不得被续接为活跃会话")
|
||||
|
||||
dropCloseUpdateFailure(t)
|
||||
r.SweepStale(atReconnect.Add(time.Second), func(uint64) (time.Time, bool) {
|
||||
return atReconnect, true
|
||||
})
|
||||
|
||||
closed := getOnlineSession(t, legacy.ID)
|
||||
require.False(t, closed.Ongoing)
|
||||
require.WithinDuration(t, staleAt, closed.EndedAt, time.Second, "旧行必须按恢复时观察到的边界闭合")
|
||||
require.Equal(t, int64(2), countOnlineSessions(t, "server_id = ?", 1))
|
||||
require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
|
||||
}
|
||||
|
||||
func TestRecordOnlineOpensSingleSession(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
r := NewOnlineHistoryRecorder()
|
||||
|
||||
now := time.Now()
|
||||
r.RecordOnline(1, now)
|
||||
r.RecordOnline(1, now.Add(time.Second))
|
||||
|
||||
require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ?", 1))
|
||||
require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
|
||||
}
|
||||
|
||||
func TestRecordOnlinePersistsLastActiveThrottled(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
r := NewOnlineHistoryRecorder()
|
||||
|
||||
now := time.Now()
|
||||
r.RecordOnline(1, now)
|
||||
// 窗口内的重复上报不应触发落库
|
||||
r.RecordOnline(1, now.Add(5*time.Second))
|
||||
s := r.open[1]
|
||||
stored := getOnlineSession(t, s.id)
|
||||
require.WithinDuration(t, now, stored.LastActiveAt, time.Second)
|
||||
|
||||
// 超过窗口后落库
|
||||
r.RecordOnline(1, now.Add(model.OnlineActiveWindow+time.Second))
|
||||
stored = getOnlineSession(t, s.id)
|
||||
require.WithinDuration(t, now.Add(model.OnlineActiveWindow+time.Second), stored.LastActiveAt, time.Second)
|
||||
}
|
||||
|
||||
func TestRecordOfflineClosesSession(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
r := NewOnlineHistoryRecorder()
|
||||
|
||||
start := time.Now()
|
||||
r.RecordOnline(1, start)
|
||||
r.RecordOffline(1, start.Add(10*time.Minute))
|
||||
// 重复关闭应为无操作
|
||||
r.RecordOffline(1, start.Add(11*time.Minute))
|
||||
|
||||
require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ?", 1))
|
||||
require.Equal(t, int64(0), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
|
||||
var s model.OnlineSession
|
||||
require.NoError(t, DB.First(&s).Error)
|
||||
require.WithinDuration(t, start.Add(10*time.Minute), s.EndedAt, time.Second)
|
||||
require.WithinDuration(t, start, s.StartedAt, time.Second)
|
||||
}
|
||||
|
||||
func TestRecordOnlineContinuesSessionAfterRestart(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
|
||||
// 模拟重启前遗留的进行中会话(最后活跃时间在在线窗口内:dashboard 只是短暂重启)
|
||||
now := time.Now()
|
||||
legacy := model.OnlineSession{ServerID: 1, StartedAt: now.Add(-time.Minute), LastActiveAt: now.Add(-5 * time.Second), Ongoing: true}
|
||||
require.NoError(t, DB.Create(&legacy).Error)
|
||||
|
||||
// 新 recorder(构造时从库恢复):新近会话应保留并续接,不产生新会话
|
||||
r := NewOnlineHistoryRecorder()
|
||||
r.RecordOnline(1, now)
|
||||
|
||||
require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ?", 1))
|
||||
continued := getOnlineSession(t, legacy.ID)
|
||||
require.True(t, continued.Ongoing)
|
||||
require.WithinDuration(t, now.Add(-time.Minute), continued.StartedAt, time.Second)
|
||||
}
|
||||
|
||||
func TestRestoreOngoingSessionsClosesStaleSessions(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
|
||||
// 上次活跃时间远早于在线窗口:重启恢复时按最后已知活跃时刻结转
|
||||
staleStart := time.Now().Add(-time.Hour)
|
||||
stale := model.OnlineSession{ServerID: 1, StartedAt: staleStart, LastActiveAt: staleStart.Add(30 * time.Minute), Ongoing: true}
|
||||
require.NoError(t, DB.Create(&stale).Error)
|
||||
|
||||
r := NewOnlineHistoryRecorder()
|
||||
r.RecordOnline(1, time.Now())
|
||||
|
||||
closed := getOnlineSession(t, stale.ID)
|
||||
require.False(t, closed.Ongoing)
|
||||
require.WithinDuration(t, staleStart.Add(30*time.Minute), closed.EndedAt, time.Second)
|
||||
// 结转后新的上报开启新会话
|
||||
require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
|
||||
}
|
||||
|
||||
func TestSweepStaleClosesOnlyStaleSessions(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
r := NewOnlineHistoryRecorder()
|
||||
|
||||
now := time.Now()
|
||||
r.RecordOnline(1, now.Add(-time.Hour))
|
||||
// 第二台服务器:会话在 DB 但没有 servers 记录(服务器已被删除),手工放进内存 map
|
||||
require.NoError(t, DB.Exec("INSERT INTO online_sessions (server_id, started_at, last_active_at, ongoing, created_at, updated_at) VALUES (2, ?, ?, 1, ?, ?)", now.Add(-time.Hour), now.Add(-time.Hour), now, now).Error)
|
||||
var second model.OnlineSession
|
||||
require.NoError(t, DB.Where("server_id = ?", 2).First(&second).Error)
|
||||
r.mu.Lock()
|
||||
r.open[2] = &onlineSessionRuntime{id: second.ID, lastPersist: now.Add(-time.Hour)}
|
||||
r.mu.Unlock()
|
||||
|
||||
// 服务器 1 活跃,服务器 2 不存在 → 仅关闭服务器 2 的会话
|
||||
lastActive := func(serverID uint64) (time.Time, bool) {
|
||||
if serverID == 1 {
|
||||
return now, true
|
||||
}
|
||||
return time.Time{}, false
|
||||
}
|
||||
r.SweepStale(now, lastActive)
|
||||
|
||||
require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
|
||||
require.Equal(t, int64(0), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 2, true))
|
||||
|
||||
// 服务器 1 失联后也会被关闭,结束时刻为最后已知活跃时间
|
||||
r.SweepStale(now.Add(model.OnlineActiveWindow*2), func(uint64) (time.Time, bool) {
|
||||
return now, true
|
||||
})
|
||||
require.Equal(t, int64(0), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true))
|
||||
var closedFirst = getOnlineSession(t, func() uint64 {
|
||||
var s model.OnlineSession
|
||||
require.NoError(t, DB.Where("server_id = ?", 1).First(&s).Error)
|
||||
return s.ID
|
||||
}())
|
||||
require.WithinDuration(t, now, closedFirst.EndedAt, time.Second)
|
||||
}
|
||||
|
||||
func TestPruneOnlineSessions(t *testing.T) {
|
||||
setupOnlineHistoryTestDB(t)
|
||||
previousConf := Conf
|
||||
Conf = &ConfigClass{Config: &model.Config{}}
|
||||
Conf.OnlineStatsDays = 7
|
||||
t.Cleanup(func() { Conf = previousConf })
|
||||
|
||||
now := time.Now()
|
||||
old := now.AddDate(0, 0, -30)
|
||||
rows := []model.OnlineSession{
|
||||
{ServerID: 1, StartedAt: old, EndedAt: old.Add(time.Hour), LastActiveAt: old.Add(time.Hour)}, // 超窗口 → 清理
|
||||
{ServerID: 1, StartedAt: old, EndedAt: now.Add(-time.Hour), LastActiveAt: now.Add(-time.Hour)}, // 跨窗口边界 → 保留
|
||||
{ServerID: 1, StartedAt: now.Add(-time.Hour), EndedAt: now, LastActiveAt: now}, // 窗口内 → 保留
|
||||
{ServerID: 1, StartedAt: now, Ongoing: true, LastActiveAt: now}, // 进行中 → 保留
|
||||
{ServerID: 99, StartedAt: now.Add(-time.Hour), EndedAt: now, LastActiveAt: now}, // 服务器已删除 → 清理
|
||||
}
|
||||
for i := range rows {
|
||||
require.NoError(t, DB.Create(&rows[i]).Error)
|
||||
}
|
||||
|
||||
PruneOnlineSessions()
|
||||
|
||||
require.Equal(t, int64(3), countOnlineSessions(t, "1 = 1"))
|
||||
require.Equal(t, int64(1), countOnlineSessions(t, "ongoing = ?", true))
|
||||
}
|
||||
@@ -63,6 +63,7 @@ func LoadSingleton(bus chan<- *model.Service) (err error) {
|
||||
ServerShared = NewServerClass()
|
||||
CronShared = NewCronClass()
|
||||
ServerTransferShared = NewServerTransferClass()
|
||||
OnlineHistoryShared = NewOnlineHistoryRecorder()
|
||||
// 最后初始化 ServiceSentinel
|
||||
ServiceSentinelShared, err = NewServiceSentinel(bus)
|
||||
return
|
||||
@@ -94,7 +95,7 @@ func InitDBFromPath(path string) error {
|
||||
model.Cron{}, model.Transfer{}, model.ServerGroupServer{},
|
||||
model.NAT{}, model.DDNSProfile{}, model.NotificationGroupNotification{},
|
||||
model.WAF{}, model.Oauth2Bind{}, model.ServerTransfer{}, model.JWTSession{},
|
||||
model.APIToken{}, model.MCPAuditLog{})
|
||||
model.APIToken{}, model.MCPAuditLog{}, model.OnlineSession{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user