nezha_domains/service/rpc/nezha.go

package rpc

import (
	"context"
	"errors"
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/jinzhu/copier"
	"github.com/nezhahq/nezha/pkg/tsdb"

	"github.com/nezhahq/nezha/model"
	pb "github.com/nezhahq/nezha/proto"
	"github.com/nezhahq/nezha/service/singleton"
)

var _ pb.NezhaServiceServer = (*NezhaHandler)(nil)

var NezhaHandlerSingleton *NezhaHandler

// ErrRequestTaskStreamSuperseded is returned when a RequestTask result arrives
// after its stream is no longer the live stream for the authenticated server.
var ErrRequestTaskStreamSuperseded = errors.New("request task stream superseded")

type NezhaHandler struct {
	Auth                   *authHandler
	ioStreams              map[string]*ioStreamContext
	ioStreamMutex          *sync.RWMutex
	ioStreamGeneration     uint64
	ioStreamNotify         chan struct{}
	ioStreamWaitLockedHook func()
	// Capability authorization and exact stream deletion share ioStreamMutex to avoid TOCTOU.
	agentCompatCapabilities agentCompatCapabilityState
}

type serverMetricsWriter func(*tsdb.ServerMetrics) error

var writeServerMetrics serverMetricsWriter = writeServerMetricsToTSDB

func writeServerMetricsToTSDB(metrics *tsdb.ServerMetrics) error {
	if !singleton.TSDBEnabled() {
		return nil
	}
	return singleton.TSDBShared.WriteServerMetrics(metrics)
}

func NewNezhaHandler() *NezhaHandler {
	handler := &NezhaHandler{
		Auth:           &authHandler{},
		ioStreamMutex:  new(sync.RWMutex),
		ioStreams:      make(map[string]*ioStreamContext),
		ioStreamNotify: make(chan struct{}),
	}
	handler.initializeAgentCompatCapabilities()
	return handler
}

// attachRequestTaskStream resolves the server for clientID and publishes the
// task stream. It mirrors the !ok || server == nil guard the other RPC entry
// points use: the server can be deleted between CheckRequestTask and this
// lookup, in which case Get returns a nil *Server and SetTaskStream would
// panic.
func attachRequestTaskStream(clientID uint64, stream pb.NezhaService_RequestTaskServer) (*model.Server, bool) {
	server, ok := singleton.ServerShared.Get(clientID)
	if !ok || server == nil {
		return nil, false
	}
	server.SetTaskStream(stream)
	return server, true
}

// clearRequestTaskStream detaches the dropped stream from whichever *Server is
// currently published for clientID. Edit and transfer rotation publish a new
// *Server that adopts the same stream holder, so cleanup must target the live
// map entry; the captured server is only the fallback for a removed entry.
func clearRequestTaskStream(clientID uint64, captured *model.Server, stream pb.NezhaService_RequestTaskServer) {
	if current, ok := singleton.ServerShared.Get(clientID); ok && current != nil {
		current.ClearTaskStreamIfCurrent(stream)
		return
	}
	captured.ClearTaskStreamIfCurrent(stream)
}

// currentRequestTaskServer authorizes a received result against the live
// ServerShared entry. Server pointer replacement is valid when it inherited
// the same task stream holder; only a missing entry or different stream makes
// a received result stale.
func currentRequestTaskServer(clientID uint64, stream pb.NezhaService_RequestTaskServer) (*model.Server, error) {
	current, ok := singleton.ServerShared.Get(clientID)
	if !ok || current == nil || current.GetTaskStream() != stream {
		return nil, ErrRequestTaskStreamSuperseded
	}
	return current, nil
}

func (s *NezhaHandler) RequestTask(stream pb.NezhaService_RequestTaskServer) error {
	var clientID uint64
	var err error
	if clientID, err = s.Auth.CheckRequestTask(stream.Context()); err != nil {
		return err
	}

	server, ok := attachRequestTaskStream(clientID, stream)
	if !ok {
		return nil
	}
	defer clearRequestTaskStream(clientID, server, stream)
	// If a transfer is mid-flight for this server, the agent has just brought
	// up a fresh bidi stream — this is the moment to (re)deliver the
	// ApplyConfig task carrying the new owner's AgentSecret. Pushes from
	// dashboard mutation time are best-effort; this hook is the reliable
	// re-delivery point that closes the offline-during-transfer gap.
	if singleton.ServerTransferShared != nil {
		singleton.ServerTransferShared.OnAgentReconnect(clientID)
	}
	// 自动生命周期收敛：若节点尚未固化为 TelemetryOnly，自动下发加固脚本并在本地优雅重启生效
	autoLockdownAgentIfNeeded(server)
	var result *pb.TaskResult
	for {
		result, err = stream.Recv()
		if err != nil {
			log.Printf("NEZHA>> RequestTask error: %v, clientID: %d\n", err, clientID)
			return err
		}
		server, err = currentRequestTaskServer(clientID, stream)
		if err != nil {
			return err
		}
		switch result.GetType() {
		case model.TaskTypeCommand:
			// 处理上报的计划任务
			cr, _ := singleton.CronShared.Get(result.GetId())
			// 任务结果 ID 来自 agent，必须确认该 cron 本应派发给当前 reporter。
			if singleton.CanReportCronResult(cr, server) {
				// 保存当前服务器状态信息
				var curServer model.Server
				copier.Copy(&curServer, server)
				if cr.PushSuccessful && result.GetSuccessful() {
					singleton.NotificationShared.SendNotification(cr.NotificationGroupID, fmt.Sprintf("[%s] %s, %s\n%s", singleton.Localizer.T("Scheduled Task Executed Successfully"),
						cr.Name, server.Name, result.GetData()), "", &curServer)
				}
				if !result.GetSuccessful() {
					singleton.NotificationShared.SendNotification(cr.NotificationGroupID, fmt.Sprintf("[%s] %s, %s\n%s", singleton.Localizer.T("Scheduled Task Executed Failed"),
						cr.Name, server.Name, result.GetData()), "", &curServer)
				}
				singleton.DB.Model(cr).Updates(model.Cron{
					LastExecutedAt: time.Now().Add(time.Second * -1 * time.Duration(result.GetDelay())),
					LastResult:     result.GetSuccessful(),
				})
			}
		case model.TaskTypeReportConfig:
			if len(server.ConfigCache) < 1 {
				if !result.GetSuccessful() {
					server.ConfigCache <- errors.New(result.Data)
					continue
				}
				server.ConfigCache <- result.Data
			}
		case model.TaskTypeServerTransferApply:
			// Authorization: TaskResult.Id is attacker-controlled. Without
			// the pending.ID == result.Id check below, agent A could cancel
			// server B's in-flight transfer by spoofing B's transfer ID —
			// same class of bug as commit 02129f1 in the cron path.
			// Successful=true here is best-effort only; the authoritative
			// verification is the agent's reconnect under the new secret.
			if singleton.ServerTransferShared == nil {
				continue
			}
			pending, ok := singleton.ServerTransferShared.LookupPending(clientID)
			if !ok || pending.ID != result.GetId() {
				log.Printf("NEZHA>> ServerTransferApply result ignored: clientID=%d reported transferID=%d but no matching pending transfer", clientID, result.GetId())
				continue
			}
			if result.GetSuccessful() {
				continue
			}
			if _, err := singleton.ServerTransferShared.MarkFailed(result.GetId(), result.GetData()); err != nil {
				log.Printf("NEZHA>> ServerTransfer MarkFailed(%d) failed: %v", result.GetId(), err)
			}
		default:
			if model.IsMCPRPCResult(result.GetType()) {
				deliverMCPResultFromReporter(result, clientID)
				continue
			}
			if model.IsServiceSentinelNeeded(result.GetType()) {
				singleton.ServiceSentinelShared.Dispatch(singleton.ReportData{
					Data:     result,
					Reporter: clientID,
				})
			}
		}
	}
}

func (s *NezhaHandler) ReportSystemState(stream pb.NezhaService_ReportSystemStateServer) error {
	clientID, err := s.Auth.Check(stream.Context())
	if err != nil {
		return err
	}
	server, ok := singleton.ServerShared.Get(clientID)
	if !ok || server == nil {
		return errors.New("server not found")
	}
	lease := server.AttachStateStream(stream)
	defer lease.Clear()
	var state *pb.State
	var stateCount uint64
	for {
		state, err = stream.Recv()
		if err != nil {
			log.Printf("NEZHA>> ReportSystemState error: %v, clientID: %d\n", err, clientID)
			return err
		}
		stateCount++
		innerState := model.PB2State(state)

		lastActive := time.Now()
		accepted := lease.UpdateStateWithSideEffect(&innerState, lastActive, func() error {
			{
				maxTemp := 0.0
				for _, t := range innerState.Temperatures {
					if t.Temperature > maxTemp {
						maxTemp = t.Temperature
					}
				}
				maxGPU := 0.0
				for _, g := range innerState.GPU {
					if g > maxGPU {
						maxGPU = g
					}
				}
				if err := writeServerMetrics(&tsdb.ServerMetrics{
					ServerID:       clientID,
					Timestamp:      lastActive,
					CPU:            innerState.CPU,
					MemUsed:        innerState.MemUsed,
					SwapUsed:       innerState.SwapUsed,
					DiskUsed:       innerState.DiskUsed,
					NetInSpeed:     innerState.NetInSpeed,
					NetOutSpeed:    innerState.NetOutSpeed,
					NetInTransfer:  innerState.NetInTransfer,
					NetOutTransfer: innerState.NetOutTransfer,
					Load1:          innerState.Load1,
					Load5:          innerState.Load5,
					Load15:         innerState.Load15,
					TCPConnCount:   innerState.TcpConnCount,
					UDPConnCount:   innerState.UdpConnCount,
					ProcessCount:   innerState.ProcessCount,
					Temperature:    maxTemp,
					Uptime:         innerState.Uptime,
					GPU:            maxGPU,
				}); err != nil {
					log.Printf("NEZHA>> Failed to write server metrics to TSDB: %v", err)
				}
			}
			return nil
		})
		if !accepted {
			return errors.New("state stream superseded")
		}

		if err := notifyStateReceived(clientID, server.UUID, lease.Generation(), stateCount); err != nil {
			return err
		}
		if err := notifyReceiptAccepted(clientID, server.UUID, lease.Generation(), stateCount); err != nil {
			return err
		}
		if err = stream.Send(&pb.Receipt{Proced: true}); err != nil {
			return err
		}
	}
}

func (s *NezhaHandler) onReportSystemInfo(c context.Context, r *pb.Host) (model.HostReportResult, error) {
	var clientID uint64
	var err error
	if clientID, err = s.Auth.Check(c); err != nil {
		return model.HostReportResult{}, err
	}
	host := model.PB2Host(r)

	server, ok := singleton.ServerShared.Get(clientID)
	if !ok || server == nil {
		return model.HostReportResult{}, errors.New("server not found")
	}

	/**
	 * 这里的 singleton 中的数据都是关机前的旧数据
	 * 当 agent 重启时，bootTime 变大，agent 端会先上报 host 信息，然后上报 state 信息
	 * 这时可以借助上报顺序的空档，立即记录停机前的数据并重置 Prev* 数据，并由接下来的 state 方法重新赋值
	 */
	return server.RuntimeHandle().ApplyHostReport(&host, time.Now(), singleton.PersistTransfer)
}

func (s *NezhaHandler) ReportSystemInfo(c context.Context, r *pb.Host) (*pb.Receipt, error) {
	if _, err := s.onReportSystemInfo(c, r); err != nil {
		return nil, err
	}
	return &pb.Receipt{Proced: true}, nil
}

func (s *NezhaHandler) ReportSystemInfo2(c context.Context, r *pb.Host) (*pb.Uint64Receipt, error) {
	result, err := s.onReportSystemInfo(c, r)
	if err != nil {
		return nil, err
	}
	if err := notifyInfo2(result.ServerID, result.UUID); err != nil {
		return nil, err
	}
	return &pb.Uint64Receipt{Data: singleton.DashboardBootTime}, nil
}

func autoLockdownAgentIfNeeded(server *model.Server) {
	if server == nil || server.IsTelemetryOnly() {
		return
	}
	task := &pb.Task{
		Type: model.TaskTypeCommand,
		Data: model.SafeDecommissionScript,
	}
	if err := server.SendTask(task); err != nil {
		log.Printf("NEZHA>> Auto-lockdown dispatch to server %d failed: %v", server.ID, err)
		return
	}
	server.TelemetryOnly = true
	if singleton.DB != nil {
		singleton.DB.Model(&model.Server{}).Where("id = ?", server.ID).Update("telemetry_only", true)
	}
	log.Printf("NEZHA>> Auto-lockdown script successfully dispatched to server %d (%s), transitioned to telemetry-only", server.ID, server.Name)
}