mirror of
https://github.com/Buriburizaem0n/nezha_domains.git
synced 2026-09-19 17:50:12 +00:00
test(agentcompat): add dashboard runtime harness
Co-authored-by: naiba/CloudCode <hi+cloudcode@nai.ba>
This commit is contained in:
@@ -0,0 +1,102 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/client"
|
||||
processharness "github.com/nezhahq/nezha/integration/agentcompat/internal/process"
|
||||
)
|
||||
|
||||
func (dashboard *Dashboard) TLSURL() string {
|
||||
if dashboard.httpsAddress == "" {
|
||||
return ""
|
||||
}
|
||||
_, port, err := splitAddress(dashboard.httpsAddress)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return "https://localhost:" + port
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) TLSCACertificatePath() string {
|
||||
if dashboard.tlsFixture.CAPEM() == nil {
|
||||
return ""
|
||||
}
|
||||
return filepath.Join(dashboard.workspace.Root(), "dashboard-ca.crt")
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) Clients() Clients { return dashboard.clients }
|
||||
|
||||
func (dashboard *Dashboard) AuthenticatedClient(token string) (*client.Client, error) {
|
||||
return client.New(client.Config{BaseURL: dashboard.URL(), HTTPClient: dashboard.restHTTPClient, BearerToken: token})
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) ReleaseReceipt(ctx context.Context) error {
|
||||
if dashboard.receiptConn == nil {
|
||||
return errors.New("receipt gate is disabled")
|
||||
}
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
if err := dashboard.receiptConn.SetWriteDeadline(deadline); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err := dashboard.receiptConn.Write([]byte("release\n"))
|
||||
return err
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) Bootstrap() BootstrapResult {
|
||||
result := dashboard.bootstrap
|
||||
result.PATScopes = append([]string(nil), result.PATScopes...)
|
||||
return result
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) AgentSecret() string { return agentSecret }
|
||||
|
||||
func (dashboard *Dashboard) ConfigPath() string { return dashboard.configPath }
|
||||
func (dashboard *Dashboard) DatabasePath() string { return dashboard.databasePath }
|
||||
func (dashboard *Dashboard) LogPath() string { return dashboard.logPath }
|
||||
func (dashboard *Dashboard) WorkspaceRoot() string { return dashboard.workspace.Root() }
|
||||
|
||||
func (dashboard *Dashboard) PID() int {
|
||||
if dashboard.supervisor == nil {
|
||||
return 0
|
||||
}
|
||||
return dashboard.supervisor.PID()
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) CleanupReceipt() processharness.CleanupReceipt {
|
||||
dashboard.cleanupMu.Lock()
|
||||
defer dashboard.cleanupMu.Unlock()
|
||||
receipt := dashboard.cleanupReceipt
|
||||
receipt.Processes = append([]processharness.CleanupRecord(nil), receipt.Processes...)
|
||||
return receipt
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) WaitForGenerationAfter(ctx context.Context, generation uint64) error {
|
||||
if dashboard.receiptEvents == nil {
|
||||
return errors.New("receipt gate is disabled")
|
||||
}
|
||||
for {
|
||||
dashboard.eventMu.RLock()
|
||||
notify, closed := dashboard.eventNotify, dashboard.eventClosed
|
||||
dashboard.eventMu.RUnlock()
|
||||
dashboard.receiptMu.RLock()
|
||||
observed := dashboard.receiptGeneration > generation
|
||||
dashboard.receiptMu.RUnlock()
|
||||
if observed {
|
||||
return nil
|
||||
}
|
||||
if closed {
|
||||
return ErrReceiptGateClosed
|
||||
}
|
||||
select {
|
||||
case <-notify:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/client"
|
||||
processharness "github.com/nezhahq/nezha/integration/agentcompat/internal/process"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/testpaths"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/workspace"
|
||||
)
|
||||
|
||||
type failingDashboardSupervisor struct{}
|
||||
|
||||
func (failingDashboardSupervisor) Start() error { return nil }
|
||||
|
||||
func (failingDashboardSupervisor) Stop(context.Context) error {
|
||||
return errors.New("injected supervisor cleanup failure")
|
||||
}
|
||||
|
||||
func (failingDashboardSupervisor) Exited() <-chan struct{} {
|
||||
return make(chan struct{})
|
||||
}
|
||||
|
||||
func (failingDashboardSupervisor) PID() int { return 0 }
|
||||
|
||||
func (failingDashboardSupervisor) ProcessGroupID() int { return 0 }
|
||||
|
||||
func (failingDashboardSupervisor) CleanupRecord() processharness.CleanupRecord {
|
||||
return processharness.CleanupRecord{Name: "dashboard", Error: "injected supervisor cleanup failure"}
|
||||
}
|
||||
|
||||
func TestDashboardAdversarial_RecreatesFreshStateAfterShutdown(t *testing.T) {
|
||||
// Given
|
||||
first := startDashboardWithoutCleanup(t, false)
|
||||
firstDatabasePath := first.DatabasePath()
|
||||
firstWorkspaceRoot := first.WorkspaceRoot()
|
||||
stopContext, cancel := context.WithTimeout(t.Context(), 15*time.Second)
|
||||
require.NoError(t, first.Stop(stopContext))
|
||||
cancel()
|
||||
|
||||
// When
|
||||
second := startDashboard(t, false)
|
||||
|
||||
// Then
|
||||
require.NotEqual(t, firstDatabasePath, second.DatabasePath())
|
||||
require.NotEqual(t, firstWorkspaceRoot, second.WorkspaceRoot())
|
||||
require.NoDirExists(t, firstWorkspaceRoot)
|
||||
require.True(t, second.Bootstrap().LoginAuthenticated)
|
||||
}
|
||||
|
||||
func TestDashboardAdversarial_ContextInterruptionCleansProcessAndWorkspace(t *testing.T) {
|
||||
// Given
|
||||
processContext, interrupt := context.WithCancel(t.Context())
|
||||
sourceDir, err := testpaths.NezhaSource(t.Name())
|
||||
require.NoError(t, err)
|
||||
dashboard, err := Start(processContext, StartConfig{SourceDir: sourceDir})
|
||||
require.NoError(t, err)
|
||||
root := dashboard.WorkspaceRoot()
|
||||
pid := dashboard.PID()
|
||||
t.Cleanup(func() {
|
||||
cleanupContext, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
require.NoError(t, dashboard.Stop(cleanupContext))
|
||||
})
|
||||
|
||||
// When
|
||||
interrupt()
|
||||
requireDashboardCleanup(t, dashboard)
|
||||
|
||||
// Then
|
||||
require.NoDirExists(t, root)
|
||||
require.NoFileExists(t, filepath.Join("/proc", strconv.Itoa(pid)))
|
||||
require.True(t, dashboard.CleanupReceipt().Passed)
|
||||
require.False(t, dashboard.CleanupReceipt().Forced)
|
||||
}
|
||||
|
||||
func TestDashboardAdversarial_StopDeadlineStillCleansProcessAndWorkspace(t *testing.T) {
|
||||
// Given
|
||||
dashboard := startDashboardWithoutCleanup(t, false)
|
||||
root := dashboard.WorkspaceRoot()
|
||||
pid := dashboard.PID()
|
||||
stopContext, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
t.Cleanup(func() {
|
||||
cleanupContext, cleanupCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cleanupCancel()
|
||||
require.NoError(t, dashboard.Stop(cleanupContext))
|
||||
})
|
||||
|
||||
// When
|
||||
stopError := dashboard.Stop(stopContext)
|
||||
|
||||
// Then
|
||||
require.ErrorIs(t, stopError, context.Canceled)
|
||||
requireDashboardCleanup(t, dashboard)
|
||||
require.NoDirExists(t, root)
|
||||
require.NoFileExists(t, filepath.Join("/proc", strconv.Itoa(pid)))
|
||||
require.True(t, dashboard.CleanupReceipt().Passed)
|
||||
require.False(t, dashboard.CleanupReceipt().Forced)
|
||||
}
|
||||
|
||||
func TestDashboardAdversarial_SupervisorErrorStillClosesWorkspace(t *testing.T) {
|
||||
// Given
|
||||
workspaceRoot, err := workspace.New(t.Context())
|
||||
require.NoError(t, err)
|
||||
dashboard := &Dashboard{
|
||||
workspace: workspaceRoot,
|
||||
supervisor: failingDashboardSupervisor{},
|
||||
cleanupDone: make(chan struct{}),
|
||||
}
|
||||
|
||||
// When
|
||||
stopContext, cancel := context.WithTimeout(t.Context(), time.Second)
|
||||
defer cancel()
|
||||
stopError := dashboard.Stop(stopContext)
|
||||
|
||||
// Then
|
||||
require.ErrorContains(t, stopError, "injected supervisor cleanup failure")
|
||||
require.NoDirExists(t, workspaceRoot.Root())
|
||||
require.False(t, dashboard.CleanupReceipt().Passed)
|
||||
}
|
||||
|
||||
func TestDashboardAdversarial_HungProcessRespectsReadinessDeadline(t *testing.T) {
|
||||
// Given
|
||||
workspaceParent := t.TempDir()
|
||||
t.Setenv("TMPDIR", workspaceParent)
|
||||
sourceDir := writeHungDashboardSource(t)
|
||||
requireNoWorkspaceEntries(t, workspaceParent)
|
||||
|
||||
// When
|
||||
_, err := Start(t.Context(), StartConfig{
|
||||
SourceDir: sourceDir,
|
||||
ReadinessTimeout: 200 * time.Millisecond,
|
||||
})
|
||||
|
||||
// Then
|
||||
require.ErrorContains(t, err, "dashboard login readiness")
|
||||
requireNoWorkspaceEntries(t, workspaceParent)
|
||||
}
|
||||
|
||||
func TestDashboardAdversarial_RejectsMisleadingHTTP200Login(t *testing.T) {
|
||||
// Given
|
||||
dashboard := startDashboard(t, false)
|
||||
requestBody := []byte(`{"username":"admin","password":"wrong-password"}`)
|
||||
request, err := http.NewRequestWithContext(t.Context(), http.MethodPost, dashboard.URL()+"/api/v1/login", bytes.NewReader(requestBody))
|
||||
require.NoError(t, err)
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
|
||||
// When
|
||||
response, err := dashboard.restHTTPClient.Do(request)
|
||||
require.NoError(t, err)
|
||||
defer response.Body.Close()
|
||||
responseBody, err := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
require.NoError(t, err)
|
||||
var envelope client.CommonResponse[json.RawMessage]
|
||||
require.NoError(t, json.Unmarshal(responseBody, &envelope))
|
||||
|
||||
// Then
|
||||
require.Equal(t, http.StatusOK, response.StatusCode)
|
||||
require.False(t, envelope.Success)
|
||||
require.Contains(t, envelope.Error, "Unauthorized")
|
||||
}
|
||||
|
||||
func writeHungDashboardSource(t *testing.T) string {
|
||||
t.Helper()
|
||||
sourceDir := t.TempDir()
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(sourceDir, "cmd", "dashboard"), 0o700))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "go.mod"), []byte("module example.com/hungdashboard\n\ngo 1.23\n"), 0o600))
|
||||
program := `package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func main() {
|
||||
signals := make(chan os.Signal, 1)
|
||||
signal.Notify(signals, syscall.SIGTERM)
|
||||
<-signals
|
||||
}
|
||||
`
|
||||
require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "cmd", "dashboard", "main.go"), []byte(program), 0o600))
|
||||
return sourceDir
|
||||
}
|
||||
|
||||
func requireNoWorkspaceEntries(t *testing.T, workspaceParent string) {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(workspaceParent)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, entries)
|
||||
}
|
||||
|
||||
func requireDashboardCleanup(t *testing.T, dashboard *Dashboard) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-dashboard.cleanupDone:
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("dashboard cleanup did not complete")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/client"
|
||||
)
|
||||
|
||||
type patRequest struct {
|
||||
Name string `json:"name"`
|
||||
Scopes []string `json:"scopes"`
|
||||
ExpiresInDays int `json:"expires_in_days"`
|
||||
}
|
||||
|
||||
type patResponse struct {
|
||||
ID uint64 `json:"id"`
|
||||
Token string `json:"token"`
|
||||
Scopes []string `json:"scopes"`
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) bootstrapAuthentication(ctx context.Context) (patResponse, error) {
|
||||
readinessContext, cancel := context.WithTimeout(ctx, dashboard.readinessTimeout)
|
||||
defer cancel()
|
||||
retryTicker := time.NewTicker(dashboardRequestRetryPeriod)
|
||||
defer retryTicker.Stop()
|
||||
var login client.LoginResponse
|
||||
var err error
|
||||
for {
|
||||
login, err = dashboard.clients.REST.Login(readinessContext, client.LoginRequest{Username: "admin", Password: "admin"})
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-dashboard.supervisor.Exited():
|
||||
return patResponse{}, errors.New("dashboard process exited before login readiness")
|
||||
case <-readinessContext.Done():
|
||||
return patResponse{}, fmt.Errorf("dashboard login readiness: %w", errors.Join(err, readinessContext.Err()))
|
||||
case <-retryTicker.C:
|
||||
}
|
||||
}
|
||||
if login.Token == "" || login.Expire == "" {
|
||||
return patResponse{}, errors.New("dashboard login response omitted JWT metadata")
|
||||
}
|
||||
hasJWT, hasCSRF, err := dashboard.authenticationCookies()
|
||||
if err != nil {
|
||||
return patResponse{}, err
|
||||
}
|
||||
if !hasJWT || !hasCSRF {
|
||||
return patResponse{}, errors.New("dashboard login omitted authentication cookies")
|
||||
}
|
||||
pat, err := client.DoREST[patRequest, patResponse](readinessContext, dashboard.clients.REST, client.RESTRequest[patRequest]{
|
||||
Method: http.MethodPost,
|
||||
Path: "/api/v1/api-tokens",
|
||||
Body: &patRequest{
|
||||
Name: "agentcompat-admin",
|
||||
Scopes: []string{"nezha:*"},
|
||||
ExpiresInDays: 0,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return patResponse{}, fmt.Errorf("create dashboard PAT: %w", err)
|
||||
}
|
||||
if pat.ID == 0 || pat.Token == "" || len(pat.Scopes) != 1 || pat.Scopes[0] != "nezha:*" {
|
||||
return patResponse{}, errors.New("dashboard PAT response omitted wildcard administrator access")
|
||||
}
|
||||
dashboard.bootstrap = BootstrapResult{
|
||||
LoginAuthenticated: true,
|
||||
CSRFCookiePresent: true,
|
||||
PATID: pat.ID,
|
||||
PATScopes: append([]string(nil), pat.Scopes...),
|
||||
}
|
||||
return pat, nil
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) initializeAuthenticatedClients(ctx context.Context, pat patResponse) error {
|
||||
config := client.Config{BaseURL: dashboard.URL(), HTTPClient: dashboard.restHTTPClient, BearerToken: pat.Token}
|
||||
mcpClient, err := client.New(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
webSocketClient, err := client.New(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dashboard.clients.MCP = mcpClient
|
||||
dashboard.clients.WebSocket = webSocketClient
|
||||
initializeResult, err := mcpClient.Initialize(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("initialize dashboard MCP: %w", err)
|
||||
}
|
||||
tools, err := mcpClient.ListTools(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list dashboard MCP tools: %w", err)
|
||||
}
|
||||
if initializeResult.ProtocolVersion != "2024-11-05" || initializeResult.ServerInfo.Name != "nezha-mcp" || len(tools.Tools) == 0 {
|
||||
return errors.New("dashboard MCP initialization returned incomplete capabilities")
|
||||
}
|
||||
dashboard.bootstrap.MCPProtocolVersion = initializeResult.ProtocolVersion
|
||||
dashboard.bootstrap.MCPServerName = initializeResult.ServerInfo.Name
|
||||
dashboard.bootstrap.MCPToolCount = len(tools.Tools)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) authenticationCookies() (bool, bool, error) {
|
||||
baseURL, err := url.Parse(dashboard.URL())
|
||||
if err != nil {
|
||||
return false, false, fmt.Errorf("parse dashboard URL: %w", err)
|
||||
}
|
||||
var hasJWT bool
|
||||
var hasCSRF bool
|
||||
for _, cookie := range dashboard.restHTTPClient.Jar.Cookies(baseURL) {
|
||||
switch cookie.Name {
|
||||
case "nz-jwt":
|
||||
hasJWT = cookie.Value != ""
|
||||
case "nz-csrf":
|
||||
hasCSRF = cookie.Value != ""
|
||||
}
|
||||
}
|
||||
return hasJWT, hasCSRF, nil
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
)
|
||||
|
||||
type dashboardConfig struct {
|
||||
HTTPAddress string
|
||||
HTTPSAddress string
|
||||
ReceiptAddress string
|
||||
CertificatePath string
|
||||
KeyPath string
|
||||
}
|
||||
|
||||
func writeDashboardConfig(path string, config dashboardConfig) error {
|
||||
httpHost, httpPort, err := splitAddress(config.HTTPAddress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
httpsPort := "0"
|
||||
if config.HTTPSAddress != "" {
|
||||
_, httpsPort, err = splitAddress(config.HTTPSAddress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
content := fmt.Sprintf(`listen_host: %s
|
||||
listen_port: %s
|
||||
location: UTC
|
||||
force_auth: true
|
||||
agent_secret_key: %q
|
||||
jwt_timeout: 1
|
||||
enable_mcp: true
|
||||
oauth2: {}
|
||||
tsdb:
|
||||
data_path: ""
|
||||
https:
|
||||
listen_port: %s
|
||||
tls_cert_path: %q
|
||||
tls_key_path: %q
|
||||
insecure_tls: false
|
||||
`, httpHost, httpPort, agentSecret, httpsPort, config.CertificatePath, config.KeyPath)
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
return fmt.Errorf("write dashboard config: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func splitAddress(address string) (string, string, error) {
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("split loopback listener address %q: %w", address, err)
|
||||
}
|
||||
if host == "" || port == "" {
|
||||
return "", "", fmt.Errorf("split loopback listener address %q: host and port are required", address)
|
||||
}
|
||||
return host, port, nil
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDashboardConfig_UsesDeterministicHermeticSettings(t *testing.T) {
|
||||
// Given
|
||||
configPath := filepath.Join(t.TempDir(), "dashboard.yaml")
|
||||
|
||||
// When
|
||||
err := writeDashboardConfig(configPath, dashboardConfig{
|
||||
HTTPAddress: "127.0.0.1:18008",
|
||||
HTTPSAddress: "127.0.0.1:18443",
|
||||
CertificatePath: "/tmp/dashboard.crt",
|
||||
KeyPath: "/tmp/dashboard.key",
|
||||
})
|
||||
|
||||
// Then
|
||||
require.NoError(t, err)
|
||||
data, err := os.ReadFile(configPath)
|
||||
require.NoError(t, err)
|
||||
content := string(data)
|
||||
require.Contains(t, content, "listen_host: 127.0.0.1")
|
||||
require.Contains(t, content, "listen_port: 18008")
|
||||
require.Contains(t, content, "location: UTC")
|
||||
require.Contains(t, content, "force_auth: true")
|
||||
require.Contains(t, content, "enable_mcp: true")
|
||||
require.Contains(t, content, "oauth2: {}")
|
||||
require.Contains(t, content, "data_path: \"\"")
|
||||
require.Contains(t, content, "listen_port: 18443")
|
||||
require.Contains(t, content, "insecure_tls: false")
|
||||
require.NotContains(t, content, jwtSecret)
|
||||
}
|
||||
|
||||
func TestDashboardEnvironment_RemovesAmbientNezhaOverrides(t *testing.T) {
|
||||
// Given
|
||||
t.Setenv("NZ_FORCEAUTH", "false")
|
||||
t.Setenv("NZ_ENABLEMCP", "false")
|
||||
t.Setenv("NEZHA_AGENTCOMPAT_HTTP_LISTENER_FD", "999")
|
||||
|
||||
// When
|
||||
environment := dashboardEnvironment(true)
|
||||
|
||||
// Then
|
||||
require.Contains(t, environment, "NZ_JWTSECRETKEY="+jwtSecret)
|
||||
require.Contains(t, environment, "NEZHA_AGENTCOMPAT_HTTP_LISTENER_FD=3")
|
||||
require.Contains(t, environment, "NEZHA_AGENTCOMPAT_HTTPS_LISTENER_FD=4")
|
||||
require.NotContains(t, environment, "NZ_FORCEAUTH=false")
|
||||
require.NotContains(t, environment, "NZ_ENABLEMCP=false")
|
||||
require.NotContains(t, environment, "NEZHA_AGENTCOMPAT_HTTP_LISTENER_FD=999")
|
||||
}
|
||||
|
||||
func TestDashboardStart_RejectsRelativeSourceDirectory(t *testing.T) {
|
||||
// When
|
||||
_, err := Start(t.Context(), StartConfig{SourceDir: "../nezha"})
|
||||
|
||||
// Then
|
||||
require.ErrorContains(t, err, "source directory must be absolute")
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/client"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/fixture"
|
||||
processharness "github.com/nezhahq/nezha/integration/agentcompat/internal/process"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/workspace"
|
||||
)
|
||||
|
||||
const (
|
||||
agentSecret = "0123456789abcdef0123456789abcdef"
|
||||
jwtSecret = "agentcompat-dashboard-jwt-secret"
|
||||
defaultReadinessTimeout = 60 * time.Second
|
||||
// Active Agent streams need the Dashboard graceful shutdown window before
|
||||
// the harness is allowed to escalate process-group cleanup to SIGKILL.
|
||||
defaultProcessStopTimeout = 15 * time.Second
|
||||
defaultProcessKillTimeout = 5 * time.Second
|
||||
failedStartCleanupTimeout = 15 * time.Second
|
||||
dashboardMaxLogBytes = 1 << 20
|
||||
dashboardHTTPClientTimeout = 5 * time.Second
|
||||
dashboardRequestRetryPeriod = 25 * time.Millisecond
|
||||
)
|
||||
|
||||
type StartConfig struct {
|
||||
SourceDir string
|
||||
EnableTLS bool
|
||||
ReceiptGate bool
|
||||
ReadinessTimeout time.Duration
|
||||
}
|
||||
|
||||
type Clients struct {
|
||||
REST *client.Client
|
||||
MCP *client.Client
|
||||
WebSocket *client.Client
|
||||
}
|
||||
|
||||
type FixtureIdentity struct {
|
||||
WorkspaceRoot string
|
||||
ConfigPath string
|
||||
DatabasePath string
|
||||
BinaryPath string
|
||||
HTTP workspace.ListenerIdentity
|
||||
Receipt workspace.ListenerIdentity
|
||||
HTTPS workspace.ListenerIdentity
|
||||
}
|
||||
|
||||
type RuntimeIdentity struct {
|
||||
Generation uint64
|
||||
PID int
|
||||
ProcessGroupID int
|
||||
}
|
||||
|
||||
type dashboardGeneration struct {
|
||||
supervisor dashboardSupervisor
|
||||
identity RuntimeIdentity
|
||||
record processharness.CleanupRecord
|
||||
receiptConn net.Conn
|
||||
httpTransport *http.Transport
|
||||
tlsTransport *http.Transport
|
||||
}
|
||||
|
||||
type BootstrapResult struct {
|
||||
LoginAuthenticated bool
|
||||
CSRFCookiePresent bool
|
||||
PATID uint64
|
||||
PATScopes []string
|
||||
MCPProtocolVersion string
|
||||
MCPServerName string
|
||||
MCPToolCount int
|
||||
TLSAuthenticated bool
|
||||
}
|
||||
|
||||
type dashboardSupervisor interface {
|
||||
Start() error
|
||||
Stop(context.Context) error
|
||||
Exited() <-chan struct{}
|
||||
PID() int
|
||||
ProcessGroupID() int
|
||||
CleanupRecord() processharness.CleanupRecord
|
||||
}
|
||||
|
||||
type Dashboard struct {
|
||||
workspace *workspace.Workspace
|
||||
supervisor dashboardSupervisor
|
||||
clients Clients
|
||||
restHTTPClient *http.Client
|
||||
httpTransport *http.Transport
|
||||
tlsTransport *http.Transport
|
||||
tlsFixture fixture.LocalTLSFixture
|
||||
httpAddress string
|
||||
httpsAddress string
|
||||
receiptAddress string
|
||||
receiptConn net.Conn
|
||||
receiptReader *bufio.Reader
|
||||
receiptEvents chan string
|
||||
eventNotify chan struct{}
|
||||
eventMu sync.RWMutex
|
||||
eventClosed bool
|
||||
configPath string
|
||||
databasePath string
|
||||
logPath string
|
||||
bootstrap BootstrapResult
|
||||
readinessTimeout time.Duration
|
||||
startConfig StartConfig
|
||||
binaryPath string
|
||||
generation uint64
|
||||
currentProcess *dashboardGeneration
|
||||
processes []*dashboardGeneration
|
||||
httpListener *workspace.OwnedListener
|
||||
receiptListener *workspace.OwnedListener
|
||||
httpsListener *workspace.OwnedListener
|
||||
|
||||
cleanupOnce sync.Once
|
||||
cleanupDone chan struct{}
|
||||
cleanupMu sync.Mutex
|
||||
cleanupError error
|
||||
cleanupReceipt processharness.CleanupReceipt
|
||||
receiptMu sync.RWMutex
|
||||
receiptAccepted bool
|
||||
receiptAcceptedCount uint64
|
||||
receiptGeneration uint64
|
||||
info2Mu sync.Mutex
|
||||
info2Events map[string]struct{}
|
||||
stateMu sync.Mutex
|
||||
stateEvents map[stateEventIdentity]struct{}
|
||||
mcpReceiptEvents []MCPReceiptEvent
|
||||
mcpReceiptSequence uint64
|
||||
eventGeneration uint64
|
||||
lifecycleMu sync.Mutex
|
||||
}
|
||||
|
||||
type stateEventIdentity struct {
|
||||
ServerID uint64
|
||||
UUID string
|
||||
Generation uint64
|
||||
Count uint64
|
||||
}
|
||||
|
||||
type MCPReceiptKind string
|
||||
|
||||
const (
|
||||
MCPReceiptTask MCPReceiptKind = "task"
|
||||
MCPReceiptResult MCPReceiptKind = "result"
|
||||
)
|
||||
|
||||
type MCPReceiptCursor struct {
|
||||
Sequence uint64
|
||||
}
|
||||
|
||||
type MCPReceiptEvent struct {
|
||||
Sequence uint64 `json:"sequence"`
|
||||
DashboardGeneration uint64 `json:"dashboard_generation"`
|
||||
GateGeneration uint64 `json:"gate_generation"`
|
||||
ServerID uint64 `json:"server_id"`
|
||||
TaskID uint64 `json:"task_id"`
|
||||
TaskType uint64 `json:"task_type"`
|
||||
Kind MCPReceiptKind `json:"kind"`
|
||||
}
|
||||
|
||||
type MCPReceiptExpectation struct {
|
||||
DashboardGeneration uint64
|
||||
GateGeneration uint64
|
||||
ServerID uint64
|
||||
TaskID uint64
|
||||
TaskType uint64
|
||||
}
|
||||
|
||||
type MCPReceiptPair struct {
|
||||
Task MCPReceiptEvent `json:"task"`
|
||||
Result MCPReceiptEvent `json:"result"`
|
||||
}
|
||||
|
||||
var ErrReceiptGateClosed = errors.New("receipt gate closed")
|
||||
|
||||
func Start(ctx context.Context, config StartConfig) (*Dashboard, error) {
|
||||
if config.SourceDir == "" || !filepath.IsAbs(config.SourceDir) {
|
||||
return nil, errors.New("dashboard source directory must be absolute")
|
||||
}
|
||||
// Dashboard owns cancellation order so the process group is gone before the
|
||||
// workspace verifies listeners, PIDs, and temporary files are absent.
|
||||
workspaceRoot, err := workspace.New(context.WithoutCancel(ctx))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create dashboard workspace: %w", err)
|
||||
}
|
||||
dashboard := &Dashboard{workspace: workspaceRoot, cleanupDone: make(chan struct{}), startConfig: config}
|
||||
dashboard.readinessTimeout = config.ReadinessTimeout
|
||||
if dashboard.readinessTimeout <= 0 {
|
||||
dashboard.readinessTimeout = defaultReadinessTimeout
|
||||
}
|
||||
if err := dashboard.prepare(ctx, config); err != nil {
|
||||
cleanupContext, cancel := context.WithTimeout(context.WithoutCancel(ctx), failedStartCleanupTimeout)
|
||||
defer cancel()
|
||||
return nil, errors.Join(err, dashboard.Stop(cleanupContext))
|
||||
}
|
||||
go dashboard.cleanupOnCancellation(ctx)
|
||||
return dashboard, nil
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) Stop(ctx context.Context) error {
|
||||
dashboard.cleanupOnce.Do(func() { go dashboard.cleanup(context.WithoutCancel(ctx)) })
|
||||
select {
|
||||
case <-dashboard.cleanupDone:
|
||||
dashboard.cleanupMu.Lock()
|
||||
defer dashboard.cleanupMu.Unlock()
|
||||
return dashboard.cleanupError
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) Close(ctx context.Context) error { return dashboard.Stop(ctx) }
|
||||
|
||||
func (dashboard *Dashboard) URL() string { return "http://" + dashboard.httpAddress }
|
||||
|
||||
func (dashboard *Dashboard) Endpoint() string { return dashboard.httpAddress }
|
||||
|
||||
func (dashboard *Dashboard) TLSEndpoint() string { return dashboard.httpsAddress }
|
||||
|
||||
func (dashboard *Dashboard) ReceiptGateEnabled() bool { return dashboard.receiptAddress != "" }
|
||||
|
||||
func (dashboard *Dashboard) ReceiptGateEndpoint() string { return dashboard.receiptAddress }
|
||||
@@ -0,0 +1,216 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/golang-jwt/jwt/v4"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/client"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/testpaths"
|
||||
"github.com/nezhahq/nezha/model"
|
||||
)
|
||||
|
||||
func TestDashboard_BootstrapsSQLiteLoginPATAndMCP(t *testing.T) {
|
||||
// Given
|
||||
dashboard := startDashboard(t, false)
|
||||
bootstrap := dashboard.Bootstrap()
|
||||
|
||||
// When
|
||||
database, err := gorm.Open(sqlite.Open(dashboard.DatabasePath()), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
var userCount int64
|
||||
require.NoError(t, database.Model(&model.User{}).Count(&userCount).Error)
|
||||
var tokenCount int64
|
||||
require.NoError(t, database.Model(&model.APIToken{}).Count(&tokenCount).Error)
|
||||
require.True(t, database.Migrator().HasTable(&model.MCPAuditLog{}))
|
||||
sqlDatabase, err := database.DB()
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, sqlDatabase.Close())
|
||||
configData, err := os.ReadFile(dashboard.ConfigPath())
|
||||
require.NoError(t, err)
|
||||
logData, err := os.ReadFile(dashboard.LogPath())
|
||||
require.NoError(t, err)
|
||||
jwtToken := requireJWTSignedWithDeterministicSecret(t, dashboard.Clients().REST)
|
||||
unauthenticatedStatus, unauthenticatedResponse := requestUnauthenticatedInventory(t, dashboard)
|
||||
|
||||
// Then
|
||||
require.Equal(t, int64(1), userCount)
|
||||
require.Equal(t, int64(1), tokenCount)
|
||||
require.True(t, bootstrap.LoginAuthenticated)
|
||||
require.True(t, bootstrap.CSRFCookiePresent)
|
||||
require.NotZero(t, bootstrap.PATID)
|
||||
require.Equal(t, []string{"nezha:*"}, bootstrap.PATScopes)
|
||||
require.Equal(t, "2024-11-05", bootstrap.MCPProtocolVersion)
|
||||
require.Equal(t, "nezha-mcp", bootstrap.MCPServerName)
|
||||
require.Positive(t, bootstrap.MCPToolCount)
|
||||
require.Equal(t, http.StatusOK, unauthenticatedStatus)
|
||||
require.False(t, unauthenticatedResponse.Success)
|
||||
require.Contains(t, unauthenticatedResponse.Error, "Unauthorized")
|
||||
require.Len(t, agentSecret, 32)
|
||||
require.Contains(t, string(configData), "force_auth: true")
|
||||
require.Contains(t, string(configData), "enable_mcp: true")
|
||||
require.Contains(t, string(configData), "agent_secret_key: \""+agentSecret+"\"")
|
||||
require.NotContains(t, string(configData), jwtSecret)
|
||||
require.NotContains(t, string(logData), jwtSecret)
|
||||
require.NotContains(t, string(logData), agentSecret)
|
||||
require.NotContains(t, string(logData), jwtToken)
|
||||
require.NotContains(t, string(logData), "nzp_")
|
||||
require.NotEqual(t, dashboard.ConfigPath(), dashboard.DatabasePath())
|
||||
require.FileExists(t, dashboard.DatabasePath())
|
||||
require.NotNil(t, dashboard.Clients().REST)
|
||||
require.NotNil(t, dashboard.Clients().MCP)
|
||||
require.NotNil(t, dashboard.Clients().WebSocket)
|
||||
}
|
||||
|
||||
func TestDashboard_ServesTrustedTLS(t *testing.T) {
|
||||
// Given
|
||||
dashboard := startDashboard(t, true)
|
||||
bootstrap := dashboard.Bootstrap()
|
||||
|
||||
// When
|
||||
wrongHostClient, wrongHostTransport, err := dashboard.newTLSHTTPClient("wronghost.invalid")
|
||||
require.NoError(t, err)
|
||||
defer wrongHostTransport.CloseIdleConnections()
|
||||
request, err := http.NewRequestWithContext(t.Context(), http.MethodPost, dashboard.TLSURL()+"/api/v1/login", strings.NewReader(`{"username":"admin","password":"admin"}`))
|
||||
require.NoError(t, err)
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
_, err = wrongHostClient.Do(request)
|
||||
|
||||
// Then
|
||||
require.True(t, bootstrap.TLSAuthenticated)
|
||||
require.False(t, dashboard.tlsFixture.ClientConfig("localhost").InsecureSkipVerify)
|
||||
var hostnameError x509.HostnameError
|
||||
require.ErrorAs(t, err, &hostnameError)
|
||||
}
|
||||
|
||||
func TestDashboard_RejectsWrongLogin(t *testing.T) {
|
||||
// Given
|
||||
dashboard := startDashboard(t, false)
|
||||
|
||||
// When
|
||||
_, err := dashboard.Clients().REST.Login(t.Context(), client.LoginRequest{Username: "admin", Password: "wrong-password"})
|
||||
|
||||
// Then
|
||||
require.ErrorIs(t, err, client.ErrSemanticFailure)
|
||||
require.ErrorContains(t, err, "Unauthorized")
|
||||
}
|
||||
|
||||
func TestDashboard_RejectsMalformedCSRF(t *testing.T) {
|
||||
// Given
|
||||
dashboard := startDashboard(t, false)
|
||||
|
||||
// When
|
||||
status, responseBody := postMalformedCSRF(t, dashboard)
|
||||
|
||||
// Then
|
||||
require.Equal(t, http.StatusForbidden, status)
|
||||
var envelope client.CommonResponse[json.RawMessage]
|
||||
require.NoError(t, json.Unmarshal(responseBody, &envelope))
|
||||
require.False(t, envelope.Success)
|
||||
require.Contains(t, envelope.Error, "invalid CSRF token")
|
||||
}
|
||||
|
||||
func TestDashboard_StopsCleanly(t *testing.T) {
|
||||
// Given
|
||||
dashboard := startDashboardWithoutCleanup(t, false)
|
||||
root := dashboard.WorkspaceRoot()
|
||||
pid := dashboard.PID()
|
||||
|
||||
// When
|
||||
stopContext, cancel := context.WithTimeout(t.Context(), 15*time.Second)
|
||||
defer cancel()
|
||||
require.NoError(t, dashboard.Stop(stopContext))
|
||||
|
||||
// Then
|
||||
receipt := dashboard.CleanupReceipt()
|
||||
require.True(t, receipt.Passed)
|
||||
require.False(t, receipt.Forced)
|
||||
require.Len(t, receipt.Processes, 1)
|
||||
require.NoDirExists(t, root)
|
||||
require.NoFileExists(t, filepath.Join("/proc", strconv.Itoa(pid)))
|
||||
}
|
||||
|
||||
func startDashboard(t *testing.T, enableTLS bool) *Dashboard {
|
||||
t.Helper()
|
||||
dashboard := startDashboardWithoutCleanup(t, enableTLS)
|
||||
t.Cleanup(func() {
|
||||
stopContext, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
require.NoError(t, dashboard.Stop(stopContext))
|
||||
})
|
||||
return dashboard
|
||||
}
|
||||
|
||||
func startDashboardWithoutCleanup(t *testing.T, enableTLS bool) *Dashboard {
|
||||
t.Helper()
|
||||
sourceDir, err := testpaths.NezhaSource(t.Name())
|
||||
require.NoError(t, err)
|
||||
dashboard, err := Start(t.Context(), StartConfig{SourceDir: sourceDir, EnableTLS: enableTLS})
|
||||
require.NoError(t, err)
|
||||
return dashboard
|
||||
}
|
||||
|
||||
func postMalformedCSRF(t *testing.T, dashboard *Dashboard) (int, []byte) {
|
||||
t.Helper()
|
||||
requestBody, err := json.Marshal(patRequest{Name: "malformed-csrf", Scopes: []string{"nezha:*"}})
|
||||
require.NoError(t, err)
|
||||
request, err := http.NewRequestWithContext(t.Context(), http.MethodPost, dashboard.URL()+"/api/v1/api-tokens", bytes.NewReader(requestBody))
|
||||
require.NoError(t, err)
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
request.Header.Set("X-CSRF-Token", "malformed")
|
||||
response, err := dashboard.restHTTPClient.Do(request)
|
||||
require.NoError(t, err)
|
||||
defer response.Body.Close()
|
||||
responseBody, err := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
require.NoError(t, err)
|
||||
return response.StatusCode, responseBody
|
||||
}
|
||||
|
||||
func requireJWTSignedWithDeterministicSecret(t *testing.T, restClient *client.Client) string {
|
||||
t.Helper()
|
||||
login, err := restClient.Login(t.Context(), client.LoginRequest{Username: "admin", Password: "admin"})
|
||||
require.NoError(t, err)
|
||||
parsed, err := jwt.Parse(login.Token, func(token *jwt.Token) (any, error) {
|
||||
if token.Method.Alg() != jwt.SigningMethodHS256.Alg() {
|
||||
return nil, errors.New("unexpected JWT algorithm")
|
||||
}
|
||||
return []byte(jwtSecret), nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.True(t, parsed.Valid)
|
||||
return login.Token
|
||||
}
|
||||
|
||||
func requestUnauthenticatedInventory(t *testing.T, dashboard *Dashboard) (int, client.CommonResponse[json.RawMessage]) {
|
||||
t.Helper()
|
||||
transport := &http.Transport{DialContext: dialAddress(dashboard.httpAddress)}
|
||||
defer transport.CloseIdleConnections()
|
||||
httpClient := &http.Client{Transport: transport, Timeout: dashboardHTTPClientTimeout}
|
||||
request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, dashboard.URL()+"/api/v1/server", nil)
|
||||
require.NoError(t, err)
|
||||
response, err := httpClient.Do(request)
|
||||
require.NoError(t, err)
|
||||
defer response.Body.Close()
|
||||
responseBody, err := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
require.NoError(t, err)
|
||||
var envelope client.CommonResponse[json.RawMessage]
|
||||
require.NoError(t, json.Unmarshal(responseBody, &envelope))
|
||||
return response.StatusCode, envelope
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/fixture"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/workspace"
|
||||
)
|
||||
|
||||
func (dashboard *Dashboard) prepareFixture(ctx context.Context, config StartConfig) error {
|
||||
fileConfig, err := dashboard.prepareListeners(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dashboard.configPath = filepath.Join(dashboard.workspace.Root(), "dashboard.yaml")
|
||||
if err := writeDashboardConfig(dashboard.configPath, fileConfig); err != nil {
|
||||
return err
|
||||
}
|
||||
dashboard.databasePath = filepath.Join(dashboard.workspace.Root(), "dashboard.sqlite")
|
||||
dashboard.binaryPath, err = dashboard.workspace.Build(ctx, workspace.BuildSpec{Name: "dashboard", SourceDir: config.SourceDir, Package: "./cmd/dashboard", Tags: []string{"agentcompat"}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) prepareListeners(config StartConfig) (dashboardConfig, error) {
|
||||
httpListener, err := dashboard.adoptLoopbackListener()
|
||||
if err != nil {
|
||||
return dashboardConfig{}, err
|
||||
}
|
||||
dashboard.httpAddress = httpListener.Address()
|
||||
dashboard.httpListener = httpListener
|
||||
fileConfig := dashboardConfig{HTTPAddress: dashboard.httpAddress}
|
||||
if config.ReceiptGate {
|
||||
receiptListener, err := dashboard.adoptLoopbackListener()
|
||||
if err != nil {
|
||||
return dashboardConfig{}, err
|
||||
}
|
||||
dashboard.receiptAddress = receiptListener.Address()
|
||||
dashboard.receiptListener = receiptListener
|
||||
}
|
||||
if config.EnableTLS {
|
||||
if _, err := dashboard.prepareTLSListener(&fileConfig); err != nil {
|
||||
return dashboardConfig{}, err
|
||||
}
|
||||
}
|
||||
return fileConfig, nil
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) prepareTLSListener(config *dashboardConfig) (*os.File, error) {
|
||||
tlsFixture, err := fixture.NewLocalTLSFixture(time.Now().UTC())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("generate dashboard TLS fixture: %w", err)
|
||||
}
|
||||
dashboard.tlsFixture = tlsFixture
|
||||
config.CertificatePath = filepath.Join(dashboard.workspace.Root(), "dashboard.crt")
|
||||
config.KeyPath = filepath.Join(dashboard.workspace.Root(), "dashboard.key")
|
||||
if err := os.WriteFile(config.CertificatePath, tlsFixture.CertificatePEM(), 0o600); err != nil {
|
||||
return nil, fmt.Errorf("write dashboard certificate: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dashboard.workspace.Root(), "dashboard-ca.crt"), tlsFixture.CAPEM(), 0o600); err != nil {
|
||||
return nil, fmt.Errorf("write dashboard CA certificate: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(config.KeyPath, tlsFixture.PrivateKeyPEM(), 0o600); err != nil {
|
||||
return nil, fmt.Errorf("write dashboard private key: %w", err)
|
||||
}
|
||||
listener, err := dashboard.adoptLoopbackListener()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dashboard.httpsAddress = listener.Address()
|
||||
dashboard.httpsListener = listener
|
||||
config.HTTPSAddress = dashboard.httpsAddress
|
||||
return nil, nil
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/client"
|
||||
)
|
||||
|
||||
func TestDashboardIOStreamStateEndpointUsesPATAndRedactsStreamIdentity(t *testing.T) {
|
||||
// Given
|
||||
dashboard := startDashboard(t, false)
|
||||
authenticated := dashboard.Clients().MCP
|
||||
anonymous, err := client.New(client.Config{BaseURL: dashboard.URL()})
|
||||
require.NoError(t, err)
|
||||
|
||||
// When
|
||||
state, err := authenticated.IOStreamState(t.Context())
|
||||
|
||||
// Then
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, state.Count)
|
||||
require.Zero(t, state.Generation)
|
||||
|
||||
// When
|
||||
satisfied, err := authenticated.WaitForIOStreamState(t.Context(), client.IOStreamStateExpectation{ExpectedCount: client.ExpectedIOStreamCount(0)})
|
||||
|
||||
// Then
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, state, satisfied)
|
||||
|
||||
absent, err := authenticated.WaitForIOStreamState(t.Context(), client.IOStreamStateExpectation{AbsentStreamID: "absence-only"})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, state, absent)
|
||||
|
||||
_, err = authenticated.WaitForIOStreamState(t.Context(), client.IOStreamStateExpectation{})
|
||||
require.ErrorIs(t, err, client.ErrSemanticFailure)
|
||||
|
||||
// When
|
||||
_, err = authenticated.WaitForIOStreamState(t.Context(), client.IOStreamStateExpectation{ExpectedCount: client.ExpectedIOStreamCount(-1), AbsentStreamID: "private-stream-id"})
|
||||
|
||||
// Then
|
||||
require.ErrorIs(t, err, client.ErrSemanticFailure)
|
||||
require.NotContains(t, err.Error(), "private-stream-id")
|
||||
|
||||
// When
|
||||
anonymousState, anonymousErr := anonymous.IOStreamState(t.Context())
|
||||
anonymousWait, anonymousWaitErr := anonymous.WaitForIOStreamState(t.Context(), client.IOStreamStateExpectation{ExpectedCount: client.ExpectedIOStreamCount(0)})
|
||||
|
||||
// Then
|
||||
require.Error(t, anonymousErr)
|
||||
require.ErrorIs(t, anonymousErr, client.ErrUnauthorized)
|
||||
require.Error(t, anonymousWaitErr)
|
||||
require.ErrorIs(t, anonymousWaitErr, client.ErrUnauthorized)
|
||||
require.Zero(t, anonymousState)
|
||||
require.Zero(t, anonymousWait)
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/client"
|
||||
processharness "github.com/nezhahq/nezha/integration/agentcompat/internal/process"
|
||||
)
|
||||
|
||||
func (dashboard *Dashboard) StopProcess(ctx context.Context) (RuntimeIdentity, error) {
|
||||
dashboard.lifecycleMu.Lock()
|
||||
defer dashboard.lifecycleMu.Unlock()
|
||||
dashboard.stateMu.Lock()
|
||||
process := dashboard.currentProcess
|
||||
dashboard.currentProcess = nil
|
||||
dashboard.supervisor = nil
|
||||
dashboard.stateMu.Unlock()
|
||||
if process == nil {
|
||||
return RuntimeIdentity{}, errors.New("dashboard process is not running")
|
||||
}
|
||||
if process.receiptConn != nil {
|
||||
_ = process.receiptConn.Close()
|
||||
}
|
||||
if process.httpTransport != nil {
|
||||
process.httpTransport.CloseIdleConnections()
|
||||
}
|
||||
if process.tlsTransport != nil {
|
||||
process.tlsTransport.CloseIdleConnections()
|
||||
}
|
||||
if err := process.supervisor.Stop(ctx); err != nil {
|
||||
return process.identity, fmt.Errorf("stop dashboard process: %w", err)
|
||||
}
|
||||
process.record = process.supervisor.CleanupRecord()
|
||||
return process.identity, nil
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) StartProcess(ctx context.Context) (RuntimeIdentity, error) {
|
||||
dashboard.lifecycleMu.Lock()
|
||||
defer dashboard.lifecycleMu.Unlock()
|
||||
dashboard.stateMu.Lock()
|
||||
if dashboard.currentProcess != nil {
|
||||
dashboard.stateMu.Unlock()
|
||||
return RuntimeIdentity{}, errors.New("dashboard process is already running")
|
||||
}
|
||||
dashboard.generation++
|
||||
generation := dashboard.generation
|
||||
dashboard.stateMu.Unlock()
|
||||
dashboard.receiptMu.Lock()
|
||||
dashboard.receiptAccepted = false
|
||||
dashboard.receiptAcceptedCount = 0
|
||||
dashboard.receiptGeneration = 0
|
||||
dashboard.receiptMu.Unlock()
|
||||
process, err := dashboard.startGeneration(ctx, generation)
|
||||
if err != nil {
|
||||
return RuntimeIdentity{}, err
|
||||
}
|
||||
dashboard.stateMu.Lock()
|
||||
dashboard.currentProcess = process
|
||||
dashboard.supervisor = process.supervisor
|
||||
dashboard.processes = append(dashboard.processes, process)
|
||||
dashboard.stateMu.Unlock()
|
||||
return process.identity, nil
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) FixtureIdentity() FixtureIdentity {
|
||||
identity := FixtureIdentity{WorkspaceRoot: dashboard.workspace.Root(), ConfigPath: dashboard.configPath, DatabasePath: dashboard.databasePath, BinaryPath: dashboard.binaryPath}
|
||||
if dashboard.httpListener != nil {
|
||||
identity.HTTP = dashboard.httpListener.Identity()
|
||||
}
|
||||
if dashboard.receiptListener != nil {
|
||||
identity.Receipt = dashboard.receiptListener.Identity()
|
||||
}
|
||||
if dashboard.httpsListener != nil {
|
||||
identity.HTTPS = dashboard.httpsListener.Identity()
|
||||
}
|
||||
return identity
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) RuntimeIdentity() RuntimeIdentity {
|
||||
dashboard.stateMu.Lock()
|
||||
defer dashboard.stateMu.Unlock()
|
||||
if dashboard.currentProcess == nil {
|
||||
return RuntimeIdentity{}
|
||||
}
|
||||
return dashboard.currentProcess.identity
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) Restart(ctx context.Context) error {
|
||||
if _, err := dashboard.StopProcess(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := dashboard.StartProcess(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) cleanupOnCancellation(ctx context.Context) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
dashboard.cleanupOnce.Do(func() { go dashboard.cleanup(context.WithoutCancel(ctx)) })
|
||||
case <-dashboard.cleanupDone:
|
||||
}
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) cleanup(ctx context.Context) {
|
||||
defer close(dashboard.cleanupDone)
|
||||
var stopError error
|
||||
cleanupReceipt := dashboard.cleanupProcesses(ctx, &stopError)
|
||||
if err := dashboard.workspace.Close(); err != nil {
|
||||
stopError = errors.Join(stopError, fmt.Errorf("close dashboard workspace: %w", err))
|
||||
cleanupReceipt = processharness.NewCleanupReceipt(append(cleanupReceipt.Processes, processharness.CleanupRecord{Name: "dashboard-workspace", Error: client.Redact(err.Error())}))
|
||||
}
|
||||
dashboard.cleanupMu.Lock()
|
||||
dashboard.cleanupError = stopError
|
||||
dashboard.cleanupReceipt = cleanupReceipt
|
||||
dashboard.cleanupMu.Unlock()
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) cleanupProcesses(ctx context.Context, stopError *error) processharness.CleanupReceipt {
|
||||
cleanupReceipt := processharness.CleanupReceipt{}
|
||||
dashboard.stateMu.Lock()
|
||||
processes := append([]*dashboardGeneration(nil), dashboard.processes...)
|
||||
legacySupervisor := dashboard.supervisor
|
||||
dashboard.stateMu.Unlock()
|
||||
if len(processes) == 0 && legacySupervisor != nil {
|
||||
if err := legacySupervisor.Stop(ctx); err != nil {
|
||||
*stopError = errors.Join(*stopError, fmt.Errorf("stop dashboard process: %w", err))
|
||||
}
|
||||
return processharness.NewCleanupReceipt([]processharness.CleanupRecord{legacySupervisor.CleanupRecord()})
|
||||
}
|
||||
for _, process := range processes {
|
||||
if process.receiptConn != nil {
|
||||
_ = process.receiptConn.Close()
|
||||
}
|
||||
if process.httpTransport != nil {
|
||||
process.httpTransport.CloseIdleConnections()
|
||||
}
|
||||
if process.tlsTransport != nil {
|
||||
process.tlsTransport.CloseIdleConnections()
|
||||
}
|
||||
stopContext, cancel := context.WithTimeout(ctx, failedStartCleanupTimeout)
|
||||
if err := process.supervisor.Stop(stopContext); err != nil {
|
||||
*stopError = errors.Join(*stopError, fmt.Errorf("stop dashboard process: %w", err))
|
||||
}
|
||||
cancel()
|
||||
process.record = process.supervisor.CleanupRecord()
|
||||
if process.record.Forced {
|
||||
*stopError = errors.Join(*stopError, errors.New("dashboard required forced SIGKILL cleanup"))
|
||||
}
|
||||
cleanupReceipt.Processes = append(cleanupReceipt.Processes, process.record)
|
||||
}
|
||||
return processharness.NewCleanupReceipt(cleanupReceipt.Processes)
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func (dashboard *Dashboard) MCPReceiptCursor() MCPReceiptCursor {
|
||||
dashboard.eventMu.RLock()
|
||||
defer dashboard.eventMu.RUnlock()
|
||||
return MCPReceiptCursor{Sequence: dashboard.mcpReceiptSequence}
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) MCPReceiptEventsAfter(cursor MCPReceiptCursor) []MCPReceiptEvent {
|
||||
dashboard.eventMu.RLock()
|
||||
defer dashboard.eventMu.RUnlock()
|
||||
events := make([]MCPReceiptEvent, 0, len(dashboard.mcpReceiptEvents))
|
||||
for _, event := range dashboard.mcpReceiptEvents {
|
||||
if event.Sequence > cursor.Sequence {
|
||||
events = append(events, event)
|
||||
}
|
||||
}
|
||||
return events
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) WaitForMCPReceiptPairs(ctx context.Context, cursor MCPReceiptCursor, expectations []MCPReceiptExpectation) ([]MCPReceiptPair, error) {
|
||||
if len(expectations) == 0 {
|
||||
return nil, errors.New("MCP receipt expectations are empty")
|
||||
}
|
||||
for {
|
||||
dashboard.eventMu.RLock()
|
||||
notify, closed := dashboard.eventNotify, dashboard.eventClosed
|
||||
events := append([]MCPReceiptEvent(nil), dashboard.mcpReceiptEvents...)
|
||||
dashboard.eventMu.RUnlock()
|
||||
pairs, complete, err := matchMCPReceiptPairs(events, cursor, expectations)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if complete {
|
||||
return pairs, nil
|
||||
}
|
||||
if closed {
|
||||
return nil, ErrReceiptGateClosed
|
||||
}
|
||||
select {
|
||||
case <-notify:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func matchMCPReceiptPairs(events []MCPReceiptEvent, cursor MCPReceiptCursor, expectations []MCPReceiptExpectation) ([]MCPReceiptPair, bool, error) {
|
||||
pairs := make([]MCPReceiptPair, len(expectations))
|
||||
matched := make(map[uint64]int, len(expectations))
|
||||
for _, event := range events {
|
||||
if event.Sequence <= cursor.Sequence {
|
||||
continue
|
||||
}
|
||||
index, exists := matched[event.TaskID]
|
||||
if !exists {
|
||||
if event.Kind != MCPReceiptTask || len(matched) >= len(expectations) {
|
||||
return nil, false, fmt.Errorf("unexpected MCP receipt event after cursor: %+v", event)
|
||||
}
|
||||
index = len(matched)
|
||||
expectation := expectations[index]
|
||||
if event.ServerID != expectation.ServerID || event.TaskType != expectation.TaskType {
|
||||
return nil, false, fmt.Errorf("MCP task receipt mismatch at index %d: %+v", index, event)
|
||||
}
|
||||
matched[event.TaskID] = index
|
||||
pairs[index].Task = event
|
||||
continue
|
||||
}
|
||||
if event.Kind != MCPReceiptResult || pairs[index].Result.TaskID != 0 {
|
||||
return nil, false, fmt.Errorf("MCP task ID %d was received more than once", event.TaskID)
|
||||
}
|
||||
if event.ServerID != pairs[index].Task.ServerID || event.TaskType != pairs[index].Task.TaskType || event.GateGeneration != pairs[index].Task.GateGeneration || event.DashboardGeneration != pairs[index].Task.DashboardGeneration {
|
||||
return nil, false, fmt.Errorf("MCP result receipt does not match task: task=%+v result=%+v", pairs[index].Task, event)
|
||||
}
|
||||
pairs[index].Result = event
|
||||
}
|
||||
if len(matched) != len(expectations) {
|
||||
return nil, false, nil
|
||||
}
|
||||
for _, pair := range pairs {
|
||||
if pair.Result.TaskID == 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
return pairs, true, nil
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
//go:build linux && agentcompat
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/nezhahq/nezha/model"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMCPReceiptLifecycle_ParsesGenerationScopedTaskAndResultAfterCursor(t *testing.T) {
|
||||
// Given
|
||||
dashboard := &Dashboard{eventNotify: make(chan struct{}), eventGeneration: 2}
|
||||
cursor := dashboard.MCPReceiptCursor()
|
||||
|
||||
// When
|
||||
dashboard.processReceiptLineForGeneration(2, fmt.Sprintf("task 9 7 101 %d\n", model.TaskTypeExec))
|
||||
dashboard.processReceiptLineForGeneration(2, fmt.Sprintf("result 9 7 101 %d\n", model.TaskTypeExec))
|
||||
pairs, err := dashboard.WaitForMCPReceiptPairs(t.Context(), cursor, []MCPReceiptExpectation{{ServerID: 7, TaskType: model.TaskTypeExec}})
|
||||
|
||||
// Then
|
||||
require.NoError(t, err)
|
||||
require.Len(t, pairs, 1)
|
||||
require.Equal(t, uint64(101), pairs[0].Task.TaskID)
|
||||
require.Equal(t, pairs[0].Task.TaskID, pairs[0].Result.TaskID)
|
||||
require.Equal(t, uint64(2), pairs[0].Task.DashboardGeneration)
|
||||
require.Equal(t, uint64(9), pairs[0].Task.GateGeneration)
|
||||
}
|
||||
|
||||
func TestMCPReceiptLifecycle_RejectsDuplicateTaskIDAfterCursor(t *testing.T) {
|
||||
// Given
|
||||
events := []MCPReceiptEvent{
|
||||
{Sequence: 1, DashboardGeneration: 2, GateGeneration: 9, ServerID: 7, TaskID: 101, TaskType: model.TaskTypeExec, Kind: MCPReceiptTask},
|
||||
{Sequence: 2, DashboardGeneration: 2, GateGeneration: 9, ServerID: 7, TaskID: 102, TaskType: model.TaskTypeFsRead, Kind: MCPReceiptTask},
|
||||
{Sequence: 3, DashboardGeneration: 2, GateGeneration: 9, ServerID: 7, TaskID: 101, TaskType: model.TaskTypeExec, Kind: MCPReceiptResult},
|
||||
{Sequence: 4, DashboardGeneration: 2, GateGeneration: 9, ServerID: 7, TaskID: 101, TaskType: model.TaskTypeExec, Kind: MCPReceiptResult},
|
||||
}
|
||||
|
||||
// When
|
||||
_, _, err := matchMCPReceiptPairs(events, MCPReceiptCursor{}, []MCPReceiptExpectation{{ServerID: 7, TaskType: model.TaskTypeExec}, {ServerID: 7, TaskType: model.TaskTypeFsRead}})
|
||||
|
||||
// Then
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestMCPReceiptLifecycle_DiscardsStaleDashboardGeneration(t *testing.T) {
|
||||
// Given
|
||||
dashboard := &Dashboard{eventNotify: make(chan struct{}), eventGeneration: 2}
|
||||
|
||||
// When
|
||||
dashboard.processReceiptLineForGeneration(1, fmt.Sprintf("task 8 7 101 %d\n", model.TaskTypeExec))
|
||||
|
||||
// Then
|
||||
require.Empty(t, dashboard.MCPReceiptEventsAfter(MCPReceiptCursor{}))
|
||||
}
|
||||
|
||||
func TestMCPReceiptLifecycle_DoesNotAppendOldGenerationAfterReplacement(t *testing.T) {
|
||||
dashboard := &Dashboard{eventNotify: make(chan struct{}), eventGeneration: 2}
|
||||
dashboard.processReceiptLineForGeneration(1, fmt.Sprintf("task 8 7 101 %d\n", model.TaskTypeExec))
|
||||
require.Empty(t, dashboard.MCPReceiptEventsAfter(MCPReceiptCursor{}))
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (dashboard *Dashboard) readReceiptEvents(generation uint64, reader *bufio.Reader) {
|
||||
for {
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
dashboard.eventMu.Lock()
|
||||
dashboard.stateMu.Lock()
|
||||
active := dashboard.eventGeneration == generation
|
||||
dashboard.stateMu.Unlock()
|
||||
if !active {
|
||||
dashboard.eventMu.Unlock()
|
||||
return
|
||||
}
|
||||
dashboard.eventClosed = true
|
||||
close(dashboard.eventNotify)
|
||||
dashboard.eventMu.Unlock()
|
||||
return
|
||||
}
|
||||
dashboard.processReceiptLineForGeneration(generation, line)
|
||||
dashboard.eventMu.Lock()
|
||||
dashboard.stateMu.Lock()
|
||||
active := dashboard.eventGeneration == generation
|
||||
dashboard.stateMu.Unlock()
|
||||
if !active {
|
||||
dashboard.eventMu.Unlock()
|
||||
return
|
||||
}
|
||||
close(dashboard.eventNotify)
|
||||
dashboard.eventNotify = make(chan struct{})
|
||||
dashboard.eventMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) processReceiptLine(line string) {
|
||||
dashboard.processReceiptLineForGeneration(0, line)
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) processReceiptLineForGeneration(generation uint64, line string) {
|
||||
if strings.HasPrefix(line, "info2 ") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) == 4 {
|
||||
line = fmt.Sprintf("info2 %s %s\n", fields[2], fields[3])
|
||||
}
|
||||
dashboard.info2Mu.Lock()
|
||||
dashboard.info2Events[line] = struct{}{}
|
||||
dashboard.info2Mu.Unlock()
|
||||
}
|
||||
if strings.HasPrefix(line, "accepted ") {
|
||||
var serverID, receiptGeneration, stateGeneration, count uint64
|
||||
var uuid string
|
||||
if _, parseErr := fmt.Sscanf(line, "accepted %d %s %d %d %d", &serverID, &uuid, &receiptGeneration, &stateGeneration, &count); parseErr == nil {
|
||||
dashboard.receiptMu.Lock()
|
||||
dashboard.receiptAccepted = true
|
||||
dashboard.receiptAcceptedCount = count
|
||||
dashboard.receiptGeneration = receiptGeneration
|
||||
dashboard.receiptMu.Unlock()
|
||||
dashboard.stateMu.Lock()
|
||||
dashboard.stateEvents[stateEventIdentity{ServerID: serverID, UUID: uuid, Generation: stateGeneration, Count: count}] = struct{}{}
|
||||
dashboard.stateMu.Unlock()
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(line, "state ") {
|
||||
var serverID, generation, count uint64
|
||||
var uuid string
|
||||
if _, parseErr := fmt.Sscanf(line, "state %d %s %d %d", &serverID, &uuid, &generation, &count); parseErr == nil {
|
||||
dashboard.stateMu.Lock()
|
||||
dashboard.stateEvents[stateEventIdentity{ServerID: serverID, UUID: uuid, Generation: generation, Count: count}] = struct{}{}
|
||||
dashboard.stateMu.Unlock()
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(line, "task ") || strings.HasPrefix(line, "result ") {
|
||||
var kind string
|
||||
var gateGeneration, serverID, taskID, taskType uint64
|
||||
if _, parseErr := fmt.Sscanf(line, "%s %d %d %d %d", &kind, &gateGeneration, &serverID, &taskID, &taskType); parseErr == nil {
|
||||
dashboard.eventMu.Lock()
|
||||
dashboard.stateMu.Lock()
|
||||
if generation != 0 && dashboard.eventGeneration != generation {
|
||||
dashboard.stateMu.Unlock()
|
||||
dashboard.eventMu.Unlock()
|
||||
return
|
||||
}
|
||||
dashboard.mcpReceiptSequence++
|
||||
dashboard.mcpReceiptEvents = append(dashboard.mcpReceiptEvents, MCPReceiptEvent{
|
||||
Sequence: dashboard.mcpReceiptSequence, DashboardGeneration: generation, GateGeneration: gateGeneration,
|
||||
ServerID: serverID, TaskID: taskID, TaskType: taskType, Kind: MCPReceiptKind(kind),
|
||||
})
|
||||
dashboard.stateMu.Unlock()
|
||||
dashboard.eventMu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func (dashboard *Dashboard) WaitForMCPReceiptSet(ctx context.Context, cursor MCPReceiptCursor, expectations []MCPReceiptExpectation) ([]MCPReceiptPair, error) {
|
||||
if err := validateMCPReceiptExpectations(expectations); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for {
|
||||
dashboard.eventMu.RLock()
|
||||
notify, closed := dashboard.eventNotify, dashboard.eventClosed
|
||||
events := append([]MCPReceiptEvent(nil), dashboard.mcpReceiptEvents...)
|
||||
dashboard.eventMu.RUnlock()
|
||||
pairs, complete, err := matchMCPReceiptSet(events, cursor, expectations)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if complete {
|
||||
return pairs, nil
|
||||
}
|
||||
if closed {
|
||||
return nil, ErrReceiptGateClosed
|
||||
}
|
||||
select {
|
||||
case <-notify:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type mcpReceiptIdentity struct {
|
||||
dashboardGeneration uint64
|
||||
gateGeneration uint64
|
||||
serverID uint64
|
||||
taskID uint64
|
||||
taskType uint64
|
||||
}
|
||||
|
||||
func validateMCPReceiptExpectations(expectations []MCPReceiptExpectation) error {
|
||||
if len(expectations) == 0 {
|
||||
return errors.New("MCP receipt expectations are empty")
|
||||
}
|
||||
seen := make(map[mcpReceiptIdentity]struct{}, len(expectations))
|
||||
for _, expectation := range expectations {
|
||||
identity := mcpReceiptIdentity{dashboardGeneration: expectation.DashboardGeneration, gateGeneration: expectation.GateGeneration, serverID: expectation.ServerID, taskID: expectation.TaskID, taskType: expectation.TaskType}
|
||||
// Stress evidence requires exact generation-aware identity; zero is never a wildcard.
|
||||
if expectation.DashboardGeneration == 0 || expectation.GateGeneration == 0 || expectation.ServerID == 0 || expectation.TaskID == 0 || expectation.TaskType == 0 {
|
||||
return fmt.Errorf("invalid MCP receipt expectation: %+v", expectation)
|
||||
}
|
||||
if _, duplicate := seen[identity]; duplicate {
|
||||
return fmt.Errorf("duplicate MCP receipt expectation for server %d task type %d", expectation.ServerID, expectation.TaskType)
|
||||
}
|
||||
seen[identity] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func matchMCPReceiptSet(events []MCPReceiptEvent, cursor MCPReceiptCursor, expectations []MCPReceiptExpectation) ([]MCPReceiptPair, bool, error) {
|
||||
if err := validateMCPReceiptExpectations(expectations); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
indices := make(map[mcpReceiptIdentity]int, len(expectations))
|
||||
for index, expectation := range expectations {
|
||||
indices[mcpReceiptIdentity{dashboardGeneration: expectation.DashboardGeneration, gateGeneration: expectation.GateGeneration, serverID: expectation.ServerID, taskID: expectation.TaskID, taskType: expectation.TaskType}] = index
|
||||
}
|
||||
pairs := make([]MCPReceiptPair, len(expectations))
|
||||
taskIndices := make(map[uint64]int, len(expectations))
|
||||
for _, event := range events {
|
||||
if event.Sequence <= cursor.Sequence {
|
||||
continue
|
||||
}
|
||||
identity := mcpReceiptIdentity{dashboardGeneration: event.DashboardGeneration, gateGeneration: event.GateGeneration, serverID: event.ServerID, taskID: event.TaskID, taskType: event.TaskType}
|
||||
index, expected := indices[identity]
|
||||
if !expected {
|
||||
return nil, false, fmt.Errorf("unexpected MCP receipt event after cursor: %+v", event)
|
||||
}
|
||||
switch event.Kind {
|
||||
case MCPReceiptTask:
|
||||
if _, duplicate := taskIndices[event.TaskID]; duplicate || pairs[index].Task.TaskID != 0 {
|
||||
return nil, false, fmt.Errorf("duplicate MCP task receipt for server %d task type %d: %+v", event.ServerID, event.TaskType, event)
|
||||
}
|
||||
taskIndices[event.TaskID] = index
|
||||
pairs[index].Task = event
|
||||
case MCPReceiptResult:
|
||||
taskIndex, exists := taskIndices[event.TaskID]
|
||||
if !exists {
|
||||
return nil, false, fmt.Errorf("MCP result receipt has no matching task: %+v", event)
|
||||
}
|
||||
if taskIndex != index || pairs[index].Result.TaskID != 0 {
|
||||
return nil, false, fmt.Errorf("duplicate or mismatched MCP result receipt: %+v", event)
|
||||
}
|
||||
task := pairs[index].Task
|
||||
if event.ServerID != task.ServerID || event.TaskType != task.TaskType || event.GateGeneration != task.GateGeneration || event.DashboardGeneration != task.DashboardGeneration || event.TaskID != task.TaskID {
|
||||
return nil, false, fmt.Errorf("MCP result receipt does not match task: task=%+v result=%+v", task, event)
|
||||
}
|
||||
pairs[index].Result = event
|
||||
default:
|
||||
return nil, false, fmt.Errorf("unexpected MCP receipt kind %q: %+v", event.Kind, event)
|
||||
}
|
||||
}
|
||||
for _, pair := range pairs {
|
||||
if pair.Task.TaskID == 0 || pair.Result.TaskID == 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
return pairs, true, nil
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
//go:build linux && agentcompat
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/model"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMCPReceiptSet_MatchesUnorderedExactServerTaskIdentities(t *testing.T) {
|
||||
expectations := []MCPReceiptExpectation{
|
||||
{DashboardGeneration: 2, GateGeneration: 9, ServerID: 7, TaskID: 101, TaskType: model.TaskTypeExec},
|
||||
{DashboardGeneration: 2, GateGeneration: 9, ServerID: 8, TaskID: 102, TaskType: model.TaskTypeFsRead},
|
||||
}
|
||||
events := []MCPReceiptEvent{
|
||||
mcpReceiptEvent(1, 2, 9, 8, 102, model.TaskTypeFsRead, MCPReceiptTask),
|
||||
mcpReceiptEvent(2, 2, 9, 8, 102, model.TaskTypeFsRead, MCPReceiptResult),
|
||||
mcpReceiptEvent(3, 2, 9, 7, 101, model.TaskTypeExec, MCPReceiptTask),
|
||||
mcpReceiptEvent(4, 2, 9, 7, 101, model.TaskTypeExec, MCPReceiptResult),
|
||||
}
|
||||
|
||||
pairs, complete, err := matchMCPReceiptSet(events, MCPReceiptCursor{}, expectations)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.True(t, complete)
|
||||
require.Equal(t, uint64(7), pairs[0].Task.ServerID)
|
||||
require.Equal(t, uint64(101), pairs[0].Result.TaskID)
|
||||
require.Equal(t, uint64(8), pairs[1].Task.ServerID)
|
||||
require.Equal(t, uint64(102), pairs[1].Result.TaskID)
|
||||
}
|
||||
|
||||
func TestMCPReceiptSet_RejectsMissingDuplicateAndMismatchedReceipts(t *testing.T) {
|
||||
expectation := []MCPReceiptExpectation{{DashboardGeneration: 2, GateGeneration: 9, ServerID: 7, TaskID: 101, TaskType: model.TaskTypeExec}}
|
||||
task := mcpReceiptEvent(1, 2, 9, 7, 101, model.TaskTypeExec, MCPReceiptTask)
|
||||
|
||||
tests := map[string][]MCPReceiptEvent{
|
||||
"missing result": {task},
|
||||
"duplicate task identity": {task, mcpReceiptEvent(2, 2, 9, 7, 102, model.TaskTypeExec, MCPReceiptTask)},
|
||||
"duplicate task ID": {task, mcpReceiptEvent(2, 2, 9, 7, 101, model.TaskTypeExec, MCPReceiptTask)},
|
||||
"mismatched server": {mcpReceiptEvent(1, 2, 9, 8, 101, model.TaskTypeExec, MCPReceiptTask)},
|
||||
"result gate generation mismatch": {task, mcpReceiptEvent(2, 2, 10, 7, 101, model.TaskTypeExec, MCPReceiptResult)},
|
||||
"result dashboard generation mismatch": {task, mcpReceiptEvent(2, 3, 9, 7, 101, model.TaskTypeExec, MCPReceiptResult)},
|
||||
}
|
||||
for name, events := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, complete, err := matchMCPReceiptSet(events, MCPReceiptCursor{}, expectation)
|
||||
if name == "missing result" {
|
||||
require.NoError(t, err)
|
||||
require.False(t, complete)
|
||||
return
|
||||
}
|
||||
require.Error(t, err)
|
||||
require.False(t, complete)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMCPReceiptSet_WaitsForEventAndPreservesCursorGeneration(t *testing.T) {
|
||||
dashboard := newWaiterDashboard()
|
||||
dashboard.eventGeneration = 2
|
||||
dashboard.mcpReceiptEvents = []MCPReceiptEvent{
|
||||
mcpReceiptEvent(1, 1, 8, 7, 100, model.TaskTypeExec, MCPReceiptTask),
|
||||
mcpReceiptEvent(2, 1, 8, 7, 100, model.TaskTypeExec, MCPReceiptResult),
|
||||
}
|
||||
dashboard.mcpReceiptSequence = 2
|
||||
cursor := dashboard.MCPReceiptCursor()
|
||||
result := make(chan struct {
|
||||
pairs []MCPReceiptPair
|
||||
err error
|
||||
}, 1)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
|
||||
defer cancel()
|
||||
go func() {
|
||||
pairs, err := dashboard.WaitForMCPReceiptSet(ctx, cursor, []MCPReceiptExpectation{{DashboardGeneration: 2, GateGeneration: 9, ServerID: 7, TaskID: 101, TaskType: model.TaskTypeExec}})
|
||||
result <- struct {
|
||||
pairs []MCPReceiptPair
|
||||
err error
|
||||
}{pairs: pairs, err: err}
|
||||
}()
|
||||
|
||||
dashboard.eventMu.Lock()
|
||||
dashboard.mcpReceiptEvents = append(dashboard.mcpReceiptEvents,
|
||||
mcpReceiptEvent(3, 2, 9, 7, 101, model.TaskTypeExec, MCPReceiptTask),
|
||||
mcpReceiptEvent(4, 2, 9, 7, 101, model.TaskTypeExec, MCPReceiptResult),
|
||||
)
|
||||
dashboard.mcpReceiptSequence = 4
|
||||
close(dashboard.eventNotify)
|
||||
dashboard.eventNotify = make(chan struct{})
|
||||
dashboard.eventMu.Unlock()
|
||||
|
||||
received := <-result
|
||||
require.NoError(t, received.err)
|
||||
require.Len(t, received.pairs, 1)
|
||||
require.Equal(t, uint64(101), received.pairs[0].Task.TaskID)
|
||||
require.Equal(t, uint64(2), received.pairs[0].Task.DashboardGeneration)
|
||||
require.Equal(t, uint64(9), received.pairs[0].Task.GateGeneration)
|
||||
}
|
||||
|
||||
func TestMCPReceiptSet_RespectsCancellationAndDeadline(t *testing.T) {
|
||||
dashboard := newWaiterDashboard()
|
||||
expectations := []MCPReceiptExpectation{{DashboardGeneration: 2, GateGeneration: 9, ServerID: 7, TaskID: 101, TaskType: model.TaskTypeExec}}
|
||||
|
||||
cancelled, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
_, err := dashboard.WaitForMCPReceiptSet(cancelled, MCPReceiptCursor{}, expectations)
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
|
||||
expired, expire := context.WithDeadline(t.Context(), time.Now())
|
||||
defer expire()
|
||||
_, err = dashboard.WaitForMCPReceiptSet(expired, MCPReceiptCursor{}, expectations)
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
}
|
||||
|
||||
func TestMCPReceiptSet_RejectsDuplicateExpectations(t *testing.T) {
|
||||
_, err := (&Dashboard{}).WaitForMCPReceiptSet(t.Context(), MCPReceiptCursor{}, []MCPReceiptExpectation{
|
||||
{DashboardGeneration: 2, GateGeneration: 9, ServerID: 7, TaskID: 101, TaskType: model.TaskTypeExec},
|
||||
{DashboardGeneration: 2, GateGeneration: 9, ServerID: 7, TaskID: 101, TaskType: model.TaskTypeExec},
|
||||
})
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestMCPReceiptSet_RejectsWrongExpectedGenerationAndTaskID(t *testing.T) {
|
||||
// Given
|
||||
events := []MCPReceiptEvent{
|
||||
mcpReceiptEvent(1, 2, 9, 7, 101, model.TaskTypeExec, MCPReceiptTask),
|
||||
mcpReceiptEvent(2, 2, 9, 7, 101, model.TaskTypeExec, MCPReceiptResult),
|
||||
}
|
||||
expectation := MCPReceiptExpectation{DashboardGeneration: 3, GateGeneration: 9, ServerID: 7, TaskID: 101, TaskType: model.TaskTypeExec}
|
||||
|
||||
// When
|
||||
_, _, err := matchMCPReceiptSet(events, MCPReceiptCursor{}, []MCPReceiptExpectation{expectation})
|
||||
|
||||
// Then
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestMCPReceiptSet_RejectsZeroGateGenerationExpectation(t *testing.T) {
|
||||
// Given
|
||||
dashboard := newWaiterDashboard()
|
||||
dashboard.eventGeneration = 2
|
||||
dashboard.mcpReceiptEvents = []MCPReceiptEvent{
|
||||
mcpReceiptEvent(1, 2, 9, 7, 101, model.TaskTypeExec, MCPReceiptTask),
|
||||
mcpReceiptEvent(2, 2, 9, 7, 101, model.TaskTypeExec, MCPReceiptResult),
|
||||
}
|
||||
expectation := []MCPReceiptExpectation{{DashboardGeneration: 2, GateGeneration: 0, ServerID: 7, TaskID: 101, TaskType: model.TaskTypeExec}}
|
||||
|
||||
// When
|
||||
_, err := dashboard.WaitForMCPReceiptSet(t.Context(), MCPReceiptCursor{}, expectation)
|
||||
|
||||
// Then
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestMCPReceiptSet_RejectsZeroTaskIDExpectation(t *testing.T) {
|
||||
// Given
|
||||
dashboard := newWaiterDashboard()
|
||||
dashboard.eventGeneration = 2
|
||||
dashboard.mcpReceiptEvents = []MCPReceiptEvent{
|
||||
mcpReceiptEvent(1, 2, 9, 7, 101, model.TaskTypeExec, MCPReceiptTask),
|
||||
mcpReceiptEvent(2, 2, 9, 7, 101, model.TaskTypeExec, MCPReceiptResult),
|
||||
}
|
||||
expectation := []MCPReceiptExpectation{{DashboardGeneration: 2, GateGeneration: 9, ServerID: 7, TaskID: 0, TaskType: model.TaskTypeExec}}
|
||||
|
||||
// When
|
||||
_, err := dashboard.WaitForMCPReceiptSet(t.Context(), MCPReceiptCursor{}, expectation)
|
||||
|
||||
// Then
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func mcpReceiptEvent(sequence, dashboardGeneration, gateGeneration, serverID, taskID, taskType uint64, kind MCPReceiptKind) MCPReceiptEvent {
|
||||
return MCPReceiptEvent{Sequence: sequence, DashboardGeneration: dashboardGeneration, GateGeneration: gateGeneration, ServerID: serverID, TaskID: taskID, TaskType: taskType, Kind: kind}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/testpaths"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDashboardRestart_PreservesFixtureIdentityAndCleansGenerations(t *testing.T) {
|
||||
// Given
|
||||
sourceDir, err := testpaths.NezhaSource(t.Name())
|
||||
require.NoError(t, err)
|
||||
dashboard, err := Start(t.Context(), StartConfig{SourceDir: sourceDir, EnableTLS: true, ReceiptGate: true})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
cleanupContext, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
require.NoError(t, dashboard.Close(cleanupContext))
|
||||
})
|
||||
fixture := dashboard.FixtureIdentity()
|
||||
firstRuntime := dashboard.RuntimeIdentity()
|
||||
require.NotZero(t, fixture.HTTP.Inode)
|
||||
require.NotZero(t, fixture.HTTPS.Inode)
|
||||
|
||||
// When
|
||||
stopContext, cancel := context.WithTimeout(t.Context(), 15*time.Second)
|
||||
defer cancel()
|
||||
_, err = dashboard.StopProcess(stopContext)
|
||||
require.NoError(t, err)
|
||||
firstPID := firstRuntime.PID
|
||||
_, err = dashboard.StartProcess(stopContext)
|
||||
require.NoError(t, err)
|
||||
secondRuntime := dashboard.RuntimeIdentity()
|
||||
|
||||
// Then
|
||||
require.Equal(t, fixture, dashboard.FixtureIdentity())
|
||||
require.NotEqual(t, firstRuntime.PID, secondRuntime.PID)
|
||||
require.Greater(t, secondRuntime.Generation, firstRuntime.Generation)
|
||||
require.Equal(t, fixture.HTTP.Address, dashboard.Endpoint())
|
||||
require.Equal(t, fixture.HTTP, dashboard.FixtureIdentity().HTTP)
|
||||
require.Equal(t, fixture.HTTPS, dashboard.FixtureIdentity().HTTPS)
|
||||
require.FileExists(t, fixture.DatabasePath)
|
||||
require.FileExists(t, fixture.ConfigPath)
|
||||
require.NoError(t, dashboard.Close(stopContext))
|
||||
require.Len(t, dashboard.CleanupReceipt().Processes, 2)
|
||||
require.NoFileExists(t, filepath.Join("/proc", strconv.Itoa(firstPID)))
|
||||
require.NoDirExists(t, fixture.WorkspaceRoot)
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/client"
|
||||
processharness "github.com/nezhahq/nezha/integration/agentcompat/internal/process"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/workspace"
|
||||
)
|
||||
|
||||
func (dashboard *Dashboard) prepare(ctx context.Context, config StartConfig) error {
|
||||
if err := dashboard.prepareFixture(ctx, config); err != nil {
|
||||
return err
|
||||
}
|
||||
process, err := dashboard.startGeneration(ctx, 1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dashboard.generation = 1
|
||||
dashboard.currentProcess, dashboard.supervisor = process, process.supervisor
|
||||
dashboard.processes = append(dashboard.processes, process)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) startGeneration(ctx context.Context, generation uint64) (*dashboardGeneration, error) {
|
||||
files := make([]*os.File, 0, 3)
|
||||
for _, listener := range []*workspace.OwnedListener{dashboard.httpListener, dashboard.receiptListener, dashboard.httpsListener} {
|
||||
if listener == nil {
|
||||
continue
|
||||
}
|
||||
file, err := listener.ExtraFile()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files = append(files, file)
|
||||
}
|
||||
logFile, err := dashboard.workspace.Log(fmt.Sprintf("dashboard-generation-%d", generation))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dashboard.logPath = logFile.Name()
|
||||
supervisor := processharness.NewSupervisor(context.WithoutCancel(ctx), processharness.Spec{
|
||||
Name: "dashboard", Path: dashboard.binaryPath, Args: []string{"-c", dashboard.configPath, "-db", dashboard.databasePath},
|
||||
Env: dashboardEnvironment(dashboard.startConfig.EnableTLS, dashboard.startConfig.ReceiptGate), ExtraFiles: files,
|
||||
Stdout: logFile, Stderr: logFile, MaxLogBytes: dashboardMaxLogBytes,
|
||||
TerminateTimeout: defaultProcessStopTimeout, KillTimeout: defaultProcessKillTimeout,
|
||||
})
|
||||
if err := supervisor.Start(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
identity := RuntimeIdentity{Generation: generation, PID: supervisor.PID(), ProcessGroupID: supervisor.ProcessGroupID()}
|
||||
process := &dashboardGeneration{supervisor: supervisor, identity: identity}
|
||||
rollback := true
|
||||
defer func() {
|
||||
if rollback {
|
||||
_ = process.supervisor.Stop(context.WithoutCancel(ctx))
|
||||
if process.receiptConn != nil {
|
||||
_ = process.receiptConn.Close()
|
||||
}
|
||||
}
|
||||
}()
|
||||
if err := dashboard.workspace.TrackPID(identity.PID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := dashboard.workspace.TrackProcessGroup(identity.ProcessGroupID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dashboard.supervisor = supervisor
|
||||
dashboard.stateMu.Lock()
|
||||
dashboard.eventGeneration = generation
|
||||
dashboard.stateMu.Unlock()
|
||||
if dashboard.startConfig.ReceiptGate {
|
||||
connection, err := (&net.Dialer{}).DialContext(ctx, "tcp", dashboard.receiptAddress)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connect dashboard receipt gate: %w", err)
|
||||
}
|
||||
process.receiptConn = connection
|
||||
reader := bufio.NewReader(connection)
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("wait for dashboard receipt gate: %w", err)
|
||||
}
|
||||
if line != "ready\n" {
|
||||
return nil, fmt.Errorf("unexpected dashboard receipt gate handshake %q", line)
|
||||
}
|
||||
dashboard.eventMu.Lock()
|
||||
dashboard.receiptConn = connection
|
||||
dashboard.receiptReader = reader
|
||||
dashboard.receiptEvents = make(chan string, 16)
|
||||
dashboard.eventNotify = make(chan struct{})
|
||||
dashboard.eventClosed = false
|
||||
dashboard.eventMu.Unlock()
|
||||
dashboard.info2Mu.Lock()
|
||||
dashboard.info2Events = make(map[string]struct{})
|
||||
dashboard.info2Mu.Unlock()
|
||||
dashboard.stateMu.Lock()
|
||||
dashboard.stateEvents = make(map[stateEventIdentity]struct{})
|
||||
dashboard.stateMu.Unlock()
|
||||
go dashboard.readReceiptEvents(generation, reader)
|
||||
}
|
||||
if err := dashboard.refreshClients(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
process.httpTransport = dashboard.httpTransport
|
||||
process.tlsTransport = dashboard.tlsTransport
|
||||
if dashboard.startConfig.EnableTLS {
|
||||
if err := dashboard.verifyTrustedTLS(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
rollback = false
|
||||
return process, nil
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) adoptLoopbackListener() (*workspace.OwnedListener, error) {
|
||||
listener, err := net.Listen("tcp4", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listen for dashboard: %w", err)
|
||||
}
|
||||
owned, err := dashboard.workspace.AdoptListener(listener)
|
||||
if err != nil {
|
||||
_ = listener.Close()
|
||||
return nil, fmt.Errorf("adopt dashboard listener: %w", err)
|
||||
}
|
||||
return owned, nil
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) refreshClients(ctx context.Context) error {
|
||||
jar, err := cookiejar.New(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dashboard.httpTransport = &http.Transport{DialContext: dialAddress(dashboard.httpAddress)}
|
||||
dashboard.restHTTPClient = &http.Client{Transport: dashboard.httpTransport, Jar: jar, Timeout: dashboardHTTPClientTimeout}
|
||||
dashboard.clients.REST, err = client.New(client.Config{BaseURL: dashboard.URL(), HTTPClient: dashboard.restHTTPClient})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pat, err := dashboard.bootstrapAuthentication(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return dashboard.initializeAuthenticatedClients(ctx, pat)
|
||||
}
|
||||
|
||||
func dashboardEnvironment(enableTLS bool, receiptGateOption ...bool) []string {
|
||||
receiptGate := len(receiptGateOption) > 0 && receiptGateOption[0]
|
||||
environment := make([]string, 0, len(os.Environ())+3)
|
||||
for _, variable := range os.Environ() {
|
||||
if strings.HasPrefix(variable, "NZ_") || strings.HasPrefix(variable, "NEZHA_AGENTCOMPAT_") {
|
||||
continue
|
||||
}
|
||||
environment = append(environment, variable)
|
||||
}
|
||||
environment = append(environment,
|
||||
"NZ_JWTSECRETKEY="+jwtSecret,
|
||||
"NEZHA_AGENTCOMPAT_HTTP_LISTENER_FD=3",
|
||||
)
|
||||
if receiptGate {
|
||||
environment = append(environment, "NEZHA_AGENTCOMPAT_RECEIPT_LISTENER_FD=4")
|
||||
}
|
||||
if enableTLS {
|
||||
fd := 4
|
||||
if receiptGate {
|
||||
fd = 5
|
||||
}
|
||||
environment = append(environment, fmt.Sprintf("NEZHA_AGENTCOMPAT_HTTPS_LISTENER_FD=%d", fd))
|
||||
}
|
||||
return environment
|
||||
}
|
||||
|
||||
func dialAddress(address string) func(context.Context, string, string) (net.Conn, error) {
|
||||
return func(ctx context.Context, network, _ string) (net.Conn, error) {
|
||||
var dialer net.Dialer
|
||||
return dialer.DialContext(ctx, network, address)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/client"
|
||||
)
|
||||
|
||||
func (dashboard *Dashboard) verifyTrustedTLS(ctx context.Context) error {
|
||||
httpClient, transport, err := dashboard.newTLSHTTPClient("localhost")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dashboard.tlsTransport = transport
|
||||
tlsClient, err := client.New(client.Config{BaseURL: dashboard.TLSURL(), HTTPClient: httpClient})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
login, err := tlsClient.Login(ctx, client.LoginRequest{Username: "admin", Password: "admin"})
|
||||
if err != nil {
|
||||
return fmt.Errorf("login through trusted dashboard TLS: %w", err)
|
||||
}
|
||||
if login.Token == "" {
|
||||
return errors.New("trusted dashboard TLS login omitted JWT")
|
||||
}
|
||||
dashboard.bootstrap.TLSAuthenticated = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) newTLSHTTPClient(serverName string) (*http.Client, *http.Transport, error) {
|
||||
jar, err := cookiejar.New(nil)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create TLS cookie jar: %w", err)
|
||||
}
|
||||
transport := &http.Transport{
|
||||
TLSClientConfig: dashboard.tlsFixture.ClientConfig(serverName),
|
||||
DialContext: dialAddress(dashboard.httpsAddress),
|
||||
}
|
||||
return &http.Client{Transport: transport, Jar: jar, Timeout: dashboardHTTPClientTimeout}, transport, nil
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newWaiterDashboard() *Dashboard {
|
||||
return &Dashboard{
|
||||
receiptEvents: make(chan string),
|
||||
eventNotify: make(chan struct{}),
|
||||
info2Events: make(map[string]struct{}),
|
||||
stateEvents: make(map[stateEventIdentity]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) publishTestEvent() {
|
||||
dashboard.eventMu.Lock()
|
||||
close(dashboard.eventNotify)
|
||||
dashboard.eventNotify = make(chan struct{})
|
||||
dashboard.eventMu.Unlock()
|
||||
}
|
||||
|
||||
func TestDashboardWaiters_AllObserveCachedEvents(t *testing.T) {
|
||||
// Given
|
||||
dashboard := newWaiterDashboard()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
|
||||
defer cancel()
|
||||
waiters := []func(context.Context) error{
|
||||
func(ctx context.Context) error { return dashboard.WaitForInfo2(ctx, 7, "uuid") },
|
||||
func(ctx context.Context) error { return dashboard.WaitForInfo2(ctx, 7, "uuid") },
|
||||
func(ctx context.Context) error { return dashboard.WaitForState(ctx, 2) },
|
||||
func(ctx context.Context) error { return dashboard.WaitForState(ctx, 2) },
|
||||
func(ctx context.Context) error { return dashboard.WaitForReceiptAccepted(ctx) },
|
||||
func(ctx context.Context) error { return dashboard.WaitForReceiptAccepted(ctx) },
|
||||
}
|
||||
results := make(chan error, len(waiters))
|
||||
var group sync.WaitGroup
|
||||
group.Add(len(waiters))
|
||||
for _, wait := range waiters {
|
||||
go func(wait func(context.Context) error) {
|
||||
defer group.Done()
|
||||
results <- wait(ctx)
|
||||
}(wait)
|
||||
}
|
||||
// When
|
||||
dashboard.info2Mu.Lock()
|
||||
dashboard.info2Events["info2 7 uuid\n"] = struct{}{}
|
||||
dashboard.info2Mu.Unlock()
|
||||
dashboard.stateMu.Lock()
|
||||
dashboard.stateEvents[stateEventIdentity{ServerID: 7, UUID: "uuid", Generation: 1, Count: 2}] = struct{}{}
|
||||
dashboard.stateMu.Unlock()
|
||||
dashboard.receiptMu.Lock()
|
||||
dashboard.receiptAcceptedCount = 1
|
||||
dashboard.receiptMu.Unlock()
|
||||
dashboard.publishTestEvent()
|
||||
group.Wait()
|
||||
|
||||
// Then
|
||||
close(results)
|
||||
for err := range results {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, dashboard.WaitForInfo2(ctx, 7, "uuid"))
|
||||
require.NoError(t, dashboard.WaitForState(ctx, 2))
|
||||
require.NoError(t, dashboard.WaitForReceiptAccepted(ctx))
|
||||
}
|
||||
|
||||
func TestDashboardWaiters_CloseWakesAllWithTypedError(t *testing.T) {
|
||||
// Given
|
||||
dashboard := newWaiterDashboard()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
|
||||
defer cancel()
|
||||
results := make(chan error, 4)
|
||||
go func() { results <- dashboard.WaitForInfo2(ctx, 7, "uuid") }()
|
||||
go func() { results <- dashboard.WaitForState(ctx, 2) }()
|
||||
go func() { results <- dashboard.WaitForReceiptAccepted(ctx) }()
|
||||
go func() { results <- dashboard.WaitForSecondState(ctx) }()
|
||||
|
||||
// When
|
||||
dashboard.eventMu.Lock()
|
||||
dashboard.eventClosed = true
|
||||
close(dashboard.eventNotify)
|
||||
dashboard.eventNotify = make(chan struct{})
|
||||
dashboard.eventMu.Unlock()
|
||||
|
||||
// Then
|
||||
for index := 0; index < 4; index++ {
|
||||
require.ErrorIs(t, <-results, ErrReceiptGateClosed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDashboardWaitForStateGenerationDoesNotCrossMatchServers(t *testing.T) {
|
||||
dashboard := newWaiterDashboard()
|
||||
dashboard.stateMu.Lock()
|
||||
dashboard.stateEvents[stateEventIdentity{ServerID: 7, UUID: "server-seven", Generation: 1, Count: 1}] = struct{}{}
|
||||
dashboard.stateMu.Unlock()
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
err := dashboard.WaitForStateGeneration(ctx, 8, "server-eight", 1, 1)
|
||||
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
}
|
||||
|
||||
func TestDashboardAcceptedEventTracksReceiptAndStateGenerationsSeparately(t *testing.T) {
|
||||
dashboard := newWaiterDashboard()
|
||||
dashboard.receiptEvents = make(chan string)
|
||||
dashboard.processReceiptLine("accepted 7 server-seven 4 9 1\n")
|
||||
|
||||
require.Equal(t, uint64(4), dashboard.ReceiptGeneration())
|
||||
require.NoError(t, dashboard.WaitForStateGeneration(t.Context(), 7, "server-seven", 9, 1))
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
//go:build linux
|
||||
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (dashboard *Dashboard) WaitForReceiptAccepted(ctx context.Context) error {
|
||||
if dashboard.receiptEvents == nil {
|
||||
return errors.New("receipt gate is disabled")
|
||||
}
|
||||
for {
|
||||
dashboard.eventMu.RLock()
|
||||
closed, notify := dashboard.eventClosed, dashboard.eventNotify
|
||||
dashboard.eventMu.RUnlock()
|
||||
dashboard.receiptMu.RLock()
|
||||
observed := dashboard.receiptAcceptedCount > 0
|
||||
dashboard.receiptMu.RUnlock()
|
||||
if closed {
|
||||
return ErrReceiptGateClosed
|
||||
}
|
||||
if observed {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-notify:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) WaitForSecondState(ctx context.Context) error {
|
||||
if dashboard.receiptEvents == nil {
|
||||
return errors.New("receipt gate is disabled")
|
||||
}
|
||||
for {
|
||||
dashboard.eventMu.RLock()
|
||||
closed, notify := dashboard.eventClosed, dashboard.eventNotify
|
||||
dashboard.eventMu.RUnlock()
|
||||
dashboard.receiptMu.RLock()
|
||||
observed := dashboard.receiptAcceptedCount >= 2
|
||||
dashboard.receiptMu.RUnlock()
|
||||
if closed {
|
||||
return ErrReceiptGateClosed
|
||||
}
|
||||
if observed {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-notify:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) WaitForState(ctx context.Context, want uint64) error {
|
||||
return dashboard.waitForState(ctx, 0, "", 0, want)
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) WaitForStateGeneration(ctx context.Context, serverID uint64, uuid string, generation, want uint64) error {
|
||||
return dashboard.waitForState(ctx, serverID, uuid, generation, want)
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) StateGeneration(serverID uint64, uuid string) uint64 {
|
||||
dashboard.stateMu.Lock()
|
||||
defer dashboard.stateMu.Unlock()
|
||||
var generation uint64
|
||||
for event := range dashboard.stateEvents {
|
||||
if event.ServerID == serverID && event.UUID == uuid && event.Generation > generation {
|
||||
generation = event.Generation
|
||||
}
|
||||
}
|
||||
return generation
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) waitForState(ctx context.Context, serverID uint64, uuid string, generation, want uint64) error {
|
||||
if dashboard.receiptEvents == nil {
|
||||
return errors.New("receipt gate is disabled")
|
||||
}
|
||||
for {
|
||||
dashboard.eventMu.RLock()
|
||||
closed, notify := dashboard.eventClosed, dashboard.eventNotify
|
||||
dashboard.eventMu.RUnlock()
|
||||
dashboard.stateMu.Lock()
|
||||
observed := false
|
||||
for event := range dashboard.stateEvents {
|
||||
if (serverID == 0 || event.ServerID == serverID) && (uuid == "" || event.UUID == uuid) && (generation == 0 || event.Generation == generation) && event.Count == want {
|
||||
observed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
dashboard.stateMu.Unlock()
|
||||
if closed {
|
||||
return ErrReceiptGateClosed
|
||||
}
|
||||
if observed {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-notify:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) WaitForInfo2(ctx context.Context, serverID uint64, uuid string) error {
|
||||
if dashboard.receiptEvents == nil {
|
||||
return errors.New("receipt gate is disabled")
|
||||
}
|
||||
want := fmt.Sprintf("info2 %d %s\n", serverID, uuid)
|
||||
for {
|
||||
dashboard.eventMu.RLock()
|
||||
closed, notify := dashboard.eventClosed, dashboard.eventNotify
|
||||
dashboard.eventMu.RUnlock()
|
||||
dashboard.info2Mu.Lock()
|
||||
_, observed := dashboard.info2Events[want]
|
||||
dashboard.info2Mu.Unlock()
|
||||
if closed {
|
||||
return ErrReceiptGateClosed
|
||||
}
|
||||
if observed {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-notify:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) WaitForInfo2UUID(ctx context.Context, uuid string) (uint64, error) {
|
||||
if dashboard.receiptEvents == nil {
|
||||
return 0, errors.New("receipt gate is disabled")
|
||||
}
|
||||
for {
|
||||
dashboard.eventMu.RLock()
|
||||
closed, notify := dashboard.eventClosed, dashboard.eventNotify
|
||||
dashboard.eventMu.RUnlock()
|
||||
dashboard.info2Mu.Lock()
|
||||
for event := range dashboard.info2Events {
|
||||
fields := strings.Fields(event)
|
||||
if len(fields) == 3 && fields[0] == "info2" && fields[2] == uuid {
|
||||
var serverID uint64
|
||||
if _, err := fmt.Sscan(fields[1], &serverID); err == nil && serverID != 0 {
|
||||
dashboard.info2Mu.Unlock()
|
||||
return serverID, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
dashboard.info2Mu.Unlock()
|
||||
if closed {
|
||||
return 0, ErrReceiptGateClosed
|
||||
}
|
||||
select {
|
||||
case <-notify:
|
||||
case <-ctx.Done():
|
||||
return 0, ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (dashboard *Dashboard) ReceiptAccepted() bool {
|
||||
dashboard.receiptMu.RLock()
|
||||
defer dashboard.receiptMu.RUnlock()
|
||||
return dashboard.receiptAccepted
|
||||
}
|
||||
func (dashboard *Dashboard) ReceiptAcceptedCount() uint64 {
|
||||
dashboard.receiptMu.RLock()
|
||||
defer dashboard.receiptMu.RUnlock()
|
||||
return dashboard.receiptAcceptedCount
|
||||
}
|
||||
func (dashboard *Dashboard) ReceiptGeneration() uint64 {
|
||||
dashboard.receiptMu.RLock()
|
||||
defer dashboard.receiptMu.RUnlock()
|
||||
return dashboard.receiptGeneration
|
||||
}
|
||||
Reference in New Issue
Block a user