mirror of
https://github.com/Buriburizaem0n/nezha_domains.git
synced 2026-09-19 09:40:12 +00:00
test(agentcompat): add scenario CLI wiring
Co-authored-by: naiba/CloudCode <hi+cloudcode@nai.ba>
This commit is contained in:
naiba
naiba/CloudCode
@@ -0,0 +1,163 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/evidence"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/scenario"
|
||||
)
|
||||
|
||||
func TestCLI_ArtifactPublicationReplacesPublicFileAndFinalSymlink(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(*testing.T, string, string)
|
||||
}{
|
||||
{"public file", func(t *testing.T, path, _ string) {
|
||||
if err := os.WriteFile(path, []byte("old"), 0o644); err != nil {
|
||||
t.Fatalf("write old file: %v", err)
|
||||
}
|
||||
}},
|
||||
{"final symlink", func(t *testing.T, path, sentinel string) {
|
||||
if err := os.Symlink(sentinel, path); err != nil {
|
||||
t.Fatalf("symlink final path: %v", err)
|
||||
}
|
||||
}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "cleanup.json")
|
||||
sentinel := filepath.Join(t.TempDir(), "sentinel")
|
||||
if err := os.WriteFile(sentinel, []byte("unchanged"), 0o600); err != nil {
|
||||
t.Fatalf("write sentinel: %v", err)
|
||||
}
|
||||
test.setup(t, path, sentinel)
|
||||
if err := writeJSONArtifact(dir, "cleanup.json", map[string]bool{"passed": true}); err != nil {
|
||||
t.Fatalf("publish artifact: %v", err)
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("lstat artifact: %v", err)
|
||||
}
|
||||
if !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("artifact mode=%v", info.Mode())
|
||||
}
|
||||
content, err := os.ReadFile(sentinel)
|
||||
if err != nil || string(content) != "unchanged" {
|
||||
t.Fatalf("outside sentinel changed: content=%q err=%v", content, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_InterruptedScenarioPublicationLeavesAtomicInvalidDirectory(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
config := testCLIConfig(t, contract.ScenarioTransfer100MiB, contract.FaultTransferHash)
|
||||
paths, err := contract.NewPaths(config.Paths.NezhaSource().String(), config.Paths.AgentSource().String(), dir)
|
||||
if err != nil {
|
||||
t.Fatalf("paths: %v", err)
|
||||
}
|
||||
config.Paths = paths
|
||||
if err := writeMetadata(t.Context(), config, time.Now()); err != nil {
|
||||
t.Fatalf("metadata: %v", err)
|
||||
}
|
||||
previous := scenarioArtifactPublished
|
||||
scenarioArtifactPublished = func(name string) error {
|
||||
if name == "results.json" {
|
||||
return errors.New("injected publication interruption")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
t.Cleanup(func() { scenarioArtifactPublished = previous })
|
||||
output := scenarioExecutionOutput{Result: scenario.Result{Name: contract.ScenarioTransfer100MiB, Passed: false, CleanupOK: true, Error: "transfer scenario: injected hash mismatch"}, Transfer: &scenario.TransferEvidence{WarmupUploadBytes: 65536, WarmupDownloadBytes: 65536, WarmupSHA256: "abc", WarmupDuration: time.Nanosecond, WarmupDeadlineRemaining: time.Second, WarmupQuiescent: true, OutsideRootSentinelsUnchanged: true}}
|
||||
if err := writeScenarioEvidence(config, output, time.Now()); err == nil {
|
||||
t.Fatal("publication interruption accepted")
|
||||
}
|
||||
if err := evidence.ValidateDirectory(dir); err == nil {
|
||||
t.Fatal("partial evidence directory validated")
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(dir, "results.json"))
|
||||
if !errors.Is(err, os.ErrNotExist) || len(data) != 0 {
|
||||
t.Fatalf("interrupted final file published: bytes=%d err=%v", len(data), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_PrivateArtifactJoinsPrimaryAndCleanupErrors(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "results.json")
|
||||
primaryErr := errors.New("publication hook failed")
|
||||
removeErr := errors.New("temporary removal failed")
|
||||
previousClose := privateArtifactClose
|
||||
previousRemove := privateArtifactRemove
|
||||
closeCalls := 0
|
||||
privateArtifactClose = func(file *os.File) error {
|
||||
closeCalls++
|
||||
return file.Close()
|
||||
}
|
||||
privateArtifactRemove = func(string) error { return removeErr }
|
||||
t.Cleanup(func() {
|
||||
privateArtifactClose = previousClose
|
||||
privateArtifactRemove = previousRemove
|
||||
})
|
||||
|
||||
err := writePrivateArtifactWithSeam(path, []byte("payload"), func() error { return primaryErr })
|
||||
if !errors.Is(err, primaryErr) || !errors.Is(err, removeErr) {
|
||||
t.Fatalf("publication error=%v, want primary and removal errors", err)
|
||||
}
|
||||
if errors.Is(err, os.ErrClosed) || closeCalls != 1 {
|
||||
t.Fatalf("successful close repeated: calls=%d err=%v", closeCalls, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_PrivateArtifactJoinsCloseAndRemoveErrors(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "results.json")
|
||||
closeErr := errors.New("temporary close failed")
|
||||
removeErr := errors.New("temporary removal failed")
|
||||
previousClose := privateArtifactClose
|
||||
previousRemove := privateArtifactRemove
|
||||
closeCalls := 0
|
||||
privateArtifactClose = func(file *os.File) error {
|
||||
closeCalls++
|
||||
if err := file.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return closeErr
|
||||
}
|
||||
privateArtifactRemove = func(string) error { return removeErr }
|
||||
t.Cleanup(func() {
|
||||
privateArtifactClose = previousClose
|
||||
privateArtifactRemove = previousRemove
|
||||
})
|
||||
|
||||
err := writePrivateArtifactWithSeam(path, []byte("payload"), func() error { return nil })
|
||||
if !errors.Is(err, closeErr) || !errors.Is(err, removeErr) {
|
||||
t.Fatalf("publication error=%v, want close and removal errors", err)
|
||||
}
|
||||
if errors.Is(err, os.ErrClosed) || closeCalls != 1 {
|
||||
t.Fatalf("close failure retried: calls=%d err=%v", closeCalls, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_PrivateArtifactRemovesTemporaryFileAfterHookFailure(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "results.json")
|
||||
err := writePrivateArtifactWithSeam(path, []byte("payload"), func() error {
|
||||
return errors.New("publication hook failed")
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("publication hook failure accepted")
|
||||
}
|
||||
entries, readErr := os.ReadDir(dir)
|
||||
if readErr != nil {
|
||||
t.Fatalf("read artifact directory: %v", readErr)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("temporary artifact survived hook failure: %v", entries)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/evidence"
|
||||
)
|
||||
|
||||
type scenarioFlags []contract.Scenario
|
||||
|
||||
func (scenarios *scenarioFlags) String() string {
|
||||
values := make([]string, 0, len(*scenarios))
|
||||
for _, scenario := range *scenarios {
|
||||
values = append(values, scenario.String())
|
||||
}
|
||||
return strings.Join(values, ",")
|
||||
}
|
||||
|
||||
func (scenarios *scenarioFlags) Set(value string) error {
|
||||
scenario, err := contract.NewScenario(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*scenarios = append(*scenarios, scenario)
|
||||
return nil
|
||||
}
|
||||
|
||||
type cliConfig struct {
|
||||
Paths contract.Paths
|
||||
Profile contract.Profile
|
||||
Seed contract.Seed
|
||||
Scenarios scenarioFlags
|
||||
Fault contract.Fault
|
||||
}
|
||||
|
||||
func parseFlags(args []string, stderr io.Writer) (cliConfig, error) {
|
||||
flags := flag.NewFlagSet("agentcompat", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
nezhaSource := flags.String("nezha-source", "", "Nezha source directory")
|
||||
agentSource := flags.String("agent-source", "", "Agent source directory")
|
||||
profileName := flags.String("profile", "", "compatibility profile")
|
||||
resultsDir := flags.String("results-dir", "", "evidence output directory")
|
||||
seedValue := flags.String("seed", "0x4e5a4841", "deterministic seed")
|
||||
faultName := flags.String("fault", "", "named fault injection")
|
||||
var scenarios scenarioFlags
|
||||
flags.Var(&scenarios, "scenario", "run only a named scenario; omit for the complete profile")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return cliConfig{}, errors.New("invalid command-line arguments")
|
||||
}
|
||||
paths, err := contract.NewPaths(*nezhaSource, *agentSource, *resultsDir)
|
||||
if err != nil {
|
||||
return cliConfig{}, err
|
||||
}
|
||||
if err := prepareResultsDirBeforeParse(paths.ResultsDir().String()); err != nil {
|
||||
return cliConfig{}, err
|
||||
}
|
||||
profile, err := contract.ProfileByName(*profileName)
|
||||
if err != nil {
|
||||
return cliConfig{}, err
|
||||
}
|
||||
seed, err := contract.ParseSeed(*seedValue)
|
||||
if err != nil {
|
||||
return cliConfig{}, err
|
||||
}
|
||||
fault := contract.Fault{}
|
||||
if *faultName != "" {
|
||||
fault, err = contract.NewFault(*faultName)
|
||||
if err != nil {
|
||||
return cliConfig{}, err
|
||||
}
|
||||
}
|
||||
return cliConfig{Paths: paths, Profile: profile, Seed: seed, Scenarios: scenarios, Fault: fault}, nil
|
||||
}
|
||||
|
||||
func prepareResultsDir(resultsDir string) error {
|
||||
return prepareEvidenceArtifacts(resultsDir, evidence.FixedEvidenceFiles())
|
||||
}
|
||||
|
||||
func prepareResultsDirBeforeParse(resultsDir string) error {
|
||||
return prepareEvidenceArtifacts(resultsDir, evidence.FixedEvidenceFiles()[1:])
|
||||
}
|
||||
|
||||
func prepareEvidenceArtifacts(resultsDir string, artifactNames []string) error {
|
||||
if info, err := os.Lstat(resultsDir); err == nil && info.Mode()&os.ModeSymlink != 0 {
|
||||
return errors.New("results directory must not be a symbolic link")
|
||||
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("inspect results directory: %w", err)
|
||||
}
|
||||
for _, name := range artifactNames {
|
||||
if err := os.Remove(filepath.Join(resultsDir, name)); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("remove previous artifact %s: %w", name, err)
|
||||
}
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(resultsDir, "agents")); err != nil {
|
||||
return fmt.Errorf("remove previous agent logs: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/scenario"
|
||||
)
|
||||
|
||||
func TestCLI_SelectsTransferAndReconnectWithTypedEvidence(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fault string
|
||||
}{
|
||||
{contract.ScenarioTransfer100MiB, ""},
|
||||
{contract.ScenarioTransfer100MiB, contract.FaultTransferHash},
|
||||
{contract.ScenarioReconnect, ""},
|
||||
{contract.ScenarioReconnect, contract.FaultDashboardExit},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name+"/"+test.fault, func(t *testing.T) {
|
||||
config := testCLIConfig(t, test.name, test.fault)
|
||||
runnerErr := errors.New("dedicated runner sentinel")
|
||||
wantResult := scenario.Result{Name: test.name, Passed: false, CleanupOK: true, Error: runnerErr.Error(), Assertions: []scenario.Assertion{{Name: "typed dispatch", Passed: false, Details: test.fault}}}
|
||||
wantTransfer := scenario.TransferEvidence{WarmupUploadBytes: 65536, WarmupDownloadBytes: 65536, WarmupSHA256: "warmup", WarmupDuration: time.Second, WarmupDeadlineRemaining: 2 * time.Second, WarmupQuiescent: true, UploadBytes: contract.TransferBytes, DownloadBytes: contract.TransferBytes, UploadSHA256: "transfer-hash", DownloadSHA256: "transfer-hash", UploadChunks: 3, DownloadChunks: 4, UploadDuration: 5 * time.Second, DownloadDuration: 6 * time.Second, RetainedHeapBytes: 7, Mode: "0640", CreateDirs: true, UploadReplayRejected: true, DownloadReplayRejected: true, OversizeRejected: true, OutsideRootSentinelsUnchanged: true}
|
||||
wantReconnect := completeReconnectDispatchEvidence(t)
|
||||
previousTransfer := runTransferScenario
|
||||
previousReconnect := runReconnectScenario
|
||||
var receivedTransfer *scenario.TransferInput
|
||||
var receivedReconnect *scenario.ReconnectInput
|
||||
runTransferScenario = func(_ context.Context, input scenario.TransferInput) (scenario.Result, scenario.TransferEvidence, error) {
|
||||
receivedTransfer = &input
|
||||
return wantResult, wantTransfer, runnerErr
|
||||
}
|
||||
runReconnectScenario = func(_ context.Context, input scenario.ReconnectInput) (scenario.Result, scenario.ReconnectEvidence, error) {
|
||||
receivedReconnect = &input
|
||||
return wantResult, wantReconnect, runnerErr
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
runTransferScenario = previousTransfer
|
||||
runReconnectScenario = previousReconnect
|
||||
})
|
||||
|
||||
execution, err := selectScenarioExecution(config)
|
||||
if err != nil {
|
||||
t.Fatalf("select execution: %v", err)
|
||||
}
|
||||
output, runErr := execution.run(context.Background())
|
||||
if !errors.Is(runErr, runnerErr) {
|
||||
t.Fatalf("runner error=%v, want sentinel", runErr)
|
||||
}
|
||||
if err := output.Validate(); err != nil {
|
||||
t.Fatalf("validate typed output: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(output.Result, wantResult) {
|
||||
t.Fatalf("result=%#v, want %#v", output.Result, wantResult)
|
||||
}
|
||||
if test.name == contract.ScenarioTransfer100MiB {
|
||||
wantInput := scenario.TransferInput{Paths: config.Paths, Fault: config.Fault}
|
||||
if receivedTransfer == nil || *receivedTransfer != wantInput || receivedReconnect != nil {
|
||||
t.Fatalf("transfer inputs: received=%#v reconnect=%#v want=%#v", receivedTransfer, receivedReconnect, wantInput)
|
||||
}
|
||||
if output.Transfer == nil || !reflect.DeepEqual(*output.Transfer, wantTransfer) || output.Reconnect != nil {
|
||||
t.Fatalf("transfer evidence=%#v reconnect=%#v", output.Transfer, output.Reconnect)
|
||||
}
|
||||
} else {
|
||||
wantInput := scenario.ReconnectInput{Paths: config.Paths, DashboardFault: config.Fault.String()}
|
||||
if receivedReconnect == nil || *receivedReconnect != wantInput || receivedTransfer != nil {
|
||||
t.Fatalf("reconnect inputs: received=%#v transfer=%#v want=%#v", receivedReconnect, receivedTransfer, wantInput)
|
||||
}
|
||||
if output.Reconnect == nil || !reflect.DeepEqual(*output.Reconnect, wantReconnect) || output.Transfer != nil {
|
||||
t.Fatalf("reconnect evidence=%#v transfer=%#v", output.Reconnect, output.Transfer)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_RejectsUnsupportedScenarioFaultPairsBeforeRunner(t *testing.T) {
|
||||
tests := []struct{ scenario, fault string }{
|
||||
{contract.ScenarioTransfer100MiB, contract.FaultDashboardExit},
|
||||
{contract.ScenarioReconnect, contract.FaultTransferHash},
|
||||
{contract.ScenarioMCPFilesystem, contract.FaultAgentBadSecret},
|
||||
}
|
||||
for _, test := range tests {
|
||||
called := false
|
||||
previousTransfer := runTransferScenario
|
||||
runTransferScenario = func(context.Context, scenario.TransferInput) (scenario.Result, scenario.TransferEvidence, error) {
|
||||
called = true
|
||||
return scenario.Result{}, scenario.TransferEvidence{}, errors.New("unexpected")
|
||||
}
|
||||
_, err := selectScenarioExecution(testCLIConfig(t, test.scenario, test.fault))
|
||||
runTransferScenario = previousTransfer
|
||||
if err == nil || called {
|
||||
t.Fatalf("unsupported pair started runner: scenario=%q fault=%q err=%v", test.scenario, test.fault, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_WritesDedicatedArtifactWithPrivateMode(t *testing.T) {
|
||||
resultsDir := t.TempDir()
|
||||
output := scenarioExecutionOutput{
|
||||
Result: scenario.Result{Name: contract.ScenarioTransfer100MiB, Passed: false, CleanupOK: true, Error: "transfer scenario: injected hash mismatch"},
|
||||
Transfer: &scenario.TransferEvidence{WarmupUploadBytes: 65536, WarmupDownloadBytes: 65536, WarmupSHA256: "abc", WarmupDuration: time.Nanosecond, WarmupDeadlineRemaining: time.Second, WarmupQuiescent: true, OutsideRootSentinelsUnchanged: true},
|
||||
}
|
||||
config := testCLIConfig(t, contract.ScenarioTransfer100MiB, contract.FaultTransferHash)
|
||||
paths, err := contract.NewPaths(config.Paths.NezhaSource().String(), config.Paths.AgentSource().String(), resultsDir)
|
||||
if err != nil {
|
||||
t.Fatalf("paths: %v", err)
|
||||
}
|
||||
config.Paths = paths
|
||||
if err := writeScenarioEvidence(config, output, time.Now()); err != nil {
|
||||
t.Fatalf("write evidence: %v", err)
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(resultsDir, "transfer.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("stat transfer evidence: %v", err)
|
||||
}
|
||||
if info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("transfer evidence mode=%o", info.Mode().Perm())
|
||||
}
|
||||
}
|
||||
|
||||
func testCLIConfig(t *testing.T, scenarioName, faultName string) cliConfig {
|
||||
t.Helper()
|
||||
paths, err := contract.NewPaths("/src/nezha", "/src/agent", t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("paths: %v", err)
|
||||
}
|
||||
profile, err := contract.ProfileByName("pr-full")
|
||||
if err != nil {
|
||||
t.Fatalf("profile: %v", err)
|
||||
}
|
||||
scenarioValue, err := contract.NewScenario(scenarioName)
|
||||
if err != nil {
|
||||
t.Fatalf("scenario: %v", err)
|
||||
}
|
||||
fault := contract.Fault{}
|
||||
if faultName != "" {
|
||||
fault, err = contract.NewFault(faultName)
|
||||
if err != nil {
|
||||
t.Fatalf("fault: %v", err)
|
||||
}
|
||||
}
|
||||
return cliConfig{Paths: paths, Profile: profile, Seed: contract.DefaultSeed, Scenarios: scenarioFlags{scenarioValue}, Fault: fault}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/evidence"
|
||||
)
|
||||
|
||||
func runContext(ctx context.Context, args []string, stdout io.Writer, stderr io.Writer, now time.Time) error {
|
||||
config, err := parseFlags(args, stderr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeMetadata(ctx, config, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(config.Scenarios) == 1 && config.Scenarios[0].String() == contract.ScenarioMetadata && config.Fault.IsZero() {
|
||||
fmt.Fprintf(stdout, "metadata written for profile %s\n", config.Profile.Name())
|
||||
return nil
|
||||
}
|
||||
execution, err := selectScenarioExecution(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
scenarioContext, cancel := context.WithTimeout(ctx, config.Profile.SuiteDeadline())
|
||||
defer cancel()
|
||||
output, runErr := execution.run(scenarioContext)
|
||||
if err := writeScenarioEvidence(config, output, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := evidence.ValidateDirectory(config.Paths.ResultsDir().String()); err != nil {
|
||||
return fmt.Errorf("validate scenario evidence: %w", err)
|
||||
}
|
||||
if runErr != nil {
|
||||
return runErr
|
||||
}
|
||||
fmt.Fprintf(stdout, "scenario %s passed for profile %s\n", execution.name, config.Profile.Name())
|
||||
return nil
|
||||
}
|
||||
|
||||
func run(args []string, stdout io.Writer, stderr io.Writer, now time.Time) error {
|
||||
return runContext(context.Background(), args, stdout, stderr, now)
|
||||
}
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := runContext(ctx, os.Args[1:], os.Stdout, os.Stderr, time.Now().UTC()); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "agentcompat: %s\n", evidence.Redact(err.Error()))
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/evidence"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/scenario"
|
||||
)
|
||||
|
||||
func TestCLI_ParsesTypedFlagsAndWritesMetadata(t *testing.T) {
|
||||
resultsDir := t.TempDir()
|
||||
var stdout, stderr bytes.Buffer
|
||||
err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--seed", "0x4e5a4841", "--scenario", "metadata"}, &stdout, &stderr, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC))
|
||||
if err != nil {
|
||||
t.Fatalf("run CLI: %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(resultsDir, "metadata.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read metadata: %v", err)
|
||||
}
|
||||
var metadata struct {
|
||||
Profile struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"profile"`
|
||||
Seed string `json:"seed"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
t.Fatalf("parse metadata: %v", err)
|
||||
}
|
||||
if metadata.Profile.Name != "pr-full" || metadata.Seed != "0x4e5a4841" || !strings.Contains(stdout.String(), "metadata written") {
|
||||
t.Fatalf("unexpected metadata-only output: %#v %q", metadata, stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_MetadataEvidenceValidatesAsCurrentMetadataProfile(t *testing.T) {
|
||||
resultsDir := t.TempDir()
|
||||
var stdout, stderr bytes.Buffer
|
||||
err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "metadata"}, &stdout, &stderr, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC))
|
||||
if err != nil {
|
||||
t.Fatalf("run metadata CLI: %v", err)
|
||||
}
|
||||
if err := evidence.ValidateDirectory(resultsDir); err != nil {
|
||||
t.Fatalf("validate metadata evidence: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_RejectsInvalidProfileAndSeedWithoutSecretEcho(t *testing.T) {
|
||||
for name, args := range map[string][]string{
|
||||
"profile": {"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "private-profile-secret", "--results-dir", t.TempDir()},
|
||||
"seed": {"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", t.TempDir(), "--seed", "not-a-seed-secret"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
err := run(args, &stdout, &stderr, time.Now())
|
||||
if err == nil {
|
||||
t.Fatal("invalid CLI input accepted")
|
||||
}
|
||||
if strings.Contains(err.Error(), "not-a-seed-secret") || strings.Contains(err.Error(), "private-profile-secret") {
|
||||
t.Fatal("invalid input echoed in error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_RejectsMissingPaths(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
err := run([]string{"--profile", "pr-full", "--scenario", "metadata"}, &stdout, &stderr, time.Now())
|
||||
if err == nil || !strings.Contains(err.Error(), "--nezha-source") {
|
||||
t.Fatalf("missing source paths were not rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_ParsesRepeatableScenariosAndFault(t *testing.T) {
|
||||
var stderr bytes.Buffer
|
||||
config, err := parseFlags([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "soak", "--results-dir", "/tmp/results", "--scenario", "metadata", "--scenario", "transfer-100mib", "--fault", "transfer-hash"}, &stderr)
|
||||
if err != nil {
|
||||
t.Fatalf("parse flags: %v", err)
|
||||
}
|
||||
if len(config.Scenarios) != 2 || config.Scenarios[0].String() != "metadata" || config.Scenarios[1].String() != "transfer-100mib" || config.Fault.String() != "transfer-hash" {
|
||||
t.Fatalf("unexpected typed flags: %#v", config)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_WritesMetadataBeforeRejectingUnsupportedRuntime(t *testing.T) {
|
||||
resultsDir := t.TempDir()
|
||||
var stdout, stderr bytes.Buffer
|
||||
err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "future-scenario"}, &stdout, &stderr, time.Now())
|
||||
if err == nil {
|
||||
t.Fatal("unimplemented runtime reported success")
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(resultsDir, "metadata.json")); statErr != nil {
|
||||
t.Fatalf("metadata was not written before runtime rejection: %v", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_RecognizesOnlyMCPFilesystemRuntimeScenario(t *testing.T) {
|
||||
var stderr bytes.Buffer
|
||||
config, err := parseFlags([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", t.TempDir(), "--scenario", "mcp-filesystem"}, &stderr)
|
||||
if err != nil {
|
||||
t.Fatalf("parse mcp-filesystem flags: %v", err)
|
||||
}
|
||||
execution, err := selectScenarioExecution(config)
|
||||
if err != nil || execution.name != "mcp-filesystem" {
|
||||
t.Fatalf("mcp-filesystem runtime registration: name=%q err=%v", execution.name, err)
|
||||
}
|
||||
|
||||
config.Scenarios = append(config.Scenarios, config.Scenarios[0])
|
||||
if _, err := selectScenarioExecution(config); err == nil || !strings.Contains(err.Error(), "exactly one") {
|
||||
t.Fatalf("multi-scenario runtime was unexpectedly accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_SelectsTerminalRuntimeScenario(t *testing.T) {
|
||||
var stderr bytes.Buffer
|
||||
config, err := parseFlags([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", t.TempDir(), "--scenario", "terminal"}, &stderr)
|
||||
if err != nil {
|
||||
t.Fatalf("parse terminal flags: %v", err)
|
||||
}
|
||||
previous := runTerminalScenario
|
||||
var received scenario.TerminalInput
|
||||
runTerminalScenario = func(_ context.Context, input scenario.TerminalInput) (scenario.Result, error) {
|
||||
received = input
|
||||
return scenario.Result{Name: "terminal", Passed: true}, nil
|
||||
}
|
||||
t.Cleanup(func() { runTerminalScenario = previous })
|
||||
|
||||
execution, err := selectScenarioExecution(config)
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("select terminal scenario: %v", err)
|
||||
}
|
||||
if execution.name != "terminal" || execution.run == nil {
|
||||
t.Fatalf("unexpected terminal execution: %#v", execution)
|
||||
}
|
||||
if _, err := execution.run(context.Background()); err != nil {
|
||||
t.Fatalf("run terminal execution: %v", err)
|
||||
}
|
||||
if received.Paths.NezhaSource().String() != "/src/nezha" || received.Paths.AgentSource().String() != "/src/agent" || !received.Fault.IsZero() {
|
||||
t.Fatalf("terminal input was not forwarded: %#v", received)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_MetadataWriteReplacesSymlinkWithoutChangingTarget(t *testing.T) {
|
||||
resultsDir := t.TempDir()
|
||||
target := filepath.Join(t.TempDir(), "target")
|
||||
if err := os.WriteFile(target, []byte("sentinel"), 0o644); err != nil {
|
||||
t.Fatalf("write target: %v", err)
|
||||
}
|
||||
if err := os.Symlink(target, filepath.Join(resultsDir, "metadata.json")); err != nil {
|
||||
t.Fatalf("create metadata symlink: %v", err)
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "metadata"}, &stdout, &stderr, time.Now())
|
||||
if err != nil {
|
||||
t.Fatalf("replace metadata symlink: %v", err)
|
||||
}
|
||||
data, readErr := os.ReadFile(target)
|
||||
if readErr != nil || string(data) != "sentinel" {
|
||||
t.Fatalf("symlink target changed: %q %v", data, readErr)
|
||||
}
|
||||
info, statErr := os.Lstat(filepath.Join(resultsDir, "metadata.json"))
|
||||
if statErr != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("metadata replacement mode=%v err=%v", info, statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_MetadataWriteUsesPrivatePermissions(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
resultsDir := filepath.Join(root, "results")
|
||||
var stdout, stderr bytes.Buffer
|
||||
if err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "metadata"}, &stdout, &stderr, time.Now()); err != nil {
|
||||
t.Fatalf("write metadata: %v", err)
|
||||
}
|
||||
directoryInfo, err := os.Stat(resultsDir)
|
||||
if err != nil {
|
||||
t.Fatalf("stat results directory: %v", err)
|
||||
}
|
||||
fileInfo, err := os.Stat(filepath.Join(resultsDir, "metadata.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("stat metadata: %v", err)
|
||||
}
|
||||
if directoryInfo.Mode().Perm() != 0o700 || fileInfo.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("unexpected permissions: directory=%o file=%o", directoryInfo.Mode().Perm(), fileInfo.Mode().Perm())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/evidence"
|
||||
)
|
||||
|
||||
var metadataArtifactReady = func(context.Context) error { return nil }
|
||||
|
||||
func writeMetadata(ctx context.Context, config cliConfig, now time.Time) error {
|
||||
resultsDir := config.Paths.ResultsDir().String()
|
||||
if info, err := os.Lstat(resultsDir); err == nil && info.Mode()&os.ModeSymlink != 0 {
|
||||
return errors.New("results directory must not be a symbolic link")
|
||||
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("inspect results directory: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(resultsDir, 0o700); err != nil {
|
||||
return fmt.Errorf("create results directory: %w", err)
|
||||
}
|
||||
if err := os.Chmod(resultsDir, 0o700); err != nil {
|
||||
return fmt.Errorf("secure results directory: %w", err)
|
||||
}
|
||||
if err := prepareResultsDir(resultsDir); err != nil {
|
||||
return err
|
||||
}
|
||||
metadata, err := evidence.NewMetadata(evidence.MetadataInput{Profile: config.Profile, Seed: config.Seed, Paths: config.Paths, ResourceBudget: contract.DefaultResourceBudget(), Scenarios: config.Scenarios, Fault: config.Fault, StartedAt: now, EvidenceFiles: evidence.EvidenceFiles()})
|
||||
if err != nil {
|
||||
return fmt.Errorf("build metadata: %w", err)
|
||||
}
|
||||
data, err := json.MarshalIndent(metadata, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal metadata: %w", err)
|
||||
}
|
||||
path := filepath.Join(resultsDir, "metadata.json")
|
||||
return writePrivateArtifactWithSeam(path, data, func() error { return metadataArtifactReady(ctx) })
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
)
|
||||
|
||||
func TestCLI_MetadataCancellationRemovesTemporaryArtifact(t *testing.T) {
|
||||
resultsDir := t.TempDir()
|
||||
config := testCLIConfig(t, contract.ScenarioMetadata, "")
|
||||
paths, err := contract.NewPaths(config.Paths.NezhaSource().String(), config.Paths.AgentSource().String(), resultsDir)
|
||||
if err != nil {
|
||||
t.Fatalf("paths: %v", err)
|
||||
}
|
||||
config.Paths = paths
|
||||
ready := make(chan struct{})
|
||||
previous := metadataArtifactReady
|
||||
metadataArtifactReady = func(ctx context.Context) error {
|
||||
close(ready)
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
t.Cleanup(func() { metadataArtifactReady = previous })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
writeDone := make(chan error, 1)
|
||||
go func() { writeDone <- writeMetadata(ctx, config, time.Now()) }()
|
||||
<-ready
|
||||
cancel()
|
||||
|
||||
if err := <-writeDone; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("metadata cancellation error=%v, want context canceled", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(resultsDir, "metadata.json")); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("metadata final file exists after cancellation: %v", err)
|
||||
}
|
||||
entries, err := os.ReadDir(resultsDir)
|
||||
if err != nil {
|
||||
t.Fatalf("read results directory: %v", err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if strings.HasPrefix(entry.Name(), ".artifact-") {
|
||||
t.Fatalf("metadata temporary artifact survived cancellation: %s", entry.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCLI_SelectsNATRuntimeScenario(t *testing.T) {
|
||||
// Given
|
||||
var stderr bytes.Buffer
|
||||
config, err := parseFlags([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", t.TempDir(), "--scenario", "nat"}, &stderr)
|
||||
if err != nil {
|
||||
t.Fatalf("parse NAT flags: %v", err)
|
||||
}
|
||||
|
||||
// When
|
||||
execution, err := selectScenarioExecution(config)
|
||||
|
||||
// Then
|
||||
if err != nil {
|
||||
t.Fatalf("select NAT scenario: %v", err)
|
||||
}
|
||||
if execution.name != "nat" || execution.run == nil {
|
||||
t.Fatalf("unexpected NAT execution: %#v", execution)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
var (
|
||||
scenarioArtifactPublished = func(string) error { return nil }
|
||||
privateArtifactClose = (*os.File).Close
|
||||
privateArtifactRemove = os.Remove
|
||||
)
|
||||
|
||||
func writePrivateArtifact(path string, data []byte) (err error) {
|
||||
return writePrivateArtifactWithSeam(path, data, func() error {
|
||||
return scenarioArtifactPublished(filepath.Base(path))
|
||||
})
|
||||
}
|
||||
|
||||
func writePrivateArtifactWithSeam(path string, data []byte, beforeRename func() error) (err error) {
|
||||
directory := filepath.Dir(path)
|
||||
temporary, err := os.CreateTemp(directory, ".artifact-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create temporary artifact: %w", err)
|
||||
}
|
||||
temporaryPath := temporary.Name()
|
||||
committed := false
|
||||
closed := false
|
||||
defer func() {
|
||||
if !committed {
|
||||
if !closed {
|
||||
err = errors.Join(err, privateArtifactClose(temporary))
|
||||
}
|
||||
err = errors.Join(err, privateArtifactRemove(temporaryPath))
|
||||
}
|
||||
}()
|
||||
if err := temporary.Chmod(0o600); err != nil {
|
||||
return fmt.Errorf("secure temporary artifact: %w", err)
|
||||
}
|
||||
if _, err := temporary.Write(append(data, '\n')); err != nil {
|
||||
return fmt.Errorf("write temporary artifact: %w", err)
|
||||
}
|
||||
if err := temporary.Sync(); err != nil {
|
||||
return fmt.Errorf("sync temporary artifact: %w", err)
|
||||
}
|
||||
closeErr := privateArtifactClose(temporary)
|
||||
closed = true
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("close temporary artifact: %w", closeErr)
|
||||
}
|
||||
if err := beforeRename(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(temporaryPath, path); err != nil {
|
||||
return fmt.Errorf("rename temporary artifact: %w", err)
|
||||
}
|
||||
committed = true
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/agent"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/dashboard"
|
||||
processharness "github.com/nezhahq/nezha/integration/agentcompat/internal/process"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/scenario"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/workspace"
|
||||
)
|
||||
|
||||
func completeReconnectDispatchEvidence(t *testing.T) scenario.ReconnectEvidence {
|
||||
t.Helper()
|
||||
disconnectAt := time.Date(2026, 2, 3, 4, 5, 6, 700, time.UTC)
|
||||
reconnectAt := disconnectAt.Add(7 * time.Second)
|
||||
dashboardReceipt := dashboard.MCPReceiptPair{
|
||||
Task: dashboard.MCPReceiptEvent{Sequence: 51, DashboardGeneration: 12, GateGeneration: 61, ServerID: 41, TaskID: 71, TaskType: 81, Kind: dashboard.MCPReceiptTask},
|
||||
Result: dashboard.MCPReceiptEvent{Sequence: 52, DashboardGeneration: 12, GateGeneration: 61, ServerID: 41, TaskID: 71, TaskType: 81, Kind: dashboard.MCPReceiptResult},
|
||||
}
|
||||
agentReceipt := dashboard.MCPReceiptPair{
|
||||
Task: dashboard.MCPReceiptEvent{Sequence: 53, DashboardGeneration: 12, GateGeneration: 62, ServerID: 41, TaskID: 72, TaskType: 82, Kind: dashboard.MCPReceiptTask},
|
||||
Result: dashboard.MCPReceiptEvent{Sequence: 54, DashboardGeneration: 12, GateGeneration: 62, ServerID: 41, TaskID: 72, TaskType: 82, Kind: dashboard.MCPReceiptResult},
|
||||
}
|
||||
evidence := scenario.ReconnectEvidence{
|
||||
Fixture: scenario.ReconnectFixtureEvidence{
|
||||
Dashboard: dashboard.FixtureIdentity{
|
||||
WorkspaceRoot: "/typed/dashboard-workspace", ConfigPath: "/typed/dashboard.yaml", DatabasePath: "/typed/dashboard.sqlite", BinaryPath: "/typed/dashboard",
|
||||
HTTP: workspace.ListenerIdentity{Address: "127.0.0.1:41001", Inode: 1101}, Receipt: workspace.ListenerIdentity{Address: "127.0.0.1:41002", Inode: 1102}, HTTPS: workspace.ListenerIdentity{Address: "127.0.0.1:41003", Inode: 1103},
|
||||
},
|
||||
AgentRoot: "/typed/agent-workspace", AgentConfigPath: "/typed/agent.yaml", AgentBinaryPath: "/typed/agent",
|
||||
},
|
||||
Runtime: scenario.ReconnectRuntimeEvidence{
|
||||
DashboardBefore: dashboard.RuntimeIdentity{Generation: 11, PID: 2101, ProcessGroupID: 3101}, DashboardAfter: dashboard.RuntimeIdentity{Generation: 12, PID: 2102, ProcessGroupID: 3102},
|
||||
AgentBefore: agent.ProcessIdentity{Generation: 21, PID: 2201, ProcessGroupID: 3201}, AgentAfter: agent.ProcessIdentity{Generation: 22, PID: 2202, ProcessGroupID: 3202},
|
||||
StateGenerationBeforeAgentRestart: 31, StateGenerationAfterAgentRestart: 32,
|
||||
},
|
||||
Identity: scenario.ReconnectIdentityEvidence{ServerID: 41, UUID: "00000000-0000-0000-0000-000000000041", DashboardConfigUnchanged: true, AgentConfigUnchanged: true, DashboardFixtureUnchanged: true, ClientsRecreated: true, BootstrapRecreated: true},
|
||||
Lifecycle: scenario.ReconnectLifecycleEvidence{
|
||||
DisconnectAt: disconnectAt, ReconnectAt: reconnectAt, ReconnectInterval: 7 * time.Second,
|
||||
DashboardReceipts: []dashboard.MCPReceiptPair{dashboardReceipt}, AgentReceipts: []dashboard.MCPReceiptPair{agentReceipt},
|
||||
StaleGenerationReceipts: 0, DuplicateTaskIDs: 0, LostResultIDs: 0, OutsideRootSentinelUnchanged: true,
|
||||
},
|
||||
Observation: scenario.ReconnectObservation{ServerID: 41, UUID: "00000000-0000-0000-0000-000000000041", OldGeneration: 11, NewGeneration: 12, DisconnectAt: disconnectAt, ReconnectAt: reconnectAt, TaskIDs: []uint64{71, 72}, ResultIDs: []uint64{71, 72}, PostReconnect: true, AgentRestarted: true},
|
||||
AgentCleanup: processharness.CleanupReceipt{Passed: true, Forced: false, Processes: []processharness.CleanupRecord{{Name: "agent-generation-21", PID: 2201, Forced: false, Error: ""}, {Name: "agent-generation-22", PID: 2202, Forced: false, Error: ""}}},
|
||||
DashboardCleanup: processharness.CleanupReceipt{Passed: true, Forced: false, Processes: []processharness.CleanupRecord{{Name: "dashboard-generation-11", PID: 2101, Forced: false, Error: ""}, {Name: "dashboard-generation-12", PID: 2102, Forced: false, Error: ""}}},
|
||||
}
|
||||
assertCompleteReconnectDispatchEvidence(t, evidence)
|
||||
return evidence
|
||||
}
|
||||
|
||||
func assertCompleteReconnectDispatchEvidence(t *testing.T, evidence scenario.ReconnectEvidence) {
|
||||
t.Helper()
|
||||
if err := evidence.Validate(); err != nil {
|
||||
t.Fatalf("incomplete reconnect dispatch fixture: %v", err)
|
||||
}
|
||||
fixture := evidence.Fixture
|
||||
if fixture.Dashboard.WorkspaceRoot == "" || fixture.Dashboard.ConfigPath == "" || fixture.Dashboard.DatabasePath == "" || fixture.Dashboard.BinaryPath == "" || fixture.AgentRoot == "" || fixture.AgentConfigPath == "" || fixture.AgentBinaryPath == "" {
|
||||
t.Fatal("incomplete reconnect dispatch fixture paths")
|
||||
}
|
||||
for name, listener := range map[string]struct {
|
||||
address string
|
||||
inode uint64
|
||||
}{
|
||||
"http": {fixture.Dashboard.HTTP.Address, fixture.Dashboard.HTTP.Inode},
|
||||
"receipt": {fixture.Dashboard.Receipt.Address, fixture.Dashboard.Receipt.Inode},
|
||||
"https": {fixture.Dashboard.HTTPS.Address, fixture.Dashboard.HTTPS.Inode},
|
||||
} {
|
||||
if listener.address == "" || listener.inode == 0 {
|
||||
t.Fatalf("incomplete reconnect dispatch %s listener", name)
|
||||
}
|
||||
}
|
||||
if len(evidence.Lifecycle.DashboardReceipts) == 0 || len(evidence.Lifecycle.AgentReceipts) == 0 {
|
||||
t.Fatal("incomplete reconnect dispatch receipt pairs")
|
||||
}
|
||||
for _, pairs := range [][]dashboard.MCPReceiptPair{evidence.Lifecycle.DashboardReceipts, evidence.Lifecycle.AgentReceipts} {
|
||||
for _, pair := range pairs {
|
||||
if pair.Task.Sequence == 0 || pair.Task.DashboardGeneration == 0 || pair.Task.GateGeneration == 0 || pair.Task.ServerID == 0 || pair.Task.TaskID == 0 || pair.Task.TaskType == 0 || pair.Task.Kind != dashboard.MCPReceiptTask {
|
||||
t.Fatalf("incomplete reconnect dispatch task receipt: %#v", pair.Task)
|
||||
}
|
||||
if pair.Result.Sequence == 0 || pair.Result.DashboardGeneration == 0 || pair.Result.GateGeneration == 0 || pair.Result.ServerID == 0 || pair.Result.TaskID == 0 || pair.Result.TaskType == 0 || pair.Result.Kind != dashboard.MCPReceiptResult {
|
||||
t.Fatalf("incomplete reconnect dispatch result receipt: %#v", pair.Result)
|
||||
}
|
||||
}
|
||||
}
|
||||
assertCompleteCleanupReceipt(t, "agent", evidence.AgentCleanup)
|
||||
assertCompleteCleanupReceipt(t, "dashboard", evidence.DashboardCleanup)
|
||||
}
|
||||
|
||||
func assertCompleteCleanupReceipt(t *testing.T, name string, receipt processharness.CleanupReceipt) {
|
||||
t.Helper()
|
||||
if !receipt.Passed || receipt.Forced || len(receipt.Processes) == 0 {
|
||||
t.Fatalf("incomplete reconnect dispatch %s cleanup receipt: %#v", name, receipt)
|
||||
}
|
||||
for _, record := range receipt.Processes {
|
||||
if record.Name == "" || record.PID == 0 || record.Forced || record.Error != "" {
|
||||
t.Fatalf("incomplete reconnect dispatch %s cleanup record: %#v", name, record)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/scenario"
|
||||
)
|
||||
|
||||
func TestCLI_RegisteredScenariosHaveExhaustiveRuntimeRouting(t *testing.T) {
|
||||
for _, definition := range contract.ScenarioDefinitions() {
|
||||
t.Run(definition.Name, func(t *testing.T) {
|
||||
config := testCLIConfig(t, definition.Name, "")
|
||||
execution, err := selectScenarioExecution(config)
|
||||
if definition.Execution == contract.ScenarioExecutionMetadata {
|
||||
if err == nil {
|
||||
t.Fatal("metadata unexpectedly received runtime execution")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("select registered scenario: %v", err)
|
||||
}
|
||||
if execution.name != definition.Name || execution.run == nil {
|
||||
t.Fatalf("incomplete runtime execution: %#v", execution)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_Todos11To15DispatchPropagatesTypedInputsErrorsAndOutputs(t *testing.T) {
|
||||
runnerError := errors.New("injected runner error")
|
||||
tests := []struct {
|
||||
name string
|
||||
fault string
|
||||
set func(*testing.T, scenario.Result, error, *bool)
|
||||
}{
|
||||
{contract.ScenarioRegistrationConfigExec, contract.FaultAgentBadSecret, func(t *testing.T, want scenario.Result, wantErr error, called *bool) {
|
||||
previous := runRegistrationConfigExecScenario
|
||||
runRegistrationConfigExecScenario = func(_ context.Context, input scenario.RegistrationConfigExecInput) (scenario.Result, error) {
|
||||
*called = true
|
||||
assertPathsAndFault(t, input.Paths, input.Fault, contract.FaultAgentBadSecret)
|
||||
return want, wantErr
|
||||
}
|
||||
t.Cleanup(func() { runRegistrationConfigExecScenario = previous })
|
||||
}},
|
||||
{contract.ScenarioNAT, "", func(t *testing.T, want scenario.Result, wantErr error, called *bool) {
|
||||
previous := runNATScenario
|
||||
runNATScenario = func(_ context.Context, input scenario.NATInput) (scenario.Result, error) {
|
||||
*called = true
|
||||
assertPathsAndFault(t, input.Paths, input.Fault, "")
|
||||
return want, wantErr
|
||||
}
|
||||
t.Cleanup(func() { runNATScenario = previous })
|
||||
}},
|
||||
{contract.ScenarioLegacyFM, contract.FaultAgentBadSecret, func(t *testing.T, want scenario.Result, wantErr error, called *bool) {
|
||||
previous := runLegacyFMScenario
|
||||
runLegacyFMScenario = func(_ context.Context, input scenario.LegacyFMInput) (scenario.Result, error) {
|
||||
*called = true
|
||||
assertPathsAndFault(t, input.Paths, input.Fault, contract.FaultAgentBadSecret)
|
||||
return want, wantErr
|
||||
}
|
||||
t.Cleanup(func() { runLegacyFMScenario = previous })
|
||||
}},
|
||||
{contract.ScenarioTerminal, "", func(t *testing.T, want scenario.Result, wantErr error, called *bool) {
|
||||
previous := runTerminalScenario
|
||||
runTerminalScenario = func(_ context.Context, input scenario.TerminalInput) (scenario.Result, error) {
|
||||
*called = true
|
||||
assertPathsAndFault(t, input.Paths, input.Fault, "")
|
||||
return want, wantErr
|
||||
}
|
||||
t.Cleanup(func() { runTerminalScenario = previous })
|
||||
}},
|
||||
{contract.ScenarioMCPFilesystem, "", func(t *testing.T, want scenario.Result, wantErr error, called *bool) {
|
||||
previous := runMCPFilesystemScenario
|
||||
runMCPFilesystemScenario = func(_ context.Context, input scenario.MCPFilesystemInput) (scenario.Result, error) {
|
||||
*called = true
|
||||
if input.Paths.NezhaSource().String() != "/src/nezha" || input.Paths.AgentSource().String() != "/src/agent" {
|
||||
t.Fatalf("MCP filesystem paths=%#v", input.Paths)
|
||||
}
|
||||
return want, wantErr
|
||||
}
|
||||
t.Cleanup(func() { runMCPFilesystemScenario = previous })
|
||||
}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
want := scenario.Result{Name: test.name, Passed: false, Assertions: []scenario.Assertion{{Name: "runner assertion", Passed: false}}, Error: runnerError.Error()}
|
||||
called := false
|
||||
test.set(t, want, runnerError, &called)
|
||||
execution, err := selectScenarioExecution(testCLIConfig(t, test.name, test.fault))
|
||||
if err != nil {
|
||||
t.Fatalf("select execution: %v", err)
|
||||
}
|
||||
output, err := execution.run(t.Context())
|
||||
if !errors.Is(err, runnerError) {
|
||||
t.Fatalf("runner error=%v", err)
|
||||
}
|
||||
if !called || output.Result.Name != want.Name || output.Result.Error != want.Error || output.Transfer != nil || output.Reconnect != nil {
|
||||
t.Fatalf("runner propagation called=%t output=%#v", called, output)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertPathsAndFault(t *testing.T, paths contract.Paths, fault contract.Fault, wantFault string) {
|
||||
t.Helper()
|
||||
if paths.NezhaSource().String() != "/src/nezha" || paths.AgentSource().String() != "/src/agent" || fault.String() != wantFault {
|
||||
t.Fatalf("paths/fault propagation paths=%#v fault=%q", paths, fault.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/evidence"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/scenario"
|
||||
)
|
||||
|
||||
type transferArtifact struct {
|
||||
Scenario string `json:"scenario"`
|
||||
Fault string `json:"fault,omitempty"`
|
||||
Passed bool `json:"passed"`
|
||||
CleanupOK bool `json:"cleanup_ok"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Evidence scenario.TransferEvidence `json:"evidence"`
|
||||
}
|
||||
|
||||
type reconnectArtifact struct {
|
||||
Scenario string `json:"scenario"`
|
||||
Fault string `json:"fault,omitempty"`
|
||||
Passed bool `json:"passed"`
|
||||
CleanupOK bool `json:"cleanup_ok"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Evidence scenario.ReconnectEvidence `json:"evidence"`
|
||||
}
|
||||
|
||||
func writeScenarioEvidence(config cliConfig, output scenarioExecutionOutput, now time.Time) error {
|
||||
if err := output.Validate(); err != nil {
|
||||
return fmt.Errorf("validate scenario execution output: %w", err)
|
||||
}
|
||||
result := output.Result
|
||||
if !result.Passed && config.Fault.String() != "" && allAssertionsPassed(result.Assertions) {
|
||||
result.Assertions = append(result.Assertions, scenario.Assertion{Name: contract.AssertionInjectedFault, Passed: false, Details: result.Error})
|
||||
}
|
||||
if output.Transfer != nil {
|
||||
artifact := transferArtifact{Scenario: result.Name, Fault: config.Fault.String(), Passed: result.Passed, CleanupOK: result.CleanupOK, Error: evidence.Redact(result.Error), Evidence: *output.Transfer}
|
||||
if err := writeJSONArtifact(config.Paths.ResultsDir().String(), "transfer.json", artifact); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if output.Reconnect != nil {
|
||||
artifact := reconnectArtifact{Scenario: result.Name, Fault: config.Fault.String(), Passed: result.Passed, CleanupOK: result.CleanupOK, Error: evidence.Redact(result.Error), Evidence: *output.Reconnect}
|
||||
if err := writeJSONArtifact(config.Paths.ResultsDir().String(), "reconnect.json", artifact); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
assertions := make([]evidence.Assertion, 0, len(result.Assertions))
|
||||
for _, assertion := range result.Assertions {
|
||||
assertions = append(assertions, evidence.Assertion{Name: assertion.Name, Passed: assertion.Passed, Details: assertion.Details})
|
||||
}
|
||||
results := evidence.Results{Profile: string(config.Profile.Name()), Passed: result.Passed, Scenarios: []evidence.ScenarioResult{{Name: result.Name, Passed: result.Passed, Assertions: assertions, Error: result.Error}}}
|
||||
data, err := evidence.MarshalResults(results)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal scenario results: %w", err)
|
||||
}
|
||||
if err := writePrivateFile(filepath.Join(config.Paths.ResultsDir().String(), "results.json"), data); err != nil {
|
||||
return fmt.Errorf("write scenario results: %w", err)
|
||||
}
|
||||
junit, err := evidence.JUnit(results)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal scenario junit: %w", err)
|
||||
}
|
||||
if err := writePrivateFile(filepath.Join(config.Paths.ResultsDir().String(), "junit.xml"), junit); err != nil {
|
||||
return fmt.Errorf("write scenario junit: %w", err)
|
||||
}
|
||||
cleanup := struct {
|
||||
Passed bool `json:"passed"`
|
||||
Scenario string `json:"scenario"`
|
||||
FinishedAt string `json:"finished_at"`
|
||||
}{Passed: result.CleanupOK, Scenario: result.Name, FinishedAt: now.UTC().Format(time.RFC3339)}
|
||||
return writeJSONArtifact(config.Paths.ResultsDir().String(), "cleanup.json", cleanup)
|
||||
}
|
||||
|
||||
func writeJSONArtifact(resultsDir, name string, artifact any) error {
|
||||
data, err := json.MarshalIndent(artifact, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal %s: %w", name, err)
|
||||
}
|
||||
if redacted := evidence.Redact(string(data)); redacted != string(data) {
|
||||
return fmt.Errorf("credential detected while marshaling %s", name)
|
||||
}
|
||||
if err := writePrivateFile(filepath.Join(resultsDir, name), data); err != nil {
|
||||
return fmt.Errorf("write %s: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writePrivateFile(path string, data []byte) error {
|
||||
return writePrivateArtifact(path, data)
|
||||
}
|
||||
|
||||
func allAssertionsPassed(assertions []scenario.Assertion) bool {
|
||||
for _, assertion := range assertions {
|
||||
if !assertion.Passed {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/scenario"
|
||||
)
|
||||
|
||||
type scenarioExecutionOutput struct {
|
||||
Result scenario.Result
|
||||
Transfer *scenario.TransferEvidence
|
||||
Reconnect *scenario.ReconnectEvidence
|
||||
}
|
||||
|
||||
func (output scenarioExecutionOutput) Validate() error {
|
||||
if output.Result.Name == "" {
|
||||
return errors.New("scenario execution result is missing")
|
||||
}
|
||||
dedicatedCount := 0
|
||||
if output.Transfer != nil {
|
||||
dedicatedCount++
|
||||
}
|
||||
if output.Reconnect != nil {
|
||||
dedicatedCount++
|
||||
}
|
||||
switch output.Result.Name {
|
||||
case contract.ScenarioTransfer100MiB:
|
||||
if dedicatedCount != 1 || output.Transfer == nil {
|
||||
return errors.New("transfer execution requires only transfer evidence")
|
||||
}
|
||||
case contract.ScenarioReconnect:
|
||||
if dedicatedCount != 1 || output.Reconnect == nil {
|
||||
return errors.New("reconnect execution requires only reconnect evidence")
|
||||
}
|
||||
default:
|
||||
if dedicatedCount != 0 {
|
||||
return errors.New("scenario execution has mismatched dedicated evidence")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type scenarioExecution struct {
|
||||
name string
|
||||
run func(context.Context) (scenarioExecutionOutput, error)
|
||||
}
|
||||
|
||||
var (
|
||||
runRegistrationConfigExecScenario = (scenario.RegistrationConfigExec{}).Run
|
||||
runNATScenario = (scenario.NAT{}).Run
|
||||
runLegacyFMScenario = (scenario.LegacyFM{}).Run
|
||||
runTerminalScenario = (scenario.Terminal{}).Run
|
||||
runMCPFilesystemScenario = (scenario.MCPFilesystem{}).Run
|
||||
runTransferScenario = (scenario.Transfer{}).RunWithEvidence
|
||||
runReconnectScenario = (scenario.Reconnect{}).RunWithEvidence
|
||||
)
|
||||
|
||||
func selectScenarioExecution(config cliConfig) (scenarioExecution, error) {
|
||||
if len(config.Scenarios) != 1 {
|
||||
return scenarioExecution{}, errors.New("runtime execution requires exactly one --scenario")
|
||||
}
|
||||
selected := config.Scenarios[0]
|
||||
if err := contract.ValidateScenarioFault(selected, config.Fault); err != nil {
|
||||
return scenarioExecution{}, err
|
||||
}
|
||||
definition, err := contract.ScenarioDefinitionByName(selected.String())
|
||||
if err != nil {
|
||||
return scenarioExecution{}, err
|
||||
}
|
||||
switch definition.Execution {
|
||||
case contract.ScenarioExecutionMetadata:
|
||||
return scenarioExecution{}, errors.New("metadata scenario does not have a runtime execution")
|
||||
case contract.ScenarioExecutionRegistrationConfigExec:
|
||||
return standardExecution(selected.String(), func(ctx context.Context) (scenario.Result, error) {
|
||||
return runRegistrationConfigExecScenario(ctx, scenario.RegistrationConfigExecInput{Paths: config.Paths, Fault: config.Fault})
|
||||
}), nil
|
||||
case contract.ScenarioExecutionNAT:
|
||||
return standardExecution(selected.String(), func(ctx context.Context) (scenario.Result, error) {
|
||||
return runNATScenario(ctx, scenario.NATInput{Paths: config.Paths, Fault: config.Fault})
|
||||
}), nil
|
||||
case contract.ScenarioExecutionLegacyFM:
|
||||
return standardExecution(selected.String(), func(ctx context.Context) (scenario.Result, error) {
|
||||
return runLegacyFMScenario(ctx, scenario.LegacyFMInput{Paths: config.Paths, Fault: config.Fault})
|
||||
}), nil
|
||||
case contract.ScenarioExecutionTerminal:
|
||||
return standardExecution(selected.String(), func(ctx context.Context) (scenario.Result, error) {
|
||||
return runTerminalScenario(ctx, scenario.TerminalInput{Paths: config.Paths, Fault: config.Fault})
|
||||
}), nil
|
||||
case contract.ScenarioExecutionMCPFilesystem:
|
||||
return standardExecution(selected.String(), func(ctx context.Context) (scenario.Result, error) {
|
||||
return runMCPFilesystemScenario(ctx, scenario.MCPFilesystemInput{Paths: config.Paths})
|
||||
}), nil
|
||||
case contract.ScenarioExecutionTransfer:
|
||||
return scenarioExecution{name: selected.String(), run: func(ctx context.Context) (scenarioExecutionOutput, error) {
|
||||
result, transferEvidence, err := runTransferScenario(ctx, scenario.TransferInput{Paths: config.Paths, Fault: config.Fault})
|
||||
return scenarioExecutionOutput{Result: result, Transfer: &transferEvidence}, err
|
||||
}}, nil
|
||||
case contract.ScenarioExecutionReconnect:
|
||||
return scenarioExecution{name: selected.String(), run: func(ctx context.Context) (scenarioExecutionOutput, error) {
|
||||
result, reconnectEvidence, err := runReconnectScenario(ctx, scenario.ReconnectInput{Paths: config.Paths, DashboardFault: config.Fault.String()})
|
||||
return scenarioExecutionOutput{Result: result, Reconnect: &reconnectEvidence}, err
|
||||
}}, nil
|
||||
default:
|
||||
return scenarioExecution{}, errors.New("runtime execution is not implemented for the selected scenario")
|
||||
}
|
||||
}
|
||||
|
||||
func standardExecution(name string, run func(context.Context) (scenario.Result, error)) scenarioExecution {
|
||||
return scenarioExecution{name: name, run: func(ctx context.Context) (scenarioExecutionOutput, error) {
|
||||
result, err := run(ctx)
|
||||
return scenarioExecutionOutput{Result: result}, err
|
||||
}}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/evidence"
|
||||
)
|
||||
|
||||
func TestCLI_FailedGenerationRemovesPriorEvidence(t *testing.T) {
|
||||
resultsDir := t.TempDir()
|
||||
seedStaleEvidence(t, resultsDir)
|
||||
var stdout, stderr bytes.Buffer
|
||||
if err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "future-scenario"}, &stdout, &stderr, time.Now()); err == nil {
|
||||
t.Fatal("unimplemented runtime unexpectedly succeeded")
|
||||
}
|
||||
for _, name := range append(evidence.FixedEvidenceFiles()[1:], "agents") {
|
||||
if _, err := os.Stat(filepath.Join(resultsDir, name)); !os.IsNotExist(err) {
|
||||
t.Fatalf("stale artifact remains: %s (%v)", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_FailedGenerationRemovesPriorEvidenceWhenMetadataIsSymlink(t *testing.T) {
|
||||
resultsDir := t.TempDir()
|
||||
target := filepath.Join(t.TempDir(), "metadata-target")
|
||||
if err := os.WriteFile(target, []byte("old metadata"), 0o600); err != nil {
|
||||
t.Fatalf("seed metadata target: %v", err)
|
||||
}
|
||||
if err := os.Symlink(target, filepath.Join(resultsDir, "metadata.json")); err != nil {
|
||||
t.Fatalf("create metadata symlink: %v", err)
|
||||
}
|
||||
for _, name := range []string{"results.json", "junit.xml"} {
|
||||
if err := os.WriteFile(filepath.Join(resultsDir, name), []byte("stale success"), 0o600); err != nil {
|
||||
t.Fatalf("seed stale artifact %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
if err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "future-scenario"}, &stdout, &stderr, time.Now()); err == nil {
|
||||
t.Fatal("unimplemented runtime unexpectedly succeeded")
|
||||
}
|
||||
metadataInfo, err := os.Lstat(filepath.Join(resultsDir, "metadata.json"))
|
||||
if err != nil || !metadataInfo.Mode().IsRegular() || metadataInfo.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("replacement metadata is not a private regular file: mode=%v err=%v", metadataInfo, err)
|
||||
}
|
||||
for _, name := range []string{"results.json", "junit.xml"} {
|
||||
if _, err := os.Stat(filepath.Join(resultsDir, name)); !os.IsNotExist(err) {
|
||||
t.Fatalf("stale artifact remains after failed invocation: %s (%v)", name, err)
|
||||
}
|
||||
}
|
||||
data, err := os.ReadFile(target)
|
||||
if err != nil || string(data) != "old metadata" {
|
||||
t.Fatalf("metadata symlink target changed: %q %v", data, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLI_ParseFailureRemovesPriorEvidence(t *testing.T) {
|
||||
for name, extraArgs := range map[string][]string{
|
||||
"invalid profile": {"--profile", "invalid-profile"},
|
||||
"invalid seed": {"--profile", "pr-full", "--seed", "invalid-seed"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
resultsDir := t.TempDir()
|
||||
seedStaleEvidence(t, resultsDir)
|
||||
args := append([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--results-dir", resultsDir}, extraArgs...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
if err := run(args, &stdout, &stderr, time.Now()); err == nil {
|
||||
t.Fatal("invalid CLI input unexpectedly succeeded")
|
||||
}
|
||||
for _, artifact := range evidence.FixedEvidenceFiles()[1:] {
|
||||
if _, err := os.Stat(filepath.Join(resultsDir, artifact)); !os.IsNotExist(err) {
|
||||
t.Fatalf("stale artifact remains after parse failure: %s (%v)", artifact, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func seedStaleEvidence(t *testing.T, resultsDir string) {
|
||||
t.Helper()
|
||||
for _, name := range evidence.FixedEvidenceFiles()[1:] {
|
||||
if err := os.WriteFile(filepath.Join(resultsDir, name), []byte("stale success"), 0o600); err != nil {
|
||||
t.Fatalf("seed stale artifact %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
if err := os.Mkdir(filepath.Join(resultsDir, "agents"), 0o700); err != nil {
|
||||
t.Fatalf("create stale agents directory: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(resultsDir, "agents", "old.log"), []byte("old success"), 0o600); err != nil {
|
||||
t.Fatalf("seed stale agent log: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user