From 64daa6e9aec87e727fb9a4c612a6588ee7ffac69 Mon Sep 17 00:00:00 2001 From: naiba Date: Mon, 20 Jul 2026 04:52:28 +0000 Subject: [PATCH] test(agentcompat): add scenario CLI wiring Co-authored-by: naiba/CloudCode --- .../agentcompat/artifact_publication_test.go | 163 +++++++++++++++ .../agentcompat/cmd/agentcompat/cli_config.go | 105 ++++++++++ .../cmd/agentcompat/dedicated_wiring_test.go | 152 ++++++++++++++ .../agentcompat/cmd/agentcompat/main.go | 59 ++++++ .../agentcompat/cmd/agentcompat/main_test.go | 192 ++++++++++++++++++ .../cmd/agentcompat/metadata_writer.go | 44 ++++ .../cmd/agentcompat/metadata_writer_test.go | 52 +++++ .../cmd/agentcompat/nat_wiring_test.go | 26 +++ .../agentcompat/private_artifact_writer.go | 61 ++++++ .../reconnect_dispatch_fixture_test.go | 101 +++++++++ .../cmd/agentcompat/scenario_dispatch_test.go | 113 +++++++++++ .../agentcompat/scenario_evidence_writer.go | 104 ++++++++++ .../cmd/agentcompat/scenario_execution.go | 115 +++++++++++ .../cmd/agentcompat/stale_artifact_test.go | 95 +++++++++ 14 files changed, 1382 insertions(+) create mode 100644 integration/agentcompat/cmd/agentcompat/artifact_publication_test.go create mode 100644 integration/agentcompat/cmd/agentcompat/cli_config.go create mode 100644 integration/agentcompat/cmd/agentcompat/dedicated_wiring_test.go create mode 100644 integration/agentcompat/cmd/agentcompat/main.go create mode 100644 integration/agentcompat/cmd/agentcompat/main_test.go create mode 100644 integration/agentcompat/cmd/agentcompat/metadata_writer.go create mode 100644 integration/agentcompat/cmd/agentcompat/metadata_writer_test.go create mode 100644 integration/agentcompat/cmd/agentcompat/nat_wiring_test.go create mode 100644 integration/agentcompat/cmd/agentcompat/private_artifact_writer.go create mode 100644 integration/agentcompat/cmd/agentcompat/reconnect_dispatch_fixture_test.go create mode 100644 integration/agentcompat/cmd/agentcompat/scenario_dispatch_test.go create mode 100644 integration/agentcompat/cmd/agentcompat/scenario_evidence_writer.go create mode 100644 integration/agentcompat/cmd/agentcompat/scenario_execution.go create mode 100644 integration/agentcompat/cmd/agentcompat/stale_artifact_test.go diff --git a/integration/agentcompat/cmd/agentcompat/artifact_publication_test.go b/integration/agentcompat/cmd/agentcompat/artifact_publication_test.go new file mode 100644 index 00000000..cc2501cc --- /dev/null +++ b/integration/agentcompat/cmd/agentcompat/artifact_publication_test.go @@ -0,0 +1,163 @@ +package main + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" + "github.com/nezhahq/nezha/integration/agentcompat/internal/evidence" + "github.com/nezhahq/nezha/integration/agentcompat/internal/scenario" +) + +func TestCLI_ArtifactPublicationReplacesPublicFileAndFinalSymlink(t *testing.T) { + tests := []struct { + name string + setup func(*testing.T, string, string) + }{ + {"public file", func(t *testing.T, path, _ string) { + if err := os.WriteFile(path, []byte("old"), 0o644); err != nil { + t.Fatalf("write old file: %v", err) + } + }}, + {"final symlink", func(t *testing.T, path, sentinel string) { + if err := os.Symlink(sentinel, path); err != nil { + t.Fatalf("symlink final path: %v", err) + } + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "cleanup.json") + sentinel := filepath.Join(t.TempDir(), "sentinel") + if err := os.WriteFile(sentinel, []byte("unchanged"), 0o600); err != nil { + t.Fatalf("write sentinel: %v", err) + } + test.setup(t, path, sentinel) + if err := writeJSONArtifact(dir, "cleanup.json", map[string]bool{"passed": true}); err != nil { + t.Fatalf("publish artifact: %v", err) + } + info, err := os.Lstat(path) + if err != nil { + t.Fatalf("lstat artifact: %v", err) + } + if !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 { + t.Fatalf("artifact mode=%v", info.Mode()) + } + content, err := os.ReadFile(sentinel) + if err != nil || string(content) != "unchanged" { + t.Fatalf("outside sentinel changed: content=%q err=%v", content, err) + } + }) + } +} + +func TestCLI_InterruptedScenarioPublicationLeavesAtomicInvalidDirectory(t *testing.T) { + dir := t.TempDir() + config := testCLIConfig(t, contract.ScenarioTransfer100MiB, contract.FaultTransferHash) + paths, err := contract.NewPaths(config.Paths.NezhaSource().String(), config.Paths.AgentSource().String(), dir) + if err != nil { + t.Fatalf("paths: %v", err) + } + config.Paths = paths + if err := writeMetadata(t.Context(), config, time.Now()); err != nil { + t.Fatalf("metadata: %v", err) + } + previous := scenarioArtifactPublished + scenarioArtifactPublished = func(name string) error { + if name == "results.json" { + return errors.New("injected publication interruption") + } + return nil + } + t.Cleanup(func() { scenarioArtifactPublished = previous }) + output := scenarioExecutionOutput{Result: scenario.Result{Name: contract.ScenarioTransfer100MiB, Passed: false, CleanupOK: true, Error: "transfer scenario: injected hash mismatch"}, Transfer: &scenario.TransferEvidence{WarmupUploadBytes: 65536, WarmupDownloadBytes: 65536, WarmupSHA256: "abc", WarmupDuration: time.Nanosecond, WarmupDeadlineRemaining: time.Second, WarmupQuiescent: true, OutsideRootSentinelsUnchanged: true}} + if err := writeScenarioEvidence(config, output, time.Now()); err == nil { + t.Fatal("publication interruption accepted") + } + if err := evidence.ValidateDirectory(dir); err == nil { + t.Fatal("partial evidence directory validated") + } + data, err := os.ReadFile(filepath.Join(dir, "results.json")) + if !errors.Is(err, os.ErrNotExist) || len(data) != 0 { + t.Fatalf("interrupted final file published: bytes=%d err=%v", len(data), err) + } +} + +func TestCLI_PrivateArtifactJoinsPrimaryAndCleanupErrors(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "results.json") + primaryErr := errors.New("publication hook failed") + removeErr := errors.New("temporary removal failed") + previousClose := privateArtifactClose + previousRemove := privateArtifactRemove + closeCalls := 0 + privateArtifactClose = func(file *os.File) error { + closeCalls++ + return file.Close() + } + privateArtifactRemove = func(string) error { return removeErr } + t.Cleanup(func() { + privateArtifactClose = previousClose + privateArtifactRemove = previousRemove + }) + + err := writePrivateArtifactWithSeam(path, []byte("payload"), func() error { return primaryErr }) + if !errors.Is(err, primaryErr) || !errors.Is(err, removeErr) { + t.Fatalf("publication error=%v, want primary and removal errors", err) + } + if errors.Is(err, os.ErrClosed) || closeCalls != 1 { + t.Fatalf("successful close repeated: calls=%d err=%v", closeCalls, err) + } +} + +func TestCLI_PrivateArtifactJoinsCloseAndRemoveErrors(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "results.json") + closeErr := errors.New("temporary close failed") + removeErr := errors.New("temporary removal failed") + previousClose := privateArtifactClose + previousRemove := privateArtifactRemove + closeCalls := 0 + privateArtifactClose = func(file *os.File) error { + closeCalls++ + if err := file.Close(); err != nil { + return err + } + return closeErr + } + privateArtifactRemove = func(string) error { return removeErr } + t.Cleanup(func() { + privateArtifactClose = previousClose + privateArtifactRemove = previousRemove + }) + + err := writePrivateArtifactWithSeam(path, []byte("payload"), func() error { return nil }) + if !errors.Is(err, closeErr) || !errors.Is(err, removeErr) { + t.Fatalf("publication error=%v, want close and removal errors", err) + } + if errors.Is(err, os.ErrClosed) || closeCalls != 1 { + t.Fatalf("close failure retried: calls=%d err=%v", closeCalls, err) + } +} + +func TestCLI_PrivateArtifactRemovesTemporaryFileAfterHookFailure(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "results.json") + err := writePrivateArtifactWithSeam(path, []byte("payload"), func() error { + return errors.New("publication hook failed") + }) + if err == nil { + t.Fatal("publication hook failure accepted") + } + entries, readErr := os.ReadDir(dir) + if readErr != nil { + t.Fatalf("read artifact directory: %v", readErr) + } + if len(entries) != 0 { + t.Fatalf("temporary artifact survived hook failure: %v", entries) + } +} diff --git a/integration/agentcompat/cmd/agentcompat/cli_config.go b/integration/agentcompat/cmd/agentcompat/cli_config.go new file mode 100644 index 00000000..29244c13 --- /dev/null +++ b/integration/agentcompat/cmd/agentcompat/cli_config.go @@ -0,0 +1,105 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" + "github.com/nezhahq/nezha/integration/agentcompat/internal/evidence" +) + +type scenarioFlags []contract.Scenario + +func (scenarios *scenarioFlags) String() string { + values := make([]string, 0, len(*scenarios)) + for _, scenario := range *scenarios { + values = append(values, scenario.String()) + } + return strings.Join(values, ",") +} + +func (scenarios *scenarioFlags) Set(value string) error { + scenario, err := contract.NewScenario(value) + if err != nil { + return err + } + *scenarios = append(*scenarios, scenario) + return nil +} + +type cliConfig struct { + Paths contract.Paths + Profile contract.Profile + Seed contract.Seed + Scenarios scenarioFlags + Fault contract.Fault +} + +func parseFlags(args []string, stderr io.Writer) (cliConfig, error) { + flags := flag.NewFlagSet("agentcompat", flag.ContinueOnError) + flags.SetOutput(io.Discard) + nezhaSource := flags.String("nezha-source", "", "Nezha source directory") + agentSource := flags.String("agent-source", "", "Agent source directory") + profileName := flags.String("profile", "", "compatibility profile") + resultsDir := flags.String("results-dir", "", "evidence output directory") + seedValue := flags.String("seed", "0x4e5a4841", "deterministic seed") + faultName := flags.String("fault", "", "named fault injection") + var scenarios scenarioFlags + flags.Var(&scenarios, "scenario", "run only a named scenario; omit for the complete profile") + if err := flags.Parse(args); err != nil { + return cliConfig{}, errors.New("invalid command-line arguments") + } + paths, err := contract.NewPaths(*nezhaSource, *agentSource, *resultsDir) + if err != nil { + return cliConfig{}, err + } + if err := prepareResultsDirBeforeParse(paths.ResultsDir().String()); err != nil { + return cliConfig{}, err + } + profile, err := contract.ProfileByName(*profileName) + if err != nil { + return cliConfig{}, err + } + seed, err := contract.ParseSeed(*seedValue) + if err != nil { + return cliConfig{}, err + } + fault := contract.Fault{} + if *faultName != "" { + fault, err = contract.NewFault(*faultName) + if err != nil { + return cliConfig{}, err + } + } + return cliConfig{Paths: paths, Profile: profile, Seed: seed, Scenarios: scenarios, Fault: fault}, nil +} + +func prepareResultsDir(resultsDir string) error { + return prepareEvidenceArtifacts(resultsDir, evidence.FixedEvidenceFiles()) +} + +func prepareResultsDirBeforeParse(resultsDir string) error { + return prepareEvidenceArtifacts(resultsDir, evidence.FixedEvidenceFiles()[1:]) +} + +func prepareEvidenceArtifacts(resultsDir string, artifactNames []string) error { + if info, err := os.Lstat(resultsDir); err == nil && info.Mode()&os.ModeSymlink != 0 { + return errors.New("results directory must not be a symbolic link") + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect results directory: %w", err) + } + for _, name := range artifactNames { + if err := os.Remove(filepath.Join(resultsDir, name)); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove previous artifact %s: %w", name, err) + } + } + if err := os.RemoveAll(filepath.Join(resultsDir, "agents")); err != nil { + return fmt.Errorf("remove previous agent logs: %w", err) + } + return nil +} diff --git a/integration/agentcompat/cmd/agentcompat/dedicated_wiring_test.go b/integration/agentcompat/cmd/agentcompat/dedicated_wiring_test.go new file mode 100644 index 00000000..3cdb13c7 --- /dev/null +++ b/integration/agentcompat/cmd/agentcompat/dedicated_wiring_test.go @@ -0,0 +1,152 @@ +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" + "github.com/nezhahq/nezha/integration/agentcompat/internal/scenario" +) + +func TestCLI_SelectsTransferAndReconnectWithTypedEvidence(t *testing.T) { + tests := []struct { + name string + fault string + }{ + {contract.ScenarioTransfer100MiB, ""}, + {contract.ScenarioTransfer100MiB, contract.FaultTransferHash}, + {contract.ScenarioReconnect, ""}, + {contract.ScenarioReconnect, contract.FaultDashboardExit}, + } + for _, test := range tests { + t.Run(test.name+"/"+test.fault, func(t *testing.T) { + config := testCLIConfig(t, test.name, test.fault) + runnerErr := errors.New("dedicated runner sentinel") + wantResult := scenario.Result{Name: test.name, Passed: false, CleanupOK: true, Error: runnerErr.Error(), Assertions: []scenario.Assertion{{Name: "typed dispatch", Passed: false, Details: test.fault}}} + wantTransfer := scenario.TransferEvidence{WarmupUploadBytes: 65536, WarmupDownloadBytes: 65536, WarmupSHA256: "warmup", WarmupDuration: time.Second, WarmupDeadlineRemaining: 2 * time.Second, WarmupQuiescent: true, UploadBytes: contract.TransferBytes, DownloadBytes: contract.TransferBytes, UploadSHA256: "transfer-hash", DownloadSHA256: "transfer-hash", UploadChunks: 3, DownloadChunks: 4, UploadDuration: 5 * time.Second, DownloadDuration: 6 * time.Second, RetainedHeapBytes: 7, Mode: "0640", CreateDirs: true, UploadReplayRejected: true, DownloadReplayRejected: true, OversizeRejected: true, OutsideRootSentinelsUnchanged: true} + wantReconnect := completeReconnectDispatchEvidence(t) + previousTransfer := runTransferScenario + previousReconnect := runReconnectScenario + var receivedTransfer *scenario.TransferInput + var receivedReconnect *scenario.ReconnectInput + runTransferScenario = func(_ context.Context, input scenario.TransferInput) (scenario.Result, scenario.TransferEvidence, error) { + receivedTransfer = &input + return wantResult, wantTransfer, runnerErr + } + runReconnectScenario = func(_ context.Context, input scenario.ReconnectInput) (scenario.Result, scenario.ReconnectEvidence, error) { + receivedReconnect = &input + return wantResult, wantReconnect, runnerErr + } + t.Cleanup(func() { + runTransferScenario = previousTransfer + runReconnectScenario = previousReconnect + }) + + execution, err := selectScenarioExecution(config) + if err != nil { + t.Fatalf("select execution: %v", err) + } + output, runErr := execution.run(context.Background()) + if !errors.Is(runErr, runnerErr) { + t.Fatalf("runner error=%v, want sentinel", runErr) + } + if err := output.Validate(); err != nil { + t.Fatalf("validate typed output: %v", err) + } + if !reflect.DeepEqual(output.Result, wantResult) { + t.Fatalf("result=%#v, want %#v", output.Result, wantResult) + } + if test.name == contract.ScenarioTransfer100MiB { + wantInput := scenario.TransferInput{Paths: config.Paths, Fault: config.Fault} + if receivedTransfer == nil || *receivedTransfer != wantInput || receivedReconnect != nil { + t.Fatalf("transfer inputs: received=%#v reconnect=%#v want=%#v", receivedTransfer, receivedReconnect, wantInput) + } + if output.Transfer == nil || !reflect.DeepEqual(*output.Transfer, wantTransfer) || output.Reconnect != nil { + t.Fatalf("transfer evidence=%#v reconnect=%#v", output.Transfer, output.Reconnect) + } + } else { + wantInput := scenario.ReconnectInput{Paths: config.Paths, DashboardFault: config.Fault.String()} + if receivedReconnect == nil || *receivedReconnect != wantInput || receivedTransfer != nil { + t.Fatalf("reconnect inputs: received=%#v transfer=%#v want=%#v", receivedReconnect, receivedTransfer, wantInput) + } + if output.Reconnect == nil || !reflect.DeepEqual(*output.Reconnect, wantReconnect) || output.Transfer != nil { + t.Fatalf("reconnect evidence=%#v transfer=%#v", output.Reconnect, output.Transfer) + } + } + }) + } +} + +func TestCLI_RejectsUnsupportedScenarioFaultPairsBeforeRunner(t *testing.T) { + tests := []struct{ scenario, fault string }{ + {contract.ScenarioTransfer100MiB, contract.FaultDashboardExit}, + {contract.ScenarioReconnect, contract.FaultTransferHash}, + {contract.ScenarioMCPFilesystem, contract.FaultAgentBadSecret}, + } + for _, test := range tests { + called := false + previousTransfer := runTransferScenario + runTransferScenario = func(context.Context, scenario.TransferInput) (scenario.Result, scenario.TransferEvidence, error) { + called = true + return scenario.Result{}, scenario.TransferEvidence{}, errors.New("unexpected") + } + _, err := selectScenarioExecution(testCLIConfig(t, test.scenario, test.fault)) + runTransferScenario = previousTransfer + if err == nil || called { + t.Fatalf("unsupported pair started runner: scenario=%q fault=%q err=%v", test.scenario, test.fault, err) + } + } +} + +func TestCLI_WritesDedicatedArtifactWithPrivateMode(t *testing.T) { + resultsDir := t.TempDir() + output := scenarioExecutionOutput{ + Result: scenario.Result{Name: contract.ScenarioTransfer100MiB, Passed: false, CleanupOK: true, Error: "transfer scenario: injected hash mismatch"}, + Transfer: &scenario.TransferEvidence{WarmupUploadBytes: 65536, WarmupDownloadBytes: 65536, WarmupSHA256: "abc", WarmupDuration: time.Nanosecond, WarmupDeadlineRemaining: time.Second, WarmupQuiescent: true, OutsideRootSentinelsUnchanged: true}, + } + config := testCLIConfig(t, contract.ScenarioTransfer100MiB, contract.FaultTransferHash) + paths, err := contract.NewPaths(config.Paths.NezhaSource().String(), config.Paths.AgentSource().String(), resultsDir) + if err != nil { + t.Fatalf("paths: %v", err) + } + config.Paths = paths + if err := writeScenarioEvidence(config, output, time.Now()); err != nil { + t.Fatalf("write evidence: %v", err) + } + info, err := os.Stat(filepath.Join(resultsDir, "transfer.json")) + if err != nil { + t.Fatalf("stat transfer evidence: %v", err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("transfer evidence mode=%o", info.Mode().Perm()) + } +} + +func testCLIConfig(t *testing.T, scenarioName, faultName string) cliConfig { + t.Helper() + paths, err := contract.NewPaths("/src/nezha", "/src/agent", t.TempDir()) + if err != nil { + t.Fatalf("paths: %v", err) + } + profile, err := contract.ProfileByName("pr-full") + if err != nil { + t.Fatalf("profile: %v", err) + } + scenarioValue, err := contract.NewScenario(scenarioName) + if err != nil { + t.Fatalf("scenario: %v", err) + } + fault := contract.Fault{} + if faultName != "" { + fault, err = contract.NewFault(faultName) + if err != nil { + t.Fatalf("fault: %v", err) + } + } + return cliConfig{Paths: paths, Profile: profile, Seed: contract.DefaultSeed, Scenarios: scenarioFlags{scenarioValue}, Fault: fault} +} diff --git a/integration/agentcompat/cmd/agentcompat/main.go b/integration/agentcompat/cmd/agentcompat/main.go new file mode 100644 index 00000000..658692c2 --- /dev/null +++ b/integration/agentcompat/cmd/agentcompat/main.go @@ -0,0 +1,59 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + "os/signal" + "syscall" + "time" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" + "github.com/nezhahq/nezha/integration/agentcompat/internal/evidence" +) + +func runContext(ctx context.Context, args []string, stdout io.Writer, stderr io.Writer, now time.Time) error { + config, err := parseFlags(args, stderr) + if err != nil { + return err + } + if err := writeMetadata(ctx, config, now); err != nil { + return err + } + if len(config.Scenarios) == 1 && config.Scenarios[0].String() == contract.ScenarioMetadata && config.Fault.IsZero() { + fmt.Fprintf(stdout, "metadata written for profile %s\n", config.Profile.Name()) + return nil + } + execution, err := selectScenarioExecution(config) + if err != nil { + return err + } + scenarioContext, cancel := context.WithTimeout(ctx, config.Profile.SuiteDeadline()) + defer cancel() + output, runErr := execution.run(scenarioContext) + if err := writeScenarioEvidence(config, output, now); err != nil { + return err + } + if err := evidence.ValidateDirectory(config.Paths.ResultsDir().String()); err != nil { + return fmt.Errorf("validate scenario evidence: %w", err) + } + if runErr != nil { + return runErr + } + fmt.Fprintf(stdout, "scenario %s passed for profile %s\n", execution.name, config.Profile.Name()) + return nil +} + +func run(args []string, stdout io.Writer, stderr io.Writer, now time.Time) error { + return runContext(context.Background(), args, stdout, stderr, now) +} + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + if err := runContext(ctx, os.Args[1:], os.Stdout, os.Stderr, time.Now().UTC()); err != nil { + fmt.Fprintf(os.Stderr, "agentcompat: %s\n", evidence.Redact(err.Error())) + os.Exit(2) + } +} diff --git a/integration/agentcompat/cmd/agentcompat/main_test.go b/integration/agentcompat/cmd/agentcompat/main_test.go new file mode 100644 index 00000000..f9dee82f --- /dev/null +++ b/integration/agentcompat/cmd/agentcompat/main_test.go @@ -0,0 +1,192 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/evidence" + "github.com/nezhahq/nezha/integration/agentcompat/internal/scenario" +) + +func TestCLI_ParsesTypedFlagsAndWritesMetadata(t *testing.T) { + resultsDir := t.TempDir() + var stdout, stderr bytes.Buffer + err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--seed", "0x4e5a4841", "--scenario", "metadata"}, &stdout, &stderr, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) + if err != nil { + t.Fatalf("run CLI: %v", err) + } + data, err := os.ReadFile(filepath.Join(resultsDir, "metadata.json")) + if err != nil { + t.Fatalf("read metadata: %v", err) + } + var metadata struct { + Profile struct { + Name string `json:"name"` + } `json:"profile"` + Seed string `json:"seed"` + } + if err := json.Unmarshal(data, &metadata); err != nil { + t.Fatalf("parse metadata: %v", err) + } + if metadata.Profile.Name != "pr-full" || metadata.Seed != "0x4e5a4841" || !strings.Contains(stdout.String(), "metadata written") { + t.Fatalf("unexpected metadata-only output: %#v %q", metadata, stdout.String()) + } +} + +func TestCLI_MetadataEvidenceValidatesAsCurrentMetadataProfile(t *testing.T) { + resultsDir := t.TempDir() + var stdout, stderr bytes.Buffer + err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "metadata"}, &stdout, &stderr, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) + if err != nil { + t.Fatalf("run metadata CLI: %v", err) + } + if err := evidence.ValidateDirectory(resultsDir); err != nil { + t.Fatalf("validate metadata evidence: %v", err) + } +} + +func TestCLI_RejectsInvalidProfileAndSeedWithoutSecretEcho(t *testing.T) { + for name, args := range map[string][]string{ + "profile": {"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "private-profile-secret", "--results-dir", t.TempDir()}, + "seed": {"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", t.TempDir(), "--seed", "not-a-seed-secret"}, + } { + t.Run(name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + err := run(args, &stdout, &stderr, time.Now()) + if err == nil { + t.Fatal("invalid CLI input accepted") + } + if strings.Contains(err.Error(), "not-a-seed-secret") || strings.Contains(err.Error(), "private-profile-secret") { + t.Fatal("invalid input echoed in error") + } + }) + } +} + +func TestCLI_RejectsMissingPaths(t *testing.T) { + var stdout, stderr bytes.Buffer + err := run([]string{"--profile", "pr-full", "--scenario", "metadata"}, &stdout, &stderr, time.Now()) + if err == nil || !strings.Contains(err.Error(), "--nezha-source") { + t.Fatalf("missing source paths were not rejected: %v", err) + } +} + +func TestCLI_ParsesRepeatableScenariosAndFault(t *testing.T) { + var stderr bytes.Buffer + config, err := parseFlags([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "soak", "--results-dir", "/tmp/results", "--scenario", "metadata", "--scenario", "transfer-100mib", "--fault", "transfer-hash"}, &stderr) + if err != nil { + t.Fatalf("parse flags: %v", err) + } + if len(config.Scenarios) != 2 || config.Scenarios[0].String() != "metadata" || config.Scenarios[1].String() != "transfer-100mib" || config.Fault.String() != "transfer-hash" { + t.Fatalf("unexpected typed flags: %#v", config) + } +} + +func TestCLI_WritesMetadataBeforeRejectingUnsupportedRuntime(t *testing.T) { + resultsDir := t.TempDir() + var stdout, stderr bytes.Buffer + err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "future-scenario"}, &stdout, &stderr, time.Now()) + if err == nil { + t.Fatal("unimplemented runtime reported success") + } + if _, statErr := os.Stat(filepath.Join(resultsDir, "metadata.json")); statErr != nil { + t.Fatalf("metadata was not written before runtime rejection: %v", statErr) + } +} + +func TestCLI_RecognizesOnlyMCPFilesystemRuntimeScenario(t *testing.T) { + var stderr bytes.Buffer + config, err := parseFlags([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", t.TempDir(), "--scenario", "mcp-filesystem"}, &stderr) + if err != nil { + t.Fatalf("parse mcp-filesystem flags: %v", err) + } + execution, err := selectScenarioExecution(config) + if err != nil || execution.name != "mcp-filesystem" { + t.Fatalf("mcp-filesystem runtime registration: name=%q err=%v", execution.name, err) + } + + config.Scenarios = append(config.Scenarios, config.Scenarios[0]) + if _, err := selectScenarioExecution(config); err == nil || !strings.Contains(err.Error(), "exactly one") { + t.Fatalf("multi-scenario runtime was unexpectedly accepted: %v", err) + } +} + +func TestCLI_SelectsTerminalRuntimeScenario(t *testing.T) { + var stderr bytes.Buffer + config, err := parseFlags([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", t.TempDir(), "--scenario", "terminal"}, &stderr) + if err != nil { + t.Fatalf("parse terminal flags: %v", err) + } + previous := runTerminalScenario + var received scenario.TerminalInput + runTerminalScenario = func(_ context.Context, input scenario.TerminalInput) (scenario.Result, error) { + received = input + return scenario.Result{Name: "terminal", Passed: true}, nil + } + t.Cleanup(func() { runTerminalScenario = previous }) + + execution, err := selectScenarioExecution(config) + + if err != nil { + t.Fatalf("select terminal scenario: %v", err) + } + if execution.name != "terminal" || execution.run == nil { + t.Fatalf("unexpected terminal execution: %#v", execution) + } + if _, err := execution.run(context.Background()); err != nil { + t.Fatalf("run terminal execution: %v", err) + } + if received.Paths.NezhaSource().String() != "/src/nezha" || received.Paths.AgentSource().String() != "/src/agent" || !received.Fault.IsZero() { + t.Fatalf("terminal input was not forwarded: %#v", received) + } +} + +func TestCLI_MetadataWriteReplacesSymlinkWithoutChangingTarget(t *testing.T) { + resultsDir := t.TempDir() + target := filepath.Join(t.TempDir(), "target") + if err := os.WriteFile(target, []byte("sentinel"), 0o644); err != nil { + t.Fatalf("write target: %v", err) + } + if err := os.Symlink(target, filepath.Join(resultsDir, "metadata.json")); err != nil { + t.Fatalf("create metadata symlink: %v", err) + } + var stdout, stderr bytes.Buffer + err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "metadata"}, &stdout, &stderr, time.Now()) + if err != nil { + t.Fatalf("replace metadata symlink: %v", err) + } + data, readErr := os.ReadFile(target) + if readErr != nil || string(data) != "sentinel" { + t.Fatalf("symlink target changed: %q %v", data, readErr) + } + info, statErr := os.Lstat(filepath.Join(resultsDir, "metadata.json")) + if statErr != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 { + t.Fatalf("metadata replacement mode=%v err=%v", info, statErr) + } +} + +func TestCLI_MetadataWriteUsesPrivatePermissions(t *testing.T) { + root := t.TempDir() + resultsDir := filepath.Join(root, "results") + var stdout, stderr bytes.Buffer + if err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "metadata"}, &stdout, &stderr, time.Now()); err != nil { + t.Fatalf("write metadata: %v", err) + } + directoryInfo, err := os.Stat(resultsDir) + if err != nil { + t.Fatalf("stat results directory: %v", err) + } + fileInfo, err := os.Stat(filepath.Join(resultsDir, "metadata.json")) + if err != nil { + t.Fatalf("stat metadata: %v", err) + } + if directoryInfo.Mode().Perm() != 0o700 || fileInfo.Mode().Perm() != 0o600 { + t.Fatalf("unexpected permissions: directory=%o file=%o", directoryInfo.Mode().Perm(), fileInfo.Mode().Perm()) + } +} diff --git a/integration/agentcompat/cmd/agentcompat/metadata_writer.go b/integration/agentcompat/cmd/agentcompat/metadata_writer.go new file mode 100644 index 00000000..a9bbf142 --- /dev/null +++ b/integration/agentcompat/cmd/agentcompat/metadata_writer.go @@ -0,0 +1,44 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" + "github.com/nezhahq/nezha/integration/agentcompat/internal/evidence" +) + +var metadataArtifactReady = func(context.Context) error { return nil } + +func writeMetadata(ctx context.Context, config cliConfig, now time.Time) error { + resultsDir := config.Paths.ResultsDir().String() + if info, err := os.Lstat(resultsDir); err == nil && info.Mode()&os.ModeSymlink != 0 { + return errors.New("results directory must not be a symbolic link") + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect results directory: %w", err) + } + if err := os.MkdirAll(resultsDir, 0o700); err != nil { + return fmt.Errorf("create results directory: %w", err) + } + if err := os.Chmod(resultsDir, 0o700); err != nil { + return fmt.Errorf("secure results directory: %w", err) + } + if err := prepareResultsDir(resultsDir); err != nil { + return err + } + metadata, err := evidence.NewMetadata(evidence.MetadataInput{Profile: config.Profile, Seed: config.Seed, Paths: config.Paths, ResourceBudget: contract.DefaultResourceBudget(), Scenarios: config.Scenarios, Fault: config.Fault, StartedAt: now, EvidenceFiles: evidence.EvidenceFiles()}) + if err != nil { + return fmt.Errorf("build metadata: %w", err) + } + data, err := json.MarshalIndent(metadata, "", " ") + if err != nil { + return fmt.Errorf("marshal metadata: %w", err) + } + path := filepath.Join(resultsDir, "metadata.json") + return writePrivateArtifactWithSeam(path, data, func() error { return metadataArtifactReady(ctx) }) +} diff --git a/integration/agentcompat/cmd/agentcompat/metadata_writer_test.go b/integration/agentcompat/cmd/agentcompat/metadata_writer_test.go new file mode 100644 index 00000000..9b047b68 --- /dev/null +++ b/integration/agentcompat/cmd/agentcompat/metadata_writer_test.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" +) + +func TestCLI_MetadataCancellationRemovesTemporaryArtifact(t *testing.T) { + resultsDir := t.TempDir() + config := testCLIConfig(t, contract.ScenarioMetadata, "") + paths, err := contract.NewPaths(config.Paths.NezhaSource().String(), config.Paths.AgentSource().String(), resultsDir) + if err != nil { + t.Fatalf("paths: %v", err) + } + config.Paths = paths + ready := make(chan struct{}) + previous := metadataArtifactReady + metadataArtifactReady = func(ctx context.Context) error { + close(ready) + <-ctx.Done() + return ctx.Err() + } + t.Cleanup(func() { metadataArtifactReady = previous }) + ctx, cancel := context.WithCancel(context.Background()) + writeDone := make(chan error, 1) + go func() { writeDone <- writeMetadata(ctx, config, time.Now()) }() + <-ready + cancel() + + if err := <-writeDone; !errors.Is(err, context.Canceled) { + t.Fatalf("metadata cancellation error=%v, want context canceled", err) + } + if _, err := os.Stat(filepath.Join(resultsDir, "metadata.json")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("metadata final file exists after cancellation: %v", err) + } + entries, err := os.ReadDir(resultsDir) + if err != nil { + t.Fatalf("read results directory: %v", err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), ".artifact-") { + t.Fatalf("metadata temporary artifact survived cancellation: %s", entry.Name()) + } + } +} diff --git a/integration/agentcompat/cmd/agentcompat/nat_wiring_test.go b/integration/agentcompat/cmd/agentcompat/nat_wiring_test.go new file mode 100644 index 00000000..9b78d233 --- /dev/null +++ b/integration/agentcompat/cmd/agentcompat/nat_wiring_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "bytes" + "testing" +) + +func TestCLI_SelectsNATRuntimeScenario(t *testing.T) { + // Given + var stderr bytes.Buffer + config, err := parseFlags([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", t.TempDir(), "--scenario", "nat"}, &stderr) + if err != nil { + t.Fatalf("parse NAT flags: %v", err) + } + + // When + execution, err := selectScenarioExecution(config) + + // Then + if err != nil { + t.Fatalf("select NAT scenario: %v", err) + } + if execution.name != "nat" || execution.run == nil { + t.Fatalf("unexpected NAT execution: %#v", execution) + } +} diff --git a/integration/agentcompat/cmd/agentcompat/private_artifact_writer.go b/integration/agentcompat/cmd/agentcompat/private_artifact_writer.go new file mode 100644 index 00000000..264b34c7 --- /dev/null +++ b/integration/agentcompat/cmd/agentcompat/private_artifact_writer.go @@ -0,0 +1,61 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +var ( + scenarioArtifactPublished = func(string) error { return nil } + privateArtifactClose = (*os.File).Close + privateArtifactRemove = os.Remove +) + +func writePrivateArtifact(path string, data []byte) (err error) { + return writePrivateArtifactWithSeam(path, data, func() error { + return scenarioArtifactPublished(filepath.Base(path)) + }) +} + +func writePrivateArtifactWithSeam(path string, data []byte, beforeRename func() error) (err error) { + directory := filepath.Dir(path) + temporary, err := os.CreateTemp(directory, ".artifact-*") + if err != nil { + return fmt.Errorf("create temporary artifact: %w", err) + } + temporaryPath := temporary.Name() + committed := false + closed := false + defer func() { + if !committed { + if !closed { + err = errors.Join(err, privateArtifactClose(temporary)) + } + err = errors.Join(err, privateArtifactRemove(temporaryPath)) + } + }() + if err := temporary.Chmod(0o600); err != nil { + return fmt.Errorf("secure temporary artifact: %w", err) + } + if _, err := temporary.Write(append(data, '\n')); err != nil { + return fmt.Errorf("write temporary artifact: %w", err) + } + if err := temporary.Sync(); err != nil { + return fmt.Errorf("sync temporary artifact: %w", err) + } + closeErr := privateArtifactClose(temporary) + closed = true + if closeErr != nil { + return fmt.Errorf("close temporary artifact: %w", closeErr) + } + if err := beforeRename(); err != nil { + return err + } + if err := os.Rename(temporaryPath, path); err != nil { + return fmt.Errorf("rename temporary artifact: %w", err) + } + committed = true + return nil +} diff --git a/integration/agentcompat/cmd/agentcompat/reconnect_dispatch_fixture_test.go b/integration/agentcompat/cmd/agentcompat/reconnect_dispatch_fixture_test.go new file mode 100644 index 00000000..fd356324 --- /dev/null +++ b/integration/agentcompat/cmd/agentcompat/reconnect_dispatch_fixture_test.go @@ -0,0 +1,101 @@ +package main + +import ( + "testing" + "time" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/agent" + "github.com/nezhahq/nezha/integration/agentcompat/internal/dashboard" + processharness "github.com/nezhahq/nezha/integration/agentcompat/internal/process" + "github.com/nezhahq/nezha/integration/agentcompat/internal/scenario" + "github.com/nezhahq/nezha/integration/agentcompat/internal/workspace" +) + +func completeReconnectDispatchEvidence(t *testing.T) scenario.ReconnectEvidence { + t.Helper() + disconnectAt := time.Date(2026, 2, 3, 4, 5, 6, 700, time.UTC) + reconnectAt := disconnectAt.Add(7 * time.Second) + dashboardReceipt := dashboard.MCPReceiptPair{ + Task: dashboard.MCPReceiptEvent{Sequence: 51, DashboardGeneration: 12, GateGeneration: 61, ServerID: 41, TaskID: 71, TaskType: 81, Kind: dashboard.MCPReceiptTask}, + Result: dashboard.MCPReceiptEvent{Sequence: 52, DashboardGeneration: 12, GateGeneration: 61, ServerID: 41, TaskID: 71, TaskType: 81, Kind: dashboard.MCPReceiptResult}, + } + agentReceipt := dashboard.MCPReceiptPair{ + Task: dashboard.MCPReceiptEvent{Sequence: 53, DashboardGeneration: 12, GateGeneration: 62, ServerID: 41, TaskID: 72, TaskType: 82, Kind: dashboard.MCPReceiptTask}, + Result: dashboard.MCPReceiptEvent{Sequence: 54, DashboardGeneration: 12, GateGeneration: 62, ServerID: 41, TaskID: 72, TaskType: 82, Kind: dashboard.MCPReceiptResult}, + } + evidence := scenario.ReconnectEvidence{ + Fixture: scenario.ReconnectFixtureEvidence{ + Dashboard: dashboard.FixtureIdentity{ + WorkspaceRoot: "/typed/dashboard-workspace", ConfigPath: "/typed/dashboard.yaml", DatabasePath: "/typed/dashboard.sqlite", BinaryPath: "/typed/dashboard", + HTTP: workspace.ListenerIdentity{Address: "127.0.0.1:41001", Inode: 1101}, Receipt: workspace.ListenerIdentity{Address: "127.0.0.1:41002", Inode: 1102}, HTTPS: workspace.ListenerIdentity{Address: "127.0.0.1:41003", Inode: 1103}, + }, + AgentRoot: "/typed/agent-workspace", AgentConfigPath: "/typed/agent.yaml", AgentBinaryPath: "/typed/agent", + }, + Runtime: scenario.ReconnectRuntimeEvidence{ + DashboardBefore: dashboard.RuntimeIdentity{Generation: 11, PID: 2101, ProcessGroupID: 3101}, DashboardAfter: dashboard.RuntimeIdentity{Generation: 12, PID: 2102, ProcessGroupID: 3102}, + AgentBefore: agent.ProcessIdentity{Generation: 21, PID: 2201, ProcessGroupID: 3201}, AgentAfter: agent.ProcessIdentity{Generation: 22, PID: 2202, ProcessGroupID: 3202}, + StateGenerationBeforeAgentRestart: 31, StateGenerationAfterAgentRestart: 32, + }, + Identity: scenario.ReconnectIdentityEvidence{ServerID: 41, UUID: "00000000-0000-0000-0000-000000000041", DashboardConfigUnchanged: true, AgentConfigUnchanged: true, DashboardFixtureUnchanged: true, ClientsRecreated: true, BootstrapRecreated: true}, + Lifecycle: scenario.ReconnectLifecycleEvidence{ + DisconnectAt: disconnectAt, ReconnectAt: reconnectAt, ReconnectInterval: 7 * time.Second, + DashboardReceipts: []dashboard.MCPReceiptPair{dashboardReceipt}, AgentReceipts: []dashboard.MCPReceiptPair{agentReceipt}, + StaleGenerationReceipts: 0, DuplicateTaskIDs: 0, LostResultIDs: 0, OutsideRootSentinelUnchanged: true, + }, + Observation: scenario.ReconnectObservation{ServerID: 41, UUID: "00000000-0000-0000-0000-000000000041", OldGeneration: 11, NewGeneration: 12, DisconnectAt: disconnectAt, ReconnectAt: reconnectAt, TaskIDs: []uint64{71, 72}, ResultIDs: []uint64{71, 72}, PostReconnect: true, AgentRestarted: true}, + AgentCleanup: processharness.CleanupReceipt{Passed: true, Forced: false, Processes: []processharness.CleanupRecord{{Name: "agent-generation-21", PID: 2201, Forced: false, Error: ""}, {Name: "agent-generation-22", PID: 2202, Forced: false, Error: ""}}}, + DashboardCleanup: processharness.CleanupReceipt{Passed: true, Forced: false, Processes: []processharness.CleanupRecord{{Name: "dashboard-generation-11", PID: 2101, Forced: false, Error: ""}, {Name: "dashboard-generation-12", PID: 2102, Forced: false, Error: ""}}}, + } + assertCompleteReconnectDispatchEvidence(t, evidence) + return evidence +} + +func assertCompleteReconnectDispatchEvidence(t *testing.T, evidence scenario.ReconnectEvidence) { + t.Helper() + if err := evidence.Validate(); err != nil { + t.Fatalf("incomplete reconnect dispatch fixture: %v", err) + } + fixture := evidence.Fixture + if fixture.Dashboard.WorkspaceRoot == "" || fixture.Dashboard.ConfigPath == "" || fixture.Dashboard.DatabasePath == "" || fixture.Dashboard.BinaryPath == "" || fixture.AgentRoot == "" || fixture.AgentConfigPath == "" || fixture.AgentBinaryPath == "" { + t.Fatal("incomplete reconnect dispatch fixture paths") + } + for name, listener := range map[string]struct { + address string + inode uint64 + }{ + "http": {fixture.Dashboard.HTTP.Address, fixture.Dashboard.HTTP.Inode}, + "receipt": {fixture.Dashboard.Receipt.Address, fixture.Dashboard.Receipt.Inode}, + "https": {fixture.Dashboard.HTTPS.Address, fixture.Dashboard.HTTPS.Inode}, + } { + if listener.address == "" || listener.inode == 0 { + t.Fatalf("incomplete reconnect dispatch %s listener", name) + } + } + if len(evidence.Lifecycle.DashboardReceipts) == 0 || len(evidence.Lifecycle.AgentReceipts) == 0 { + t.Fatal("incomplete reconnect dispatch receipt pairs") + } + for _, pairs := range [][]dashboard.MCPReceiptPair{evidence.Lifecycle.DashboardReceipts, evidence.Lifecycle.AgentReceipts} { + for _, pair := range pairs { + if pair.Task.Sequence == 0 || pair.Task.DashboardGeneration == 0 || pair.Task.GateGeneration == 0 || pair.Task.ServerID == 0 || pair.Task.TaskID == 0 || pair.Task.TaskType == 0 || pair.Task.Kind != dashboard.MCPReceiptTask { + t.Fatalf("incomplete reconnect dispatch task receipt: %#v", pair.Task) + } + if pair.Result.Sequence == 0 || pair.Result.DashboardGeneration == 0 || pair.Result.GateGeneration == 0 || pair.Result.ServerID == 0 || pair.Result.TaskID == 0 || pair.Result.TaskType == 0 || pair.Result.Kind != dashboard.MCPReceiptResult { + t.Fatalf("incomplete reconnect dispatch result receipt: %#v", pair.Result) + } + } + } + assertCompleteCleanupReceipt(t, "agent", evidence.AgentCleanup) + assertCompleteCleanupReceipt(t, "dashboard", evidence.DashboardCleanup) +} + +func assertCompleteCleanupReceipt(t *testing.T, name string, receipt processharness.CleanupReceipt) { + t.Helper() + if !receipt.Passed || receipt.Forced || len(receipt.Processes) == 0 { + t.Fatalf("incomplete reconnect dispatch %s cleanup receipt: %#v", name, receipt) + } + for _, record := range receipt.Processes { + if record.Name == "" || record.PID == 0 || record.Forced || record.Error != "" { + t.Fatalf("incomplete reconnect dispatch %s cleanup record: %#v", name, record) + } + } +} diff --git a/integration/agentcompat/cmd/agentcompat/scenario_dispatch_test.go b/integration/agentcompat/cmd/agentcompat/scenario_dispatch_test.go new file mode 100644 index 00000000..9a43d2de --- /dev/null +++ b/integration/agentcompat/cmd/agentcompat/scenario_dispatch_test.go @@ -0,0 +1,113 @@ +package main + +import ( + "context" + "errors" + "testing" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" + "github.com/nezhahq/nezha/integration/agentcompat/internal/scenario" +) + +func TestCLI_RegisteredScenariosHaveExhaustiveRuntimeRouting(t *testing.T) { + for _, definition := range contract.ScenarioDefinitions() { + t.Run(definition.Name, func(t *testing.T) { + config := testCLIConfig(t, definition.Name, "") + execution, err := selectScenarioExecution(config) + if definition.Execution == contract.ScenarioExecutionMetadata { + if err == nil { + t.Fatal("metadata unexpectedly received runtime execution") + } + return + } + if err != nil { + t.Fatalf("select registered scenario: %v", err) + } + if execution.name != definition.Name || execution.run == nil { + t.Fatalf("incomplete runtime execution: %#v", execution) + } + }) + } +} + +func TestCLI_Todos11To15DispatchPropagatesTypedInputsErrorsAndOutputs(t *testing.T) { + runnerError := errors.New("injected runner error") + tests := []struct { + name string + fault string + set func(*testing.T, scenario.Result, error, *bool) + }{ + {contract.ScenarioRegistrationConfigExec, contract.FaultAgentBadSecret, func(t *testing.T, want scenario.Result, wantErr error, called *bool) { + previous := runRegistrationConfigExecScenario + runRegistrationConfigExecScenario = func(_ context.Context, input scenario.RegistrationConfigExecInput) (scenario.Result, error) { + *called = true + assertPathsAndFault(t, input.Paths, input.Fault, contract.FaultAgentBadSecret) + return want, wantErr + } + t.Cleanup(func() { runRegistrationConfigExecScenario = previous }) + }}, + {contract.ScenarioNAT, "", func(t *testing.T, want scenario.Result, wantErr error, called *bool) { + previous := runNATScenario + runNATScenario = func(_ context.Context, input scenario.NATInput) (scenario.Result, error) { + *called = true + assertPathsAndFault(t, input.Paths, input.Fault, "") + return want, wantErr + } + t.Cleanup(func() { runNATScenario = previous }) + }}, + {contract.ScenarioLegacyFM, contract.FaultAgentBadSecret, func(t *testing.T, want scenario.Result, wantErr error, called *bool) { + previous := runLegacyFMScenario + runLegacyFMScenario = func(_ context.Context, input scenario.LegacyFMInput) (scenario.Result, error) { + *called = true + assertPathsAndFault(t, input.Paths, input.Fault, contract.FaultAgentBadSecret) + return want, wantErr + } + t.Cleanup(func() { runLegacyFMScenario = previous }) + }}, + {contract.ScenarioTerminal, "", func(t *testing.T, want scenario.Result, wantErr error, called *bool) { + previous := runTerminalScenario + runTerminalScenario = func(_ context.Context, input scenario.TerminalInput) (scenario.Result, error) { + *called = true + assertPathsAndFault(t, input.Paths, input.Fault, "") + return want, wantErr + } + t.Cleanup(func() { runTerminalScenario = previous }) + }}, + {contract.ScenarioMCPFilesystem, "", func(t *testing.T, want scenario.Result, wantErr error, called *bool) { + previous := runMCPFilesystemScenario + runMCPFilesystemScenario = func(_ context.Context, input scenario.MCPFilesystemInput) (scenario.Result, error) { + *called = true + if input.Paths.NezhaSource().String() != "/src/nezha" || input.Paths.AgentSource().String() != "/src/agent" { + t.Fatalf("MCP filesystem paths=%#v", input.Paths) + } + return want, wantErr + } + t.Cleanup(func() { runMCPFilesystemScenario = previous }) + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + want := scenario.Result{Name: test.name, Passed: false, Assertions: []scenario.Assertion{{Name: "runner assertion", Passed: false}}, Error: runnerError.Error()} + called := false + test.set(t, want, runnerError, &called) + execution, err := selectScenarioExecution(testCLIConfig(t, test.name, test.fault)) + if err != nil { + t.Fatalf("select execution: %v", err) + } + output, err := execution.run(t.Context()) + if !errors.Is(err, runnerError) { + t.Fatalf("runner error=%v", err) + } + if !called || output.Result.Name != want.Name || output.Result.Error != want.Error || output.Transfer != nil || output.Reconnect != nil { + t.Fatalf("runner propagation called=%t output=%#v", called, output) + } + }) + } +} + +func assertPathsAndFault(t *testing.T, paths contract.Paths, fault contract.Fault, wantFault string) { + t.Helper() + if paths.NezhaSource().String() != "/src/nezha" || paths.AgentSource().String() != "/src/agent" || fault.String() != wantFault { + t.Fatalf("paths/fault propagation paths=%#v fault=%q", paths, fault.String()) + } +} diff --git a/integration/agentcompat/cmd/agentcompat/scenario_evidence_writer.go b/integration/agentcompat/cmd/agentcompat/scenario_evidence_writer.go new file mode 100644 index 00000000..8fc7ede6 --- /dev/null +++ b/integration/agentcompat/cmd/agentcompat/scenario_evidence_writer.go @@ -0,0 +1,104 @@ +package main + +import ( + "encoding/json" + "fmt" + "path/filepath" + "time" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" + "github.com/nezhahq/nezha/integration/agentcompat/internal/evidence" + "github.com/nezhahq/nezha/integration/agentcompat/internal/scenario" +) + +type transferArtifact struct { + Scenario string `json:"scenario"` + Fault string `json:"fault,omitempty"` + Passed bool `json:"passed"` + CleanupOK bool `json:"cleanup_ok"` + Error string `json:"error,omitempty"` + Evidence scenario.TransferEvidence `json:"evidence"` +} + +type reconnectArtifact struct { + Scenario string `json:"scenario"` + Fault string `json:"fault,omitempty"` + Passed bool `json:"passed"` + CleanupOK bool `json:"cleanup_ok"` + Error string `json:"error,omitempty"` + Evidence scenario.ReconnectEvidence `json:"evidence"` +} + +func writeScenarioEvidence(config cliConfig, output scenarioExecutionOutput, now time.Time) error { + if err := output.Validate(); err != nil { + return fmt.Errorf("validate scenario execution output: %w", err) + } + result := output.Result + if !result.Passed && config.Fault.String() != "" && allAssertionsPassed(result.Assertions) { + result.Assertions = append(result.Assertions, scenario.Assertion{Name: contract.AssertionInjectedFault, Passed: false, Details: result.Error}) + } + if output.Transfer != nil { + artifact := transferArtifact{Scenario: result.Name, Fault: config.Fault.String(), Passed: result.Passed, CleanupOK: result.CleanupOK, Error: evidence.Redact(result.Error), Evidence: *output.Transfer} + if err := writeJSONArtifact(config.Paths.ResultsDir().String(), "transfer.json", artifact); err != nil { + return err + } + } + if output.Reconnect != nil { + artifact := reconnectArtifact{Scenario: result.Name, Fault: config.Fault.String(), Passed: result.Passed, CleanupOK: result.CleanupOK, Error: evidence.Redact(result.Error), Evidence: *output.Reconnect} + if err := writeJSONArtifact(config.Paths.ResultsDir().String(), "reconnect.json", artifact); err != nil { + return err + } + } + assertions := make([]evidence.Assertion, 0, len(result.Assertions)) + for _, assertion := range result.Assertions { + assertions = append(assertions, evidence.Assertion{Name: assertion.Name, Passed: assertion.Passed, Details: assertion.Details}) + } + results := evidence.Results{Profile: string(config.Profile.Name()), Passed: result.Passed, Scenarios: []evidence.ScenarioResult{{Name: result.Name, Passed: result.Passed, Assertions: assertions, Error: result.Error}}} + data, err := evidence.MarshalResults(results) + if err != nil { + return fmt.Errorf("marshal scenario results: %w", err) + } + if err := writePrivateFile(filepath.Join(config.Paths.ResultsDir().String(), "results.json"), data); err != nil { + return fmt.Errorf("write scenario results: %w", err) + } + junit, err := evidence.JUnit(results) + if err != nil { + return fmt.Errorf("marshal scenario junit: %w", err) + } + if err := writePrivateFile(filepath.Join(config.Paths.ResultsDir().String(), "junit.xml"), junit); err != nil { + return fmt.Errorf("write scenario junit: %w", err) + } + cleanup := struct { + Passed bool `json:"passed"` + Scenario string `json:"scenario"` + FinishedAt string `json:"finished_at"` + }{Passed: result.CleanupOK, Scenario: result.Name, FinishedAt: now.UTC().Format(time.RFC3339)} + return writeJSONArtifact(config.Paths.ResultsDir().String(), "cleanup.json", cleanup) +} + +func writeJSONArtifact(resultsDir, name string, artifact any) error { + data, err := json.MarshalIndent(artifact, "", " ") + if err != nil { + return fmt.Errorf("marshal %s: %w", name, err) + } + if redacted := evidence.Redact(string(data)); redacted != string(data) { + return fmt.Errorf("credential detected while marshaling %s", name) + } + if err := writePrivateFile(filepath.Join(resultsDir, name), data); err != nil { + return fmt.Errorf("write %s: %w", name, err) + } + return nil +} + +func writePrivateFile(path string, data []byte) error { + return writePrivateArtifact(path, data) +} + +func allAssertionsPassed(assertions []scenario.Assertion) bool { + for _, assertion := range assertions { + if !assertion.Passed { + return false + } + } + return true +} diff --git a/integration/agentcompat/cmd/agentcompat/scenario_execution.go b/integration/agentcompat/cmd/agentcompat/scenario_execution.go new file mode 100644 index 00000000..4c1e5e18 --- /dev/null +++ b/integration/agentcompat/cmd/agentcompat/scenario_execution.go @@ -0,0 +1,115 @@ +package main + +import ( + "context" + "errors" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" + "github.com/nezhahq/nezha/integration/agentcompat/internal/scenario" +) + +type scenarioExecutionOutput struct { + Result scenario.Result + Transfer *scenario.TransferEvidence + Reconnect *scenario.ReconnectEvidence +} + +func (output scenarioExecutionOutput) Validate() error { + if output.Result.Name == "" { + return errors.New("scenario execution result is missing") + } + dedicatedCount := 0 + if output.Transfer != nil { + dedicatedCount++ + } + if output.Reconnect != nil { + dedicatedCount++ + } + switch output.Result.Name { + case contract.ScenarioTransfer100MiB: + if dedicatedCount != 1 || output.Transfer == nil { + return errors.New("transfer execution requires only transfer evidence") + } + case contract.ScenarioReconnect: + if dedicatedCount != 1 || output.Reconnect == nil { + return errors.New("reconnect execution requires only reconnect evidence") + } + default: + if dedicatedCount != 0 { + return errors.New("scenario execution has mismatched dedicated evidence") + } + } + return nil +} + +type scenarioExecution struct { + name string + run func(context.Context) (scenarioExecutionOutput, error) +} + +var ( + runRegistrationConfigExecScenario = (scenario.RegistrationConfigExec{}).Run + runNATScenario = (scenario.NAT{}).Run + runLegacyFMScenario = (scenario.LegacyFM{}).Run + runTerminalScenario = (scenario.Terminal{}).Run + runMCPFilesystemScenario = (scenario.MCPFilesystem{}).Run + runTransferScenario = (scenario.Transfer{}).RunWithEvidence + runReconnectScenario = (scenario.Reconnect{}).RunWithEvidence +) + +func selectScenarioExecution(config cliConfig) (scenarioExecution, error) { + if len(config.Scenarios) != 1 { + return scenarioExecution{}, errors.New("runtime execution requires exactly one --scenario") + } + selected := config.Scenarios[0] + if err := contract.ValidateScenarioFault(selected, config.Fault); err != nil { + return scenarioExecution{}, err + } + definition, err := contract.ScenarioDefinitionByName(selected.String()) + if err != nil { + return scenarioExecution{}, err + } + switch definition.Execution { + case contract.ScenarioExecutionMetadata: + return scenarioExecution{}, errors.New("metadata scenario does not have a runtime execution") + case contract.ScenarioExecutionRegistrationConfigExec: + return standardExecution(selected.String(), func(ctx context.Context) (scenario.Result, error) { + return runRegistrationConfigExecScenario(ctx, scenario.RegistrationConfigExecInput{Paths: config.Paths, Fault: config.Fault}) + }), nil + case contract.ScenarioExecutionNAT: + return standardExecution(selected.String(), func(ctx context.Context) (scenario.Result, error) { + return runNATScenario(ctx, scenario.NATInput{Paths: config.Paths, Fault: config.Fault}) + }), nil + case contract.ScenarioExecutionLegacyFM: + return standardExecution(selected.String(), func(ctx context.Context) (scenario.Result, error) { + return runLegacyFMScenario(ctx, scenario.LegacyFMInput{Paths: config.Paths, Fault: config.Fault}) + }), nil + case contract.ScenarioExecutionTerminal: + return standardExecution(selected.String(), func(ctx context.Context) (scenario.Result, error) { + return runTerminalScenario(ctx, scenario.TerminalInput{Paths: config.Paths, Fault: config.Fault}) + }), nil + case contract.ScenarioExecutionMCPFilesystem: + return standardExecution(selected.String(), func(ctx context.Context) (scenario.Result, error) { + return runMCPFilesystemScenario(ctx, scenario.MCPFilesystemInput{Paths: config.Paths}) + }), nil + case contract.ScenarioExecutionTransfer: + return scenarioExecution{name: selected.String(), run: func(ctx context.Context) (scenarioExecutionOutput, error) { + result, transferEvidence, err := runTransferScenario(ctx, scenario.TransferInput{Paths: config.Paths, Fault: config.Fault}) + return scenarioExecutionOutput{Result: result, Transfer: &transferEvidence}, err + }}, nil + case contract.ScenarioExecutionReconnect: + return scenarioExecution{name: selected.String(), run: func(ctx context.Context) (scenarioExecutionOutput, error) { + result, reconnectEvidence, err := runReconnectScenario(ctx, scenario.ReconnectInput{Paths: config.Paths, DashboardFault: config.Fault.String()}) + return scenarioExecutionOutput{Result: result, Reconnect: &reconnectEvidence}, err + }}, nil + default: + return scenarioExecution{}, errors.New("runtime execution is not implemented for the selected scenario") + } +} + +func standardExecution(name string, run func(context.Context) (scenario.Result, error)) scenarioExecution { + return scenarioExecution{name: name, run: func(ctx context.Context) (scenarioExecutionOutput, error) { + result, err := run(ctx) + return scenarioExecutionOutput{Result: result}, err + }} +} diff --git a/integration/agentcompat/cmd/agentcompat/stale_artifact_test.go b/integration/agentcompat/cmd/agentcompat/stale_artifact_test.go new file mode 100644 index 00000000..b2c0b3df --- /dev/null +++ b/integration/agentcompat/cmd/agentcompat/stale_artifact_test.go @@ -0,0 +1,95 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "testing" + "time" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/evidence" +) + +func TestCLI_FailedGenerationRemovesPriorEvidence(t *testing.T) { + resultsDir := t.TempDir() + seedStaleEvidence(t, resultsDir) + var stdout, stderr bytes.Buffer + if err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "future-scenario"}, &stdout, &stderr, time.Now()); err == nil { + t.Fatal("unimplemented runtime unexpectedly succeeded") + } + for _, name := range append(evidence.FixedEvidenceFiles()[1:], "agents") { + if _, err := os.Stat(filepath.Join(resultsDir, name)); !os.IsNotExist(err) { + t.Fatalf("stale artifact remains: %s (%v)", name, err) + } + } +} + +func TestCLI_FailedGenerationRemovesPriorEvidenceWhenMetadataIsSymlink(t *testing.T) { + resultsDir := t.TempDir() + target := filepath.Join(t.TempDir(), "metadata-target") + if err := os.WriteFile(target, []byte("old metadata"), 0o600); err != nil { + t.Fatalf("seed metadata target: %v", err) + } + if err := os.Symlink(target, filepath.Join(resultsDir, "metadata.json")); err != nil { + t.Fatalf("create metadata symlink: %v", err) + } + for _, name := range []string{"results.json", "junit.xml"} { + if err := os.WriteFile(filepath.Join(resultsDir, name), []byte("stale success"), 0o600); err != nil { + t.Fatalf("seed stale artifact %s: %v", name, err) + } + } + var stdout, stderr bytes.Buffer + if err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "future-scenario"}, &stdout, &stderr, time.Now()); err == nil { + t.Fatal("unimplemented runtime unexpectedly succeeded") + } + metadataInfo, err := os.Lstat(filepath.Join(resultsDir, "metadata.json")) + if err != nil || !metadataInfo.Mode().IsRegular() || metadataInfo.Mode().Perm() != 0o600 { + t.Fatalf("replacement metadata is not a private regular file: mode=%v err=%v", metadataInfo, err) + } + for _, name := range []string{"results.json", "junit.xml"} { + if _, err := os.Stat(filepath.Join(resultsDir, name)); !os.IsNotExist(err) { + t.Fatalf("stale artifact remains after failed invocation: %s (%v)", name, err) + } + } + data, err := os.ReadFile(target) + if err != nil || string(data) != "old metadata" { + t.Fatalf("metadata symlink target changed: %q %v", data, err) + } +} + +func TestCLI_ParseFailureRemovesPriorEvidence(t *testing.T) { + for name, extraArgs := range map[string][]string{ + "invalid profile": {"--profile", "invalid-profile"}, + "invalid seed": {"--profile", "pr-full", "--seed", "invalid-seed"}, + } { + t.Run(name, func(t *testing.T) { + resultsDir := t.TempDir() + seedStaleEvidence(t, resultsDir) + args := append([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--results-dir", resultsDir}, extraArgs...) + var stdout, stderr bytes.Buffer + if err := run(args, &stdout, &stderr, time.Now()); err == nil { + t.Fatal("invalid CLI input unexpectedly succeeded") + } + for _, artifact := range evidence.FixedEvidenceFiles()[1:] { + if _, err := os.Stat(filepath.Join(resultsDir, artifact)); !os.IsNotExist(err) { + t.Fatalf("stale artifact remains after parse failure: %s (%v)", artifact, err) + } + } + }) + } +} + +func seedStaleEvidence(t *testing.T, resultsDir string) { + t.Helper() + for _, name := range evidence.FixedEvidenceFiles()[1:] { + if err := os.WriteFile(filepath.Join(resultsDir, name), []byte("stale success"), 0o600); err != nil { + t.Fatalf("seed stale artifact %s: %v", name, err) + } + } + if err := os.Mkdir(filepath.Join(resultsDir, "agents"), 0o700); err != nil { + t.Fatalf("create stale agents directory: %v", err) + } + if err := os.WriteFile(filepath.Join(resultsDir, "agents", "old.log"), []byte("old success"), 0o600); err != nil { + t.Fatalf("seed stale agent log: %v", err) + } +}