test(agentcompat): add scenario CLI wiring

Co-authored-by: naiba/CloudCode <hi+cloudcode@nai.ba>

integration/agentcompat/cmd/agentcompat/artifact_publication_test.go

@@ -0,0 +1,163 @@
+package main
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/evidence"
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/scenario"
+)
+
+func TestCLI_ArtifactPublicationReplacesPublicFileAndFinalSymlink(t *testing.T) {
+	tests := []struct {
+		name  string
+		setup func(*testing.T, string, string)
+	}{
+		{"public file", func(t *testing.T, path, _ string) {
+			if err := os.WriteFile(path, []byte("old"), 0o644); err != nil {
+				t.Fatalf("write old file: %v", err)
+			}
+		}},
+		{"final symlink", func(t *testing.T, path, sentinel string) {
+			if err := os.Symlink(sentinel, path); err != nil {
+				t.Fatalf("symlink final path: %v", err)
+			}
+		}},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			dir := t.TempDir()
+			path := filepath.Join(dir, "cleanup.json")
+			sentinel := filepath.Join(t.TempDir(), "sentinel")
+			if err := os.WriteFile(sentinel, []byte("unchanged"), 0o600); err != nil {
+				t.Fatalf("write sentinel: %v", err)
+			}
+			test.setup(t, path, sentinel)
+			if err := writeJSONArtifact(dir, "cleanup.json", map[string]bool{"passed": true}); err != nil {
+				t.Fatalf("publish artifact: %v", err)
+			}
+			info, err := os.Lstat(path)
+			if err != nil {
+				t.Fatalf("lstat artifact: %v", err)
+			}
+			if !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 {
+				t.Fatalf("artifact mode=%v", info.Mode())
+			}
+			content, err := os.ReadFile(sentinel)
+			if err != nil || string(content) != "unchanged" {
+				t.Fatalf("outside sentinel changed: content=%q err=%v", content, err)
+			}
+		})
+	}
+}
+
+func TestCLI_InterruptedScenarioPublicationLeavesAtomicInvalidDirectory(t *testing.T) {
+	dir := t.TempDir()
+	config := testCLIConfig(t, contract.ScenarioTransfer100MiB, contract.FaultTransferHash)
+	paths, err := contract.NewPaths(config.Paths.NezhaSource().String(), config.Paths.AgentSource().String(), dir)
+	if err != nil {
+		t.Fatalf("paths: %v", err)
+	}
+	config.Paths = paths
+	if err := writeMetadata(t.Context(), config, time.Now()); err != nil {
+		t.Fatalf("metadata: %v", err)
+	}
+	previous := scenarioArtifactPublished
+	scenarioArtifactPublished = func(name string) error {
+		if name == "results.json" {
+			return errors.New("injected publication interruption")
+		}
+		return nil
+	}
+	t.Cleanup(func() { scenarioArtifactPublished = previous })
+	output := scenarioExecutionOutput{Result: scenario.Result{Name: contract.ScenarioTransfer100MiB, Passed: false, CleanupOK: true, Error: "transfer scenario: injected hash mismatch"}, Transfer: &scenario.TransferEvidence{WarmupUploadBytes: 65536, WarmupDownloadBytes: 65536, WarmupSHA256: "abc", WarmupDuration: time.Nanosecond, WarmupDeadlineRemaining: time.Second, WarmupQuiescent: true, OutsideRootSentinelsUnchanged: true}}
+	if err := writeScenarioEvidence(config, output, time.Now()); err == nil {
+		t.Fatal("publication interruption accepted")
+	}
+	if err := evidence.ValidateDirectory(dir); err == nil {
+		t.Fatal("partial evidence directory validated")
+	}
+	data, err := os.ReadFile(filepath.Join(dir, "results.json"))
+	if !errors.Is(err, os.ErrNotExist) || len(data) != 0 {
+		t.Fatalf("interrupted final file published: bytes=%d err=%v", len(data), err)
+	}
+}
+
+func TestCLI_PrivateArtifactJoinsPrimaryAndCleanupErrors(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "results.json")
+	primaryErr := errors.New("publication hook failed")
+	removeErr := errors.New("temporary removal failed")
+	previousClose := privateArtifactClose
+	previousRemove := privateArtifactRemove
+	closeCalls := 0
+	privateArtifactClose = func(file *os.File) error {
+		closeCalls++
+		return file.Close()
+	}
+	privateArtifactRemove = func(string) error { return removeErr }
+	t.Cleanup(func() {
+		privateArtifactClose = previousClose
+		privateArtifactRemove = previousRemove
+	})
+
+	err := writePrivateArtifactWithSeam(path, []byte("payload"), func() error { return primaryErr })
+	if !errors.Is(err, primaryErr) || !errors.Is(err, removeErr) {
+		t.Fatalf("publication error=%v, want primary and removal errors", err)
+	}
+	if errors.Is(err, os.ErrClosed) || closeCalls != 1 {
+		t.Fatalf("successful close repeated: calls=%d err=%v", closeCalls, err)
+	}
+}
+
+func TestCLI_PrivateArtifactJoinsCloseAndRemoveErrors(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "results.json")
+	closeErr := errors.New("temporary close failed")
+	removeErr := errors.New("temporary removal failed")
+	previousClose := privateArtifactClose
+	previousRemove := privateArtifactRemove
+	closeCalls := 0
+	privateArtifactClose = func(file *os.File) error {
+		closeCalls++
+		if err := file.Close(); err != nil {
+			return err
+		}
+		return closeErr
+	}
+	privateArtifactRemove = func(string) error { return removeErr }
+	t.Cleanup(func() {
+		privateArtifactClose = previousClose
+		privateArtifactRemove = previousRemove
+	})
+
+	err := writePrivateArtifactWithSeam(path, []byte("payload"), func() error { return nil })
+	if !errors.Is(err, closeErr) || !errors.Is(err, removeErr) {
+		t.Fatalf("publication error=%v, want close and removal errors", err)
+	}
+	if errors.Is(err, os.ErrClosed) || closeCalls != 1 {
+		t.Fatalf("close failure retried: calls=%d err=%v", closeCalls, err)
+	}
+}
+
+func TestCLI_PrivateArtifactRemovesTemporaryFileAfterHookFailure(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "results.json")
+	err := writePrivateArtifactWithSeam(path, []byte("payload"), func() error {
+		return errors.New("publication hook failed")
+	})
+	if err == nil {
+		t.Fatal("publication hook failure accepted")
+	}
+	entries, readErr := os.ReadDir(dir)
+	if readErr != nil {
+		t.Fatalf("read artifact directory: %v", readErr)
+	}
+	if len(entries) != 0 {
+		t.Fatalf("temporary artifact survived hook failure: %v", entries)
+	}
+}

integration/agentcompat/cmd/agentcompat/cli_config.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/evidence"
+)
+
+type scenarioFlags []contract.Scenario
+
+func (scenarios *scenarioFlags) String() string {
+	values := make([]string, 0, len(*scenarios))
+	for _, scenario := range *scenarios {
+		values = append(values, scenario.String())
+	}
+	return strings.Join(values, ",")
+}
+
+func (scenarios *scenarioFlags) Set(value string) error {
+	scenario, err := contract.NewScenario(value)
+	if err != nil {
+		return err
+	}
+	*scenarios = append(*scenarios, scenario)
+	return nil
+}
+
+type cliConfig struct {
+	Paths     contract.Paths
+	Profile   contract.Profile
+	Seed      contract.Seed
+	Scenarios scenarioFlags
+	Fault     contract.Fault
+}
+
+func parseFlags(args []string, stderr io.Writer) (cliConfig, error) {
+	flags := flag.NewFlagSet("agentcompat", flag.ContinueOnError)
+	flags.SetOutput(io.Discard)
+	nezhaSource := flags.String("nezha-source", "", "Nezha source directory")
+	agentSource := flags.String("agent-source", "", "Agent source directory")
+	profileName := flags.String("profile", "", "compatibility profile")
+	resultsDir := flags.String("results-dir", "", "evidence output directory")
+	seedValue := flags.String("seed", "0x4e5a4841", "deterministic seed")
+	faultName := flags.String("fault", "", "named fault injection")
+	var scenarios scenarioFlags
+	flags.Var(&scenarios, "scenario", "run only a named scenario; omit for the complete profile")
+	if err := flags.Parse(args); err != nil {
+		return cliConfig{}, errors.New("invalid command-line arguments")
+	}
+	paths, err := contract.NewPaths(*nezhaSource, *agentSource, *resultsDir)
+	if err != nil {
+		return cliConfig{}, err
+	}
+	if err := prepareResultsDirBeforeParse(paths.ResultsDir().String()); err != nil {
+		return cliConfig{}, err
+	}
+	profile, err := contract.ProfileByName(*profileName)
+	if err != nil {
+		return cliConfig{}, err
+	}
+	seed, err := contract.ParseSeed(*seedValue)
+	if err != nil {
+		return cliConfig{}, err
+	}
+	fault := contract.Fault{}
+	if *faultName != "" {
+		fault, err = contract.NewFault(*faultName)
+		if err != nil {
+			return cliConfig{}, err
+		}
+	}
+	return cliConfig{Paths: paths, Profile: profile, Seed: seed, Scenarios: scenarios, Fault: fault}, nil
+}
+
+func prepareResultsDir(resultsDir string) error {
+	return prepareEvidenceArtifacts(resultsDir, evidence.FixedEvidenceFiles())
+}
+
+func prepareResultsDirBeforeParse(resultsDir string) error {
+	return prepareEvidenceArtifacts(resultsDir, evidence.FixedEvidenceFiles()[1:])
+}
+
+func prepareEvidenceArtifacts(resultsDir string, artifactNames []string) error {
+	if info, err := os.Lstat(resultsDir); err == nil && info.Mode()&os.ModeSymlink != 0 {
+		return errors.New("results directory must not be a symbolic link")
+	} else if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("inspect results directory: %w", err)
+	}
+	for _, name := range artifactNames {
+		if err := os.Remove(filepath.Join(resultsDir, name)); err != nil && !errors.Is(err, os.ErrNotExist) {
+			return fmt.Errorf("remove previous artifact %s: %w", name, err)
+		}
+	}
+	if err := os.RemoveAll(filepath.Join(resultsDir, "agents")); err != nil {
+		return fmt.Errorf("remove previous agent logs: %w", err)
+	}
+	return nil
+}

integration/agentcompat/cmd/agentcompat/dedicated_wiring_test.go

@@ -0,0 +1,152 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/scenario"
+)
+
+func TestCLI_SelectsTransferAndReconnectWithTypedEvidence(t *testing.T) {
+	tests := []struct {
+		name  string
+		fault string
+	}{
+		{contract.ScenarioTransfer100MiB, ""},
+		{contract.ScenarioTransfer100MiB, contract.FaultTransferHash},
+		{contract.ScenarioReconnect, ""},
+		{contract.ScenarioReconnect, contract.FaultDashboardExit},
+	}
+	for _, test := range tests {
+		t.Run(test.name+"/"+test.fault, func(t *testing.T) {
+			config := testCLIConfig(t, test.name, test.fault)
+			runnerErr := errors.New("dedicated runner sentinel")
+			wantResult := scenario.Result{Name: test.name, Passed: false, CleanupOK: true, Error: runnerErr.Error(), Assertions: []scenario.Assertion{{Name: "typed dispatch", Passed: false, Details: test.fault}}}
+			wantTransfer := scenario.TransferEvidence{WarmupUploadBytes: 65536, WarmupDownloadBytes: 65536, WarmupSHA256: "warmup", WarmupDuration: time.Second, WarmupDeadlineRemaining: 2 * time.Second, WarmupQuiescent: true, UploadBytes: contract.TransferBytes, DownloadBytes: contract.TransferBytes, UploadSHA256: "transfer-hash", DownloadSHA256: "transfer-hash", UploadChunks: 3, DownloadChunks: 4, UploadDuration: 5 * time.Second, DownloadDuration: 6 * time.Second, RetainedHeapBytes: 7, Mode: "0640", CreateDirs: true, UploadReplayRejected: true, DownloadReplayRejected: true, OversizeRejected: true, OutsideRootSentinelsUnchanged: true}
+			wantReconnect := completeReconnectDispatchEvidence(t)
+			previousTransfer := runTransferScenario
+			previousReconnect := runReconnectScenario
+			var receivedTransfer *scenario.TransferInput
+			var receivedReconnect *scenario.ReconnectInput
+			runTransferScenario = func(_ context.Context, input scenario.TransferInput) (scenario.Result, scenario.TransferEvidence, error) {
+				receivedTransfer = &input
+				return wantResult, wantTransfer, runnerErr
+			}
+			runReconnectScenario = func(_ context.Context, input scenario.ReconnectInput) (scenario.Result, scenario.ReconnectEvidence, error) {
+				receivedReconnect = &input
+				return wantResult, wantReconnect, runnerErr
+			}
+			t.Cleanup(func() {
+				runTransferScenario = previousTransfer
+				runReconnectScenario = previousReconnect
+			})
+
+			execution, err := selectScenarioExecution(config)
+			if err != nil {
+				t.Fatalf("select execution: %v", err)
+			}
+			output, runErr := execution.run(context.Background())
+			if !errors.Is(runErr, runnerErr) {
+				t.Fatalf("runner error=%v, want sentinel", runErr)
+			}
+			if err := output.Validate(); err != nil {
+				t.Fatalf("validate typed output: %v", err)
+			}
+			if !reflect.DeepEqual(output.Result, wantResult) {
+				t.Fatalf("result=%#v, want %#v", output.Result, wantResult)
+			}
+			if test.name == contract.ScenarioTransfer100MiB {
+				wantInput := scenario.TransferInput{Paths: config.Paths, Fault: config.Fault}
+				if receivedTransfer == nil || *receivedTransfer != wantInput || receivedReconnect != nil {
+					t.Fatalf("transfer inputs: received=%#v reconnect=%#v want=%#v", receivedTransfer, receivedReconnect, wantInput)
+				}
+				if output.Transfer == nil || !reflect.DeepEqual(*output.Transfer, wantTransfer) || output.Reconnect != nil {
+					t.Fatalf("transfer evidence=%#v reconnect=%#v", output.Transfer, output.Reconnect)
+				}
+			} else {
+				wantInput := scenario.ReconnectInput{Paths: config.Paths, DashboardFault: config.Fault.String()}
+				if receivedReconnect == nil || *receivedReconnect != wantInput || receivedTransfer != nil {
+					t.Fatalf("reconnect inputs: received=%#v transfer=%#v want=%#v", receivedReconnect, receivedTransfer, wantInput)
+				}
+				if output.Reconnect == nil || !reflect.DeepEqual(*output.Reconnect, wantReconnect) || output.Transfer != nil {
+					t.Fatalf("reconnect evidence=%#v transfer=%#v", output.Reconnect, output.Transfer)
+				}
+			}
+		})
+	}
+}
+
+func TestCLI_RejectsUnsupportedScenarioFaultPairsBeforeRunner(t *testing.T) {
+	tests := []struct{ scenario, fault string }{
+		{contract.ScenarioTransfer100MiB, contract.FaultDashboardExit},
+		{contract.ScenarioReconnect, contract.FaultTransferHash},
+		{contract.ScenarioMCPFilesystem, contract.FaultAgentBadSecret},
+	}
+	for _, test := range tests {
+		called := false
+		previousTransfer := runTransferScenario
+		runTransferScenario = func(context.Context, scenario.TransferInput) (scenario.Result, scenario.TransferEvidence, error) {
+			called = true
+			return scenario.Result{}, scenario.TransferEvidence{}, errors.New("unexpected")
+		}
+		_, err := selectScenarioExecution(testCLIConfig(t, test.scenario, test.fault))
+		runTransferScenario = previousTransfer
+		if err == nil || called {
+			t.Fatalf("unsupported pair started runner: scenario=%q fault=%q err=%v", test.scenario, test.fault, err)
+		}
+	}
+}
+
+func TestCLI_WritesDedicatedArtifactWithPrivateMode(t *testing.T) {
+	resultsDir := t.TempDir()
+	output := scenarioExecutionOutput{
+		Result:   scenario.Result{Name: contract.ScenarioTransfer100MiB, Passed: false, CleanupOK: true, Error: "transfer scenario: injected hash mismatch"},
+		Transfer: &scenario.TransferEvidence{WarmupUploadBytes: 65536, WarmupDownloadBytes: 65536, WarmupSHA256: "abc", WarmupDuration: time.Nanosecond, WarmupDeadlineRemaining: time.Second, WarmupQuiescent: true, OutsideRootSentinelsUnchanged: true},
+	}
+	config := testCLIConfig(t, contract.ScenarioTransfer100MiB, contract.FaultTransferHash)
+	paths, err := contract.NewPaths(config.Paths.NezhaSource().String(), config.Paths.AgentSource().String(), resultsDir)
+	if err != nil {
+		t.Fatalf("paths: %v", err)
+	}
+	config.Paths = paths
+	if err := writeScenarioEvidence(config, output, time.Now()); err != nil {
+		t.Fatalf("write evidence: %v", err)
+	}
+	info, err := os.Stat(filepath.Join(resultsDir, "transfer.json"))
+	if err != nil {
+		t.Fatalf("stat transfer evidence: %v", err)
+	}
+	if info.Mode().Perm() != 0o600 {
+		t.Fatalf("transfer evidence mode=%o", info.Mode().Perm())
+	}
+}
+
+func testCLIConfig(t *testing.T, scenarioName, faultName string) cliConfig {
+	t.Helper()
+	paths, err := contract.NewPaths("/src/nezha", "/src/agent", t.TempDir())
+	if err != nil {
+		t.Fatalf("paths: %v", err)
+	}
+	profile, err := contract.ProfileByName("pr-full")
+	if err != nil {
+		t.Fatalf("profile: %v", err)
+	}
+	scenarioValue, err := contract.NewScenario(scenarioName)
+	if err != nil {
+		t.Fatalf("scenario: %v", err)
+	}
+	fault := contract.Fault{}
+	if faultName != "" {
+		fault, err = contract.NewFault(faultName)
+		if err != nil {
+			t.Fatalf("fault: %v", err)
+		}
+	}
+	return cliConfig{Paths: paths, Profile: profile, Seed: contract.DefaultSeed, Scenarios: scenarioFlags{scenarioValue}, Fault: fault}
+}

integration/agentcompat/cmd/agentcompat/main.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/evidence"
+)
+
+func runContext(ctx context.Context, args []string, stdout io.Writer, stderr io.Writer, now time.Time) error {
+	config, err := parseFlags(args, stderr)
+	if err != nil {
+		return err
+	}
+	if err := writeMetadata(ctx, config, now); err != nil {
+		return err
+	}
+	if len(config.Scenarios) == 1 && config.Scenarios[0].String() == contract.ScenarioMetadata && config.Fault.IsZero() {
+		fmt.Fprintf(stdout, "metadata written for profile %s\n", config.Profile.Name())
+		return nil
+	}
+	execution, err := selectScenarioExecution(config)
+	if err != nil {
+		return err
+	}
+	scenarioContext, cancel := context.WithTimeout(ctx, config.Profile.SuiteDeadline())
+	defer cancel()
+	output, runErr := execution.run(scenarioContext)
+	if err := writeScenarioEvidence(config, output, now); err != nil {
+		return err
+	}
+	if err := evidence.ValidateDirectory(config.Paths.ResultsDir().String()); err != nil {
+		return fmt.Errorf("validate scenario evidence: %w", err)
+	}
+	if runErr != nil {
+		return runErr
+	}
+	fmt.Fprintf(stdout, "scenario %s passed for profile %s\n", execution.name, config.Profile.Name())
+	return nil
+}
+
+func run(args []string, stdout io.Writer, stderr io.Writer, now time.Time) error {
+	return runContext(context.Background(), args, stdout, stderr, now)
+}
+
+func main() {
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+	if err := runContext(ctx, os.Args[1:], os.Stdout, os.Stderr, time.Now().UTC()); err != nil {
+		fmt.Fprintf(os.Stderr, "agentcompat: %s\n", evidence.Redact(err.Error()))
+		os.Exit(2)
+	}
+}

integration/agentcompat/cmd/agentcompat/main_test.go

@@ -0,0 +1,192 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/evidence"
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/scenario"
+)
+
+func TestCLI_ParsesTypedFlagsAndWritesMetadata(t *testing.T) {
+	resultsDir := t.TempDir()
+	var stdout, stderr bytes.Buffer
+	err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--seed", "0x4e5a4841", "--scenario", "metadata"}, &stdout, &stderr, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC))
+	if err != nil {
+		t.Fatalf("run CLI: %v", err)
+	}
+	data, err := os.ReadFile(filepath.Join(resultsDir, "metadata.json"))
+	if err != nil {
+		t.Fatalf("read metadata: %v", err)
+	}
+	var metadata struct {
+		Profile struct {
+			Name string `json:"name"`
+		} `json:"profile"`
+		Seed string `json:"seed"`
+	}
+	if err := json.Unmarshal(data, &metadata); err != nil {
+		t.Fatalf("parse metadata: %v", err)
+	}
+	if metadata.Profile.Name != "pr-full" || metadata.Seed != "0x4e5a4841" || !strings.Contains(stdout.String(), "metadata written") {
+		t.Fatalf("unexpected metadata-only output: %#v %q", metadata, stdout.String())
+	}
+}
+
+func TestCLI_MetadataEvidenceValidatesAsCurrentMetadataProfile(t *testing.T) {
+	resultsDir := t.TempDir()
+	var stdout, stderr bytes.Buffer
+	err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "metadata"}, &stdout, &stderr, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC))
+	if err != nil {
+		t.Fatalf("run metadata CLI: %v", err)
+	}
+	if err := evidence.ValidateDirectory(resultsDir); err != nil {
+		t.Fatalf("validate metadata evidence: %v", err)
+	}
+}
+
+func TestCLI_RejectsInvalidProfileAndSeedWithoutSecretEcho(t *testing.T) {
+	for name, args := range map[string][]string{
+		"profile": {"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "private-profile-secret", "--results-dir", t.TempDir()},
+		"seed":    {"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", t.TempDir(), "--seed", "not-a-seed-secret"},
+	} {
+		t.Run(name, func(t *testing.T) {
+			var stdout, stderr bytes.Buffer
+			err := run(args, &stdout, &stderr, time.Now())
+			if err == nil {
+				t.Fatal("invalid CLI input accepted")
+			}
+			if strings.Contains(err.Error(), "not-a-seed-secret") || strings.Contains(err.Error(), "private-profile-secret") {
+				t.Fatal("invalid input echoed in error")
+			}
+		})
+	}
+}
+
+func TestCLI_RejectsMissingPaths(t *testing.T) {
+	var stdout, stderr bytes.Buffer
+	err := run([]string{"--profile", "pr-full", "--scenario", "metadata"}, &stdout, &stderr, time.Now())
+	if err == nil || !strings.Contains(err.Error(), "--nezha-source") {
+		t.Fatalf("missing source paths were not rejected: %v", err)
+	}
+}
+
+func TestCLI_ParsesRepeatableScenariosAndFault(t *testing.T) {
+	var stderr bytes.Buffer
+	config, err := parseFlags([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "soak", "--results-dir", "/tmp/results", "--scenario", "metadata", "--scenario", "transfer-100mib", "--fault", "transfer-hash"}, &stderr)
+	if err != nil {
+		t.Fatalf("parse flags: %v", err)
+	}
+	if len(config.Scenarios) != 2 || config.Scenarios[0].String() != "metadata" || config.Scenarios[1].String() != "transfer-100mib" || config.Fault.String() != "transfer-hash" {
+		t.Fatalf("unexpected typed flags: %#v", config)
+	}
+}
+
+func TestCLI_WritesMetadataBeforeRejectingUnsupportedRuntime(t *testing.T) {
+	resultsDir := t.TempDir()
+	var stdout, stderr bytes.Buffer
+	err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "future-scenario"}, &stdout, &stderr, time.Now())
+	if err == nil {
+		t.Fatal("unimplemented runtime reported success")
+	}
+	if _, statErr := os.Stat(filepath.Join(resultsDir, "metadata.json")); statErr != nil {
+		t.Fatalf("metadata was not written before runtime rejection: %v", statErr)
+	}
+}
+
+func TestCLI_RecognizesOnlyMCPFilesystemRuntimeScenario(t *testing.T) {
+	var stderr bytes.Buffer
+	config, err := parseFlags([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", t.TempDir(), "--scenario", "mcp-filesystem"}, &stderr)
+	if err != nil {
+		t.Fatalf("parse mcp-filesystem flags: %v", err)
+	}
+	execution, err := selectScenarioExecution(config)
+	if err != nil || execution.name != "mcp-filesystem" {
+		t.Fatalf("mcp-filesystem runtime registration: name=%q err=%v", execution.name, err)
+	}
+
+	config.Scenarios = append(config.Scenarios, config.Scenarios[0])
+	if _, err := selectScenarioExecution(config); err == nil || !strings.Contains(err.Error(), "exactly one") {
+		t.Fatalf("multi-scenario runtime was unexpectedly accepted: %v", err)
+	}
+}
+
+func TestCLI_SelectsTerminalRuntimeScenario(t *testing.T) {
+	var stderr bytes.Buffer
+	config, err := parseFlags([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", t.TempDir(), "--scenario", "terminal"}, &stderr)
+	if err != nil {
+		t.Fatalf("parse terminal flags: %v", err)
+	}
+	previous := runTerminalScenario
+	var received scenario.TerminalInput
+	runTerminalScenario = func(_ context.Context, input scenario.TerminalInput) (scenario.Result, error) {
+		received = input
+		return scenario.Result{Name: "terminal", Passed: true}, nil
+	}
+	t.Cleanup(func() { runTerminalScenario = previous })
+
+	execution, err := selectScenarioExecution(config)
+
+	if err != nil {
+		t.Fatalf("select terminal scenario: %v", err)
+	}
+	if execution.name != "terminal" || execution.run == nil {
+		t.Fatalf("unexpected terminal execution: %#v", execution)
+	}
+	if _, err := execution.run(context.Background()); err != nil {
+		t.Fatalf("run terminal execution: %v", err)
+	}
+	if received.Paths.NezhaSource().String() != "/src/nezha" || received.Paths.AgentSource().String() != "/src/agent" || !received.Fault.IsZero() {
+		t.Fatalf("terminal input was not forwarded: %#v", received)
+	}
+}
+
+func TestCLI_MetadataWriteReplacesSymlinkWithoutChangingTarget(t *testing.T) {
+	resultsDir := t.TempDir()
+	target := filepath.Join(t.TempDir(), "target")
+	if err := os.WriteFile(target, []byte("sentinel"), 0o644); err != nil {
+		t.Fatalf("write target: %v", err)
+	}
+	if err := os.Symlink(target, filepath.Join(resultsDir, "metadata.json")); err != nil {
+		t.Fatalf("create metadata symlink: %v", err)
+	}
+	var stdout, stderr bytes.Buffer
+	err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "metadata"}, &stdout, &stderr, time.Now())
+	if err != nil {
+		t.Fatalf("replace metadata symlink: %v", err)
+	}
+	data, readErr := os.ReadFile(target)
+	if readErr != nil || string(data) != "sentinel" {
+		t.Fatalf("symlink target changed: %q %v", data, readErr)
+	}
+	info, statErr := os.Lstat(filepath.Join(resultsDir, "metadata.json"))
+	if statErr != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 {
+		t.Fatalf("metadata replacement mode=%v err=%v", info, statErr)
+	}
+}
+
+func TestCLI_MetadataWriteUsesPrivatePermissions(t *testing.T) {
+	root := t.TempDir()
+	resultsDir := filepath.Join(root, "results")
+	var stdout, stderr bytes.Buffer
+	if err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "metadata"}, &stdout, &stderr, time.Now()); err != nil {
+		t.Fatalf("write metadata: %v", err)
+	}
+	directoryInfo, err := os.Stat(resultsDir)
+	if err != nil {
+		t.Fatalf("stat results directory: %v", err)
+	}
+	fileInfo, err := os.Stat(filepath.Join(resultsDir, "metadata.json"))
+	if err != nil {
+		t.Fatalf("stat metadata: %v", err)
+	}
+	if directoryInfo.Mode().Perm() != 0o700 || fileInfo.Mode().Perm() != 0o600 {
+		t.Fatalf("unexpected permissions: directory=%o file=%o", directoryInfo.Mode().Perm(), fileInfo.Mode().Perm())
+	}
+}

integration/agentcompat/cmd/agentcompat/metadata_writer.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/evidence"
+)
+
+var metadataArtifactReady = func(context.Context) error { return nil }
+
+func writeMetadata(ctx context.Context, config cliConfig, now time.Time) error {
+	resultsDir := config.Paths.ResultsDir().String()
+	if info, err := os.Lstat(resultsDir); err == nil && info.Mode()&os.ModeSymlink != 0 {
+		return errors.New("results directory must not be a symbolic link")
+	} else if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("inspect results directory: %w", err)
+	}
+	if err := os.MkdirAll(resultsDir, 0o700); err != nil {
+		return fmt.Errorf("create results directory: %w", err)
+	}
+	if err := os.Chmod(resultsDir, 0o700); err != nil {
+		return fmt.Errorf("secure results directory: %w", err)
+	}
+	if err := prepareResultsDir(resultsDir); err != nil {
+		return err
+	}
+	metadata, err := evidence.NewMetadata(evidence.MetadataInput{Profile: config.Profile, Seed: config.Seed, Paths: config.Paths, ResourceBudget: contract.DefaultResourceBudget(), Scenarios: config.Scenarios, Fault: config.Fault, StartedAt: now, EvidenceFiles: evidence.EvidenceFiles()})
+	if err != nil {
+		return fmt.Errorf("build metadata: %w", err)
+	}
+	data, err := json.MarshalIndent(metadata, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal metadata: %w", err)
+	}
+	path := filepath.Join(resultsDir, "metadata.json")
+	return writePrivateArtifactWithSeam(path, data, func() error { return metadataArtifactReady(ctx) })
+}

integration/agentcompat/cmd/agentcompat/metadata_writer_test.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+)
+
+func TestCLI_MetadataCancellationRemovesTemporaryArtifact(t *testing.T) {
+	resultsDir := t.TempDir()
+	config := testCLIConfig(t, contract.ScenarioMetadata, "")
+	paths, err := contract.NewPaths(config.Paths.NezhaSource().String(), config.Paths.AgentSource().String(), resultsDir)
+	if err != nil {
+		t.Fatalf("paths: %v", err)
+	}
+	config.Paths = paths
+	ready := make(chan struct{})
+	previous := metadataArtifactReady
+	metadataArtifactReady = func(ctx context.Context) error {
+		close(ready)
+		<-ctx.Done()
+		return ctx.Err()
+	}
+	t.Cleanup(func() { metadataArtifactReady = previous })
+	ctx, cancel := context.WithCancel(context.Background())
+	writeDone := make(chan error, 1)
+	go func() { writeDone <- writeMetadata(ctx, config, time.Now()) }()
+	<-ready
+	cancel()
+
+	if err := <-writeDone; !errors.Is(err, context.Canceled) {
+		t.Fatalf("metadata cancellation error=%v, want context canceled", err)
+	}
+	if _, err := os.Stat(filepath.Join(resultsDir, "metadata.json")); !errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("metadata final file exists after cancellation: %v", err)
+	}
+	entries, err := os.ReadDir(resultsDir)
+	if err != nil {
+		t.Fatalf("read results directory: %v", err)
+	}
+	for _, entry := range entries {
+		if strings.HasPrefix(entry.Name(), ".artifact-") {
+			t.Fatalf("metadata temporary artifact survived cancellation: %s", entry.Name())
+		}
+	}
+}

integration/agentcompat/cmd/agentcompat/nat_wiring_test.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestCLI_SelectsNATRuntimeScenario(t *testing.T) {
+	// Given
+	var stderr bytes.Buffer
+	config, err := parseFlags([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", t.TempDir(), "--scenario", "nat"}, &stderr)
+	if err != nil {
+		t.Fatalf("parse NAT flags: %v", err)
+	}
+
+	// When
+	execution, err := selectScenarioExecution(config)
+
+	// Then
+	if err != nil {
+		t.Fatalf("select NAT scenario: %v", err)
+	}
+	if execution.name != "nat" || execution.run == nil {
+		t.Fatalf("unexpected NAT execution: %#v", execution)
+	}
+}

integration/agentcompat/cmd/agentcompat/private_artifact_writer.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+var (
+	scenarioArtifactPublished = func(string) error { return nil }
+	privateArtifactClose      = (*os.File).Close
+	privateArtifactRemove     = os.Remove
+)
+
+func writePrivateArtifact(path string, data []byte) (err error) {
+	return writePrivateArtifactWithSeam(path, data, func() error {
+		return scenarioArtifactPublished(filepath.Base(path))
+	})
+}
+
+func writePrivateArtifactWithSeam(path string, data []byte, beforeRename func() error) (err error) {
+	directory := filepath.Dir(path)
+	temporary, err := os.CreateTemp(directory, ".artifact-*")
+	if err != nil {
+		return fmt.Errorf("create temporary artifact: %w", err)
+	}
+	temporaryPath := temporary.Name()
+	committed := false
+	closed := false
+	defer func() {
+		if !committed {
+			if !closed {
+				err = errors.Join(err, privateArtifactClose(temporary))
+			}
+			err = errors.Join(err, privateArtifactRemove(temporaryPath))
+		}
+	}()
+	if err := temporary.Chmod(0o600); err != nil {
+		return fmt.Errorf("secure temporary artifact: %w", err)
+	}
+	if _, err := temporary.Write(append(data, '\n')); err != nil {
+		return fmt.Errorf("write temporary artifact: %w", err)
+	}
+	if err := temporary.Sync(); err != nil {
+		return fmt.Errorf("sync temporary artifact: %w", err)
+	}
+	closeErr := privateArtifactClose(temporary)
+	closed = true
+	if closeErr != nil {
+		return fmt.Errorf("close temporary artifact: %w", closeErr)
+	}
+	if err := beforeRename(); err != nil {
+		return err
+	}
+	if err := os.Rename(temporaryPath, path); err != nil {
+		return fmt.Errorf("rename temporary artifact: %w", err)
+	}
+	committed = true
+	return nil
+}

integration/agentcompat/cmd/agentcompat/reconnect_dispatch_fixture_test.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"testing"
+	"time"
+
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/agent"
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/dashboard"
+	processharness "github.com/nezhahq/nezha/integration/agentcompat/internal/process"
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/scenario"
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/workspace"
+)
+
+func completeReconnectDispatchEvidence(t *testing.T) scenario.ReconnectEvidence {
+	t.Helper()
+	disconnectAt := time.Date(2026, 2, 3, 4, 5, 6, 700, time.UTC)
+	reconnectAt := disconnectAt.Add(7 * time.Second)
+	dashboardReceipt := dashboard.MCPReceiptPair{
+		Task:   dashboard.MCPReceiptEvent{Sequence: 51, DashboardGeneration: 12, GateGeneration: 61, ServerID: 41, TaskID: 71, TaskType: 81, Kind: dashboard.MCPReceiptTask},
+		Result: dashboard.MCPReceiptEvent{Sequence: 52, DashboardGeneration: 12, GateGeneration: 61, ServerID: 41, TaskID: 71, TaskType: 81, Kind: dashboard.MCPReceiptResult},
+	}
+	agentReceipt := dashboard.MCPReceiptPair{
+		Task:   dashboard.MCPReceiptEvent{Sequence: 53, DashboardGeneration: 12, GateGeneration: 62, ServerID: 41, TaskID: 72, TaskType: 82, Kind: dashboard.MCPReceiptTask},
+		Result: dashboard.MCPReceiptEvent{Sequence: 54, DashboardGeneration: 12, GateGeneration: 62, ServerID: 41, TaskID: 72, TaskType: 82, Kind: dashboard.MCPReceiptResult},
+	}
+	evidence := scenario.ReconnectEvidence{
+		Fixture: scenario.ReconnectFixtureEvidence{
+			Dashboard: dashboard.FixtureIdentity{
+				WorkspaceRoot: "/typed/dashboard-workspace", ConfigPath: "/typed/dashboard.yaml", DatabasePath: "/typed/dashboard.sqlite", BinaryPath: "/typed/dashboard",
+				HTTP: workspace.ListenerIdentity{Address: "127.0.0.1:41001", Inode: 1101}, Receipt: workspace.ListenerIdentity{Address: "127.0.0.1:41002", Inode: 1102}, HTTPS: workspace.ListenerIdentity{Address: "127.0.0.1:41003", Inode: 1103},
+			},
+			AgentRoot: "/typed/agent-workspace", AgentConfigPath: "/typed/agent.yaml", AgentBinaryPath: "/typed/agent",
+		},
+		Runtime: scenario.ReconnectRuntimeEvidence{
+			DashboardBefore: dashboard.RuntimeIdentity{Generation: 11, PID: 2101, ProcessGroupID: 3101}, DashboardAfter: dashboard.RuntimeIdentity{Generation: 12, PID: 2102, ProcessGroupID: 3102},
+			AgentBefore: agent.ProcessIdentity{Generation: 21, PID: 2201, ProcessGroupID: 3201}, AgentAfter: agent.ProcessIdentity{Generation: 22, PID: 2202, ProcessGroupID: 3202},
+			StateGenerationBeforeAgentRestart: 31, StateGenerationAfterAgentRestart: 32,
+		},
+		Identity: scenario.ReconnectIdentityEvidence{ServerID: 41, UUID: "00000000-0000-0000-0000-000000000041", DashboardConfigUnchanged: true, AgentConfigUnchanged: true, DashboardFixtureUnchanged: true, ClientsRecreated: true, BootstrapRecreated: true},
+		Lifecycle: scenario.ReconnectLifecycleEvidence{
+			DisconnectAt: disconnectAt, ReconnectAt: reconnectAt, ReconnectInterval: 7 * time.Second,
+			DashboardReceipts: []dashboard.MCPReceiptPair{dashboardReceipt}, AgentReceipts: []dashboard.MCPReceiptPair{agentReceipt},
+			StaleGenerationReceipts: 0, DuplicateTaskIDs: 0, LostResultIDs: 0, OutsideRootSentinelUnchanged: true,
+		},
+		Observation:      scenario.ReconnectObservation{ServerID: 41, UUID: "00000000-0000-0000-0000-000000000041", OldGeneration: 11, NewGeneration: 12, DisconnectAt: disconnectAt, ReconnectAt: reconnectAt, TaskIDs: []uint64{71, 72}, ResultIDs: []uint64{71, 72}, PostReconnect: true, AgentRestarted: true},
+		AgentCleanup:     processharness.CleanupReceipt{Passed: true, Forced: false, Processes: []processharness.CleanupRecord{{Name: "agent-generation-21", PID: 2201, Forced: false, Error: ""}, {Name: "agent-generation-22", PID: 2202, Forced: false, Error: ""}}},
+		DashboardCleanup: processharness.CleanupReceipt{Passed: true, Forced: false, Processes: []processharness.CleanupRecord{{Name: "dashboard-generation-11", PID: 2101, Forced: false, Error: ""}, {Name: "dashboard-generation-12", PID: 2102, Forced: false, Error: ""}}},
+	}
+	assertCompleteReconnectDispatchEvidence(t, evidence)
+	return evidence
+}
+
+func assertCompleteReconnectDispatchEvidence(t *testing.T, evidence scenario.ReconnectEvidence) {
+	t.Helper()
+	if err := evidence.Validate(); err != nil {
+		t.Fatalf("incomplete reconnect dispatch fixture: %v", err)
+	}
+	fixture := evidence.Fixture
+	if fixture.Dashboard.WorkspaceRoot == "" || fixture.Dashboard.ConfigPath == "" || fixture.Dashboard.DatabasePath == "" || fixture.Dashboard.BinaryPath == "" || fixture.AgentRoot == "" || fixture.AgentConfigPath == "" || fixture.AgentBinaryPath == "" {
+		t.Fatal("incomplete reconnect dispatch fixture paths")
+	}
+	for name, listener := range map[string]struct {
+		address string
+		inode   uint64
+	}{
+		"http":    {fixture.Dashboard.HTTP.Address, fixture.Dashboard.HTTP.Inode},
+		"receipt": {fixture.Dashboard.Receipt.Address, fixture.Dashboard.Receipt.Inode},
+		"https":   {fixture.Dashboard.HTTPS.Address, fixture.Dashboard.HTTPS.Inode},
+	} {
+		if listener.address == "" || listener.inode == 0 {
+			t.Fatalf("incomplete reconnect dispatch %s listener", name)
+		}
+	}
+	if len(evidence.Lifecycle.DashboardReceipts) == 0 || len(evidence.Lifecycle.AgentReceipts) == 0 {
+		t.Fatal("incomplete reconnect dispatch receipt pairs")
+	}
+	for _, pairs := range [][]dashboard.MCPReceiptPair{evidence.Lifecycle.DashboardReceipts, evidence.Lifecycle.AgentReceipts} {
+		for _, pair := range pairs {
+			if pair.Task.Sequence == 0 || pair.Task.DashboardGeneration == 0 || pair.Task.GateGeneration == 0 || pair.Task.ServerID == 0 || pair.Task.TaskID == 0 || pair.Task.TaskType == 0 || pair.Task.Kind != dashboard.MCPReceiptTask {
+				t.Fatalf("incomplete reconnect dispatch task receipt: %#v", pair.Task)
+			}
+			if pair.Result.Sequence == 0 || pair.Result.DashboardGeneration == 0 || pair.Result.GateGeneration == 0 || pair.Result.ServerID == 0 || pair.Result.TaskID == 0 || pair.Result.TaskType == 0 || pair.Result.Kind != dashboard.MCPReceiptResult {
+				t.Fatalf("incomplete reconnect dispatch result receipt: %#v", pair.Result)
+			}
+		}
+	}
+	assertCompleteCleanupReceipt(t, "agent", evidence.AgentCleanup)
+	assertCompleteCleanupReceipt(t, "dashboard", evidence.DashboardCleanup)
+}
+
+func assertCompleteCleanupReceipt(t *testing.T, name string, receipt processharness.CleanupReceipt) {
+	t.Helper()
+	if !receipt.Passed || receipt.Forced || len(receipt.Processes) == 0 {
+		t.Fatalf("incomplete reconnect dispatch %s cleanup receipt: %#v", name, receipt)
+	}
+	for _, record := range receipt.Processes {
+		if record.Name == "" || record.PID == 0 || record.Forced || record.Error != "" {
+			t.Fatalf("incomplete reconnect dispatch %s cleanup record: %#v", name, record)
+		}
+	}
+}

integration/agentcompat/cmd/agentcompat/scenario_dispatch_test.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/scenario"
+)
+
+func TestCLI_RegisteredScenariosHaveExhaustiveRuntimeRouting(t *testing.T) {
+	for _, definition := range contract.ScenarioDefinitions() {
+		t.Run(definition.Name, func(t *testing.T) {
+			config := testCLIConfig(t, definition.Name, "")
+			execution, err := selectScenarioExecution(config)
+			if definition.Execution == contract.ScenarioExecutionMetadata {
+				if err == nil {
+					t.Fatal("metadata unexpectedly received runtime execution")
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("select registered scenario: %v", err)
+			}
+			if execution.name != definition.Name || execution.run == nil {
+				t.Fatalf("incomplete runtime execution: %#v", execution)
+			}
+		})
+	}
+}
+
+func TestCLI_Todos11To15DispatchPropagatesTypedInputsErrorsAndOutputs(t *testing.T) {
+	runnerError := errors.New("injected runner error")
+	tests := []struct {
+		name  string
+		fault string
+		set   func(*testing.T, scenario.Result, error, *bool)
+	}{
+		{contract.ScenarioRegistrationConfigExec, contract.FaultAgentBadSecret, func(t *testing.T, want scenario.Result, wantErr error, called *bool) {
+			previous := runRegistrationConfigExecScenario
+			runRegistrationConfigExecScenario = func(_ context.Context, input scenario.RegistrationConfigExecInput) (scenario.Result, error) {
+				*called = true
+				assertPathsAndFault(t, input.Paths, input.Fault, contract.FaultAgentBadSecret)
+				return want, wantErr
+			}
+			t.Cleanup(func() { runRegistrationConfigExecScenario = previous })
+		}},
+		{contract.ScenarioNAT, "", func(t *testing.T, want scenario.Result, wantErr error, called *bool) {
+			previous := runNATScenario
+			runNATScenario = func(_ context.Context, input scenario.NATInput) (scenario.Result, error) {
+				*called = true
+				assertPathsAndFault(t, input.Paths, input.Fault, "")
+				return want, wantErr
+			}
+			t.Cleanup(func() { runNATScenario = previous })
+		}},
+		{contract.ScenarioLegacyFM, contract.FaultAgentBadSecret, func(t *testing.T, want scenario.Result, wantErr error, called *bool) {
+			previous := runLegacyFMScenario
+			runLegacyFMScenario = func(_ context.Context, input scenario.LegacyFMInput) (scenario.Result, error) {
+				*called = true
+				assertPathsAndFault(t, input.Paths, input.Fault, contract.FaultAgentBadSecret)
+				return want, wantErr
+			}
+			t.Cleanup(func() { runLegacyFMScenario = previous })
+		}},
+		{contract.ScenarioTerminal, "", func(t *testing.T, want scenario.Result, wantErr error, called *bool) {
+			previous := runTerminalScenario
+			runTerminalScenario = func(_ context.Context, input scenario.TerminalInput) (scenario.Result, error) {
+				*called = true
+				assertPathsAndFault(t, input.Paths, input.Fault, "")
+				return want, wantErr
+			}
+			t.Cleanup(func() { runTerminalScenario = previous })
+		}},
+		{contract.ScenarioMCPFilesystem, "", func(t *testing.T, want scenario.Result, wantErr error, called *bool) {
+			previous := runMCPFilesystemScenario
+			runMCPFilesystemScenario = func(_ context.Context, input scenario.MCPFilesystemInput) (scenario.Result, error) {
+				*called = true
+				if input.Paths.NezhaSource().String() != "/src/nezha" || input.Paths.AgentSource().String() != "/src/agent" {
+					t.Fatalf("MCP filesystem paths=%#v", input.Paths)
+				}
+				return want, wantErr
+			}
+			t.Cleanup(func() { runMCPFilesystemScenario = previous })
+		}},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			want := scenario.Result{Name: test.name, Passed: false, Assertions: []scenario.Assertion{{Name: "runner assertion", Passed: false}}, Error: runnerError.Error()}
+			called := false
+			test.set(t, want, runnerError, &called)
+			execution, err := selectScenarioExecution(testCLIConfig(t, test.name, test.fault))
+			if err != nil {
+				t.Fatalf("select execution: %v", err)
+			}
+			output, err := execution.run(t.Context())
+			if !errors.Is(err, runnerError) {
+				t.Fatalf("runner error=%v", err)
+			}
+			if !called || output.Result.Name != want.Name || output.Result.Error != want.Error || output.Transfer != nil || output.Reconnect != nil {
+				t.Fatalf("runner propagation called=%t output=%#v", called, output)
+			}
+		})
+	}
+}
+
+func assertPathsAndFault(t *testing.T, paths contract.Paths, fault contract.Fault, wantFault string) {
+	t.Helper()
+	if paths.NezhaSource().String() != "/src/nezha" || paths.AgentSource().String() != "/src/agent" || fault.String() != wantFault {
+		t.Fatalf("paths/fault propagation paths=%#v fault=%q", paths, fault.String())
+	}
+}

integration/agentcompat/cmd/agentcompat/scenario_evidence_writer.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+	"time"
+
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/evidence"
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/scenario"
+)
+
+type transferArtifact struct {
+	Scenario  string                    `json:"scenario"`
+	Fault     string                    `json:"fault,omitempty"`
+	Passed    bool                      `json:"passed"`
+	CleanupOK bool                      `json:"cleanup_ok"`
+	Error     string                    `json:"error,omitempty"`
+	Evidence  scenario.TransferEvidence `json:"evidence"`
+}
+
+type reconnectArtifact struct {
+	Scenario  string                     `json:"scenario"`
+	Fault     string                     `json:"fault,omitempty"`
+	Passed    bool                       `json:"passed"`
+	CleanupOK bool                       `json:"cleanup_ok"`
+	Error     string                     `json:"error,omitempty"`
+	Evidence  scenario.ReconnectEvidence `json:"evidence"`
+}
+
+func writeScenarioEvidence(config cliConfig, output scenarioExecutionOutput, now time.Time) error {
+	if err := output.Validate(); err != nil {
+		return fmt.Errorf("validate scenario execution output: %w", err)
+	}
+	result := output.Result
+	if !result.Passed && config.Fault.String() != "" && allAssertionsPassed(result.Assertions) {
+		result.Assertions = append(result.Assertions, scenario.Assertion{Name: contract.AssertionInjectedFault, Passed: false, Details: result.Error})
+	}
+	if output.Transfer != nil {
+		artifact := transferArtifact{Scenario: result.Name, Fault: config.Fault.String(), Passed: result.Passed, CleanupOK: result.CleanupOK, Error: evidence.Redact(result.Error), Evidence: *output.Transfer}
+		if err := writeJSONArtifact(config.Paths.ResultsDir().String(), "transfer.json", artifact); err != nil {
+			return err
+		}
+	}
+	if output.Reconnect != nil {
+		artifact := reconnectArtifact{Scenario: result.Name, Fault: config.Fault.String(), Passed: result.Passed, CleanupOK: result.CleanupOK, Error: evidence.Redact(result.Error), Evidence: *output.Reconnect}
+		if err := writeJSONArtifact(config.Paths.ResultsDir().String(), "reconnect.json", artifact); err != nil {
+			return err
+		}
+	}
+	assertions := make([]evidence.Assertion, 0, len(result.Assertions))
+	for _, assertion := range result.Assertions {
+		assertions = append(assertions, evidence.Assertion{Name: assertion.Name, Passed: assertion.Passed, Details: assertion.Details})
+	}
+	results := evidence.Results{Profile: string(config.Profile.Name()), Passed: result.Passed, Scenarios: []evidence.ScenarioResult{{Name: result.Name, Passed: result.Passed, Assertions: assertions, Error: result.Error}}}
+	data, err := evidence.MarshalResults(results)
+	if err != nil {
+		return fmt.Errorf("marshal scenario results: %w", err)
+	}
+	if err := writePrivateFile(filepath.Join(config.Paths.ResultsDir().String(), "results.json"), data); err != nil {
+		return fmt.Errorf("write scenario results: %w", err)
+	}
+	junit, err := evidence.JUnit(results)
+	if err != nil {
+		return fmt.Errorf("marshal scenario junit: %w", err)
+	}
+	if err := writePrivateFile(filepath.Join(config.Paths.ResultsDir().String(), "junit.xml"), junit); err != nil {
+		return fmt.Errorf("write scenario junit: %w", err)
+	}
+	cleanup := struct {
+		Passed     bool   `json:"passed"`
+		Scenario   string `json:"scenario"`
+		FinishedAt string `json:"finished_at"`
+	}{Passed: result.CleanupOK, Scenario: result.Name, FinishedAt: now.UTC().Format(time.RFC3339)}
+	return writeJSONArtifact(config.Paths.ResultsDir().String(), "cleanup.json", cleanup)
+}
+
+func writeJSONArtifact(resultsDir, name string, artifact any) error {
+	data, err := json.MarshalIndent(artifact, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal %s: %w", name, err)
+	}
+	if redacted := evidence.Redact(string(data)); redacted != string(data) {
+		return fmt.Errorf("credential detected while marshaling %s", name)
+	}
+	if err := writePrivateFile(filepath.Join(resultsDir, name), data); err != nil {
+		return fmt.Errorf("write %s: %w", name, err)
+	}
+	return nil
+}
+
+func writePrivateFile(path string, data []byte) error {
+	return writePrivateArtifact(path, data)
+}
+
+func allAssertionsPassed(assertions []scenario.Assertion) bool {
+	for _, assertion := range assertions {
+		if !assertion.Passed {
+			return false
+		}
+	}
+	return true
+}

integration/agentcompat/cmd/agentcompat/scenario_execution.go

@@ -0,0 +1,115 @@
+package main
+
+import (
+	"context"
+	"errors"
+
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/scenario"
+)
+
+type scenarioExecutionOutput struct {
+	Result    scenario.Result
+	Transfer  *scenario.TransferEvidence
+	Reconnect *scenario.ReconnectEvidence
+}
+
+func (output scenarioExecutionOutput) Validate() error {
+	if output.Result.Name == "" {
+		return errors.New("scenario execution result is missing")
+	}
+	dedicatedCount := 0
+	if output.Transfer != nil {
+		dedicatedCount++
+	}
+	if output.Reconnect != nil {
+		dedicatedCount++
+	}
+	switch output.Result.Name {
+	case contract.ScenarioTransfer100MiB:
+		if dedicatedCount != 1 || output.Transfer == nil {
+			return errors.New("transfer execution requires only transfer evidence")
+		}
+	case contract.ScenarioReconnect:
+		if dedicatedCount != 1 || output.Reconnect == nil {
+			return errors.New("reconnect execution requires only reconnect evidence")
+		}
+	default:
+		if dedicatedCount != 0 {
+			return errors.New("scenario execution has mismatched dedicated evidence")
+		}
+	}
+	return nil
+}
+
+type scenarioExecution struct {
+	name string
+	run  func(context.Context) (scenarioExecutionOutput, error)
+}
+
+var (
+	runRegistrationConfigExecScenario = (scenario.RegistrationConfigExec{}).Run
+	runNATScenario                    = (scenario.NAT{}).Run
+	runLegacyFMScenario               = (scenario.LegacyFM{}).Run
+	runTerminalScenario               = (scenario.Terminal{}).Run
+	runMCPFilesystemScenario          = (scenario.MCPFilesystem{}).Run
+	runTransferScenario               = (scenario.Transfer{}).RunWithEvidence
+	runReconnectScenario              = (scenario.Reconnect{}).RunWithEvidence
+)
+
+func selectScenarioExecution(config cliConfig) (scenarioExecution, error) {
+	if len(config.Scenarios) != 1 {
+		return scenarioExecution{}, errors.New("runtime execution requires exactly one --scenario")
+	}
+	selected := config.Scenarios[0]
+	if err := contract.ValidateScenarioFault(selected, config.Fault); err != nil {
+		return scenarioExecution{}, err
+	}
+	definition, err := contract.ScenarioDefinitionByName(selected.String())
+	if err != nil {
+		return scenarioExecution{}, err
+	}
+	switch definition.Execution {
+	case contract.ScenarioExecutionMetadata:
+		return scenarioExecution{}, errors.New("metadata scenario does not have a runtime execution")
+	case contract.ScenarioExecutionRegistrationConfigExec:
+		return standardExecution(selected.String(), func(ctx context.Context) (scenario.Result, error) {
+			return runRegistrationConfigExecScenario(ctx, scenario.RegistrationConfigExecInput{Paths: config.Paths, Fault: config.Fault})
+		}), nil
+	case contract.ScenarioExecutionNAT:
+		return standardExecution(selected.String(), func(ctx context.Context) (scenario.Result, error) {
+			return runNATScenario(ctx, scenario.NATInput{Paths: config.Paths, Fault: config.Fault})
+		}), nil
+	case contract.ScenarioExecutionLegacyFM:
+		return standardExecution(selected.String(), func(ctx context.Context) (scenario.Result, error) {
+			return runLegacyFMScenario(ctx, scenario.LegacyFMInput{Paths: config.Paths, Fault: config.Fault})
+		}), nil
+	case contract.ScenarioExecutionTerminal:
+		return standardExecution(selected.String(), func(ctx context.Context) (scenario.Result, error) {
+			return runTerminalScenario(ctx, scenario.TerminalInput{Paths: config.Paths, Fault: config.Fault})
+		}), nil
+	case contract.ScenarioExecutionMCPFilesystem:
+		return standardExecution(selected.String(), func(ctx context.Context) (scenario.Result, error) {
+			return runMCPFilesystemScenario(ctx, scenario.MCPFilesystemInput{Paths: config.Paths})
+		}), nil
+	case contract.ScenarioExecutionTransfer:
+		return scenarioExecution{name: selected.String(), run: func(ctx context.Context) (scenarioExecutionOutput, error) {
+			result, transferEvidence, err := runTransferScenario(ctx, scenario.TransferInput{Paths: config.Paths, Fault: config.Fault})
+			return scenarioExecutionOutput{Result: result, Transfer: &transferEvidence}, err
+		}}, nil
+	case contract.ScenarioExecutionReconnect:
+		return scenarioExecution{name: selected.String(), run: func(ctx context.Context) (scenarioExecutionOutput, error) {
+			result, reconnectEvidence, err := runReconnectScenario(ctx, scenario.ReconnectInput{Paths: config.Paths, DashboardFault: config.Fault.String()})
+			return scenarioExecutionOutput{Result: result, Reconnect: &reconnectEvidence}, err
+		}}, nil
+	default:
+		return scenarioExecution{}, errors.New("runtime execution is not implemented for the selected scenario")
+	}
+}
+
+func standardExecution(name string, run func(context.Context) (scenario.Result, error)) scenarioExecution {
+	return scenarioExecution{name: name, run: func(ctx context.Context) (scenarioExecutionOutput, error) {
+		result, err := run(ctx)
+		return scenarioExecutionOutput{Result: result}, err
+	}}
+}

integration/agentcompat/cmd/agentcompat/stale_artifact_test.go

@@ -0,0 +1,95 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/nezhahq/nezha/integration/agentcompat/internal/evidence"
+)
+
+func TestCLI_FailedGenerationRemovesPriorEvidence(t *testing.T) {
+	resultsDir := t.TempDir()
+	seedStaleEvidence(t, resultsDir)
+	var stdout, stderr bytes.Buffer
+	if err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "future-scenario"}, &stdout, &stderr, time.Now()); err == nil {
+		t.Fatal("unimplemented runtime unexpectedly succeeded")
+	}
+	for _, name := range append(evidence.FixedEvidenceFiles()[1:], "agents") {
+		if _, err := os.Stat(filepath.Join(resultsDir, name)); !os.IsNotExist(err) {
+			t.Fatalf("stale artifact remains: %s (%v)", name, err)
+		}
+	}
+}
+
+func TestCLI_FailedGenerationRemovesPriorEvidenceWhenMetadataIsSymlink(t *testing.T) {
+	resultsDir := t.TempDir()
+	target := filepath.Join(t.TempDir(), "metadata-target")
+	if err := os.WriteFile(target, []byte("old metadata"), 0o600); err != nil {
+		t.Fatalf("seed metadata target: %v", err)
+	}
+	if err := os.Symlink(target, filepath.Join(resultsDir, "metadata.json")); err != nil {
+		t.Fatalf("create metadata symlink: %v", err)
+	}
+	for _, name := range []string{"results.json", "junit.xml"} {
+		if err := os.WriteFile(filepath.Join(resultsDir, name), []byte("stale success"), 0o600); err != nil {
+			t.Fatalf("seed stale artifact %s: %v", name, err)
+		}
+	}
+	var stdout, stderr bytes.Buffer
+	if err := run([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--profile", "pr-full", "--results-dir", resultsDir, "--scenario", "future-scenario"}, &stdout, &stderr, time.Now()); err == nil {
+		t.Fatal("unimplemented runtime unexpectedly succeeded")
+	}
+	metadataInfo, err := os.Lstat(filepath.Join(resultsDir, "metadata.json"))
+	if err != nil || !metadataInfo.Mode().IsRegular() || metadataInfo.Mode().Perm() != 0o600 {
+		t.Fatalf("replacement metadata is not a private regular file: mode=%v err=%v", metadataInfo, err)
+	}
+	for _, name := range []string{"results.json", "junit.xml"} {
+		if _, err := os.Stat(filepath.Join(resultsDir, name)); !os.IsNotExist(err) {
+			t.Fatalf("stale artifact remains after failed invocation: %s (%v)", name, err)
+		}
+	}
+	data, err := os.ReadFile(target)
+	if err != nil || string(data) != "old metadata" {
+		t.Fatalf("metadata symlink target changed: %q %v", data, err)
+	}
+}
+
+func TestCLI_ParseFailureRemovesPriorEvidence(t *testing.T) {
+	for name, extraArgs := range map[string][]string{
+		"invalid profile": {"--profile", "invalid-profile"},
+		"invalid seed":    {"--profile", "pr-full", "--seed", "invalid-seed"},
+	} {
+		t.Run(name, func(t *testing.T) {
+			resultsDir := t.TempDir()
+			seedStaleEvidence(t, resultsDir)
+			args := append([]string{"--nezha-source", "/src/nezha", "--agent-source", "/src/agent", "--results-dir", resultsDir}, extraArgs...)
+			var stdout, stderr bytes.Buffer
+			if err := run(args, &stdout, &stderr, time.Now()); err == nil {
+				t.Fatal("invalid CLI input unexpectedly succeeded")
+			}
+			for _, artifact := range evidence.FixedEvidenceFiles()[1:] {
+				if _, err := os.Stat(filepath.Join(resultsDir, artifact)); !os.IsNotExist(err) {
+					t.Fatalf("stale artifact remains after parse failure: %s (%v)", artifact, err)
+				}
+			}
+		})
+	}
+}
+
+func seedStaleEvidence(t *testing.T, resultsDir string) {
+	t.Helper()
+	for _, name := range evidence.FixedEvidenceFiles()[1:] {
+		if err := os.WriteFile(filepath.Join(resultsDir, name), []byte("stale success"), 0o600); err != nil {
+			t.Fatalf("seed stale artifact %s: %v", name, err)
+		}
+	}
+	if err := os.Mkdir(filepath.Join(resultsDir, "agents"), 0o700); err != nil {
+		t.Fatalf("create stale agents directory: %v", err)
+	}
+	if err := os.WriteFile(filepath.Join(resultsDir, "agents", "old.log"), []byte("old success"), 0o600); err != nil {
+		t.Fatalf("seed stale agent log: %v", err)
+	}
+}