diff --git a/integration/agentcompat/internal/evidence/contract_metadata.go b/integration/agentcompat/internal/evidence/contract_metadata.go
new file mode 100644
index 00000000..65e93d25
--- /dev/null
+++ b/integration/agentcompat/internal/evidence/contract_metadata.go
@@ -0,0 +1,59 @@
+package evidence
+
+import (
+ "fmt"
+
+ "github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+)
+
+type ProfileMetadata struct {
+ Name string `json:"name"`
+ JobTimeoutSeconds int64 `json:"job_timeout_seconds"`
+ SuiteDeadlineSeconds int64 `json:"suite_deadline_seconds"`
+ DefaultSeed string `json:"default_seed"`
+ AgentCount int `json:"agent_count"`
+ StressRounds int `json:"stress_rounds"`
+ ConcurrentOperations int `json:"concurrent_operations"`
+ ConcurrentSessionsPerKind int `json:"concurrent_sessions_per_kind"`
+ TransferPairs int `json:"transfer_pairs"`
+ TransferBytes uint64 `json:"transfer_bytes"`
+ DashboardRestartCycles int `json:"dashboard_restart_cycles"`
+ Iterations int `json:"iterations"`
+ StreamBoundaryAllowed int `json:"stream_boundary_allowed"`
+ StreamBoundaryRejected int `json:"stream_boundary_rejected"`
+}
+
+type ResourceBudgetMetadata struct {
+ WarmupRunsPerPath int `json:"warmup_runs_per_path"`
+ BaselineSampleCount int `json:"baseline_sample_count"`
+ EndSampleCount int `json:"end_sample_count"`
+ SampleIntervalMilliseconds int64 `json:"sample_interval_milliseconds"`
+ ChildProcessCountDrift int `json:"child_process_count_drift"`
+ ListenerCountDrift int `json:"listener_count_drift"`
+ NonStdioFDCountDrift int `json:"non_stdio_fd_count_drift"`
+ DashboardRSSDeltaBytes uint64 `json:"dashboard_rss_delta_bytes"`
+ AgentRSSDeltaBytes uint64 `json:"agent_rss_delta_bytes"`
+ TransferHeapBytes uint64 `json:"transfer_heap_bytes"`
+}
+
+func profileMetadata(profile contract.Profile) ProfileMetadata {
+ return ProfileMetadata{Name: string(profile.Name()), JobTimeoutSeconds: int64(profile.JobTimeout().Seconds()), SuiteDeadlineSeconds: int64(profile.SuiteDeadline().Seconds()), DefaultSeed: fmt.Sprintf("0x%x", uint64(profile.Seed())), AgentCount: profile.AgentCount(), StressRounds: profile.StressRounds(), ConcurrentOperations: profile.ConcurrentOperations(), ConcurrentSessionsPerKind: profile.ConcurrentSessions(), TransferPairs: profile.TransferPairs(), TransferBytes: profile.TransferBytes(), DashboardRestartCycles: profile.DashboardRestartCycles(), Iterations: profile.Iterations(), StreamBoundaryAllowed: profile.StreamBoundaryAllowed(), StreamBoundaryRejected: profile.StreamBoundaryRejected()}
+}
+
+func resourceBudgetMetadata(budget contract.ResourceBudget) ResourceBudgetMetadata {
+ return ResourceBudgetMetadata{WarmupRunsPerPath: budget.WarmupRuns(), BaselineSampleCount: budget.SampleCount(), EndSampleCount: budget.SampleCount(), SampleIntervalMilliseconds: budget.SampleInterval().Milliseconds(), ChildProcessCountDrift: budget.ChildProcessCountDrift(), ListenerCountDrift: budget.ListenerCountDrift(), NonStdioFDCountDrift: budget.NonStdioFDCountDrift(), DashboardRSSDeltaBytes: budget.DashboardRSSDeltaBytes(), AgentRSSDeltaBytes: budget.AgentRSSDeltaBytes(), TransferHeapBytes: budget.TransferHeapBytes()}
+}
+
+func (p ProfileMetadata) Validate() error {
+ if p.Name == "" || p.JobTimeoutSeconds < 1 || p.SuiteDeadlineSeconds < 1 || p.DefaultSeed == "" || p.DefaultSeed == "0x0" || p.AgentCount < 1 || p.StressRounds < 1 || p.ConcurrentOperations < 1 || p.ConcurrentSessionsPerKind < 1 || p.TransferPairs < 1 || p.TransferBytes == 0 || p.DashboardRestartCycles < 1 || p.Iterations < 1 || p.StreamBoundaryAllowed < 1 || p.StreamBoundaryRejected <= p.StreamBoundaryAllowed {
+ return fmt.Errorf("profile fields are invalid")
+ }
+ return nil
+}
+
+func (b ResourceBudgetMetadata) Validate() error {
+ if b.WarmupRunsPerPath < 1 || b.BaselineSampleCount < 1 || b.EndSampleCount < 1 || b.SampleIntervalMilliseconds < 1 || b.ChildProcessCountDrift < 0 || b.ListenerCountDrift < 0 || b.NonStdioFDCountDrift < 0 || b.DashboardRSSDeltaBytes == 0 || b.AgentRSSDeltaBytes == 0 || b.TransferHeapBytes == 0 {
+ return fmt.Errorf("resource budget fields are invalid")
+ }
+ return nil
+}
diff --git a/integration/agentcompat/internal/evidence/dedicated_artifacts.go b/integration/agentcompat/internal/evidence/dedicated_artifacts.go
new file mode 100644
index 00000000..bdef7046
--- /dev/null
+++ b/integration/agentcompat/internal/evidence/dedicated_artifacts.go
@@ -0,0 +1,213 @@
+package evidence
+
+import (
+ "errors"
+ "strings"
+ "time"
+
+ "github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+)
+
+type transferEvidence struct {
+ WarmupUploadBytes uint64 `json:"warmup_upload_bytes"`
+ WarmupDownloadBytes uint64 `json:"warmup_download_bytes"`
+ WarmupSHA256 string `json:"warmup_sha256"`
+ WarmupDuration time.Duration `json:"warmup_duration"`
+ WarmupDeadlineRemaining time.Duration `json:"warmup_deadline_remaining"`
+ WarmupQuiescent bool `json:"warmup_quiescent"`
+ UploadBytes uint64 `json:"upload_bytes"`
+ DownloadBytes uint64 `json:"download_bytes"`
+ UploadSHA256 string `json:"upload_sha256"`
+ DownloadSHA256 string `json:"download_sha256"`
+ UploadChunks uint64 `json:"upload_chunks"`
+ DownloadChunks uint64 `json:"download_chunks"`
+ UploadDuration time.Duration `json:"upload_duration"`
+ DownloadDuration time.Duration `json:"download_duration"`
+ RetainedHeapBytes uint64 `json:"retained_heap_bytes"`
+ Mode string `json:"mode"`
+ CreateDirs bool `json:"create_dirs"`
+ UploadReplayRejected bool `json:"upload_replay_rejected"`
+ DownloadReplayRejected bool `json:"download_replay_rejected"`
+ OversizeRejected bool `json:"oversize_rejected"`
+ AgentTempResidue int `json:"agent_temp_residue"`
+ DashboardSpoolResidue int `json:"dashboard_spool_residue"`
+ OutsideRootSentinelsUnchanged bool `json:"outside_root_sentinels_unchanged"`
+}
+
+type transferArtifact struct {
+ Scenario string `json:"scenario"`
+ Fault string `json:"fault,omitempty"`
+ Passed bool `json:"passed"`
+ CleanupOK bool `json:"cleanup_ok"`
+ Error string `json:"error,omitempty"`
+ Evidence transferEvidence `json:"evidence"`
+}
+
+type reconnectObservation struct {
+ ServerID uint64
+ UUID string
+ OldGeneration uint64
+ NewGeneration uint64
+ DisconnectAt time.Time
+ ReconnectAt time.Time
+ TaskIDs []uint64
+ ResultIDs []uint64
+ PostReconnect bool
+ AgentRestarted bool
+}
+
+type listenerIdentity struct {
+ Address string
+ Inode uint64
+}
+
+type fixtureIdentity struct {
+ WorkspaceRoot string
+ ConfigPath string
+ DatabasePath string
+ BinaryPath string
+ HTTP listenerIdentity
+ Receipt listenerIdentity
+ HTTPS listenerIdentity
+}
+
+type runtimeIdentity struct {
+ Generation uint64
+ PID int
+ ProcessGroupID int
+}
+
+type receiptEvent struct {
+ Sequence uint64 `json:"sequence"`
+ DashboardGeneration uint64 `json:"dashboard_generation"`
+ GateGeneration uint64 `json:"gate_generation"`
+ ServerID uint64 `json:"server_id"`
+ TaskID uint64 `json:"task_id"`
+ TaskType uint64 `json:"task_type"`
+ Kind string `json:"kind"`
+}
+
+type receiptPair struct {
+ Task receiptEvent `json:"task"`
+ Result receiptEvent `json:"result"`
+}
+
+type cleanupRecord struct {
+ Name string `json:"name"`
+ PID int `json:"pid"`
+ Forced bool `json:"forced"`
+ Error string `json:"error,omitempty"`
+}
+
+type cleanupReceipt struct {
+ Passed bool `json:"passed"`
+ Forced bool `json:"forced"`
+ Processes []cleanupRecord `json:"processes"`
+}
+
+type reconnectEvidence struct {
+ Fixture struct {
+ Dashboard fixtureIdentity `json:"dashboard"`
+ AgentRoot string `json:"agent_root"`
+ AgentConfigPath string `json:"agent_config_path"`
+ AgentBinaryPath string `json:"agent_binary_path"`
+ } `json:"fixture"`
+ Runtime struct {
+ DashboardBefore runtimeIdentity `json:"dashboard_before"`
+ DashboardAfter runtimeIdentity `json:"dashboard_after"`
+ AgentBefore runtimeIdentity `json:"agent_before"`
+ AgentAfter runtimeIdentity `json:"agent_after"`
+ StateGenerationBeforeAgentRestart uint64 `json:"state_generation_before_agent_restart"`
+ StateGenerationAfterAgentRestart uint64 `json:"state_generation_after_agent_restart"`
+ } `json:"runtime"`
+ Identity struct {
+ ServerID uint64 `json:"server_id"`
+ UUID string `json:"uuid"`
+ DashboardConfigUnchanged bool `json:"dashboard_config_unchanged"`
+ AgentConfigUnchanged bool `json:"agent_config_unchanged"`
+ DashboardFixtureUnchanged bool `json:"dashboard_fixture_unchanged"`
+ ClientsRecreated bool `json:"clients_recreated"`
+ BootstrapRecreated bool `json:"bootstrap_recreated"`
+ } `json:"identity"`
+ Lifecycle struct {
+ DisconnectAt time.Time `json:"disconnect_at"`
+ ReconnectAt time.Time `json:"reconnect_at"`
+ ReconnectInterval time.Duration `json:"reconnect_interval"`
+ DashboardReceipts []receiptPair `json:"dashboard_receipts"`
+ AgentReceipts []receiptPair `json:"agent_receipts"`
+ StaleGenerationReceipts int `json:"stale_generation_receipts"`
+ DuplicateTaskIDs int `json:"duplicate_task_ids"`
+ LostResultIDs int `json:"lost_result_ids"`
+ OutsideRootSentinelUnchanged bool `json:"outside_root_sentinel_unchanged"`
+ } `json:"lifecycle"`
+ Observation reconnectObservation `json:"observation"`
+ AgentCleanup cleanupReceipt `json:"agent_cleanup"`
+ DashboardCleanup cleanupReceipt `json:"dashboard_cleanup"`
+}
+
+type reconnectArtifact struct {
+ Scenario string `json:"scenario"`
+ Fault string `json:"fault,omitempty"`
+ Passed bool `json:"passed"`
+ CleanupOK bool `json:"cleanup_ok"`
+ Error string `json:"error,omitempty"`
+ Evidence reconnectEvidence `json:"evidence"`
+}
+
+func validateTransferArtifact(metadata Metadata, result ScenarioResult, artifact transferArtifact) error {
+ if err := validateArtifactHeader(metadata, result, artifact.Scenario, artifact.Fault, artifact.Passed, artifact.CleanupOK, artifact.Error); err != nil {
+ return err
+ }
+ switch artifact.Fault {
+ case "":
+ if !artifact.Passed {
+ return errors.New("transfer success artifact reports failure")
+ }
+ if err := validateTransferSuccess(artifact.Evidence); err != nil {
+ return err
+ }
+ case contract.FaultTransferHash:
+ if artifact.Passed || !strings.Contains(artifact.Error, "injected hash mismatch") {
+ return errors.New("transfer-hash artifact does not identify the injected failure")
+ }
+ evidence := artifact.Evidence
+ if evidence.WarmupUploadBytes != 65536 || evidence.WarmupDownloadBytes != 65536 || evidence.WarmupSHA256 == "" || evidence.WarmupDuration <= 0 || evidence.WarmupDeadlineRemaining <= 0 || !evidence.WarmupQuiescent {
+ return errors.New("transfer-hash artifact omitted warmup evidence")
+ }
+ if evidence.UploadBytes != 0 || evidence.DownloadBytes != 0 || evidence.UploadSHA256 != "" || evidence.DownloadSHA256 != "" || evidence.UploadChunks != 0 || evidence.DownloadChunks != 0 || evidence.UploadDuration != 0 || evidence.DownloadDuration != 0 || evidence.RetainedHeapBytes != 0 || evidence.Mode != "" || evidence.CreateDirs || evidence.UploadReplayRejected || evidence.DownloadReplayRejected || evidence.OversizeRejected {
+ return errors.New("transfer-hash artifact presents stale success evidence")
+ }
+ if evidence.AgentTempResidue != 0 || evidence.DashboardSpoolResidue != 0 || !evidence.OutsideRootSentinelsUnchanged {
+ return errors.New("transfer-hash artifact cleanup or sentinel evidence failed")
+ }
+ default:
+ return errors.New("transfer artifact has unsupported fault")
+ }
+ return nil
+}
+
+func validateArtifactHeader(metadata Metadata, result ScenarioResult, scenarioName, fault string, passed, cleanupOK bool, errorText string) error {
+ if scenarioName != result.Name || scenarioName != metadata.Scenarios[0] || fault != metadata.Fault || passed != result.Passed || !cleanupOK || errorText != result.Error {
+ return errors.New("dedicated artifact does not agree with metadata, results, or cleanup")
+ }
+ if passed && errorText != "" || !passed && errorText == "" {
+ return errors.New("dedicated artifact pass/error state is inconsistent")
+ }
+ return nil
+}
+
+func validateTransferSuccess(evidence transferEvidence) error {
+ if evidence.WarmupUploadBytes != 65536 || evidence.WarmupDownloadBytes != 65536 || evidence.WarmupSHA256 == "" || evidence.WarmupDuration <= 0 || evidence.WarmupDeadlineRemaining <= 0 || !evidence.WarmupQuiescent {
+ return errors.New("transfer warmup evidence is invalid")
+ }
+ if evidence.UploadBytes != contract.TransferBytes || evidence.DownloadBytes != contract.TransferBytes || evidence.UploadSHA256 == "" || !strings.EqualFold(evidence.UploadSHA256, evidence.DownloadSHA256) {
+ return errors.New("transfer byte counts or hashes are invalid")
+ }
+ if evidence.UploadChunks == 0 || evidence.DownloadChunks == 0 || evidence.UploadDuration <= 0 || evidence.DownloadDuration <= 0 || evidence.RetainedHeapBytes > contract.TransferHeapBytes {
+ return errors.New("transfer measurement evidence is invalid")
+ }
+ if evidence.Mode != "0640" || !evidence.CreateDirs || !evidence.UploadReplayRejected || !evidence.DownloadReplayRejected || !evidence.OversizeRejected || evidence.AgentTempResidue != 0 || evidence.DashboardSpoolResidue != 0 || !evidence.OutsideRootSentinelsUnchanged {
+ return errors.New("transfer contract evidence is invalid")
+ }
+ return nil
+}
diff --git a/integration/agentcompat/internal/evidence/dedicated_assertion_test.go b/integration/agentcompat/internal/evidence/dedicated_assertion_test.go
new file mode 100644
index 00000000..fa5f49e6
--- /dev/null
+++ b/integration/agentcompat/internal/evidence/dedicated_assertion_test.go
@@ -0,0 +1,46 @@
+package evidence
+
+import (
+ "testing"
+
+ "github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+)
+
+func TestEvidence_DedicatedArtifactsRequireExactScenarioAssertions(t *testing.T) {
+ tests := []struct {
+ name string
+ scenario string
+ assertion string
+ artifact func(*testing.T, string)
+ }{
+ {"transfer generic", contract.ScenarioTransfer100MiB, "scenario passed", func(t *testing.T, dir string) {
+ writeJSONEvidenceFile(t, dir, "transfer.json", validTransferArtifact("", true))
+ }},
+ {"transfer mismatched", contract.ScenarioTransfer100MiB, contract.AssertionReconnectDisconnect, func(t *testing.T, dir string) {
+ writeJSONEvidenceFile(t, dir, "transfer.json", validTransferArtifact("", true))
+ }},
+ {"reconnect generic", contract.ScenarioReconnect, "scenario passed", func(t *testing.T, dir string) {
+ writeJSONEvidenceFile(t, dir, "reconnect.json", validReconnectArtifact("", true))
+ }},
+ {"reconnect mismatched", contract.ScenarioReconnect, contract.AssertionTransferWarmup, func(t *testing.T, dir string) {
+ writeJSONEvidenceFile(t, dir, "reconnect.json", validReconnectArtifact("", true))
+ }},
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dir := t.TempDir()
+ writeDedicatedExecutableEvidence(t, dir, test.scenario, "", true)
+ results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: test.scenario, Passed: true, Assertions: []Assertion{{Name: test.assertion, Passed: true}}}}}
+ writeJSONEvidenceFile(t, dir, "results.json", results)
+ junit, err := JUnit(results)
+ if err != nil {
+ t.Fatalf("JUnit: %v", err)
+ }
+ writeEvidenceFile(t, dir, "junit.xml", string(junit))
+ test.artifact(t, dir)
+ if err := ValidateDirectory(dir); err == nil {
+ t.Fatal("generic assertion accepted for dedicated evidence")
+ }
+ })
+ }
+}
diff --git a/integration/agentcompat/internal/evidence/dedicated_fault_fields_test.go b/integration/agentcompat/internal/evidence/dedicated_fault_fields_test.go
new file mode 100644
index 00000000..7f929637
--- /dev/null
+++ b/integration/agentcompat/internal/evidence/dedicated_fault_fields_test.go
@@ -0,0 +1,86 @@
+package evidence
+
+import (
+ "testing"
+ "time"
+)
+
+func TestEvidence_TransferHashRejectsEverySuccessOnlyField(t *testing.T) {
+ tests := []struct {
+ name string
+ mutate func(*transferEvidence)
+ }{
+ {"upload bytes", func(value *transferEvidence) { value.UploadBytes = 1 }},
+ {"download bytes", func(value *transferEvidence) { value.DownloadBytes = 1 }},
+ {"upload hash", func(value *transferEvidence) { value.UploadSHA256 = "stale" }},
+ {"download hash", func(value *transferEvidence) { value.DownloadSHA256 = "stale" }},
+ {"upload chunks", func(value *transferEvidence) { value.UploadChunks = 1 }},
+ {"download chunks", func(value *transferEvidence) { value.DownloadChunks = 1 }},
+ {"upload duration", func(value *transferEvidence) { value.UploadDuration = time.Nanosecond }},
+ {"download duration", func(value *transferEvidence) { value.DownloadDuration = time.Nanosecond }},
+ {"retained heap", func(value *transferEvidence) { value.RetainedHeapBytes = 1 }},
+ {"mode", func(value *transferEvidence) { value.Mode = "0640" }},
+ {"create dirs", func(value *transferEvidence) { value.CreateDirs = true }},
+ {"upload replay", func(value *transferEvidence) { value.UploadReplayRejected = true }},
+ {"download replay", func(value *transferEvidence) { value.DownloadReplayRejected = true }},
+ {"oversize", func(value *transferEvidence) { value.OversizeRejected = true }},
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dir := t.TempDir()
+ writeDedicatedExecutableEvidence(t, dir, "transfer-100mib", "transfer-hash", false)
+ artifact := validTransferArtifact("transfer-hash", false)
+ test.mutate(&artifact.Evidence)
+ writeJSONEvidenceFile(t, dir, "transfer.json", artifact)
+ if err := ValidateDirectory(dir); err == nil {
+ t.Fatal("stale transfer success field accepted")
+ }
+ })
+ }
+}
+
+func TestEvidence_DashboardExitRejectsEveryPostFaultField(t *testing.T) {
+ tests := []struct {
+ name string
+ mutate func(*reconnectEvidence)
+ }{
+ {"dashboard after", func(value *reconnectEvidence) { value.Runtime.DashboardAfter.PID = 1 }},
+ {"agent after", func(value *reconnectEvidence) { value.Runtime.AgentAfter.PID = 1 }},
+ {"state before restart", func(value *reconnectEvidence) { value.Runtime.StateGenerationBeforeAgentRestart = 1 }},
+ {"state after restart", func(value *reconnectEvidence) { value.Runtime.StateGenerationAfterAgentRestart = 1 }},
+ {"identity dashboard config", func(value *reconnectEvidence) { value.Identity.DashboardConfigUnchanged = true }},
+ {"identity agent config", func(value *reconnectEvidence) { value.Identity.AgentConfigUnchanged = true }},
+ {"identity fixture", func(value *reconnectEvidence) { value.Identity.DashboardFixtureUnchanged = true }},
+ {"clients recreated", func(value *reconnectEvidence) { value.Identity.ClientsRecreated = true }},
+ {"bootstrap recreated", func(value *reconnectEvidence) { value.Identity.BootstrapRecreated = true }},
+ {"reconnect timestamp", func(value *reconnectEvidence) { value.Lifecycle.ReconnectAt = time.Now() }},
+ {"reconnect interval", func(value *reconnectEvidence) { value.Lifecycle.ReconnectInterval = time.Second }},
+ {"dashboard receipts", func(value *reconnectEvidence) { value.Lifecycle.DashboardReceipts = []receiptPair{{}} }},
+ {"agent receipts", func(value *reconnectEvidence) { value.Lifecycle.AgentReceipts = []receiptPair{{}} }},
+ {"stale accounting", func(value *reconnectEvidence) { value.Lifecycle.StaleGenerationReceipts = 1 }},
+ {"duplicate accounting", func(value *reconnectEvidence) { value.Lifecycle.DuplicateTaskIDs = 1 }},
+ {"lost accounting", func(value *reconnectEvidence) { value.Lifecycle.LostResultIDs = 1 }},
+ {"observation server", func(value *reconnectEvidence) { value.Observation.ServerID = 1 }},
+ {"observation uuid", func(value *reconnectEvidence) { value.Observation.UUID = "stale" }},
+ {"old generation", func(value *reconnectEvidence) { value.Observation.OldGeneration = 1 }},
+ {"new generation", func(value *reconnectEvidence) { value.Observation.NewGeneration = 1 }},
+ {"disconnect observation", func(value *reconnectEvidence) { value.Observation.DisconnectAt = time.Now() }},
+ {"reconnect observation", func(value *reconnectEvidence) { value.Observation.ReconnectAt = time.Now() }},
+ {"task ids", func(value *reconnectEvidence) { value.Observation.TaskIDs = []uint64{1} }},
+ {"result ids", func(value *reconnectEvidence) { value.Observation.ResultIDs = []uint64{1} }},
+ {"post reconnect", func(value *reconnectEvidence) { value.Observation.PostReconnect = true }},
+ {"agent restarted", func(value *reconnectEvidence) { value.Observation.AgentRestarted = true }},
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dir := t.TempDir()
+ writeDedicatedExecutableEvidence(t, dir, "reconnect", "dashboard-exit", false)
+ artifact := validReconnectArtifact("dashboard-exit", false)
+ test.mutate(&artifact.Evidence)
+ writeJSONEvidenceFile(t, dir, "reconnect.json", artifact)
+ if err := ValidateDirectory(dir); err == nil {
+ t.Fatal("stale reconnect success field accepted")
+ }
+ })
+ }
+}
diff --git a/integration/agentcompat/internal/evidence/dedicated_validation_test.go b/integration/agentcompat/internal/evidence/dedicated_validation_test.go
new file mode 100644
index 00000000..3e488c3f
--- /dev/null
+++ b/integration/agentcompat/internal/evidence/dedicated_validation_test.go
@@ -0,0 +1,212 @@
+package evidence
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+)
+
+func TestEvidence_TransferAndReconnectRequireDedicatedArtifacts(t *testing.T) {
+ for _, scenarioName := range []string{"transfer-100mib", "reconnect"} {
+ t.Run(scenarioName, func(t *testing.T) {
+ dir := t.TempDir()
+ writeExecutableEvidence(t, dir, "pr-full", scenarioName, true, true, true)
+ if err := ValidateDirectory(dir); err == nil {
+ t.Fatal("missing dedicated artifact accepted")
+ }
+ })
+ }
+}
+
+func TestEvidence_ValidatesTypedTransferSuccessAndFaultArtifacts(t *testing.T) {
+ tests := []struct {
+ name string
+ fault string
+ passed bool
+ }{
+ {"success", "", true},
+ {"hash fault", "transfer-hash", false},
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dir := t.TempDir()
+ writeDedicatedExecutableEvidence(t, dir, "transfer-100mib", test.fault, test.passed)
+ artifact := validTransferArtifact(test.fault, test.passed)
+ writeJSONEvidenceFile(t, dir, "transfer.json", artifact)
+ if err := ValidateDirectory(dir); err != nil {
+ t.Fatalf("validate transfer evidence: %v", err)
+ }
+ })
+ }
+}
+
+func TestEvidence_ValidatesTypedReconnectSuccessAndFaultArtifacts(t *testing.T) {
+ tests := []struct {
+ name string
+ fault string
+ passed bool
+ }{
+ {"success", "", true},
+ {"dashboard fault", "dashboard-exit", false},
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dir := t.TempDir()
+ writeDedicatedExecutableEvidence(t, dir, "reconnect", test.fault, test.passed)
+ artifact := validReconnectArtifact(test.fault, test.passed)
+ writeJSONEvidenceFile(t, dir, "reconnect.json", artifact)
+ if err := ValidateDirectory(dir); err != nil {
+ t.Fatalf("validate reconnect evidence: %v", err)
+ }
+ })
+ }
+}
+
+func TestEvidence_RejectsMisleadingDedicatedArtifact(t *testing.T) {
+ dir := t.TempDir()
+ writeDedicatedExecutableEvidence(t, dir, "transfer-100mib", "transfer-hash", false)
+ artifact := validTransferArtifact("transfer-hash", false)
+ artifact.Evidence.UploadBytes = 104857600
+ writeJSONEvidenceFile(t, dir, "transfer.json", artifact)
+ if err := ValidateDirectory(dir); err == nil {
+ t.Fatal("fault artifact containing stale success accepted")
+ }
+}
+
+func TestEvidence_RejectsWrongOrStaleDedicatedArtifact(t *testing.T) {
+ dir := t.TempDir()
+ writeExecutableEvidence(t, dir, "pr-full", "transfer-100mib", true, true, true)
+ writeEvidenceFile(t, dir, "reconnect.json", `{}`)
+ if err := ValidateDirectory(dir); err == nil {
+ t.Fatal("wrong dedicated artifact accepted")
+ }
+}
+
+func TestEvidence_RejectsDedicatedArtifactWithPublicMode(t *testing.T) {
+ dir := t.TempDir()
+ writeExecutableEvidence(t, dir, "pr-full", "transfer-100mib", true, true, true)
+ path := filepath.Join(dir, "transfer.json")
+ if err := os.WriteFile(path, []byte(`{}`), 0o644); err != nil {
+ t.Fatalf("write transfer artifact: %v", err)
+ }
+ if err := ValidateDirectory(dir); err == nil {
+ t.Fatal("public dedicated artifact mode accepted")
+ }
+}
+
+func writeDedicatedExecutableEvidence(t *testing.T, dir, scenarioName, fault string, passed bool) {
+ t.Helper()
+ metadata := validMetadata(t, dir, scenarioName)
+ metadata.Fault = fault
+ writeJSONEvidenceFile(t, dir, "metadata.json", metadata)
+ errorText := ""
+ if !passed {
+ if scenarioName == "transfer-100mib" {
+ errorText = "transfer scenario: injected hash mismatch"
+ } else {
+ errorText = "reconnect scenario: injected Dashboard exit"
+ }
+ }
+ definition, err := contract.ScenarioDefinitionByName(scenarioName)
+ if err != nil {
+ t.Fatalf("scenario definition: %v", err)
+ }
+ assertions := make([]Assertion, 0, len(definition.Assertions(fault)))
+ for _, assertion := range definition.Assertions(fault) {
+ assertions = append(assertions, Assertion{Name: assertion.Name, Passed: assertion.Passed})
+ }
+ results := Results{Profile: "pr-full", Passed: passed, Scenarios: []ScenarioResult{{Name: scenarioName, Passed: passed, Assertions: assertions, Error: errorText}}}
+ writeJSONEvidenceFile(t, dir, "results.json", results)
+ junit, err := JUnit(results)
+ if err != nil {
+ t.Fatalf("JUnit: %v", err)
+ }
+ writeEvidenceFile(t, dir, "junit.xml", string(junit))
+ writeEvidenceFile(t, dir, "cleanup.json", fmt.Sprintf(`{"passed":true,"scenario":%q,"finished_at":"2026-01-02T03:04:05Z"}`, scenarioName))
+}
+
+func validTransferArtifact(fault string, passed bool) transferArtifact {
+ errorText := ""
+ if !passed {
+ errorText = "transfer scenario: injected hash mismatch"
+ }
+ evidence := transferEvidence{WarmupUploadBytes: 65536, WarmupDownloadBytes: 65536, WarmupSHA256: "abc", WarmupDuration: time.Nanosecond, WarmupDeadlineRemaining: time.Second, WarmupQuiescent: true, OutsideRootSentinelsUnchanged: true}
+ if passed {
+ evidence.UploadBytes = 104857600
+ evidence.DownloadBytes = 104857600
+ evidence.UploadSHA256 = "abc"
+ evidence.DownloadSHA256 = "abc"
+ evidence.UploadChunks = 2
+ evidence.DownloadChunks = 2
+ evidence.UploadDuration = time.Second
+ evidence.DownloadDuration = time.Second
+ evidence.Mode = "0640"
+ evidence.CreateDirs = true
+ evidence.UploadReplayRejected = true
+ evidence.DownloadReplayRejected = true
+ evidence.OversizeRejected = true
+ }
+ return transferArtifact{Scenario: "transfer-100mib", Fault: fault, Passed: passed, CleanupOK: true, Error: errorText, Evidence: evidence}
+}
+
+func validReconnectArtifact(fault string, passed bool) reconnectArtifact {
+ var artifact reconnectArtifact
+ artifact.Scenario = "reconnect"
+ artifact.Fault = fault
+ artifact.Passed = passed
+ artifact.CleanupOK = true
+ if !passed {
+ artifact.Error = "reconnect scenario: injected Dashboard exit"
+ }
+ evidence := &artifact.Evidence
+ evidence.Fixture.Dashboard.WorkspaceRoot = "/tmp/dashboard"
+ evidence.Fixture.Dashboard.ConfigPath = "/tmp/dashboard/config"
+ evidence.Fixture.Dashboard.BinaryPath = "/tmp/dashboard/bin"
+ evidence.Fixture.AgentRoot = "/tmp/agent"
+ evidence.Fixture.AgentConfigPath = "/tmp/agent/config"
+ evidence.Fixture.AgentBinaryPath = "/tmp/agent/bin"
+ evidence.Runtime.DashboardBefore = runtimeIdentity{Generation: 1, PID: 10, ProcessGroupID: 10}
+ evidence.Runtime.AgentBefore = runtimeIdentity{Generation: 1, PID: 20, ProcessGroupID: 20}
+ evidence.Identity.ServerID = 1
+ evidence.Identity.UUID = "uuid"
+ evidence.Lifecycle.DisconnectAt = time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+ evidence.Lifecycle.OutsideRootSentinelUnchanged = true
+ evidence.AgentCleanup = validCleanupReceipt("agent", expectedProcessCount(passed))
+ evidence.DashboardCleanup = validCleanupReceipt("dashboard", expectedProcessCount(passed))
+ if passed {
+ evidence.Runtime.DashboardAfter = runtimeIdentity{Generation: 2, PID: 11, ProcessGroupID: 11}
+ evidence.Runtime.AgentAfter = runtimeIdentity{Generation: 2, PID: 21, ProcessGroupID: 21}
+ evidence.Runtime.StateGenerationBeforeAgentRestart = 1
+ evidence.Runtime.StateGenerationAfterAgentRestart = 2
+ evidence.Identity.DashboardConfigUnchanged = true
+ evidence.Identity.AgentConfigUnchanged = true
+ evidence.Identity.DashboardFixtureUnchanged = true
+ evidence.Identity.ClientsRecreated = true
+ evidence.Identity.BootstrapRecreated = true
+ evidence.Lifecycle.ReconnectAt = evidence.Lifecycle.DisconnectAt.Add(time.Second)
+ evidence.Lifecycle.ReconnectInterval = time.Second
+ evidence.Lifecycle.DashboardReceipts = make([]receiptPair, 3)
+ evidence.Lifecycle.AgentReceipts = make([]receiptPair, 2)
+ evidence.Observation = reconnectObservation{ServerID: 1, UUID: "uuid", OldGeneration: 1, NewGeneration: 2, DisconnectAt: evidence.Lifecycle.DisconnectAt, ReconnectAt: evidence.Lifecycle.ReconnectAt, TaskIDs: []uint64{1, 2, 3, 4, 5}, ResultIDs: []uint64{1, 2, 3, 4, 5}, PostReconnect: true, AgentRestarted: true}
+ }
+ return artifact
+}
+
+func validCleanupReceipt(name string, count int) cleanupReceipt {
+ receipt := cleanupReceipt{Passed: true, Processes: make([]cleanupRecord, count)}
+ for index := range receipt.Processes {
+ receipt.Processes[index] = cleanupRecord{Name: name, PID: index + 1}
+ }
+ return receipt
+}
+
+func expectedProcessCount(passed bool) int {
+ if passed {
+ return 2
+ }
+ return 1
+}
diff --git a/integration/agentcompat/internal/evidence/directory_scan.go b/integration/agentcompat/internal/evidence/directory_scan.go
new file mode 100644
index 00000000..0e02b4ca
--- /dev/null
+++ b/integration/agentcompat/internal/evidence/directory_scan.go
@@ -0,0 +1,133 @@
+package evidence
+
+import (
+ "encoding/json"
+ "encoding/xml"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+const (
+ maxEvidenceFiles = 128
+ maxEvidenceBytes = 32 << 20
+)
+
+func scanDirectory(resultsDir string) (map[string]os.FileInfo, error) {
+ if strings.TrimSpace(resultsDir) == "" {
+ return nil, errors.New("evidence directory is required")
+ }
+ info, err := os.Lstat(resultsDir)
+ if err != nil {
+ return nil, fmt.Errorf("stat evidence directory: %w", err)
+ }
+ if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
+ return nil, errors.New("evidence path must be a directory")
+ }
+ if info.Mode().Perm() != 0o700 {
+ return nil, errors.New("evidence directory must use mode 0700")
+ }
+ seen := make(map[string]os.FileInfo)
+ var totalBytes int64
+ err = filepath.WalkDir(resultsDir, func(path string, entry os.DirEntry, walkErr error) error {
+ if walkErr != nil {
+ return walkErr
+ }
+ if path == resultsDir {
+ return nil
+ }
+ if entry.IsDir() {
+ relative, err := filepath.Rel(resultsDir, path)
+ if err != nil {
+ return err
+ }
+ if relative != "agents" {
+ return fmt.Errorf("evidence path is not allowed: %s", relative)
+ }
+ return nil
+ }
+ if entry.Type()&os.ModeSymlink != 0 {
+ return fmt.Errorf("evidence symlink is not allowed: %s", path)
+ }
+ if !entry.Type().IsRegular() {
+ return fmt.Errorf("evidence file is not regular: %s", path)
+ }
+ relative, err := filepath.Rel(resultsDir, path)
+ if err != nil {
+ return err
+ }
+ fileInfo, err := entry.Info()
+ if err != nil {
+ return err
+ }
+ if !allowedEvidencePath(relative) {
+ return fmt.Errorf("evidence path is not allowed: %s", relative)
+ }
+ if fileInfo.Mode().Perm() != 0o600 {
+ return fmt.Errorf("evidence file must use mode 0600: %s", relative)
+ }
+ seen[relative] = fileInfo
+ if len(seen) > maxEvidenceFiles {
+ return errors.New("too many evidence files")
+ }
+ totalBytes += fileInfo.Size()
+ if totalBytes > maxEvidenceBytes {
+ return errors.New("evidence files exceed size limit")
+ }
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return err
+ }
+ if Redact(string(data)) != string(data) {
+ return fmt.Errorf("credential detected in evidence file: %s", path)
+ }
+ switch filepath.Ext(path) {
+ case ".json":
+ if !json.Valid(data) {
+ return fmt.Errorf("invalid JSON evidence file: %s", path)
+ }
+ case ".xml":
+ var document any
+ if err := xml.Unmarshal(data, &document); err != nil {
+ return fmt.Errorf("invalid XML evidence file: %s: %w", path, err)
+ }
+ }
+ return nil
+ })
+ return seen, err
+}
+
+func allowedEvidencePath(relative string) bool {
+ for _, name := range EvidenceFiles() {
+ if name == relative {
+ return true
+ }
+ }
+ if filepath.Dir(relative) != "agents" || filepath.Ext(relative) != ".log" {
+ return false
+ }
+ base := strings.TrimSuffix(filepath.Base(relative), ".log")
+ if base == "" || base == "." {
+ return false
+ }
+ for _, character := range base {
+ if character != '-' && character != '_' && character != '.' && (character < '0' || character > '9') && (character < 'A' || character > 'Z') && (character < 'a' || character > 'z') {
+ return false
+ }
+ }
+ return true
+}
+
+func readJSONFile[T any](resultsDir, name string) (T, error) {
+ var value T
+ data, err := os.ReadFile(filepath.Join(resultsDir, name))
+ if err != nil {
+ return value, fmt.Errorf("read %s: %w", name, err)
+ }
+ if err := json.Unmarshal(data, &value); err != nil {
+ return value, fmt.Errorf("parse %s: %w", name, err)
+ }
+ return value, nil
+}
diff --git a/integration/agentcompat/internal/evidence/directory_security_test.go b/integration/agentcompat/internal/evidence/directory_security_test.go
new file mode 100644
index 00000000..dcdc339c
--- /dev/null
+++ b/integration/agentcompat/internal/evidence/directory_security_test.go
@@ -0,0 +1,69 @@
+package evidence
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+)
+
+func TestEvidence_ValidateDirectoryRejectsPublicRootOrEvidenceFile(t *testing.T) {
+ tests := []struct {
+ name string
+ path string
+ }{
+ {name: "root", path: ""},
+ {name: "metadata", path: "metadata.json"},
+ {name: "results", path: "results.json"},
+ {name: "junit", path: "junit.xml"},
+ {name: "cleanup", path: "cleanup.json"},
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dir := t.TempDir()
+ writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, true, true, true)
+ path := dir
+ if test.path != "" {
+ path = filepath.Join(dir, test.path)
+ }
+ if err := os.Chmod(path, 0o644); err != nil {
+ t.Fatalf("chmod fixture: %v", err)
+ }
+ if err := ValidateDirectory(dir); err == nil {
+ t.Fatal("public evidence mode accepted")
+ }
+ })
+ }
+}
+
+func TestEvidence_ValidateDirectoryRejectsUnexpectedPaths(t *testing.T) {
+ tests := []struct {
+ name string
+ path string
+ mode os.FileMode
+ }{
+ {name: "payload", path: "payload.bin", mode: 0o600},
+ {name: "executable", path: "agentcompat", mode: 0o700},
+ {name: "nested log", path: "agents/nested/agent.log", mode: 0o600},
+ {name: "unknown agent file", path: "agents/agent.bin", mode: 0o600},
+ {name: "unknown directory", path: "unexpected/file.log", mode: 0o600},
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dir := t.TempDir()
+ writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, true, true, true)
+ path := filepath.Join(dir, test.path)
+ if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+ t.Fatalf("mkdir unexpected parent: %v", err)
+ }
+ if err := os.WriteFile(path, []byte("unexpected"), test.mode); err != nil {
+ t.Fatalf("write unexpected file: %v", err)
+ }
+ if err := ValidateDirectory(dir); err == nil || !strings.Contains(err.Error(), "not allowed") {
+ t.Fatalf("unexpected path rejection err=%v", err)
+ }
+ })
+ }
+}
diff --git a/integration/agentcompat/internal/evidence/directory_validation.go b/integration/agentcompat/internal/evidence/directory_validation.go
new file mode 100644
index 00000000..4cc40c79
--- /dev/null
+++ b/integration/agentcompat/internal/evidence/directory_validation.go
@@ -0,0 +1,174 @@
+package evidence
+
+import (
+ "encoding/xml"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "slices"
+ "time"
+
+ "github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+)
+
+type cleanupEvidence struct {
+ Passed bool `json:"passed"`
+ Scenario string `json:"scenario"`
+ FinishedAt string `json:"finished_at"`
+}
+
+type currentEvidenceProfile struct {
+ RequiredFiles []string
+ DedicatedFile string
+ Executable bool
+}
+
+func currentProfile(metadata Metadata) (currentEvidenceProfile, error) {
+ if len(metadata.Scenarios) == 0 {
+ return currentEvidenceProfile{}, errors.New("metadata scenarios are required")
+ }
+ if len(metadata.Scenarios) == 1 && metadata.Scenarios[0] == contract.ScenarioMetadata {
+ if metadata.Fault != "" {
+ return currentEvidenceProfile{}, errors.New("metadata scenario does not support fault injection")
+ }
+ return currentEvidenceProfile{RequiredFiles: []string{"metadata.json"}}, nil
+ }
+ if len(metadata.Scenarios) != 1 {
+ return currentEvidenceProfile{}, errors.New("current evidence profile requires one supported scenario")
+ }
+ scenarioValue, err := contract.NewScenario(metadata.Scenarios[0])
+ if err != nil {
+ return currentEvidenceProfile{}, fmt.Errorf("construct metadata scenario: %w", err)
+ }
+ if !contract.IsSupportedScenario(metadata.Scenarios[0]) || metadata.Scenarios[0] == contract.ScenarioMetadata {
+ return currentEvidenceProfile{}, errors.New("current evidence profile requires one supported scenario")
+ }
+ faultValue := contract.Fault{}
+ if metadata.Fault != "" {
+ faultValue, err = contract.NewFault(metadata.Fault)
+ if err != nil {
+ return currentEvidenceProfile{}, fmt.Errorf("construct metadata fault: %w", err)
+ }
+ }
+ if err := contract.ValidateScenarioFault(scenarioValue, faultValue); err != nil {
+ return currentEvidenceProfile{}, err
+ }
+ definition, err := contract.ScenarioDefinitionByName(metadata.Scenarios[0])
+ if err != nil {
+ return currentEvidenceProfile{}, err
+ }
+ profile := currentEvidenceProfile{RequiredFiles: []string{"metadata.json", "results.json", "junit.xml", "cleanup.json"}, DedicatedFile: definition.DedicatedArtifactName(), Executable: true}
+ if profile.DedicatedFile != "" {
+ profile.RequiredFiles = append(profile.RequiredFiles, profile.DedicatedFile)
+ }
+ return profile, nil
+}
+
+func ValidateDirectory(resultsDir string) error {
+ files, err := scanDirectory(resultsDir)
+ if err != nil {
+ return err
+ }
+ if len(files) == 0 {
+ return errors.New("evidence directory contains no files")
+ }
+ metadata, err := readJSONFile[Metadata](resultsDir, "metadata.json")
+ if err != nil {
+ return err
+ }
+ if err := metadata.Validate(); err != nil {
+ return fmt.Errorf("validate metadata evidence: %w", err)
+ }
+ profile, err := currentProfile(metadata)
+ if err != nil {
+ return err
+ }
+ for _, required := range profile.RequiredFiles {
+ if _, exists := files[required]; !exists {
+ return fmt.Errorf("required evidence file is missing: %s", required)
+ }
+ }
+ if err := rejectStaleDedicatedFiles(files, profile.DedicatedFile); err != nil {
+ return err
+ }
+ if !profile.Executable {
+ return nil
+ }
+ results, err := readJSONFile[Results](resultsDir, "results.json")
+ if err != nil {
+ return err
+ }
+ if err := results.Validate(); err != nil {
+ return fmt.Errorf("validate results evidence: %w", err)
+ }
+ if results.Profile != metadata.Profile.Name || !slices.Equal(metadata.Scenarios, scenarioResultNames(results.Scenarios)) {
+ return errors.New("metadata and results do not agree")
+ }
+ if err := validateJUnit(resultsDir, results); err != nil {
+ return err
+ }
+ cleanup, err := readJSONFile[cleanupEvidence](resultsDir, "cleanup.json")
+ if err != nil {
+ return err
+ }
+ if !cleanup.Passed || cleanup.Scenario != results.Scenarios[0].Name {
+ return errors.New("cleanup evidence is missing or failed")
+ }
+ if _, err := time.Parse(time.RFC3339, cleanup.FinishedAt); err != nil {
+ return errors.New("cleanup finish time is invalid")
+ }
+ if profile.DedicatedFile != "" {
+ definition, err := contract.ScenarioDefinitionByName(results.Scenarios[0].Name)
+ if err != nil {
+ return err
+ }
+ if err := validateScenarioAssertions(results.Scenarios[0], definition.Assertions(metadata.Fault)); err != nil {
+ return err
+ }
+ return validateDedicatedArtifact(resultsDir, metadata, results.Scenarios[0])
+ }
+ return nil
+}
+
+func validateScenarioAssertions(result ScenarioResult, expected []contract.AssertionDefinition) error {
+ if len(result.Assertions) != len(expected) {
+ return errors.New("scenario assertions do not match dedicated evidence contract")
+ }
+ for index, assertion := range result.Assertions {
+ if assertion.Name != expected[index].Name || assertion.Passed != expected[index].Passed {
+ return errors.New("scenario assertions do not match dedicated evidence contract")
+ }
+ }
+ return nil
+}
+
+func rejectStaleDedicatedFiles(files map[string]os.FileInfo, expected string) error {
+ for _, name := range []string{"transfer.json", "reconnect.json"} {
+ if _, exists := files[name]; exists && name != expected {
+ return fmt.Errorf("stale or wrong dedicated evidence file: %s", name)
+ }
+ }
+ return nil
+}
+
+func validateJUnit(resultsDir string, results Results) error {
+ data, err := os.ReadFile(filepath.Join(resultsDir, "junit.xml"))
+ if err != nil {
+ return fmt.Errorf("read JUnit evidence: %w", err)
+ }
+ var suite junitSuite
+ if err := xml.Unmarshal(data, &suite); err != nil {
+ return fmt.Errorf("parse JUnit evidence: %w", err)
+ }
+ if suite.Name != results.Profile || suite.Tests != len(results.Scenarios) || suite.Failures != countFailedScenarios(results.Scenarios) || len(suite.Cases) != len(results.Scenarios) {
+ return errors.New("results and JUnit evidence do not agree")
+ }
+ for index, scenario := range results.Scenarios {
+ caseResult := suite.Cases[index]
+ if caseResult.Name != scenario.Name || (caseResult.Failure != nil) != !scenario.Passed {
+ return errors.New("results and JUnit scenario states do not agree")
+ }
+ }
+ return nil
+}
diff --git a/integration/agentcompat/internal/evidence/directory_validation_test.go b/integration/agentcompat/internal/evidence/directory_validation_test.go
new file mode 100644
index 00000000..091b6d81
--- /dev/null
+++ b/integration/agentcompat/internal/evidence/directory_validation_test.go
@@ -0,0 +1,204 @@
+package evidence
+
+import (
+ "encoding/json"
+ "flag"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+)
+
+func TestEvidence_CurrentProfilePropagatesMalformedScenarioAndFault(t *testing.T) {
+ tests := []struct {
+ name string
+ metadata Metadata
+ wantContext string
+ wantCause string
+ }{
+ {name: "scenario", metadata: Metadata{Scenarios: []string{"invalid scenario"}}, wantContext: "construct metadata scenario", wantCause: "invalid scenario name"},
+ {name: "fault", metadata: Metadata{Scenarios: []string{contract.ScenarioTransfer100MiB}, Fault: "invalid fault"}, wantContext: "construct metadata fault", wantCause: "invalid fault name"},
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ _, err := currentProfile(test.metadata)
+ if err == nil || !strings.Contains(err.Error(), test.wantContext) || !strings.Contains(err.Error(), test.wantCause) {
+ t.Fatalf("currentProfile error=%v, want context %q and cause %q", err, test.wantContext, test.wantCause)
+ }
+ })
+ }
+}
+
+func TestEvidence_NoCredentialsInDirectory(t *testing.T) {
+ if len(flag.Args()) == 0 {
+ t.Skip("requires a results directory argument: go test ./integration/agentcompat/internal/evidence -run TestEvidence_NoCredentialsInDirectory -args RESULTS")
+ }
+ if err := ValidateDirectory(flag.Args()[0]); err != nil {
+ t.Fatalf("validate evidence directory: %v", err)
+ }
+}
+
+func TestEvidence_NoCredentialsInDirectoryFixture(t *testing.T) {
+ resultsDir := t.TempDir()
+ metadata := validMetadata(t, resultsDir, contract.ScenarioRegistrationConfigExec)
+ writeJSONEvidenceFile(t, resultsDir, "metadata.json", metadata)
+ results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: contract.ScenarioRegistrationConfigExec, Passed: true, Assertions: []Assertion{{Name: "safe fixture", Passed: true}}}}}
+ writeJSONEvidenceFile(t, resultsDir, "results.json", results)
+ junit, err := JUnit(results)
+ if err != nil {
+ t.Fatalf("marshal safe JUnit: %v", err)
+ }
+ writeEvidenceFile(t, resultsDir, "junit.xml", string(junit))
+ writeEvidenceFile(t, resultsDir, "cleanup.json", `{"passed":true,"scenario":"registration-config-exec","finished_at":"2026-01-02T03:04:05Z"}`)
+ if err := ValidateDirectory(resultsDir); err != nil {
+ t.Fatalf("validate evidence directory: %v", err)
+ }
+}
+
+func TestEvidence_ValidateDirectoryRejectsCrossFileMismatches(t *testing.T) {
+ tests := map[string]func(*testing.T, string){
+ "profile mismatch": func(t *testing.T, dir string) {
+ writeExecutableEvidence(t, dir, "soak", contract.ScenarioRegistrationConfigExec, true, true, true)
+ },
+ "scenario mismatch": func(t *testing.T, dir string) {
+ writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, true, false, true)
+ },
+ "junit mismatch": func(t *testing.T, dir string) {
+ writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, false, true, true)
+ },
+ "missing cleanup": func(t *testing.T, dir string) {
+ writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, true, true, false)
+ },
+ }
+ for name, setup := range tests {
+ t.Run(name, func(t *testing.T) {
+ dir := t.TempDir()
+ setup(t, dir)
+ if err := ValidateDirectory(dir); err == nil {
+ t.Fatal("inconsistent evidence accepted")
+ }
+ })
+ }
+}
+
+func TestEvidence_ValidateDirectoryRejectsFailedCleanup(t *testing.T) {
+ dir := t.TempDir()
+ writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, true, true, true)
+ writeEvidenceFile(t, dir, "cleanup.json", `{"passed":false,"scenario":"registration-config-exec","finished_at":"2026-01-02T03:04:05Z"}`)
+ if err := ValidateDirectory(dir); err == nil {
+ t.Fatal("failed cleanup accepted")
+ }
+}
+
+func TestEvidence_MCPFilesystemExecutableProfileValidates(t *testing.T) {
+ dir := t.TempDir()
+ writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioMCPFilesystem, true, true, true)
+ if err := ValidateDirectory(dir); err != nil {
+ t.Fatalf("validate mcp-filesystem evidence: %v", err)
+ }
+}
+
+func TestEvidence_NoCredentialsInDirectoryRejectsMissingPathAndFiles(t *testing.T) {
+ if err := ValidateDirectory(filepath.Join(t.TempDir(), "missing")); err == nil {
+ t.Fatal("missing evidence directory accepted")
+ }
+ for name, dir := range map[string]string{"empty": t.TempDir(), "incomplete": t.TempDir()} {
+ if name == "incomplete" {
+ writeEvidenceFile(t, dir, "metadata.json", `{ "ok": true }`)
+ }
+ if err := ValidateDirectory(dir); err == nil {
+ t.Fatalf("%s evidence directory accepted", name)
+ }
+ }
+}
+
+func TestEvidence_NoCredentialsInDirectoryRejectsCredentialsAndMalformedDocuments(t *testing.T) {
+ tests := []struct{ name, results, junit string }{
+ {"credentials", `{ "token": "secret-value" }`, ``},
+ {"malformed json", `{ malformed`, ``},
+ {"malformed xml", `{ "ok": true }`, ``},
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ dir := t.TempDir()
+ writeEvidenceFile(t, dir, "metadata.json", `{ "ok": true }`)
+ writeEvidenceFile(t, dir, "results.json", test.results)
+ writeEvidenceFile(t, dir, "junit.xml", test.junit)
+ if err := ValidateDirectory(dir); err == nil {
+ t.Fatal("invalid evidence accepted")
+ }
+ })
+ }
+}
+
+func writeEvidenceFile(t *testing.T, resultsDir, name, content string) {
+ t.Helper()
+ if err := os.Chmod(resultsDir, 0o700); err != nil {
+ t.Fatalf("secure evidence root: %v", err)
+ }
+ path := filepath.Join(resultsDir, name)
+ if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+ t.Fatalf("create evidence parent: %v", err)
+ }
+ if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
+ t.Fatalf("write evidence file: %v", err)
+ }
+}
+
+func writeJSONEvidenceFile(t *testing.T, resultsDir, name string, value any) {
+ t.Helper()
+ data, err := json.Marshal(value)
+ if err != nil {
+ t.Fatalf("marshal %s: %v", name, err)
+ }
+ writeEvidenceFile(t, resultsDir, name, string(data))
+}
+
+func validMetadata(t *testing.T, resultsDir, scenarioName string) Metadata {
+ t.Helper()
+ profile, err := contract.ProfileByName("pr-full")
+ if err != nil {
+ t.Fatalf("profile: %v", err)
+ }
+ paths, err := contract.NewPaths("/src/nezha", "/src/agent", resultsDir)
+ if err != nil {
+ t.Fatalf("paths: %v", err)
+ }
+ scenarioValue, err := contract.NewScenario(scenarioName)
+ if err != nil {
+ t.Fatalf("scenario: %v", err)
+ }
+ metadata, err := NewMetadata(MetadataInput{Profile: profile, Seed: contract.DefaultSeed, Paths: paths, ResourceBudget: contract.DefaultResourceBudget(), Scenarios: []contract.Scenario{scenarioValue}, StartedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), EvidenceFiles: EvidenceFiles()})
+ if err != nil {
+ t.Fatalf("metadata: %v", err)
+ }
+ return metadata
+}
+
+func writeExecutableEvidence(t *testing.T, dir, profileName, scenarioName string, matchingJUnit, matchingScenario, writeCleanup bool) {
+ t.Helper()
+ metadata := validMetadata(t, dir, scenarioName)
+ metadata.Profile.Name = profileName
+ writeJSONEvidenceFile(t, dir, "metadata.json", metadata)
+ resultsScenario := scenarioName
+ if !matchingScenario {
+ resultsScenario = contract.ScenarioMetadata
+ }
+ results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: resultsScenario, Passed: true, Assertions: []Assertion{{Name: "fixture", Passed: true}}}}}
+ writeJSONEvidenceFile(t, dir, "results.json", results)
+ junit, err := JUnit(results)
+ if err != nil {
+ t.Fatalf("JUnit: %v", err)
+ }
+ if !matchingJUnit {
+ junit = []byte(``)
+ }
+ writeEvidenceFile(t, dir, "junit.xml", string(junit))
+ if writeCleanup {
+ writeEvidenceFile(t, dir, "cleanup.json", fmt.Sprintf(`{"passed":true,"scenario":%q,"finished_at":"2026-01-02T03:04:05Z"}`, scenarioName))
+ }
+}
diff --git a/integration/agentcompat/internal/evidence/evidence_test.go b/integration/agentcompat/internal/evidence/evidence_test.go
new file mode 100644
index 00000000..0cf4d158
--- /dev/null
+++ b/integration/agentcompat/internal/evidence/evidence_test.go
@@ -0,0 +1,223 @@
+package evidence
+
+import (
+ "encoding/json"
+ "encoding/xml"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+)
+
+func TestEvidence_Redaction(t *testing.T) {
+ input := `"Authorization":"Basic authorization-secret" JWT=eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.signature PAT=pat-secret agent_secret_key=agent-secret client_secret=config-secret https://local/mcp/upload/path-secret?token=query-secret`
+ redacted := Redact(input)
+ for _, secret := range []string{"authorization-secret", "eyJhbGciOiJIUzI1NiJ9", "pat-secret", "agent-secret", "config-secret", "path-secret", "query-secret"} {
+ if strings.Contains(redacted, secret) {
+ t.Fatalf("secret survived redaction: %q", secret)
+ }
+ }
+ if !strings.Contains(redacted, "[REDACTED]") {
+ t.Fatalf("redaction marker missing: %q", redacted)
+ }
+ junit, err := JUnit(Results{Profile: "pr-full", Passed: false, Scenarios: []ScenarioResult{{Name: "secret", Passed: false, Assertions: []Assertion{{Name: "secret failure", Passed: false}}, Error: input}}})
+ if err != nil {
+ t.Fatalf("marshal redacted junit: %v", err)
+ }
+ if strings.Contains(string(junit), "authorization-secret") || strings.Contains(string(junit), "path-secret") {
+ t.Fatalf("secret survived JUnit redaction: %s", junit)
+ }
+}
+
+func TestEvidence_Golden(t *testing.T) {
+ profile, err := contract.ProfileByName("pr-full")
+ if err != nil {
+ t.Fatalf("construct profile: %v", err)
+ }
+ paths, err := contract.NewPaths("/src/nezha", "/src/agent", "/tmp/results")
+ if err != nil {
+ t.Fatalf("construct paths: %v", err)
+ }
+ metadata, err := NewMetadata(MetadataInput{
+ Profile: profile,
+ Seed: contract.DefaultSeed,
+ Paths: paths,
+ ResourceBudget: contract.DefaultResourceBudget(),
+ StartedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
+ EvidenceFiles: EvidenceFiles(),
+ })
+ if err != nil {
+ t.Fatalf("construct metadata: %v", err)
+ }
+ data, err := json.Marshal(metadata)
+ if err != nil {
+ t.Fatalf("marshal metadata: %v", err)
+ }
+ const wantJSON = `{"agent_source":"/src/agent","evidence_files":["metadata.json","results.json","junit.xml","dashboard.log","agents/*.log","transfer.json","reconnect.json","stress.json","cleanup.json","step-summary.md"],"load_classification":"regression loads, not capacity claims","nezha_source":"/src/nezha","profile":{"name":"pr-full","job_timeout_seconds":4500,"suite_deadline_seconds":3300,"default_seed":"0x4e5a4841","agent_count":8,"stress_rounds":4,"concurrent_operations":64,"concurrent_sessions_per_kind":4,"transfer_pairs":1,"transfer_bytes":104857600,"dashboard_restart_cycles":1,"iterations":1,"stream_boundary_allowed":40,"stream_boundary_rejected":41},"resource_budget":{"warmup_runs_per_path":1,"baseline_sample_count":5,"end_sample_count":5,"sample_interval_milliseconds":250,"child_process_count_drift":0,"listener_count_drift":0,"non_stdio_fd_count_drift":0,"dashboard_rss_delta_bytes":67108864,"agent_rss_delta_bytes":33554432,"transfer_heap_bytes":16777216},"results_dir":"/tmp/results","scenarios":[],"seed":"0x4e5a4841","started_at":"2026-01-02T03:04:05Z"}`
+ if string(data) != wantJSON {
+ t.Fatalf("metadata golden mismatch\nwant: %s\ngot: %s", wantJSON, data)
+ }
+
+ results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: "metadata", Passed: true, Assertions: []Assertion{{Name: "metadata written", Passed: true}}}}}
+ resultsJSON, err := json.Marshal(results)
+ if err != nil {
+ t.Fatalf("marshal results: %v", err)
+ }
+ const wantResultsJSON = `{"profile":"pr-full","passed":true,"scenarios":[{"name":"metadata","passed":true,"assertions":[{"name":"metadata written","passed":true}]}]}`
+ if string(resultsJSON) != wantResultsJSON {
+ t.Fatalf("results golden mismatch\nwant: %s\ngot: %s", wantResultsJSON, resultsJSON)
+ }
+ junitXML, err := JUnit(results)
+ if err != nil {
+ t.Fatalf("marshal junit: %v", err)
+ }
+ const wantXML = ``
+ if string(junitXML) != wantXML {
+ t.Fatalf("junit golden mismatch\nwant: %s\ngot: %s", wantXML, junitXML)
+ }
+ var parsedJUnit junitSuite
+ if err := xml.Unmarshal(junitXML, &parsedJUnit); err != nil {
+ t.Fatalf("parse junit golden: %v", err)
+ }
+ if parsedJUnit.Tests != 1 || parsedJUnit.Failures != 0 || len(parsedJUnit.Cases) != 1 {
+ t.Fatalf("unexpected parsed junit: %#v", parsedJUnit)
+ }
+}
+
+func TestEvidence_SchemaValidation(t *testing.T) {
+ invalid := Metadata{}
+ if err := invalid.Validate(); err == nil {
+ t.Fatal("invalid metadata accepted")
+ }
+ invalidBudget := ResourceBudgetMetadata{WarmupRunsPerPath: 1, BaselineSampleCount: 5, EndSampleCount: 5, SampleIntervalMilliseconds: 250, DashboardRSSDeltaBytes: 1, AgentRSSDeltaBytes: 1}
+ if err := invalidBudget.Validate(); err == nil {
+ t.Fatal("missing transfer heap threshold accepted")
+ }
+ results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: "failed", Passed: false, Assertions: []Assertion{{Name: "failed assertion", Passed: false}}, Error: "failure"}}}
+ if err := results.Validate(); err == nil {
+ t.Fatal("inconsistent results accepted")
+ }
+ for _, result := range []Results{
+ {Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: "../escape", Passed: true}}},
+ {Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: "duplicate", Passed: true}, {Name: "duplicate", Passed: true}}},
+ } {
+ if err := result.Validate(); err == nil {
+ t.Fatalf("invalid scenario results accepted: %#v", result)
+ }
+ }
+}
+
+func TestEvidence_ResultsRejectsSuccessfulScenarioWithoutAssertions(t *testing.T) {
+ results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: "registration-config-exec", Passed: true}}}
+
+ if err := results.Validate(); err == nil {
+ t.Fatal("successful executable scenario without assertions accepted")
+ }
+}
+
+func TestEvidence_ResultsJSONRedactsSecrets(t *testing.T) {
+ results := Results{Profile: "pr-full", Passed: false, Scenarios: []ScenarioResult{
+ {Name: "authorization", Passed: false, Assertions: []Assertion{{Name: "authorization failure", Passed: false}}, Error: "Authorization: Bearer jwt-secret"},
+ {Name: "credential-field", Passed: false, Assertions: []Assertion{{Name: "credential failure", Passed: false}}, Error: "agent_secret=agent-secret"},
+ {Name: "transfer-token", Passed: false, Assertions: []Assertion{{Name: "transfer failure", Passed: false}}, Error: "/mcp/download/path-token?token=query-token"},
+ {Name: "access-token", Passed: false, Assertions: []Assertion{{Name: "access failure", Passed: false}}, Error: "https://local/file?access_token=access-secret"},
+ {Name: "api-key", Passed: false, Assertions: []Assertion{{Name: "api failure", Passed: false}}, Error: "https://local/file?api_key=api-secret"},
+ {Name: "signature", Passed: false, Assertions: []Assertion{{Name: "signature failure", Passed: false}}, Error: "https://local/file?sig=sig-secret&signature=signature-secret&X-Amz-Signature=aws-secret"},
+ }}
+ data, err := MarshalResults(results)
+ if err != nil {
+ t.Fatalf("marshal results: %v", err)
+ }
+ for _, secret := range []string{"jwt-secret", "agent-secret", "path-token", "query-token", "access-secret", "api-secret", "sig-secret", "signature-secret", "aws-secret"} {
+ if strings.Contains(string(data), secret) {
+ t.Fatalf("secret survived results JSON: %q in %s", secret, data)
+ }
+ }
+ directData, err := json.Marshal(results)
+ if err != nil {
+ t.Fatalf("direct marshal results: %v", err)
+ }
+ if strings.Contains(string(directData), "jwt-secret") || strings.Contains(string(directData), "path-token") {
+ t.Fatalf("direct JSON marshal bypassed redaction: %s", directData)
+ }
+}
+
+func TestEvidence_RedactsXMLTextAttributesAndMalformedFragments(t *testing.T) {
+ input := `agent-secrettransfer-secretpassword-secretcredential-secret`
+ redacted := Redact(input)
+ for _, secret := range []string{"authorization-secret", "agent-secret", "transfer-secret", "password-secret", "credential-secret"} {
+ if strings.Contains(redacted, secret) {
+ t.Fatalf("XML secret survived redaction: %q in %s", secret, redacted)
+ }
+ }
+ var parsed struct {
+ XMLName xml.Name `xml:"record"`
+ Secret string `xml:"agent_secret_key"`
+ }
+ if err := xml.Unmarshal([]byte(redacted), &parsed); err != nil {
+ t.Fatalf("redacted XML is not parseable: %v; output=%s", err, redacted)
+ }
+ if parsed.Secret != "[REDACTED]" {
+ t.Fatalf("unexpected redacted element text: %q", parsed.Secret)
+ }
+ results := Results{Profile: "pr-full", Passed: false, Scenarios: []ScenarioResult{{Name: "xml", Passed: false, Assertions: []Assertion{{Name: "xml failure", Passed: false}}, Error: input}}}
+ junit, err := JUnit(results)
+ if err != nil {
+ t.Fatalf("marshal XML evidence: %v", err)
+ }
+ var parsedSuite junitSuite
+ if err := xml.Unmarshal(junit, &parsedSuite); err != nil {
+ t.Fatalf("redacted JUnit is not parseable: %v; output=%s", err, junit)
+ }
+ if strings.Contains(string(junit), "agent-secret") || strings.Contains(string(junit), "credential-secret") {
+ t.Fatalf("XML evidence secret survived: %s", junit)
+ }
+
+ malformed := `config-secret`
+ malformedRedacted := Redact(malformed)
+ if strings.Contains(malformedRedacted, "config-secret") {
+ t.Fatalf("malformed XML secret survived redaction: %s", malformedRedacted)
+ }
+}
+
+func TestEvidence_RedactsXMLSensitiveAttributesWithWhitespaceAndMalformedTags(t *testing.T) {
+ input := `password text`
+ redacted := Redact(input)
+ for _, secret := range []string{"agent secret with spaces", "config-secret", "password secret with spaces", "password text"} {
+ if strings.Contains(redacted, secret) {
+ t.Fatalf("XML attribute or text secret survived redaction: %q in %s", secret, redacted)
+ }
+ }
+ if !strings.Contains(redacted, ``) {
+ t.Fatalf("sensitive XML attribute was not structurally redacted: %s", redacted)
+ }
+}
+
+func TestEvidence_RedactsCredentialAttributesWithSpacesAndTruncatedXML(t *testing.T) {
+ input := `ok`
+ redacted := Redact(input)
+ if strings.Contains(redacted, "self-closing secret") {
+ t.Fatalf("self-closing XML secret survived redaction: %s", redacted)
+ }
+ var parsed struct {
+ XMLName xml.Name `xml:"record"`
+ Result string `xml:"result"`
+ }
+ if err := xml.Unmarshal([]byte(redacted), &parsed); err != nil {
+ t.Fatalf("redacted self-closing XML is not parseable: %v; output=%s", err, redacted)
+ }
+ if parsed.Result != "ok" {
+ t.Fatalf("redacted self-closing XML lost sibling content: %#v", parsed)
+ }
+}
diff --git a/integration/agentcompat/internal/evidence/junit.go b/integration/agentcompat/internal/evidence/junit.go
new file mode 100644
index 00000000..1424aed0
--- /dev/null
+++ b/integration/agentcompat/internal/evidence/junit.go
@@ -0,0 +1,46 @@
+package evidence
+
+import "encoding/xml"
+
+type junitSuite struct {
+ XMLName xml.Name `xml:"testsuite"`
+ Name string `xml:"name,attr"`
+ Tests int `xml:"tests,attr"`
+ Failures int `xml:"failures,attr"`
+ Cases []junitCase `xml:"testcase"`
+}
+
+type junitCase struct {
+ Name string `xml:"name,attr"`
+ Failure *junitFailure `xml:"failure,omitempty"`
+}
+
+type junitFailure struct {
+ Message string `xml:"message,attr"`
+}
+
+func JUnit(results Results) ([]byte, error) {
+ if err := results.Validate(); err != nil {
+ return nil, err
+ }
+ suite := junitSuite{Name: results.Profile, Tests: len(results.Scenarios)}
+ for _, scenario := range results.Scenarios {
+ caseResult := junitCase{Name: scenario.Name}
+ if !scenario.Passed {
+ suite.Failures++
+ caseResult.Failure = &junitFailure{Message: Redact(scenario.Error)}
+ }
+ suite.Cases = append(suite.Cases, caseResult)
+ }
+ return xml.Marshal(suite)
+}
+
+func countFailedScenarios(scenarios []ScenarioResult) int {
+ failed := 0
+ for _, scenario := range scenarios {
+ if !scenario.Passed {
+ failed++
+ }
+ }
+ return failed
+}
diff --git a/integration/agentcompat/internal/evidence/metadata.go b/integration/agentcompat/internal/evidence/metadata.go
new file mode 100644
index 00000000..4270de00
--- /dev/null
+++ b/integration/agentcompat/internal/evidence/metadata.go
@@ -0,0 +1,134 @@
+package evidence
+
+import (
+ "encoding/json"
+ "fmt"
+ "path/filepath"
+ "slices"
+ "time"
+
+ "github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+)
+
+type MetadataInput struct {
+ Profile contract.Profile
+ Seed contract.Seed
+ Paths contract.Paths
+ ResourceBudget contract.ResourceBudget
+ Scenarios []contract.Scenario
+ Fault contract.Fault
+ StartedAt time.Time
+ EvidenceFiles []string
+}
+
+type Metadata struct {
+ AgentSource string `json:"agent_source"`
+ EvidenceFiles []string `json:"evidence_files"`
+ LoadClassification string `json:"load_classification"`
+ NezhaSource string `json:"nezha_source"`
+ Profile ProfileMetadata `json:"profile"`
+ ResourceBudget ResourceBudgetMetadata `json:"resource_budget"`
+ ResultsDir string `json:"results_dir"`
+ Scenarios []string `json:"scenarios"`
+ Seed string `json:"seed"`
+ Fault string `json:"fault,omitempty"`
+ StartedAt string `json:"started_at"`
+}
+
+func NewMetadata(input MetadataInput) (Metadata, error) {
+ metadata := Metadata{
+ AgentSource: Redact(input.Paths.AgentSource().String()),
+ EvidenceFiles: append([]string(nil), input.EvidenceFiles...),
+ LoadClassification: "regression loads, not capacity claims",
+ NezhaSource: Redact(input.Paths.NezhaSource().String()),
+ Profile: profileMetadata(input.Profile),
+ ResourceBudget: resourceBudgetMetadata(input.ResourceBudget),
+ ResultsDir: Redact(input.Paths.ResultsDir().String()),
+ Scenarios: scenarioNames(input.Scenarios),
+ Seed: fmt.Sprintf("0x%x", uint64(input.Seed)),
+ Fault: input.Fault.String(),
+ StartedAt: input.StartedAt.UTC().Format(time.RFC3339),
+ }
+ if err := metadata.Validate(); err != nil {
+ return Metadata{}, err
+ }
+ return metadata, nil
+}
+
+func EvidenceFiles() []string {
+ files := []string{"metadata.json", "results.json", "junit.xml", "dashboard.log", "agents/*.log"}
+ for _, definition := range contract.ScenarioDefinitions() {
+ if name := definition.DedicatedArtifactName(); name != "" {
+ files = append(files, name)
+ }
+ }
+ return append(files, "stress.json", "cleanup.json", "step-summary.md")
+}
+
+func FixedEvidenceFiles() []string {
+ files := make([]string, 0, len(EvidenceFiles()))
+ for _, name := range EvidenceFiles() {
+ if !slices.Contains([]rune(name), '*') {
+ files = append(files, name)
+ }
+ }
+ return files
+}
+
+func scenarioNames(scenarios []contract.Scenario) []string {
+ names := make([]string, 0, len(scenarios))
+ for _, scenario := range scenarios {
+ names = append(names, scenario.String())
+ }
+ return names
+}
+
+func (metadata Metadata) Validate() error {
+ if metadata.AgentSource == "" || len(metadata.EvidenceFiles) == 0 || metadata.LoadClassification == "" || metadata.NezhaSource == "" || metadata.ResultsDir == "" || metadata.Seed == "" || metadata.Seed == "0x0" || metadata.StartedAt == "" {
+ return fmt.Errorf("metadata fields are incomplete")
+ }
+ if err := metadata.Profile.Validate(); err != nil {
+ return err
+ }
+ if _, err := contract.ProfileByName(metadata.Profile.Name); err != nil {
+ return fmt.Errorf("metadata profile is invalid: %w", err)
+ }
+ if err := metadata.ResourceBudget.Validate(); err != nil {
+ return err
+ }
+ if !filepath.IsAbs(metadata.AgentSource) || !filepath.IsAbs(metadata.NezhaSource) || !filepath.IsAbs(metadata.ResultsDir) {
+ return fmt.Errorf("metadata paths must be absolute")
+ }
+ if _, err := time.Parse(time.RFC3339, metadata.StartedAt); err != nil {
+ return fmt.Errorf("metadata start time is invalid: %w", err)
+ }
+ if !slices.Equal(metadata.EvidenceFiles, EvidenceFiles()) {
+ return fmt.Errorf("metadata evidence files are invalid")
+ }
+ seen := make(map[string]struct{}, len(metadata.Scenarios))
+ for _, scenarioName := range metadata.Scenarios {
+ _, err := contract.NewScenario(scenarioName)
+ if err != nil {
+ return fmt.Errorf("metadata scenario is invalid: %w", err)
+ }
+ if _, exists := seen[scenarioName]; exists {
+ return fmt.Errorf("metadata scenario is duplicated")
+ }
+ seen[scenarioName] = struct{}{}
+ }
+ if metadata.Fault != "" {
+ if _, err := contract.NewFault(metadata.Fault); err != nil {
+ return fmt.Errorf("metadata fault is invalid: %w", err)
+ }
+ }
+ return nil
+}
+
+func (metadata Metadata) MarshalJSON() ([]byte, error) {
+ type metadataWire Metadata
+ redacted := metadataWire(metadata)
+ redacted.AgentSource = Redact(redacted.AgentSource)
+ redacted.NezhaSource = Redact(redacted.NezhaSource)
+ redacted.ResultsDir = Redact(redacted.ResultsDir)
+ return json.Marshal(redacted)
+}
diff --git a/integration/agentcompat/internal/evidence/reconnect_artifact_validation.go b/integration/agentcompat/internal/evidence/reconnect_artifact_validation.go
new file mode 100644
index 00000000..54eb1119
--- /dev/null
+++ b/integration/agentcompat/internal/evidence/reconnect_artifact_validation.go
@@ -0,0 +1,124 @@
+package evidence
+
+import (
+ "errors"
+ "path/filepath"
+ "slices"
+ "strings"
+
+ "github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+)
+
+func validateDedicatedArtifact(resultsDir string, metadata Metadata, result ScenarioResult) error {
+ definition, err := contract.ScenarioDefinitionByName(result.Name)
+ if err != nil {
+ return err
+ }
+ switch definition.DedicatedArtifact {
+ case contract.DedicatedArtifactTransfer:
+ artifact, err := readJSONFile[transferArtifact](resultsDir, "transfer.json")
+ if err != nil {
+ return err
+ }
+ return validateTransferArtifact(metadata, result, artifact)
+ case contract.DedicatedArtifactReconnect:
+ artifact, err := readJSONFile[reconnectArtifact](resultsDir, "reconnect.json")
+ if err != nil {
+ return err
+ }
+ return validateReconnectArtifact(metadata, result, artifact)
+ case contract.DedicatedArtifactNone:
+ return nil
+ default:
+ return errors.New("unsupported dedicated artifact kind")
+ }
+}
+
+func validateReconnectArtifact(metadata Metadata, result ScenarioResult, artifact reconnectArtifact) error {
+ if err := validateArtifactHeader(metadata, result, artifact.Scenario, artifact.Fault, artifact.Passed, artifact.CleanupOK, artifact.Error); err != nil {
+ return err
+ }
+ switch artifact.Fault {
+ case "":
+ if !artifact.Passed {
+ return errors.New("reconnect success artifact reports failure")
+ }
+ if err := validateReconnectSuccess(artifact.Evidence); err != nil {
+ return err
+ }
+ if err := validateReconnectSuccessReceipts(artifact.Evidence); err != nil {
+ return err
+ }
+ return validateReconnectFinalEvidence(artifact.Evidence, 2)
+ case contract.FaultDashboardExit:
+ if artifact.Passed || !strings.Contains(artifact.Error, "injected Dashboard exit") {
+ return errors.New("dashboard-exit artifact does not identify the injected failure")
+ }
+ evidence := artifact.Evidence
+ if reconnectPostFaultEvidencePresent(evidence) {
+ return errors.New("dashboard-exit artifact presents stale reconnect success")
+ }
+ if evidence.Lifecycle.DisconnectAt.IsZero() || !evidence.Lifecycle.ReconnectAt.IsZero() || !evidence.Lifecycle.OutsideRootSentinelUnchanged {
+ return errors.New("dashboard-exit lifecycle evidence is invalid")
+ }
+ return validateReconnectFinalEvidence(evidence, 1)
+ default:
+ return errors.New("reconnect artifact has unsupported fault")
+ }
+}
+
+func reconnectPostFaultEvidencePresent(evidence reconnectEvidence) bool {
+ return evidence.Runtime.DashboardAfter != (runtimeIdentity{}) || evidence.Runtime.AgentAfter != (runtimeIdentity{}) ||
+ evidence.Runtime.StateGenerationBeforeAgentRestart != 0 || evidence.Runtime.StateGenerationAfterAgentRestart != 0 ||
+ evidence.Identity.DashboardConfigUnchanged || evidence.Identity.AgentConfigUnchanged || evidence.Identity.DashboardFixtureUnchanged || evidence.Identity.ClientsRecreated || evidence.Identity.BootstrapRecreated ||
+ !evidence.Lifecycle.ReconnectAt.IsZero() || evidence.Lifecycle.ReconnectInterval != 0 || len(evidence.Lifecycle.DashboardReceipts) != 0 || len(evidence.Lifecycle.AgentReceipts) != 0 ||
+ evidence.Lifecycle.StaleGenerationReceipts != 0 || evidence.Lifecycle.DuplicateTaskIDs != 0 || evidence.Lifecycle.LostResultIDs != 0 ||
+ evidence.Observation.ServerID != 0 || evidence.Observation.UUID != "" || evidence.Observation.OldGeneration != 0 || evidence.Observation.NewGeneration != 0 ||
+ !evidence.Observation.DisconnectAt.IsZero() || !evidence.Observation.ReconnectAt.IsZero() || len(evidence.Observation.TaskIDs) != 0 || len(evidence.Observation.ResultIDs) != 0 || evidence.Observation.PostReconnect || evidence.Observation.AgentRestarted
+}
+
+func validateReconnectSuccess(evidence reconnectEvidence) error {
+ observation := evidence.Observation
+ if observation.ServerID == 0 || observation.UUID == "" || observation.OldGeneration == 0 || observation.NewGeneration <= observation.OldGeneration || observation.DisconnectAt.IsZero() || !observation.ReconnectAt.After(observation.DisconnectAt) || len(observation.TaskIDs) != 5 || !slices.Equal(observation.TaskIDs, observation.ResultIDs) || !observation.PostReconnect || !observation.AgentRestarted {
+ return errors.New("reconnect observation evidence is invalid")
+ }
+ if evidence.Runtime.DashboardAfter.Generation <= evidence.Runtime.DashboardBefore.Generation || evidence.Runtime.DashboardAfter.PID == evidence.Runtime.DashboardBefore.PID || evidence.Runtime.AgentAfter.Generation <= evidence.Runtime.AgentBefore.Generation || evidence.Runtime.AgentAfter.PID == evidence.Runtime.AgentBefore.PID || evidence.Runtime.StateGenerationAfterAgentRestart <= evidence.Runtime.StateGenerationBeforeAgentRestart {
+ return errors.New("reconnect runtime generations are invalid")
+ }
+ identity := evidence.Identity
+ if identity.ServerID != observation.ServerID || identity.UUID != observation.UUID || !identity.DashboardConfigUnchanged || !identity.AgentConfigUnchanged || !identity.DashboardFixtureUnchanged || !identity.ClientsRecreated || !identity.BootstrapRecreated {
+ return errors.New("reconnect identity evidence is invalid")
+ }
+ if evidence.Lifecycle.StaleGenerationReceipts != 0 || evidence.Lifecycle.DuplicateTaskIDs != 0 || evidence.Lifecycle.LostResultIDs != 0 || !evidence.Lifecycle.OutsideRootSentinelUnchanged {
+ return errors.New("reconnect lifecycle accounting is invalid")
+ }
+ return nil
+}
+
+func validateReconnectSuccessReceipts(evidence reconnectEvidence) error {
+ if len(evidence.Lifecycle.DashboardReceipts) != 3 || len(evidence.Lifecycle.AgentReceipts) != 2 || len(evidence.Observation.TaskIDs) != 5 {
+ return errors.New("reconnect receipt accounting is not exact")
+ }
+ if evidence.Lifecycle.DisconnectAt.IsZero() || evidence.Lifecycle.ReconnectAt.IsZero() || evidence.Lifecycle.ReconnectAt.Sub(evidence.Lifecycle.DisconnectAt) != evidence.Lifecycle.ReconnectInterval {
+ return errors.New("reconnect timestamp interval is inconsistent")
+ }
+ if !evidence.Observation.DisconnectAt.Equal(evidence.Lifecycle.DisconnectAt) || !evidence.Observation.ReconnectAt.Equal(evidence.Lifecycle.ReconnectAt) {
+ return errors.New("reconnect lifecycle and observation timestamps differ")
+ }
+ return nil
+}
+
+func validateReconnectFinalEvidence(evidence reconnectEvidence, expectedProcessCount int) error {
+ if !evidence.AgentCleanup.Passed || evidence.AgentCleanup.Forced || !evidence.DashboardCleanup.Passed || evidence.DashboardCleanup.Forced {
+ return errors.New("reconnect cleanup receipts failed")
+ }
+ if len(evidence.AgentCleanup.Processes) != expectedProcessCount || len(evidence.DashboardCleanup.Processes) != expectedProcessCount {
+ return errors.New("reconnect cleanup receipt process count is invalid")
+ }
+ for _, path := range []string{evidence.Fixture.Dashboard.WorkspaceRoot, evidence.Fixture.AgentRoot, evidence.Fixture.Dashboard.ConfigPath, evidence.Fixture.AgentConfigPath, evidence.Fixture.Dashboard.BinaryPath, evidence.Fixture.AgentBinaryPath} {
+ if path == "" || !filepath.IsAbs(path) {
+ return errors.New("reconnect fixture paths are invalid")
+ }
+ }
+ return nil
+}
diff --git a/integration/agentcompat/internal/evidence/redaction.go b/integration/agentcompat/internal/evidence/redaction.go
new file mode 100644
index 00000000..918a0747
--- /dev/null
+++ b/integration/agentcompat/internal/evidence/redaction.go
@@ -0,0 +1,41 @@
+package evidence
+
+import (
+ "regexp"
+ "strings"
+)
+
+type redactionRule struct {
+ pattern *regexp.Regexp
+ replace string
+}
+
+var redactionRules = []redactionRule{
+ {pattern: regexp.MustCompile(`(?i)(["']?authorization["']?\s*[:=]\s*["']?)[^"'\r\n,}]+(["']?)`), replace: `${1}[REDACTED]${2}`},
+ {pattern: regexp.MustCompile(`(?i)(\b["']?(?:token|access[_-]?token|api[_-]?(?:key|token)|jwt[_-]?secret(?:[_-]?key)?|jwt[_-]?token|pat|agent[_-]?secret(?:[_-]?key)?|client[_-]?secret|handshake[_-]?secret|revert[_-]?handshake[_-]?secret|password|credential|transfer[_-]?token)["']?\s*[:=]\s*["']?)[^"'\s,;&}]+(["']?)`), replace: `${1}[REDACTED]${2}`},
+ {pattern: regexp.MustCompile(`(?i)([?&](?:token|access_token|api_key|jwt|pat|secret|authorization|sig|signature|x-amz-signature)=)[^&#\s]+`), replace: `${1}[REDACTED]`},
+ {pattern: regexp.MustCompile(`(?i)(/mcp/(?:download|upload)/)[A-Za-z0-9._-]+`), replace: `${1}[REDACTED]`},
+ {pattern: regexp.MustCompile(`\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b`), replace: `[REDACTED]`},
+ {pattern: regexp.MustCompile(`\b(?:ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})\b`), replace: `[REDACTED]`},
+ {pattern: regexp.MustCompile(`(?is)(<\s*(?:agent[_-]?secret(?:[_-]?key)?|authorization|transfer[_-]?token|password|credential|config[_-]?secret)\b)[^>]*?(/?)>`), replace: `${1} value="[REDACTED]"${2}>`},
+ {pattern: regexp.MustCompile(`(?is)(<\s*(?:agent[_-]?secret(?:[_-]?key)?|authorization|transfer[_-]?token|password|credential|config[_-]?secret)\b[^>]*>)[^<]*()`), replace: `${1}[REDACTED]${2}`},
+ {pattern: regexp.MustCompile(`(?is)(<\s*(?:agent[_-]?secret(?:[_-]?key)?|authorization|transfer[_-]?token|password|credential|config[_-]?secret)\b[^>]*>)[^<]*$`), replace: `${1}[REDACTED]`},
+ {pattern: regexp.MustCompile(`(?is)(<\s*(?:agent[_-]?secret(?:[_-]?key)?|authorization|transfer[_-]?token|password|credential|config[_-]?secret)\b)[^>]*$`), replace: `${1} value="[REDACTED]"`},
+}
+
+var sensitiveXMLAttribute = regexp.MustCompile(`(?i)(\b(?:agent[_-]?secret(?:[_-]?key)?|authorization|transfer[_-]?token|password|credential|config[_-]?secret)\b\s*=\s*)(?:"[^"]*"|'[^']*'|"[^\r\n>]*|'[^'\r\n>]*'|[^\s/>][^/>]*)`)
+
+func Redact(input string) string {
+ redacted := sensitiveXMLAttribute.ReplaceAllStringFunc(input, func(attribute string) string {
+ separator := strings.IndexByte(attribute, '=')
+ if separator < 0 {
+ return attribute
+ }
+ prefix := attribute[:separator+1]
+ return prefix + "\"[REDACTED]\""
+ })
+ for _, rule := range redactionRules {
+ redacted = rule.pattern.ReplaceAllString(redacted, rule.replace)
+ }
+ return redacted
+}
diff --git a/integration/agentcompat/internal/evidence/registry_coverage_test.go b/integration/agentcompat/internal/evidence/registry_coverage_test.go
new file mode 100644
index 00000000..35c31064
--- /dev/null
+++ b/integration/agentcompat/internal/evidence/registry_coverage_test.go
@@ -0,0 +1,32 @@
+package evidence
+
+import (
+ "slices"
+ "testing"
+
+ "github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+)
+
+func TestEvidence_RegistryDefinitionsDriveManifestAndProfiles(t *testing.T) {
+ manifest := EvidenceFiles()
+ for _, definition := range contract.ScenarioDefinitions() {
+ dedicatedName := definition.DedicatedArtifactName()
+ if dedicatedName != "" && !slices.Contains(manifest, dedicatedName) {
+ t.Fatalf("dedicated artifact %q absent from manifest", dedicatedName)
+ }
+ metadata := validMetadata(t, t.TempDir(), definition.Name)
+ profile, err := currentProfile(metadata)
+ if err != nil {
+ t.Fatalf("profile for %q: %v", definition.Name, err)
+ }
+ if profile.DedicatedFile != dedicatedName {
+ t.Fatalf("scenario %q dedicated file=%q want=%q", definition.Name, profile.DedicatedFile, dedicatedName)
+ }
+ if definition.Execution == contract.ScenarioExecutionMetadata && profile.Executable {
+ t.Fatal("metadata profile unexpectedly executable")
+ }
+ if definition.Execution != contract.ScenarioExecutionMetadata && !profile.Executable {
+ t.Fatalf("scenario %q profile is not executable", definition.Name)
+ }
+ }
+}
diff --git a/integration/agentcompat/internal/evidence/results.go b/integration/agentcompat/internal/evidence/results.go
new file mode 100644
index 00000000..c5d52733
--- /dev/null
+++ b/integration/agentcompat/internal/evidence/results.go
@@ -0,0 +1,105 @@
+package evidence
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
+)
+
+type ScenarioResult struct {
+ Name string `json:"name"`
+ Passed bool `json:"passed"`
+ Assertions []Assertion `json:"assertions,omitempty"`
+ Error string `json:"error,omitempty"`
+}
+
+type Assertion struct {
+ Name string `json:"name"`
+ Passed bool `json:"passed"`
+ Details string `json:"details,omitempty"`
+}
+
+type Results struct {
+ Profile string `json:"profile"`
+ Passed bool `json:"passed"`
+ Scenarios []ScenarioResult `json:"scenarios"`
+}
+
+func (results Results) Validate() error {
+ if results.Profile == "" || len(results.Scenarios) == 0 {
+ return fmt.Errorf("results fields are incomplete")
+ }
+ if _, err := contract.ProfileByName(results.Profile); err != nil {
+ return fmt.Errorf("results profile is invalid: %w", err)
+ }
+ allPassed := true
+ seenScenarios := make(map[string]struct{}, len(results.Scenarios))
+ for _, scenario := range results.Scenarios {
+ if _, err := contract.NewScenario(scenario.Name); err != nil {
+ return fmt.Errorf("scenario name is invalid: %w", err)
+ }
+ if _, exists := seenScenarios[scenario.Name]; exists {
+ return fmt.Errorf("scenario name is duplicated")
+ }
+ seenScenarios[scenario.Name] = struct{}{}
+ if len(scenario.Assertions) == 0 {
+ return fmt.Errorf("scenario must contain assertions")
+ }
+ assertionsPassed := true
+ seenAssertions := make(map[string]struct{}, len(scenario.Assertions))
+ for _, assertion := range scenario.Assertions {
+ if assertion.Name == "" {
+ return fmt.Errorf("assertion name is required")
+ }
+ if _, exists := seenAssertions[assertion.Name]; exists {
+ return fmt.Errorf("assertion name is duplicated")
+ }
+ seenAssertions[assertion.Name] = struct{}{}
+ assertionsPassed = assertionsPassed && assertion.Passed
+ }
+ if scenario.Passed != assertionsPassed {
+ return fmt.Errorf("scenario pass state is inconsistent with assertions")
+ }
+ if !scenario.Passed {
+ if scenario.Error == "" {
+ return fmt.Errorf("failed scenario error is required")
+ }
+ allPassed = false
+ } else if scenario.Error != "" {
+ return fmt.Errorf("passed scenario cannot contain an error")
+ }
+ }
+ if results.Passed != allPassed {
+ return fmt.Errorf("results pass state is inconsistent")
+ }
+ return nil
+}
+
+func MarshalResults(results Results) ([]byte, error) {
+ if err := results.Validate(); err != nil {
+ return nil, err
+ }
+ return json.Marshal(results)
+}
+
+func (results Results) MarshalJSON() ([]byte, error) {
+ type resultsWire Results
+ redacted := resultsWire{Profile: results.Profile, Passed: results.Passed, Scenarios: make([]ScenarioResult, 0, len(results.Scenarios))}
+ for _, scenario := range results.Scenarios {
+ assertions := make([]Assertion, 0, len(scenario.Assertions))
+ for _, assertion := range scenario.Assertions {
+ assertions = append(assertions, Assertion{Name: assertion.Name, Passed: assertion.Passed, Details: Redact(assertion.Details)})
+ }
+ redacted.Scenarios = append(redacted.Scenarios, ScenarioResult{Name: scenario.Name, Passed: scenario.Passed, Assertions: assertions, Error: Redact(scenario.Error)})
+ }
+ return json.Marshal(redacted)
+}
+
+func scenarioResultNames(scenarios []ScenarioResult) []string {
+ names := make([]string, 0, len(scenarios))
+ for _, scenario := range scenarios {
+ names = append(names, scenario.Name)
+ }
+ return names
+}