From b3b3d9289505abb850408e9b40e62d529ae0cc84 Mon Sep 17 00:00:00 2001 From: naiba Date: Mon, 20 Jul 2026 04:50:03 +0000 Subject: [PATCH] test(agentcompat): add evidence validation Co-authored-by: naiba/CloudCode --- .../internal/evidence/contract_metadata.go | 59 +++++ .../internal/evidence/dedicated_artifacts.go | 213 +++++++++++++++++ .../evidence/dedicated_assertion_test.go | 46 ++++ .../evidence/dedicated_fault_fields_test.go | 86 +++++++ .../evidence/dedicated_validation_test.go | 212 +++++++++++++++++ .../internal/evidence/directory_scan.go | 133 +++++++++++ .../evidence/directory_security_test.go | 69 ++++++ .../internal/evidence/directory_validation.go | 174 ++++++++++++++ .../evidence/directory_validation_test.go | 204 ++++++++++++++++ .../internal/evidence/evidence_test.go | 223 ++++++++++++++++++ .../agentcompat/internal/evidence/junit.go | 46 ++++ .../agentcompat/internal/evidence/metadata.go | 134 +++++++++++ .../evidence/reconnect_artifact_validation.go | 124 ++++++++++ .../internal/evidence/redaction.go | 41 ++++ .../evidence/registry_coverage_test.go | 32 +++ .../agentcompat/internal/evidence/results.go | 105 +++++++++ 16 files changed, 1901 insertions(+) create mode 100644 integration/agentcompat/internal/evidence/contract_metadata.go create mode 100644 integration/agentcompat/internal/evidence/dedicated_artifacts.go create mode 100644 integration/agentcompat/internal/evidence/dedicated_assertion_test.go create mode 100644 integration/agentcompat/internal/evidence/dedicated_fault_fields_test.go create mode 100644 integration/agentcompat/internal/evidence/dedicated_validation_test.go create mode 100644 integration/agentcompat/internal/evidence/directory_scan.go create mode 100644 integration/agentcompat/internal/evidence/directory_security_test.go create mode 100644 integration/agentcompat/internal/evidence/directory_validation.go create mode 100644 integration/agentcompat/internal/evidence/directory_validation_test.go create mode 100644 integration/agentcompat/internal/evidence/evidence_test.go create mode 100644 integration/agentcompat/internal/evidence/junit.go create mode 100644 integration/agentcompat/internal/evidence/metadata.go create mode 100644 integration/agentcompat/internal/evidence/reconnect_artifact_validation.go create mode 100644 integration/agentcompat/internal/evidence/redaction.go create mode 100644 integration/agentcompat/internal/evidence/registry_coverage_test.go create mode 100644 integration/agentcompat/internal/evidence/results.go diff --git a/integration/agentcompat/internal/evidence/contract_metadata.go b/integration/agentcompat/internal/evidence/contract_metadata.go new file mode 100644 index 00000000..65e93d25 --- /dev/null +++ b/integration/agentcompat/internal/evidence/contract_metadata.go @@ -0,0 +1,59 @@ +package evidence + +import ( + "fmt" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" +) + +type ProfileMetadata struct { + Name string `json:"name"` + JobTimeoutSeconds int64 `json:"job_timeout_seconds"` + SuiteDeadlineSeconds int64 `json:"suite_deadline_seconds"` + DefaultSeed string `json:"default_seed"` + AgentCount int `json:"agent_count"` + StressRounds int `json:"stress_rounds"` + ConcurrentOperations int `json:"concurrent_operations"` + ConcurrentSessionsPerKind int `json:"concurrent_sessions_per_kind"` + TransferPairs int `json:"transfer_pairs"` + TransferBytes uint64 `json:"transfer_bytes"` + DashboardRestartCycles int `json:"dashboard_restart_cycles"` + Iterations int `json:"iterations"` + StreamBoundaryAllowed int `json:"stream_boundary_allowed"` + StreamBoundaryRejected int `json:"stream_boundary_rejected"` +} + +type ResourceBudgetMetadata struct { + WarmupRunsPerPath int `json:"warmup_runs_per_path"` + BaselineSampleCount int `json:"baseline_sample_count"` + EndSampleCount int `json:"end_sample_count"` + SampleIntervalMilliseconds int64 `json:"sample_interval_milliseconds"` + ChildProcessCountDrift int `json:"child_process_count_drift"` + ListenerCountDrift int `json:"listener_count_drift"` + NonStdioFDCountDrift int `json:"non_stdio_fd_count_drift"` + DashboardRSSDeltaBytes uint64 `json:"dashboard_rss_delta_bytes"` + AgentRSSDeltaBytes uint64 `json:"agent_rss_delta_bytes"` + TransferHeapBytes uint64 `json:"transfer_heap_bytes"` +} + +func profileMetadata(profile contract.Profile) ProfileMetadata { + return ProfileMetadata{Name: string(profile.Name()), JobTimeoutSeconds: int64(profile.JobTimeout().Seconds()), SuiteDeadlineSeconds: int64(profile.SuiteDeadline().Seconds()), DefaultSeed: fmt.Sprintf("0x%x", uint64(profile.Seed())), AgentCount: profile.AgentCount(), StressRounds: profile.StressRounds(), ConcurrentOperations: profile.ConcurrentOperations(), ConcurrentSessionsPerKind: profile.ConcurrentSessions(), TransferPairs: profile.TransferPairs(), TransferBytes: profile.TransferBytes(), DashboardRestartCycles: profile.DashboardRestartCycles(), Iterations: profile.Iterations(), StreamBoundaryAllowed: profile.StreamBoundaryAllowed(), StreamBoundaryRejected: profile.StreamBoundaryRejected()} +} + +func resourceBudgetMetadata(budget contract.ResourceBudget) ResourceBudgetMetadata { + return ResourceBudgetMetadata{WarmupRunsPerPath: budget.WarmupRuns(), BaselineSampleCount: budget.SampleCount(), EndSampleCount: budget.SampleCount(), SampleIntervalMilliseconds: budget.SampleInterval().Milliseconds(), ChildProcessCountDrift: budget.ChildProcessCountDrift(), ListenerCountDrift: budget.ListenerCountDrift(), NonStdioFDCountDrift: budget.NonStdioFDCountDrift(), DashboardRSSDeltaBytes: budget.DashboardRSSDeltaBytes(), AgentRSSDeltaBytes: budget.AgentRSSDeltaBytes(), TransferHeapBytes: budget.TransferHeapBytes()} +} + +func (p ProfileMetadata) Validate() error { + if p.Name == "" || p.JobTimeoutSeconds < 1 || p.SuiteDeadlineSeconds < 1 || p.DefaultSeed == "" || p.DefaultSeed == "0x0" || p.AgentCount < 1 || p.StressRounds < 1 || p.ConcurrentOperations < 1 || p.ConcurrentSessionsPerKind < 1 || p.TransferPairs < 1 || p.TransferBytes == 0 || p.DashboardRestartCycles < 1 || p.Iterations < 1 || p.StreamBoundaryAllowed < 1 || p.StreamBoundaryRejected <= p.StreamBoundaryAllowed { + return fmt.Errorf("profile fields are invalid") + } + return nil +} + +func (b ResourceBudgetMetadata) Validate() error { + if b.WarmupRunsPerPath < 1 || b.BaselineSampleCount < 1 || b.EndSampleCount < 1 || b.SampleIntervalMilliseconds < 1 || b.ChildProcessCountDrift < 0 || b.ListenerCountDrift < 0 || b.NonStdioFDCountDrift < 0 || b.DashboardRSSDeltaBytes == 0 || b.AgentRSSDeltaBytes == 0 || b.TransferHeapBytes == 0 { + return fmt.Errorf("resource budget fields are invalid") + } + return nil +} diff --git a/integration/agentcompat/internal/evidence/dedicated_artifacts.go b/integration/agentcompat/internal/evidence/dedicated_artifacts.go new file mode 100644 index 00000000..bdef7046 --- /dev/null +++ b/integration/agentcompat/internal/evidence/dedicated_artifacts.go @@ -0,0 +1,213 @@ +package evidence + +import ( + "errors" + "strings" + "time" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" +) + +type transferEvidence struct { + WarmupUploadBytes uint64 `json:"warmup_upload_bytes"` + WarmupDownloadBytes uint64 `json:"warmup_download_bytes"` + WarmupSHA256 string `json:"warmup_sha256"` + WarmupDuration time.Duration `json:"warmup_duration"` + WarmupDeadlineRemaining time.Duration `json:"warmup_deadline_remaining"` + WarmupQuiescent bool `json:"warmup_quiescent"` + UploadBytes uint64 `json:"upload_bytes"` + DownloadBytes uint64 `json:"download_bytes"` + UploadSHA256 string `json:"upload_sha256"` + DownloadSHA256 string `json:"download_sha256"` + UploadChunks uint64 `json:"upload_chunks"` + DownloadChunks uint64 `json:"download_chunks"` + UploadDuration time.Duration `json:"upload_duration"` + DownloadDuration time.Duration `json:"download_duration"` + RetainedHeapBytes uint64 `json:"retained_heap_bytes"` + Mode string `json:"mode"` + CreateDirs bool `json:"create_dirs"` + UploadReplayRejected bool `json:"upload_replay_rejected"` + DownloadReplayRejected bool `json:"download_replay_rejected"` + OversizeRejected bool `json:"oversize_rejected"` + AgentTempResidue int `json:"agent_temp_residue"` + DashboardSpoolResidue int `json:"dashboard_spool_residue"` + OutsideRootSentinelsUnchanged bool `json:"outside_root_sentinels_unchanged"` +} + +type transferArtifact struct { + Scenario string `json:"scenario"` + Fault string `json:"fault,omitempty"` + Passed bool `json:"passed"` + CleanupOK bool `json:"cleanup_ok"` + Error string `json:"error,omitempty"` + Evidence transferEvidence `json:"evidence"` +} + +type reconnectObservation struct { + ServerID uint64 + UUID string + OldGeneration uint64 + NewGeneration uint64 + DisconnectAt time.Time + ReconnectAt time.Time + TaskIDs []uint64 + ResultIDs []uint64 + PostReconnect bool + AgentRestarted bool +} + +type listenerIdentity struct { + Address string + Inode uint64 +} + +type fixtureIdentity struct { + WorkspaceRoot string + ConfigPath string + DatabasePath string + BinaryPath string + HTTP listenerIdentity + Receipt listenerIdentity + HTTPS listenerIdentity +} + +type runtimeIdentity struct { + Generation uint64 + PID int + ProcessGroupID int +} + +type receiptEvent struct { + Sequence uint64 `json:"sequence"` + DashboardGeneration uint64 `json:"dashboard_generation"` + GateGeneration uint64 `json:"gate_generation"` + ServerID uint64 `json:"server_id"` + TaskID uint64 `json:"task_id"` + TaskType uint64 `json:"task_type"` + Kind string `json:"kind"` +} + +type receiptPair struct { + Task receiptEvent `json:"task"` + Result receiptEvent `json:"result"` +} + +type cleanupRecord struct { + Name string `json:"name"` + PID int `json:"pid"` + Forced bool `json:"forced"` + Error string `json:"error,omitempty"` +} + +type cleanupReceipt struct { + Passed bool `json:"passed"` + Forced bool `json:"forced"` + Processes []cleanupRecord `json:"processes"` +} + +type reconnectEvidence struct { + Fixture struct { + Dashboard fixtureIdentity `json:"dashboard"` + AgentRoot string `json:"agent_root"` + AgentConfigPath string `json:"agent_config_path"` + AgentBinaryPath string `json:"agent_binary_path"` + } `json:"fixture"` + Runtime struct { + DashboardBefore runtimeIdentity `json:"dashboard_before"` + DashboardAfter runtimeIdentity `json:"dashboard_after"` + AgentBefore runtimeIdentity `json:"agent_before"` + AgentAfter runtimeIdentity `json:"agent_after"` + StateGenerationBeforeAgentRestart uint64 `json:"state_generation_before_agent_restart"` + StateGenerationAfterAgentRestart uint64 `json:"state_generation_after_agent_restart"` + } `json:"runtime"` + Identity struct { + ServerID uint64 `json:"server_id"` + UUID string `json:"uuid"` + DashboardConfigUnchanged bool `json:"dashboard_config_unchanged"` + AgentConfigUnchanged bool `json:"agent_config_unchanged"` + DashboardFixtureUnchanged bool `json:"dashboard_fixture_unchanged"` + ClientsRecreated bool `json:"clients_recreated"` + BootstrapRecreated bool `json:"bootstrap_recreated"` + } `json:"identity"` + Lifecycle struct { + DisconnectAt time.Time `json:"disconnect_at"` + ReconnectAt time.Time `json:"reconnect_at"` + ReconnectInterval time.Duration `json:"reconnect_interval"` + DashboardReceipts []receiptPair `json:"dashboard_receipts"` + AgentReceipts []receiptPair `json:"agent_receipts"` + StaleGenerationReceipts int `json:"stale_generation_receipts"` + DuplicateTaskIDs int `json:"duplicate_task_ids"` + LostResultIDs int `json:"lost_result_ids"` + OutsideRootSentinelUnchanged bool `json:"outside_root_sentinel_unchanged"` + } `json:"lifecycle"` + Observation reconnectObservation `json:"observation"` + AgentCleanup cleanupReceipt `json:"agent_cleanup"` + DashboardCleanup cleanupReceipt `json:"dashboard_cleanup"` +} + +type reconnectArtifact struct { + Scenario string `json:"scenario"` + Fault string `json:"fault,omitempty"` + Passed bool `json:"passed"` + CleanupOK bool `json:"cleanup_ok"` + Error string `json:"error,omitempty"` + Evidence reconnectEvidence `json:"evidence"` +} + +func validateTransferArtifact(metadata Metadata, result ScenarioResult, artifact transferArtifact) error { + if err := validateArtifactHeader(metadata, result, artifact.Scenario, artifact.Fault, artifact.Passed, artifact.CleanupOK, artifact.Error); err != nil { + return err + } + switch artifact.Fault { + case "": + if !artifact.Passed { + return errors.New("transfer success artifact reports failure") + } + if err := validateTransferSuccess(artifact.Evidence); err != nil { + return err + } + case contract.FaultTransferHash: + if artifact.Passed || !strings.Contains(artifact.Error, "injected hash mismatch") { + return errors.New("transfer-hash artifact does not identify the injected failure") + } + evidence := artifact.Evidence + if evidence.WarmupUploadBytes != 65536 || evidence.WarmupDownloadBytes != 65536 || evidence.WarmupSHA256 == "" || evidence.WarmupDuration <= 0 || evidence.WarmupDeadlineRemaining <= 0 || !evidence.WarmupQuiescent { + return errors.New("transfer-hash artifact omitted warmup evidence") + } + if evidence.UploadBytes != 0 || evidence.DownloadBytes != 0 || evidence.UploadSHA256 != "" || evidence.DownloadSHA256 != "" || evidence.UploadChunks != 0 || evidence.DownloadChunks != 0 || evidence.UploadDuration != 0 || evidence.DownloadDuration != 0 || evidence.RetainedHeapBytes != 0 || evidence.Mode != "" || evidence.CreateDirs || evidence.UploadReplayRejected || evidence.DownloadReplayRejected || evidence.OversizeRejected { + return errors.New("transfer-hash artifact presents stale success evidence") + } + if evidence.AgentTempResidue != 0 || evidence.DashboardSpoolResidue != 0 || !evidence.OutsideRootSentinelsUnchanged { + return errors.New("transfer-hash artifact cleanup or sentinel evidence failed") + } + default: + return errors.New("transfer artifact has unsupported fault") + } + return nil +} + +func validateArtifactHeader(metadata Metadata, result ScenarioResult, scenarioName, fault string, passed, cleanupOK bool, errorText string) error { + if scenarioName != result.Name || scenarioName != metadata.Scenarios[0] || fault != metadata.Fault || passed != result.Passed || !cleanupOK || errorText != result.Error { + return errors.New("dedicated artifact does not agree with metadata, results, or cleanup") + } + if passed && errorText != "" || !passed && errorText == "" { + return errors.New("dedicated artifact pass/error state is inconsistent") + } + return nil +} + +func validateTransferSuccess(evidence transferEvidence) error { + if evidence.WarmupUploadBytes != 65536 || evidence.WarmupDownloadBytes != 65536 || evidence.WarmupSHA256 == "" || evidence.WarmupDuration <= 0 || evidence.WarmupDeadlineRemaining <= 0 || !evidence.WarmupQuiescent { + return errors.New("transfer warmup evidence is invalid") + } + if evidence.UploadBytes != contract.TransferBytes || evidence.DownloadBytes != contract.TransferBytes || evidence.UploadSHA256 == "" || !strings.EqualFold(evidence.UploadSHA256, evidence.DownloadSHA256) { + return errors.New("transfer byte counts or hashes are invalid") + } + if evidence.UploadChunks == 0 || evidence.DownloadChunks == 0 || evidence.UploadDuration <= 0 || evidence.DownloadDuration <= 0 || evidence.RetainedHeapBytes > contract.TransferHeapBytes { + return errors.New("transfer measurement evidence is invalid") + } + if evidence.Mode != "0640" || !evidence.CreateDirs || !evidence.UploadReplayRejected || !evidence.DownloadReplayRejected || !evidence.OversizeRejected || evidence.AgentTempResidue != 0 || evidence.DashboardSpoolResidue != 0 || !evidence.OutsideRootSentinelsUnchanged { + return errors.New("transfer contract evidence is invalid") + } + return nil +} diff --git a/integration/agentcompat/internal/evidence/dedicated_assertion_test.go b/integration/agentcompat/internal/evidence/dedicated_assertion_test.go new file mode 100644 index 00000000..fa5f49e6 --- /dev/null +++ b/integration/agentcompat/internal/evidence/dedicated_assertion_test.go @@ -0,0 +1,46 @@ +package evidence + +import ( + "testing" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" +) + +func TestEvidence_DedicatedArtifactsRequireExactScenarioAssertions(t *testing.T) { + tests := []struct { + name string + scenario string + assertion string + artifact func(*testing.T, string) + }{ + {"transfer generic", contract.ScenarioTransfer100MiB, "scenario passed", func(t *testing.T, dir string) { + writeJSONEvidenceFile(t, dir, "transfer.json", validTransferArtifact("", true)) + }}, + {"transfer mismatched", contract.ScenarioTransfer100MiB, contract.AssertionReconnectDisconnect, func(t *testing.T, dir string) { + writeJSONEvidenceFile(t, dir, "transfer.json", validTransferArtifact("", true)) + }}, + {"reconnect generic", contract.ScenarioReconnect, "scenario passed", func(t *testing.T, dir string) { + writeJSONEvidenceFile(t, dir, "reconnect.json", validReconnectArtifact("", true)) + }}, + {"reconnect mismatched", contract.ScenarioReconnect, contract.AssertionTransferWarmup, func(t *testing.T, dir string) { + writeJSONEvidenceFile(t, dir, "reconnect.json", validReconnectArtifact("", true)) + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + writeDedicatedExecutableEvidence(t, dir, test.scenario, "", true) + results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: test.scenario, Passed: true, Assertions: []Assertion{{Name: test.assertion, Passed: true}}}}} + writeJSONEvidenceFile(t, dir, "results.json", results) + junit, err := JUnit(results) + if err != nil { + t.Fatalf("JUnit: %v", err) + } + writeEvidenceFile(t, dir, "junit.xml", string(junit)) + test.artifact(t, dir) + if err := ValidateDirectory(dir); err == nil { + t.Fatal("generic assertion accepted for dedicated evidence") + } + }) + } +} diff --git a/integration/agentcompat/internal/evidence/dedicated_fault_fields_test.go b/integration/agentcompat/internal/evidence/dedicated_fault_fields_test.go new file mode 100644 index 00000000..7f929637 --- /dev/null +++ b/integration/agentcompat/internal/evidence/dedicated_fault_fields_test.go @@ -0,0 +1,86 @@ +package evidence + +import ( + "testing" + "time" +) + +func TestEvidence_TransferHashRejectsEverySuccessOnlyField(t *testing.T) { + tests := []struct { + name string + mutate func(*transferEvidence) + }{ + {"upload bytes", func(value *transferEvidence) { value.UploadBytes = 1 }}, + {"download bytes", func(value *transferEvidence) { value.DownloadBytes = 1 }}, + {"upload hash", func(value *transferEvidence) { value.UploadSHA256 = "stale" }}, + {"download hash", func(value *transferEvidence) { value.DownloadSHA256 = "stale" }}, + {"upload chunks", func(value *transferEvidence) { value.UploadChunks = 1 }}, + {"download chunks", func(value *transferEvidence) { value.DownloadChunks = 1 }}, + {"upload duration", func(value *transferEvidence) { value.UploadDuration = time.Nanosecond }}, + {"download duration", func(value *transferEvidence) { value.DownloadDuration = time.Nanosecond }}, + {"retained heap", func(value *transferEvidence) { value.RetainedHeapBytes = 1 }}, + {"mode", func(value *transferEvidence) { value.Mode = "0640" }}, + {"create dirs", func(value *transferEvidence) { value.CreateDirs = true }}, + {"upload replay", func(value *transferEvidence) { value.UploadReplayRejected = true }}, + {"download replay", func(value *transferEvidence) { value.DownloadReplayRejected = true }}, + {"oversize", func(value *transferEvidence) { value.OversizeRejected = true }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + writeDedicatedExecutableEvidence(t, dir, "transfer-100mib", "transfer-hash", false) + artifact := validTransferArtifact("transfer-hash", false) + test.mutate(&artifact.Evidence) + writeJSONEvidenceFile(t, dir, "transfer.json", artifact) + if err := ValidateDirectory(dir); err == nil { + t.Fatal("stale transfer success field accepted") + } + }) + } +} + +func TestEvidence_DashboardExitRejectsEveryPostFaultField(t *testing.T) { + tests := []struct { + name string + mutate func(*reconnectEvidence) + }{ + {"dashboard after", func(value *reconnectEvidence) { value.Runtime.DashboardAfter.PID = 1 }}, + {"agent after", func(value *reconnectEvidence) { value.Runtime.AgentAfter.PID = 1 }}, + {"state before restart", func(value *reconnectEvidence) { value.Runtime.StateGenerationBeforeAgentRestart = 1 }}, + {"state after restart", func(value *reconnectEvidence) { value.Runtime.StateGenerationAfterAgentRestart = 1 }}, + {"identity dashboard config", func(value *reconnectEvidence) { value.Identity.DashboardConfigUnchanged = true }}, + {"identity agent config", func(value *reconnectEvidence) { value.Identity.AgentConfigUnchanged = true }}, + {"identity fixture", func(value *reconnectEvidence) { value.Identity.DashboardFixtureUnchanged = true }}, + {"clients recreated", func(value *reconnectEvidence) { value.Identity.ClientsRecreated = true }}, + {"bootstrap recreated", func(value *reconnectEvidence) { value.Identity.BootstrapRecreated = true }}, + {"reconnect timestamp", func(value *reconnectEvidence) { value.Lifecycle.ReconnectAt = time.Now() }}, + {"reconnect interval", func(value *reconnectEvidence) { value.Lifecycle.ReconnectInterval = time.Second }}, + {"dashboard receipts", func(value *reconnectEvidence) { value.Lifecycle.DashboardReceipts = []receiptPair{{}} }}, + {"agent receipts", func(value *reconnectEvidence) { value.Lifecycle.AgentReceipts = []receiptPair{{}} }}, + {"stale accounting", func(value *reconnectEvidence) { value.Lifecycle.StaleGenerationReceipts = 1 }}, + {"duplicate accounting", func(value *reconnectEvidence) { value.Lifecycle.DuplicateTaskIDs = 1 }}, + {"lost accounting", func(value *reconnectEvidence) { value.Lifecycle.LostResultIDs = 1 }}, + {"observation server", func(value *reconnectEvidence) { value.Observation.ServerID = 1 }}, + {"observation uuid", func(value *reconnectEvidence) { value.Observation.UUID = "stale" }}, + {"old generation", func(value *reconnectEvidence) { value.Observation.OldGeneration = 1 }}, + {"new generation", func(value *reconnectEvidence) { value.Observation.NewGeneration = 1 }}, + {"disconnect observation", func(value *reconnectEvidence) { value.Observation.DisconnectAt = time.Now() }}, + {"reconnect observation", func(value *reconnectEvidence) { value.Observation.ReconnectAt = time.Now() }}, + {"task ids", func(value *reconnectEvidence) { value.Observation.TaskIDs = []uint64{1} }}, + {"result ids", func(value *reconnectEvidence) { value.Observation.ResultIDs = []uint64{1} }}, + {"post reconnect", func(value *reconnectEvidence) { value.Observation.PostReconnect = true }}, + {"agent restarted", func(value *reconnectEvidence) { value.Observation.AgentRestarted = true }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + writeDedicatedExecutableEvidence(t, dir, "reconnect", "dashboard-exit", false) + artifact := validReconnectArtifact("dashboard-exit", false) + test.mutate(&artifact.Evidence) + writeJSONEvidenceFile(t, dir, "reconnect.json", artifact) + if err := ValidateDirectory(dir); err == nil { + t.Fatal("stale reconnect success field accepted") + } + }) + } +} diff --git a/integration/agentcompat/internal/evidence/dedicated_validation_test.go b/integration/agentcompat/internal/evidence/dedicated_validation_test.go new file mode 100644 index 00000000..3e488c3f --- /dev/null +++ b/integration/agentcompat/internal/evidence/dedicated_validation_test.go @@ -0,0 +1,212 @@ +package evidence + +import ( + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" +) + +func TestEvidence_TransferAndReconnectRequireDedicatedArtifacts(t *testing.T) { + for _, scenarioName := range []string{"transfer-100mib", "reconnect"} { + t.Run(scenarioName, func(t *testing.T) { + dir := t.TempDir() + writeExecutableEvidence(t, dir, "pr-full", scenarioName, true, true, true) + if err := ValidateDirectory(dir); err == nil { + t.Fatal("missing dedicated artifact accepted") + } + }) + } +} + +func TestEvidence_ValidatesTypedTransferSuccessAndFaultArtifacts(t *testing.T) { + tests := []struct { + name string + fault string + passed bool + }{ + {"success", "", true}, + {"hash fault", "transfer-hash", false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + writeDedicatedExecutableEvidence(t, dir, "transfer-100mib", test.fault, test.passed) + artifact := validTransferArtifact(test.fault, test.passed) + writeJSONEvidenceFile(t, dir, "transfer.json", artifact) + if err := ValidateDirectory(dir); err != nil { + t.Fatalf("validate transfer evidence: %v", err) + } + }) + } +} + +func TestEvidence_ValidatesTypedReconnectSuccessAndFaultArtifacts(t *testing.T) { + tests := []struct { + name string + fault string + passed bool + }{ + {"success", "", true}, + {"dashboard fault", "dashboard-exit", false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + writeDedicatedExecutableEvidence(t, dir, "reconnect", test.fault, test.passed) + artifact := validReconnectArtifact(test.fault, test.passed) + writeJSONEvidenceFile(t, dir, "reconnect.json", artifact) + if err := ValidateDirectory(dir); err != nil { + t.Fatalf("validate reconnect evidence: %v", err) + } + }) + } +} + +func TestEvidence_RejectsMisleadingDedicatedArtifact(t *testing.T) { + dir := t.TempDir() + writeDedicatedExecutableEvidence(t, dir, "transfer-100mib", "transfer-hash", false) + artifact := validTransferArtifact("transfer-hash", false) + artifact.Evidence.UploadBytes = 104857600 + writeJSONEvidenceFile(t, dir, "transfer.json", artifact) + if err := ValidateDirectory(dir); err == nil { + t.Fatal("fault artifact containing stale success accepted") + } +} + +func TestEvidence_RejectsWrongOrStaleDedicatedArtifact(t *testing.T) { + dir := t.TempDir() + writeExecutableEvidence(t, dir, "pr-full", "transfer-100mib", true, true, true) + writeEvidenceFile(t, dir, "reconnect.json", `{}`) + if err := ValidateDirectory(dir); err == nil { + t.Fatal("wrong dedicated artifact accepted") + } +} + +func TestEvidence_RejectsDedicatedArtifactWithPublicMode(t *testing.T) { + dir := t.TempDir() + writeExecutableEvidence(t, dir, "pr-full", "transfer-100mib", true, true, true) + path := filepath.Join(dir, "transfer.json") + if err := os.WriteFile(path, []byte(`{}`), 0o644); err != nil { + t.Fatalf("write transfer artifact: %v", err) + } + if err := ValidateDirectory(dir); err == nil { + t.Fatal("public dedicated artifact mode accepted") + } +} + +func writeDedicatedExecutableEvidence(t *testing.T, dir, scenarioName, fault string, passed bool) { + t.Helper() + metadata := validMetadata(t, dir, scenarioName) + metadata.Fault = fault + writeJSONEvidenceFile(t, dir, "metadata.json", metadata) + errorText := "" + if !passed { + if scenarioName == "transfer-100mib" { + errorText = "transfer scenario: injected hash mismatch" + } else { + errorText = "reconnect scenario: injected Dashboard exit" + } + } + definition, err := contract.ScenarioDefinitionByName(scenarioName) + if err != nil { + t.Fatalf("scenario definition: %v", err) + } + assertions := make([]Assertion, 0, len(definition.Assertions(fault))) + for _, assertion := range definition.Assertions(fault) { + assertions = append(assertions, Assertion{Name: assertion.Name, Passed: assertion.Passed}) + } + results := Results{Profile: "pr-full", Passed: passed, Scenarios: []ScenarioResult{{Name: scenarioName, Passed: passed, Assertions: assertions, Error: errorText}}} + writeJSONEvidenceFile(t, dir, "results.json", results) + junit, err := JUnit(results) + if err != nil { + t.Fatalf("JUnit: %v", err) + } + writeEvidenceFile(t, dir, "junit.xml", string(junit)) + writeEvidenceFile(t, dir, "cleanup.json", fmt.Sprintf(`{"passed":true,"scenario":%q,"finished_at":"2026-01-02T03:04:05Z"}`, scenarioName)) +} + +func validTransferArtifact(fault string, passed bool) transferArtifact { + errorText := "" + if !passed { + errorText = "transfer scenario: injected hash mismatch" + } + evidence := transferEvidence{WarmupUploadBytes: 65536, WarmupDownloadBytes: 65536, WarmupSHA256: "abc", WarmupDuration: time.Nanosecond, WarmupDeadlineRemaining: time.Second, WarmupQuiescent: true, OutsideRootSentinelsUnchanged: true} + if passed { + evidence.UploadBytes = 104857600 + evidence.DownloadBytes = 104857600 + evidence.UploadSHA256 = "abc" + evidence.DownloadSHA256 = "abc" + evidence.UploadChunks = 2 + evidence.DownloadChunks = 2 + evidence.UploadDuration = time.Second + evidence.DownloadDuration = time.Second + evidence.Mode = "0640" + evidence.CreateDirs = true + evidence.UploadReplayRejected = true + evidence.DownloadReplayRejected = true + evidence.OversizeRejected = true + } + return transferArtifact{Scenario: "transfer-100mib", Fault: fault, Passed: passed, CleanupOK: true, Error: errorText, Evidence: evidence} +} + +func validReconnectArtifact(fault string, passed bool) reconnectArtifact { + var artifact reconnectArtifact + artifact.Scenario = "reconnect" + artifact.Fault = fault + artifact.Passed = passed + artifact.CleanupOK = true + if !passed { + artifact.Error = "reconnect scenario: injected Dashboard exit" + } + evidence := &artifact.Evidence + evidence.Fixture.Dashboard.WorkspaceRoot = "/tmp/dashboard" + evidence.Fixture.Dashboard.ConfigPath = "/tmp/dashboard/config" + evidence.Fixture.Dashboard.BinaryPath = "/tmp/dashboard/bin" + evidence.Fixture.AgentRoot = "/tmp/agent" + evidence.Fixture.AgentConfigPath = "/tmp/agent/config" + evidence.Fixture.AgentBinaryPath = "/tmp/agent/bin" + evidence.Runtime.DashboardBefore = runtimeIdentity{Generation: 1, PID: 10, ProcessGroupID: 10} + evidence.Runtime.AgentBefore = runtimeIdentity{Generation: 1, PID: 20, ProcessGroupID: 20} + evidence.Identity.ServerID = 1 + evidence.Identity.UUID = "uuid" + evidence.Lifecycle.DisconnectAt = time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + evidence.Lifecycle.OutsideRootSentinelUnchanged = true + evidence.AgentCleanup = validCleanupReceipt("agent", expectedProcessCount(passed)) + evidence.DashboardCleanup = validCleanupReceipt("dashboard", expectedProcessCount(passed)) + if passed { + evidence.Runtime.DashboardAfter = runtimeIdentity{Generation: 2, PID: 11, ProcessGroupID: 11} + evidence.Runtime.AgentAfter = runtimeIdentity{Generation: 2, PID: 21, ProcessGroupID: 21} + evidence.Runtime.StateGenerationBeforeAgentRestart = 1 + evidence.Runtime.StateGenerationAfterAgentRestart = 2 + evidence.Identity.DashboardConfigUnchanged = true + evidence.Identity.AgentConfigUnchanged = true + evidence.Identity.DashboardFixtureUnchanged = true + evidence.Identity.ClientsRecreated = true + evidence.Identity.BootstrapRecreated = true + evidence.Lifecycle.ReconnectAt = evidence.Lifecycle.DisconnectAt.Add(time.Second) + evidence.Lifecycle.ReconnectInterval = time.Second + evidence.Lifecycle.DashboardReceipts = make([]receiptPair, 3) + evidence.Lifecycle.AgentReceipts = make([]receiptPair, 2) + evidence.Observation = reconnectObservation{ServerID: 1, UUID: "uuid", OldGeneration: 1, NewGeneration: 2, DisconnectAt: evidence.Lifecycle.DisconnectAt, ReconnectAt: evidence.Lifecycle.ReconnectAt, TaskIDs: []uint64{1, 2, 3, 4, 5}, ResultIDs: []uint64{1, 2, 3, 4, 5}, PostReconnect: true, AgentRestarted: true} + } + return artifact +} + +func validCleanupReceipt(name string, count int) cleanupReceipt { + receipt := cleanupReceipt{Passed: true, Processes: make([]cleanupRecord, count)} + for index := range receipt.Processes { + receipt.Processes[index] = cleanupRecord{Name: name, PID: index + 1} + } + return receipt +} + +func expectedProcessCount(passed bool) int { + if passed { + return 2 + } + return 1 +} diff --git a/integration/agentcompat/internal/evidence/directory_scan.go b/integration/agentcompat/internal/evidence/directory_scan.go new file mode 100644 index 00000000..0e02b4ca --- /dev/null +++ b/integration/agentcompat/internal/evidence/directory_scan.go @@ -0,0 +1,133 @@ +package evidence + +import ( + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +const ( + maxEvidenceFiles = 128 + maxEvidenceBytes = 32 << 20 +) + +func scanDirectory(resultsDir string) (map[string]os.FileInfo, error) { + if strings.TrimSpace(resultsDir) == "" { + return nil, errors.New("evidence directory is required") + } + info, err := os.Lstat(resultsDir) + if err != nil { + return nil, fmt.Errorf("stat evidence directory: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return nil, errors.New("evidence path must be a directory") + } + if info.Mode().Perm() != 0o700 { + return nil, errors.New("evidence directory must use mode 0700") + } + seen := make(map[string]os.FileInfo) + var totalBytes int64 + err = filepath.WalkDir(resultsDir, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if path == resultsDir { + return nil + } + if entry.IsDir() { + relative, err := filepath.Rel(resultsDir, path) + if err != nil { + return err + } + if relative != "agents" { + return fmt.Errorf("evidence path is not allowed: %s", relative) + } + return nil + } + if entry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("evidence symlink is not allowed: %s", path) + } + if !entry.Type().IsRegular() { + return fmt.Errorf("evidence file is not regular: %s", path) + } + relative, err := filepath.Rel(resultsDir, path) + if err != nil { + return err + } + fileInfo, err := entry.Info() + if err != nil { + return err + } + if !allowedEvidencePath(relative) { + return fmt.Errorf("evidence path is not allowed: %s", relative) + } + if fileInfo.Mode().Perm() != 0o600 { + return fmt.Errorf("evidence file must use mode 0600: %s", relative) + } + seen[relative] = fileInfo + if len(seen) > maxEvidenceFiles { + return errors.New("too many evidence files") + } + totalBytes += fileInfo.Size() + if totalBytes > maxEvidenceBytes { + return errors.New("evidence files exceed size limit") + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + if Redact(string(data)) != string(data) { + return fmt.Errorf("credential detected in evidence file: %s", path) + } + switch filepath.Ext(path) { + case ".json": + if !json.Valid(data) { + return fmt.Errorf("invalid JSON evidence file: %s", path) + } + case ".xml": + var document any + if err := xml.Unmarshal(data, &document); err != nil { + return fmt.Errorf("invalid XML evidence file: %s: %w", path, err) + } + } + return nil + }) + return seen, err +} + +func allowedEvidencePath(relative string) bool { + for _, name := range EvidenceFiles() { + if name == relative { + return true + } + } + if filepath.Dir(relative) != "agents" || filepath.Ext(relative) != ".log" { + return false + } + base := strings.TrimSuffix(filepath.Base(relative), ".log") + if base == "" || base == "." { + return false + } + for _, character := range base { + if character != '-' && character != '_' && character != '.' && (character < '0' || character > '9') && (character < 'A' || character > 'Z') && (character < 'a' || character > 'z') { + return false + } + } + return true +} + +func readJSONFile[T any](resultsDir, name string) (T, error) { + var value T + data, err := os.ReadFile(filepath.Join(resultsDir, name)) + if err != nil { + return value, fmt.Errorf("read %s: %w", name, err) + } + if err := json.Unmarshal(data, &value); err != nil { + return value, fmt.Errorf("parse %s: %w", name, err) + } + return value, nil +} diff --git a/integration/agentcompat/internal/evidence/directory_security_test.go b/integration/agentcompat/internal/evidence/directory_security_test.go new file mode 100644 index 00000000..dcdc339c --- /dev/null +++ b/integration/agentcompat/internal/evidence/directory_security_test.go @@ -0,0 +1,69 @@ +package evidence + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" +) + +func TestEvidence_ValidateDirectoryRejectsPublicRootOrEvidenceFile(t *testing.T) { + tests := []struct { + name string + path string + }{ + {name: "root", path: ""}, + {name: "metadata", path: "metadata.json"}, + {name: "results", path: "results.json"}, + {name: "junit", path: "junit.xml"}, + {name: "cleanup", path: "cleanup.json"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, true, true, true) + path := dir + if test.path != "" { + path = filepath.Join(dir, test.path) + } + if err := os.Chmod(path, 0o644); err != nil { + t.Fatalf("chmod fixture: %v", err) + } + if err := ValidateDirectory(dir); err == nil { + t.Fatal("public evidence mode accepted") + } + }) + } +} + +func TestEvidence_ValidateDirectoryRejectsUnexpectedPaths(t *testing.T) { + tests := []struct { + name string + path string + mode os.FileMode + }{ + {name: "payload", path: "payload.bin", mode: 0o600}, + {name: "executable", path: "agentcompat", mode: 0o700}, + {name: "nested log", path: "agents/nested/agent.log", mode: 0o600}, + {name: "unknown agent file", path: "agents/agent.bin", mode: 0o600}, + {name: "unknown directory", path: "unexpected/file.log", mode: 0o600}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, true, true, true) + path := filepath.Join(dir, test.path) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatalf("mkdir unexpected parent: %v", err) + } + if err := os.WriteFile(path, []byte("unexpected"), test.mode); err != nil { + t.Fatalf("write unexpected file: %v", err) + } + if err := ValidateDirectory(dir); err == nil || !strings.Contains(err.Error(), "not allowed") { + t.Fatalf("unexpected path rejection err=%v", err) + } + }) + } +} diff --git a/integration/agentcompat/internal/evidence/directory_validation.go b/integration/agentcompat/internal/evidence/directory_validation.go new file mode 100644 index 00000000..4cc40c79 --- /dev/null +++ b/integration/agentcompat/internal/evidence/directory_validation.go @@ -0,0 +1,174 @@ +package evidence + +import ( + "encoding/xml" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "time" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" +) + +type cleanupEvidence struct { + Passed bool `json:"passed"` + Scenario string `json:"scenario"` + FinishedAt string `json:"finished_at"` +} + +type currentEvidenceProfile struct { + RequiredFiles []string + DedicatedFile string + Executable bool +} + +func currentProfile(metadata Metadata) (currentEvidenceProfile, error) { + if len(metadata.Scenarios) == 0 { + return currentEvidenceProfile{}, errors.New("metadata scenarios are required") + } + if len(metadata.Scenarios) == 1 && metadata.Scenarios[0] == contract.ScenarioMetadata { + if metadata.Fault != "" { + return currentEvidenceProfile{}, errors.New("metadata scenario does not support fault injection") + } + return currentEvidenceProfile{RequiredFiles: []string{"metadata.json"}}, nil + } + if len(metadata.Scenarios) != 1 { + return currentEvidenceProfile{}, errors.New("current evidence profile requires one supported scenario") + } + scenarioValue, err := contract.NewScenario(metadata.Scenarios[0]) + if err != nil { + return currentEvidenceProfile{}, fmt.Errorf("construct metadata scenario: %w", err) + } + if !contract.IsSupportedScenario(metadata.Scenarios[0]) || metadata.Scenarios[0] == contract.ScenarioMetadata { + return currentEvidenceProfile{}, errors.New("current evidence profile requires one supported scenario") + } + faultValue := contract.Fault{} + if metadata.Fault != "" { + faultValue, err = contract.NewFault(metadata.Fault) + if err != nil { + return currentEvidenceProfile{}, fmt.Errorf("construct metadata fault: %w", err) + } + } + if err := contract.ValidateScenarioFault(scenarioValue, faultValue); err != nil { + return currentEvidenceProfile{}, err + } + definition, err := contract.ScenarioDefinitionByName(metadata.Scenarios[0]) + if err != nil { + return currentEvidenceProfile{}, err + } + profile := currentEvidenceProfile{RequiredFiles: []string{"metadata.json", "results.json", "junit.xml", "cleanup.json"}, DedicatedFile: definition.DedicatedArtifactName(), Executable: true} + if profile.DedicatedFile != "" { + profile.RequiredFiles = append(profile.RequiredFiles, profile.DedicatedFile) + } + return profile, nil +} + +func ValidateDirectory(resultsDir string) error { + files, err := scanDirectory(resultsDir) + if err != nil { + return err + } + if len(files) == 0 { + return errors.New("evidence directory contains no files") + } + metadata, err := readJSONFile[Metadata](resultsDir, "metadata.json") + if err != nil { + return err + } + if err := metadata.Validate(); err != nil { + return fmt.Errorf("validate metadata evidence: %w", err) + } + profile, err := currentProfile(metadata) + if err != nil { + return err + } + for _, required := range profile.RequiredFiles { + if _, exists := files[required]; !exists { + return fmt.Errorf("required evidence file is missing: %s", required) + } + } + if err := rejectStaleDedicatedFiles(files, profile.DedicatedFile); err != nil { + return err + } + if !profile.Executable { + return nil + } + results, err := readJSONFile[Results](resultsDir, "results.json") + if err != nil { + return err + } + if err := results.Validate(); err != nil { + return fmt.Errorf("validate results evidence: %w", err) + } + if results.Profile != metadata.Profile.Name || !slices.Equal(metadata.Scenarios, scenarioResultNames(results.Scenarios)) { + return errors.New("metadata and results do not agree") + } + if err := validateJUnit(resultsDir, results); err != nil { + return err + } + cleanup, err := readJSONFile[cleanupEvidence](resultsDir, "cleanup.json") + if err != nil { + return err + } + if !cleanup.Passed || cleanup.Scenario != results.Scenarios[0].Name { + return errors.New("cleanup evidence is missing or failed") + } + if _, err := time.Parse(time.RFC3339, cleanup.FinishedAt); err != nil { + return errors.New("cleanup finish time is invalid") + } + if profile.DedicatedFile != "" { + definition, err := contract.ScenarioDefinitionByName(results.Scenarios[0].Name) + if err != nil { + return err + } + if err := validateScenarioAssertions(results.Scenarios[0], definition.Assertions(metadata.Fault)); err != nil { + return err + } + return validateDedicatedArtifact(resultsDir, metadata, results.Scenarios[0]) + } + return nil +} + +func validateScenarioAssertions(result ScenarioResult, expected []contract.AssertionDefinition) error { + if len(result.Assertions) != len(expected) { + return errors.New("scenario assertions do not match dedicated evidence contract") + } + for index, assertion := range result.Assertions { + if assertion.Name != expected[index].Name || assertion.Passed != expected[index].Passed { + return errors.New("scenario assertions do not match dedicated evidence contract") + } + } + return nil +} + +func rejectStaleDedicatedFiles(files map[string]os.FileInfo, expected string) error { + for _, name := range []string{"transfer.json", "reconnect.json"} { + if _, exists := files[name]; exists && name != expected { + return fmt.Errorf("stale or wrong dedicated evidence file: %s", name) + } + } + return nil +} + +func validateJUnit(resultsDir string, results Results) error { + data, err := os.ReadFile(filepath.Join(resultsDir, "junit.xml")) + if err != nil { + return fmt.Errorf("read JUnit evidence: %w", err) + } + var suite junitSuite + if err := xml.Unmarshal(data, &suite); err != nil { + return fmt.Errorf("parse JUnit evidence: %w", err) + } + if suite.Name != results.Profile || suite.Tests != len(results.Scenarios) || suite.Failures != countFailedScenarios(results.Scenarios) || len(suite.Cases) != len(results.Scenarios) { + return errors.New("results and JUnit evidence do not agree") + } + for index, scenario := range results.Scenarios { + caseResult := suite.Cases[index] + if caseResult.Name != scenario.Name || (caseResult.Failure != nil) != !scenario.Passed { + return errors.New("results and JUnit scenario states do not agree") + } + } + return nil +} diff --git a/integration/agentcompat/internal/evidence/directory_validation_test.go b/integration/agentcompat/internal/evidence/directory_validation_test.go new file mode 100644 index 00000000..091b6d81 --- /dev/null +++ b/integration/agentcompat/internal/evidence/directory_validation_test.go @@ -0,0 +1,204 @@ +package evidence + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" +) + +func TestEvidence_CurrentProfilePropagatesMalformedScenarioAndFault(t *testing.T) { + tests := []struct { + name string + metadata Metadata + wantContext string + wantCause string + }{ + {name: "scenario", metadata: Metadata{Scenarios: []string{"invalid scenario"}}, wantContext: "construct metadata scenario", wantCause: "invalid scenario name"}, + {name: "fault", metadata: Metadata{Scenarios: []string{contract.ScenarioTransfer100MiB}, Fault: "invalid fault"}, wantContext: "construct metadata fault", wantCause: "invalid fault name"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := currentProfile(test.metadata) + if err == nil || !strings.Contains(err.Error(), test.wantContext) || !strings.Contains(err.Error(), test.wantCause) { + t.Fatalf("currentProfile error=%v, want context %q and cause %q", err, test.wantContext, test.wantCause) + } + }) + } +} + +func TestEvidence_NoCredentialsInDirectory(t *testing.T) { + if len(flag.Args()) == 0 { + t.Skip("requires a results directory argument: go test ./integration/agentcompat/internal/evidence -run TestEvidence_NoCredentialsInDirectory -args RESULTS") + } + if err := ValidateDirectory(flag.Args()[0]); err != nil { + t.Fatalf("validate evidence directory: %v", err) + } +} + +func TestEvidence_NoCredentialsInDirectoryFixture(t *testing.T) { + resultsDir := t.TempDir() + metadata := validMetadata(t, resultsDir, contract.ScenarioRegistrationConfigExec) + writeJSONEvidenceFile(t, resultsDir, "metadata.json", metadata) + results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: contract.ScenarioRegistrationConfigExec, Passed: true, Assertions: []Assertion{{Name: "safe fixture", Passed: true}}}}} + writeJSONEvidenceFile(t, resultsDir, "results.json", results) + junit, err := JUnit(results) + if err != nil { + t.Fatalf("marshal safe JUnit: %v", err) + } + writeEvidenceFile(t, resultsDir, "junit.xml", string(junit)) + writeEvidenceFile(t, resultsDir, "cleanup.json", `{"passed":true,"scenario":"registration-config-exec","finished_at":"2026-01-02T03:04:05Z"}`) + if err := ValidateDirectory(resultsDir); err != nil { + t.Fatalf("validate evidence directory: %v", err) + } +} + +func TestEvidence_ValidateDirectoryRejectsCrossFileMismatches(t *testing.T) { + tests := map[string]func(*testing.T, string){ + "profile mismatch": func(t *testing.T, dir string) { + writeExecutableEvidence(t, dir, "soak", contract.ScenarioRegistrationConfigExec, true, true, true) + }, + "scenario mismatch": func(t *testing.T, dir string) { + writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, true, false, true) + }, + "junit mismatch": func(t *testing.T, dir string) { + writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, false, true, true) + }, + "missing cleanup": func(t *testing.T, dir string) { + writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, true, true, false) + }, + } + for name, setup := range tests { + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + setup(t, dir) + if err := ValidateDirectory(dir); err == nil { + t.Fatal("inconsistent evidence accepted") + } + }) + } +} + +func TestEvidence_ValidateDirectoryRejectsFailedCleanup(t *testing.T) { + dir := t.TempDir() + writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, true, true, true) + writeEvidenceFile(t, dir, "cleanup.json", `{"passed":false,"scenario":"registration-config-exec","finished_at":"2026-01-02T03:04:05Z"}`) + if err := ValidateDirectory(dir); err == nil { + t.Fatal("failed cleanup accepted") + } +} + +func TestEvidence_MCPFilesystemExecutableProfileValidates(t *testing.T) { + dir := t.TempDir() + writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioMCPFilesystem, true, true, true) + if err := ValidateDirectory(dir); err != nil { + t.Fatalf("validate mcp-filesystem evidence: %v", err) + } +} + +func TestEvidence_NoCredentialsInDirectoryRejectsMissingPathAndFiles(t *testing.T) { + if err := ValidateDirectory(filepath.Join(t.TempDir(), "missing")); err == nil { + t.Fatal("missing evidence directory accepted") + } + for name, dir := range map[string]string{"empty": t.TempDir(), "incomplete": t.TempDir()} { + if name == "incomplete" { + writeEvidenceFile(t, dir, "metadata.json", `{ "ok": true }`) + } + if err := ValidateDirectory(dir); err == nil { + t.Fatalf("%s evidence directory accepted", name) + } + } +} + +func TestEvidence_NoCredentialsInDirectoryRejectsCredentialsAndMalformedDocuments(t *testing.T) { + tests := []struct{ name, results, junit string }{ + {"credentials", `{ "token": "secret-value" }`, ``}, + {"malformed json", `{ malformed`, ``}, + {"malformed xml", `{ "ok": true }`, ``}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + writeEvidenceFile(t, dir, "metadata.json", `{ "ok": true }`) + writeEvidenceFile(t, dir, "results.json", test.results) + writeEvidenceFile(t, dir, "junit.xml", test.junit) + if err := ValidateDirectory(dir); err == nil { + t.Fatal("invalid evidence accepted") + } + }) + } +} + +func writeEvidenceFile(t *testing.T, resultsDir, name, content string) { + t.Helper() + if err := os.Chmod(resultsDir, 0o700); err != nil { + t.Fatalf("secure evidence root: %v", err) + } + path := filepath.Join(resultsDir, name) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatalf("create evidence parent: %v", err) + } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write evidence file: %v", err) + } +} + +func writeJSONEvidenceFile(t *testing.T, resultsDir, name string, value any) { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal %s: %v", name, err) + } + writeEvidenceFile(t, resultsDir, name, string(data)) +} + +func validMetadata(t *testing.T, resultsDir, scenarioName string) Metadata { + t.Helper() + profile, err := contract.ProfileByName("pr-full") + if err != nil { + t.Fatalf("profile: %v", err) + } + paths, err := contract.NewPaths("/src/nezha", "/src/agent", resultsDir) + if err != nil { + t.Fatalf("paths: %v", err) + } + scenarioValue, err := contract.NewScenario(scenarioName) + if err != nil { + t.Fatalf("scenario: %v", err) + } + metadata, err := NewMetadata(MetadataInput{Profile: profile, Seed: contract.DefaultSeed, Paths: paths, ResourceBudget: contract.DefaultResourceBudget(), Scenarios: []contract.Scenario{scenarioValue}, StartedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), EvidenceFiles: EvidenceFiles()}) + if err != nil { + t.Fatalf("metadata: %v", err) + } + return metadata +} + +func writeExecutableEvidence(t *testing.T, dir, profileName, scenarioName string, matchingJUnit, matchingScenario, writeCleanup bool) { + t.Helper() + metadata := validMetadata(t, dir, scenarioName) + metadata.Profile.Name = profileName + writeJSONEvidenceFile(t, dir, "metadata.json", metadata) + resultsScenario := scenarioName + if !matchingScenario { + resultsScenario = contract.ScenarioMetadata + } + results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: resultsScenario, Passed: true, Assertions: []Assertion{{Name: "fixture", Passed: true}}}}} + writeJSONEvidenceFile(t, dir, "results.json", results) + junit, err := JUnit(results) + if err != nil { + t.Fatalf("JUnit: %v", err) + } + if !matchingJUnit { + junit = []byte(``) + } + writeEvidenceFile(t, dir, "junit.xml", string(junit)) + if writeCleanup { + writeEvidenceFile(t, dir, "cleanup.json", fmt.Sprintf(`{"passed":true,"scenario":%q,"finished_at":"2026-01-02T03:04:05Z"}`, scenarioName)) + } +} diff --git a/integration/agentcompat/internal/evidence/evidence_test.go b/integration/agentcompat/internal/evidence/evidence_test.go new file mode 100644 index 00000000..0cf4d158 --- /dev/null +++ b/integration/agentcompat/internal/evidence/evidence_test.go @@ -0,0 +1,223 @@ +package evidence + +import ( + "encoding/json" + "encoding/xml" + "strings" + "testing" + "time" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" +) + +func TestEvidence_Redaction(t *testing.T) { + input := `"Authorization":"Basic authorization-secret" JWT=eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.signature PAT=pat-secret agent_secret_key=agent-secret client_secret=config-secret https://local/mcp/upload/path-secret?token=query-secret` + redacted := Redact(input) + for _, secret := range []string{"authorization-secret", "eyJhbGciOiJIUzI1NiJ9", "pat-secret", "agent-secret", "config-secret", "path-secret", "query-secret"} { + if strings.Contains(redacted, secret) { + t.Fatalf("secret survived redaction: %q", secret) + } + } + if !strings.Contains(redacted, "[REDACTED]") { + t.Fatalf("redaction marker missing: %q", redacted) + } + junit, err := JUnit(Results{Profile: "pr-full", Passed: false, Scenarios: []ScenarioResult{{Name: "secret", Passed: false, Assertions: []Assertion{{Name: "secret failure", Passed: false}}, Error: input}}}) + if err != nil { + t.Fatalf("marshal redacted junit: %v", err) + } + if strings.Contains(string(junit), "authorization-secret") || strings.Contains(string(junit), "path-secret") { + t.Fatalf("secret survived JUnit redaction: %s", junit) + } +} + +func TestEvidence_Golden(t *testing.T) { + profile, err := contract.ProfileByName("pr-full") + if err != nil { + t.Fatalf("construct profile: %v", err) + } + paths, err := contract.NewPaths("/src/nezha", "/src/agent", "/tmp/results") + if err != nil { + t.Fatalf("construct paths: %v", err) + } + metadata, err := NewMetadata(MetadataInput{ + Profile: profile, + Seed: contract.DefaultSeed, + Paths: paths, + ResourceBudget: contract.DefaultResourceBudget(), + StartedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + EvidenceFiles: EvidenceFiles(), + }) + if err != nil { + t.Fatalf("construct metadata: %v", err) + } + data, err := json.Marshal(metadata) + if err != nil { + t.Fatalf("marshal metadata: %v", err) + } + const wantJSON = `{"agent_source":"/src/agent","evidence_files":["metadata.json","results.json","junit.xml","dashboard.log","agents/*.log","transfer.json","reconnect.json","stress.json","cleanup.json","step-summary.md"],"load_classification":"regression loads, not capacity claims","nezha_source":"/src/nezha","profile":{"name":"pr-full","job_timeout_seconds":4500,"suite_deadline_seconds":3300,"default_seed":"0x4e5a4841","agent_count":8,"stress_rounds":4,"concurrent_operations":64,"concurrent_sessions_per_kind":4,"transfer_pairs":1,"transfer_bytes":104857600,"dashboard_restart_cycles":1,"iterations":1,"stream_boundary_allowed":40,"stream_boundary_rejected":41},"resource_budget":{"warmup_runs_per_path":1,"baseline_sample_count":5,"end_sample_count":5,"sample_interval_milliseconds":250,"child_process_count_drift":0,"listener_count_drift":0,"non_stdio_fd_count_drift":0,"dashboard_rss_delta_bytes":67108864,"agent_rss_delta_bytes":33554432,"transfer_heap_bytes":16777216},"results_dir":"/tmp/results","scenarios":[],"seed":"0x4e5a4841","started_at":"2026-01-02T03:04:05Z"}` + if string(data) != wantJSON { + t.Fatalf("metadata golden mismatch\nwant: %s\ngot: %s", wantJSON, data) + } + + results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: "metadata", Passed: true, Assertions: []Assertion{{Name: "metadata written", Passed: true}}}}} + resultsJSON, err := json.Marshal(results) + if err != nil { + t.Fatalf("marshal results: %v", err) + } + const wantResultsJSON = `{"profile":"pr-full","passed":true,"scenarios":[{"name":"metadata","passed":true,"assertions":[{"name":"metadata written","passed":true}]}]}` + if string(resultsJSON) != wantResultsJSON { + t.Fatalf("results golden mismatch\nwant: %s\ngot: %s", wantResultsJSON, resultsJSON) + } + junitXML, err := JUnit(results) + if err != nil { + t.Fatalf("marshal junit: %v", err) + } + const wantXML = `` + if string(junitXML) != wantXML { + t.Fatalf("junit golden mismatch\nwant: %s\ngot: %s", wantXML, junitXML) + } + var parsedJUnit junitSuite + if err := xml.Unmarshal(junitXML, &parsedJUnit); err != nil { + t.Fatalf("parse junit golden: %v", err) + } + if parsedJUnit.Tests != 1 || parsedJUnit.Failures != 0 || len(parsedJUnit.Cases) != 1 { + t.Fatalf("unexpected parsed junit: %#v", parsedJUnit) + } +} + +func TestEvidence_SchemaValidation(t *testing.T) { + invalid := Metadata{} + if err := invalid.Validate(); err == nil { + t.Fatal("invalid metadata accepted") + } + invalidBudget := ResourceBudgetMetadata{WarmupRunsPerPath: 1, BaselineSampleCount: 5, EndSampleCount: 5, SampleIntervalMilliseconds: 250, DashboardRSSDeltaBytes: 1, AgentRSSDeltaBytes: 1} + if err := invalidBudget.Validate(); err == nil { + t.Fatal("missing transfer heap threshold accepted") + } + results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: "failed", Passed: false, Assertions: []Assertion{{Name: "failed assertion", Passed: false}}, Error: "failure"}}} + if err := results.Validate(); err == nil { + t.Fatal("inconsistent results accepted") + } + for _, result := range []Results{ + {Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: "../escape", Passed: true}}}, + {Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: "duplicate", Passed: true}, {Name: "duplicate", Passed: true}}}, + } { + if err := result.Validate(); err == nil { + t.Fatalf("invalid scenario results accepted: %#v", result) + } + } +} + +func TestEvidence_ResultsRejectsSuccessfulScenarioWithoutAssertions(t *testing.T) { + results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: "registration-config-exec", Passed: true}}} + + if err := results.Validate(); err == nil { + t.Fatal("successful executable scenario without assertions accepted") + } +} + +func TestEvidence_ResultsJSONRedactsSecrets(t *testing.T) { + results := Results{Profile: "pr-full", Passed: false, Scenarios: []ScenarioResult{ + {Name: "authorization", Passed: false, Assertions: []Assertion{{Name: "authorization failure", Passed: false}}, Error: "Authorization: Bearer jwt-secret"}, + {Name: "credential-field", Passed: false, Assertions: []Assertion{{Name: "credential failure", Passed: false}}, Error: "agent_secret=agent-secret"}, + {Name: "transfer-token", Passed: false, Assertions: []Assertion{{Name: "transfer failure", Passed: false}}, Error: "/mcp/download/path-token?token=query-token"}, + {Name: "access-token", Passed: false, Assertions: []Assertion{{Name: "access failure", Passed: false}}, Error: "https://local/file?access_token=access-secret"}, + {Name: "api-key", Passed: false, Assertions: []Assertion{{Name: "api failure", Passed: false}}, Error: "https://local/file?api_key=api-secret"}, + {Name: "signature", Passed: false, Assertions: []Assertion{{Name: "signature failure", Passed: false}}, Error: "https://local/file?sig=sig-secret&signature=signature-secret&X-Amz-Signature=aws-secret"}, + }} + data, err := MarshalResults(results) + if err != nil { + t.Fatalf("marshal results: %v", err) + } + for _, secret := range []string{"jwt-secret", "agent-secret", "path-token", "query-token", "access-secret", "api-secret", "sig-secret", "signature-secret", "aws-secret"} { + if strings.Contains(string(data), secret) { + t.Fatalf("secret survived results JSON: %q in %s", secret, data) + } + } + directData, err := json.Marshal(results) + if err != nil { + t.Fatalf("direct marshal results: %v", err) + } + if strings.Contains(string(directData), "jwt-secret") || strings.Contains(string(directData), "path-token") { + t.Fatalf("direct JSON marshal bypassed redaction: %s", directData) + } +} + +func TestEvidence_RedactsXMLTextAttributesAndMalformedFragments(t *testing.T) { + input := `agent-secrettransfer-secretpassword-secretcredential-secret` + redacted := Redact(input) + for _, secret := range []string{"authorization-secret", "agent-secret", "transfer-secret", "password-secret", "credential-secret"} { + if strings.Contains(redacted, secret) { + t.Fatalf("XML secret survived redaction: %q in %s", secret, redacted) + } + } + var parsed struct { + XMLName xml.Name `xml:"record"` + Secret string `xml:"agent_secret_key"` + } + if err := xml.Unmarshal([]byte(redacted), &parsed); err != nil { + t.Fatalf("redacted XML is not parseable: %v; output=%s", err, redacted) + } + if parsed.Secret != "[REDACTED]" { + t.Fatalf("unexpected redacted element text: %q", parsed.Secret) + } + results := Results{Profile: "pr-full", Passed: false, Scenarios: []ScenarioResult{{Name: "xml", Passed: false, Assertions: []Assertion{{Name: "xml failure", Passed: false}}, Error: input}}} + junit, err := JUnit(results) + if err != nil { + t.Fatalf("marshal XML evidence: %v", err) + } + var parsedSuite junitSuite + if err := xml.Unmarshal(junit, &parsedSuite); err != nil { + t.Fatalf("redacted JUnit is not parseable: %v; output=%s", err, junit) + } + if strings.Contains(string(junit), "agent-secret") || strings.Contains(string(junit), "credential-secret") { + t.Fatalf("XML evidence secret survived: %s", junit) + } + + malformed := `config-secret` + malformedRedacted := Redact(malformed) + if strings.Contains(malformedRedacted, "config-secret") { + t.Fatalf("malformed XML secret survived redaction: %s", malformedRedacted) + } +} + +func TestEvidence_RedactsXMLSensitiveAttributesWithWhitespaceAndMalformedTags(t *testing.T) { + input := `password text` + redacted := Redact(input) + for _, secret := range []string{"agent secret with spaces", "config-secret", "password secret with spaces", "password text"} { + if strings.Contains(redacted, secret) { + t.Fatalf("XML attribute or text secret survived redaction: %q in %s", secret, redacted) + } + } + if !strings.Contains(redacted, ``) { + t.Fatalf("sensitive XML attribute was not structurally redacted: %s", redacted) + } +} + +func TestEvidence_RedactsCredentialAttributesWithSpacesAndTruncatedXML(t *testing.T) { + input := `ok` + redacted := Redact(input) + if strings.Contains(redacted, "self-closing secret") { + t.Fatalf("self-closing XML secret survived redaction: %s", redacted) + } + var parsed struct { + XMLName xml.Name `xml:"record"` + Result string `xml:"result"` + } + if err := xml.Unmarshal([]byte(redacted), &parsed); err != nil { + t.Fatalf("redacted self-closing XML is not parseable: %v; output=%s", err, redacted) + } + if parsed.Result != "ok" { + t.Fatalf("redacted self-closing XML lost sibling content: %#v", parsed) + } +} diff --git a/integration/agentcompat/internal/evidence/junit.go b/integration/agentcompat/internal/evidence/junit.go new file mode 100644 index 00000000..1424aed0 --- /dev/null +++ b/integration/agentcompat/internal/evidence/junit.go @@ -0,0 +1,46 @@ +package evidence + +import "encoding/xml" + +type junitSuite struct { + XMLName xml.Name `xml:"testsuite"` + Name string `xml:"name,attr"` + Tests int `xml:"tests,attr"` + Failures int `xml:"failures,attr"` + Cases []junitCase `xml:"testcase"` +} + +type junitCase struct { + Name string `xml:"name,attr"` + Failure *junitFailure `xml:"failure,omitempty"` +} + +type junitFailure struct { + Message string `xml:"message,attr"` +} + +func JUnit(results Results) ([]byte, error) { + if err := results.Validate(); err != nil { + return nil, err + } + suite := junitSuite{Name: results.Profile, Tests: len(results.Scenarios)} + for _, scenario := range results.Scenarios { + caseResult := junitCase{Name: scenario.Name} + if !scenario.Passed { + suite.Failures++ + caseResult.Failure = &junitFailure{Message: Redact(scenario.Error)} + } + suite.Cases = append(suite.Cases, caseResult) + } + return xml.Marshal(suite) +} + +func countFailedScenarios(scenarios []ScenarioResult) int { + failed := 0 + for _, scenario := range scenarios { + if !scenario.Passed { + failed++ + } + } + return failed +} diff --git a/integration/agentcompat/internal/evidence/metadata.go b/integration/agentcompat/internal/evidence/metadata.go new file mode 100644 index 00000000..4270de00 --- /dev/null +++ b/integration/agentcompat/internal/evidence/metadata.go @@ -0,0 +1,134 @@ +package evidence + +import ( + "encoding/json" + "fmt" + "path/filepath" + "slices" + "time" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" +) + +type MetadataInput struct { + Profile contract.Profile + Seed contract.Seed + Paths contract.Paths + ResourceBudget contract.ResourceBudget + Scenarios []contract.Scenario + Fault contract.Fault + StartedAt time.Time + EvidenceFiles []string +} + +type Metadata struct { + AgentSource string `json:"agent_source"` + EvidenceFiles []string `json:"evidence_files"` + LoadClassification string `json:"load_classification"` + NezhaSource string `json:"nezha_source"` + Profile ProfileMetadata `json:"profile"` + ResourceBudget ResourceBudgetMetadata `json:"resource_budget"` + ResultsDir string `json:"results_dir"` + Scenarios []string `json:"scenarios"` + Seed string `json:"seed"` + Fault string `json:"fault,omitempty"` + StartedAt string `json:"started_at"` +} + +func NewMetadata(input MetadataInput) (Metadata, error) { + metadata := Metadata{ + AgentSource: Redact(input.Paths.AgentSource().String()), + EvidenceFiles: append([]string(nil), input.EvidenceFiles...), + LoadClassification: "regression loads, not capacity claims", + NezhaSource: Redact(input.Paths.NezhaSource().String()), + Profile: profileMetadata(input.Profile), + ResourceBudget: resourceBudgetMetadata(input.ResourceBudget), + ResultsDir: Redact(input.Paths.ResultsDir().String()), + Scenarios: scenarioNames(input.Scenarios), + Seed: fmt.Sprintf("0x%x", uint64(input.Seed)), + Fault: input.Fault.String(), + StartedAt: input.StartedAt.UTC().Format(time.RFC3339), + } + if err := metadata.Validate(); err != nil { + return Metadata{}, err + } + return metadata, nil +} + +func EvidenceFiles() []string { + files := []string{"metadata.json", "results.json", "junit.xml", "dashboard.log", "agents/*.log"} + for _, definition := range contract.ScenarioDefinitions() { + if name := definition.DedicatedArtifactName(); name != "" { + files = append(files, name) + } + } + return append(files, "stress.json", "cleanup.json", "step-summary.md") +} + +func FixedEvidenceFiles() []string { + files := make([]string, 0, len(EvidenceFiles())) + for _, name := range EvidenceFiles() { + if !slices.Contains([]rune(name), '*') { + files = append(files, name) + } + } + return files +} + +func scenarioNames(scenarios []contract.Scenario) []string { + names := make([]string, 0, len(scenarios)) + for _, scenario := range scenarios { + names = append(names, scenario.String()) + } + return names +} + +func (metadata Metadata) Validate() error { + if metadata.AgentSource == "" || len(metadata.EvidenceFiles) == 0 || metadata.LoadClassification == "" || metadata.NezhaSource == "" || metadata.ResultsDir == "" || metadata.Seed == "" || metadata.Seed == "0x0" || metadata.StartedAt == "" { + return fmt.Errorf("metadata fields are incomplete") + } + if err := metadata.Profile.Validate(); err != nil { + return err + } + if _, err := contract.ProfileByName(metadata.Profile.Name); err != nil { + return fmt.Errorf("metadata profile is invalid: %w", err) + } + if err := metadata.ResourceBudget.Validate(); err != nil { + return err + } + if !filepath.IsAbs(metadata.AgentSource) || !filepath.IsAbs(metadata.NezhaSource) || !filepath.IsAbs(metadata.ResultsDir) { + return fmt.Errorf("metadata paths must be absolute") + } + if _, err := time.Parse(time.RFC3339, metadata.StartedAt); err != nil { + return fmt.Errorf("metadata start time is invalid: %w", err) + } + if !slices.Equal(metadata.EvidenceFiles, EvidenceFiles()) { + return fmt.Errorf("metadata evidence files are invalid") + } + seen := make(map[string]struct{}, len(metadata.Scenarios)) + for _, scenarioName := range metadata.Scenarios { + _, err := contract.NewScenario(scenarioName) + if err != nil { + return fmt.Errorf("metadata scenario is invalid: %w", err) + } + if _, exists := seen[scenarioName]; exists { + return fmt.Errorf("metadata scenario is duplicated") + } + seen[scenarioName] = struct{}{} + } + if metadata.Fault != "" { + if _, err := contract.NewFault(metadata.Fault); err != nil { + return fmt.Errorf("metadata fault is invalid: %w", err) + } + } + return nil +} + +func (metadata Metadata) MarshalJSON() ([]byte, error) { + type metadataWire Metadata + redacted := metadataWire(metadata) + redacted.AgentSource = Redact(redacted.AgentSource) + redacted.NezhaSource = Redact(redacted.NezhaSource) + redacted.ResultsDir = Redact(redacted.ResultsDir) + return json.Marshal(redacted) +} diff --git a/integration/agentcompat/internal/evidence/reconnect_artifact_validation.go b/integration/agentcompat/internal/evidence/reconnect_artifact_validation.go new file mode 100644 index 00000000..54eb1119 --- /dev/null +++ b/integration/agentcompat/internal/evidence/reconnect_artifact_validation.go @@ -0,0 +1,124 @@ +package evidence + +import ( + "errors" + "path/filepath" + "slices" + "strings" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" +) + +func validateDedicatedArtifact(resultsDir string, metadata Metadata, result ScenarioResult) error { + definition, err := contract.ScenarioDefinitionByName(result.Name) + if err != nil { + return err + } + switch definition.DedicatedArtifact { + case contract.DedicatedArtifactTransfer: + artifact, err := readJSONFile[transferArtifact](resultsDir, "transfer.json") + if err != nil { + return err + } + return validateTransferArtifact(metadata, result, artifact) + case contract.DedicatedArtifactReconnect: + artifact, err := readJSONFile[reconnectArtifact](resultsDir, "reconnect.json") + if err != nil { + return err + } + return validateReconnectArtifact(metadata, result, artifact) + case contract.DedicatedArtifactNone: + return nil + default: + return errors.New("unsupported dedicated artifact kind") + } +} + +func validateReconnectArtifact(metadata Metadata, result ScenarioResult, artifact reconnectArtifact) error { + if err := validateArtifactHeader(metadata, result, artifact.Scenario, artifact.Fault, artifact.Passed, artifact.CleanupOK, artifact.Error); err != nil { + return err + } + switch artifact.Fault { + case "": + if !artifact.Passed { + return errors.New("reconnect success artifact reports failure") + } + if err := validateReconnectSuccess(artifact.Evidence); err != nil { + return err + } + if err := validateReconnectSuccessReceipts(artifact.Evidence); err != nil { + return err + } + return validateReconnectFinalEvidence(artifact.Evidence, 2) + case contract.FaultDashboardExit: + if artifact.Passed || !strings.Contains(artifact.Error, "injected Dashboard exit") { + return errors.New("dashboard-exit artifact does not identify the injected failure") + } + evidence := artifact.Evidence + if reconnectPostFaultEvidencePresent(evidence) { + return errors.New("dashboard-exit artifact presents stale reconnect success") + } + if evidence.Lifecycle.DisconnectAt.IsZero() || !evidence.Lifecycle.ReconnectAt.IsZero() || !evidence.Lifecycle.OutsideRootSentinelUnchanged { + return errors.New("dashboard-exit lifecycle evidence is invalid") + } + return validateReconnectFinalEvidence(evidence, 1) + default: + return errors.New("reconnect artifact has unsupported fault") + } +} + +func reconnectPostFaultEvidencePresent(evidence reconnectEvidence) bool { + return evidence.Runtime.DashboardAfter != (runtimeIdentity{}) || evidence.Runtime.AgentAfter != (runtimeIdentity{}) || + evidence.Runtime.StateGenerationBeforeAgentRestart != 0 || evidence.Runtime.StateGenerationAfterAgentRestart != 0 || + evidence.Identity.DashboardConfigUnchanged || evidence.Identity.AgentConfigUnchanged || evidence.Identity.DashboardFixtureUnchanged || evidence.Identity.ClientsRecreated || evidence.Identity.BootstrapRecreated || + !evidence.Lifecycle.ReconnectAt.IsZero() || evidence.Lifecycle.ReconnectInterval != 0 || len(evidence.Lifecycle.DashboardReceipts) != 0 || len(evidence.Lifecycle.AgentReceipts) != 0 || + evidence.Lifecycle.StaleGenerationReceipts != 0 || evidence.Lifecycle.DuplicateTaskIDs != 0 || evidence.Lifecycle.LostResultIDs != 0 || + evidence.Observation.ServerID != 0 || evidence.Observation.UUID != "" || evidence.Observation.OldGeneration != 0 || evidence.Observation.NewGeneration != 0 || + !evidence.Observation.DisconnectAt.IsZero() || !evidence.Observation.ReconnectAt.IsZero() || len(evidence.Observation.TaskIDs) != 0 || len(evidence.Observation.ResultIDs) != 0 || evidence.Observation.PostReconnect || evidence.Observation.AgentRestarted +} + +func validateReconnectSuccess(evidence reconnectEvidence) error { + observation := evidence.Observation + if observation.ServerID == 0 || observation.UUID == "" || observation.OldGeneration == 0 || observation.NewGeneration <= observation.OldGeneration || observation.DisconnectAt.IsZero() || !observation.ReconnectAt.After(observation.DisconnectAt) || len(observation.TaskIDs) != 5 || !slices.Equal(observation.TaskIDs, observation.ResultIDs) || !observation.PostReconnect || !observation.AgentRestarted { + return errors.New("reconnect observation evidence is invalid") + } + if evidence.Runtime.DashboardAfter.Generation <= evidence.Runtime.DashboardBefore.Generation || evidence.Runtime.DashboardAfter.PID == evidence.Runtime.DashboardBefore.PID || evidence.Runtime.AgentAfter.Generation <= evidence.Runtime.AgentBefore.Generation || evidence.Runtime.AgentAfter.PID == evidence.Runtime.AgentBefore.PID || evidence.Runtime.StateGenerationAfterAgentRestart <= evidence.Runtime.StateGenerationBeforeAgentRestart { + return errors.New("reconnect runtime generations are invalid") + } + identity := evidence.Identity + if identity.ServerID != observation.ServerID || identity.UUID != observation.UUID || !identity.DashboardConfigUnchanged || !identity.AgentConfigUnchanged || !identity.DashboardFixtureUnchanged || !identity.ClientsRecreated || !identity.BootstrapRecreated { + return errors.New("reconnect identity evidence is invalid") + } + if evidence.Lifecycle.StaleGenerationReceipts != 0 || evidence.Lifecycle.DuplicateTaskIDs != 0 || evidence.Lifecycle.LostResultIDs != 0 || !evidence.Lifecycle.OutsideRootSentinelUnchanged { + return errors.New("reconnect lifecycle accounting is invalid") + } + return nil +} + +func validateReconnectSuccessReceipts(evidence reconnectEvidence) error { + if len(evidence.Lifecycle.DashboardReceipts) != 3 || len(evidence.Lifecycle.AgentReceipts) != 2 || len(evidence.Observation.TaskIDs) != 5 { + return errors.New("reconnect receipt accounting is not exact") + } + if evidence.Lifecycle.DisconnectAt.IsZero() || evidence.Lifecycle.ReconnectAt.IsZero() || evidence.Lifecycle.ReconnectAt.Sub(evidence.Lifecycle.DisconnectAt) != evidence.Lifecycle.ReconnectInterval { + return errors.New("reconnect timestamp interval is inconsistent") + } + if !evidence.Observation.DisconnectAt.Equal(evidence.Lifecycle.DisconnectAt) || !evidence.Observation.ReconnectAt.Equal(evidence.Lifecycle.ReconnectAt) { + return errors.New("reconnect lifecycle and observation timestamps differ") + } + return nil +} + +func validateReconnectFinalEvidence(evidence reconnectEvidence, expectedProcessCount int) error { + if !evidence.AgentCleanup.Passed || evidence.AgentCleanup.Forced || !evidence.DashboardCleanup.Passed || evidence.DashboardCleanup.Forced { + return errors.New("reconnect cleanup receipts failed") + } + if len(evidence.AgentCleanup.Processes) != expectedProcessCount || len(evidence.DashboardCleanup.Processes) != expectedProcessCount { + return errors.New("reconnect cleanup receipt process count is invalid") + } + for _, path := range []string{evidence.Fixture.Dashboard.WorkspaceRoot, evidence.Fixture.AgentRoot, evidence.Fixture.Dashboard.ConfigPath, evidence.Fixture.AgentConfigPath, evidence.Fixture.Dashboard.BinaryPath, evidence.Fixture.AgentBinaryPath} { + if path == "" || !filepath.IsAbs(path) { + return errors.New("reconnect fixture paths are invalid") + } + } + return nil +} diff --git a/integration/agentcompat/internal/evidence/redaction.go b/integration/agentcompat/internal/evidence/redaction.go new file mode 100644 index 00000000..918a0747 --- /dev/null +++ b/integration/agentcompat/internal/evidence/redaction.go @@ -0,0 +1,41 @@ +package evidence + +import ( + "regexp" + "strings" +) + +type redactionRule struct { + pattern *regexp.Regexp + replace string +} + +var redactionRules = []redactionRule{ + {pattern: regexp.MustCompile(`(?i)(["']?authorization["']?\s*[:=]\s*["']?)[^"'\r\n,}]+(["']?)`), replace: `${1}[REDACTED]${2}`}, + {pattern: regexp.MustCompile(`(?i)(\b["']?(?:token|access[_-]?token|api[_-]?(?:key|token)|jwt[_-]?secret(?:[_-]?key)?|jwt[_-]?token|pat|agent[_-]?secret(?:[_-]?key)?|client[_-]?secret|handshake[_-]?secret|revert[_-]?handshake[_-]?secret|password|credential|transfer[_-]?token)["']?\s*[:=]\s*["']?)[^"'\s,;&}]+(["']?)`), replace: `${1}[REDACTED]${2}`}, + {pattern: regexp.MustCompile(`(?i)([?&](?:token|access_token|api_key|jwt|pat|secret|authorization|sig|signature|x-amz-signature)=)[^&#\s]+`), replace: `${1}[REDACTED]`}, + {pattern: regexp.MustCompile(`(?i)(/mcp/(?:download|upload)/)[A-Za-z0-9._-]+`), replace: `${1}[REDACTED]`}, + {pattern: regexp.MustCompile(`\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b`), replace: `[REDACTED]`}, + {pattern: regexp.MustCompile(`\b(?:ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})\b`), replace: `[REDACTED]`}, + {pattern: regexp.MustCompile(`(?is)(<\s*(?:agent[_-]?secret(?:[_-]?key)?|authorization|transfer[_-]?token|password|credential|config[_-]?secret)\b)[^>]*?(/?)>`), replace: `${1} value="[REDACTED]"${2}>`}, + {pattern: regexp.MustCompile(`(?is)(<\s*(?:agent[_-]?secret(?:[_-]?key)?|authorization|transfer[_-]?token|password|credential|config[_-]?secret)\b[^>]*>)[^<]*()`), replace: `${1}[REDACTED]${2}`}, + {pattern: regexp.MustCompile(`(?is)(<\s*(?:agent[_-]?secret(?:[_-]?key)?|authorization|transfer[_-]?token|password|credential|config[_-]?secret)\b[^>]*>)[^<]*$`), replace: `${1}[REDACTED]`}, + {pattern: regexp.MustCompile(`(?is)(<\s*(?:agent[_-]?secret(?:[_-]?key)?|authorization|transfer[_-]?token|password|credential|config[_-]?secret)\b)[^>]*$`), replace: `${1} value="[REDACTED]"`}, +} + +var sensitiveXMLAttribute = regexp.MustCompile(`(?i)(\b(?:agent[_-]?secret(?:[_-]?key)?|authorization|transfer[_-]?token|password|credential|config[_-]?secret)\b\s*=\s*)(?:"[^"]*"|'[^']*'|"[^\r\n>]*|'[^'\r\n>]*'|[^\s/>][^/>]*)`) + +func Redact(input string) string { + redacted := sensitiveXMLAttribute.ReplaceAllStringFunc(input, func(attribute string) string { + separator := strings.IndexByte(attribute, '=') + if separator < 0 { + return attribute + } + prefix := attribute[:separator+1] + return prefix + "\"[REDACTED]\"" + }) + for _, rule := range redactionRules { + redacted = rule.pattern.ReplaceAllString(redacted, rule.replace) + } + return redacted +} diff --git a/integration/agentcompat/internal/evidence/registry_coverage_test.go b/integration/agentcompat/internal/evidence/registry_coverage_test.go new file mode 100644 index 00000000..35c31064 --- /dev/null +++ b/integration/agentcompat/internal/evidence/registry_coverage_test.go @@ -0,0 +1,32 @@ +package evidence + +import ( + "slices" + "testing" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" +) + +func TestEvidence_RegistryDefinitionsDriveManifestAndProfiles(t *testing.T) { + manifest := EvidenceFiles() + for _, definition := range contract.ScenarioDefinitions() { + dedicatedName := definition.DedicatedArtifactName() + if dedicatedName != "" && !slices.Contains(manifest, dedicatedName) { + t.Fatalf("dedicated artifact %q absent from manifest", dedicatedName) + } + metadata := validMetadata(t, t.TempDir(), definition.Name) + profile, err := currentProfile(metadata) + if err != nil { + t.Fatalf("profile for %q: %v", definition.Name, err) + } + if profile.DedicatedFile != dedicatedName { + t.Fatalf("scenario %q dedicated file=%q want=%q", definition.Name, profile.DedicatedFile, dedicatedName) + } + if definition.Execution == contract.ScenarioExecutionMetadata && profile.Executable { + t.Fatal("metadata profile unexpectedly executable") + } + if definition.Execution != contract.ScenarioExecutionMetadata && !profile.Executable { + t.Fatalf("scenario %q profile is not executable", definition.Name) + } + } +} diff --git a/integration/agentcompat/internal/evidence/results.go b/integration/agentcompat/internal/evidence/results.go new file mode 100644 index 00000000..c5d52733 --- /dev/null +++ b/integration/agentcompat/internal/evidence/results.go @@ -0,0 +1,105 @@ +package evidence + +import ( + "encoding/json" + "fmt" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/contract" +) + +type ScenarioResult struct { + Name string `json:"name"` + Passed bool `json:"passed"` + Assertions []Assertion `json:"assertions,omitempty"` + Error string `json:"error,omitempty"` +} + +type Assertion struct { + Name string `json:"name"` + Passed bool `json:"passed"` + Details string `json:"details,omitempty"` +} + +type Results struct { + Profile string `json:"profile"` + Passed bool `json:"passed"` + Scenarios []ScenarioResult `json:"scenarios"` +} + +func (results Results) Validate() error { + if results.Profile == "" || len(results.Scenarios) == 0 { + return fmt.Errorf("results fields are incomplete") + } + if _, err := contract.ProfileByName(results.Profile); err != nil { + return fmt.Errorf("results profile is invalid: %w", err) + } + allPassed := true + seenScenarios := make(map[string]struct{}, len(results.Scenarios)) + for _, scenario := range results.Scenarios { + if _, err := contract.NewScenario(scenario.Name); err != nil { + return fmt.Errorf("scenario name is invalid: %w", err) + } + if _, exists := seenScenarios[scenario.Name]; exists { + return fmt.Errorf("scenario name is duplicated") + } + seenScenarios[scenario.Name] = struct{}{} + if len(scenario.Assertions) == 0 { + return fmt.Errorf("scenario must contain assertions") + } + assertionsPassed := true + seenAssertions := make(map[string]struct{}, len(scenario.Assertions)) + for _, assertion := range scenario.Assertions { + if assertion.Name == "" { + return fmt.Errorf("assertion name is required") + } + if _, exists := seenAssertions[assertion.Name]; exists { + return fmt.Errorf("assertion name is duplicated") + } + seenAssertions[assertion.Name] = struct{}{} + assertionsPassed = assertionsPassed && assertion.Passed + } + if scenario.Passed != assertionsPassed { + return fmt.Errorf("scenario pass state is inconsistent with assertions") + } + if !scenario.Passed { + if scenario.Error == "" { + return fmt.Errorf("failed scenario error is required") + } + allPassed = false + } else if scenario.Error != "" { + return fmt.Errorf("passed scenario cannot contain an error") + } + } + if results.Passed != allPassed { + return fmt.Errorf("results pass state is inconsistent") + } + return nil +} + +func MarshalResults(results Results) ([]byte, error) { + if err := results.Validate(); err != nil { + return nil, err + } + return json.Marshal(results) +} + +func (results Results) MarshalJSON() ([]byte, error) { + type resultsWire Results + redacted := resultsWire{Profile: results.Profile, Passed: results.Passed, Scenarios: make([]ScenarioResult, 0, len(results.Scenarios))} + for _, scenario := range results.Scenarios { + assertions := make([]Assertion, 0, len(scenario.Assertions)) + for _, assertion := range scenario.Assertions { + assertions = append(assertions, Assertion{Name: assertion.Name, Passed: assertion.Passed, Details: Redact(assertion.Details)}) + } + redacted.Scenarios = append(redacted.Scenarios, ScenarioResult{Name: scenario.Name, Passed: scenario.Passed, Assertions: assertions, Error: Redact(scenario.Error)}) + } + return json.Marshal(redacted) +} + +func scenarioResultNames(scenarios []ScenarioResult) []string { + names := make([]string, 0, len(scenarios)) + for _, scenario := range scenarios { + names = append(names, scenario.Name) + } + return names +}