mirror of
https://github.com/Buriburizaem0n/nezha_domains.git
synced 2026-09-19 09:40:12 +00:00
test(agentcompat): add evidence validation
Co-authored-by: naiba/CloudCode <hi+cloudcode@nai.ba>
This commit is contained in:
naiba
naiba/CloudCode
@@ -0,0 +1,59 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
)
|
||||
|
||||
type ProfileMetadata struct {
|
||||
Name string `json:"name"`
|
||||
JobTimeoutSeconds int64 `json:"job_timeout_seconds"`
|
||||
SuiteDeadlineSeconds int64 `json:"suite_deadline_seconds"`
|
||||
DefaultSeed string `json:"default_seed"`
|
||||
AgentCount int `json:"agent_count"`
|
||||
StressRounds int `json:"stress_rounds"`
|
||||
ConcurrentOperations int `json:"concurrent_operations"`
|
||||
ConcurrentSessionsPerKind int `json:"concurrent_sessions_per_kind"`
|
||||
TransferPairs int `json:"transfer_pairs"`
|
||||
TransferBytes uint64 `json:"transfer_bytes"`
|
||||
DashboardRestartCycles int `json:"dashboard_restart_cycles"`
|
||||
Iterations int `json:"iterations"`
|
||||
StreamBoundaryAllowed int `json:"stream_boundary_allowed"`
|
||||
StreamBoundaryRejected int `json:"stream_boundary_rejected"`
|
||||
}
|
||||
|
||||
type ResourceBudgetMetadata struct {
|
||||
WarmupRunsPerPath int `json:"warmup_runs_per_path"`
|
||||
BaselineSampleCount int `json:"baseline_sample_count"`
|
||||
EndSampleCount int `json:"end_sample_count"`
|
||||
SampleIntervalMilliseconds int64 `json:"sample_interval_milliseconds"`
|
||||
ChildProcessCountDrift int `json:"child_process_count_drift"`
|
||||
ListenerCountDrift int `json:"listener_count_drift"`
|
||||
NonStdioFDCountDrift int `json:"non_stdio_fd_count_drift"`
|
||||
DashboardRSSDeltaBytes uint64 `json:"dashboard_rss_delta_bytes"`
|
||||
AgentRSSDeltaBytes uint64 `json:"agent_rss_delta_bytes"`
|
||||
TransferHeapBytes uint64 `json:"transfer_heap_bytes"`
|
||||
}
|
||||
|
||||
func profileMetadata(profile contract.Profile) ProfileMetadata {
|
||||
return ProfileMetadata{Name: string(profile.Name()), JobTimeoutSeconds: int64(profile.JobTimeout().Seconds()), SuiteDeadlineSeconds: int64(profile.SuiteDeadline().Seconds()), DefaultSeed: fmt.Sprintf("0x%x", uint64(profile.Seed())), AgentCount: profile.AgentCount(), StressRounds: profile.StressRounds(), ConcurrentOperations: profile.ConcurrentOperations(), ConcurrentSessionsPerKind: profile.ConcurrentSessions(), TransferPairs: profile.TransferPairs(), TransferBytes: profile.TransferBytes(), DashboardRestartCycles: profile.DashboardRestartCycles(), Iterations: profile.Iterations(), StreamBoundaryAllowed: profile.StreamBoundaryAllowed(), StreamBoundaryRejected: profile.StreamBoundaryRejected()}
|
||||
}
|
||||
|
||||
func resourceBudgetMetadata(budget contract.ResourceBudget) ResourceBudgetMetadata {
|
||||
return ResourceBudgetMetadata{WarmupRunsPerPath: budget.WarmupRuns(), BaselineSampleCount: budget.SampleCount(), EndSampleCount: budget.SampleCount(), SampleIntervalMilliseconds: budget.SampleInterval().Milliseconds(), ChildProcessCountDrift: budget.ChildProcessCountDrift(), ListenerCountDrift: budget.ListenerCountDrift(), NonStdioFDCountDrift: budget.NonStdioFDCountDrift(), DashboardRSSDeltaBytes: budget.DashboardRSSDeltaBytes(), AgentRSSDeltaBytes: budget.AgentRSSDeltaBytes(), TransferHeapBytes: budget.TransferHeapBytes()}
|
||||
}
|
||||
|
||||
func (p ProfileMetadata) Validate() error {
|
||||
if p.Name == "" || p.JobTimeoutSeconds < 1 || p.SuiteDeadlineSeconds < 1 || p.DefaultSeed == "" || p.DefaultSeed == "0x0" || p.AgentCount < 1 || p.StressRounds < 1 || p.ConcurrentOperations < 1 || p.ConcurrentSessionsPerKind < 1 || p.TransferPairs < 1 || p.TransferBytes == 0 || p.DashboardRestartCycles < 1 || p.Iterations < 1 || p.StreamBoundaryAllowed < 1 || p.StreamBoundaryRejected <= p.StreamBoundaryAllowed {
|
||||
return fmt.Errorf("profile fields are invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b ResourceBudgetMetadata) Validate() error {
|
||||
if b.WarmupRunsPerPath < 1 || b.BaselineSampleCount < 1 || b.EndSampleCount < 1 || b.SampleIntervalMilliseconds < 1 || b.ChildProcessCountDrift < 0 || b.ListenerCountDrift < 0 || b.NonStdioFDCountDrift < 0 || b.DashboardRSSDeltaBytes == 0 || b.AgentRSSDeltaBytes == 0 || b.TransferHeapBytes == 0 {
|
||||
return fmt.Errorf("resource budget fields are invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
)
|
||||
|
||||
type transferEvidence struct {
|
||||
WarmupUploadBytes uint64 `json:"warmup_upload_bytes"`
|
||||
WarmupDownloadBytes uint64 `json:"warmup_download_bytes"`
|
||||
WarmupSHA256 string `json:"warmup_sha256"`
|
||||
WarmupDuration time.Duration `json:"warmup_duration"`
|
||||
WarmupDeadlineRemaining time.Duration `json:"warmup_deadline_remaining"`
|
||||
WarmupQuiescent bool `json:"warmup_quiescent"`
|
||||
UploadBytes uint64 `json:"upload_bytes"`
|
||||
DownloadBytes uint64 `json:"download_bytes"`
|
||||
UploadSHA256 string `json:"upload_sha256"`
|
||||
DownloadSHA256 string `json:"download_sha256"`
|
||||
UploadChunks uint64 `json:"upload_chunks"`
|
||||
DownloadChunks uint64 `json:"download_chunks"`
|
||||
UploadDuration time.Duration `json:"upload_duration"`
|
||||
DownloadDuration time.Duration `json:"download_duration"`
|
||||
RetainedHeapBytes uint64 `json:"retained_heap_bytes"`
|
||||
Mode string `json:"mode"`
|
||||
CreateDirs bool `json:"create_dirs"`
|
||||
UploadReplayRejected bool `json:"upload_replay_rejected"`
|
||||
DownloadReplayRejected bool `json:"download_replay_rejected"`
|
||||
OversizeRejected bool `json:"oversize_rejected"`
|
||||
AgentTempResidue int `json:"agent_temp_residue"`
|
||||
DashboardSpoolResidue int `json:"dashboard_spool_residue"`
|
||||
OutsideRootSentinelsUnchanged bool `json:"outside_root_sentinels_unchanged"`
|
||||
}
|
||||
|
||||
type transferArtifact struct {
|
||||
Scenario string `json:"scenario"`
|
||||
Fault string `json:"fault,omitempty"`
|
||||
Passed bool `json:"passed"`
|
||||
CleanupOK bool `json:"cleanup_ok"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Evidence transferEvidence `json:"evidence"`
|
||||
}
|
||||
|
||||
type reconnectObservation struct {
|
||||
ServerID uint64
|
||||
UUID string
|
||||
OldGeneration uint64
|
||||
NewGeneration uint64
|
||||
DisconnectAt time.Time
|
||||
ReconnectAt time.Time
|
||||
TaskIDs []uint64
|
||||
ResultIDs []uint64
|
||||
PostReconnect bool
|
||||
AgentRestarted bool
|
||||
}
|
||||
|
||||
type listenerIdentity struct {
|
||||
Address string
|
||||
Inode uint64
|
||||
}
|
||||
|
||||
type fixtureIdentity struct {
|
||||
WorkspaceRoot string
|
||||
ConfigPath string
|
||||
DatabasePath string
|
||||
BinaryPath string
|
||||
HTTP listenerIdentity
|
||||
Receipt listenerIdentity
|
||||
HTTPS listenerIdentity
|
||||
}
|
||||
|
||||
type runtimeIdentity struct {
|
||||
Generation uint64
|
||||
PID int
|
||||
ProcessGroupID int
|
||||
}
|
||||
|
||||
type receiptEvent struct {
|
||||
Sequence uint64 `json:"sequence"`
|
||||
DashboardGeneration uint64 `json:"dashboard_generation"`
|
||||
GateGeneration uint64 `json:"gate_generation"`
|
||||
ServerID uint64 `json:"server_id"`
|
||||
TaskID uint64 `json:"task_id"`
|
||||
TaskType uint64 `json:"task_type"`
|
||||
Kind string `json:"kind"`
|
||||
}
|
||||
|
||||
type receiptPair struct {
|
||||
Task receiptEvent `json:"task"`
|
||||
Result receiptEvent `json:"result"`
|
||||
}
|
||||
|
||||
type cleanupRecord struct {
|
||||
Name string `json:"name"`
|
||||
PID int `json:"pid"`
|
||||
Forced bool `json:"forced"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type cleanupReceipt struct {
|
||||
Passed bool `json:"passed"`
|
||||
Forced bool `json:"forced"`
|
||||
Processes []cleanupRecord `json:"processes"`
|
||||
}
|
||||
|
||||
type reconnectEvidence struct {
|
||||
Fixture struct {
|
||||
Dashboard fixtureIdentity `json:"dashboard"`
|
||||
AgentRoot string `json:"agent_root"`
|
||||
AgentConfigPath string `json:"agent_config_path"`
|
||||
AgentBinaryPath string `json:"agent_binary_path"`
|
||||
} `json:"fixture"`
|
||||
Runtime struct {
|
||||
DashboardBefore runtimeIdentity `json:"dashboard_before"`
|
||||
DashboardAfter runtimeIdentity `json:"dashboard_after"`
|
||||
AgentBefore runtimeIdentity `json:"agent_before"`
|
||||
AgentAfter runtimeIdentity `json:"agent_after"`
|
||||
StateGenerationBeforeAgentRestart uint64 `json:"state_generation_before_agent_restart"`
|
||||
StateGenerationAfterAgentRestart uint64 `json:"state_generation_after_agent_restart"`
|
||||
} `json:"runtime"`
|
||||
Identity struct {
|
||||
ServerID uint64 `json:"server_id"`
|
||||
UUID string `json:"uuid"`
|
||||
DashboardConfigUnchanged bool `json:"dashboard_config_unchanged"`
|
||||
AgentConfigUnchanged bool `json:"agent_config_unchanged"`
|
||||
DashboardFixtureUnchanged bool `json:"dashboard_fixture_unchanged"`
|
||||
ClientsRecreated bool `json:"clients_recreated"`
|
||||
BootstrapRecreated bool `json:"bootstrap_recreated"`
|
||||
} `json:"identity"`
|
||||
Lifecycle struct {
|
||||
DisconnectAt time.Time `json:"disconnect_at"`
|
||||
ReconnectAt time.Time `json:"reconnect_at"`
|
||||
ReconnectInterval time.Duration `json:"reconnect_interval"`
|
||||
DashboardReceipts []receiptPair `json:"dashboard_receipts"`
|
||||
AgentReceipts []receiptPair `json:"agent_receipts"`
|
||||
StaleGenerationReceipts int `json:"stale_generation_receipts"`
|
||||
DuplicateTaskIDs int `json:"duplicate_task_ids"`
|
||||
LostResultIDs int `json:"lost_result_ids"`
|
||||
OutsideRootSentinelUnchanged bool `json:"outside_root_sentinel_unchanged"`
|
||||
} `json:"lifecycle"`
|
||||
Observation reconnectObservation `json:"observation"`
|
||||
AgentCleanup cleanupReceipt `json:"agent_cleanup"`
|
||||
DashboardCleanup cleanupReceipt `json:"dashboard_cleanup"`
|
||||
}
|
||||
|
||||
type reconnectArtifact struct {
|
||||
Scenario string `json:"scenario"`
|
||||
Fault string `json:"fault,omitempty"`
|
||||
Passed bool `json:"passed"`
|
||||
CleanupOK bool `json:"cleanup_ok"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Evidence reconnectEvidence `json:"evidence"`
|
||||
}
|
||||
|
||||
func validateTransferArtifact(metadata Metadata, result ScenarioResult, artifact transferArtifact) error {
|
||||
if err := validateArtifactHeader(metadata, result, artifact.Scenario, artifact.Fault, artifact.Passed, artifact.CleanupOK, artifact.Error); err != nil {
|
||||
return err
|
||||
}
|
||||
switch artifact.Fault {
|
||||
case "":
|
||||
if !artifact.Passed {
|
||||
return errors.New("transfer success artifact reports failure")
|
||||
}
|
||||
if err := validateTransferSuccess(artifact.Evidence); err != nil {
|
||||
return err
|
||||
}
|
||||
case contract.FaultTransferHash:
|
||||
if artifact.Passed || !strings.Contains(artifact.Error, "injected hash mismatch") {
|
||||
return errors.New("transfer-hash artifact does not identify the injected failure")
|
||||
}
|
||||
evidence := artifact.Evidence
|
||||
if evidence.WarmupUploadBytes != 65536 || evidence.WarmupDownloadBytes != 65536 || evidence.WarmupSHA256 == "" || evidence.WarmupDuration <= 0 || evidence.WarmupDeadlineRemaining <= 0 || !evidence.WarmupQuiescent {
|
||||
return errors.New("transfer-hash artifact omitted warmup evidence")
|
||||
}
|
||||
if evidence.UploadBytes != 0 || evidence.DownloadBytes != 0 || evidence.UploadSHA256 != "" || evidence.DownloadSHA256 != "" || evidence.UploadChunks != 0 || evidence.DownloadChunks != 0 || evidence.UploadDuration != 0 || evidence.DownloadDuration != 0 || evidence.RetainedHeapBytes != 0 || evidence.Mode != "" || evidence.CreateDirs || evidence.UploadReplayRejected || evidence.DownloadReplayRejected || evidence.OversizeRejected {
|
||||
return errors.New("transfer-hash artifact presents stale success evidence")
|
||||
}
|
||||
if evidence.AgentTempResidue != 0 || evidence.DashboardSpoolResidue != 0 || !evidence.OutsideRootSentinelsUnchanged {
|
||||
return errors.New("transfer-hash artifact cleanup or sentinel evidence failed")
|
||||
}
|
||||
default:
|
||||
return errors.New("transfer artifact has unsupported fault")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateArtifactHeader(metadata Metadata, result ScenarioResult, scenarioName, fault string, passed, cleanupOK bool, errorText string) error {
|
||||
if scenarioName != result.Name || scenarioName != metadata.Scenarios[0] || fault != metadata.Fault || passed != result.Passed || !cleanupOK || errorText != result.Error {
|
||||
return errors.New("dedicated artifact does not agree with metadata, results, or cleanup")
|
||||
}
|
||||
if passed && errorText != "" || !passed && errorText == "" {
|
||||
return errors.New("dedicated artifact pass/error state is inconsistent")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateTransferSuccess(evidence transferEvidence) error {
|
||||
if evidence.WarmupUploadBytes != 65536 || evidence.WarmupDownloadBytes != 65536 || evidence.WarmupSHA256 == "" || evidence.WarmupDuration <= 0 || evidence.WarmupDeadlineRemaining <= 0 || !evidence.WarmupQuiescent {
|
||||
return errors.New("transfer warmup evidence is invalid")
|
||||
}
|
||||
if evidence.UploadBytes != contract.TransferBytes || evidence.DownloadBytes != contract.TransferBytes || evidence.UploadSHA256 == "" || !strings.EqualFold(evidence.UploadSHA256, evidence.DownloadSHA256) {
|
||||
return errors.New("transfer byte counts or hashes are invalid")
|
||||
}
|
||||
if evidence.UploadChunks == 0 || evidence.DownloadChunks == 0 || evidence.UploadDuration <= 0 || evidence.DownloadDuration <= 0 || evidence.RetainedHeapBytes > contract.TransferHeapBytes {
|
||||
return errors.New("transfer measurement evidence is invalid")
|
||||
}
|
||||
if evidence.Mode != "0640" || !evidence.CreateDirs || !evidence.UploadReplayRejected || !evidence.DownloadReplayRejected || !evidence.OversizeRejected || evidence.AgentTempResidue != 0 || evidence.DashboardSpoolResidue != 0 || !evidence.OutsideRootSentinelsUnchanged {
|
||||
return errors.New("transfer contract evidence is invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
)
|
||||
|
||||
func TestEvidence_DedicatedArtifactsRequireExactScenarioAssertions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
scenario string
|
||||
assertion string
|
||||
artifact func(*testing.T, string)
|
||||
}{
|
||||
{"transfer generic", contract.ScenarioTransfer100MiB, "scenario passed", func(t *testing.T, dir string) {
|
||||
writeJSONEvidenceFile(t, dir, "transfer.json", validTransferArtifact("", true))
|
||||
}},
|
||||
{"transfer mismatched", contract.ScenarioTransfer100MiB, contract.AssertionReconnectDisconnect, func(t *testing.T, dir string) {
|
||||
writeJSONEvidenceFile(t, dir, "transfer.json", validTransferArtifact("", true))
|
||||
}},
|
||||
{"reconnect generic", contract.ScenarioReconnect, "scenario passed", func(t *testing.T, dir string) {
|
||||
writeJSONEvidenceFile(t, dir, "reconnect.json", validReconnectArtifact("", true))
|
||||
}},
|
||||
{"reconnect mismatched", contract.ScenarioReconnect, contract.AssertionTransferWarmup, func(t *testing.T, dir string) {
|
||||
writeJSONEvidenceFile(t, dir, "reconnect.json", validReconnectArtifact("", true))
|
||||
}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeDedicatedExecutableEvidence(t, dir, test.scenario, "", true)
|
||||
results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: test.scenario, Passed: true, Assertions: []Assertion{{Name: test.assertion, Passed: true}}}}}
|
||||
writeJSONEvidenceFile(t, dir, "results.json", results)
|
||||
junit, err := JUnit(results)
|
||||
if err != nil {
|
||||
t.Fatalf("JUnit: %v", err)
|
||||
}
|
||||
writeEvidenceFile(t, dir, "junit.xml", string(junit))
|
||||
test.artifact(t, dir)
|
||||
if err := ValidateDirectory(dir); err == nil {
|
||||
t.Fatal("generic assertion accepted for dedicated evidence")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestEvidence_TransferHashRejectsEverySuccessOnlyField(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*transferEvidence)
|
||||
}{
|
||||
{"upload bytes", func(value *transferEvidence) { value.UploadBytes = 1 }},
|
||||
{"download bytes", func(value *transferEvidence) { value.DownloadBytes = 1 }},
|
||||
{"upload hash", func(value *transferEvidence) { value.UploadSHA256 = "stale" }},
|
||||
{"download hash", func(value *transferEvidence) { value.DownloadSHA256 = "stale" }},
|
||||
{"upload chunks", func(value *transferEvidence) { value.UploadChunks = 1 }},
|
||||
{"download chunks", func(value *transferEvidence) { value.DownloadChunks = 1 }},
|
||||
{"upload duration", func(value *transferEvidence) { value.UploadDuration = time.Nanosecond }},
|
||||
{"download duration", func(value *transferEvidence) { value.DownloadDuration = time.Nanosecond }},
|
||||
{"retained heap", func(value *transferEvidence) { value.RetainedHeapBytes = 1 }},
|
||||
{"mode", func(value *transferEvidence) { value.Mode = "0640" }},
|
||||
{"create dirs", func(value *transferEvidence) { value.CreateDirs = true }},
|
||||
{"upload replay", func(value *transferEvidence) { value.UploadReplayRejected = true }},
|
||||
{"download replay", func(value *transferEvidence) { value.DownloadReplayRejected = true }},
|
||||
{"oversize", func(value *transferEvidence) { value.OversizeRejected = true }},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeDedicatedExecutableEvidence(t, dir, "transfer-100mib", "transfer-hash", false)
|
||||
artifact := validTransferArtifact("transfer-hash", false)
|
||||
test.mutate(&artifact.Evidence)
|
||||
writeJSONEvidenceFile(t, dir, "transfer.json", artifact)
|
||||
if err := ValidateDirectory(dir); err == nil {
|
||||
t.Fatal("stale transfer success field accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_DashboardExitRejectsEveryPostFaultField(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*reconnectEvidence)
|
||||
}{
|
||||
{"dashboard after", func(value *reconnectEvidence) { value.Runtime.DashboardAfter.PID = 1 }},
|
||||
{"agent after", func(value *reconnectEvidence) { value.Runtime.AgentAfter.PID = 1 }},
|
||||
{"state before restart", func(value *reconnectEvidence) { value.Runtime.StateGenerationBeforeAgentRestart = 1 }},
|
||||
{"state after restart", func(value *reconnectEvidence) { value.Runtime.StateGenerationAfterAgentRestart = 1 }},
|
||||
{"identity dashboard config", func(value *reconnectEvidence) { value.Identity.DashboardConfigUnchanged = true }},
|
||||
{"identity agent config", func(value *reconnectEvidence) { value.Identity.AgentConfigUnchanged = true }},
|
||||
{"identity fixture", func(value *reconnectEvidence) { value.Identity.DashboardFixtureUnchanged = true }},
|
||||
{"clients recreated", func(value *reconnectEvidence) { value.Identity.ClientsRecreated = true }},
|
||||
{"bootstrap recreated", func(value *reconnectEvidence) { value.Identity.BootstrapRecreated = true }},
|
||||
{"reconnect timestamp", func(value *reconnectEvidence) { value.Lifecycle.ReconnectAt = time.Now() }},
|
||||
{"reconnect interval", func(value *reconnectEvidence) { value.Lifecycle.ReconnectInterval = time.Second }},
|
||||
{"dashboard receipts", func(value *reconnectEvidence) { value.Lifecycle.DashboardReceipts = []receiptPair{{}} }},
|
||||
{"agent receipts", func(value *reconnectEvidence) { value.Lifecycle.AgentReceipts = []receiptPair{{}} }},
|
||||
{"stale accounting", func(value *reconnectEvidence) { value.Lifecycle.StaleGenerationReceipts = 1 }},
|
||||
{"duplicate accounting", func(value *reconnectEvidence) { value.Lifecycle.DuplicateTaskIDs = 1 }},
|
||||
{"lost accounting", func(value *reconnectEvidence) { value.Lifecycle.LostResultIDs = 1 }},
|
||||
{"observation server", func(value *reconnectEvidence) { value.Observation.ServerID = 1 }},
|
||||
{"observation uuid", func(value *reconnectEvidence) { value.Observation.UUID = "stale" }},
|
||||
{"old generation", func(value *reconnectEvidence) { value.Observation.OldGeneration = 1 }},
|
||||
{"new generation", func(value *reconnectEvidence) { value.Observation.NewGeneration = 1 }},
|
||||
{"disconnect observation", func(value *reconnectEvidence) { value.Observation.DisconnectAt = time.Now() }},
|
||||
{"reconnect observation", func(value *reconnectEvidence) { value.Observation.ReconnectAt = time.Now() }},
|
||||
{"task ids", func(value *reconnectEvidence) { value.Observation.TaskIDs = []uint64{1} }},
|
||||
{"result ids", func(value *reconnectEvidence) { value.Observation.ResultIDs = []uint64{1} }},
|
||||
{"post reconnect", func(value *reconnectEvidence) { value.Observation.PostReconnect = true }},
|
||||
{"agent restarted", func(value *reconnectEvidence) { value.Observation.AgentRestarted = true }},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeDedicatedExecutableEvidence(t, dir, "reconnect", "dashboard-exit", false)
|
||||
artifact := validReconnectArtifact("dashboard-exit", false)
|
||||
test.mutate(&artifact.Evidence)
|
||||
writeJSONEvidenceFile(t, dir, "reconnect.json", artifact)
|
||||
if err := ValidateDirectory(dir); err == nil {
|
||||
t.Fatal("stale reconnect success field accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
)
|
||||
|
||||
func TestEvidence_TransferAndReconnectRequireDedicatedArtifacts(t *testing.T) {
|
||||
for _, scenarioName := range []string{"transfer-100mib", "reconnect"} {
|
||||
t.Run(scenarioName, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeExecutableEvidence(t, dir, "pr-full", scenarioName, true, true, true)
|
||||
if err := ValidateDirectory(dir); err == nil {
|
||||
t.Fatal("missing dedicated artifact accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_ValidatesTypedTransferSuccessAndFaultArtifacts(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fault string
|
||||
passed bool
|
||||
}{
|
||||
{"success", "", true},
|
||||
{"hash fault", "transfer-hash", false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeDedicatedExecutableEvidence(t, dir, "transfer-100mib", test.fault, test.passed)
|
||||
artifact := validTransferArtifact(test.fault, test.passed)
|
||||
writeJSONEvidenceFile(t, dir, "transfer.json", artifact)
|
||||
if err := ValidateDirectory(dir); err != nil {
|
||||
t.Fatalf("validate transfer evidence: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_ValidatesTypedReconnectSuccessAndFaultArtifacts(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fault string
|
||||
passed bool
|
||||
}{
|
||||
{"success", "", true},
|
||||
{"dashboard fault", "dashboard-exit", false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeDedicatedExecutableEvidence(t, dir, "reconnect", test.fault, test.passed)
|
||||
artifact := validReconnectArtifact(test.fault, test.passed)
|
||||
writeJSONEvidenceFile(t, dir, "reconnect.json", artifact)
|
||||
if err := ValidateDirectory(dir); err != nil {
|
||||
t.Fatalf("validate reconnect evidence: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_RejectsMisleadingDedicatedArtifact(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeDedicatedExecutableEvidence(t, dir, "transfer-100mib", "transfer-hash", false)
|
||||
artifact := validTransferArtifact("transfer-hash", false)
|
||||
artifact.Evidence.UploadBytes = 104857600
|
||||
writeJSONEvidenceFile(t, dir, "transfer.json", artifact)
|
||||
if err := ValidateDirectory(dir); err == nil {
|
||||
t.Fatal("fault artifact containing stale success accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_RejectsWrongOrStaleDedicatedArtifact(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeExecutableEvidence(t, dir, "pr-full", "transfer-100mib", true, true, true)
|
||||
writeEvidenceFile(t, dir, "reconnect.json", `{}`)
|
||||
if err := ValidateDirectory(dir); err == nil {
|
||||
t.Fatal("wrong dedicated artifact accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_RejectsDedicatedArtifactWithPublicMode(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeExecutableEvidence(t, dir, "pr-full", "transfer-100mib", true, true, true)
|
||||
path := filepath.Join(dir, "transfer.json")
|
||||
if err := os.WriteFile(path, []byte(`{}`), 0o644); err != nil {
|
||||
t.Fatalf("write transfer artifact: %v", err)
|
||||
}
|
||||
if err := ValidateDirectory(dir); err == nil {
|
||||
t.Fatal("public dedicated artifact mode accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func writeDedicatedExecutableEvidence(t *testing.T, dir, scenarioName, fault string, passed bool) {
|
||||
t.Helper()
|
||||
metadata := validMetadata(t, dir, scenarioName)
|
||||
metadata.Fault = fault
|
||||
writeJSONEvidenceFile(t, dir, "metadata.json", metadata)
|
||||
errorText := ""
|
||||
if !passed {
|
||||
if scenarioName == "transfer-100mib" {
|
||||
errorText = "transfer scenario: injected hash mismatch"
|
||||
} else {
|
||||
errorText = "reconnect scenario: injected Dashboard exit"
|
||||
}
|
||||
}
|
||||
definition, err := contract.ScenarioDefinitionByName(scenarioName)
|
||||
if err != nil {
|
||||
t.Fatalf("scenario definition: %v", err)
|
||||
}
|
||||
assertions := make([]Assertion, 0, len(definition.Assertions(fault)))
|
||||
for _, assertion := range definition.Assertions(fault) {
|
||||
assertions = append(assertions, Assertion{Name: assertion.Name, Passed: assertion.Passed})
|
||||
}
|
||||
results := Results{Profile: "pr-full", Passed: passed, Scenarios: []ScenarioResult{{Name: scenarioName, Passed: passed, Assertions: assertions, Error: errorText}}}
|
||||
writeJSONEvidenceFile(t, dir, "results.json", results)
|
||||
junit, err := JUnit(results)
|
||||
if err != nil {
|
||||
t.Fatalf("JUnit: %v", err)
|
||||
}
|
||||
writeEvidenceFile(t, dir, "junit.xml", string(junit))
|
||||
writeEvidenceFile(t, dir, "cleanup.json", fmt.Sprintf(`{"passed":true,"scenario":%q,"finished_at":"2026-01-02T03:04:05Z"}`, scenarioName))
|
||||
}
|
||||
|
||||
func validTransferArtifact(fault string, passed bool) transferArtifact {
|
||||
errorText := ""
|
||||
if !passed {
|
||||
errorText = "transfer scenario: injected hash mismatch"
|
||||
}
|
||||
evidence := transferEvidence{WarmupUploadBytes: 65536, WarmupDownloadBytes: 65536, WarmupSHA256: "abc", WarmupDuration: time.Nanosecond, WarmupDeadlineRemaining: time.Second, WarmupQuiescent: true, OutsideRootSentinelsUnchanged: true}
|
||||
if passed {
|
||||
evidence.UploadBytes = 104857600
|
||||
evidence.DownloadBytes = 104857600
|
||||
evidence.UploadSHA256 = "abc"
|
||||
evidence.DownloadSHA256 = "abc"
|
||||
evidence.UploadChunks = 2
|
||||
evidence.DownloadChunks = 2
|
||||
evidence.UploadDuration = time.Second
|
||||
evidence.DownloadDuration = time.Second
|
||||
evidence.Mode = "0640"
|
||||
evidence.CreateDirs = true
|
||||
evidence.UploadReplayRejected = true
|
||||
evidence.DownloadReplayRejected = true
|
||||
evidence.OversizeRejected = true
|
||||
}
|
||||
return transferArtifact{Scenario: "transfer-100mib", Fault: fault, Passed: passed, CleanupOK: true, Error: errorText, Evidence: evidence}
|
||||
}
|
||||
|
||||
func validReconnectArtifact(fault string, passed bool) reconnectArtifact {
|
||||
var artifact reconnectArtifact
|
||||
artifact.Scenario = "reconnect"
|
||||
artifact.Fault = fault
|
||||
artifact.Passed = passed
|
||||
artifact.CleanupOK = true
|
||||
if !passed {
|
||||
artifact.Error = "reconnect scenario: injected Dashboard exit"
|
||||
}
|
||||
evidence := &artifact.Evidence
|
||||
evidence.Fixture.Dashboard.WorkspaceRoot = "/tmp/dashboard"
|
||||
evidence.Fixture.Dashboard.ConfigPath = "/tmp/dashboard/config"
|
||||
evidence.Fixture.Dashboard.BinaryPath = "/tmp/dashboard/bin"
|
||||
evidence.Fixture.AgentRoot = "/tmp/agent"
|
||||
evidence.Fixture.AgentConfigPath = "/tmp/agent/config"
|
||||
evidence.Fixture.AgentBinaryPath = "/tmp/agent/bin"
|
||||
evidence.Runtime.DashboardBefore = runtimeIdentity{Generation: 1, PID: 10, ProcessGroupID: 10}
|
||||
evidence.Runtime.AgentBefore = runtimeIdentity{Generation: 1, PID: 20, ProcessGroupID: 20}
|
||||
evidence.Identity.ServerID = 1
|
||||
evidence.Identity.UUID = "uuid"
|
||||
evidence.Lifecycle.DisconnectAt = time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
evidence.Lifecycle.OutsideRootSentinelUnchanged = true
|
||||
evidence.AgentCleanup = validCleanupReceipt("agent", expectedProcessCount(passed))
|
||||
evidence.DashboardCleanup = validCleanupReceipt("dashboard", expectedProcessCount(passed))
|
||||
if passed {
|
||||
evidence.Runtime.DashboardAfter = runtimeIdentity{Generation: 2, PID: 11, ProcessGroupID: 11}
|
||||
evidence.Runtime.AgentAfter = runtimeIdentity{Generation: 2, PID: 21, ProcessGroupID: 21}
|
||||
evidence.Runtime.StateGenerationBeforeAgentRestart = 1
|
||||
evidence.Runtime.StateGenerationAfterAgentRestart = 2
|
||||
evidence.Identity.DashboardConfigUnchanged = true
|
||||
evidence.Identity.AgentConfigUnchanged = true
|
||||
evidence.Identity.DashboardFixtureUnchanged = true
|
||||
evidence.Identity.ClientsRecreated = true
|
||||
evidence.Identity.BootstrapRecreated = true
|
||||
evidence.Lifecycle.ReconnectAt = evidence.Lifecycle.DisconnectAt.Add(time.Second)
|
||||
evidence.Lifecycle.ReconnectInterval = time.Second
|
||||
evidence.Lifecycle.DashboardReceipts = make([]receiptPair, 3)
|
||||
evidence.Lifecycle.AgentReceipts = make([]receiptPair, 2)
|
||||
evidence.Observation = reconnectObservation{ServerID: 1, UUID: "uuid", OldGeneration: 1, NewGeneration: 2, DisconnectAt: evidence.Lifecycle.DisconnectAt, ReconnectAt: evidence.Lifecycle.ReconnectAt, TaskIDs: []uint64{1, 2, 3, 4, 5}, ResultIDs: []uint64{1, 2, 3, 4, 5}, PostReconnect: true, AgentRestarted: true}
|
||||
}
|
||||
return artifact
|
||||
}
|
||||
|
||||
func validCleanupReceipt(name string, count int) cleanupReceipt {
|
||||
receipt := cleanupReceipt{Passed: true, Processes: make([]cleanupRecord, count)}
|
||||
for index := range receipt.Processes {
|
||||
receipt.Processes[index] = cleanupRecord{Name: name, PID: index + 1}
|
||||
}
|
||||
return receipt
|
||||
}
|
||||
|
||||
func expectedProcessCount(passed bool) int {
|
||||
if passed {
|
||||
return 2
|
||||
}
|
||||
return 1
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
maxEvidenceFiles = 128
|
||||
maxEvidenceBytes = 32 << 20
|
||||
)
|
||||
|
||||
func scanDirectory(resultsDir string) (map[string]os.FileInfo, error) {
|
||||
if strings.TrimSpace(resultsDir) == "" {
|
||||
return nil, errors.New("evidence directory is required")
|
||||
}
|
||||
info, err := os.Lstat(resultsDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat evidence directory: %w", err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
|
||||
return nil, errors.New("evidence path must be a directory")
|
||||
}
|
||||
if info.Mode().Perm() != 0o700 {
|
||||
return nil, errors.New("evidence directory must use mode 0700")
|
||||
}
|
||||
seen := make(map[string]os.FileInfo)
|
||||
var totalBytes int64
|
||||
err = filepath.WalkDir(resultsDir, func(path string, entry os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if path == resultsDir {
|
||||
return nil
|
||||
}
|
||||
if entry.IsDir() {
|
||||
relative, err := filepath.Rel(resultsDir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if relative != "agents" {
|
||||
return fmt.Errorf("evidence path is not allowed: %s", relative)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if entry.Type()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("evidence symlink is not allowed: %s", path)
|
||||
}
|
||||
if !entry.Type().IsRegular() {
|
||||
return fmt.Errorf("evidence file is not regular: %s", path)
|
||||
}
|
||||
relative, err := filepath.Rel(resultsDir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileInfo, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !allowedEvidencePath(relative) {
|
||||
return fmt.Errorf("evidence path is not allowed: %s", relative)
|
||||
}
|
||||
if fileInfo.Mode().Perm() != 0o600 {
|
||||
return fmt.Errorf("evidence file must use mode 0600: %s", relative)
|
||||
}
|
||||
seen[relative] = fileInfo
|
||||
if len(seen) > maxEvidenceFiles {
|
||||
return errors.New("too many evidence files")
|
||||
}
|
||||
totalBytes += fileInfo.Size()
|
||||
if totalBytes > maxEvidenceBytes {
|
||||
return errors.New("evidence files exceed size limit")
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if Redact(string(data)) != string(data) {
|
||||
return fmt.Errorf("credential detected in evidence file: %s", path)
|
||||
}
|
||||
switch filepath.Ext(path) {
|
||||
case ".json":
|
||||
if !json.Valid(data) {
|
||||
return fmt.Errorf("invalid JSON evidence file: %s", path)
|
||||
}
|
||||
case ".xml":
|
||||
var document any
|
||||
if err := xml.Unmarshal(data, &document); err != nil {
|
||||
return fmt.Errorf("invalid XML evidence file: %s: %w", path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return seen, err
|
||||
}
|
||||
|
||||
func allowedEvidencePath(relative string) bool {
|
||||
for _, name := range EvidenceFiles() {
|
||||
if name == relative {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if filepath.Dir(relative) != "agents" || filepath.Ext(relative) != ".log" {
|
||||
return false
|
||||
}
|
||||
base := strings.TrimSuffix(filepath.Base(relative), ".log")
|
||||
if base == "" || base == "." {
|
||||
return false
|
||||
}
|
||||
for _, character := range base {
|
||||
if character != '-' && character != '_' && character != '.' && (character < '0' || character > '9') && (character < 'A' || character > 'Z') && (character < 'a' || character > 'z') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func readJSONFile[T any](resultsDir, name string) (T, error) {
|
||||
var value T
|
||||
data, err := os.ReadFile(filepath.Join(resultsDir, name))
|
||||
if err != nil {
|
||||
return value, fmt.Errorf("read %s: %w", name, err)
|
||||
}
|
||||
if err := json.Unmarshal(data, &value); err != nil {
|
||||
return value, fmt.Errorf("parse %s: %w", name, err)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
)
|
||||
|
||||
func TestEvidence_ValidateDirectoryRejectsPublicRootOrEvidenceFile(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
}{
|
||||
{name: "root", path: ""},
|
||||
{name: "metadata", path: "metadata.json"},
|
||||
{name: "results", path: "results.json"},
|
||||
{name: "junit", path: "junit.xml"},
|
||||
{name: "cleanup", path: "cleanup.json"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, true, true, true)
|
||||
path := dir
|
||||
if test.path != "" {
|
||||
path = filepath.Join(dir, test.path)
|
||||
}
|
||||
if err := os.Chmod(path, 0o644); err != nil {
|
||||
t.Fatalf("chmod fixture: %v", err)
|
||||
}
|
||||
if err := ValidateDirectory(dir); err == nil {
|
||||
t.Fatal("public evidence mode accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_ValidateDirectoryRejectsUnexpectedPaths(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
mode os.FileMode
|
||||
}{
|
||||
{name: "payload", path: "payload.bin", mode: 0o600},
|
||||
{name: "executable", path: "agentcompat", mode: 0o700},
|
||||
{name: "nested log", path: "agents/nested/agent.log", mode: 0o600},
|
||||
{name: "unknown agent file", path: "agents/agent.bin", mode: 0o600},
|
||||
{name: "unknown directory", path: "unexpected/file.log", mode: 0o600},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, true, true, true)
|
||||
path := filepath.Join(dir, test.path)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
t.Fatalf("mkdir unexpected parent: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte("unexpected"), test.mode); err != nil {
|
||||
t.Fatalf("write unexpected file: %v", err)
|
||||
}
|
||||
if err := ValidateDirectory(dir); err == nil || !strings.Contains(err.Error(), "not allowed") {
|
||||
t.Fatalf("unexpected path rejection err=%v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
)
|
||||
|
||||
type cleanupEvidence struct {
|
||||
Passed bool `json:"passed"`
|
||||
Scenario string `json:"scenario"`
|
||||
FinishedAt string `json:"finished_at"`
|
||||
}
|
||||
|
||||
type currentEvidenceProfile struct {
|
||||
RequiredFiles []string
|
||||
DedicatedFile string
|
||||
Executable bool
|
||||
}
|
||||
|
||||
func currentProfile(metadata Metadata) (currentEvidenceProfile, error) {
|
||||
if len(metadata.Scenarios) == 0 {
|
||||
return currentEvidenceProfile{}, errors.New("metadata scenarios are required")
|
||||
}
|
||||
if len(metadata.Scenarios) == 1 && metadata.Scenarios[0] == contract.ScenarioMetadata {
|
||||
if metadata.Fault != "" {
|
||||
return currentEvidenceProfile{}, errors.New("metadata scenario does not support fault injection")
|
||||
}
|
||||
return currentEvidenceProfile{RequiredFiles: []string{"metadata.json"}}, nil
|
||||
}
|
||||
if len(metadata.Scenarios) != 1 {
|
||||
return currentEvidenceProfile{}, errors.New("current evidence profile requires one supported scenario")
|
||||
}
|
||||
scenarioValue, err := contract.NewScenario(metadata.Scenarios[0])
|
||||
if err != nil {
|
||||
return currentEvidenceProfile{}, fmt.Errorf("construct metadata scenario: %w", err)
|
||||
}
|
||||
if !contract.IsSupportedScenario(metadata.Scenarios[0]) || metadata.Scenarios[0] == contract.ScenarioMetadata {
|
||||
return currentEvidenceProfile{}, errors.New("current evidence profile requires one supported scenario")
|
||||
}
|
||||
faultValue := contract.Fault{}
|
||||
if metadata.Fault != "" {
|
||||
faultValue, err = contract.NewFault(metadata.Fault)
|
||||
if err != nil {
|
||||
return currentEvidenceProfile{}, fmt.Errorf("construct metadata fault: %w", err)
|
||||
}
|
||||
}
|
||||
if err := contract.ValidateScenarioFault(scenarioValue, faultValue); err != nil {
|
||||
return currentEvidenceProfile{}, err
|
||||
}
|
||||
definition, err := contract.ScenarioDefinitionByName(metadata.Scenarios[0])
|
||||
if err != nil {
|
||||
return currentEvidenceProfile{}, err
|
||||
}
|
||||
profile := currentEvidenceProfile{RequiredFiles: []string{"metadata.json", "results.json", "junit.xml", "cleanup.json"}, DedicatedFile: definition.DedicatedArtifactName(), Executable: true}
|
||||
if profile.DedicatedFile != "" {
|
||||
profile.RequiredFiles = append(profile.RequiredFiles, profile.DedicatedFile)
|
||||
}
|
||||
return profile, nil
|
||||
}
|
||||
|
||||
func ValidateDirectory(resultsDir string) error {
|
||||
files, err := scanDirectory(resultsDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(files) == 0 {
|
||||
return errors.New("evidence directory contains no files")
|
||||
}
|
||||
metadata, err := readJSONFile[Metadata](resultsDir, "metadata.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := metadata.Validate(); err != nil {
|
||||
return fmt.Errorf("validate metadata evidence: %w", err)
|
||||
}
|
||||
profile, err := currentProfile(metadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, required := range profile.RequiredFiles {
|
||||
if _, exists := files[required]; !exists {
|
||||
return fmt.Errorf("required evidence file is missing: %s", required)
|
||||
}
|
||||
}
|
||||
if err := rejectStaleDedicatedFiles(files, profile.DedicatedFile); err != nil {
|
||||
return err
|
||||
}
|
||||
if !profile.Executable {
|
||||
return nil
|
||||
}
|
||||
results, err := readJSONFile[Results](resultsDir, "results.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := results.Validate(); err != nil {
|
||||
return fmt.Errorf("validate results evidence: %w", err)
|
||||
}
|
||||
if results.Profile != metadata.Profile.Name || !slices.Equal(metadata.Scenarios, scenarioResultNames(results.Scenarios)) {
|
||||
return errors.New("metadata and results do not agree")
|
||||
}
|
||||
if err := validateJUnit(resultsDir, results); err != nil {
|
||||
return err
|
||||
}
|
||||
cleanup, err := readJSONFile[cleanupEvidence](resultsDir, "cleanup.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !cleanup.Passed || cleanup.Scenario != results.Scenarios[0].Name {
|
||||
return errors.New("cleanup evidence is missing or failed")
|
||||
}
|
||||
if _, err := time.Parse(time.RFC3339, cleanup.FinishedAt); err != nil {
|
||||
return errors.New("cleanup finish time is invalid")
|
||||
}
|
||||
if profile.DedicatedFile != "" {
|
||||
definition, err := contract.ScenarioDefinitionByName(results.Scenarios[0].Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateScenarioAssertions(results.Scenarios[0], definition.Assertions(metadata.Fault)); err != nil {
|
||||
return err
|
||||
}
|
||||
return validateDedicatedArtifact(resultsDir, metadata, results.Scenarios[0])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateScenarioAssertions(result ScenarioResult, expected []contract.AssertionDefinition) error {
|
||||
if len(result.Assertions) != len(expected) {
|
||||
return errors.New("scenario assertions do not match dedicated evidence contract")
|
||||
}
|
||||
for index, assertion := range result.Assertions {
|
||||
if assertion.Name != expected[index].Name || assertion.Passed != expected[index].Passed {
|
||||
return errors.New("scenario assertions do not match dedicated evidence contract")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func rejectStaleDedicatedFiles(files map[string]os.FileInfo, expected string) error {
|
||||
for _, name := range []string{"transfer.json", "reconnect.json"} {
|
||||
if _, exists := files[name]; exists && name != expected {
|
||||
return fmt.Errorf("stale or wrong dedicated evidence file: %s", name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateJUnit(resultsDir string, results Results) error {
|
||||
data, err := os.ReadFile(filepath.Join(resultsDir, "junit.xml"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("read JUnit evidence: %w", err)
|
||||
}
|
||||
var suite junitSuite
|
||||
if err := xml.Unmarshal(data, &suite); err != nil {
|
||||
return fmt.Errorf("parse JUnit evidence: %w", err)
|
||||
}
|
||||
if suite.Name != results.Profile || suite.Tests != len(results.Scenarios) || suite.Failures != countFailedScenarios(results.Scenarios) || len(suite.Cases) != len(results.Scenarios) {
|
||||
return errors.New("results and JUnit evidence do not agree")
|
||||
}
|
||||
for index, scenario := range results.Scenarios {
|
||||
caseResult := suite.Cases[index]
|
||||
if caseResult.Name != scenario.Name || (caseResult.Failure != nil) != !scenario.Passed {
|
||||
return errors.New("results and JUnit scenario states do not agree")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
)
|
||||
|
||||
func TestEvidence_CurrentProfilePropagatesMalformedScenarioAndFault(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
metadata Metadata
|
||||
wantContext string
|
||||
wantCause string
|
||||
}{
|
||||
{name: "scenario", metadata: Metadata{Scenarios: []string{"invalid scenario"}}, wantContext: "construct metadata scenario", wantCause: "invalid scenario name"},
|
||||
{name: "fault", metadata: Metadata{Scenarios: []string{contract.ScenarioTransfer100MiB}, Fault: "invalid fault"}, wantContext: "construct metadata fault", wantCause: "invalid fault name"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
_, err := currentProfile(test.metadata)
|
||||
if err == nil || !strings.Contains(err.Error(), test.wantContext) || !strings.Contains(err.Error(), test.wantCause) {
|
||||
t.Fatalf("currentProfile error=%v, want context %q and cause %q", err, test.wantContext, test.wantCause)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_NoCredentialsInDirectory(t *testing.T) {
|
||||
if len(flag.Args()) == 0 {
|
||||
t.Skip("requires a results directory argument: go test ./integration/agentcompat/internal/evidence -run TestEvidence_NoCredentialsInDirectory -args RESULTS")
|
||||
}
|
||||
if err := ValidateDirectory(flag.Args()[0]); err != nil {
|
||||
t.Fatalf("validate evidence directory: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_NoCredentialsInDirectoryFixture(t *testing.T) {
|
||||
resultsDir := t.TempDir()
|
||||
metadata := validMetadata(t, resultsDir, contract.ScenarioRegistrationConfigExec)
|
||||
writeJSONEvidenceFile(t, resultsDir, "metadata.json", metadata)
|
||||
results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: contract.ScenarioRegistrationConfigExec, Passed: true, Assertions: []Assertion{{Name: "safe fixture", Passed: true}}}}}
|
||||
writeJSONEvidenceFile(t, resultsDir, "results.json", results)
|
||||
junit, err := JUnit(results)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal safe JUnit: %v", err)
|
||||
}
|
||||
writeEvidenceFile(t, resultsDir, "junit.xml", string(junit))
|
||||
writeEvidenceFile(t, resultsDir, "cleanup.json", `{"passed":true,"scenario":"registration-config-exec","finished_at":"2026-01-02T03:04:05Z"}`)
|
||||
if err := ValidateDirectory(resultsDir); err != nil {
|
||||
t.Fatalf("validate evidence directory: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_ValidateDirectoryRejectsCrossFileMismatches(t *testing.T) {
|
||||
tests := map[string]func(*testing.T, string){
|
||||
"profile mismatch": func(t *testing.T, dir string) {
|
||||
writeExecutableEvidence(t, dir, "soak", contract.ScenarioRegistrationConfigExec, true, true, true)
|
||||
},
|
||||
"scenario mismatch": func(t *testing.T, dir string) {
|
||||
writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, true, false, true)
|
||||
},
|
||||
"junit mismatch": func(t *testing.T, dir string) {
|
||||
writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, false, true, true)
|
||||
},
|
||||
"missing cleanup": func(t *testing.T, dir string) {
|
||||
writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, true, true, false)
|
||||
},
|
||||
}
|
||||
for name, setup := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
setup(t, dir)
|
||||
if err := ValidateDirectory(dir); err == nil {
|
||||
t.Fatal("inconsistent evidence accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_ValidateDirectoryRejectsFailedCleanup(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, true, true, true)
|
||||
writeEvidenceFile(t, dir, "cleanup.json", `{"passed":false,"scenario":"registration-config-exec","finished_at":"2026-01-02T03:04:05Z"}`)
|
||||
if err := ValidateDirectory(dir); err == nil {
|
||||
t.Fatal("failed cleanup accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_MCPFilesystemExecutableProfileValidates(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioMCPFilesystem, true, true, true)
|
||||
if err := ValidateDirectory(dir); err != nil {
|
||||
t.Fatalf("validate mcp-filesystem evidence: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_NoCredentialsInDirectoryRejectsMissingPathAndFiles(t *testing.T) {
|
||||
if err := ValidateDirectory(filepath.Join(t.TempDir(), "missing")); err == nil {
|
||||
t.Fatal("missing evidence directory accepted")
|
||||
}
|
||||
for name, dir := range map[string]string{"empty": t.TempDir(), "incomplete": t.TempDir()} {
|
||||
if name == "incomplete" {
|
||||
writeEvidenceFile(t, dir, "metadata.json", `{ "ok": true }`)
|
||||
}
|
||||
if err := ValidateDirectory(dir); err == nil {
|
||||
t.Fatalf("%s evidence directory accepted", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_NoCredentialsInDirectoryRejectsCredentialsAndMalformedDocuments(t *testing.T) {
|
||||
tests := []struct{ name, results, junit string }{
|
||||
{"credentials", `{ "token": "secret-value" }`, `<testsuite></testsuite>`},
|
||||
{"malformed json", `{ malformed`, `<testsuite></testsuite>`},
|
||||
{"malformed xml", `{ "ok": true }`, `<testsuite>`},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writeEvidenceFile(t, dir, "metadata.json", `{ "ok": true }`)
|
||||
writeEvidenceFile(t, dir, "results.json", test.results)
|
||||
writeEvidenceFile(t, dir, "junit.xml", test.junit)
|
||||
if err := ValidateDirectory(dir); err == nil {
|
||||
t.Fatal("invalid evidence accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func writeEvidenceFile(t *testing.T, resultsDir, name, content string) {
|
||||
t.Helper()
|
||||
if err := os.Chmod(resultsDir, 0o700); err != nil {
|
||||
t.Fatalf("secure evidence root: %v", err)
|
||||
}
|
||||
path := filepath.Join(resultsDir, name)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
t.Fatalf("create evidence parent: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatalf("write evidence file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeJSONEvidenceFile(t *testing.T, resultsDir, name string, value any) {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal %s: %v", name, err)
|
||||
}
|
||||
writeEvidenceFile(t, resultsDir, name, string(data))
|
||||
}
|
||||
|
||||
func validMetadata(t *testing.T, resultsDir, scenarioName string) Metadata {
|
||||
t.Helper()
|
||||
profile, err := contract.ProfileByName("pr-full")
|
||||
if err != nil {
|
||||
t.Fatalf("profile: %v", err)
|
||||
}
|
||||
paths, err := contract.NewPaths("/src/nezha", "/src/agent", resultsDir)
|
||||
if err != nil {
|
||||
t.Fatalf("paths: %v", err)
|
||||
}
|
||||
scenarioValue, err := contract.NewScenario(scenarioName)
|
||||
if err != nil {
|
||||
t.Fatalf("scenario: %v", err)
|
||||
}
|
||||
metadata, err := NewMetadata(MetadataInput{Profile: profile, Seed: contract.DefaultSeed, Paths: paths, ResourceBudget: contract.DefaultResourceBudget(), Scenarios: []contract.Scenario{scenarioValue}, StartedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), EvidenceFiles: EvidenceFiles()})
|
||||
if err != nil {
|
||||
t.Fatalf("metadata: %v", err)
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
func writeExecutableEvidence(t *testing.T, dir, profileName, scenarioName string, matchingJUnit, matchingScenario, writeCleanup bool) {
|
||||
t.Helper()
|
||||
metadata := validMetadata(t, dir, scenarioName)
|
||||
metadata.Profile.Name = profileName
|
||||
writeJSONEvidenceFile(t, dir, "metadata.json", metadata)
|
||||
resultsScenario := scenarioName
|
||||
if !matchingScenario {
|
||||
resultsScenario = contract.ScenarioMetadata
|
||||
}
|
||||
results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: resultsScenario, Passed: true, Assertions: []Assertion{{Name: "fixture", Passed: true}}}}}
|
||||
writeJSONEvidenceFile(t, dir, "results.json", results)
|
||||
junit, err := JUnit(results)
|
||||
if err != nil {
|
||||
t.Fatalf("JUnit: %v", err)
|
||||
}
|
||||
if !matchingJUnit {
|
||||
junit = []byte(`<testsuite name="pr-full" tests="1" failures="1"><testcase name="other"><failure message="bad"></failure></testcase></testsuite>`)
|
||||
}
|
||||
writeEvidenceFile(t, dir, "junit.xml", string(junit))
|
||||
if writeCleanup {
|
||||
writeEvidenceFile(t, dir, "cleanup.json", fmt.Sprintf(`{"passed":true,"scenario":%q,"finished_at":"2026-01-02T03:04:05Z"}`, scenarioName))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
)
|
||||
|
||||
func TestEvidence_Redaction(t *testing.T) {
|
||||
input := `"Authorization":"Basic authorization-secret" JWT=eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.signature PAT=pat-secret agent_secret_key=agent-secret client_secret=config-secret https://local/mcp/upload/path-secret?token=query-secret`
|
||||
redacted := Redact(input)
|
||||
for _, secret := range []string{"authorization-secret", "eyJhbGciOiJIUzI1NiJ9", "pat-secret", "agent-secret", "config-secret", "path-secret", "query-secret"} {
|
||||
if strings.Contains(redacted, secret) {
|
||||
t.Fatalf("secret survived redaction: %q", secret)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(redacted, "[REDACTED]") {
|
||||
t.Fatalf("redaction marker missing: %q", redacted)
|
||||
}
|
||||
junit, err := JUnit(Results{Profile: "pr-full", Passed: false, Scenarios: []ScenarioResult{{Name: "secret", Passed: false, Assertions: []Assertion{{Name: "secret failure", Passed: false}}, Error: input}}})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal redacted junit: %v", err)
|
||||
}
|
||||
if strings.Contains(string(junit), "authorization-secret") || strings.Contains(string(junit), "path-secret") {
|
||||
t.Fatalf("secret survived JUnit redaction: %s", junit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_Golden(t *testing.T) {
|
||||
profile, err := contract.ProfileByName("pr-full")
|
||||
if err != nil {
|
||||
t.Fatalf("construct profile: %v", err)
|
||||
}
|
||||
paths, err := contract.NewPaths("/src/nezha", "/src/agent", "/tmp/results")
|
||||
if err != nil {
|
||||
t.Fatalf("construct paths: %v", err)
|
||||
}
|
||||
metadata, err := NewMetadata(MetadataInput{
|
||||
Profile: profile,
|
||||
Seed: contract.DefaultSeed,
|
||||
Paths: paths,
|
||||
ResourceBudget: contract.DefaultResourceBudget(),
|
||||
StartedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
|
||||
EvidenceFiles: EvidenceFiles(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("construct metadata: %v", err)
|
||||
}
|
||||
data, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal metadata: %v", err)
|
||||
}
|
||||
const wantJSON = `{"agent_source":"/src/agent","evidence_files":["metadata.json","results.json","junit.xml","dashboard.log","agents/*.log","transfer.json","reconnect.json","stress.json","cleanup.json","step-summary.md"],"load_classification":"regression loads, not capacity claims","nezha_source":"/src/nezha","profile":{"name":"pr-full","job_timeout_seconds":4500,"suite_deadline_seconds":3300,"default_seed":"0x4e5a4841","agent_count":8,"stress_rounds":4,"concurrent_operations":64,"concurrent_sessions_per_kind":4,"transfer_pairs":1,"transfer_bytes":104857600,"dashboard_restart_cycles":1,"iterations":1,"stream_boundary_allowed":40,"stream_boundary_rejected":41},"resource_budget":{"warmup_runs_per_path":1,"baseline_sample_count":5,"end_sample_count":5,"sample_interval_milliseconds":250,"child_process_count_drift":0,"listener_count_drift":0,"non_stdio_fd_count_drift":0,"dashboard_rss_delta_bytes":67108864,"agent_rss_delta_bytes":33554432,"transfer_heap_bytes":16777216},"results_dir":"/tmp/results","scenarios":[],"seed":"0x4e5a4841","started_at":"2026-01-02T03:04:05Z"}`
|
||||
if string(data) != wantJSON {
|
||||
t.Fatalf("metadata golden mismatch\nwant: %s\ngot: %s", wantJSON, data)
|
||||
}
|
||||
|
||||
results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: "metadata", Passed: true, Assertions: []Assertion{{Name: "metadata written", Passed: true}}}}}
|
||||
resultsJSON, err := json.Marshal(results)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal results: %v", err)
|
||||
}
|
||||
const wantResultsJSON = `{"profile":"pr-full","passed":true,"scenarios":[{"name":"metadata","passed":true,"assertions":[{"name":"metadata written","passed":true}]}]}`
|
||||
if string(resultsJSON) != wantResultsJSON {
|
||||
t.Fatalf("results golden mismatch\nwant: %s\ngot: %s", wantResultsJSON, resultsJSON)
|
||||
}
|
||||
junitXML, err := JUnit(results)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal junit: %v", err)
|
||||
}
|
||||
const wantXML = `<testsuite name="pr-full" tests="1" failures="0"><testcase name="metadata"></testcase></testsuite>`
|
||||
if string(junitXML) != wantXML {
|
||||
t.Fatalf("junit golden mismatch\nwant: %s\ngot: %s", wantXML, junitXML)
|
||||
}
|
||||
var parsedJUnit junitSuite
|
||||
if err := xml.Unmarshal(junitXML, &parsedJUnit); err != nil {
|
||||
t.Fatalf("parse junit golden: %v", err)
|
||||
}
|
||||
if parsedJUnit.Tests != 1 || parsedJUnit.Failures != 0 || len(parsedJUnit.Cases) != 1 {
|
||||
t.Fatalf("unexpected parsed junit: %#v", parsedJUnit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_SchemaValidation(t *testing.T) {
|
||||
invalid := Metadata{}
|
||||
if err := invalid.Validate(); err == nil {
|
||||
t.Fatal("invalid metadata accepted")
|
||||
}
|
||||
invalidBudget := ResourceBudgetMetadata{WarmupRunsPerPath: 1, BaselineSampleCount: 5, EndSampleCount: 5, SampleIntervalMilliseconds: 250, DashboardRSSDeltaBytes: 1, AgentRSSDeltaBytes: 1}
|
||||
if err := invalidBudget.Validate(); err == nil {
|
||||
t.Fatal("missing transfer heap threshold accepted")
|
||||
}
|
||||
results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: "failed", Passed: false, Assertions: []Assertion{{Name: "failed assertion", Passed: false}}, Error: "failure"}}}
|
||||
if err := results.Validate(); err == nil {
|
||||
t.Fatal("inconsistent results accepted")
|
||||
}
|
||||
for _, result := range []Results{
|
||||
{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: "../escape", Passed: true}}},
|
||||
{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: "duplicate", Passed: true}, {Name: "duplicate", Passed: true}}},
|
||||
} {
|
||||
if err := result.Validate(); err == nil {
|
||||
t.Fatalf("invalid scenario results accepted: %#v", result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_ResultsRejectsSuccessfulScenarioWithoutAssertions(t *testing.T) {
|
||||
results := Results{Profile: "pr-full", Passed: true, Scenarios: []ScenarioResult{{Name: "registration-config-exec", Passed: true}}}
|
||||
|
||||
if err := results.Validate(); err == nil {
|
||||
t.Fatal("successful executable scenario without assertions accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_ResultsJSONRedactsSecrets(t *testing.T) {
|
||||
results := Results{Profile: "pr-full", Passed: false, Scenarios: []ScenarioResult{
|
||||
{Name: "authorization", Passed: false, Assertions: []Assertion{{Name: "authorization failure", Passed: false}}, Error: "Authorization: Bearer jwt-secret"},
|
||||
{Name: "credential-field", Passed: false, Assertions: []Assertion{{Name: "credential failure", Passed: false}}, Error: "agent_secret=agent-secret"},
|
||||
{Name: "transfer-token", Passed: false, Assertions: []Assertion{{Name: "transfer failure", Passed: false}}, Error: "/mcp/download/path-token?token=query-token"},
|
||||
{Name: "access-token", Passed: false, Assertions: []Assertion{{Name: "access failure", Passed: false}}, Error: "https://local/file?access_token=access-secret"},
|
||||
{Name: "api-key", Passed: false, Assertions: []Assertion{{Name: "api failure", Passed: false}}, Error: "https://local/file?api_key=api-secret"},
|
||||
{Name: "signature", Passed: false, Assertions: []Assertion{{Name: "signature failure", Passed: false}}, Error: "https://local/file?sig=sig-secret&signature=signature-secret&X-Amz-Signature=aws-secret"},
|
||||
}}
|
||||
data, err := MarshalResults(results)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal results: %v", err)
|
||||
}
|
||||
for _, secret := range []string{"jwt-secret", "agent-secret", "path-token", "query-token", "access-secret", "api-secret", "sig-secret", "signature-secret", "aws-secret"} {
|
||||
if strings.Contains(string(data), secret) {
|
||||
t.Fatalf("secret survived results JSON: %q in %s", secret, data)
|
||||
}
|
||||
}
|
||||
directData, err := json.Marshal(results)
|
||||
if err != nil {
|
||||
t.Fatalf("direct marshal results: %v", err)
|
||||
}
|
||||
if strings.Contains(string(directData), "jwt-secret") || strings.Contains(string(directData), "path-token") {
|
||||
t.Fatalf("direct JSON marshal bypassed redaction: %s", directData)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_RedactsXMLTextAttributesAndMalformedFragments(t *testing.T) {
|
||||
input := `<record Authorization="Bearer authorization-secret"><agent_secret_key>agent-secret</agent_secret_key><transfer_token>transfer-secret</transfer_token><password>password-secret</password><credential>credential-secret</credential></record>`
|
||||
redacted := Redact(input)
|
||||
for _, secret := range []string{"authorization-secret", "agent-secret", "transfer-secret", "password-secret", "credential-secret"} {
|
||||
if strings.Contains(redacted, secret) {
|
||||
t.Fatalf("XML secret survived redaction: %q in %s", secret, redacted)
|
||||
}
|
||||
}
|
||||
var parsed struct {
|
||||
XMLName xml.Name `xml:"record"`
|
||||
Secret string `xml:"agent_secret_key"`
|
||||
}
|
||||
if err := xml.Unmarshal([]byte(redacted), &parsed); err != nil {
|
||||
t.Fatalf("redacted XML is not parseable: %v; output=%s", err, redacted)
|
||||
}
|
||||
if parsed.Secret != "[REDACTED]" {
|
||||
t.Fatalf("unexpected redacted element text: %q", parsed.Secret)
|
||||
}
|
||||
results := Results{Profile: "pr-full", Passed: false, Scenarios: []ScenarioResult{{Name: "xml", Passed: false, Assertions: []Assertion{{Name: "xml failure", Passed: false}}, Error: input}}}
|
||||
junit, err := JUnit(results)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal XML evidence: %v", err)
|
||||
}
|
||||
var parsedSuite junitSuite
|
||||
if err := xml.Unmarshal(junit, &parsedSuite); err != nil {
|
||||
t.Fatalf("redacted JUnit is not parseable: %v; output=%s", err, junit)
|
||||
}
|
||||
if strings.Contains(string(junit), "agent-secret") || strings.Contains(string(junit), "credential-secret") {
|
||||
t.Fatalf("XML evidence secret survived: %s", junit)
|
||||
}
|
||||
|
||||
malformed := `<credential>config-secret`
|
||||
malformedRedacted := Redact(malformed)
|
||||
if strings.Contains(malformedRedacted, "config-secret") {
|
||||
t.Fatalf("malformed XML secret survived redaction: %s", malformedRedacted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_RedactsXMLSensitiveAttributesWithWhitespaceAndMalformedTags(t *testing.T) {
|
||||
input := `<record><agent_secret_key value="agent secret with spaces"><credential value=config-secret></credential><password data="password secret with spaces">password text</password></record>`
|
||||
redacted := Redact(input)
|
||||
for _, secret := range []string{"agent secret with spaces", "config-secret", "password secret with spaces", "password text"} {
|
||||
if strings.Contains(redacted, secret) {
|
||||
t.Fatalf("XML attribute or text secret survived redaction: %q in %s", secret, redacted)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(redacted, `<agent_secret_key value="[REDACTED]">`) {
|
||||
t.Fatalf("sensitive XML attribute was not structurally redacted: %s", redacted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_RedactsCredentialAttributesWithSpacesAndTruncatedXML(t *testing.T) {
|
||||
input := `<record credential="credential secret with spaces"><credential value="truncated secret`
|
||||
redacted := Redact(input)
|
||||
for _, secret := range []string{"credential secret with spaces", "truncated secret"} {
|
||||
if strings.Contains(redacted, secret) {
|
||||
t.Fatalf("credential secret survived redaction: %q in %s", secret, redacted)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidence_RedactsSelfClosingSensitiveXMLWithoutBreakingDocument(t *testing.T) {
|
||||
input := `<record><credential value="self-closing secret"/><result>ok</result></record>`
|
||||
redacted := Redact(input)
|
||||
if strings.Contains(redacted, "self-closing secret") {
|
||||
t.Fatalf("self-closing XML secret survived redaction: %s", redacted)
|
||||
}
|
||||
var parsed struct {
|
||||
XMLName xml.Name `xml:"record"`
|
||||
Result string `xml:"result"`
|
||||
}
|
||||
if err := xml.Unmarshal([]byte(redacted), &parsed); err != nil {
|
||||
t.Fatalf("redacted self-closing XML is not parseable: %v; output=%s", err, redacted)
|
||||
}
|
||||
if parsed.Result != "ok" {
|
||||
t.Fatalf("redacted self-closing XML lost sibling content: %#v", parsed)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package evidence
|
||||
|
||||
import "encoding/xml"
|
||||
|
||||
type junitSuite struct {
|
||||
XMLName xml.Name `xml:"testsuite"`
|
||||
Name string `xml:"name,attr"`
|
||||
Tests int `xml:"tests,attr"`
|
||||
Failures int `xml:"failures,attr"`
|
||||
Cases []junitCase `xml:"testcase"`
|
||||
}
|
||||
|
||||
type junitCase struct {
|
||||
Name string `xml:"name,attr"`
|
||||
Failure *junitFailure `xml:"failure,omitempty"`
|
||||
}
|
||||
|
||||
type junitFailure struct {
|
||||
Message string `xml:"message,attr"`
|
||||
}
|
||||
|
||||
func JUnit(results Results) ([]byte, error) {
|
||||
if err := results.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
suite := junitSuite{Name: results.Profile, Tests: len(results.Scenarios)}
|
||||
for _, scenario := range results.Scenarios {
|
||||
caseResult := junitCase{Name: scenario.Name}
|
||||
if !scenario.Passed {
|
||||
suite.Failures++
|
||||
caseResult.Failure = &junitFailure{Message: Redact(scenario.Error)}
|
||||
}
|
||||
suite.Cases = append(suite.Cases, caseResult)
|
||||
}
|
||||
return xml.Marshal(suite)
|
||||
}
|
||||
|
||||
func countFailedScenarios(scenarios []ScenarioResult) int {
|
||||
failed := 0
|
||||
for _, scenario := range scenarios {
|
||||
if !scenario.Passed {
|
||||
failed++
|
||||
}
|
||||
}
|
||||
return failed
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
)
|
||||
|
||||
type MetadataInput struct {
|
||||
Profile contract.Profile
|
||||
Seed contract.Seed
|
||||
Paths contract.Paths
|
||||
ResourceBudget contract.ResourceBudget
|
||||
Scenarios []contract.Scenario
|
||||
Fault contract.Fault
|
||||
StartedAt time.Time
|
||||
EvidenceFiles []string
|
||||
}
|
||||
|
||||
type Metadata struct {
|
||||
AgentSource string `json:"agent_source"`
|
||||
EvidenceFiles []string `json:"evidence_files"`
|
||||
LoadClassification string `json:"load_classification"`
|
||||
NezhaSource string `json:"nezha_source"`
|
||||
Profile ProfileMetadata `json:"profile"`
|
||||
ResourceBudget ResourceBudgetMetadata `json:"resource_budget"`
|
||||
ResultsDir string `json:"results_dir"`
|
||||
Scenarios []string `json:"scenarios"`
|
||||
Seed string `json:"seed"`
|
||||
Fault string `json:"fault,omitempty"`
|
||||
StartedAt string `json:"started_at"`
|
||||
}
|
||||
|
||||
func NewMetadata(input MetadataInput) (Metadata, error) {
|
||||
metadata := Metadata{
|
||||
AgentSource: Redact(input.Paths.AgentSource().String()),
|
||||
EvidenceFiles: append([]string(nil), input.EvidenceFiles...),
|
||||
LoadClassification: "regression loads, not capacity claims",
|
||||
NezhaSource: Redact(input.Paths.NezhaSource().String()),
|
||||
Profile: profileMetadata(input.Profile),
|
||||
ResourceBudget: resourceBudgetMetadata(input.ResourceBudget),
|
||||
ResultsDir: Redact(input.Paths.ResultsDir().String()),
|
||||
Scenarios: scenarioNames(input.Scenarios),
|
||||
Seed: fmt.Sprintf("0x%x", uint64(input.Seed)),
|
||||
Fault: input.Fault.String(),
|
||||
StartedAt: input.StartedAt.UTC().Format(time.RFC3339),
|
||||
}
|
||||
if err := metadata.Validate(); err != nil {
|
||||
return Metadata{}, err
|
||||
}
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
func EvidenceFiles() []string {
|
||||
files := []string{"metadata.json", "results.json", "junit.xml", "dashboard.log", "agents/*.log"}
|
||||
for _, definition := range contract.ScenarioDefinitions() {
|
||||
if name := definition.DedicatedArtifactName(); name != "" {
|
||||
files = append(files, name)
|
||||
}
|
||||
}
|
||||
return append(files, "stress.json", "cleanup.json", "step-summary.md")
|
||||
}
|
||||
|
||||
func FixedEvidenceFiles() []string {
|
||||
files := make([]string, 0, len(EvidenceFiles()))
|
||||
for _, name := range EvidenceFiles() {
|
||||
if !slices.Contains([]rune(name), '*') {
|
||||
files = append(files, name)
|
||||
}
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
func scenarioNames(scenarios []contract.Scenario) []string {
|
||||
names := make([]string, 0, len(scenarios))
|
||||
for _, scenario := range scenarios {
|
||||
names = append(names, scenario.String())
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func (metadata Metadata) Validate() error {
|
||||
if metadata.AgentSource == "" || len(metadata.EvidenceFiles) == 0 || metadata.LoadClassification == "" || metadata.NezhaSource == "" || metadata.ResultsDir == "" || metadata.Seed == "" || metadata.Seed == "0x0" || metadata.StartedAt == "" {
|
||||
return fmt.Errorf("metadata fields are incomplete")
|
||||
}
|
||||
if err := metadata.Profile.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := contract.ProfileByName(metadata.Profile.Name); err != nil {
|
||||
return fmt.Errorf("metadata profile is invalid: %w", err)
|
||||
}
|
||||
if err := metadata.ResourceBudget.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !filepath.IsAbs(metadata.AgentSource) || !filepath.IsAbs(metadata.NezhaSource) || !filepath.IsAbs(metadata.ResultsDir) {
|
||||
return fmt.Errorf("metadata paths must be absolute")
|
||||
}
|
||||
if _, err := time.Parse(time.RFC3339, metadata.StartedAt); err != nil {
|
||||
return fmt.Errorf("metadata start time is invalid: %w", err)
|
||||
}
|
||||
if !slices.Equal(metadata.EvidenceFiles, EvidenceFiles()) {
|
||||
return fmt.Errorf("metadata evidence files are invalid")
|
||||
}
|
||||
seen := make(map[string]struct{}, len(metadata.Scenarios))
|
||||
for _, scenarioName := range metadata.Scenarios {
|
||||
_, err := contract.NewScenario(scenarioName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("metadata scenario is invalid: %w", err)
|
||||
}
|
||||
if _, exists := seen[scenarioName]; exists {
|
||||
return fmt.Errorf("metadata scenario is duplicated")
|
||||
}
|
||||
seen[scenarioName] = struct{}{}
|
||||
}
|
||||
if metadata.Fault != "" {
|
||||
if _, err := contract.NewFault(metadata.Fault); err != nil {
|
||||
return fmt.Errorf("metadata fault is invalid: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (metadata Metadata) MarshalJSON() ([]byte, error) {
|
||||
type metadataWire Metadata
|
||||
redacted := metadataWire(metadata)
|
||||
redacted.AgentSource = Redact(redacted.AgentSource)
|
||||
redacted.NezhaSource = Redact(redacted.NezhaSource)
|
||||
redacted.ResultsDir = Redact(redacted.ResultsDir)
|
||||
return json.Marshal(redacted)
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
)
|
||||
|
||||
func validateDedicatedArtifact(resultsDir string, metadata Metadata, result ScenarioResult) error {
|
||||
definition, err := contract.ScenarioDefinitionByName(result.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch definition.DedicatedArtifact {
|
||||
case contract.DedicatedArtifactTransfer:
|
||||
artifact, err := readJSONFile[transferArtifact](resultsDir, "transfer.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return validateTransferArtifact(metadata, result, artifact)
|
||||
case contract.DedicatedArtifactReconnect:
|
||||
artifact, err := readJSONFile[reconnectArtifact](resultsDir, "reconnect.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return validateReconnectArtifact(metadata, result, artifact)
|
||||
case contract.DedicatedArtifactNone:
|
||||
return nil
|
||||
default:
|
||||
return errors.New("unsupported dedicated artifact kind")
|
||||
}
|
||||
}
|
||||
|
||||
func validateReconnectArtifact(metadata Metadata, result ScenarioResult, artifact reconnectArtifact) error {
|
||||
if err := validateArtifactHeader(metadata, result, artifact.Scenario, artifact.Fault, artifact.Passed, artifact.CleanupOK, artifact.Error); err != nil {
|
||||
return err
|
||||
}
|
||||
switch artifact.Fault {
|
||||
case "":
|
||||
if !artifact.Passed {
|
||||
return errors.New("reconnect success artifact reports failure")
|
||||
}
|
||||
if err := validateReconnectSuccess(artifact.Evidence); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateReconnectSuccessReceipts(artifact.Evidence); err != nil {
|
||||
return err
|
||||
}
|
||||
return validateReconnectFinalEvidence(artifact.Evidence, 2)
|
||||
case contract.FaultDashboardExit:
|
||||
if artifact.Passed || !strings.Contains(artifact.Error, "injected Dashboard exit") {
|
||||
return errors.New("dashboard-exit artifact does not identify the injected failure")
|
||||
}
|
||||
evidence := artifact.Evidence
|
||||
if reconnectPostFaultEvidencePresent(evidence) {
|
||||
return errors.New("dashboard-exit artifact presents stale reconnect success")
|
||||
}
|
||||
if evidence.Lifecycle.DisconnectAt.IsZero() || !evidence.Lifecycle.ReconnectAt.IsZero() || !evidence.Lifecycle.OutsideRootSentinelUnchanged {
|
||||
return errors.New("dashboard-exit lifecycle evidence is invalid")
|
||||
}
|
||||
return validateReconnectFinalEvidence(evidence, 1)
|
||||
default:
|
||||
return errors.New("reconnect artifact has unsupported fault")
|
||||
}
|
||||
}
|
||||
|
||||
func reconnectPostFaultEvidencePresent(evidence reconnectEvidence) bool {
|
||||
return evidence.Runtime.DashboardAfter != (runtimeIdentity{}) || evidence.Runtime.AgentAfter != (runtimeIdentity{}) ||
|
||||
evidence.Runtime.StateGenerationBeforeAgentRestart != 0 || evidence.Runtime.StateGenerationAfterAgentRestart != 0 ||
|
||||
evidence.Identity.DashboardConfigUnchanged || evidence.Identity.AgentConfigUnchanged || evidence.Identity.DashboardFixtureUnchanged || evidence.Identity.ClientsRecreated || evidence.Identity.BootstrapRecreated ||
|
||||
!evidence.Lifecycle.ReconnectAt.IsZero() || evidence.Lifecycle.ReconnectInterval != 0 || len(evidence.Lifecycle.DashboardReceipts) != 0 || len(evidence.Lifecycle.AgentReceipts) != 0 ||
|
||||
evidence.Lifecycle.StaleGenerationReceipts != 0 || evidence.Lifecycle.DuplicateTaskIDs != 0 || evidence.Lifecycle.LostResultIDs != 0 ||
|
||||
evidence.Observation.ServerID != 0 || evidence.Observation.UUID != "" || evidence.Observation.OldGeneration != 0 || evidence.Observation.NewGeneration != 0 ||
|
||||
!evidence.Observation.DisconnectAt.IsZero() || !evidence.Observation.ReconnectAt.IsZero() || len(evidence.Observation.TaskIDs) != 0 || len(evidence.Observation.ResultIDs) != 0 || evidence.Observation.PostReconnect || evidence.Observation.AgentRestarted
|
||||
}
|
||||
|
||||
func validateReconnectSuccess(evidence reconnectEvidence) error {
|
||||
observation := evidence.Observation
|
||||
if observation.ServerID == 0 || observation.UUID == "" || observation.OldGeneration == 0 || observation.NewGeneration <= observation.OldGeneration || observation.DisconnectAt.IsZero() || !observation.ReconnectAt.After(observation.DisconnectAt) || len(observation.TaskIDs) != 5 || !slices.Equal(observation.TaskIDs, observation.ResultIDs) || !observation.PostReconnect || !observation.AgentRestarted {
|
||||
return errors.New("reconnect observation evidence is invalid")
|
||||
}
|
||||
if evidence.Runtime.DashboardAfter.Generation <= evidence.Runtime.DashboardBefore.Generation || evidence.Runtime.DashboardAfter.PID == evidence.Runtime.DashboardBefore.PID || evidence.Runtime.AgentAfter.Generation <= evidence.Runtime.AgentBefore.Generation || evidence.Runtime.AgentAfter.PID == evidence.Runtime.AgentBefore.PID || evidence.Runtime.StateGenerationAfterAgentRestart <= evidence.Runtime.StateGenerationBeforeAgentRestart {
|
||||
return errors.New("reconnect runtime generations are invalid")
|
||||
}
|
||||
identity := evidence.Identity
|
||||
if identity.ServerID != observation.ServerID || identity.UUID != observation.UUID || !identity.DashboardConfigUnchanged || !identity.AgentConfigUnchanged || !identity.DashboardFixtureUnchanged || !identity.ClientsRecreated || !identity.BootstrapRecreated {
|
||||
return errors.New("reconnect identity evidence is invalid")
|
||||
}
|
||||
if evidence.Lifecycle.StaleGenerationReceipts != 0 || evidence.Lifecycle.DuplicateTaskIDs != 0 || evidence.Lifecycle.LostResultIDs != 0 || !evidence.Lifecycle.OutsideRootSentinelUnchanged {
|
||||
return errors.New("reconnect lifecycle accounting is invalid")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateReconnectSuccessReceipts(evidence reconnectEvidence) error {
|
||||
if len(evidence.Lifecycle.DashboardReceipts) != 3 || len(evidence.Lifecycle.AgentReceipts) != 2 || len(evidence.Observation.TaskIDs) != 5 {
|
||||
return errors.New("reconnect receipt accounting is not exact")
|
||||
}
|
||||
if evidence.Lifecycle.DisconnectAt.IsZero() || evidence.Lifecycle.ReconnectAt.IsZero() || evidence.Lifecycle.ReconnectAt.Sub(evidence.Lifecycle.DisconnectAt) != evidence.Lifecycle.ReconnectInterval {
|
||||
return errors.New("reconnect timestamp interval is inconsistent")
|
||||
}
|
||||
if !evidence.Observation.DisconnectAt.Equal(evidence.Lifecycle.DisconnectAt) || !evidence.Observation.ReconnectAt.Equal(evidence.Lifecycle.ReconnectAt) {
|
||||
return errors.New("reconnect lifecycle and observation timestamps differ")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateReconnectFinalEvidence(evidence reconnectEvidence, expectedProcessCount int) error {
|
||||
if !evidence.AgentCleanup.Passed || evidence.AgentCleanup.Forced || !evidence.DashboardCleanup.Passed || evidence.DashboardCleanup.Forced {
|
||||
return errors.New("reconnect cleanup receipts failed")
|
||||
}
|
||||
if len(evidence.AgentCleanup.Processes) != expectedProcessCount || len(evidence.DashboardCleanup.Processes) != expectedProcessCount {
|
||||
return errors.New("reconnect cleanup receipt process count is invalid")
|
||||
}
|
||||
for _, path := range []string{evidence.Fixture.Dashboard.WorkspaceRoot, evidence.Fixture.AgentRoot, evidence.Fixture.Dashboard.ConfigPath, evidence.Fixture.AgentConfigPath, evidence.Fixture.Dashboard.BinaryPath, evidence.Fixture.AgentBinaryPath} {
|
||||
if path == "" || !filepath.IsAbs(path) {
|
||||
return errors.New("reconnect fixture paths are invalid")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type redactionRule struct {
|
||||
pattern *regexp.Regexp
|
||||
replace string
|
||||
}
|
||||
|
||||
var redactionRules = []redactionRule{
|
||||
{pattern: regexp.MustCompile(`(?i)(["']?authorization["']?\s*[:=]\s*["']?)[^"'\r\n,}]+(["']?)`), replace: `${1}[REDACTED]${2}`},
|
||||
{pattern: regexp.MustCompile(`(?i)(\b["']?(?:token|access[_-]?token|api[_-]?(?:key|token)|jwt[_-]?secret(?:[_-]?key)?|jwt[_-]?token|pat|agent[_-]?secret(?:[_-]?key)?|client[_-]?secret|handshake[_-]?secret|revert[_-]?handshake[_-]?secret|password|credential|transfer[_-]?token)["']?\s*[:=]\s*["']?)[^"'\s,;&}]+(["']?)`), replace: `${1}[REDACTED]${2}`},
|
||||
{pattern: regexp.MustCompile(`(?i)([?&](?:token|access_token|api_key|jwt|pat|secret|authorization|sig|signature|x-amz-signature)=)[^&#\s]+`), replace: `${1}[REDACTED]`},
|
||||
{pattern: regexp.MustCompile(`(?i)(/mcp/(?:download|upload)/)[A-Za-z0-9._-]+`), replace: `${1}[REDACTED]`},
|
||||
{pattern: regexp.MustCompile(`\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b`), replace: `[REDACTED]`},
|
||||
{pattern: regexp.MustCompile(`\b(?:ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})\b`), replace: `[REDACTED]`},
|
||||
{pattern: regexp.MustCompile(`(?is)(<\s*(?:agent[_-]?secret(?:[_-]?key)?|authorization|transfer[_-]?token|password|credential|config[_-]?secret)\b)[^>]*?(/?)>`), replace: `${1} value="[REDACTED]"${2}>`},
|
||||
{pattern: regexp.MustCompile(`(?is)(<\s*(?:agent[_-]?secret(?:[_-]?key)?|authorization|transfer[_-]?token|password|credential|config[_-]?secret)\b[^>]*>)[^<]*(</\s*(?:agent[_-]?secret(?:[_-]?key)?|authorization|transfer[_-]?token|password|credential|config[_-]?secret)\s*>)`), replace: `${1}[REDACTED]${2}`},
|
||||
{pattern: regexp.MustCompile(`(?is)(<\s*(?:agent[_-]?secret(?:[_-]?key)?|authorization|transfer[_-]?token|password|credential|config[_-]?secret)\b[^>]*>)[^<]*$`), replace: `${1}[REDACTED]`},
|
||||
{pattern: regexp.MustCompile(`(?is)(<\s*(?:agent[_-]?secret(?:[_-]?key)?|authorization|transfer[_-]?token|password|credential|config[_-]?secret)\b)[^>]*$`), replace: `${1} value="[REDACTED]"`},
|
||||
}
|
||||
|
||||
var sensitiveXMLAttribute = regexp.MustCompile(`(?i)(\b(?:agent[_-]?secret(?:[_-]?key)?|authorization|transfer[_-]?token|password|credential|config[_-]?secret)\b\s*=\s*)(?:"[^"]*"|'[^']*'|"[^\r\n>]*|'[^'\r\n>]*'|[^\s/>][^/>]*)`)
|
||||
|
||||
func Redact(input string) string {
|
||||
redacted := sensitiveXMLAttribute.ReplaceAllStringFunc(input, func(attribute string) string {
|
||||
separator := strings.IndexByte(attribute, '=')
|
||||
if separator < 0 {
|
||||
return attribute
|
||||
}
|
||||
prefix := attribute[:separator+1]
|
||||
return prefix + "\"[REDACTED]\""
|
||||
})
|
||||
for _, rule := range redactionRules {
|
||||
redacted = rule.pattern.ReplaceAllString(redacted, rule.replace)
|
||||
}
|
||||
return redacted
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
)
|
||||
|
||||
func TestEvidence_RegistryDefinitionsDriveManifestAndProfiles(t *testing.T) {
|
||||
manifest := EvidenceFiles()
|
||||
for _, definition := range contract.ScenarioDefinitions() {
|
||||
dedicatedName := definition.DedicatedArtifactName()
|
||||
if dedicatedName != "" && !slices.Contains(manifest, dedicatedName) {
|
||||
t.Fatalf("dedicated artifact %q absent from manifest", dedicatedName)
|
||||
}
|
||||
metadata := validMetadata(t, t.TempDir(), definition.Name)
|
||||
profile, err := currentProfile(metadata)
|
||||
if err != nil {
|
||||
t.Fatalf("profile for %q: %v", definition.Name, err)
|
||||
}
|
||||
if profile.DedicatedFile != dedicatedName {
|
||||
t.Fatalf("scenario %q dedicated file=%q want=%q", definition.Name, profile.DedicatedFile, dedicatedName)
|
||||
}
|
||||
if definition.Execution == contract.ScenarioExecutionMetadata && profile.Executable {
|
||||
t.Fatal("metadata profile unexpectedly executable")
|
||||
}
|
||||
if definition.Execution != contract.ScenarioExecutionMetadata && !profile.Executable {
|
||||
t.Fatalf("scenario %q profile is not executable", definition.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package evidence
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
|
||||
)
|
||||
|
||||
type ScenarioResult struct {
|
||||
Name string `json:"name"`
|
||||
Passed bool `json:"passed"`
|
||||
Assertions []Assertion `json:"assertions,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type Assertion struct {
|
||||
Name string `json:"name"`
|
||||
Passed bool `json:"passed"`
|
||||
Details string `json:"details,omitempty"`
|
||||
}
|
||||
|
||||
type Results struct {
|
||||
Profile string `json:"profile"`
|
||||
Passed bool `json:"passed"`
|
||||
Scenarios []ScenarioResult `json:"scenarios"`
|
||||
}
|
||||
|
||||
func (results Results) Validate() error {
|
||||
if results.Profile == "" || len(results.Scenarios) == 0 {
|
||||
return fmt.Errorf("results fields are incomplete")
|
||||
}
|
||||
if _, err := contract.ProfileByName(results.Profile); err != nil {
|
||||
return fmt.Errorf("results profile is invalid: %w", err)
|
||||
}
|
||||
allPassed := true
|
||||
seenScenarios := make(map[string]struct{}, len(results.Scenarios))
|
||||
for _, scenario := range results.Scenarios {
|
||||
if _, err := contract.NewScenario(scenario.Name); err != nil {
|
||||
return fmt.Errorf("scenario name is invalid: %w", err)
|
||||
}
|
||||
if _, exists := seenScenarios[scenario.Name]; exists {
|
||||
return fmt.Errorf("scenario name is duplicated")
|
||||
}
|
||||
seenScenarios[scenario.Name] = struct{}{}
|
||||
if len(scenario.Assertions) == 0 {
|
||||
return fmt.Errorf("scenario must contain assertions")
|
||||
}
|
||||
assertionsPassed := true
|
||||
seenAssertions := make(map[string]struct{}, len(scenario.Assertions))
|
||||
for _, assertion := range scenario.Assertions {
|
||||
if assertion.Name == "" {
|
||||
return fmt.Errorf("assertion name is required")
|
||||
}
|
||||
if _, exists := seenAssertions[assertion.Name]; exists {
|
||||
return fmt.Errorf("assertion name is duplicated")
|
||||
}
|
||||
seenAssertions[assertion.Name] = struct{}{}
|
||||
assertionsPassed = assertionsPassed && assertion.Passed
|
||||
}
|
||||
if scenario.Passed != assertionsPassed {
|
||||
return fmt.Errorf("scenario pass state is inconsistent with assertions")
|
||||
}
|
||||
if !scenario.Passed {
|
||||
if scenario.Error == "" {
|
||||
return fmt.Errorf("failed scenario error is required")
|
||||
}
|
||||
allPassed = false
|
||||
} else if scenario.Error != "" {
|
||||
return fmt.Errorf("passed scenario cannot contain an error")
|
||||
}
|
||||
}
|
||||
if results.Passed != allPassed {
|
||||
return fmt.Errorf("results pass state is inconsistent")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func MarshalResults(results Results) ([]byte, error) {
|
||||
if err := results.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(results)
|
||||
}
|
||||
|
||||
func (results Results) MarshalJSON() ([]byte, error) {
|
||||
type resultsWire Results
|
||||
redacted := resultsWire{Profile: results.Profile, Passed: results.Passed, Scenarios: make([]ScenarioResult, 0, len(results.Scenarios))}
|
||||
for _, scenario := range results.Scenarios {
|
||||
assertions := make([]Assertion, 0, len(scenario.Assertions))
|
||||
for _, assertion := range scenario.Assertions {
|
||||
assertions = append(assertions, Assertion{Name: assertion.Name, Passed: assertion.Passed, Details: Redact(assertion.Details)})
|
||||
}
|
||||
redacted.Scenarios = append(redacted.Scenarios, ScenarioResult{Name: scenario.Name, Passed: scenario.Passed, Assertions: assertions, Error: Redact(scenario.Error)})
|
||||
}
|
||||
return json.Marshal(redacted)
|
||||
}
|
||||
|
||||
func scenarioResultNames(scenarios []ScenarioResult) []string {
|
||||
names := make([]string, 0, len(scenarios))
|
||||
for _, scenario := range scenarios {
|
||||
names = append(names, scenario.Name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
Reference in New Issue
Block a user