fix(agentcompat): snapshot evidence through rooted handles

Co-authored-by: naiba/CloudCode <hi+cloudcode@nai.ba>

integration/agentcompat/internal/evidence/directory_scan.go

@@ -5,6 +5,8 @@ import (
 	"encoding/xml"
 	"errors"
 	"fmt"
+	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
@@ -15,11 +17,15 @@ const (
 	maxEvidenceBytes = 32 << 20
 )
 
-func scanDirectory(resultsDir string) (map[string]os.FileInfo, error) {
-	if strings.TrimSpace(resultsDir) == "" {
-		return nil, errors.New("evidence directory is required")
+type evidenceFile struct {
+	info os.FileInfo
+	data []byte
 }
-	info, err := os.Lstat(resultsDir)
+
+type evidenceSnapshot map[string]evidenceFile
+
+func scanDirectory(root *os.Root) (evidenceSnapshot, error) {
+	info, err := root.Lstat(".")
 	if err != nil {
 		return nil, fmt.Errorf("stat evidence directory: %w", err)
 	}
@@ -29,61 +35,65 @@ func scanDirectory(resultsDir string) (map[string]os.FileInfo, error) {
 	if err := validateEvidenceDirectoryMode(info); err != nil {
 		return nil, err
 	}
-	seen := make(map[string]os.FileInfo)
+	seen := make(evidenceSnapshot)
 	var totalBytes int64
-	err = filepath.WalkDir(resultsDir, func(path string, entry os.DirEntry, walkErr error) error {
+	err = fs.WalkDir(root.FS(), ".", func(path string, entry fs.DirEntry, walkErr error) error {
 		if walkErr != nil {
 			return walkErr
 		}
-		if path == resultsDir {
+		if path == "." {
 			return nil
 		}
 		if entry.IsDir() {
-			relative, err := filepath.Rel(resultsDir, path)
-			if err != nil {
-				return err
-			}
-			if relative != "agents" {
-				return fmt.Errorf("evidence path is not allowed: %s", relative)
+			if path != "agents" {
+				return fmt.Errorf("evidence path is not allowed: %s", path)
 			}
 			return nil
 		}
 		if entry.Type()&os.ModeSymlink != 0 {
 			return fmt.Errorf("evidence symlink is not allowed: %s", path)
 		}
-		if !entry.Type().IsRegular() {
+		if !allowedEvidencePath(path) {
+			return fmt.Errorf("evidence path is not allowed: %s", path)
+		}
+		file, err := root.Open(path)
+		if err != nil {
+			return fmt.Errorf("open evidence file %s: %w", path, err)
+		}
+		fileInfo, err := file.Stat()
+		if err != nil {
+			_ = file.Close()
+			return fmt.Errorf("stat evidence file %s: %w", path, err)
+		}
+		if !fileInfo.Mode().IsRegular() {
+			_ = file.Close()
 			return fmt.Errorf("evidence file is not regular: %s", path)
 		}
-		relative, err := filepath.Rel(resultsDir, path)
-		if err != nil {
+		if err := validateEvidenceFileMode(fileInfo, path); err != nil {
+			_ = file.Close()
 			return err
 		}
-		fileInfo, err := entry.Info()
-		if err != nil {
-			return err
-		}
-		if !allowedEvidencePath(relative) {
-			return fmt.Errorf("evidence path is not allowed: %s", relative)
-		}
-		if err := validateEvidenceFileMode(fileInfo, relative); err != nil {
-			return err
-		}
-		seen[relative] = fileInfo
-		if len(seen) > maxEvidenceFiles {
+		if len(seen)+1 > maxEvidenceFiles {
+			_ = file.Close()
 			return errors.New("too many evidence files")
 		}
-		totalBytes += fileInfo.Size()
-		if totalBytes > maxEvidenceBytes {
+		remaining := int64(maxEvidenceBytes) - totalBytes
+		data, readErr := io.ReadAll(io.LimitReader(file, remaining+1))
+		closeErr := file.Close()
+		if readErr != nil {
+			return fmt.Errorf("read evidence file %s: %w", path, readErr)
+		}
+		if closeErr != nil {
+			return fmt.Errorf("close evidence file %s: %w", path, closeErr)
+		}
+		if int64(len(data)) > remaining {
 			return errors.New("evidence files exceed size limit")
 		}
-		data, err := os.ReadFile(path)
-		if err != nil {
-			return err
-		}
+		totalBytes += int64(len(data))
 		if Redact(string(data)) != string(data) {
 			return fmt.Errorf("credential detected in evidence file: %s", path)
 		}
-		switch filepath.Ext(path) {
+		switch extension(path) {
 		case ".json":
 			if !json.Valid(data) {
 				return fmt.Errorf("invalid JSON evidence file: %s", path)
@@ -94,6 +104,7 @@ func scanDirectory(resultsDir string) (map[string]os.FileInfo, error) {
 				return fmt.Errorf("invalid XML evidence file: %s: %w", path, err)
 			}
 		}
+		seen[path] = evidenceFile{info: fileInfo, data: data}
 		return nil
 	})
 	return seen, err
@@ -120,14 +131,22 @@ func allowedEvidencePath(relative string) bool {
 	return true
 }
 
-func readJSONFile[T any](resultsDir, name string) (T, error) {
+func readJSONFile[T any](files evidenceSnapshot, name string) (T, error) {
 	var value T
-	data, err := os.ReadFile(filepath.Join(resultsDir, name))
-	if err != nil {
-		return value, fmt.Errorf("read %s: %w", name, err)
+	file, exists := files[name]
+	if !exists {
+		return value, fmt.Errorf("read %s: evidence snapshot is missing", name)
 	}
-	if err := json.Unmarshal(data, &value); err != nil {
+	if err := json.Unmarshal(file.data, &value); err != nil {
 		return value, fmt.Errorf("parse %s: %w", name, err)
 	}
 	return value, nil
 }
+
+func extension(path string) string {
+	index := strings.LastIndexByte(path, '.')
+	if index < 0 {
+		return ""
+	}
+	return path[index:]
+}

integration/agentcompat/internal/evidence/directory_security_test.go

@@ -38,3 +38,73 @@ func TestEvidence_ValidateDirectoryRejectsUnexpectedPaths(t *testing.T) {
 		})
 	}
 }
+
+func TestEvidence_ValidateDirectoryRejectsSymlinkReplacementOutsideRoot(t *testing.T) {
+	// Given
+	dir := t.TempDir()
+	writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, true, true, true)
+	outside := filepath.Join(t.TempDir(), "outside-metadata.json")
+	if err := os.WriteFile(outside, []byte(`{"profile":"outside"}`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	metadata := filepath.Join(dir, "metadata.json")
+	if err := os.Remove(metadata); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Symlink(outside, metadata); err != nil {
+		t.Fatal(err)
+	}
+
+	// When
+	err := ValidateDirectory(dir)
+
+	// Then
+	if err == nil || !strings.Contains(err.Error(), "symlink") {
+		t.Fatalf("symlink replacement error=%v", err)
+	}
+}
+
+func TestEvidence_ValidateDirectoryRejectsResultsRootSymlink(t *testing.T) {
+	// Given
+	resultsDir := t.TempDir()
+	writeExecutableEvidence(t, resultsDir, "pr-full", contract.ScenarioRegistrationConfigExec, true, true, true)
+	symlink := filepath.Join(t.TempDir(), "results-link")
+	if err := os.Symlink(resultsDir, symlink); err != nil {
+		t.Fatal(err)
+	}
+
+	// When
+	err := ValidateDirectory(symlink)
+
+	// Then
+	if err == nil || !strings.Contains(err.Error(), "must be a directory") {
+		t.Fatalf("results root symlink error=%v", err)
+	}
+}
+
+func TestEvidence_ValidateSnapshotUsesCapturedBytesAfterDiskMutation(t *testing.T) {
+	// Given
+	dir := t.TempDir()
+	writeExecutableEvidence(t, dir, "pr-full", contract.ScenarioRegistrationConfigExec, true, true, true)
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer root.Close()
+	snapshot, err := scanDirectory(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	writeEvidenceFile(t, dir, "metadata.json", `{}`)
+	writeEvidenceFile(t, dir, "results.json", `{}`)
+	writeEvidenceFile(t, dir, "junit.xml", `<testsuite name="replaced"></testsuite>`)
+	writeEvidenceFile(t, dir, "cleanup.json", `{}`)
+
+	// When
+	err = validateSnapshot(snapshot)
+
+	// Then
+	if err != nil {
+		t.Fatalf("captured evidence was changed by later disk mutation: %v", err)
+	}
+}

integration/agentcompat/internal/evidence/directory_validation.go

@@ -5,8 +5,8 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"path/filepath"
 	"slices"
+	"strings"
 	"time"
 
 	"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
@@ -66,14 +66,33 @@ func currentProfile(metadata Metadata) (currentEvidenceProfile, error) {
 }
 
 func ValidateDirectory(resultsDir string) error {
-	files, err := scanDirectory(resultsDir)
+	if strings.TrimSpace(resultsDir) == "" {
+		return errors.New("evidence directory is required")
+	}
+	info, err := os.Lstat(resultsDir)
+	if err != nil {
+		return fmt.Errorf("stat evidence directory: %w", err)
+	}
+	if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
+		return errors.New("evidence path must be a directory")
+	}
+	root, err := os.OpenRoot(resultsDir)
+	if err != nil {
+		return fmt.Errorf("open evidence directory: %w", err)
+	}
+	defer root.Close()
+	files, err := scanDirectory(root)
 	if err != nil {
 		return err
 	}
+	return validateSnapshot(files)
+}
+
+func validateSnapshot(files evidenceSnapshot) error {
 	if len(files) == 0 {
 		return errors.New("evidence directory contains no files")
 	}
-	metadata, err := readJSONFile[Metadata](resultsDir, "metadata.json")
+	metadata, err := readJSONFile[Metadata](files, "metadata.json")
 	if err != nil {
 		return err
 	}
@@ -95,7 +114,7 @@ func ValidateDirectory(resultsDir string) error {
 	if !profile.Executable {
 		return nil
 	}
-	results, err := readJSONFile[Results](resultsDir, "results.json")
+	results, err := readJSONFile[Results](files, "results.json")
 	if err != nil {
 		return err
 	}
@@ -105,10 +124,10 @@ func ValidateDirectory(resultsDir string) error {
 	if results.Profile != metadata.Profile.Name || !slices.Equal(metadata.Scenarios, scenarioResultNames(results.Scenarios)) {
 		return errors.New("metadata and results do not agree")
 	}
-	if err := validateJUnit(resultsDir, results); err != nil {
+	if err := validateJUnit(files, results); err != nil {
 		return err
 	}
-	cleanup, err := readJSONFile[cleanupEvidence](resultsDir, "cleanup.json")
+	cleanup, err := readJSONFile[cleanupEvidence](files, "cleanup.json")
 	if err != nil {
 		return err
 	}
@@ -126,7 +145,7 @@ func ValidateDirectory(resultsDir string) error {
 		if err := validateScenarioAssertions(results.Scenarios[0], definition.Assertions(metadata.Fault)); err != nil {
 			return err
 		}
-		return validateDedicatedArtifact(resultsDir, metadata, results.Scenarios[0])
+		return validateDedicatedArtifact(files, metadata, results.Scenarios[0])
 	}
 	return nil
 }
@@ -143,7 +162,7 @@ func validateScenarioAssertions(result ScenarioResult, expected []contract.Asser
 	return nil
 }
 
-func rejectStaleDedicatedFiles(files map[string]os.FileInfo, expected string) error {
+func rejectStaleDedicatedFiles(files evidenceSnapshot, expected string) error {
 	for _, name := range []string{"transfer.json", "reconnect.json"} {
 		if _, exists := files[name]; exists && name != expected {
 			return fmt.Errorf("stale or wrong dedicated evidence file: %s", name)
@@ -152,13 +171,13 @@ func rejectStaleDedicatedFiles(files map[string]os.FileInfo, expected string) er
 	return nil
 }
 
-func validateJUnit(resultsDir string, results Results) error {
-	data, err := os.ReadFile(filepath.Join(resultsDir, "junit.xml"))
-	if err != nil {
-		return fmt.Errorf("read JUnit evidence: %w", err)
+func validateJUnit(files evidenceSnapshot, results Results) error {
+	file, exists := files["junit.xml"]
+	if !exists {
+		return errors.New("read JUnit evidence: evidence snapshot is missing")
 	}
 	var suite junitSuite
-	if err := xml.Unmarshal(data, &suite); err != nil {
+	if err := xml.Unmarshal(file.data, &suite); err != nil {
 		return fmt.Errorf("parse JUnit evidence: %w", err)
 	}
 	if suite.Name != results.Profile || suite.Tests != len(results.Scenarios) || suite.Failures != countFailedScenarios(results.Scenarios) || len(suite.Cases) != len(results.Scenarios) {

integration/agentcompat/internal/evidence/reconnect_artifact_validation.go

@@ -9,20 +9,20 @@ import (
 	"github.com/nezhahq/nezha/integration/agentcompat/internal/contract"
 )
 
-func validateDedicatedArtifact(resultsDir string, metadata Metadata, result ScenarioResult) error {
+func validateDedicatedArtifact(files evidenceSnapshot, metadata Metadata, result ScenarioResult) error {
 	definition, err := contract.ScenarioDefinitionByName(result.Name)
 	if err != nil {
 		return err
 	}
 	switch definition.DedicatedArtifact {
 	case contract.DedicatedArtifactTransfer:
-		artifact, err := readJSONFile[transferArtifact](resultsDir, "transfer.json")
+		artifact, err := readJSONFile[transferArtifact](files, "transfer.json")
 		if err != nil {
 			return err
 		}
 		return validateTransferArtifact(metadata, result, artifact)
 	case contract.DedicatedArtifactReconnect:
-		artifact, err := readJSONFile[reconnectArtifact](resultsDir, "reconnect.json")
+		artifact, err := readJSONFile[reconnectArtifact](files, "reconnect.json")
 		if err != nil {
 			return err
 		}