test: add integration tests for commands

.github/workflows/build.yaml

@@ -1,12 +1,60 @@
 name: app
 on:
   push:
+    branches:
+      - '*'
     tags:
       - v*.*.*
 
 jobs:
+  test:
+    permissions:
+      contents: read
+      id-token: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        # match Kubectl version to kind cluster version used in scaffolding setup
+        # https://github.com/sigstore/scaffolding/blob/8f5907d5dc59c1af0e3c757486105179eab3301a/hack/setup-kind.sh#L80-L81
+      - name: Install kubectl
+        run: |
+          curl -LO "https://dl.k8s.io/release/v1.25.0/bin/linux/amd64/kubectl"
+          chmod +x kubectl
+          mkdir -p $HOME/.bin/kubectl
+          mv kubectl $HOME/.bin/kubectl/
+          echo "$HOME/.bin/kubectl" >> $GITHUB_PATH
+      - name: Detect Workflow
+        id: detect-workflow
+        uses: slsa-framework/slsa-github-generator/.github/actions/detect-workflow-js@v1.6.0
+      - name: Sigstore Setup
+        uses: sigstore/scaffolding/actions/setup@v0.6.4
+        with:
+          k8s-version: "v1.25.x"
+          version: "v0.6.4"
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: 'go.mod'
+      - name: Download Deps
+        run: go mod download
+      - name: Build
+        run: make build
+      - name: Move TUF root
+        run: mv root.json test/
+      - name: Run Tests
+        env:
+          KEYLESS_ISSUER: "https://token.actions.githubusercontent.com"
+          KEYLESS_SUBJECT: ${{ github.server_url }}/${{ github.repository }}/${{ steps.detect-workflow.outputs.workflow }}@${{ steps.detect-workflow.outputs.ref }}
+          # the attestation tool expects a value for GITHUB_TOKEN, but interactions with the GitHub API are replayed from fixtures
+          GITHUB_TOKEN: "invalid"
+          REGISTRY_URL: "registry.local:5000"
+        run: make test
+
   release:
+    if: startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-latest
+    needs: [test]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -15,7 +63,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: "1.20"
+          go-version-file: 'go.mod'
       - name: GoReleaser
         uses: goreleaser/goreleaser-action@v4
         with:

.gitignore

@@ -1,4 +1,5 @@
 .idea/
 bundle.tar.gz
 *.predicate.json
-attestation
+attestation
+test/root.json

Makefile

@@ -1,6 +1,6 @@
 MAKEFLAGS += --silent
 
-BUILD := $(shell git describe --dirty)
+BUILD := $(shell git describe --always --dirty)
 LDFLAGS=-ldflags "-X github.com/liatrio/gh-trusted-builds-attestations/build.Version=$(BUILD)"
 
 .PHONY: build
@@ -31,4 +31,16 @@ version:
 
 .PHONY: help
 help:
-	go run $(LDFLAGS) main.go help
+	go run $(LDFLAGS) main.go help
+
+.PHONY: test-setup
+test-setup:
+	./hack/test-setup.sh
+
+.PHONY: test
+test:
+	./hack/test-run.sh
+
+.PHONY:
+test-teardown:
+	./hack/test-teardown.sh

README.md

@@ -144,3 +144,78 @@ In order to build the project, you'll need Go 1.20+.
 Export any environment variables as described for the command being tested.
 
 Each attestor should have a [Makefile](Makefile) target to invoke it, like this: `make github-pull-request`
+
+### Integration Tests
+
+This application includes a suite of integration tests that verify the different attestation commands. In order to run the tests, you'll need to have these tools installed locally:
+
+- [`docker`](https://www.docker.com/products/docker-desktop/)
+- [`kind`](https://kind.sigs.k8s.io/)
+- [`yq`](https://github.com/mikefarah/yq)
+- [`kubectl`](https://kubernetes.io/docs/reference/kubectl/)
+
+First, add the following entries to `/etc/hosts`:
+
+```
+127.0.0.1 registry.local
+127.0.0.1 rekor.rekor-system.svc
+127.0.0.1 fulcio.fulcio-system.svc
+127.0.0.1 ctlog.ctlog-system.svc
+127.0.0.1 gettoken.default.svc
+127.0.0.1 tuf.tuf-system.svc
+```
+
+Next, run `make test-setup`. This will download resources from the Sigstore [scaffolding repo](https://github.com/sigstore/scaffolding), stand up a kind cluster, and deploy Rekor & Fulcio.
+It will also create a TUF root that's used by the tests. The setup should take 5-10 minutes. It only needs to be run once.
+
+⚠️ WARNING: The tests run the equivalent of `cosign initialize`, meaning that if you have a custom TUF root configured, it will be temporarily overwritten in place of the TUF root created
+by the scaffolding setup. The tests will attempt to save the TUF root in `~/.sigstore-backup` before running, and restore it after. If the tests fail to restore the custom root, you can remove it by running `rm -rf ~/.sigstore` and `mv ~/.sigstore-backup ~/.sigstore`.
+If you're not using a custom TUF root, deleting the `~/.sigstore` directory should suffice.
+
+Next, run `make test` to start the tests. Unfortunately, there's some noise in the output, but you can usually ignore these logs about port-forwarding:
+
+> Handling connection for 8080
+
+as well as the logs about issues with the TUF root metadata:
+
+```
+**Warning** Custom metadata not configured properly for target tsa_intermediate_0.crt.pem, skipping target
+**Warning** Custom metadata not configured properly for target tsa_leaf.crt.pem, skipping target
+```
+
+You'll also see certificates printed in the output from keyless signing, these are generated during the tests:
+
+```
+Successfully verified SCT...
+using ephemeral certificate:
+-----BEGIN CERTIFICATE-----
+MIIExTCCAq2gAwIBAgIUa1P4DGiAjiFev2fx+KCZ2NrK9VMwDQYJKoZIhvcNAQEL
+BQAwfjEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+...
+```
+
+Lastly, once you're done testing, you can run `make test-teardown` to destroy the kind cluster. Optionally, you can also remove the entries from `/etc/hosts` that were added in the first step.
+
+#### GitHub API
+
+The `github-pull-request` attestor uses the GitHub API to populate the attestation.
+In order to avoid depending on the live GitHub.com service, the tests use [`go-vcr`](https://github.com/dnaeon/go-vcr) to replay past responses.
+These responses are stored in `test/fixtures/github`, organized by test name.
+
+The fixture data is from [`liatrio/pr-attestation-fixtures`](https://github.com/liatrio/pr-attestation-fixtures).
+If you need to add a fixture for a new scenario, you can make changes in that repository.
+
+Next, set `GITHUB_TOKEN` to a fine-grained personal access token with the following scopes for the `liatrio/pr-attestation-fixtures` repository:
+ - `contents` (read-only)
+ - `metadata` (read-only)
+ - `pull-requests` (read-only)
+
+Finally, change the mode on GitHub API recorder to `recorder.ModeRecordOnce` (if you're adding a new test) or `recorder.ModeReplayWithNewEpisodes` (if you're making changes to an existing test).
+
+```go
+r, err := recorder.NewWithOptions(&recorder.Options{
+    CassetteName:  filepath.Join("fixtures", "github", t.Name()),
+    Mode:          recorder.ModeRecordOnce,
+    RealTransport: oauth2Transport,
+})
+```

cmd/github_pull_request.go

@@ -2,14 +2,15 @@ package cmd
 
 import (
 	"context"
-	"github.com/liatrio/gh-trusted-builds-attestations/internal/attestors"
+
+	"github.com/liatrio/gh-trusted-builds-attestations/internal/attestors/github_pull_request"
 	"github.com/liatrio/gh-trusted-builds-attestations/internal/config"
 )
 
 type GitHubPullRequest struct {
 	ctx      context.Context
 	opts     *config.GitHubPullRequestCommandOptions
-	attestor *attestors.GitHubPullRequestAttestor
+	attestor *github_pull_request.Attestor
 }
 
 func (g *GitHubPullRequest) Is(s string) bool {
@@ -30,7 +31,7 @@ func (g *GitHubPullRequest) Init(ctx context.Context, flags []string) error {
 	}
 	g.opts = opts
 
-	attestor, err := attestors.NewGitHubPullRequestAttestor(ctx, opts)
+	attestor, err := github_pull_request.NewAttestor(ctx, opts)
 	if err != nil {
 		return err
 	}

go.mod

@@ -13,8 +13,10 @@ require (
 	github.com/sigstore/cosign/v2 v2.0.2
 	github.com/sigstore/rekor v1.1.0
 	github.com/sigstore/sigstore v1.6.3
+	github.com/stretchr/testify v1.8.4
 	golang.org/x/oauth2 v0.7.0
 	google.golang.org/protobuf v1.30.0
+	gopkg.in/dnaeon/go-vcr.v3 v3.1.2
 )
 
 require (
@@ -152,6 +154,7 @@ require (
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/sassoftware/relic v7.2.1+incompatible // indirect
 	github.com/segmentio/ksuid v1.0.4 // indirect

go.sum

@@ -832,8 +832,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=
 github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
@@ -1348,6 +1349,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/dnaeon/go-vcr.v3 v3.1.2 h1:F1smfXBqQqwpVifDfUBQG6zzaGjzT+EnVZakrOdr5wA=
+gopkg.in/dnaeon/go-vcr.v3 v3.1.2/go.mod h1:2IMOnnlx9I6u9x+YBsM3tAMx6AlOxnJ0pWxQAzZ79Ag=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=

hack/test-run.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [ -n "${CI-}" ]; then
+    export REKOR_URL=$(kubectl -n rekor-system get ksvc rekor -ojsonpath='{.status.url}')
+    export FULCIO_URL=$(kubectl -n fulcio-system get ksvc fulcio -ojsonpath='{.status.url}')
+    export TUF_MIRROR=$(kubectl -n tuf-system get ksvc tuf -ojsonpath='{.status.url}')
+else
+    echo "Port-forwarding kourier service"
+    kubectl -n kourier-system port-forward service/kourier-internal 8080:80 &
+    KUBECTL_PID=$!
+    trap "echo 'Stopping port-forward' && kill -9 $KUBECTL_PID" EXIT
+
+    echo "Waiting for port-forwarding to start"
+    sleep 3
+
+    # When running locally, use the scaffolding gettoken service. In CI, use the ambient OIDC flow
+    export ID_TOKEN=$(curl --fail -s "http://gettoken.default.svc:8080")
+fi
+
+# a valid token is only needed when recording new fixtures
+export GITHUB_TOKEN="${GITHUB_TOKEN-"invalid"}"
+go test -v -count 1 ./...

hack/test-setup.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SIGSTORE_SCAFFOLDING_VERSION="v0.6.4"
+K8S_VERSION="v1.25.x"
+REGISTRY_URL="registry.local:5001"
+
+tmpDir=$(mktemp -d)
+
+echo "Using ${tmpDir} for test setup"
+pushd "${tmpDir}"
+
+curl --fail --remote-name-all -sL \
+  "https://github.com/sigstore/scaffolding/releases/download/${SIGSTORE_SCAFFOLDING_VERSION}/{setup-kind.sh,setup-scaffolding-from-release.sh,testrelease.yaml}"
+
+chmod +x setup-kind.sh setup-scaffolding-from-release.sh
+
+echo "Setting up kind cluster"
+./setup-kind.sh  --k8s-version "${K8S_VERSION}" --registry-url "${REGISTRY_URL}"
+
+echo "Installing scaffolding"
+./setup-scaffolding-from-release.sh --release-version "${SIGSTORE_SCAFFOLDING_VERSION}"
+
+echo "Installing OIDC Issuer & Testing Setup"
+
+# copy TUF root to default namespace because the sigstore/scaffolding test jobs will use it
+# https://github.com/sigstore/scaffolding/blob/84f4140ad89fd7ea270f9862941228b2d0fa72e6/actions/setup/action.yml#L111-L112
+kubectl -n tuf-system get secrets tuf-root -oyaml | sed 's/namespace: .*/namespace: default/' | kubectl apply -f -
+
+# overwrite the hard-coded registry in the release
+# https://github.com/sigstore/scaffolding/pull/547
+sed -i.bak "s/registry.local:5000/${REGISTRY_URL}/g" testrelease.yaml
+
+kubectl apply -f testrelease.yaml
+
+echo "Waiting for sign & verify setup test jobs to finish"
+kubectl wait --for=condition=Complete --timeout=180s job/sign-job
+kubectl wait --for=condition=Complete --timeout=180s job/verify-job
+
+popd
+
+cp "${tmpDir}/root.json" "test/root.json"
+
+echo "Finished test setup"

hack/test-teardown.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+CLUSTER_NAME="kind"
+REGISTRY_NAME="registry.local"
+TUF_ROOT="test/root.json"
+
+for cluster in $(kind get clusters); do
+  if [[ "$cluster" == "${CLUSTER_NAME}" ]]; then
+    echo "Removing kind cluster"
+    kind delete clusters kind
+  fi
+done
+
+registryContainer=$(docker ps -q --filter "name=${REGISTRY_NAME}")
+if [[ -n "${registryContainer}" ]]; then
+    echo "Stopping registry container"
+    docker stop "${registryContainer}"
+    docker rm "${registryContainer}"
+fi
+
+dockerNetwork=$(docker network ls -q --filter name="${CLUSTER_NAME}")
+
+if [[ -n "${dockerNetwork}" ]]; then
+  echo "Deleting kind network"
+  docker network rm "${dockerNetwork}"
+fi
+
+if [[ -f "${TUF_ROOT}" ]]; then
+  echo "Deleting TUF root"
+  rm "${TUF_ROOT}"
+fi