Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

✨ Pull builder's binary instead of compiling it #86

Merged
merged 28 commits into from
May 25, 2022
Merged
Show file tree
Hide file tree
Changes from 23 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 41 additions & 31 deletions .github/workflows/builder_go_slsa3.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,13 @@ env:
GENERATED_BINARY_NAME: compiled-binary
# Builder
BUILDER_BINARY: builder
BUILDER_RELEASE_BINARY: slsa-builder-go-linux-amd64
BUILDER_REPOSITORY: slsa-framework/slsa-github-generator
# Verifier
VERIFIER_REPOSITORY: slsa-framework/slsa-verifier
VERIFIER_RELEASE_BINARY: slsa-verifier-linux-amd64
VERIFIER_RELEASE_BINARY_SHA256: fb743bc6bb56908d590da66bfe5c266d003aa226b30fcada5f7b9e4aea43b52b
Copy link
Collaborator Author

@laurentsimon laurentsimon May 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note: this hash is invalid. We will update it once we (the maintainers) have cut a release for the verifier and verified the hash is valid thru provenance - see #74 (comment)

VERIFIER_RELEASE: v0.0.1
# Builder location
BUILDER_DIR: builders

Expand All @@ -34,11 +41,11 @@ on:
workflow_call:
inputs:
go-version:
description: "The go version to use"
description: "The go version to use."
required: true
type: string
upload-assets:
description: "Whether to upload assets to a GitHub release or not"
description: "Whether to upload assets to a GitHub release or not."
required: false
type: boolean
default: true
Expand All @@ -48,12 +55,17 @@ on:
type: string
default: ".slsa-goreleaser.yml"
evaluated-envs:
description: "Evaluated env variables to pass to the builder"
description: "Evaluated env variables to pass to the builder."
required: false
type: string
compile-builder:
description: "Build the builder from source. This increases build time by ~2mn."
required: false
type: boolean
default: false
outputs:
go-binary-name:
description: "The name of the generated binary uploaded to the artifact registry"
description: "The name of the generated binary uploaded to the artifact registry."
value: ${{ jobs.build-dry.outputs.go-binary-name }}

jobs:
Expand Down Expand Up @@ -125,40 +137,44 @@ jobs:
with:
go-version: 1.18

- name: Download dependencies
- name: Build builder
shell: bash
id: builder-gen
env:
COMPILE_BUILDER: "${{ inputs.compile-builder }}"
BUILDER_REF: "${{ needs.detect-env.outputs.builder_ref }}"
# Needed for the gh CLI used in builder-fetch.sh.
GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
run: |
set -euo pipefail

cd "$BUILDER_DIR"/go/
if [[ "$COMPILE_BUILDER" = true ]]; then
echo "Building the builder"

#TODO(reproducible)
go mod vendor
cd "$BUILDER_DIR"/go/

# TODO(hermeticity) OS-level.
# - name: Disable hermeticity
# uses: slsa/hermeticity@xxx
# with:
# to-state: enabled
#TODO(reproducible)
go mod vendor

- name: Build builder
shell: bash
id: builder-gen
env:
BUILDER_BINARY: "${{ env.BUILDER_BINARY }}"
run: |
set -euo pipefail
# https://go.dev/ref/mod#build-commands.
go build -mod=vendor -o "$BUILDER_BINARY"

mv "$BUILDER_BINARY" ../../"$BUILDER_BINARY"

cd "$BUILDER_DIR"/go/
cd -
else
echo "Fetching the builder with ref: $BUILDER_REF"

.github/workflows/scripts/builder-fetch.sh

# https://go.dev/ref/mod#build-commands.
go build -mod=vendor -o "$BUILDER_BINARY"
mv "$BUILDER_RELEASE_BINARY" "$BUILDER_BINARY"

fi

BUILDER_DIGEST=$(sha256sum "$BUILDER_BINARY" | awk '{print $1}')
echo "::set-output name=go-builder-sha256::$BUILDER_DIGEST"
echo "hash of $BUILDER_BINARY is $BUILDER_DIGEST"

mv "$BUILDER_BINARY" ../../"$BUILDER_BINARY"

- name: Upload the builder
uses: actions/upload-artifact@6673cd052c4cd6fcf4b4e6e60ea986c889389535 # v2.3.1
with:
Expand Down Expand Up @@ -202,7 +218,6 @@ jobs:
- name: Verify builder
env:
BUILDER_HASH: "${{ needs.builder.outputs.go-builder-sha256 }}"
BUILDER_BINARY: "${{ env.BUILDER_BINARY }}"
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note: I've removed all occurrences of BUILDER_BINARY: "${{ env.BUILDER_BINARY }}" because they are not needed

run: |
set -euo pipefail

Expand All @@ -221,7 +236,6 @@ jobs:
id: build-dry
shell: bash
env:
BUILDER_BINARY: "${{ env.BUILDER_BINARY }}"
CONFIG_FILE: "${{ inputs.config-file }}"
UNTRUSTED_ENVS: "${{ inputs.evaluated-envs }}"
run: |
Expand Down Expand Up @@ -264,7 +278,6 @@ jobs:
- name: Verify builder
env:
BUILDER_HASH: "${{ needs.builder.outputs.go-builder-sha256 }}"
BUILDER_BINARY: "${{ env.BUILDER_BINARY }}"
run: |
set -euo pipefail

Expand Down Expand Up @@ -300,7 +313,6 @@ jobs:
id: build-gen
shell: bash
env:
BUILDER_BINARY: "${{ env.BUILDER_BINARY }}"
CONFIG_FILE: "${{ inputs.config-file }}"
UNTRUSTED_ENVS: "${{ inputs.evaluated-envs }}"
run: |
Expand Down Expand Up @@ -398,7 +410,6 @@ jobs:
- name: Verify builder
env:
BUILDER_HASH: "${{ needs.builder.outputs.go-builder-sha256 }}"
BUILDER_BINARY: "${{ env.BUILDER_BINARY }}"
run: |
set -euo pipefail

Expand All @@ -422,7 +433,6 @@ jobs:
UNTRUSTED_COMMAND: "${{ needs.build-dry.outputs.go-command }}"
UNTRUSTED_ENV: "${{ needs.build-dry.outputs.go-env }}"
UNTRUSTED_WORKING_DIR: "${{ needs.build-dry.outputs.go-working-dir }}"
BUILDER_BINARY: "${{ env.BUILDER_BINARY }}"
GITHUB_CONTEXT: "${{ toJSON(github) }}"
run: |
set -euo pipefail
Expand Down
14 changes: 14 additions & 0 deletions .github/workflows/configs-go/config-release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# Used for pre-submit tests.
version: 1
env:
- GO111MODULE=on
- CGO_ENABLED=0

flags:
- -trimpath
- -tags=netgo

goos: linux
goarch: amd64
dir: builders/go/
binary: slsa-builder-go-{{ .Os }}-{{ .Arch }}
23 changes: 23 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
name: Builders realeaser

on:
push:
tags:
- "*" # triggers only if push new tag version, like `0.8.4` or else

permissions: read-all

env:
GH_TOKEN: ${{ secrets.E2E_GO_TOKEN }}

jobs:
# Go builder.
go-builder:
permissions:
id-token: write # For signing.
contents: write # For asset uploads.
uses: laurentsimon/slsa-github-generator/.github/workflows/builder_go_slsa3.yml@feat/fastbuilds
with:
go-version: 1.18
config-file: .github/workflows/configs-go/config-release.yml
compile-builder: true
89 changes: 89 additions & 0 deletions .github/workflows/scripts/builder-fetch.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
#!/usr/bin/env bash

set -euo pipefail

# Caller sets the following:
Copy link
Collaborator Author

@laurentsimon laurentsimon May 24, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note: We will replace this script with Golang code in the future #87

#BUILDER_REPOSITORY="slsa-framework/slsa-github-generator"
#BUILDER_TAG="v13.0.10" or "6a1e642a8689671a2cec9287149eb50bd9fe5ef6"
#BUILDER_RELEASE_BINARY="builder-linux-amd64"
#VERIFIER_REPOSITORY="slsa-framework/slsa-verifier"
#VERIFIER_RELEASE="v13.0.10"
#VERIFIER_RELEASE_BINARY="slsa-verifier-linux-amd64"
#VERIFIER_RELEASE_BINARY_SHA256="89fbcba9aed67d5146ea99946c7e4e5a80e3767871f0e3ffcd0b582134efd010"

PREFIX="refs/tags/"

# Extract version.
if [[ "$BUILDER_REF" =~ "^$PREFIX*" ]]; then
echo "Invalid ref: $BUILDER_REF"
exit 2
fi

BUILDER_TAG="${BUILDER_REF#"$PREFIX"}"

if [[ "$BUILDER_TAG" = "$(echo -n "$BUILDER_TAG" | grep -P '^[a-f\d]{40}$')" ]]; then
echo "Builder referenced by hash: $BUILDER_TAG"
echo "Resolving..."

RELEASE_TAG=""

# List the releases and find the corepsonding hash.
RELEASE_LIST=$(gh release -R "$BUILDER_REPOSITORY" -L 50 list)
laurentsimon marked this conversation as resolved.
Show resolved Hide resolved
while read line; do
TAG=$(echo "$line" | cut -f1)
BRANCH=$(gh release -R "$BUILDER_REPOSITORY" view "$TAG" --json targetCommitish --jq '.targetCommitish')
if [[ "$BRANCH" != "main" ]]; then
continue
fi
COMMIT=$(gh api /repos/"$BUILDER_REPOSITORY"/git/ref/tags/"$TAG" | jq -r '.object.sha')
if [[ "$COMMIT" == "$BUILDER_TAG" ]]; then
RELEASE_TAG="$TAG"
echo "Found tag $BUILDER_TAG match at tag $TAG and commit $COMMIT"
break
fi
done <<< "$RELEASE_LIST"

if [[ -z "$RELEASE_TAG" ]]; then
echo "Tag not found for $BUILDER_TAG"
exit 3
fi

BUILDER_TAG="$RELEASE_TAG"
fi

if [[ "$BUILDER_TAG" != "$(echo -n "$BUILDER_TAG" | grep -P '^v\d*(\.([\d]{1,})){0,2}$')" ]]; then
echo "Invalid ref: $BUILDER_TAG"
exit 7
fi

echo "Builder version: $BUILDER_TAG"

echo "BUILDER_REPOSITORY: $BUILDER_REPOSITORY"

# Fetch the release binary and provenance.
gh release -R "$BUILDER_REPOSITORY" download "$BUILDER_TAG" -p "$BUILDER_RELEASE_BINARY*" || exit 10

# Fetch the verifier at the right hash.
gh release -R "$VERIFIER_REPOSITORY" download "$VERIFIER_RELEASE" -p "$VERIFIER_RELEASE_BINARY" || exit 11
COMPUTED_HASH=$(sha256sum "$VERIFIER_RELEASE_BINARY" | awk '{print $1}')
echo "verifier hash computed is $COMPUTED_HASH"
echo "$VERIFIER_RELEASE_BINARY_SHA256 $VERIFIER_RELEASE_BINARY" | sha256sum --strict --check --status || exit 4
echo "verifier hash verification has passed"

# Verify the provenance of the builder.
chmod a+x "$VERIFIER_RELEASE_BINARY"
./"$VERIFIER_RELEASE_BINARY" --branch "main" \
--tag "$BUILDER_TAG" \
--artifact-path "$BUILDER_RELEASE_BINARY" \
--provenance "$BUILDER_RELEASE_BINARY.intoto.jsonl" \
--source "github.com/$BUILDER_REPOSITORY" || exit 6

BUILDER_COMMIT=$(gh api /repos/"$BUILDER_REPOSITORY"/git/ref/tags/"$BUILDER_TAG" | jq -r '.object.sha')
PROVENANCE_COMMIT=$(cat "$BUILDER_RELEASE_BINARY.intoto.jsonl" | jq -r '.payload' | base64 -d | jq -r '.predicate.materials[0].digest.sha1')
if [[ "$BUILDER_COMMIT" != "$PROVENANCE_COMMIT" ]]; then
echo "Builder commit sha $BUILDER_COMMIT != provenance material $PROVENANCE_COMMIT"
exit 5
fi

#TODO: verify the command
echo "Builder provenance verified at tag $BUILDER_TAG and commit $BUILDER_COMMIT"