From d7fa12569310cf0602343a0555a812e859f072d4 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Wed, 14 Feb 2024 10:42:45 -0800 Subject: [PATCH 1/2] Add guide for multiple operating systems Signed-off-by: Mihai Maruseac --- internal/builders/generic/README.md | 170 ++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) diff --git a/internal/builders/generic/README.md b/internal/builders/generic/README.md index af51c0224b..09e1273481 100644 --- a/internal/builders/generic/README.md +++ b/internal/builders/generic/README.md @@ -42,6 +42,7 @@ project simply generates provenance as a separate step in an existing workflow. - [Provenance for matrix strategy builds](#provenance-for-matrix-strategy-builds) - [A single provenance attestation for all artifacts](#a-single-provenance-attestation-for-all-artifacts) - [A different attestation for each iteration](#a-different-attestation-for-each-iteration) +- [Provenance for artifacts built across multiple operating systems](#provenance-for-artifacts-built-across-multiple-operating-systems) - [Known Issues](#known-issues) - [Skip output 'hashes' since it may contain secret](#skip-output-hashes-since-it-may-contain-secret) - ['internal error' when using `upload-assets`](#internal-error-when-using-upload-assets) @@ -1408,6 +1409,175 @@ jobs: upload-assets: true # Optional: Upload to a new release ``` +## Provenance for artifacts built across multiple operating systems + +If a single release job produces artifacts for multiple operating systems (using +matrix strategy), there are a few more caveats to consider: + +1. First, it is ideal to make Windows behave the same as Linux and MacOS by + making the runner use `bash` as the shell to execute commands in: + + ```yaml + defaults: + run: + shell: bash + ``` + +2. Optionally, you might also want to make the workflow use LF as line + terminator even on Windows: + + ```yaml + - run: git config --global core.autocrlf input + # Alternatively, also force line endings for every file + # - run: | + # git config --global core.eol lf + # git config --global core.autocrlf input + ``` + +The other complexity arises from the fact that the utilities used to compute the +digest (`sha256sum`) and the base 64 encoding (`base64`) have different +behaviors across the operating systems: + +* On MacOS, the utlity to compute the digest is called `shasum` and the + algorithm is passed via the `-a 256` algorithm +* On Windows, we need to tell `sha256sum` to treat all files as text by using + the `-t` switch, otherwise the output will contain an extra `*` in front of + the filename. This will later be wrongly interpretted as a glob pattern, so we + should avoid it. +* On MacOS, `base64` does not have a `-w0` option, the line wrapping is + implicit. + +One way to merge all these differences is to use the bash `||` operator: + +```yaml + - id: hash + run: | + set -euo pipefail + (sha256sum -t release_artifact_${{ runner.os }} || shasum -a 256 release_artifact_${{ runner.os }}) > checksum + echo "hash-${{ matrix.os }}=$(base64 -w0 checksum || base64 checksum)" >> "${GITHUB_OUTPUT}" +``` + +Thus, to generate a single provenance for artifacts built on all 3 operating +systems, you would use the following example: + +```yaml +defaults: + run: + shell: bash + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false # Don't cancel other jobs if one fails + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + outputs: + hash-ubuntu-latest: ${{ steps.hash.outputs.hash-ubuntu-latest }} + hash-macos-latest: ${{ steps.hash.outputs.hash-macos-latest }} + hash-windows-latest: ${{ steps.hash.outputs.hash-windows-latest }} + steps: + - run: git config --global core.autocrlf input + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + persist-credentials: false + + # Do the build to create release_artifact_${{ runner.os }} + - run: ... + + - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + with: + path: release_artifact_${{ runner.os }} + name: release_artifact_${{ runner.os }} + if-no-files-found: error + - id: hash + run: | + set -euo pipefail + (sha256sum -t release_artifact_${{ runner.os }} || shasum -a 256 release_artifact_${{ runner.os }}) > checksum + echo "hash-${{ matrix.os }}=$(base64 -w0 checksum || base64 checksum)" >> "${GITHUB_OUTPUT}" + + provenance: + needs: [build] + strategy: + fail-fast: false # Don't cancel other jobs if one fails + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + permissions: + actions: read + id-token: write + contents: write + uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.9.0 + with: + base64-subjects: "${{ needs.build.outputs[format('hash-{0}', matrix.os)] }}" + upload-assets: true # NOTE: This does nothing unless 'upload-tag-name' parameter is also set to an existing tag +``` + +Alternatively, to generate 3 different provenance statements, one per each +artifact, you would use the following example: + +```yaml +defaults: + run: + shell: bash + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false # Don't cancel other jobs if one fails + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + outputs: + hash-ubuntu-latest: ${{ steps.hash.outputs.hash-ubuntu-latest }} + hash-macos-latest: ${{ steps.hash.outputs.hash-macos-latest }} + hash-windows-latest: ${{ steps.hash.outputs.hash-windows-latest }} + steps: + - run: git config --global core.autocrlf input + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + persist-credentials: false + + # Do the build to create release_artifact_${{ runner.os }} + - run: ... + + - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 + with: + path: release_artifact_${{ runner.os }} + name: release_artifact_${{ runner.os }} + if-no-files-found: error + - id: hash + run: | + set -euo pipefail + (sha256sum -t release_artifact_${{ runner.os }} || shasum -a 256 release_artifact_${{ runner.os }}) > checksum + echo "hash-${{ matrix.os }}=$(base64 -w0 checksum || base64 checksum)" >> "${GITHUB_OUTPUT}" + + combine_hashes: + needs: [build] + runs-on: ubuntu-latest + outputs: + hashes: ${{ steps.combine_hashes.outputs.hashes }} + env: + HASHES: ${{ toJSON(needs.build.outputs) }} + steps: + - id: combine_hashes + run: | + set -euo pipefail + echo "${HASHES}" | jq -r '.[] | @base64d' | sed "/^$/d" > hashes + echo "hashes=$(base64 -w0 hashes)" >> "${GITHUB_OUTPUT}" + + provenance: + needs: [combine_hashes] + permissions: + actions: read + id-token: write + contents: write + uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.9.0 + with: + base64-subjects: "${{ needs.combine_hashes.outputs.hashes }}" + upload-assets: true # NOTE: This does nothing unless 'upload-tag-name' parameter is also set to an existing tag + +``` + ## Known Issues ### Skip output 'hashes' since it may contain secret From 27bc1eb67e1b79deea411ab406b8d492c7d7ef87 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Wed, 14 Feb 2024 10:49:54 -0800 Subject: [PATCH 2/2] Fix lint Signed-off-by: Mihai Maruseac --- internal/builders/generic/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/builders/generic/README.md b/internal/builders/generic/README.md index 09e1273481..1e809fff43 100644 --- a/internal/builders/generic/README.md +++ b/internal/builders/generic/README.md @@ -1438,13 +1438,13 @@ The other complexity arises from the fact that the utilities used to compute the digest (`sha256sum`) and the base 64 encoding (`base64`) have different behaviors across the operating systems: -* On MacOS, the utlity to compute the digest is called `shasum` and the +- On MacOS, the utlity to compute the digest is called `shasum` and the algorithm is passed via the `-a 256` algorithm -* On Windows, we need to tell `sha256sum` to treat all files as text by using +- On Windows, we need to tell `sha256sum` to treat all files as text by using the `-t` switch, otherwise the output will contain an extra `*` in front of the filename. This will later be wrongly interpretted as a glob pattern, so we should avoid it. -* On MacOS, `base64` does not have a `-w0` option, the line wrapping is +- On MacOS, `base64` does not have a `-w0` option, the line wrapping is implicit. One way to merge all these differences is to use the bash `||` operator: