diff --git a/.github/workflows/build_rust_docs.yaml b/.github/workflows/build_rust_docs.yaml deleted file mode 100644 index a8ad7442105..00000000000 --- a/.github/workflows/build_rust_docs.yaml +++ /dev/null @@ -1,78 +0,0 @@ -name: Build Rust docs - -on: - push: - branches: [main] - pull_request: - branches: [main] - -# See https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#example-using-concurrency-to-cancel-any-in-progress-job-or-run -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - build_rust_docs: - runs-on: ubuntu-20.04 - - permissions: - contents: write - - steps: - - name: Checkout branch - uses: actions/checkout@v3 - - - name: Checkout gh-pages - uses: actions/checkout@v3 - with: - ref: gh-pages - path: out - - # We need to set up git user details before we can perform git operations. - - name: Git setup - run: | - git config --global user.email "actions@github.com" - git config --global user.name "GitHub Actions" - - # Copied from https://github.com/jens-maus/RaspberryMatic/blob/ea6b8ce0dd2d53ea88b2766ba8d7f8e1d667281f/.github/workflows/ci.yml#L34-L40 - - name: Free disk space - run: | - df --human-readable - sudo swapoff --all - sudo rm --force /swapfile - sudo apt clean - docker rmi $(docker image ls --all --quiet) - df --human-readable - - - uses: DeterminateSystems/nix-installer-action@677cbc8aa1fe7e80b1fd15243ec41d5e0c7e3669 - - uses: DeterminateSystems/magic-nix-cache-action@5cc3a5e24b688cf7e0ea986ab07a9d3d32c34a95 - - # Remove all files from the "out" folder. - - name: Clear "out" folder - run: rm --recursive --force ./out/* - - # Build the Nix shell in a separate step first, so we can keep track of its build time. - # We use the `rust` shell here and below, since it avoids building unnecessary dependencies. - - name: Build nix shell - run: | - nix develop .#rust --command echo done - - # Generate docs from within the Nix shell. - - name: Generate docs - run: | - nix develop .#rust --command ./scripts/build_gh_pages ./out - - # From the "out" folder, commit the results and push to the `gh-pages` branch. - # This step only applies to `push` events (not `pull_request`), and only if there are actual - # changes to commit in the "out" folder. - - name: Commit and push (post-merge only) - if: github.event_name == 'push' - run: | - cd ./out - git add . - if [[ -n "$(git status --porcelain)" ]]; then - git commit --message="Update gh-pages from ${GITHUB_SHA}" - git push - else - echo 'no changes to commit' - fi diff --git a/.github/workflows/build_toolchain.yaml b/.github/workflows/build_toolchain.yaml deleted file mode 100644 index b086b32539e..00000000000 --- a/.github/workflows/build_toolchain.yaml +++ /dev/null @@ -1,82 +0,0 @@ -name: Build Cross Compiler Toolchain - -on: workflow_dispatch - -jobs: - build_toolchain: - runs-on: ubuntu-20.04 - timeout-minutes: 120 - strategy: - fail-fast: false - - steps: - - name: Checkout branch - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - # The runner comes with all this software pre-installed: https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md - # so we delete some large packages to make sure we have more space available. - # - # We print the remaining disk space between commands in order to have an idea of which of them - # have the most significant impact. - # - # Copied from https://github.com/jens-maus/RaspberryMatic/blob/ea6b8ce0dd2d53ea88b2766ba8d7f8e1d667281f/.github/workflows/ci.yml#L34-L40 - # and https://github.com/actions/virtual-environments/issues/709#issuecomment-612569242 - - name: Free disk space - run: | - df --human-readable - sudo apt-get remove --yes '^dotnet-.*' '^llvm-.*' 'php.*' azure-cli google-cloud-sdk hhvm google-chrome-stable firefox powershell - df --human-readable - sudo apt-get autoremove --yes - df --human-readable - sudo apt clean - df --human-readable - docker rmi $(docker image ls --all --quiet) - df --human-readable - rm --recursive --force "$AGENT_TOOLSDIRECTORY" - df --human-readable - rm --recursive --force /usr/local/share/boost - df --human-readable - sudo swapoff --all - df --human-readable - sudo rm --force /swapfile - df --human-readable - - name: Docker pull - timeout-minutes: 10 - run: | - ./scripts/docker_pull - df --human-readable - - name: Run command - run: | - ./scripts/docker_run toolchain/build.sh - df --human-readable - - name: Download Ent CLI - if: steps.cache-ent.outputs.cache-hit != 'true' - env: - ENT_URL: https://ent-server-62sa4xcfia-ew.a.run.app - ENT_DIGEST_SHA_2_256: ca76e76e32e0b984a8d55d1a27d587865774b29a2ae2fffd756ffc2dbd4880bd - run: | - set -o errexit - set -o nounset - set -o xtrace - set -o pipefail - curl --fail ${ENT_URL}/raw/sha2-256:${ENT_DIGEST_SHA_2_256} > /usr/local/bin/ent - echo "${ENT_DIGEST_SHA_2_256} /usr/local/bin/ent" | sha256sum --check - chmod +x /usr/local/bin/ent - ent - cat < ~/.config/ent.toml - [[remotes]] - name = 'ent-store' - url = '${ENT_URL}' - write = true - api_key = '${{ secrets.ENT_API_KEY }}' - EOF - - name: Upload to Ent - run: | - ent put ./toolchain/toolchain.tar.bz2 - - # Ensure that the previous steps did not modify our source-code and that - # relevant build artifacts are ignored by git. - - name: Git check diff - run: ./scripts/docker_run ./scripts/git_check_diff diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml deleted file mode 100644 index c8a12739986..00000000000 --- a/.github/workflows/ci.yaml +++ /dev/null @@ -1,89 +0,0 @@ -name: Continuous Integration - -on: - push: - branches: [main] - pull_request: - branches: [main] - -# See https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#example-using-concurrency-to-cancel-any-in-progress-job-or-run -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - xtask: - runs-on: ubuntu-20.04 - timeout-minutes: 120 - # We use the same job template, but parametrize the actual command to be passed to the xtask - # binary using the matrix strategy, so that we get the commands running in parallel. - strategy: - fail-fast: false - matrix: - # Do not add `run-ci` to these commands. We parse them for `xtask run-ci`. - cmd: - # format tries to format all the source files, and relies on the git diff step to determine whether anything was missed. - - format - # check-format checks for additional issues (e.g. missing license headers) that format does not enforce. - - check-format - - run-cargo-deny - - run-cargo-udeps - - run-bazel-tests - - run-cargo-fuzz -- -max_total_time=2 - - completion - - run-cargo-clippy - - steps: - - name: Checkout branch - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - # The runner comes with all this software pre-installed: https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md - # so we delete some large packages to make sure we have more space available. - # - # We print the remaining disk space between commands in order to have an idea of which of them - # have the most significant impact. - # - # Copied from https://github.com/jens-maus/RaspberryMatic/blob/ea6b8ce0dd2d53ea88b2766ba8d7f8e1d667281f/.github/workflows/ci.yml#L34-L40 - # and https://github.com/actions/virtual-environments/issues/709#issuecomment-612569242 - - name: Free disk space - run: | - df --human-readable - sudo apt-get remove --yes '^dotnet-.*' '^llvm-.*' 'php.*' azure-cli google-cloud-cli hhvm google-chrome-stable firefox powershell - df --human-readable - sudo apt-get autoremove --yes - df --human-readable - sudo apt clean - df --human-readable - docker rmi $(docker image ls --all --quiet) - df --human-readable - rm --recursive --force "$AGENT_TOOLSDIRECTORY" - df --human-readable - rm --recursive --force /usr/local/share/boost - df --human-readable - sudo swapoff --all - df --human-readable - sudo rm --force /swapfile - df --human-readable - - - name: Docker pull - timeout-minutes: 10 - run: | - ./scripts/docker_pull - df --human-readable - - - name: Run command - env: - RUST_BACKTRACE: 1 - RUST_LOG: debug - # Do not run tests that require KVM on GitHub Actions, since nested virtualization is not supported. - OAK_KVM_TESTS: skip - run: | - ./scripts/docker_run nix develop .#ci --command ./scripts/xtask ${{ matrix.cmd }} - df --human-readable - - # Ensure that the previous steps did not modify our source-code and that - # relevant build artifacts are ignored by git. - - name: Git check diff - run: ./scripts/docker_run ./scripts/git_check_diff diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml deleted file mode 100644 index 87331a81efb..00000000000 --- a/.github/workflows/cifuzz.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: CIFuzz -on: [pull_request] - -# See https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#example-using-concurrency-to-cancel-any-in-progress-job-or-run -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - Fuzzing: - runs-on: ubuntu-latest - steps: - - name: Build Fuzzers - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master - with: - oss-fuzz-project-name: 'oak' - dry-run: false - language: rust - - name: Run Fuzzers - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master - with: - oss-fuzz-project-name: 'oak' - fuzz-seconds: 600 - dry-run: false - - name: Upload Crash - uses: actions/upload-artifact@v1 - if: failure() && steps.build.outcome == 'success' - with: - name: artifacts - path: ./out/artifacts diff --git a/.github/workflows/ent.yaml b/.github/workflows/ent.yaml new file mode 100644 index 00000000000..3658ad19c54 --- /dev/null +++ b/.github/workflows/ent.yaml @@ -0,0 +1,87 @@ +name: Ent + +on: + push: + branches: [main] + pull_request: + branches: [main] + +# See https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#example-using-concurrency-to-cancel-any-in-progress-job-or-run +# cc +# xx +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + xtask: + runs-on: ubuntu-20.04 + timeout-minutes: 120 + + steps: + - name: Checkout branch + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - uses: 'google-github-actions/auth@v2' + with: + credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY_JSON }} + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v2' + + - id: 'info' + name: 'info' + run: | + set -o errexit + set -o nounset + set -o xtrace + set -o pipefail + + gcloud --version + gsutil --version + + # - id: 'upload-file' + # name: 'upload files' + # uses: 'google-github-actions/upload-cloud-storage@v2' + # with: + # process_gcloudignore: false + # path: './buildconfigs' + # destination: 'oak-bins/${{ github.sha }}/${{ github.action }}-${{ github.run_id }}-${{ github.run_attempt }}' + + - id: 'index-files' + name: 'index files' + run: | + set -o errexit + set -o nounset + set -o xtrace + set -o pipefail + + bin_name=oak-bins + destination='${{ github.sha }}/${{ github.action }}-${{ github.run_id }}-${{ github.run_attempt }}' + + IFS=, + files=(./buildconfigs/*) + echo $files + for file in "${files[@]}"; do + base_file_name=$(basename ${file}) + echo $file + gsutil cp $file "gs://${bin_name}/${destination}/${base_file_name}" + # https://storage.googleapis.com/oak-bins/16df3ddbf14af9b0c5839aece766fb72fcc1c240/upload-file-8754169623-1/buildconfigs/key_xor_test_app.toml + # xxy + file_url="https://storage.googleapis.com/${bin_name}/${destination}/${base_file_name}" + body=$(cat <> $GITHUB_OUTPUT - - - name: Print values - run: | - echo "${{ steps.builder-digest.outputs.builder-digest }}" - - generate_provenance: - needs: [get_inputs] - permissions: - actions: read - id-token: write - contents: write - # We are a pre-release of the slsa-github-generator. This is because the - # current release does not include fix for the action deadlocking. - # Ref: https://github.com/slsa-framework/slsa-github-generator/issues/3571 - # TODO: b/335461780 - Switch to using a released tag once there is one that includes the fix. - uses: slsa-framework/slsa-github-generator/.github/workflows/builder_container-based_slsa3.yml@v2.0.0-rc.0 - with: - builder-image: 'europe-west2-docker.pkg.dev/oak-ci/oak-development/oak-development' - builder-digest: ${{ needs.get_inputs.outputs.builder-digest }} - config-path: ${{ inputs.build-config-path }} - provenance-name: attestation.intoto - compile-builder: true diff --git a/.github/workflows/reusable_provenance.yaml b/.github/workflows/reusable_provenance.yaml deleted file mode 100644 index 029e0afcc0d..00000000000 --- a/.github/workflows/reusable_provenance.yaml +++ /dev/null @@ -1,205 +0,0 @@ -# Container-based SLSA provenance generator that works with multiple provenance subjects. -name: Build SLSA3 Provenance - -on: - workflow_call: - inputs: - build-config-path: - required: true - type: string - # Public key corresponding to the private key used to sign Ent tags. - ent-public-key: - required: true - type: string - secrets: - # The Ent API key is used to upload the binary and its provenance to Ent. - ENT_API_KEY: - required: true - # The Ent secret key is used to sign Ent tags, mapping the identifier - # of a binary to its provenance. - ENT_SECRET_KEY: - required: true - -jobs: - get_inputs: - outputs: - # Resolves to a single file which is passed to the SLSA provenance - # generator. - # TODO: b/301399862 - We need to figure out how we can pass more than - # one file to the provenance generation, without having Ent tags that - # depend on filenames. - artifact-path: ${{ steps.parse-build-config.outputs.artifact-path }} - # The name of the internal TR package. This must coincide with the - # basename of the buildconfig. - package-name: ${{ steps.parse-build-config.outputs.package-name }} - builder-digest: ${{ steps.builder-digest.outputs.builder-digest }} - - runs-on: ubuntu-20.04 - - steps: - - name: Mount main branch - uses: actions/checkout@v3 - - - name: Parse build config - id: parse-build-config - run: | - set -o errexit - set -o nounset - set -o xtrace - set -o pipefail - artifact_path="$(tail -1 ${{ inputs.build-config-path }} | grep -oP 'artifact_path = \K(.*)')" - package_name="$(basename ${{ inputs.build-config-path }} .toml)" - echo "artifact-path=${artifact_path}" >> $GITHUB_OUTPUT - echo "package-name=${package_name}" >> $GITHUB_OUTPUT - - - name: Get builder image info - id: builder-digest - run: | - set -o errexit - set -o nounset - set -o xtrace - set -o pipefail - source ./scripts/common - digest="$(echo "${DOCKER_IMAGE_REPO_DIGEST}" | cut -d'@' -f2)" - echo "builder-digest=${digest}" >> $GITHUB_OUTPUT - - - name: Print values - run: | - echo "${{ steps.parse-build-config.outputs.artifact-path }}" - echo "${{ steps.parse-build-config.outputs.package-name }}" - echo "${{ steps.builder-digest.outputs.builder-digest }}" - - generate_provenance: - needs: [get_inputs] - permissions: - actions: read - id-token: write - contents: write # For uploading provenances. - # We are a pre-release of the slsa-github-generator. This is because the - # current release does not include fix for the action deadlocking. - # Ref: https://github.com/slsa-framework/slsa-github-generator/issues/3571 - # TODO: b/335461780 - Switch to using a released tag once there is one that includes the fix. - uses: slsa-framework/slsa-github-generator/.github/workflows/builder_container-based_slsa3.yml@v2.0.0-rc.0 - with: - builder-image: 'europe-west2-docker.pkg.dev/oak-ci/oak-development/oak-development' - builder-digest: ${{ needs.get_inputs.outputs.builder-digest }} - config-path: ${{ inputs.build-config-path }} - provenance-name: attestation.intoto - compile-builder: true - - # This job uploads the signed provenance from the previous step to Ent, and - # publishes a comment for each binary on the PR. - upload_provenance: - if: | - github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'provenance:force-run') - needs: [get_inputs, generate_provenance] - runs-on: ubuntu-20.04 - permissions: - # Allow the job to update the repo with the latest provenance info and index. - contents: write - # Allow the job to add a comment to the PR. - pull-requests: write - - steps: - # See https://github.com/google/ent - - name: Download Ent CLI - if: steps.cache-ent.outputs.cache-hit != 'true' - env: - ENT_URL: https://ent-server-62sa4xcfia-ew.a.run.app - ENT_DIGEST_SHA_2_256: ca76e76e32e0b984a8d55d1a27d587865774b29a2ae2fffd756ffc2dbd4880bd - run: | - set -o errexit - set -o nounset - set -o xtrace - set -o pipefail - curl --fail "${ENT_URL}/raw/sha2-256:${ENT_DIGEST_SHA_2_256}" > /usr/local/bin/ent - echo "${ENT_DIGEST_SHA_2_256} /usr/local/bin/ent" | sha256sum --check - chmod +x /usr/local/bin/ent - ent - cat < ~/.config/ent.toml - secret_key = '${{ secrets.ENT_SECRET_KEY }}' - [[remotes]] - name = 'ent-store' - url = '${ENT_URL}' - write = true - api_key = '${{ secrets.ENT_API_KEY }}' - EOF - - - name: Download the built artifact - uses: actions/download-artifact@v3 - with: - name: ${{ needs.generate_provenance.outputs.build-outputs-name }} - path: downloads - - - name: Download the DSSE document - uses: actions/download-artifact@v3 - with: - name: - ${{ needs.generate_provenance.outputs.attestations-download-name }} - path: downloads - - - name: Debug step - Display structure of downloaded files - run: ls -R - working-directory: downloads - - - name: Upload binary, provenance, signed tag to Ent - id: ent_upload_binary - working-directory: downloads - # The output on any trigger other than "pull_request" has an additional - # ".sigstore" suffix. However, that suffix appears to be ".build.slsa". - # See https://github.com/slsa-framework/slsa-github-generator/tree/main/internal/builders/docker#workflow-outputs - # The artifact path may be a wildcard that resolves to multiple files. - run: | - set -o errexit - set -o nounset - set -o xtrace - set -o pipefail - provenance_file="attestation.intoto" - if [[ "${{ github.event_name }}" != "pull_request" ]]; then - provenance_file="${provenance_file}.build.slsa" - fi - package_name=${{ needs.get_inputs.outputs.package-name }} - binary_file=${{ needs.get_inputs.outputs.artifact-path }} - binary_digest="$(ent put --digest-format=human --porcelain "${binary_file}")" - provenance_digest="$(ent put --digest-format=human --porcelain "${provenance_file}")" - ent tag set --public-key=${{ inputs.ent-public-key }} \ - --label="artifact_${GITHUB_SHA}_${package_name}" \ - --target="${binary_digest}" - ent tag set --public-key=${{ inputs.ent-public-key }} \ - --label="provenance_${GITHUB_SHA}_${package_name}" \ - --target=${provenance_digest} - - # Debug step similar to `upload_provenance`, but runs on pull-request events. - # Differs from `upload_provenance` in that it does not publish the binary - # and its provenance into Ent and that it does not post a comment on the PR. - debug_provenance: - if: github.event_name == 'pull_request' - needs: [get_inputs, generate_provenance] - runs-on: ubuntu-20.04 - - steps: - - name: Download the built artifact - uses: actions/download-artifact@v3 - with: - name: ${{ needs.generate_provenance.outputs.build-outputs-name }} - path: downloads - - - name: Download the DSSE document - uses: actions/download-artifact@v3 - with: - name: - ${{ needs.generate_provenance.outputs.attestations-download-name }} - path: downloads - - - name: Display structure after downloading the files (debug step) - run: ls -R - working-directory: downloads - - - name: Print binary digest - working-directory: downloads - run: echo "$(sha256sum ${{ needs.get_inputs.outputs.artifact-path }})" - - - name: Print provenance digest - working-directory: downloads - run: | - echo "$(sha256sum attestation.intoto)"