From b1f05e8d3f8b323a811a0413bece92cb1a561227 Mon Sep 17 00:00:00 2001 From: Brandon Schurman Date: Fri, 31 May 2024 14:05:04 -0400 Subject: [PATCH] reusable smoke test for BYOC and self-hosted satellites --- .github/actions/stage2-setup/action.yml | 20 +- .github/workflows/README.md | 14 +- .github/workflows/build-earthly.yml | 14 +- .github/workflows/ci-docker-satellites.yml | 26 ++ .github/workflows/ci-docker-ubuntu.yml | 161 ++----- .../ci-earthly-next-docker-ubuntu.yml | 196 ++++++++ .github/workflows/ci-lint-docs.yml | 4 +- .github/workflows/ci-podman-ubuntu.yml | 13 + .github/workflows/ci-staging-deploy.yml | 1 + .github/workflows/reusable-misc-tests-1.yml | 2 +- .../reusable-satellite-smoke-test.yml | 70 +++ .github/workflows/reusable-test.yml | 11 + CHANGELOG.md | 58 +++ Earthfile | 23 +- ast/commandflag/flags.go | 1 + builder/builder.go | 8 +- builder/builder_error.go | 36 -- builder/image_solver.go | 29 +- builder/solver.go | 22 +- buildkitd/Earthfile | 2 +- buildkitd/buildkitd.go | 34 +- buildkitd/dockerd-wrapper.sh | 15 +- cloud/client.go | 4 + cloud/cloud_installation.go | 75 +++- cloud/github.go | 2 +- cloud/http.go | 19 - cloud/log.go | 33 -- cloud/secret.go | 29 ++ cmd/earthly/app/before.go | 80 ++-- cmd/earthly/base/init_frontend.go | 11 + cmd/earthly/flag/global.go | 29 +- cmd/earthly/subcmd/build_cmd.go | 45 +- cmd/earthly/subcmd/cloud_installation_cmds.go | 109 ++++- conslogging/bundle.go | 309 ------------- conslogging/conslogging.go | 69 --- docs/SUMMARY.md | 1 + .../part-6-using-docker-with-earthly.md | 16 +- docs/basics/part-7-using-remote-runners.md | 6 +- ...art-8a-using-earthly-in-your-current-ci.md | 4 +- docs/caching/managing-cache.md | 2 +- .../build-an-earthly-ci-image.md | 8 +- .../guides/bitbucket-pipelines-integration.md | 2 +- .../guides/circle-integration.md | 2 +- .../guides/codebuild-integration.md | 2 +- .../guides/gh-actions-integration.md | 7 +- .../guides/gitlab-integration.md | 2 +- .../guides/google-cloud-build.md | 4 +- docs/ci-integration/guides/kubernetes.md | 4 +- .../guides/woodpecker-integration.md | 2 +- docs/ci-integration/overview.md | 2 +- docs/ci-integration/pull-through-cache.md | 2 +- docs/ci-integration/remote-buildkit.md | 4 +- docs/ci-integration/use-earthly-ci-image.md | 16 +- docs/cloud/oidc.md | 84 ++++ docs/cloud/overview.md | 3 +- docs/cloud/satellites/best-practices.md | 4 +- docs/cloud/satellites/gha-runners.md | 125 ++++++ docs/cloud/satellites/self-hosted.md | 16 +- docs/cloud/satellites/using.md | 2 +- docs/docker-images/all-in-one.md | 24 +- docs/docker-images/buildkit-standalone.md | 2 +- docs/earthfile/earthfile.md | 68 ++- docs/earthfile/features.md | 81 ++-- docs/earthly-command/earthly-command.md | 2 +- docs/earthly-config/earthly-config.md | 12 +- docs/examples/examples.md | 4 +- docs/guides/auth.md | 10 +- docs/guides/best-practices.md | 10 +- docs/guides/cloud-providers/aws.md | 33 ++ docs/guides/docker-in-earthly.md | 12 +- docs/guides/importing.md | 10 +- docs/guides/integration.md | 4 +- docs/guides/podman.md | 2 +- docs/guides/registries/aws-ecr.md | 8 +- docs/guides/registries/self-signed.md | 2 +- docs/guides/target-ref.md | 10 +- docs/remote-runners.md | 20 +- earthfile2llb/converter.go | 25 +- earthfile2llb/interpreter.go | 41 +- earthly-next | 1 + examples/grpc/Earthfile | 2 +- examples/integration-test/Earthfile | 2 +- examples/mkdocs/Pipfile.lock | 207 +++++---- examples/next-js-netlify/package-lock.json | 265 ++++++----- examples/react/Earthfile | 2 +- examples/ruby-on-rails/Gemfile.lock | 4 +- examples/tutorial/go/part6/Earthfile | 2 +- examples/tutorial/java/part6/Earthfile | 2 +- examples/tutorial/js/part6/Earthfile | 2 +- examples/tutorial/python/part6/Earthfile | 2 +- features/features.go | 1 + go.mod | 12 +- go.sum | 20 +- logbus/formatter/formatter.go | 17 +- outmon/solvermon.go | 420 ------------------ outmon/vertexmon.go | 269 ----------- release/Earthfile | 26 +- release/README.md | 6 +- release/release.sh | 30 +- scripts/tests/backwards-compatability.sh | 66 +++ tests/Earthfile | 35 +- tests/aws-flag.earth | 6 + tests/oidc/Earthfile | 39 ++ tests/oidc/aws.earth | 15 + tests/oidc/test-aws.sh | 13 + tests/run-no-cache-save-artifact.earth | 20 + util/containerutil/frontend.go | 4 +- util/containerutil/settings_test.go | 14 +- util/containerutil/shell_shared.go | 8 +- util/containerutil/types.go | 8 +- util/deltautil/go.mod | 2 +- util/deltautil/go.sum | 4 +- util/fsutilprogress/progress.go | 2 +- util/llbutil/secretprovider/aws_creds.go | 169 ++++++- util/oidcutil/aws.go | 152 +++++++ util/oidcutil/aws_test.go | 215 +++++++++ util/parseutil/map.go | 24 + util/parseutil/map_test.go | 49 ++ util/stringutil/scrub.go | 7 + util/stringutil/scrub_test.go | 5 + 120 files changed, 2472 insertions(+), 1951 deletions(-) create mode 100644 .github/workflows/ci-earthly-next-docker-ubuntu.yml create mode 100644 .github/workflows/reusable-satellite-smoke-test.yml delete mode 100644 builder/builder_error.go delete mode 100644 cloud/log.go delete mode 100644 conslogging/bundle.go create mode 100644 docs/cloud/oidc.md create mode 100644 docs/cloud/satellites/gha-runners.md create mode 100644 docs/guides/cloud-providers/aws.md create mode 100644 earthly-next delete mode 100644 outmon/solvermon.go delete mode 100644 outmon/vertexmon.go create mode 100755 scripts/tests/backwards-compatability.sh create mode 100644 tests/oidc/Earthfile create mode 100644 tests/oidc/aws.earth create mode 100755 tests/oidc/test-aws.sh create mode 100644 tests/run-no-cache-save-artifact.earth create mode 100644 util/oidcutil/aws.go create mode 100644 util/oidcutil/aws_test.go create mode 100644 util/parseutil/map.go create mode 100644 util/parseutil/map_test.go diff --git a/.github/actions/stage2-setup/action.yml b/.github/actions/stage2-setup/action.yml index 0e11630036..727f4f1ffe 100644 --- a/.github/actions/stage2-setup/action.yml +++ b/.github/actions/stage2-setup/action.yml @@ -18,6 +18,10 @@ inputs: required: false type: boolean default: false + USE_NEXT: + required: false + type: boolean + default: false SATELLITE_NAME: required: false type: string @@ -67,16 +71,26 @@ runs: shell: bash - run: |- echo "Extracting earthly binary from stage1 of build" - BUILDKITD_IMAGE=docker.io/earthly/buildkitd-staging TAG=${GITHUB_SHA}-ubuntu-latest-${{inputs.BINARY}} ${{inputs.SUDO}} ./earthly upgrade + export TAG=${GITHUB_SHA}-ubuntu-latest-${{inputs.BINARY}} + if [ "${{inputs.USE_NEXT}}" = "true" ]; then export TAG="$TAG-ticktock"; fi + BUILDKITD_IMAGE=docker.io/earthly/buildkitd-staging ${{inputs.SUDO}} ./earthly upgrade ${{inputs.SUDO}} chown -R $USER ~/.earthly # restore non-sudo user ownership test -n "${{inputs.BUILT_EARTHLY_PATH}}" || (echo "BUILT_EARTHLY_PATH is empty" && exit 1) mkdir -p $(dirname "${{inputs.BUILT_EARTHLY_PATH}}") - ${{inputs.SUDO}} mv ${HOME}/.earthly/earthly-${GITHUB_SHA}-ubuntu-latest-${{inputs.BINARY}} "${{inputs.BUILT_EARTHLY_PATH}}" + ${{inputs.SUDO}} ls "${HOME}/.earthly/" + ${{inputs.SUDO}} mv "${HOME}/.earthly/earthly-$TAG" "${{inputs.BUILT_EARTHLY_PATH}}" echo "extracted ${{inputs.BUILT_EARTHLY_PATH}}" shell: bash + - if: ${{ inputs.USE_NEXT == 'true' }} + run: |- + export expected_buildkit_client_sha="$(cat earthly-next | head -c 12)" + test -n "$expected_buildkit_client_sha" || ( echo "expected_buildkit_client_sha is empty" && exit 1) + (strings ${{inputs.BUILT_EARTHLY_PATH}} | grep "$expected_buildkit_client_sha" ) || ( echo "expected to find $expected_buildkit_client_sha in earthly binary" && exit 1) + echo "correctly found $expected_buildkit_client_sha in earthly binary; this confirms earthly-next was used" + shell: bash - run: |- echo "Setting up mirror credentials in .arg and .secret" - export earthly=${{inputs.BUILT_EARTHLY_PATH}} + export earthly="${{inputs.BUILT_EARTHLY_PATH}}" # setup secrets echo "DOCKERHUB_MIRROR_USER=$($earthly secret --org earthly-technologies --project core get -n dockerhub-mirror/user || kill $$)" > .secret echo "DOCKERHUB_MIRROR_PASS=$($earthly secret --org earthly-technologies --project core get -n dockerhub-mirror/pass || kill $$)" >> .secret diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 91d40e6e0e..3516e7b1f6 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -1,6 +1,6 @@ -# Github Actions +# GitHub Actions -Documentation for this repo Github actions configuration +Documentation for this repo GitHub actions configuration ## Skipping PR Workflows (DISABLED! SEE NOTE BELOW) @@ -12,8 +12,8 @@ Some PR workflows (workflows triggered by PRs) in this repo might take substanti and it is not always necessary to run them all, especially in cases where the affected files are documents or any other file that don't affect our workflows or tests. In addition, some of our tests use satellites and limiting the number of (concurrent) builds might help with their performance. -### Github Builtin Support -Github supports defining filters on workflows so that they won't get triggered, and there is a builtin filter that would do so according to affected files - [paths-ignore](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore). +### GitHub Builtin Support +GitHub supports defining filters on workflows so that they won't get triggered, and there is a builtin filter that would do so according to affected files - [paths-ignore](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore). The shortcoming of this filter is that when the workflow is marked as a `required check` and does not get triggered because of this filter, the workflow will be stuck in `pending` and won't allow the PR to be merged. Thus, using this filter is a good approach when handling workflows that are not required checks, but it's problematic otherwise. @@ -22,18 +22,18 @@ Thus, using this filter is a good approach when handling workflows that are not An alternative to `paths-ignore` is examining the affected files at the `job` level instead. Jobs allow you to set conditions on whether they should execute or not. The advantage of using conditions on the job level is that if a job does not run due to a condition evaluation to false, -Github will still mark the workflow as passing/successful instead of pending, +GitHub will still mark the workflow as passing/successful instead of pending, and will not block the PR even if the workflow is marked as a required check ([reference](https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution#overview)). To examine the files, [dorny/paths-filter](https://github.com/dorny/paths-filter) is used. -Similarly to Github's `paths-ignore`, `dorny/paths-filter` supports setting rules that match against the affected files, which evaluates to a boolean value. +Similarly to GitHub's `paths-ignore`, `dorny/paths-filter` supports setting rules that match against the affected files, which evaluates to a boolean value. We can then use that boolean value to determine whether subsequent jobs in the workflow should execute or not. ### Complexity due to reusable workflows #### Background A problem arises when the job we wish to conditionally run uses a reusable workflow. -The problem is that Github treats such configuration as a parent-child relationship, +The problem is that GitHub treats such configuration as a parent-child relationship, and any job that is skipped, by default will also cause its children jobs to skip as well. In addition, when we set `required checks`, we cannot properly select the parent job, only the child job. In this scenario, if a child job is skipped as a result of skipping its parent, the child job will get stuck in `pending` and the PR will be blocked from merging (just like in the original problem). diff --git a/.github/workflows/build-earthly.yml b/.github/workflows/build-earthly.yml index 7e96f77ece..7d6c72e3e8 100644 --- a/.github/workflows/build-earthly.yml +++ b/.github/workflows/build-earthly.yml @@ -19,11 +19,15 @@ on: required: false type: boolean default: false + USE_NEXT: + required: false + type: boolean + default: false jobs: build-earthly: - name: build (and push) earthly using ${{inputs.BINARY}} + name: build (and push) earthly using ${{inputs.BINARY}} and earthly-next=${{inputs.USE_NEXT}} if: ${{!inputs.SKIP_JOB}} runs-on: ${{inputs.RUNS_ON}} env: @@ -71,6 +75,9 @@ jobs: - name: Podman Login run: ${{inputs.SUDO}} ${{inputs.BINARY}} login docker.io --username "${{ secrets.DOCKERHUB_USERNAME }}" --password "${{ secrets.DOCKERHUB_TOKEN }}" if: inputs.BINARY == 'podman' + - name: Update Buildkit to earthly-next + if: inputs.USE_NEXT + run: ${{inputs.SUDO}} $(which earthly) +update-buildkit --BUILDKIT_GIT_SHA=$(cat earthly-next) - name: Build latest earthly using released earthly run: ${{inputs.SUDO}} $(which earthly) --use-inline-cache +for-linux - name: Earthly bootstrap using latest earthly build @@ -81,7 +88,10 @@ jobs: EARTHLY_VERSION_FLAG_OVERRIDES="$(tr -d '\n' < .earthly_version_flag_overrides)" echo "EARTHLY_VERSION_FLAG_OVERRIDES=$EARTHLY_VERSION_FLAG_OVERRIDES" >> "$GITHUB_ENV" - name: Build and push +ci-release using latest earthly build - run: ${{inputs.SUDO}} ./build/linux/amd64/earthly --ci --push +ci-release --TAG_SUFFIX="${{inputs.RUNS_ON}}-${{inputs.BINARY}}" + run: |- + export TAG_SUFFIX="${{inputs.RUNS_ON}}-${{inputs.BINARY}}" + if [ "${{inputs.USE_NEXT}}" = "true" ]; then export TAG_SUFFIX="$TAG_SUFFIX-ticktock"; fi + ${{inputs.SUDO}} ./build/linux/amd64/earthly --ci --push --build-arg TAG_SUFFIX +ci-release - name: Buildkit logs (runs on failure) run: ${{inputs.SUDO}} ${{inputs.BINARY}} logs earthly-buildkitd if: ${{ failure() }} diff --git a/.github/workflows/ci-docker-satellites.yml b/.github/workflows/ci-docker-satellites.yml index b2a9d0b2b2..de29f22d31 100644 --- a/.github/workflows/ci-docker-satellites.yml +++ b/.github/workflows/ci-docker-satellites.yml @@ -73,3 +73,29 @@ jobs: SATELLITE_NAME: "core-test" EARTHLY_ORG: "earthly-technologies" secrets: inherit + + satellites-byoc-smoke-test: + needs: build-earthly + uses: ./.github/workflows/reusable-satellite-smoke-test.yml + with: + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + SUDO: "" + SATELLITE_NAME: "core-byoc" + EARTHLY_ORG: "earthly-technologies" + USE_VPN: true + secrets: inherit + + satellites-self-hosted-smoke-test: + needs: build-earthly + uses: ./.github/workflows/reusable-satellite-smoke-test.yml + with: + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + SUDO: "" + SATELLITE_NAME: "self-hosted" + EARTHLY_ORG: "earthly-technologies" + USE_VPN: true + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/ci-docker-ubuntu.yml b/.github/workflows/ci-docker-ubuntu.yml index 847fcddfe0..230c9a31f0 100644 --- a/.github/workflows/ci-docker-ubuntu.yml +++ b/.github/workflows/ci-docker-ubuntu.yml @@ -95,34 +95,6 @@ jobs: EXTRA_ARGS: "--auto-skip" secrets: inherit - docker-tests-no-qemu-group1-no-logbus: - needs: build-earthly - if: ${{ !failure() }} - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "+test-no-qemu-group1" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "docker" - SUDO: "" - EXTRA_ARGS: "--logstream=false --logstream-upload=false --auto-skip" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - secrets: inherit - - docker-tests-no-qemu-group2-no-logbus: - needs: build-earthly - if: ${{ !failure() }} - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "+test-no-qemu-group2" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "docker" - SUDO: "" - EXTRA_ARGS: "--logstream=false --logstream-upload=false --auto-skip" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - secrets: inherit - docker-tests-no-qemu-group3: needs: build-earthly if: ${{ !failure() }} @@ -137,20 +109,6 @@ jobs: EXTRA_ARGS: "--auto-skip" secrets: inherit - docker-tests-no-qemu-group3-no-logbus: - needs: build-earthly - if: ${{ !failure() }} - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "+test-no-qemu-group3" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "docker" - SUDO: "" - EXTRA_ARGS: "--logstream=false --logstream-upload=false --auto-skip" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - secrets: inherit - docker-tests-no-qemu-group4: needs: build-earthly if: ${{ !failure() }} @@ -165,20 +123,6 @@ jobs: EXTRA_ARGS: "--auto-skip" secrets: inherit - docker-tests-no-qemu-group4-no-logbus: - needs: build-earthly - if: ${{ !failure() }} - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "+test-no-qemu-group4" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "docker" - SUDO: "" - EXTRA_ARGS: "--logstream=false --logstream-upload=false --auto-skip" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - secrets: inherit - docker-tests-no-qemu-group5: needs: build-earthly if: ${{ !failure() }} @@ -193,20 +137,6 @@ jobs: EXTRA_ARGS: "--auto-skip" secrets: inherit - docker-tests-no-qemu-group5-no-logbus: - needs: build-earthly - if: ${{ !failure() }} - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "+test-no-qemu-group5" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "docker" - SUDO: "" - EXTRA_ARGS: "--logstream=false --logstream-upload=false --auto-skip" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - secrets: inherit - docker-tests-no-qemu-group6: needs: build-earthly if: ${{ !failure() }} @@ -221,20 +151,6 @@ jobs: EXTRA_ARGS: "--auto-skip" secrets: inherit - docker-tests-no-qemu-group6-no-logbus: - needs: build-earthly - if: ${{ !failure() }} - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "+test-no-qemu-group6" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "docker" - SUDO: "" - EXTRA_ARGS: "--logstream=false --logstream-upload=false --auto-skip" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - secrets: inherit - docker-tests-no-qemu-group7: needs: build-earthly if: ${{ !failure() }} @@ -249,20 +165,6 @@ jobs: EXTRA_ARGS: "--auto-skip" secrets: inherit - docker-tests-no-qemu-group7-no-logbus: - needs: build-earthly - if: ${{ !failure() }} - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "+test-no-qemu-group7" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "docker" - SUDO: "" - EXTRA_ARGS: "--logstream=false --logstream-upload=false --auto-skip" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - secrets: inherit - docker-tests-no-qemu-group8: needs: build-earthly if: ${{ !failure() }} @@ -277,20 +179,6 @@ jobs: EXTRA_ARGS: "--auto-skip" secrets: inherit - docker-tests-no-qemu-group8-no-logbus: - needs: build-earthly - if: ${{ !failure() }} - uses: ./.github/workflows/reusable-test.yml - with: - TEST_TARGET: "+test-no-qemu-group8" - BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - RUNS_ON: "ubuntu-latest" - BINARY: "docker" - SUDO: "" - EXTRA_ARGS: "--logstream=false --logstream-upload=false --auto-skip" - SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - secrets: inherit - docker-tests-no-qemu-group9: needs: build-earthly if: ${{ !failure() }} @@ -473,6 +361,20 @@ jobs: EXTRA_ARGS: "--auto-skip" secrets: inherit + docker-test-oidc-command: + if: ${{ !failure() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} + needs: build-earthly + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "./tests/oidc+test" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + docker-tests-qemu: needs: build-earthly if: ${{ !failure() }} @@ -860,6 +762,41 @@ jobs: EXTRA_ARGS: "--auto-skip" secrets: inherit + backwards-compatability-test: + needs: build-earthly + name: Backwards Compatability + runs-on: ubuntu-latest + env: + FORCE_COLOR: 1 + EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" + EARTHLY_INSTALL_ID: "earthly-githubactions" + # Used in our github action as the token - TODO: look to change it into an input + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Docker mirror login (Earthly Only) + run: docker login registry-1.docker.io.mirror.corp.earthly.dev --username "${{ secrets.DOCKERHUB_MIRROR_USERNAME }}" --password "${{ secrets.DOCKERHUB_MIRROR_PASSWORD }}" + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository + - name: Retrieve earthly from build-earthly job + run: |- + BUILDKITD_IMAGE=docker.io/earthly/buildkitd-staging TAG=${GITHUB_SHA}-ubuntu-latest-docker ./earthly upgrade + mkdir -p $(dirname "./build/earthly") + mv ${HOME}/.earthly/earthly-${GITHUB_SHA}-ubuntu-latest-docker ./build/earthly + - name: Configure Earthly to use mirror (Earthly Only) + run: |- + ./build/earthly config global.buildkit_additional_config "'[registry.\"docker.io\"] + mirrors = [\"registry-1.docker.io.mirror.corp.earthly.dev\"]'" + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository + - name: Download older version of earthly (v0.8.0) + run: |- + sudo wget https://github.com/earthly/earthly/releases/download/v0.8.0/earthly-linux-amd64 -O /usr/bin/earthly-v0.8.0 + sudo chmod +x /usr/bin/earthly-v0.8.0 + earthly-v0.8.0 --version + - name: execute backwards compatability script + run: earthly=./build/earthly scripts/tests/backwards-compatability.sh + tutorial: needs: build-earthly name: Tutorial diff --git a/.github/workflows/ci-earthly-next-docker-ubuntu.yml b/.github/workflows/ci-earthly-next-docker-ubuntu.yml new file mode 100644 index 0000000000..1589d7e1c7 --- /dev/null +++ b/.github/workflows/ci-earthly-next-docker-ubuntu.yml @@ -0,0 +1,196 @@ +name: Earthly Next Docker CI Ubuntu + +on: + push: + branches: [ main ] + paths-ignore: + - 'docs/**' + - '**.md' + - '.github/renovate.json5' + - '.github/CODEOWNERS' + - 'LICENSE' + pull_request: + branches: [ main ] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build-earthly-with-next: + uses: ./.github/workflows/build-earthly.yml + with: + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group1: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group1" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group2: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group2" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group3: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group3" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group4: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group4" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group5: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group5" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group6: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group6" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group7: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group7" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group8: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group8" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group9: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group9" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group10: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group10" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group11: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group11" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group12: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group12" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit diff --git a/.github/workflows/ci-lint-docs.yml b/.github/workflows/ci-lint-docs.yml index b25ee972cc..066e658e15 100644 --- a/.github/workflows/ci-lint-docs.yml +++ b/.github/workflows/ci-lint-docs.yml @@ -2,9 +2,9 @@ name: lint-docs on: push: - branches: [ main, next ] + branches: [ main, docs-0.8 ] pull_request: - branches: [ main, next ] + branches: [ main, docs-0.8 ] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} diff --git a/.github/workflows/ci-podman-ubuntu.yml b/.github/workflows/ci-podman-ubuntu.yml index c565866aa8..66cc0e4dc5 100644 --- a/.github/workflows/ci-podman-ubuntu.yml +++ b/.github/workflows/ci-podman-ubuntu.yml @@ -326,6 +326,19 @@ jobs: SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} secrets: inherit + podman-test-oidc-command: + if: ${{ !failure() && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) }} + needs: build-earthly + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "./tests/oidc+test" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "podman" + SUDO: "sudo -E" + SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} + secrets: inherit + podman-tests-qemu: needs: build-earthly if: ${{ !failure() }} diff --git a/.github/workflows/ci-staging-deploy.yml b/.github/workflows/ci-staging-deploy.yml index 452f6d5619..0142ff47da 100644 --- a/.github/workflows/ci-staging-deploy.yml +++ b/.github/workflows/ci-staging-deploy.yml @@ -72,6 +72,7 @@ jobs: export SHA_DEC="$(echo "ibase=16; $(git rev-parse --short HEAD | tr '[:lower:]' '[:upper:]')" | bc)" export RELEASE_TAG="v0.$(date +%s).$SHA_DEC" export SKIP_CHANGELOG_DATE_TEST=true + export EARTHLY_STAGING=true export S3_BUCKET="staging-pkg" export earthly="./build/linux/amd64/earthly" echo "attempting staging-release version: $RELEASE_TAG" diff --git a/.github/workflows/reusable-misc-tests-1.yml b/.github/workflows/reusable-misc-tests-1.yml index 09785f3779..21c7694128 100644 --- a/.github/workflows/reusable-misc-tests-1.yml +++ b/.github/workflows/reusable-misc-tests-1.yml @@ -79,7 +79,7 @@ jobs: - name: Execute registry-certs test run: frontend=${{inputs.BINARY}} ./tests/registry-certs/test.sh - name: Execute try-catch test - run: ./tests/try-catch/test.sh + run: GITHUB_ACTIONS="" ./tests/try-catch/test.sh - name: Buildkit logs (runs on failure) run: ${{inputs.SUDO}} ${{inputs.BINARY}} logs earthly-buildkitd || true if: ${{ failure() }} diff --git a/.github/workflows/reusable-satellite-smoke-test.yml b/.github/workflows/reusable-satellite-smoke-test.yml new file mode 100644 index 0000000000..07d2c49f14 --- /dev/null +++ b/.github/workflows/reusable-satellite-smoke-test.yml @@ -0,0 +1,70 @@ +name: Satellite Smoke Test + +on: + workflow_call: + inputs: + BUILT_EARTHLY_PATH: + required: true + type: string + RUNS_ON: + required: true + type: string + SUDO: + type: string + required: false + BINARY: + required: true + type: string + SATELLITE_NAME: + required: true + type: string + EARTHLY_ORG: + required: false + type: string + USE_VPN: + required: false + type: boolean + +jobs: + satellite-smoke-test: + runs-on: ${{inputs.RUNS_ON}} + env: + FORCE_COLOR: 1 + EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" + EARTHLY_ORG: "${{inputs.EARTHLY_ORG}}" + EARTHLY_SATELLITE: ${{ inputs.SATELLITE_NAME }} + EARTHLY_INSTALL_ID: "earthly-githubactions" + DOCKERHUB_MIRROR_USERNAME: "${{ secrets.DOCKERHUB_MIRROR_USERNAME }}" + DOCKERHUB_MIRROR_PASSWORD: "${{ secrets.DOCKERHUB_MIRROR_PASSWORD }}" + # Used in our github action as the token - TODO: look to change it into an input + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - uses: ./.github/actions/stage2-setup + with: + DOCKERHUB_MIRROR_USERNAME: "${{ secrets.DOCKERHUB_MIRROR_USERNAME }}" + DOCKERHUB_MIRROR_PASSWORD: "${{ secrets.DOCKERHUB_MIRROR_PASSWORD }}" + DOCKERHUB_USERNAME: "${{ secrets.DOCKERHUB_USERNAME }}" + DOCKERHUB_PASSWORD: "${{ secrets.DOCKERHUB_TOKEN }}" + EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" + BINARY: "${{ inputs.BINARY }}" + SUDO: "${{ inputs.SUDO }}" + BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" + SATELLITE_NAME: "${{ inputs.SATELLITE_NAME }}" + USE_SATELLITE: true + - name: Configure Tailscale + uses: tailscale/github-action@v2 + if: ${{ inputs.USE_VPN }} + with: + oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} + oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} + tags: tag:ci + - name: Set EARTHLY_VERSION_FLAG_OVERRIDES env + run: |- + set -euo pipefail + EARTHLY_VERSION_FLAG_OVERRIDES="$(tr -d '\n' < .earthly_version_flag_overrides)" + echo "EARTHLY_VERSION_FLAG_OVERRIDES=$EARTHLY_VERSION_FLAG_OVERRIDES" >> "$GITHUB_ENV" + - name: Run satellite smoke test + run: ${{inputs.BUILT_EARTHLY_PATH}} github.com/earthly/hello-world+hello \ No newline at end of file diff --git a/.github/workflows/reusable-test.yml b/.github/workflows/reusable-test.yml index 474980cab6..f320154d2b 100644 --- a/.github/workflows/reusable-test.yml +++ b/.github/workflows/reusable-test.yml @@ -38,6 +38,10 @@ on: required: false type: boolean default: false + USE_NEXT: + required: false + type: boolean + default: false jobs: test: @@ -67,6 +71,7 @@ jobs: BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" BINARY: "${{ inputs.BINARY }}" USE_QEMU: "${{ inputs.USE_QEMU }}" + USE_NEXT: "${{ inputs.USE_NEXT }}" SUDO: "${{ inputs.SUDO }}" USE_SATELLITE: "${{ inputs.USE_SATELLITE }}" SATELLITE_NAME: "${{ inputs.SATELLITE_NAME }}" @@ -78,6 +83,12 @@ jobs: - name: Execute ${{ inputs.TEST_TARGET }} (Earthly Only) run: |- ${{inputs.SUDO}} ${{ inputs.BUILT_EARTHLY_PATH }} ${{inputs.EXTRA_ARGS}} --ci -P ${{inputs.TEST_TARGET}} + - name: Display buildkit version + run: |- + ${{inputs.SUDO}} ${{inputs.BINARY}} ps -a + ${{inputs.SUDO}} ${{inputs.BINARY}} logs earthly-buildkitd |& grep 'starting earthly-buildkit' + shell: bash + if: ${{ ! inputs.USE_SATELLITE }} - name: Execute fail test run: | ! ${{inputs.SUDO}} GITHUB_ACTIONS="" ${{ inputs.BUILT_EARTHLY_PATH }} ${{inputs.EXTRA_ARGS}} --ci ./tests/fail+test-fail diff --git a/CHANGELOG.md b/CHANGELOG.md index 829874ca12..08c61df218 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,66 @@ All notable changes to [Earthly](https://github.com/earthly/earthly) will be doc ## Unreleased +## v0.8.13 - 2024-05-28 + +### Fixed +- `RUN --no-cache` doesn't always work. [#2593](https://github.com/earthly/earthly/issues/2593) +- ANSI escape codes are now removed from GitHub Action specific output. [#4131](https://github.com/earthly/earthly/issues/4131) +- Specifying the `--ticktock` option will no longer show a buildkit version mismatch warning. +- The `--ticktock` flag is now used when running the bootstrap command. +- Caching issue when the experimental `--ticktock` buildkit scheduler is enabled. + +### Changed +- Removed deprecated non-streaming log uploading mechanism; log sharing is now only supported by the streaming-upload mechanism. The hidden `--logstream` and `--logstream-upload` flags have been removed. + +### Additional Info +- This release includes changes to buildkit + +## v0.8.12 - 2024-05-23 + +### Added +- An experimental modification of the buildkit scheduler, which attempts to solve the `inconsistent graph state` error, which can be enabled locally with `earthly --ticktock ...`. + +### Changed +- The BYOC (bring your own cloud) commands have been updated to reflect server-side API changes. + +### Fixed +- The `--buildkit-container-name` flag was incorrectly being ignored when `--no-buildkit-update` was set. + +### Additional Info +- This release includes changes to buildkit + +## v0.8.11 - 2024-05-16 + +### Added +- Support for using HTTP(S) proxies when connecting to satellites. + +### Fixed +- Backwards compatability issue where `WITH DOCKER` would fail with `EARTHLY_DOCKERD_CACHE_DATA: parameter not set` when using an older version of the earthly in combination with a satellite running v0.8.10. + +### Additional Info +- This release includes changes to buildkit + +## v0.8.10 - 2024-05-14 + ### Added - New Github Actions Workflow commands integration `--github-annotations` flag or GITHUB_ACTIONS=true env. [#2189](https://github.com/earthly/earthly/issues/2189) +- Added a new `--oidc` flag to `RUN` command which allows authentication to AWS via OIDC. Enable with the `VERSION --run-with-aws-oidc` feature flag. [#3804](https://github.com/earthly/earthly/issues/3804) +- Experimental `WITH DOCKER --cache-id=` feature, which will cache the contents of the docker data root, resulting in faster `--load` and `--pull` execution. Enabled with the `VERSION --docker-cache` feature flag. [#3399](https://github.com/earthly/earthly/issues/3399) +- New `SAVE IMAGE --without-earthly-labels` feature, which will prevent any `dev.earthly.*` labels from being saved to the image. Enable with the `VERSION --allow-without-earthly-labels` feature flag. Thanks to [@3manuek](https://github.com/3manuek) for the contribution! + +### Fixed +- `WITH DOCKER` load time calculation. [#3485](https://github.com/earthly/earthly/issues/3485) +- The earthly cli was not correctly setting the exit status on failures when executing a `RUN` on a satellite which reached the max execution time limit. +- Self-hosted satellite connection issue. + +### Changed +- Earthly will now use source link format when displaying errors, e.g. `::` rather than ` line :`. +- Improved error messages for cases where a shell is required to run a command such as `IF`, `FOR`, etc. +- Earthly will now show a warning when earthly anonymously connects to a registry (which increases the chance of being rate-limited). + +### Additional Info +- This release includes changes to buildkit ## v0.8.9 - 2024-04-24 diff --git a/Earthfile b/Earthfile index d3a857ce96..4326178f28 100644 --- a/Earthfile +++ b/Earthfile @@ -63,7 +63,7 @@ code: END COPY ./ast/parser+parser/*.go ./ast/parser/ COPY --dir analytics autocomplete billing buildcontext builder logbus cleanup cloud cmd config conslogging debugger \ - dockertar docker2earthly domain features internal outmon slog states util variables regproxy ./ + dockertar docker2earthly domain features internal slog states util variables regproxy ./ COPY --dir buildkitd/buildkitd.go buildkitd/settings.go buildkitd/certificates.go buildkitd/ COPY --dir earthfile2llb/*.go earthfile2llb/ COPY --dir ast/antlrhandler ast/spec ast/hint ast/command ast/commandflag ast/*.go ast/ @@ -561,15 +561,29 @@ for-own: # the documentation on +earthly for extra detail about this option. ARG GO_GCFLAGS BUILD ./buildkitd+buildkitd --BUILDKIT_PROJECT="$BUILDKIT_PROJECT" + BUILD +build-ticktock COPY (+earthly/earthly --GO_GCFLAGS="${GO_GCFLAGS}") ./ SAVE ARTIFACT ./earthly AS LOCAL ./build/own/earthly +# build-ticktock is used for building the ticktock version of buildkit +# it is only used when BUILDKIT_PROJECT is not overridden +build-ticktock: + ARG BUILDKIT_PROJECT + IF [ -z "$BUILDKIT_PROJECT" ] + COPY earthly-next . + LET ticktock="$(cat earthly-next)" + ARG EARTHLY_TARGET_TAG_DOCKER + LET BUILDKIT_TAG="dev-$EARTHLY_TARGET_TAG_DOCKER-ticktock" + BUILD --platform=linux/amd64 ./buildkitd+buildkitd --BUILDKIT_PROJECT="github.com/earthly/buildkit:$ticktock" --TAG=$BUILDKIT_TAG + END + # for-linux builds earthly-buildkitd and the earthly CLI for the a linux amd64 system # and saves the final CLI binary locally in the ./build/linux folder. for-linux: ARG BUILDKIT_PROJECT ARG GO_GCFLAGS BUILD --platform=linux/amd64 ./buildkitd+buildkitd --BUILDKIT_PROJECT="$BUILDKIT_PROJECT" + BUILD --platform=linux/amd64 +build-ticktock BUILD ./ast/parser+parser COPY (+earthly-linux-amd64/earthly --GO_GCFLAGS="${GO_GCFLAGS}") ./ SAVE ARTIFACT ./earthly AS LOCAL ./build/linux/amd64/earthly @@ -580,6 +594,7 @@ for-linux-arm64: ARG BUILDKIT_PROJECT ARG GO_GCFLAGS BUILD --platform=linux/arm64 ./buildkitd+buildkitd --BUILDKIT_PROJECT="$BUILDKIT_PROJECT" + BUILD --platform=linux/arm64 +build-ticktock BUILD ./ast/parser+parser COPY (+earthly-linux-arm64/earthly --GO_GCFLAGS="${GO_GCFLAGS}") ./ SAVE ARTIFACT ./earthly AS LOCAL ./build/linux/arm64/earthly @@ -591,6 +606,7 @@ for-darwin: ARG BUILDKIT_PROJECT ARG GO_GCFLAGS BUILD --platform=linux/amd64 ./buildkitd+buildkitd --BUILDKIT_PROJECT="$BUILDKIT_PROJECT" + BUILD --platform=linux/amd64 +build-ticktock BUILD ./ast/parser+parser COPY (+earthly-darwin-amd64/earthly --GO_GCFLAGS="${GO_GCFLAGS}") ./ SAVE ARTIFACT ./earthly AS LOCAL ./build/darwin/amd64/earthly @@ -601,6 +617,7 @@ for-darwin-m1: ARG BUILDKIT_PROJECT ARG GO_GCFLAGS BUILD --platform=linux/arm64 ./buildkitd+buildkitd --BUILDKIT_PROJECT="$BUILDKIT_PROJECT" + BUILD --platform=linux/arm64 +build-ticktock BUILD ./ast/parser+parser COPY (+earthly-darwin-arm64/earthly --GO_GCFLAGS="${GO_GCFLAGS}") ./ SAVE ARTIFACT ./earthly AS LOCAL ./build/darwin/arm64/earthly @@ -880,7 +897,7 @@ merge-main-to-docs: ARG TARGETARCH # renovate: datasource=github-releases depName=cli/cli - ARG gh_version=v2.49.0 + ARG gh_version=v2.49.2 RUN curl -Lo ghlinux.tar.gz \ https://github.com/cli/cli/releases/download/$gh_version/gh_${gh_version#v}_linux_${TARGETARCH}.tar.gz \ && tar --strip-components=1 -xf ghlinux.tar.gz \ @@ -949,7 +966,7 @@ open-pr-for-fork: ARG TARGETARCH # renovate: datasource=github-releases depName=cli/cli - ARG gh_version=v2.49.0 + ARG gh_version=v2.49.2 RUN curl -Lo ghlinux.tar.gz \ https://github.com/cli/cli/releases/download/$gh_version/gh_${gh_version#v}_linux_${TARGETARCH}.tar.gz \ && tar --strip-components=1 -xf ghlinux.tar.gz \ diff --git a/ast/commandflag/flags.go b/ast/commandflag/flags.go index d1b3a081a0..7fc1c39baa 100644 --- a/ast/commandflag/flags.go +++ b/ast/commandflag/flags.go @@ -32,6 +32,7 @@ type RunOpts struct { WithDocker bool `long:"with-docker" description:"Deprecated"` WithSSH bool `long:"ssh" description:"Make available the SSH agent of the host"` WithAWS bool `long:"aws" description:"Make any AWS credentials set in the environment available to RUN commands"` + OIDC string `long:"oidc" description:"make credentials from oidc provider (currently only works with AWS) available to RUN commands"` NoCache bool `long:"no-cache" description:"Always run this specific item, ignoring cache"` Interactive bool `long:"interactive" description:"Run this command with an interactive session, without saving changes"` InteractiveKeep bool `long:"interactive-keep" description:"Run this command with an interactive session, saving changes"` diff --git a/builder/builder.go b/builder/builder.go index 69c896998a..931020e05f 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -20,7 +20,6 @@ import ( "github.com/earthly/earthly/earthfile2llb" "github.com/earthly/earthly/logbus" "github.com/earthly/earthly/logbus/solvermon" - "github.com/earthly/earthly/outmon" "github.com/earthly/earthly/regproxy" "github.com/earthly/earthly/states" "github.com/earthly/earthly/util/containerutil" @@ -61,7 +60,6 @@ const ( type Opt struct { BkClient *client.Client LogBusSolverMonitor *solvermon.SolverMonitor - UseLogstream bool Console conslogging.ConsoleLogger Verbose bool Attachables []session.Attachable @@ -140,9 +138,7 @@ type Builder struct { func NewBuilder(ctx context.Context, opt Opt) (*Builder, error) { b := &Builder{ s: &solver{ - sm: outmon.NewSolverMonitor(opt.Console, opt.Verbose, opt.DisableNoOutputUpdates || opt.UseLogstream), logbusSM: opt.LogBusSolverMonitor, - useLogstream: opt.UseLogstream, bkClient: opt.BkClient, cacheImports: opt.CacheImports, cacheExport: opt.CacheExport, @@ -284,8 +280,8 @@ func (b *Builder) convertAndBuild(ctx context.Context, target domain.Target, opt ImageResolveMode: b.opt.ImageResolveMode, CleanCollection: b.opt.CleanCollection, PlatformResolver: opt.PlatformResolver.SubResolver(opt.PlatformResolver.Current()), - DockerImageSolverTar: newTarImageSolver(b.opt, b.s.sm), - MultiImageSolver: newMultiImageSolver(b.opt, b.s.sm), + DockerImageSolverTar: newTarImageSolver(b.opt, b.s.logbusSM), + MultiImageSolver: newMultiImageSolver(b.opt, b.s.logbusSM), OverridingVars: b.opt.OverridingVars, BuildContextProvider: b.opt.BuildContextProvider, CacheImports: b.opt.CacheImports, diff --git a/builder/builder_error.go b/builder/builder_error.go deleted file mode 100644 index c3a59b93c8..0000000000 --- a/builder/builder_error.go +++ /dev/null @@ -1,36 +0,0 @@ -package builder - -// BuildError contains an BuildError and log -type BuildError struct { - err error - log string -} - -// NewBuildError creates a new BuildError with the additional output log of the command that failed -func NewBuildError(err error, vertexLog string) error { - if vertexLog == "" { - return err - } - return &BuildError{ - err: err, - log: vertexLog, - } -} - -// BuildError formats the BuildError as a string, omitting the vertex log -func (e *BuildError) Error() string { - if e == nil { - return "" - } - return e.err.Error() -} - -// Unwrap returns the wrapped error -func (e *BuildError) Unwrap() error { - return e.err -} - -// VertexLog returns the vertex log associated with the error -func (e *BuildError) VertexLog() string { - return e.log -} diff --git a/builder/image_solver.go b/builder/image_solver.go index 16f6300256..eead919d59 100644 --- a/builder/image_solver.go +++ b/builder/image_solver.go @@ -10,7 +10,7 @@ import ( "os" "strings" - "github.com/earthly/earthly/outmon" + "github.com/earthly/earthly/logbus/solvermon" "github.com/earthly/earthly/states" "github.com/earthly/earthly/states/image" "github.com/earthly/earthly/util/gatewaycrafter" @@ -27,7 +27,7 @@ import ( "golang.org/x/sync/errgroup" ) -func newTarImageSolver(opt Opt, sm *outmon.SolverMonitor) *tarImageSolver { +func newTarImageSolver(opt Opt, sm *solvermon.SolverMonitor) *tarImageSolver { return &tarImageSolver{ sm: sm, bkClient: opt.BkClient, @@ -39,7 +39,7 @@ func newTarImageSolver(opt Opt, sm *outmon.SolverMonitor) *tarImageSolver { type tarImageSolver struct { bkClient *client.Client - sm *outmon.SolverMonitor + sm *solvermon.SolverMonitor attachables []session.Attachable enttlmnts []entitlements.Entitlement cacheImports *states.CacheImports @@ -99,12 +99,9 @@ func (s *tarImageSolver) SolveImage(ctx context.Context, mts *states.MultiTarget } return nil }) - var vertexFailureOutput string eg.Go(func() error { - var err error if printOutput { - vertexFailureOutput, err = s.sm.MonitorProgress(ctx, ch, "", true, s.bkClient) - return err + return s.sm.MonitorProgress(ctx, ch) } // Silent case. for { @@ -148,22 +145,18 @@ func (s *tarImageSolver) SolveImage(ctx context.Context, mts *states.MultiTarget // Close read pipe on cancels, otherwise the whole thing hangs. pipeR.Close() }() - err = eg.Wait() - if err != nil { - return NewBuildError(err, vertexFailureOutput) - } - return nil + return eg.Wait() } type multiImageSolver struct { bkClient *client.Client - sm *outmon.SolverMonitor + sm *solvermon.SolverMonitor attachables []session.Attachable enttlmnts []entitlements.Entitlement cacheImports *states.CacheImports } -func newMultiImageSolver(opt Opt, sm *outmon.SolverMonitor) *multiImageSolver { +func newMultiImageSolver(opt Opt, sm *solvermon.SolverMonitor) *multiImageSolver { return &multiImageSolver{ sm: sm, bkClient: opt.BkClient, @@ -328,20 +321,18 @@ func (m *multiImageSolver) SolveImages(ctx context.Context, imageDefs []*states. AllowedEntitlements: m.enttlmnts, } - var vertexFailureOutput string - go func() { _, err := m.bkClient.Build(ctx, *solveOpt, "", buildFn, statusChan) if err != nil { - errChan <- NewBuildError(err, vertexFailureOutput) + errChan <- err } doneChan <- struct{}{} }() go func() { - vertexFailureOutput, err := m.sm.MonitorProgress(ctx, statusChan, "", true, m.bkClient) + err := m.sm.MonitorProgress(ctx, statusChan) if err != nil { - errChan <- NewBuildError(err, vertexFailureOutput) + errChan <- err } doneChan <- struct{}{} }() diff --git a/builder/solver.go b/builder/solver.go index 45e672f12d..87cab91ea9 100644 --- a/builder/solver.go +++ b/builder/solver.go @@ -9,7 +9,6 @@ import ( "github.com/earthly/earthly/domain" "github.com/earthly/earthly/earthfile2llb" "github.com/earthly/earthly/logbus/solvermon" - "github.com/earthly/earthly/outmon" "github.com/earthly/earthly/states" "github.com/earthly/earthly/util/flagutil" "github.com/earthly/earthly/util/fsutilprogress" @@ -33,9 +32,7 @@ type onArtifactFunc func(context.Context, string, domain.Artifact, string, strin type onFinalArtifactFunc func(context.Context) (string, error) type solver struct { - sm *outmon.SolverMonitor logbusSM *solvermon.SolverMonitor - useLogstream bool bkClient *client.Client attachables []session.Attachable enttlmnts []entitlements.Entitlement @@ -71,24 +68,15 @@ func (s *solver) buildMainMulti(ctx context.Context, bf gwclient.BuildFunc, onIm } return nil }) - var vertexFailureOutput string - if s.useLogstream { - eg.Go(func() error { - return s.logbusSM.MonitorProgress(ctx, ch) - }) - } else { - eg.Go(func() error { - var err error - vertexFailureOutput, err = s.sm.MonitorProgress(ctx, ch, phaseText, false, s.bkClient) - return err - }) - } + eg.Go(func() error { + return s.logbusSM.MonitorProgress(ctx, ch) + }) err = eg.Wait() if buildErr != nil { - return NewBuildError(buildErr, vertexFailureOutput) + return buildErr } if err != nil { - return NewBuildError(err, vertexFailureOutput) + return err } return nil } diff --git a/buildkitd/Earthfile b/buildkitd/Earthfile index f8cb160333..2dfa084638 100644 --- a/buildkitd/Earthfile +++ b/buildkitd/Earthfile @@ -12,7 +12,7 @@ buildkitd: ARG BUILDKIT_BASE_IMAGE=$BUILDKIT_PROJECT+build END ELSE - ARG BUILDKIT_BASE_IMAGE=github.com/earthly/buildkit:594835b598b9140741472bd8f3524cb502aa49f8+build + ARG BUILDKIT_BASE_IMAGE=github.com/earthly/buildkit:531b303aa8ec03c29c2ceaa140eb0a6d32e6f6f3+build END ARG EARTHLY_TARGET_TAG_DOCKER ARG TAG="dev-$EARTHLY_TARGET_TAG_DOCKER" diff --git a/buildkitd/buildkitd.go b/buildkitd/buildkitd.go index 138d272b20..27216a79a5 100644 --- a/buildkitd/buildkitd.go +++ b/buildkitd/buildkitd.go @@ -289,13 +289,19 @@ func maybeRestart(ctx context.Context, console conslogging.ConsoleLogger, image, if err != nil { return nil, nil, errors.Wrap(err, "could not get settings hash") } - ok, err := settings.VerifyHash(hash) + hashOK, err := settings.VerifyHash(hash) if err != nil { return nil, nil, errors.Wrap(err, "verify hash") } - if ok { - // No need to replace: images are the same and settings are the same. + useExistingContainer := false + if hashOK { bkCons.VerbosePrintf("Settings hashes match (%q), no restart required\n", hash) + useExistingContainer = true + } else if settings.NoUpdate { + bkCons.Warnf("Settings do not match; however restart was inhibited. This may cause unexpected issues, proceed with caution.\n") + useExistingContainer = true + } + if useExistingContainer { info, workerInfo, err := checkConnection(ctx, settings.BuildkitAddress, 5*time.Second, opts...) if err != nil { return nil, nil, errors.Wrap(err, "could not connect to buildkitd to shut down container") @@ -305,7 +311,7 @@ func maybeRestart(ctx context.Context, console conslogging.ConsoleLogger, image, bkCons.Printf("Settings do not match. Restarting buildkit daemon with updated settings...\n") } else { if settings.NoUpdate { - bkCons.Printf("Updated image available. But update was inhibited.\n") + bkCons.Printf("Updated image available; however update was inhibited.\n") info, workerInfo, err := checkConnection(ctx, settings.BuildkitAddress, 5*time.Second, opts...) if err != nil { return nil, nil, errors.Wrap(err, "could not verify connection to buildkitd container") @@ -556,6 +562,14 @@ func IsStarted(ctx context.Context, containerName string, fe containerutil.Conta // WaitUntilStarted waits until the buildkitd daemon has started and is healthy. func WaitUntilStarted(ctx context.Context, console conslogging.ConsoleLogger, containerName, volumeName, address string, opTimeout time.Duration, fe containerutil.ContainerFrontend, opts ...client.ClientOpt) (*client.Info, *client.WorkerInfo, error) { + // Check that containerName and address match when address connects over the docker-container:// scheme + if strings.HasPrefix(address, containerutil.DockerSchemePrefix) { + expectedAddress := containerutil.DockerSchemePrefix + containerName + if address != expectedAddress { + // This shouldn't happen unless there's a programming error + return nil, nil, errors.Errorf("expected address to be %s, but got %s", expectedAddress, address) + } + } // First, wait for the container to be marked as started. ctxTimeout, cancel := context.WithTimeout(ctx, opTimeout) defer cancel() @@ -904,7 +918,7 @@ func printBuildkitInfo(bkCons conslogging.ConsoleLogger, info *client.Info, work if info.BuildkitVersion.Package != "github.com/earthly/buildkit" { bkCons.Warnf("Using a non-Earthly version of Buildkit. This is not supported.") } else { - if info.BuildkitVersion.Version != earthlyVersion { + if strings.TrimSuffix(info.BuildkitVersion.Version, "-ticktock") != earthlyVersion { if isLocal { // For local buildkits we expect perfect version match. bkCons.Warnf( @@ -1012,10 +1026,12 @@ func addRequiredOpts(settings Settings, installationName string, isUsingPodman b } if settings.SatelliteName != "" { - opts = append(opts, client.WithAdditionalMetadataContext( - "satellite_name", settings.SatelliteName, - "satellite_org", settings.SatelliteOrgID, - "satellite_token", settings.SatelliteToken), + opts = append(opts, + client.WithDefaultGRPCDialer(), + client.WithAdditionalMetadataContext( + "satellite_name", settings.SatelliteName, + "satellite_org", settings.SatelliteOrgID, + "satellite_token", settings.SatelliteToken), ) } diff --git a/buildkitd/dockerd-wrapper.sh b/buildkitd/dockerd-wrapper.sh index bdb5f0653d..b106c2bdc7 100755 --- a/buildkitd/dockerd-wrapper.sh +++ b/buildkitd/dockerd-wrapper.sh @@ -1,6 +1,8 @@ #!/bin/sh set -eu +EARTHLY_DOCKERD_CACHE_DATA=${EARTHLY_DOCKERD_CACHE_DATA:-"false"} + EARTHLY_DOCKER_WRAPPER_DEBUG=${EARTHLY_DOCKER_WRAPPER_DEBUG:-''} if [ "$EARTHLY_DOCKER_WRAPPER_DEBUG" = "1" ]; then echo "enabling docker wrapper debug mode" @@ -51,7 +53,9 @@ execute() { fi mkdir -p "$EARTHLY_DOCKERD_DATA_ROOT" - if [ -f "/sys/fs/cgroup/cgroup.controllers" ]; then + EARTHLY_FLOCK_AQUIRED=${EARTHLY_FLOCK_AQUIRED:-''} + + if [ -f "/sys/fs/cgroup/cgroup.controllers" ] && [ -z "$EARTHLY_FLOCK_AQUIRED" ]; then if [ "$EARTHLY_DOCKER_WRAPPER_DEBUG" = "1" ]; then echo >&2 "detected cgroups v2" fi @@ -75,6 +79,15 @@ execute() { fi fi + if [ "$EARTHLY_DOCKERD_CACHE_DATA" = "true" ] && [ -z "$EARTHLY_FLOCK_AQUIRED" ]; then + FLOCK_PATH="$EARTHLY_DOCKERD_DATA_ROOT/.earthly-docker-lock" + echo "aquiring flock for $FLOCK_PATH" + export EARTHLY_FLOCK_AQUIRED="true" + # dockerd-wrapper.sh will be recursively called once the lock is aquired + flock "$FLOCK_PATH" "$0" "$@" + exit 0 + fi + # Sometimes, when dockerd starts containerd, it doesn't come up in time. This timeout is not configurable from # dockerd, therefore we retry... since most instances of this timeout seem to be related to networking or scheduling # when many WITH DOCKER commands are also running. Logs are printed for each failure. diff --git a/cloud/client.go b/cloud/client.go index c586e09efc..3b3f5e6498 100644 --- a/cloud/client.go +++ b/cloud/client.go @@ -12,6 +12,8 @@ import ( "github.com/earthly/cloud-api/compute" "github.com/earthly/cloud-api/logstream" "github.com/earthly/cloud-api/pipelines" + "github.com/earthly/cloud-api/secrets" + "github.com/google/uuid" grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry" "github.com/pkg/errors" @@ -68,6 +70,7 @@ type Client struct { analytics analytics.AnalyticsClient askv askv.AskvClient billing billing.BillingClient + secrets secrets.SecretsServiceClient requestID string installationName string logstreamAddressOverride string @@ -160,6 +163,7 @@ func NewClient(httpAddr, grpcAddr string, useInsecure bool, agentSockPath, authC c.analytics = analytics.NewAnalyticsClient(conn) c.askv = askv.NewAskvClient(conn) c.billing = billing.NewBillingClient(conn) + c.secrets = secrets.NewSecretsServiceClient(conn) logstreamAddr := grpcAddr if c.logstreamAddressOverride != "" { diff --git a/cloud/cloud_installation.go b/cloud/cloud_installation.go index b1d0a6d69f..1ecfe9d6df 100644 --- a/cloud/cloud_installation.go +++ b/cloud/cloud_installation.go @@ -8,32 +8,70 @@ import ( ) const ( - CloudStatusConnected = "Connected" - CloudStatusActive = "Active" - CloudStatusProblem = "Problem" + CloudStatusGreen = "Green" + CloudStatusYellow = "Yellow" + CloudStatusRed = "Red" + CloudStatusUnknown = "Unknown" ) type Installation struct { Name string Org string Status string + StatusMessage string NumSatellites int IsDefault bool } -func (c *Client) ConfigureCloud(ctx context.Context, orgID, cloudName string, setDefault bool) (*Installation, error) { +type CloudConfigurationOpt struct { + Name string + SetDefault bool + SshKeyName string + ComputeRoleArn string + AccountId string + AllowedSubnetIds []string + SecurityGroupId string + Region string + InstanceProfileArn string +} + +func (c *Client) ConfigureCloud(ctx context.Context, orgID string, configuration *CloudConfigurationOpt) (*Installation, error) { resp, err := c.compute.ConfigureCloud(c.withAuth(ctx), &pb.ConfigureCloudRequest{ - OrgId: orgID, - Name: cloudName, - SetDefault: setDefault, + OrgId: orgID, + Name: configuration.Name, + SetDefault: configuration.SetDefault, + SshKeyName: configuration.SshKeyName, + ComputeRoleArn: configuration.ComputeRoleArn, + AccountId: configuration.AccountId, + AllowedSubnetIds: configuration.AllowedSubnetIds, + SecurityGroupId: configuration.SecurityGroupId, + Region: configuration.Region, + InstanceProfileArn: configuration.InstanceProfileArn, }) if err != nil { return nil, errors.Wrap(err, "error from ConfigureCloud API") } return &Installation{ - Name: cloudName, - Org: orgID, - Status: installationStatus(resp.Status), + Name: configuration.Name, + Org: orgID, + Status: installationStatus(resp.Status), + StatusMessage: resp.Message, + }, nil +} + +func (c *Client) UseCloud(ctx context.Context, orgID string, configuration *CloudConfigurationOpt) (*Installation, error) { + resp, err := c.compute.UseCloud(c.withAuth(ctx), &pb.UseCloudRequest{ + OrgId: orgID, + Name: configuration.Name, + }) + if err != nil { + return nil, errors.Wrap(err, "error from UseCloud API") + } + return &Installation{ + Name: configuration.Name, + Org: orgID, + Status: installationStatus(resp.Status), + StatusMessage: resp.Message, }, nil } @@ -50,6 +88,7 @@ func (c *Client) ListClouds(ctx context.Context, orgID string) ([]Installation, Name: i.CloudName, Org: orgID, Status: installationStatus(i.Status), + StatusMessage: i.StatusContext, NumSatellites: int(i.NumSatellites), IsDefault: i.IsDefault, }) @@ -69,14 +108,14 @@ func (c *Client) DeleteCloud(ctx context.Context, orgID, cloudName string) error } func installationStatus(status pb.CloudStatus) string { - internalStatus := "UNKNOWN" switch status { - case pb.CloudStatus_CLOUD_STATUS_ACCOUNT_ACTIVE: - internalStatus = CloudStatusActive - case pb.CloudStatus_CLOUD_STATUS_ACCOUNT_CONNECTED: - internalStatus = CloudStatusConnected - case pb.CloudStatus_CLOUD_STATUS_PROBLEM: - internalStatus = CloudStatusProblem + case pb.CloudStatus_CLOUD_STATUS_GREEN: + return CloudStatusGreen + case pb.CloudStatus_CLOUD_STATUS_YELLOW: + return CloudStatusYellow + case pb.CloudStatus_CLOUD_STATUS_RED: + return CloudStatusRed + default: + return CloudStatusUnknown } - return internalStatus } diff --git a/cloud/github.go b/cloud/github.go index ced503b468..20a01ca1fe 100644 --- a/cloud/github.go +++ b/cloud/github.go @@ -19,7 +19,7 @@ func (c *Client) SetGithubToken(ctx context.Context, orgName string, ghOrg strin GithubToken: token, }) if err != nil { - return fmt.Errorf("failed setting Github token: %w", err) + return fmt.Errorf("failed setting GitHub token: %w", err) } return nil } diff --git a/cloud/http.go b/cloud/http.go index 2a11418cb6..37e53eff90 100644 --- a/cloud/http.go +++ b/cloud/http.go @@ -8,7 +8,6 @@ import ( "io" "net" "net/http" - "os" "strings" "time" @@ -57,24 +56,6 @@ func withJSONBody(body proto.Message) requestOpt { } } -func withFileBody(pathOnDisk string) requestOpt { - return func(r *request) error { - _, err := os.Stat(pathOnDisk) - if err != nil { - return errors.Wrapf(err, "could not stat file at %s", pathOnDisk) - } - - contents, err := os.ReadFile(pathOnDisk) - if err != nil { - return errors.Wrapf(err, "could not add file %s to request body", pathOnDisk) - } - - r.hasBody = true - r.body = contents - return nil - } -} - func withBody(body []byte) requestOpt { return func(r *request) error { r.hasBody = true diff --git a/cloud/log.go b/cloud/log.go deleted file mode 100644 index d785701e02..0000000000 --- a/cloud/log.go +++ /dev/null @@ -1,33 +0,0 @@ -package cloud - -import ( - "bytes" - "context" - "fmt" - "net/http" - - logsapi "github.com/earthly/cloud-api/logs" - "github.com/pkg/errors" -) - -func (c *Client) UploadLog(ctx context.Context, pathOnDisk string) (string, error) { - status, body, err := c.doCall(ctx, http.MethodPost, "/api/v0/logs", withAuth(), withFileBody(pathOnDisk), withHeader("Content-Type", "application/gzip")) - if err != nil { - return "", err - } - if status != http.StatusCreated { - msg, err := getMessageFromJSON(bytes.NewReader(body)) - if err != nil { - return "", errors.Wrap(err, fmt.Sprintf("failed to decode response body (status code: %d)", status)) - } - return "", errors.Errorf("failed to upload log: %s", msg) - } - - var uploadBundleResponse logsapi.UploadLogBundleResponse - err = c.jum.Unmarshal(body, &uploadBundleResponse) - if err != nil { - return "", errors.Wrap(err, "failed to unmarshal uploadbundle response") - } - - return fmt.Sprintf(uploadBundleResponse.ViewURL), nil -} diff --git a/cloud/secret.go b/cloud/secret.go index 0e775e5ffa..92e34db0d1 100644 --- a/cloud/secret.go +++ b/cloud/secret.go @@ -6,8 +6,12 @@ import ( "fmt" "net/http" "strings" + "time" + + "github.com/earthly/cloud-api/secrets" "github.com/pkg/errors" + "google.golang.org/protobuf/types/known/durationpb" ) func (c *Client) Remove(ctx context.Context, path string) error { @@ -84,3 +88,28 @@ func (c *Client) Set(ctx context.Context, path string, data []byte) error { } return nil } + +func (c *Client) GetAWSCredentials(ctx context.Context, sessionName string, roleARN string, orgName string, projectName string, region string, sessionDuration *time.Duration) (*secrets.GetAWSCredentialsResponse, error) { + if orgName == "" { + return nil, errors.New("org must be set in order to use AWS OIDC") + } + if projectName == "" { + return nil, errors.New("project must be set in order to use AWS OIDC") + } + var duration *durationpb.Duration + if sessionDuration != nil { + duration = durationpb.New(*sessionDuration) + } + response, err := c.secrets.GetAWSCredentials(c.withAuth(ctx), &secrets.GetAWSCredentialsRequest{ + RoleArn: roleARN, + SessionName: sessionName, + SessionDuration: duration, + Region: region, + OrgName: orgName, + ProjectName: projectName, + }) + if err != nil { + return nil, errors.Wrap(err, "failed to get aws credentials via oidc provider") + } + return response, nil +} diff --git a/cmd/earthly/app/before.go b/cmd/earthly/app/before.go index dd242aca16..529f47ab74 100644 --- a/cmd/earthly/app/before.go +++ b/cmd/earthly/app/before.go @@ -23,59 +23,65 @@ import ( ) func (app *EarthlyApp) before(cliCtx *cli.Context) error { - if app.BaseCLI.Flags().EnableProfiler { + flags := app.BaseCLI.Flags() + + if flags.EnableProfiler { go profhandler() } - if app.BaseCLI.Flags().InstallationName != "" { + if flags.InstallationName != "" { if !cliCtx.IsSet("config") { - app.BaseCLI.Flags().ConfigPath = defaultConfigPath(app.BaseCLI.Flags().InstallationName) + flags.ConfigPath = defaultConfigPath(flags.InstallationName) } if !cliCtx.IsSet("buildkit-container-name") { - app.BaseCLI.Flags().ContainerName = fmt.Sprintf("%s-buildkitd", app.BaseCLI.Flags().InstallationName) + flags.ContainerName = fmt.Sprintf("%s-buildkitd", flags.InstallationName) } if !cliCtx.IsSet("buildkit-volume-name") { - app.BaseCLI.Flags().BuildkitdSettings.VolumeName = fmt.Sprintf("%s-cache", app.BaseCLI.Flags().InstallationName) + flags.BuildkitdSettings.VolumeName = fmt.Sprintf("%s-cache", flags.InstallationName) } } - if app.BaseCLI.Flags().Debug { + if flags.Debug { app.BaseCLI.SetConsole(app.BaseCLI.Console().WithLogLevel(conslogging.Debug)) - } else if app.BaseCLI.Flags().Verbose { + } else if flags.Verbose { app.BaseCLI.SetConsole(app.BaseCLI.Console().WithLogLevel(conslogging.Verbose)) } - if app.BaseCLI.Flags().LogstreamUpload { - app.BaseCLI.Flags().Logstream = true + + app.BaseCLI.SetConsole(app.BaseCLI.Console().WithPrefixWriter(app.BaseCLI.Logbus().Run().Generic())) + if flags.BuildID == "" { + flags.BuildID = uuid.NewString() } - if app.BaseCLI.Flags().Logstream { - app.BaseCLI.SetConsole(app.BaseCLI.Console().WithPrefixWriter(app.BaseCLI.Logbus().Run().Generic())) - if app.BaseCLI.Flags().BuildID == "" { - app.BaseCLI.Flags().BuildID = uuid.NewString() - } - var execStatsTracker *execstatssummary.Tracker - if app.BaseCLI.Flags().ExecStatsSummary != "" { - execStatsTracker = execstatssummary.NewTracker(app.BaseCLI.Flags().ExecStatsSummary) - } - disableOngoingUpdates := !app.BaseCLI.Flags().Logstream || app.BaseCLI.Flags().InteractiveDebugging - forceColor := envutil.IsTrue("FORCE_COLOR") - noColor := envutil.IsTrue("NO_COLOR") - var err error - newSetup, err := logbussetup.New( - cliCtx.Context, app.BaseCLI.Logbus(), app.BaseCLI.Flags().Debug, app.BaseCLI.Flags().Verbose, app.BaseCLI.Flags().DisplayExecStats, forceColor, noColor, - disableOngoingUpdates, app.BaseCLI.Flags().LogstreamDebugFile, app.BaseCLI.Flags().BuildID, execStatsTracker, app.BaseCLI.Flags().GithubAnnotations) - app.BaseCLI.SetLogbusSetup(newSetup) - if err != nil { - return errors.Wrap(err, "logbus setup") - } + var execStatsTracker *execstatssummary.Tracker + if flags.ExecStatsSummary != "" { + execStatsTracker = execstatssummary.NewTracker(flags.ExecStatsSummary) } + busSetup, err := logbussetup.New( + cliCtx.Context, + app.BaseCLI.Logbus(), + flags.Debug, + flags.Verbose, + flags.DisplayExecStats, + envutil.IsTrue("FORCE_COLOR"), + envutil.IsTrue("NO_COLOR"), + app.BaseCLI.Flags().InteractiveDebugging, + flags.LogstreamDebugFile, + flags.BuildID, + execStatsTracker, + flags.GithubAnnotations, + ) + if err != nil { + return errors.Wrap(err, "logbus setup") + } + + app.BaseCLI.SetLogbusSetup(busSetup) if cliCtx.IsSet("config") { - app.BaseCLI.Console().Printf("loading config values from %q\n", app.BaseCLI.Flags().ConfigPath) + app.BaseCLI.Console().Printf("loading config values from %q\n", flags.ConfigPath) } var yamlData []byte - if app.BaseCLI.Flags().ConfigPath != "" { + if flags.ConfigPath != "" { var err error - yamlData, err = config.ReadConfigFile(app.BaseCLI.Flags().ConfigPath) + yamlData, err = config.ReadConfigFile(flags.ConfigPath) if err != nil { if cliCtx.IsSet("config") || !errors.Is(err, os.ErrNotExist) { return errors.Wrapf(err, "read config") @@ -83,9 +89,9 @@ func (app *EarthlyApp) before(cliCtx *cli.Context) error { } } - cfg, err := config.ParseYAML(yamlData, app.BaseCLI.Flags().InstallationName) + cfg, err := config.ParseYAML(yamlData, flags.InstallationName) if err != nil { - return errors.Wrapf(err, "failed to parse %s", app.BaseCLI.Flags().ConfigPath) + return errors.Wrapf(err, "failed to parse %s", flags.ConfigPath) } app.BaseCLI.SetCfg(&cfg) @@ -109,7 +115,7 @@ func (app *EarthlyApp) before(cliCtx *cli.Context) error { } } - if !isBootstrapCmd && !cliutil.IsBootstrapped(app.BaseCLI.Flags().InstallationName) { + if !isBootstrapCmd && !cliutil.IsBootstrapped(flags.InstallationName) { app.BaseCLI.Flags().BootstrapNoBuildkit = true // Docker may not be available, for instance... like our integration tests. newBootstrap := subcmd.NewBootstrap(app.BaseCLI) err = newBootstrap.Action(cliCtx) @@ -119,7 +125,7 @@ func (app *EarthlyApp) before(cliCtx *cli.Context) error { } if !cliCtx.IsSet("org") { - app.BaseCLI.Flags().OrgName = app.BaseCLI.Cfg().Global.Org + flags.OrgName = app.BaseCLI.Cfg().Global.Org } return nil } @@ -130,7 +136,7 @@ func (app *EarthlyApp) parseFrontend(cliCtx *cli.Context, cfg *config.Config) er BuildkitHostCLIValue: app.BaseCLI.Flags().BuildkitHost, BuildkitHostFileValue: app.BaseCLI.Cfg().Global.BuildkitHost, LocalRegistryHostFileValue: app.BaseCLI.Cfg().Global.LocalRegistryHost, - InstallationName: app.BaseCLI.Flags().InstallationName, + LocalContainerName: app.BaseCLI.Flags().ContainerName, DefaultPort: 8372 + config.PortOffset(app.BaseCLI.Flags().InstallationName), Console: console, } diff --git a/cmd/earthly/base/init_frontend.go b/cmd/earthly/base/init_frontend.go index 93f5ec46fd..626799920b 100644 --- a/cmd/earthly/base/init_frontend.go +++ b/cmd/earthly/base/init_frontend.go @@ -1,6 +1,7 @@ package base import ( + "fmt" "net/url" "path/filepath" "time" @@ -17,6 +18,16 @@ func (cli *CLI) InitFrontend(cliCtx *cli.Context) error { cli.Flags().BuildkitdImage = cli.Cfg().Global.BuildkitImage } + if cli.Flags().UseTickTockBuildkitImage { + if cliCtx.IsSet("buildkit-image") { + return fmt.Errorf("the --buildkit-image and --ticktock flags are mutually exclusive") + } + if cli.Cfg().Global.BuildkitImage != "" { + return fmt.Errorf("the --ticktock flags can not be used in combination with the buildkit_image config option") + } + cli.Flags().BuildkitdImage += "-ticktock" + } + bkURL, err := url.Parse(cli.Flags().BuildkitHost) // Not validated because we already did that when we calculated it. if err != nil { return errors.Wrap(err, "failed to parse generated buildkit URL") diff --git a/cmd/earthly/flag/global.go b/cmd/earthly/flag/global.go index 55a6a79ca0..d417b72934 100644 --- a/cmd/earthly/flag/global.go +++ b/cmd/earthly/flag/global.go @@ -25,9 +25,6 @@ const ( DefaultSecretFile = ".secret" SecretFileFlag = "secret-file-path" - - DefaultLogstream = true - DefaultLogstreamUpload = true ) // Put flags on Flags instead as there are other things in the CLI that are being called + set @@ -58,8 +55,6 @@ type Global struct { ArgFile string SecretFile string NoBuildkitUpdate bool - Logstream bool - LogstreamUpload bool LogstreamDebugFile string LogstreamDebugManifestFile string LogstreamAddressOverride string @@ -87,6 +82,7 @@ type Global struct { Pull bool Push bool CI bool + UseTickTockBuildkitImage bool Output bool NoOutput bool NoCache bool @@ -286,22 +282,6 @@ func (global *Global) RootFlags(installName string, bkImage string) []cli.Flag { Value: DefaultSecretFile, Destination: &global.SecretFile, }, - &cli.BoolFlag{ - Name: "logstream", - EnvVars: []string{"EARTHLY_LOGSTREAM"}, - Usage: "Enable log streaming only locally", - Destination: &global.Logstream, - Hidden: true, // Internal. - Value: DefaultLogstream, - }, - &cli.BoolFlag{ - Name: "logstream-upload", - EnvVars: []string{"EARTHLY_LOGSTREAM_UPLOAD"}, - Usage: "Enable log stream uploading", - Destination: &global.LogstreamUpload, - Hidden: true, // Internal. - Value: DefaultLogstreamUpload, - }, &cli.StringFlag{ Name: "logstream-debug-file", EnvVars: []string{"EARTHLY_LOGSTREAM_DEBUG_FILE"}, @@ -368,6 +348,13 @@ func (global *Global) RootFlags(installName string, bkImage string) []cli.Flag { Usage: common.Wrap("Execute in CI mode. ", "Implies --no-output --strict"), Destination: &global.CI, }, + &cli.BoolFlag{ + Name: "ticktock", + EnvVars: []string{"EARTHLY_TICKTOCK"}, + Usage: "Use earthly's experimental buildkit ticktock codebase", + Destination: &global.UseTickTockBuildkitImage, + Hidden: true, // Experimental + }, &cli.BoolFlag{ Name: "output", EnvVars: []string{"EARTHLY_OUTPUT"}, diff --git a/cmd/earthly/subcmd/build_cmd.go b/cmd/earthly/subcmd/build_cmd.go index 7174357a21..016cbdbe18 100644 --- a/cmd/earthly/subcmd/build_cmd.go +++ b/cmd/earthly/subcmd/build_cmd.go @@ -47,7 +47,6 @@ import ( "github.com/earthly/earthly/domain" "github.com/earthly/earthly/earthfile2llb" "github.com/earthly/earthly/inputgraph" - "github.com/earthly/earthly/logbus/solvermon" "github.com/earthly/earthly/states" "github.com/earthly/earthly/util/cliutil" "github.com/earthly/earthly/util/containerutil" @@ -421,7 +420,7 @@ func (a *Build) ActionBuildImp(cliCtx *cli.Context, flagArgs, nonFlagArgs []stri secretProvider := secretprovider.New( internalSecretStore, - secretprovider.NewAWSCredentialProvider(), + secretprovider.NewAWSCredentialProvider(cloudClient), secretprovider.NewMapStore(secretsMap), customSecretProviderCmd, secretprovider.NewCloudStore(cloudClient), @@ -537,17 +536,11 @@ func (a *Build) ActionBuildImp(cliCtx *cli.Context, flagArgs, nonFlagArgs []stri localRegistryAddr = u.Host } - var logbusSM *solvermon.SolverMonitor - if a.cli.Flags().Logstream { - logbusSM = a.cli.LogbusSetup().SolverMonitor - } else if a.cli.Flags().DisplayExecStats || a.cli.Flags().ExecStatsSummary != "" { - return fmt.Errorf("the --exec-stats and --exec-stats-summary features are only available when --logstream is enabled") - } + logbusSM := a.cli.LogbusSetup().SolverMonitor builderOpts := builder.Opt{ BkClient: bkClient, LogBusSolverMonitor: logbusSM, - UseLogstream: a.cli.Flags().Logstream, Console: a.cli.Console(), Verbose: a.cli.Flags().Verbose, Attachables: attachables, @@ -640,11 +633,10 @@ func (a *Build) ActionBuildImp(cliCtx *cli.Context, flagArgs, nonFlagArgs []stri if a.cli.Flags().ProjectName != "" { projectName = a.cli.Flags().ProjectName } + a.cli.Console().WithPrefix("logbus").Printf("Setting organization %q and project %q", orgName, projectName) analytics.AddEarthfileProject(orgName, projectName) - if !a.cli.Flags().Logstream { - return nil - } + setup := a.cli.LogbusSetup() setup.SetOrgAndProject(orgName, projectName) setup.SetGitAuthor(gitCommitAuthorEmail, gitConfigEmail) @@ -656,7 +648,7 @@ func (a *Build) ActionBuildImp(cliCtx *cli.Context, flagArgs, nonFlagArgs []stri return nil } - if a.cli.Flags().Logstream && doLogstreamUpload && !a.cli.LogbusSetup().LogStreamerStarted() { + if doLogstreamUpload && !a.cli.LogbusSetup().LogStreamerStarted() { a.cli.Console().ColorPrintf(color.New(color.FgHiYellow), "Streaming logs to %s\n\n", logstreamURL) } @@ -839,9 +831,7 @@ func (a *Build) platformResolver(ctx context.Context, bkClient *bkclient.Client, if err != nil { return nil, errors.Wrap(err, "get native platform via buildkit client") } - if a.cli.Flags().Logstream { - a.cli.LogbusSetup().SetDefaultPlatform(platforms.Format(nativePlatform)) - } + a.cli.LogbusSetup().SetDefaultPlatform(platforms.Format(nativePlatform)) platr := platutil.NewResolver(nativePlatform) a.cli.SetAnaMetaBKPlatform(platforms.Format(nativePlatform)) a.cli.SetAnaMetaUserPlatform(platforms.Format(platr.LLBUser())) @@ -960,29 +950,6 @@ func (a *Build) logShareLink(ctx context.Context, cloudClient *cloud.Client, tar return "", false, printLinkFn } - if !a.cli.Flags().LogstreamUpload { - // If you are logged in, then add the bundle builder code, and - // configure cleanup and post-build messages. - a.cli.SetConsole(a.cli.Console().WithLogBundleWriter(target.String(), clean)) - printLinkFn := func() { // Defer this to keep log upload code together - logPath, err := a.cli.Console().WriteBundleToDisk() - if err != nil { - err := errors.Wrapf(err, "failed to write log to disk") - a.cli.Console().Warnf(err.Error()) - return - } - - id, err := cloudClient.UploadLog(ctx, logPath) - if err != nil { - err := errors.Wrapf(err, "failed to upload log") - a.cli.Console().Warnf(err.Error()) - return - } - a.cli.Console().ColorPrintf(color.New(color.FgHiYellow), "Shareable link: %s\n", id) - } - return "", false, printLinkFn - } - logstreamURL := fmt.Sprintf("%s/builds/%s", a.cli.CIHost(), a.cli.LogbusSetup().InitialManifest.GetBuildId()) printLinkFn := func() { diff --git a/cmd/earthly/subcmd/cloud_installation_cmds.go b/cmd/earthly/subcmd/cloud_installation_cmds.go index 90e6643d6f..ef63df51e5 100644 --- a/cmd/earthly/subcmd/cloud_installation_cmds.go +++ b/cmd/earthly/subcmd/cloud_installation_cmds.go @@ -1,10 +1,15 @@ package subcmd import ( + "context" "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/fatih/color" "os" "text/tabwriter" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/cloudformation" "github.com/pkg/errors" "github.com/urfave/cli/v2" @@ -98,16 +103,22 @@ func (c *CloudInstallation) install(cliCtx *cli.Context) error { return err } + installation, err := c.getInstallationDataFromCloudFormation(ctx, cloudName) + if err != nil { + return err + } + c.cli.Console().Printf("Configuring new Cloud Installation: %s. Please wait...", cloudName) - install, err := cloudClient.ConfigureCloud(ctx, orgID, cloudName, false) + install, err := cloudClient.ConfigureCloud(ctx, orgID, installation) if err != nil { return errors.Wrap(err, "failed installing cloud") } - if install.Status == cloud.CloudStatusProblem { - c.cli.Console().Warnf("There is a problem with the cloud installation. Please contact Earthly team for support.") - return errors.New("cloud Installation failed validation") + if install.Status == cloud.CloudStatusRed || install.Status == cloud.CloudStatusYellow { + c.cli.Console().Warnf("There is a problem with the cloud installation.") + c.cli.Console().Warnf(install.StatusMessage) + return errors.New("cloud installation failed validation") } c.cli.Console().Printf("...Done\n") @@ -145,17 +156,17 @@ func (c *CloudInstallation) use(cliCtx *cli.Context) error { c.cli.Console().Printf("Validating Cloud Installation: %s. Please wait...", cloudName) - install, err := cloudClient.ConfigureCloud(ctx, orgID, cloudName, true) + install, err := cloudClient.UseCloud(ctx, orgID, &cloud.CloudConfigurationOpt{ + Name: cloudName, + SetDefault: true, + }) if err != nil { return errors.Wrap(err, "could not select cloud") } - if err != nil { - return errors.Wrap(err, "failed selecting cloud") - } - - if install.Status == cloud.CloudStatusProblem { - c.cli.Console().Warnf("There is a problem with the cloud installation. Please contact Earthly team for support.") + if install.Status == cloud.CloudStatusRed || install.Status == cloud.CloudStatusYellow { + c.cli.Console().Warnf("There is a problem with the cloud installation.") + c.cli.Console().Warnf(install.StatusMessage) return errors.New("cloud Installation failed validation") } @@ -231,18 +242,78 @@ func (c *CloudInstallation) printTable(installations []cloud.Installation) { if i.IsDefault { selected = "*" } - var description string + var coloredStatus string switch i.Status { - case cloud.CloudStatusActive: - description = "Ready to use" - case cloud.CloudStatusConnected: - description = "Reachable, but not yet validated" - case cloud.CloudStatusProblem: - description = "Please contact Earthly for support" + case cloud.CloudStatusGreen: + coloredStatus = color.GreenString(i.Status) + case cloud.CloudStatusYellow: + coloredStatus = color.YellowString(i.Status) + case cloud.CloudStatusRed: + coloredStatus = color.RedString(i.Status) + default: + coloredStatus = color.HiRedString(i.Status) + } + suffix := "" + if i.StatusMessage != "" { + suffix = fmt.Sprintf(": %s", i.StatusMessage) } - fmt.Fprintf(t, "%s\t%s\t%d\t%s: %s\t\n", selected, i.Name, i.NumSatellites, i.Status, description) + fullStatus := fmt.Sprintf("%s%s", coloredStatus, suffix) + fmt.Fprintf(t, "%s\t%s\t%d\t%s\t\n", selected, i.Name, i.NumSatellites, fullStatus) } if err := t.Flush(); err != nil { fmt.Printf("failed to print satellites: %s", err.Error()) } } + +func (c *CloudInstallation) getInstallationDataFromCloudFormation(ctx context.Context, stackName string) (*cloud.CloudConfigurationOpt, error) { + awsConfig, err := awsconfig.LoadDefaultConfig(ctx) + if err != nil { + return nil, errors.Wrap(err, "could not load aws config") + } + + client := cloudformation.NewFromConfig(awsConfig) + + describeStacksOutput, err := client.DescribeStacks(ctx, &cloudformation.DescribeStacksInput{ + StackName: aws.String(stackName), + }) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("could not describe stack %s", stackName)) + } + + if len(describeStacksOutput.Stacks) != 1 { + return nil, fmt.Errorf("unexpected number of stacks(%v) found with name %q", len(describeStacksOutput.Stacks), stackName) + } + + stack := describeStacksOutput.Stacks[0] + params := &cloud.CloudConfigurationOpt{} + + for _, output := range stack.Outputs { + if output.OutputKey == nil { + return nil, fmt.Errorf("specified stack %s has nil output key", stackName) + } + if output.OutputValue == nil { + return nil, fmt.Errorf("specified stack %s has nil value for key %s", stackName, *output.OutputKey) + } + + switch *output.OutputKey { + case "InstallationName": + params.Name = *output.OutputValue + case "SshKeyName": + params.SshKeyName = *output.OutputValue + case "ComputeRoleArn": + params.ComputeRoleArn = *output.OutputValue + case "AccountId": + params.AccountId = *output.OutputValue + case "AllowedSubnetIds": + params.AllowedSubnetIds = []string{*output.OutputValue} + case "SecurityGroupId": + params.SecurityGroupId = *output.OutputValue + case "Region": + params.Region = *output.OutputValue + case "InstanceProfileArn": + params.InstanceProfileArn = *output.OutputValue + } + } + + return params, nil +} diff --git a/conslogging/bundle.go b/conslogging/bundle.go deleted file mode 100644 index 520bce4a32..0000000000 --- a/conslogging/bundle.go +++ /dev/null @@ -1,309 +0,0 @@ -package conslogging - -import ( - "archive/tar" - "compress/gzip" - "encoding/json" - "fmt" - "io" - "net/url" - "os" - "regexp" - "strings" - "sync" - "time" - - "github.com/earthly/earthly/cleanup" - "github.com/pkg/errors" -) - -const fullLog = "_full" - -type targetLogger struct { - writer *strings.Builder - prefix string - result string - status string - started time.Time -} - -// BundleBuilder builds log bundles for local storage or upload to a logging server -type BundleBuilder struct { - entrypoint string - started time.Time - cleanup *cleanup.Collection - - mu sync.Mutex - logsForTarget map[string]*targetLogger -} - -// Write implements io.Writer as a passthrough to the underlying strings.Builder for convenience. -func (tl *targetLogger) Write(p []byte) (n int, err error) { - return tl.writer.Write(p) -} - -// NewBundleBuilder makes a new BundleBuilder, that will write logs to the targeted root directory, -// and specify the entrypoint in the resulting manifest. -func NewBundleBuilder(entrypoint string, cleanup *cleanup.Collection) *BundleBuilder { - return &BundleBuilder{ - entrypoint: entrypoint, - logsForTarget: map[string]*targetLogger{}, - started: time.Now(), - cleanup: cleanup, - } -} - -// PrefixResult sets the prefix(aka target) result as it should appear in the manifest for that specific target. -func (bb *BundleBuilder) PrefixResult(prefix, result string) { - bb.mu.Lock() - defer bb.mu.Unlock() - if builder, ok := bb.logsForTarget[prefix]; ok { - builder.result = result - } -} - -// PrefixStatus sets the prefix(aka target) result as it should appear in the manifest for that specific target. -func (bb *BundleBuilder) PrefixStatus(prefix, status string) { - bb.mu.Lock() - defer bb.mu.Unlock() - if builder, ok := bb.logsForTarget[prefix]; ok { - builder.status = status - } -} - -// PrefixWriter gets an io.Writer for a given prefix(aka target). If its a prefix we have not seen before, -// then generate a new writer to accommodate it. -func (bb *BundleBuilder) PrefixWriter(prefix string) io.Writer { - bb.mu.Lock() - defer bb.mu.Unlock() - - if builder, ok := bb.logsForTarget[prefix]; ok { - return builder - } - - writer := &targetLogger{ - writer: &strings.Builder{}, - status: StatusWaiting, - result: ResultCancelled, - started: time.Now(), - prefix: prefix, - } - bb.logsForTarget[prefix] = writer - return writer -} - -// WriteToDisk aggregates all the data in the numerous prefix writers, and generates an Earthly log bundle. -// These bundles include a manifest generated from the aggregation of the prefixes (targets). -func (bb *BundleBuilder) WriteToDisk() (string, error) { - // Build file and io.Writer for saving log data - file, err := os.CreateTemp("", "earthly-log*.tar.gz") - if err != nil { - return "", errors.Wrapf(err, "could not create tarball") - } - defer file.Close() - bb.cleanup.Add(func() error { - return os.Remove(file.Name()) - }) - - gzipWriter := gzip.NewWriter(file) - defer gzipWriter.Close() - - tarWriter := tar.NewWriter(gzipWriter) - defer tarWriter.Close() - - // Make a copy so that we keep the lock for as little time as possible. - bb.mu.Lock() - logsForTarget := make(map[string]*targetLogger, len(bb.logsForTarget)) - for k, v := range bb.logsForTarget { - logsForTarget[k] = v - } - bb.mu.Unlock() - - // Convert targets to manifest representations, get tar headers for data - targetData := make([]TargetManifest, 0) - for _, lines := range logsForTarget { - mt, err := lines.toManifestTarget() - if err != nil { - // Something was wrong with this targets logs (0 length, or blacklisted name...). Ignore it. - continue - } - - targetData = append(targetData, mt) - - trimmed := strings.TrimSpace(lines.prefix) - escaped := url.QueryEscape(trimmed) - - err = tarWriter.WriteHeader(&tar.Header{ - Name: fmt.Sprintf("target/%s", escaped), - Size: int64(lines.writer.Len()), - Mode: 0600, - ChangeTime: time.Now(), - }) - if err != nil { - return "", errors.Wrapf(err, "could not write target header") - } - _, err = tarWriter.Write([]byte(lines.writer.String())) - if err != nil { - return "", errors.Wrapf(err, "could not write target data") - } - } - - // build manifest and permissions - mani := bb.buildManifest(targetData) - manifestJSON, _ := json.Marshal(mani) - err = tarWriter.WriteHeader(&tar.Header{ - Name: "manifest", - Size: int64(len(manifestJSON)), - Mode: 0600, - ChangeTime: time.Now(), - }) - if err != nil { - return "", errors.Wrapf(err, "could not write manifest header") - } - _, err = tarWriter.Write(manifestJSON) - if err != nil { - return "", errors.Wrapf(err, "could not write manifest") - } - - perm := bb.buildPermissions() - permissionsJSON, _ := json.Marshal(perm) - err = tarWriter.WriteHeader(&tar.Header{ - Name: "permissions", - Size: int64(len(permissionsJSON)), - Mode: 0600, - ChangeTime: time.Now(), - }) - if err != nil { - return "", errors.Wrapf(err, "could not write permissions header") - } - _, err = tarWriter.Write(permissionsJSON) - if err != nil { - return "", errors.Wrapf(err, "could not write permissions") - } - - return file.Name(), nil -} - -func (bb *BundleBuilder) buildManifest(targetManifests []TargetManifest) *Manifest { - manifest := &Manifest{ - Version: 1, - Duration: int(time.Since(bb.started).Milliseconds()), - Status: StatusComplete, - Result: ResultSuccess, - CreatedAt: time.Now().In(time.UTC), - Entrypoint: bb.entrypoint, - Targets: targetManifests, - } - - for _, tm := range targetManifests { - if tm.Name == fullLog { - // Full Log reserved name should not determine whole build status. - // Really, we could go back through after determining whole build status to set _full result & status to the - // values for the whole build; but it doesn't (yet) affect or mean anything to us. So leave it as is. - continue - } - - if tm.Result != ResultSuccess { - manifest.Result = tm.Result - } - - if tm.Status != StatusComplete { - manifest.Status = tm.Status - } - } - - return manifest -} - -func (bb *BundleBuilder) buildPermissions() *Permissions { - return &Permissions{ - Version: 1, - Users: []string{"*"}, - Orgs: []string{"*"}, - } -} - -func (tl *targetLogger) toManifestTarget() (TargetManifest, error) { - if tl.writer.Len() <= 0 { - // Do not write empty logs, if the prefix didn't write anything - return TargetManifest{}, errors.New("0 length target") - } - - if tl.prefix == "ongoing" || tl.prefix == "buildkitd" { - // The ongoing & buildkitd init messages end up in here too. Since they are not updates from a vertex, we will - // never mark them as complete. Additionally, its not useful to have in the output. Ignore it here. - return TargetManifest{}, fmt.Errorf("blacklisted target name %s", tl.prefix) - } - - command, summary := tl.getCommandAndSummary() - - manifestTarget := TargetManifest{ - Name: tl.prefix, - Status: tl.status, - Result: tl.result, - Duration: int(time.Since(tl.started).Milliseconds()), - Size: tl.writer.Len(), - Command: command, - Summary: summary, - } - - return manifestTarget, nil -} - -// Nobody expects ANSI in the command/summary. -// So, even if we don't inject color we should strip it since a tool inside could have done an ANSI too. *SIGH* -const ansi = "[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))" - -var re = regexp.MustCompile(ansi) - -func (tl *targetLogger) getCommandAndSummary() (string, string) { - rawText := tl.writer.String() - text := re.ReplaceAllString(rawText, "") - - prettyPrefix := prettyPrefix(DefaultPadding, tl.prefix) - - // regex to find command lines in the output. - regexStr := fmt.Sprintf(`(?m)^%s \| (\*cached\* |\*local\* | )*--> `, regexp.QuoteMeta(prettyPrefix)) - r := regexp.MustCompile(regexStr) - matches := r.FindAllStringIndex(text, -1) - if len(matches) == 0 { - return "", "" - } - - // Take the last match, and use the first line up to 120 characters, or first newlines... whichever comes first. - lastMatch := matches[len(matches)-1] - remainder := text[lastMatch[1]:] // The rest of the log from end of the last match - command := truncateString(remainder, 120) // The line up to a newline or 120 chars - - // regex to get the last line, (ab)use groups to get the line without the prefix. Truncate it like command. - regexStr2 := fmt.Sprintf(`%s \| (.*)\n?$`, regexp.QuoteMeta(prettyPrefix)) - r2 := regexp.MustCompile(regexStr2) - matches2 := r2.FindAllStringSubmatch(remainder, -1) - if len(matches2) == 0 { - return command, "" - } - summary := truncateString(matches2[len(matches2)-1][1], 120) - - return command, summary -} - -func truncateString(str string, length int) string { - // This weird truncation is needed to support multi-byte characters, which the slice notation does not account for. - if length <= 0 { - return "" - } - - truncated := "" - count := 0 - for _, char := range str { - if char == '\n' { - break - } - truncated += string(char) - count++ - if count >= length { - break - } - } - return truncated -} diff --git a/conslogging/conslogging.go b/conslogging/conslogging.go index 1724d32948..3d508399f4 100644 --- a/conslogging/conslogging.go +++ b/conslogging/conslogging.go @@ -9,7 +9,6 @@ import ( "sync" "unicode/utf8" - "github.com/earthly/earthly/cleanup" "github.com/fatih/color" ) @@ -79,7 +78,6 @@ type ConsoleLogger struct { prefixWriter PrefixWriter trailingLine bool prefixPadding int - bb *BundleBuilder } // Current returns the current console. @@ -123,7 +121,6 @@ func (cl ConsoleLogger) clone() ConsoleLogger { nextColorIndex: cl.nextColorIndex, prefixPadding: cl.prefixPadding, mu: cl.mu, - bb: cl.bb, } } @@ -135,9 +132,6 @@ func (cl ConsoleLogger) WithPrefix(prefix string) ConsoleLogger { // WithPrefixAndSalt returns a ConsoleLogger with a prefix and a seed added. func (cl ConsoleLogger) WithPrefixAndSalt(prefix string, salt string) ConsoleLogger { ret := cl.clone() - if cl.bb != nil { - ret.errW = io.MultiWriter(cl.consoleErrW, cl.bb.PrefixWriter(prefix)) - } ret.prefix = prefix ret.salt = salt if ret.prefixWriter != nil { @@ -200,16 +194,6 @@ func (cl ConsoleLogger) WithPrefixWriter(w PrefixWriter) ConsoleLogger { return ret } -// WithLogBundleWriter returns a ConsoleLogger with a BundleWriter attached to capture output into a log bundle, for upload to log sharing. -func (cl ConsoleLogger) WithLogBundleWriter(entrypoint string, collection *cleanup.Collection) ConsoleLogger { - ret := cl.clone() - ret.bb = NewBundleBuilder(entrypoint, collection) - fullW := ret.bb.PrefixWriter(fullLog) - ret.consoleErrW = io.MultiWriter(ret.consoleErrW, fullW) - ret.errW = ret.consoleErrW - return ret -} - // PrintPhaseHeader prints the phase header. func (cl ConsoleLogger) PrintPhaseHeader(phase string, disabled bool, special string) { w := new(bytes.Buffer) @@ -577,56 +561,3 @@ func (cl ConsoleLogger) WithLogLevel(logLevel LogLevel) ConsoleLogger { ret.logLevel = logLevel return ret } - -// WriteBundleToDisk makes an attached bundle writer (if any) write the collected bundle to disk. -func (cl ConsoleLogger) WriteBundleToDisk() (string, error) { - if cl.bb == nil { - return "", nil - } - - return cl.bb.WriteToDisk() -} - -// MarkBundleBuilderResult marks the current targets result in a log bundle for a given prefix with the current result. -func (cl ConsoleLogger) MarkBundleBuilderResult(isError, isCanceled bool) { - if cl.bb == nil { - return - } - - var result string - if isCanceled { - result = ResultCancelled - } else { - if isError { - result = ResultFailure - } else { - result = ResultSuccess - } - } - - cl.bb.PrefixResult(cl.Prefix(), result) -} - -// MarkBundleBuilderStatus marks the current targets status in a log bundle for a given prefix with the current status. -func (cl ConsoleLogger) MarkBundleBuilderStatus(isStarted, isFinished, isCanceled bool) { - if cl.bb == nil { - return - } - - var status string - if isCanceled { - status = StatusCancelled - } else { - if isStarted { - if isFinished { - status = StatusComplete - } else { - status = StatusInProgress - } - } else { - status = StatusWaiting - } - } - - cl.bb.PrefixStatus(cl.Prefix(), status) -} diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 05c4332e40..31c75a831e 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -85,4 +85,5 @@ * [Managing Satellites](cloud/satellites/managing.md) * [Using Satellites](cloud/satellites/using.md) * [Self-Hosted Satellites](cloud/satellites/self-hosted.md) + * [GitHub runners](cloud/satellites/gha-runners.md) * [Best Practices](cloud/satellites/best-practices.md) diff --git a/docs/basics/part-6-using-docker-with-earthly.md b/docs/basics/part-6-using-docker-with-earthly.md index 63ea6a6328..af967a28d9 100644 --- a/docs/basics/part-6-using-docker-with-earthly.md +++ b/docs/basics/part-6-using-docker-with-earthly.md @@ -10,14 +10,14 @@ Examples in [Python](#more-examples), [JavaScript](#more-examples) and [Java](#m You may find that you need to run Docker commands inside a target. For those cases Earthly offers `WITH DOCKER`. `WITH DOCKER` will initialize a Docker daemon that can be used in the context of a `RUN` command. -Whenever you need to use `WITH DOCKER` we recommend (though it is not required) that you use Earthly's own Docker in Docker (dind) image: `earthly/dind:alpine-3.19-docker-25.0.2-r0`. +Whenever you need to use `WITH DOCKER` we recommend (though it is not required) that you use Earthly's own Docker in Docker (dind) image: `earthly/dind:alpine-3.19-docker-25.0.5-r0`. -Notice `WITH DOCKER` creates a block of code that has an `END` keyword. Everything that happens within this block is going to take place within our `earthly/dind:alpine-3.19-docker-25.0.2-r0` container. +Notice `WITH DOCKER` creates a block of code that has an `END` keyword. Everything that happens within this block is going to take place within our `earthly/dind:alpine-3.19-docker-25.0.5-r0` container. ### Pulling an Image ```Dockerfile hello: - FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 WITH DOCKER --pull hello-world RUN docker run hello-world END @@ -35,7 +35,7 @@ my-hello-world: SAVE IMAGE my-hello:latest hello: - FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 WITH DOCKER --load hello:latest=+my-hello-world RUN docker run hello:latest END @@ -137,7 +137,7 @@ test-setup: SAVE IMAGE test:latest integration-tests: - FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 COPY docker-compose.yml ./ WITH DOCKER --compose docker-compose.yml --load tests:latest=+test-setup RUN docker run --network=default_go/part6_default tests:latest @@ -346,7 +346,7 @@ api-docker: # Run your app and api side by side app-with-api: - FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 RUN apk add curl WITH DOCKER \ --load app:latest=+app-docker \ @@ -401,7 +401,7 @@ docker: SAVE IMAGE java-example:$tag with-postgresql: - FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 COPY ./docker-compose.yml . RUN apk update RUN apk add postgresql-client @@ -559,7 +559,7 @@ build: COPY . . run-tests: - FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 COPY ./docker-compose.yml . COPY ./tests ./tests RUN apk update diff --git a/docs/basics/part-7-using-remote-runners.md b/docs/basics/part-7-using-remote-runners.md index 1eb90abdd5..123cdec79b 100644 --- a/docs/basics/part-7-using-remote-runners.md +++ b/docs/basics/part-7-using-remote-runners.md @@ -15,7 +15,7 @@ Remote runners are especially useful in a few specific circumstances: There are two types of remote runners: * Earthly Satellites (managed by Earthly; free up to 6,000 minutes/month; get started now by visiting the [sign up](https://cloud.earthly.dev/login) page) -* Remote Buildkit (free, self-hosted) +* Remote BuildKit (free, self-hosted) ### Using Earthly Satellites @@ -56,9 +56,9 @@ earthly --sat my-satellite +my-target For more information, check out the [Earthly Satellites](../cloud/satellites.md) page. -### Using a Remote Buildkit +### Using a Remote BuildKit -To run your own remote Buildkit, you can follow the instructions on the [remote Buildkit page](../ci-integration/remote-buildkit.md). +To run your own remote BuildKit, you can follow the instructions on the [remote BuildKit page](../ci-integration/remote-buildkit.md). ### Secrets and remote builds diff --git a/docs/basics/part-8a-using-earthly-in-your-current-ci.md b/docs/basics/part-8a-using-earthly-in-your-current-ci.md index fce3ee6718..62d7758910 100644 --- a/docs/basics/part-8a-using-earthly-in-your-current-ci.md +++ b/docs/basics/part-8a-using-earthly-in-your-current-ci.md @@ -41,7 +41,7 @@ jobs: steps: - uses: earthly/actions/setup-earthly@v1 with: - version: v0.8.6 + version: v0.8.13 - uses: actions/checkout@v2 - name: Docker Login run: docker login --username "$DOCKERHUB_USERNAME" --password "$DOCKERHUB_TOKEN" @@ -51,7 +51,7 @@ jobs: Here is an explanation of the steps above: -* The action `earthly/actions/setup-earthly@v1` downloads and installs Earthly. Running this action is similar to running the Earthly installation one-liner `sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/download/v0.8.6/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly'` +* The action `earthly/actions/setup-earthly@v1` downloads and installs Earthly. Running this action is similar to running the Earthly installation one-liner `sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/download/v0.8.13/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly'` * The command `docker login` performs a login to the DockerHub registry. This is required, to prevent rate-limiting issues when using popular base images. * The command `earthly --org ... --sat ... --ci --push +build` executes the build. The `--ci` flag is used here, in order to force the use of `--strict` mode. In `--strict` mode, Earthly prevents the use of features that make the build less repeatable and also disables local outputs -- because artifacts and images resulting from the build are not needed within the CI environment. Any outputs should be pushed via `RUN --push` or `SAVE IMAGE --push` commands. The flags `--org` and `--sat` allow you to select the organization and satellite to use for the build. If no satellite is specified, the build will be executed in the CI environment itself, with limited caching. diff --git a/docs/caching/managing-cache.md b/docs/caching/managing-cache.md index 151d5c1c2f..6c76d10b2e 100644 --- a/docs/caching/managing-cache.md +++ b/docs/caching/managing-cache.md @@ -55,7 +55,7 @@ earthly prune --reset If you are using [Earthly Satellites](../cloud/satellites.md), you can simply launch a bigger satellite via the `--size` flag: `earthly sat launch --size ...`. -If you are using a self-hosted remote runner, you can configure the cache policy by passing the appropriate [buildkit configuration](https://github.com/moby/buildkit/blob/master/docs/buildkitd.toml.md) to the [buildkit container](../ci-integration/remote-buildkit.md). +If you are using a self-hosted remote runner, you can configure the cache policy by passing the appropriate [buildkit configuration](https://github.com/moby/buildkit/blob/master/docs/buildkitd.toml.md) to the [BuildKit container](../ci-integration/remote-buildkit.md). ### Resetting the cache on a remote runner diff --git a/docs/ci-integration/build-an-earthly-ci-image.md b/docs/ci-integration/build-an-earthly-ci-image.md index 984bd7a2ae..953fa54979 100644 --- a/docs/ci-integration/build-an-earthly-ci-image.md +++ b/docs/ci-integration/build-an-earthly-ci-image.md @@ -18,7 +18,7 @@ This guide will cover both approaches to constructing your image. This is the recommended approach when adopting Earthly into your containerized CI. Start by basing your custom image on ours: ```docker -FROM earthly/earthly:v0.8.6 +FROM earthly/earthly:v0.8.13 RUN ... # Add your agent, certificates, tools... ``` @@ -50,7 +50,7 @@ In this setup, Earthly will be allowed to manage an instance of its `earthly/bui To enable this, simply follow the installation instructions within your Dockerfile/Earthfile as you would on any other host. An example of installing this can be found below. ```docker -RUN wget https://github.com/earthly/earthly/releases/download/v0.8.6/earthly-linux-amd64 -O /usr/local/bin/earthly && \ +RUN wget https://github.com/earthly/earthly/releases/download/v0.8.13/earthly-linux-amd64 -O /usr/local/bin/earthly && \ chmod +x /usr/local/bin/earthly && \ /usr/local/bin/earthly bootstrap ``` @@ -89,11 +89,11 @@ fail with the error: `sh: write error: Resource busy`. ## An important note about running the image -When running the built image in your CI of choice, if you're not using a remote daemon, Earthly will start Buildkit within the same container. In this case, it is important to ensure that the directory used by Buildkit to cache the builds is mounted as a Docker volume. Failing to do so may result in excessive disk usage, slow builds, or Earthly not functioning properly. +When running the built image in your CI of choice, if you're not using a remote daemon, Earthly will start BuildKit within the same container. In this case, it is important to ensure that the directory used by BuildKit to cache the builds is mounted as a Docker volume. Failing to do so may result in excessive disk usage, slow builds, or Earthly not functioning properly. {% hint style='danger' %} ##### Important -We *strongly* recommend using a Docker volume for mounting `/tmp/earthly`. If you do not, Buildkit can consume excessive disk space, operate very slowly, or it might not function correctly. +We *strongly* recommend using a Docker volume for mounting `/tmp/earthly`. If you do not, BuildKit can consume excessive disk space, operate very slowly, or it might not function correctly. {% endhint %} In some environments, not mounting `/tmp/earthly` as a Docker volume results in the following error: diff --git a/docs/ci-integration/guides/bitbucket-pipelines-integration.md b/docs/ci-integration/guides/bitbucket-pipelines-integration.md index fa926c4261..3113bbc76f 100644 --- a/docs/ci-integration/guides/bitbucket-pipelines-integration.md +++ b/docs/ci-integration/guides/bitbucket-pipelines-integration.md @@ -7,7 +7,7 @@ You can however, run Earthly builds on Bitbucket pipelines via [remote runners]( ```yml # ./bitbucket-pipelines.yml -image: earthly/earthly:v0.8.6 +image: earthly/earthly:v0.8.13 pipelines: default: diff --git a/docs/ci-integration/guides/circle-integration.md b/docs/ci-integration/guides/circle-integration.md index c892a55920..6a953bc09a 100644 --- a/docs/ci-integration/guides/circle-integration.md +++ b/docs/ci-integration/guides/circle-integration.md @@ -14,7 +14,7 @@ jobs: steps: - checkout - run: docker login --username "$DOCKERHUB_USERNAME" --password "$DOCKERHUB_TOKEN" - - run: "sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/download/v0.8.6/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly'" + - run: "sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/download/v0.8.13/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly'" - run: earthly --ci --push +build ``` diff --git a/docs/ci-integration/guides/codebuild-integration.md b/docs/ci-integration/guides/codebuild-integration.md index 3166d16ec5..8ab452d1d3 100644 --- a/docs/ci-integration/guides/codebuild-integration.md +++ b/docs/ci-integration/guides/codebuild-integration.md @@ -18,7 +18,7 @@ version: 0.2 phases: install: commands: - - wget https://github.com/earthly/earthly/releases/download/v0.8.6/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly + - wget https://github.com/earthly/earthly/releases/download/v0.8.13/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly pre_build: commands: - echo Logging into Docker diff --git a/docs/ci-integration/guides/gh-actions-integration.md b/docs/ci-integration/guides/gh-actions-integration.md index 5d67fa008d..741d60cf5f 100644 --- a/docs/ci-integration/guides/gh-actions-integration.md +++ b/docs/ci-integration/guides/gh-actions-integration.md @@ -1,6 +1,11 @@ # GitHub Actions integration +{% hint style='info' %} +##### News +It is recommended to use [Satellites as GHA runners](../cloud/satellites/gha-runners.md), since they don't require GitHub Actions workers. +{% endhint %} + Here is an example of a GitHub Actions build that uses the [earthly/actions-setup](https://github.com/earthly/actions-setup). This example assumes an [Earthfile](../../earthfile/earthfile.md) exists with a `+build` target: @@ -60,7 +65,7 @@ jobs: - name: Docker Login run: docker login --username "$DOCKERHUB_USERNAME" --password "$DOCKERHUB_TOKEN" - name: Download latest earthly - run: "sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/download/v0.8.6/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly'" + run: "sudo /bin/sh -c 'wget https://github.com/earthly/earthly/releases/download/v0.8.13/earthly-linux-amd64 -O /usr/local/bin/earthly && chmod +x /usr/local/bin/earthly'" - name: Run build run: earthly --ci --push +build ``` diff --git a/docs/ci-integration/guides/gitlab-integration.md b/docs/ci-integration/guides/gitlab-integration.md index 7e902210fd..c6beda9784 100644 --- a/docs/ci-integration/guides/gitlab-integration.md +++ b/docs/ci-integration/guides/gitlab-integration.md @@ -15,7 +15,7 @@ variables: FORCE_COLOR: 1 EARTHLY_EXEC_CMD: "/bin/sh" -image: earthly/earthly:v0.8.6 +image: earthly/earthly:v0.8.13 before_script: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY diff --git a/docs/ci-integration/guides/google-cloud-build.md b/docs/ci-integration/guides/google-cloud-build.md index f51e049ed5..1535728204 100644 --- a/docs/ci-integration/guides/google-cloud-build.md +++ b/docs/ci-integration/guides/google-cloud-build.md @@ -106,7 +106,7 @@ Running this build will use the [`cloudbuild.yaml`](https://github.com/earthly/c ```yaml - id: 'build' - name: 'earthly/earthly:v0.8.6' + name: 'earthly/earthly:v0.8.13' args: - --ci - --push @@ -117,7 +117,7 @@ Running this build will use the [`cloudbuild.yaml`](https://github.com/earthly/c ```yaml - id: 'gcp-test' - name: 'earthly/earthly:v0.8.6' + name: 'earthly/earthly:v0.8.13' args: - +gcp-cloudbuild secretEnv: diff --git a/docs/ci-integration/guides/kubernetes.md b/docs/ci-integration/guides/kubernetes.md index 79aaefb72c..464841cd4c 100644 --- a/docs/ci-integration/guides/kubernetes.md +++ b/docs/ci-integration/guides/kubernetes.md @@ -2,7 +2,7 @@ {% hint style='info' %} ##### Note -This guide is related to self-hosting a remote Buildkit, however, Self-Hosted Satellites **beta** are now available. Self-Hosted Satellites provide more features, have better security, and are easier to deploy than remote Buildkit. Check out the [Self-Hosted Satellites Guide](../../cloud/satellites/self-hosted.md) for more details and instructions to deploy in Kubernetes or AWS EC2. +This guide is related to self-hosting a remote BuildKit, however, Self-Hosted Satellites **beta** are now available. Self-Hosted Satellites provide more features, have better security, and are easier to deploy than remote BuildKit. Check out the [Self-Hosted Satellites Guide](../../cloud/satellites/self-hosted.md) for more details and instructions to deploy in Kubernetes or AWS EC2. {% endhint %} @@ -84,7 +84,7 @@ There are some caveats that come with this kind of a setup, though: To mitigate some of the issues, it is recommended to run in a "sticky" mode to keep builds pinned to a single instance for the duration. You can see how to do this in our example: ```yaml -# Use session affinity to prevent "roaming" across multiple buildkit instances; if needed. +# Use session affinity to prevent "roaming" across multiple BuildKit instances; if needed. sessionAffinity: ClientIP sessionAffinityConfig: clientIP: diff --git a/docs/ci-integration/guides/woodpecker-integration.md b/docs/ci-integration/guides/woodpecker-integration.md index cca51be46d..7f7c51776d 100644 --- a/docs/ci-integration/guides/woodpecker-integration.md +++ b/docs/ci-integration/guides/woodpecker-integration.md @@ -14,7 +14,7 @@ The project needs to be [trusted](https://woodpecker-ci.org/docs/usage/project-s #.woodpecker.yml pipeline: earthly: - image: earthly/earthly:v0.8.6 + image: earthly/earthly:v0.8.13 volumes: - /var/run/docker.sock:/var/run/docker.sock environment: diff --git a/docs/ci-integration/overview.md b/docs/ci-integration/overview.md index f2a637fe9a..c5e966912c 100644 --- a/docs/ci-integration/overview.md +++ b/docs/ci-integration/overview.md @@ -27,7 +27,7 @@ Once you have ensured that the dependencies are available, you'll need to instal This is the simplest method for adding `earthly` to your CI. It will work best on dedicated computers, or in scripted/auto-provisioned build environments. You can pin it to a specific version like so: ```shell -wget https://github.com/earthly/earthly/releases/download/v0.8.6/earthly-linux-amd64 -O /usr/local/bin/earthly && \ +wget https://github.com/earthly/earthly/releases/download/v0.8.13/earthly-linux-amd64 -O /usr/local/bin/earthly && \ chmod +x /usr/local/bin/earthly && \ /usr/local/bin/earthly bootstrap ``` diff --git a/docs/ci-integration/pull-through-cache.md b/docs/ci-integration/pull-through-cache.md index 2df6de7cf6..8572202679 100644 --- a/docs/ci-integration/pull-through-cache.md +++ b/docs/ci-integration/pull-through-cache.md @@ -180,7 +180,7 @@ global: The next time earthly is run, it will detect the configuration change and will restart the `earthly-buildkitd` container to reflect these settings. -You can force these settings to be applied, and verify the mirror appears in the buildkit config by running: +You can force these settings to be applied, and verify the mirror appears in the BuildKit config by running: ```bash earthly bootstrap && docker exec earthly-buildkitd cat /etc/buildkitd.toml diff --git a/docs/ci-integration/remote-buildkit.md b/docs/ci-integration/remote-buildkit.md index e3cbe9b2b2..d61e74cd53 100644 --- a/docs/ci-integration/remote-buildkit.md +++ b/docs/ci-integration/remote-buildkit.md @@ -2,7 +2,7 @@ {% hint style='info' %} ##### Note -This guide is related to self-hosting a remote Buildkit, however, Self-Hosted Satellites **beta** are now available. Self-Hosted Satellites provide more features, have better security, and are easier to deploy than remote Buildkit. Check out the [Self-Hosted Satellites Guide](../cloud/satellites/self-hosted.md) for more details. If your use case cannot tolerate a cloud-based control plane, however, then self-hosting a remote Buildkit is still supported. +This guide is related to self-hosting a remote BuildKit, however, Self-Hosted Satellites **beta** are now available. Self-Hosted Satellites provide more features, have better security, and are easier to deploy than remote BuildKit. Check out the [Self-Hosted Satellites Guide](../cloud/satellites/self-hosted.md) for more details. If your use case cannot tolerate a cloud-based control plane, however, then self-hosting a remote BuildKit is still supported. {% endhint %} ## Introduction @@ -27,7 +27,7 @@ A remote daemon should be reachable by all clients intending to use it. Earthly **`/tmp/earthly`** -This path within the container is the location that Buildkit uses for storing the cache. Because this folder sees _a lot_ of traffic, its important that it remains fast. +This path within the container is the location that BuildKit uses for storing the cache. Because this folder sees _a lot_ of traffic, its important that it remains fast. {% hint style='danger' %} ##### Important diff --git a/docs/ci-integration/use-earthly-ci-image.md b/docs/ci-integration/use-earthly-ci-image.md index 9dfb17c400..f95204f7a0 100644 --- a/docs/ci-integration/use-earthly-ci-image.md +++ b/docs/ci-integration/use-earthly-ci-image.md @@ -6,7 +6,7 @@ This guide is intended to help you use the Earthly image for your containerized ## Prerequisites -The `earthly/earthly` image requires that it is run as `--privileged`, or alternatively, it is run without the embedded Buildkit daemon (`NO_BUILDKIT=1`). +The `earthly/earthly` image requires that it is run as `--privileged`, or alternatively, it is run without the embedded BuildKit daemon (`NO_BUILDKIT=1`). ## Getting Started @@ -29,39 +29,39 @@ An alternative option is to use the `earthly/earthly` image in conjunction with You may also use the `earthly/earthly` image to run a build against an Earthly Satellite. To achieve this you can pass along an `EARTHLY_TOKEN` environment variable, along with the command-line flags `--sat` and `--org`, to point the build to a specific satellite. -For more details on using remote execution, [see our guide on remote Buildkit](./remote-buildkit.md) or the [introduction to Satellites](../cloud/satellites.md). +For more details on using remote execution, [see our guide on remote BuildKit](./remote-buildkit.md) or the [introduction to Satellites](../cloud/satellites.md). #### Mounting the source code The image expects the source code of the application you are building in the current working directory (by default `/workspace`). You will need to copy or mount the necessary files to that directory prior to invoking the entrypoint. ```bash -docker run --privileged --rm -v "$PWD":/workspace earthly/earthly:v0.8.6 +my-target +docker run --privileged --rm -v "$PWD":/workspace earthly/earthly:v0.8.13 +my-target ``` Or, if you would like to use an alternative directory: ```bash -docker run --privileged --rm -v "$PWD":/my-dir -w /my-dir earthly/earthly:v0.8.6 +my-target +docker run --privileged --rm -v "$PWD":/my-dir -w /my-dir earthly/earthly:v0.8.13 +my-target ``` Alternatively, you may rely on Earthly to perform a git clone, by using the remote target reference format. For example: ```bash -docker run --privileged --rm earthly/earthly:v0.8.6 github.com/foo/bar:my-branch+target +docker run --privileged --rm earthly/earthly:v0.8.13 github.com/foo/bar:my-branch+target ``` #### `NO_BUILDKIT` Environment Variable -As the embedded Buildkit daemon requires `--privileged`, for some operations you may be able to use the `NO_BUILDKIT=1` environment variable to disable the embedded Buildkit daemon. This is especially useful when running against a remote buildkit (like a Satellite), or when not performing a build as part of the command (like when using `earthly account`). +As the embedded BuildKit daemon requires `--privileged`, for some operations you may be able to use the `NO_BUILDKIT=1` environment variable to disable the embedded BuildKit daemon. This is especially useful when running against a remote BuildKit (like a Satellite), or when not performing a build as part of the command (like when using `earthly account`). ## An important note about running the image -When running the built image in your CI of choice, if you're not using a remote daemon, Earthly will start Buildkit within the same container. In this case, it is important to ensure that the directory used by Buildkit to cache the builds is mounted as a Docker volume. Failing to do so may result in excessive disk usage, slow builds, or Earthly not functioning properly. +When running the built image in your CI of choice, if you're not using a remote daemon, Earthly will start BuildKit within the same container. In this case, it is important to ensure that the directory used by BuildKit to cache the builds is mounted as a Docker volume. Failing to do so may result in excessive disk usage, slow builds, or Earthly not functioning properly. {% hint style='danger' %} ##### Important -We *strongly* recommend using a Docker volume for mounting `/tmp/earthly`. If you do not, Buildkit can consume excessive disk space, operate very slowly, or it might not function correctly. +We *strongly* recommend using a Docker volume for mounting `/tmp/earthly`. If you do not, BuildKit can consume excessive disk space, operate very slowly, or it might not function correctly. {% endhint %} In some environments, not mounting `/tmp/earthly` as a Docker volume results in the following error: diff --git a/docs/cloud/oidc.md b/docs/cloud/oidc.md new file mode 100644 index 0000000000..1f66d86271 --- /dev/null +++ b/docs/cloud/oidc.md @@ -0,0 +1,84 @@ +# OpenID Connect (OIDC) Authentication + +Earthly can support cases where you might require access to a 3rd-party cloud provider as part of your build, without storing secrets in your CI or accessing credentials from your local environment. +This is especially useful in CI where otherwise, authentication requires MFA(multi-factor authentication). +The OIDC protocol allows you to access the provider without storing credentials in your local environment or CI. + +## Introduction + +This page covers how to set up OIDC with cloud providers. +At the moment the only AWS is supported. + +## Cloud Providers + +### AWS + +#### Setup + +1. Add the Earthly OIDC provider to AWS IAM - see the [AWS guide](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_oidc.html). + 1. Set https://api.earthly.dev as the provider URL. + 2. Set `sts.amazonaws.com` as the audience. +2. Create a new IAM role (or configure an existing role you'd like to reuse) - see the [AWS guide](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-idp_oidc.html). + 1. Make sure to limit the permissions for the role (these are the actions the user can perform after assuming the role) + 2. Make sure to limit who can assume the role by specifying a trust policy such as: +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "api.earthly.dev:aud": "sts.amazonaws.com", + "api.earthly.dev:sub": "/" + } + } + } + ] +} +``` + +where: +* `` is the oidc provider's arn that was configured in step 1. +* `` the earthly org the user is a member of and is set in the Earthfile or as part of the earthly build execution (see more details below). +* `` the earthly project the user has access to [read secrets](./managing-permissions.md#earthly-project-access-levels) from, and is set in the Earthfile or as part of the earthly build execution (see more details below). + +Note, a trust policy allows configuring different rules which you can mix and match to allow/disallow assuming the role by members of your team: +* To allow access to all members of the org: +```json +"Condition": { + "StringLike": { + "api.earthly.dev:sub": "/*" + } +} +``` +* To allow access only to a specific user: +```json +"Condition": { + "StringEquals": { + "api.earthly.dev:email": "" + } +} +``` +where `` is the email address associated with the earthly account. + +#### Usage + +Once OIDC is configured, you can access AWS resources from your build. +Here is an example Earthfile to list S3 objects: +```dockerfile +VERSION --run-with-aws --run-with-aws-oidc 0.8 + +PROJECT / + +aws: + FROM amazon/aws-cli + LET OIDC="role-arn=arn:aws:iam::1234567890:role/your-oidc-role,session-name=my-session,region=us-east-1" + RUN --aws --oidc=$OIDC aws s3 ls +``` + +For more information on the `RUN --aws --oidc` flags, see [here](../earthfile/earthfile.md#--oidc-oidc-spec-experimental) diff --git a/docs/cloud/overview.md b/docs/cloud/overview.md index fedcd04041..242763692f 100644 --- a/docs/cloud/overview.md +++ b/docs/cloud/overview.md @@ -2,10 +2,11 @@ Earthly Cloud is a collection of features that enrich the Earthly experience via cloud-based services. These include: -* [Earthly Satellites](./satellites.md): Cloud-based Buildkit instances managed by the Earthly team. +* [Earthly Satellites](./satellites.md): Cloud-based BuildKit instances managed by the Earthly team. * [Earthly Cloud Secrets](./cloud-secrets.md): A secret management system that allows you to store secrets in a cloud-based service and use them across builds. * [Auto-skip](../caching/caching-in-earthfiles.md#auto-skip): A feature that allows you to skip large parts of a build in certain situations. * **Log sharing**: The ability to share build logs with coworkers. +* [OIDC Authentication](./oidc.md): The ability to authenticate to 3rd-party cloud services without storing long-term credentials. ## Sign up for Earthly Cloud for free! diff --git a/docs/cloud/satellites/best-practices.md b/docs/cloud/satellites/best-practices.md index 1cababa097..b9fbe5bdd4 100644 --- a/docs/cloud/satellites/best-practices.md +++ b/docs/cloud/satellites/best-practices.md @@ -48,10 +48,10 @@ This strategy is easiest to implement when there are distinct components that ca When running Earthly on pull requests, it is common for new commits to be pushed before a previous build has finished. Since the previous results may no longer be useful, it’s good practice to cancel the previous build – freeing up resources on the satellite for the current build. -Many CI systems allow this behavior to be configured. Github Actions, for example, can be configured with: +Many CI systems allow this behavior to be configured. GitHub Actions, for example, can be configured with: ```yaml -name: Github Actions CI +name: GitHub Actions CI concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} diff --git a/docs/cloud/satellites/gha-runners.md b/docs/cloud/satellites/gha-runners.md new file mode 100644 index 0000000000..b3ce65a95a --- /dev/null +++ b/docs/cloud/satellites/gha-runners.md @@ -0,0 +1,125 @@ +# Satellites as GitHub Actions runners + +{% hint style='warning' %} +This feature is experimental. + +Not recommended for production usage since it might introduce breaking changes in the future. + +Feedback is welcome and much appreciated! + +{% endhint %} + +Earthly satellites are now bundled with a GitHub Actions runner, so they can directly pull jobs from GitHub Actions without the need of an intermediate runner. + +These runners come with the Earthly CLI preinstalled and configured to use the satellite BuildKit instance, so GitHub Actions jobs will share the same satellite cache than the traditional satellite builds. + +## Getting started + +Satellite-based GitHub Actions runners can be enabled for a particular repository or for all repositories of a GitHub organization at once. + +The integration process requires you to provide us with a GitHub token, so we can: +- register a webhook in your GitHub repository/organization to receive events associated to GitHub Actions jobs +- create GitHub self-hosted runners on demand, to process your repository/organization jobs + +Follow the next steps to create such integrations: + +### 1. Create a GitHub token +Both GitHub classic and fine-grained tokens are supported, but depending on the type of installation (organization-wide or single-repository), the provided token requires different scopes: + +| Integration type | User type | Classic token scopes | Fine-grained token permissions | +|------------------|--------------------|-------------------------------|----------------------------------------------------------------------| +| Organization | Organization admin | `admin:org_hook`, `admin:org` | `organization_hooks:write`, `organization_self_hosted_runners:write` | +| Repository | Repository admin | `admin:repo_hook`, `repo` | `repository_hooks:write`, `administration:write` | + +{% hint style='info' %} +Follow the [official docs](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) for detailed information on how to create a GitHub token, and make sure to set an expiration long enough, since the integration won't work after the token expires. +{% endhint %} + +### 2. Register the integration via CLI +Create the integration using the `earthly github add` CLI command, passing the token created in the previous step. + +#### Organization integration +``` +earthly github add \ + --org \ + --gh-org \ + --gh-token +``` + +#### Single repository integration +``` +earthly github add \ + --org \ + --gh-org \ + --gh-repo \ + --gh-token +``` + +### 3. Configure your satellites + +This feature needs to be enabled during satellite creation to be able to use it. + +#### Earthly-Cloud satellites +Launch the satellite with the `enable-gha-runner` [feature-flag](https://docs.earthly.dev/earthly-cloud/satellites/managing#changing-feature-flags) enabled. +``` +earthly satellite launch --feature-flag enable-gha-runner +``` + +#### Self-hosted satellites +To enable the GH runner for a self-hosted satellite, set this environment entry when launching it: +``` +-e RUNNER_GHA_ENABLED=true +``` +also note that the satellite container must have access to the docker daemon in order to run the GitHub Actions jobs in containers: +``` +-v /var/run/docker.sock:/var/run/docker.sock +``` + +##### Example +```shell +docker run --privileged \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v satellite-cache:/tmp/earthly:rw \ + -p 8372:8372 \ + -e EARTHLY_TOKEN= \ + -e EARTHLY_ORG= \ + -e SATELLITE_NAME= \ + -e SATELLITE_HOST= \ + -e RUNNER_GHA_ENABLED=true \ + earthly/satellite:v0.8.13 +``` +{% hint style='info' %} +**Required version:** Use at least `earthly/satellite:v0.8.13 +{% endhint %} + +##### Logs +You should see a log message like this, when the GitHub Actions runner is enabled: +``` +{...,"msg":"starting GitHub Actions job polling loop",...} +``` + +### 4. Configure your GitHub Actions jobs +In order to make a job run into the satellite, you'll need to set its [runs-on](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on) label as follows: + +``` +runs-on: [earthly-satellite#] +``` + +#### Example +The following example runs the `+build` target in the satellite. Given that the GH runner is configured to use the satellite BuildKit instance, the persistent satellite cache is implicitly used here. +```yml +earthly-job: + runs-on: [earthly-satellite#my-gha-satellite] + env: + FORCE_COLOR: 1 + EARTHLY_TOKEN: "${{ secrets.EARTHLY_TOKEN }}" + steps: + - uses: actions/checkout@v2 + - name: Earthly build + run: earthly -ci +build +``` + +{% hint style='warning' %} +For Earthly-Cloud satellites make sure you have an [EARTHLY_TOKEN](https://docs.earthly.dev/docs/earthly-command#earthly-account-create-token) available in your [GitHub Actions secrets](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions) store, and add it to the job environment, as shown in the previous example. Future versions will remove this requirement. + +{% endhint %} diff --git a/docs/cloud/satellites/self-hosted.md b/docs/cloud/satellites/self-hosted.md index 6d4d41deec..bb65a2e694 100644 --- a/docs/cloud/satellites/self-hosted.md +++ b/docs/cloud/satellites/self-hosted.md @@ -26,7 +26,7 @@ docker run --privileged \ -e EARTHLY_ORG=my-org \ -e SATELLITE_NAME=my-satellite \ -e SATELLITE_HOST=153.65.8.0 \ - earthly/satellite:v0.8.6 + earthly/satellite:v0.8.13 ``` The following environment variables are required: @@ -107,7 +107,7 @@ runcmd: -e EARTHLY_TOKEN=GuFna*****nve7e \ -e EARTHLY_ORG=my-org \ -e SATELLITE_NAME=my-satellite \ - earthly/satellite:v0.8.6 + earthly/satellite:v0.8.13 ``` Note that the `SATELLITE_HOST` variable is unset in this example so that the host is auto-discovered by the satellite when it starts. This should result in the instance’s private DNS being used as the host. @@ -134,7 +134,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: earthly/satellite:v0.8.6 + image: earthly/satellite:v0.8.13 securityContext: privileged: true ports: @@ -174,7 +174,7 @@ Here’s an example of how to attach the volume using the Docker command line: ``` docker run -v earthly-cache:/tmp/earthly:rw \ ... - earthly/satellite:v0.8.6 + earthly/satellite:v0.8.13 ``` ## Additional Environment Variables @@ -188,12 +188,12 @@ The following environment variables can also be set to tweak the performance of * `CACHE_KEEP_DURATION` - How long idle cache will be retained on disk before being pruned (in seconds). * `RUNNER_DISABLE_TLS` - Disable TLS on the satellite. Requires Earthly CLI to also disable TLS. Not recommended in most cases. * `LOG_LEVEL` - The log level of the internal "runner" process within the satellite. Set to INFO by default. -* `BUILDKIT_DEBUG` - Enable buildkit debug-level logs. False by default. -* `EARTHLY_ADDITIONAL_BUILDKIT_CONFIG` - allows additional buildkit configs to be injected in TOML format. +* `BUILDKIT_DEBUG` - Enable BuildKit debug-level logs. False by default. +* `EARTHLY_ADDITIONAL_BUILDKIT_CONFIG` - allows additional BuildKit configs to be injected in TOML format. ## Running on Localhost -For testing purposes, you may want to try your self-hosted satellite on localhost. Satellites currently do not support using `localhost` or `127.0.0.1` as an address when supplied to `SATELLITE_HOST`, in part because Earthly CLI has reserved this address for use as its own local Buildkit container. +For testing purposes, you may want to try your self-hosted satellite on localhost. Satellites currently do not support using `localhost` or `127.0.0.1` as an address when supplied to `SATELLITE_HOST`, in part because Earthly CLI has reserved this address for use as its own local BuildKit container. It’s still possible to test self-hosted satellites locally, however, by using an alternate entry in your `/etc/hosts` file that maps to `localhost`. For example, you can try adding this entry: @@ -223,7 +223,7 @@ If you are having problems using or deploying your self-hosted satellite, please ### Problem: The satellite log says that it is running on port 9372 -**Resolution:** The log message `"running server on [::]:9372"` can be misleading, however, the exposed port on the container is still 8372. Multiple processes are running inside the satellite container, including an earthly/buildkit process. This log message comes from the buildkit process, however, a separate process on port 8372 handles the incoming gRPC requests to the container. +**Resolution:** The log message `"running server on [::]:9372"` can be misleading, however, the exposed port on the container is still 8372. Multiple processes are running inside the satellite container, including an Earthly/BuildKit process. This log message comes from the BuildKit process, however, a separate process on port 8372 handles the incoming gRPC requests to the container. ### Problem: Satellite shows an `operational` state even though it is no longer running diff --git a/docs/cloud/satellites/using.md b/docs/cloud/satellites/using.md index a91225000a..1e4ef08721 100644 --- a/docs/cloud/satellites/using.md +++ b/docs/cloud/satellites/using.md @@ -54,7 +54,7 @@ The following feature flags are recommended for use with Satellites and will be satellite | Connecting to core-test... satellite | ...Done satellite | Version github.com/earthly/buildkit v0.6.21 7a6f9e1ab2a3a3ddec5f9e612ef390af218a32bd - satellite | Info: Buildkit version (v0.6.21) is different from Earthly version (prerelease) + satellite | Info: BuildKit version (v0.6.21) is different from Earthly version (prerelease) satellite | Platforms: linux/amd64 (native) linux/amd64/v2 linux/amd64/v3 linux/amd64/v4 linux/arm64 linux/riscv64 linux/ppc64le linux/s390x linux/386 linux/mips64le linux/mips64 linux/arm/v7 linux/arm/v6 satellite | Utilization: 0 other builds, 0/12 op load satellite | GC stats: 9.0 GB cache, avg GC duration 275ms, all-time GC duration 2.754s, last GC duration 0s, last cleared 0 B diff --git a/docs/docker-images/all-in-one.md b/docs/docker-images/all-in-one.md index d5a21ecd85..fabe28e42f 100644 --- a/docs/docker-images/all-in-one.md +++ b/docs/docker-images/all-in-one.md @@ -2,7 +2,7 @@ This image contains `earthly`, `buildkit`, and some extra configuration to enabl ## Tags -Currently, the `latest` tag is `v0.8.6`. +Currently, the `latest` tag is `v0.8.13`. For other available tags, please check out https://hub.docker.com/r/earthly/earthly/tags ## Quickstart @@ -11,26 +11,26 @@ Want to get started? Here are a couple sample `docker run` commands that cover t ### Usage with Docker Socket -This example shows how to use the Earthly container in conjunction with a Docker socket that Earthly can use to start up the Buildkit daemon. +This example shows how to use the Earthly container in conjunction with a Docker socket that Earthly can use to start up the BuildKit daemon. ```bash -docker run -t -v $(pwd):/workspace -v /var/run/docker.sock:/var/run/docker.sock -e NO_BUILDKIT=1 earthly/earthly:v0.8.6 +for-linux +docker run -t -v $(pwd):/workspace -v /var/run/docker.sock:/var/run/docker.sock -e NO_BUILDKIT=1 earthly/earthly:v0.8.13 +for-linux ``` Here's a quick breakdown: - `-t` tells Docker to emulate a TTY. This makes the `earthly` log output colorized. - `-v $(pwd):/workspace` mounts the source code into the conventional location within the docker container. Earthly is executed from this directory when starting the container. Any artifacts saved within this folder remain on your local machine. -- `-v /var/run/docker.sock:/var/run/docker.sock` mounts the Docker socket such that Earthly can start Buildkit as a Docker container in the host's Docker. -- `-e NO_BUILDKIT=1` tells the Earthly container not to start en embedded buildkit. A Buildkit daemon will instead be started via the Docker socket provided. +- `-v /var/run/docker.sock:/var/run/docker.sock` mounts the Docker socket such that Earthly can start BuildKit as a Docker container in the host's Docker. +- `-e NO_BUILDKIT=1` tells the Earthly container not to start en embedded BuildKit. A BuildKit daemon will instead be started via the Docker socket provided. - `+for-linux` is the target to be invoked. All arguments specified after the image tag will be passed to `earthly`. -### Usage with Embedded Buildkit +### Usage with Embedded BuildKit -This example shows how the Earthly image can start a Buildkit daemon within the same container. A Docker socket is not needed in this case, however the container will need to be run with the `--privileged` flag. +This example shows how the Earthly image can start a BuildKit daemon within the same container. A Docker socket is not needed in this case, however the container will need to be run with the `--privileged` flag. ```bash -docker run --privileged -t -v $(pwd):/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.8.6 +for-linux +docker run --privileged -t -v $(pwd):/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.8.13 +for-linux ``` Here's a quick breakdown: @@ -46,7 +46,7 @@ Here's a quick breakdown: This example utilizes an [Earthly Satellite](https://docs.earthly.dev/earthly-cloud/satellites) to perform builds. The code to be built is downloaded directly from GitHub. ```bash -docker run -t -e NO_BUILDKIT=1 -e EARTHLY_TOKEN= earthly/earthly:v0.8.6 --ci --org --sat github.com/earthly/earthly+for-linux +docker run -t -e NO_BUILDKIT=1 -e EARTHLY_TOKEN= earthly/earthly:v0.8.13 --ci --org --sat github.com/earthly/earthly+for-linux ``` Here's what this does: @@ -61,11 +61,11 @@ Here's what this does: This example shows how to use the Earthly container to run non-build commands. This is useful for running commands like `earthly account`, or `earthly secret`. ```bash -docker run -t -e NO_BUILDKIT=1 -e EARTHLY_TOKEN= earthly/earthly:v0.8.6 account list-tokens +docker run -t -e NO_BUILDKIT=1 -e EARTHLY_TOKEN= earthly/earthly:v0.8.13 account list-tokens ``` ```bash -docker run -t -e NO_BUILDKIT=1 -e EARTHLY_TOKEN= earthly/earthly:v0.8.6 secret get foo +docker run -t -e NO_BUILDKIT=1 -e EARTHLY_TOKEN= earthly/earthly:v0.8.13 secret get foo ``` ## Using This Image @@ -110,7 +110,7 @@ This is the easiest way to ensure you get the nice, colorized output from `earth |-------------------------------------|--------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | GLOBAL_CONFIG | | Any valid YAML for the top-level `global` key in `config.yml`. Example: `{disable_analytics: true, local_registry_host: 'tcp://127.0.0.1:8371'}` | | GIT_CONFIG | | Any valid YAML for the top-level `git` key in `config.yml`. Example: `{example: {pattern: 'example.com/([^/]+)', substitute: 'ssh://git@example.com:2222/var/git/repos/$1.git', auth: ssh}}` | -| NO_BUILDKIT | | Disables the embedded Buildkit daemon. | +| NO_BUILDKIT | | Disables the embedded BuildKit daemon. | | DOCKER_HOST | `/var/run/docker.sock` | From Docker's CLI. | | BUILDKIT_HOST | `tcp://:8372` | The address of your BuildKit host. Use this when you have a remote `buildkitd` you would like to connect to. | | EARTHLY_ADDITIONAL_BUILDKIT_CONFIG | | Additional `buildkitd` config to append to the generated configuration file. | diff --git a/docs/docker-images/buildkit-standalone.md b/docs/docker-images/buildkit-standalone.md index 1c9d70fbf7..7e4972f9e8 100644 --- a/docs/docker-images/buildkit-standalone.md +++ b/docs/docker-images/buildkit-standalone.md @@ -4,7 +4,7 @@ This image contains `buildkit` with some Earthly-specific setup. This is what Ea ## Tags -Currently, the `latest` tag is `v0.8.6`. +Currently, the `latest` tag is `v0.8.13`. For other available tags, please check out https://hub.docker.com/r/earthly/buildkitd/tags ## Quickstart diff --git a/docs/earthfile/earthfile.md b/docs/earthfile/earthfile.md index 0d7e5b6121..a0e8f93355 100644 --- a/docs/earthfile/earthfile.md +++ b/docs/earthfile/earthfile.md @@ -282,7 +282,7 @@ Note that `RUN --ssh` option is only used for creating a tunnel to the host's ss Mounts a file or directory in the context of the build environment. -The `` is defined as a series of comma-separated list of key-values. The following keys are allowed +The `` is defined as a series of comma-separated list of key-values. The following keys are allowed: | Key | Description | Example | |-----------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------| @@ -365,6 +365,26 @@ The `--aws` flag has experimental status. To use this feature, it must be enable Makes AWS credentials available to the executed command via the host's environment variables or ~/.aws directory. +##### `--oidc ` (experimental) + +{% hint style='info' %} +##### Note +The `--oidc` flag has experimental status and can only be used conjointly with the `--aws` flag. To use this feature, it must be enabled via `VERSION --run-with-aws --run-with-aws-oidc 0.8`. +{% endhint %} + +Makes AWS credentials available to the executed command via AWS OIDC provider. + +The `` is defined as a series of comma-separated list of key-values. The following keys are allowed: + +| Key | Description | Example | +|--------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------| +| `session-name` | The session name to identify in AWS's logs. If any `RUN ... --oidc` commands use the same `session-name`, they will share the same temporary token | `session-name=my-session` | +| `role-arn` | The AWS arn of the role for which to get credentials. | `role-arn=arn:aws:iam::123456789012:role/some-role` | +| `region` | The AWS region to connect to in order to get the credentials. This will also be the region used by the executed AWS command (though the region may be overridden in the command). If the region is not specified, the global AWS endpoint will be used | `region=us-east-1` | +| `session-duration` | The time the credentials will be valid for before they expire. Default (AWS minimum): 15 minutes. | `session-duration=20m` | + +Click [here](../cloud/oidc.md#openid-connect-oidc-authentication) for more information on how to configure OIDC in AWS for Earthly. + ##### `--raw-output` (experimental) {% hint style='info' %} @@ -1172,7 +1192,7 @@ This does not apply to Dockerfile's [RUN --security](https://docs.docker.com/ref #### Synopsis ```Dockerfile -WITH DOCKER [--pull ] [--load =] [--compose ] +WITH DOCKER [--pull ] [--load [=]] [--compose ] [--service ] [--allow-privileged] ... @@ -1192,7 +1212,7 @@ The `WITH DOCKER` clause only supports the command [`RUN`](#run). Other commands A typical example of a `WITH DOCKER` clause might be: ```Dockerfile -FROM earthly/dind:alpine-3.19-docker-25.0.3-r2 +FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 WORKDIR /test COPY docker-compose.yml ./ WITH DOCKER \ @@ -1214,7 +1234,7 @@ For information on using `WITH DOCKER` with podman see the [Podman guide](../gui ##### Note For performance reasons, it is recommended to use a Docker image that already contains `dockerd`. If `dockerd` is not found, Earthly will attempt to install it. -Earthly provides officially supported images such as `earthly/dind:alpine-3.19-docker-25.0.3-r2` and `earthly/dind:ubuntu-23.04-docker-24.0.5-1` to be used together with `WITH DOCKER`. +Earthly provides officially supported images such as `earthly/dind:alpine-3.19-docker-25.0.5-r0` and `earthly/dind:ubuntu-23.04-docker-25.0.2-1` to be used together with `WITH DOCKER`. {% endhint %} {% hint style='info' %} @@ -1235,9 +1255,9 @@ This option may be repeated in order to provide multiple images to be pulled. It is recommended that you avoid issuing `RUN docker pull ...` and use `WITH DOCKER --pull ...` instead. The classical `docker pull` command does not take into account Earthly caching and so it would redownload the image much more frequently than necessary. {% endhint %} -##### `--load =` +##### `--load [=]` -Builds the image referenced by `` and then loads it into the temporary Docker daemon created by `WITH DOCKER`. The image can be referenced as `` within `WITH DOCKER`. +Builds the image referenced by `` and then loads it into the temporary Docker daemon created by `WITH DOCKER`. Within `WITH DOCKER`, the image can be referenced as ``, if specified, or otherwise by the name of the image specified in the referenced target's `SAVE IMAGE` command. `` may be a simple target reference (`+some-target`), or a target reference with a build arg `(+some-target --SOME_BUILD_ARG=value)`. @@ -1304,6 +1324,8 @@ This option is deprecated. Please use `--load =( --` is evaluated by running it in the build environment. If the exit code of the expression is zero, then the block of that condition is executed. Otherwise, the control continues to the next `ELSE IF` condition (if any), or if no condition returns a non-zero exit code, the control continues to executing the ``, if one is provided. +#### Examples + A very common pattern is to use the POSIX shell `[ ... ]` conditions. For example the following marks port `8080` as exposed if the file `./foo` exists. ```Dockerfile @@ -1312,6 +1334,40 @@ IF [ -f ./foo ] END ``` +It is also possible to call other commands, which can be useful for more comparisons such as semantic versioning. For example: + +```Dockerfile +VERSION 0.8 + +test: + FROM python:3 + RUN pip3 install semver + + # The following python script requires two arguments (v1 and v2) + # and will return an exit code of 0 when v1 is semantically greater than v2 + # or an exit code of 1 in all other cases. + RUN echo "#!/usr/bin/env python3 +import sys +import semver +v1 = sys.argv[1] +v2 = sys.argv[2] +if semver.compare(v1, v2) > 0: + sys.exit(0) +sys.exit(1) + " > ./semver-gt && chmod +x semver-gt + + # Define two different versions + ARG A="0.3.2" + ARG B="0.10.1" + + # and compare them + IF ./semver-gt "$A" "$B" + RUN echo "A ($A) is semantically greater than B ($B)" + ELSE + RUN echo "A ($A) is NOT semantically greater than B ($B)" + END +``` + {% hint style='info' %} ##### Note Performing a condition requires that a `FROM` (or a from-like command, such as `LOCALLY`) has been issued before the condition itself. diff --git a/docs/earthfile/features.md b/docs/earthfile/features.md index b0a9fc21bb..f8704bf4fa 100644 --- a/docs/earthfile/features.md +++ b/docs/earthfile/features.md @@ -51,47 +51,48 @@ VERSION [...] ## Feature flags -| Feature flag | Status | Description | -|-----------------------------------------|---------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------| -| `--use-registry-for-with-docker` | 0.5 | Makes use of the embedded BuildKit Docker registry (instead of tar files) for `WITH DOCKER` loads and pulls | -| `--use-copy-include-patterns` | 0.6 | Speeds up COPY transfers | -| `--referenced-save-only` | 0.6 | Changes the behavior of SAVE commands in a significant way | -| `--for-in` | 0.6 | Enables support for `FOR ... IN ...` commands | -| `--require-force-for-unsafe-saves` | 0.6 | Requires `--force` for saving artifacts locally outside the Earthfile's directory | -| `--no-implicit-ignore` | 0.6 | Eliminates implicit `.earthlyignore` entries, such as `Earthfile` and `.tmp-earthly-out` | -| `--earthly-version-arg` | 0.7 | Enables builtin ARGs: `EARTHLY_VERSION` and `EARTHLY_BUILD_SHA` | -| `--shell-out-anywhere` | 0.7 | Allows shelling-out in any earthly command (including in the middle of `ARG`) | -| `--explicit-global` | 0.7 | Base target args must have a `--global` flag in order to be considered global args | -| `--check-duplicate-images` | 0.7 | Check for duplicate images during output | -| `--use-cache-command` | 0.7 | Allow use of `CACHE` command in Earthfiles | -| `--use-host-command` | 0.7 | Allow use of `HOST` command in Earthfiles | -| `--use-copy-link` | 0.7 | Use the equivalent of `COPY --link` for all copy-like operations | -| `--new-platform` | 0.7 | Enable new platform behavior | -| `--no-tar-build-output` | 0.7 | Do not print output when creating a tarball to load into `WITH DOCKER` | -| `--use-no-manifest-list` | 0.7 | Enable the `SAVE IMAGE --no-manifest-list` option | -| `--use-chmod` | 0.7 | Enable the `COPY --chmod` option | -| `--earthly-locally-arg` | 0.7 | Enable the `EARTHLY_LOCALLY` arg | -| `--use-project-secrets` | 0.7 | Enable project-based secret resolution | -| `--use-pipelines` | 0.7 | Enable the `PIPELINE` and `TRIGGER` commands | -| `--earthly-git-author-args` | 0.7 | Enable the `EARTHLY_GIT_AUTHOR` and `EARTHLY_GIT_CO_AUTHORS` args | -| `--wait-block` | 0.7 | Enable the `WAIT` / `END` block commands | -| `--no-network` | 0.8 | Allow the use of `RUN --network=none` commands | -| `--arg-scope-and-set` | 0.8 | Enable the `LET` / `SET` commands and nested `ARG` scoping | -| `--use-docker-ignore` | 0.8 | Enable the use of `.dockerignore` files in `FROM DOCKERFILE` targets | -| `--pass-args` | 0.8 | Enable the optional `--pass-args` flag for the `BUILD`, `FROM`, `COPY`, `WITH DOCKER --load` commands | -| `--global-cache` | 0.8 | Enable global caches (shared across different Earthfiles), for cache mounts and `CACHE` commands having an ID | +| Feature flag | Status | Description | +|-----------------------------------------|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------| +| `--use-registry-for-with-docker` | 0.5 | Makes use of the embedded BuildKit Docker registry (instead of tar files) for `WITH DOCKER` loads and pulls | +| `--use-copy-include-patterns` | 0.6 | Speeds up COPY transfers | +| `--referenced-save-only` | 0.6 | Changes the behavior of SAVE commands in a significant way | +| `--for-in` | 0.6 | Enables support for `FOR ... IN ...` commands | +| `--require-force-for-unsafe-saves` | 0.6 | Requires `--force` for saving artifacts locally outside the Earthfile's directory | +| `--no-implicit-ignore` | 0.6 | Eliminates implicit `.earthlyignore` entries, such as `Earthfile` and `.tmp-earthly-out` | +| `--earthly-version-arg` | 0.7 | Enables builtin ARGs: `EARTHLY_VERSION` and `EARTHLY_BUILD_SHA` | +| `--shell-out-anywhere` | 0.7 | Allows shelling-out in any earthly command (including in the middle of `ARG`) | +| `--explicit-global` | 0.7 | Base target args must have a `--global` flag in order to be considered global args | +| `--check-duplicate-images` | 0.7 | Check for duplicate images during output | +| `--use-cache-command` | 0.7 | Allow use of `CACHE` command in Earthfiles | +| `--use-host-command` | 0.7 | Allow use of `HOST` command in Earthfiles | +| `--use-copy-link` | 0.7 | Use the equivalent of `COPY --link` for all copy-like operations | +| `--new-platform` | 0.7 | Enable new platform behavior | +| `--no-tar-build-output` | 0.7 | Do not print output when creating a tarball to load into `WITH DOCKER` | +| `--use-no-manifest-list` | 0.7 | Enable the `SAVE IMAGE --no-manifest-list` option | +| `--use-chmod` | 0.7 | Enable the `COPY --chmod` option | +| `--earthly-locally-arg` | 0.7 | Enable the `EARTHLY_LOCALLY` arg | +| `--use-project-secrets` | 0.7 | Enable project-based secret resolution | +| `--use-pipelines` | 0.7 | Enable the `PIPELINE` and `TRIGGER` commands | +| `--earthly-git-author-args` | 0.7 | Enable the `EARTHLY_GIT_AUTHOR` and `EARTHLY_GIT_CO_AUTHORS` args | +| `--wait-block` | 0.7 | Enable the `WAIT` / `END` block commands | +| `--no-network` | 0.8 | Allow the use of `RUN --network=none` commands | +| `--arg-scope-and-set` | 0.8 | Enable the `LET` / `SET` commands and nested `ARG` scoping | +| `--use-docker-ignore` | 0.8 | Enable the use of `.dockerignore` files in `FROM DOCKERFILE` targets | +| `--pass-args` | 0.8 | Enable the optional `--pass-args` flag for the `BUILD`, `FROM`, `COPY`, `WITH DOCKER --load` commands | +| `--global-cache` | 0.8 | Enable global caches (shared across different Earthfiles), for cache mounts and `CACHE` commands having an ID | | `--cache-persist-option` | 0.8 | Adds `CACHE --persist` option to persist cache content in images, Changes default `CACHE` behaviour to not persist | -| `--use-function-keyword` | 0.8 | Enable using `FUNCTION` instead of `COMMAND` when declaring a function | -| `--use-visited-upfront-hash-collection` | 0.8 | Switches to a newer target parallelization algorithm | -| `--no-use-registry-for-with-docker` | Experimental | Disable `use-registry-for-with-docker` | -| `--try` | Experimental | Enable the `TRY` / `FINALLY` / `END` block commands | -| `--earthly-ci-runner-arg` | Experimental | Enable the `EARTHLY_CI_RUNNER` builtin ARG | -| `--wildcard-builds` | Experimental | Alow for the expansion of wildcard (glob) paths for BUILD commands | -| `--build-auto-skip` | Experimental | Allow for `--auto-skip` to be used on individual BUILD commands | -| `--allow-privileged-from-dockerfile` | Experimental | Allow the use of the `--allow-privileged` flag in the `FROM DOCKERFILE` command | -| `--run-with-aws` | Experimental | Make AWS credentials in the environment or ~/.aws available to `RUN` commands | -| `--wildcard-copy` | Experimental | Alow for the expansion of wildcard (glob) paths for COPY commands | -| `--raw-output` | Experimental | Enable `--raw-output` for `RUN` output. | +| `--use-function-keyword` | 0.8 | Enable using `FUNCTION` instead of `COMMAND` when declaring a function | +| `--use-visited-upfront-hash-collection` | 0.8 | Switches to a newer target parallelization algorithm | +| `--no-use-registry-for-with-docker` | Experimental | Disable `use-registry-for-with-docker` | +| `--try` | Experimental | Enable the `TRY` / `FINALLY` / `END` block commands | +| `--earthly-ci-runner-arg` | Experimental | Enable the `EARTHLY_CI_RUNNER` builtin ARG | +| `--wildcard-builds` | Experimental | Alow for the expansion of wildcard (glob) paths for BUILD commands | +| `--build-auto-skip` | Experimental | Allow for `--auto-skip` to be used on individual BUILD commands | +| `--allow-privileged-from-dockerfile` | Experimental | Allow the use of the `--allow-privileged` flag in the `FROM DOCKERFILE` command | +| `--run-with-aws` | Experimental | Make AWS credentials in the environment or ~/.aws available to `RUN` commands | +| `--wildcard-copy` | Experimental | Alow for the expansion of wildcard (glob) paths for COPY commands | +| `--raw-output` | Experimental | Enable `--raw-output` for `RUN` output. | +| `--run-with-aws-oidc` | Experimental | Make AWS credentials via OIDC provider available to `RUN` commands | Note that the features flags are disabled by default in Earthly versions lower than the version listed in the "status" column above. diff --git a/docs/earthly-command/earthly-command.md b/docs/earthly-command/earthly-command.md index 106382c82f..e316b269ce 100644 --- a/docs/earthly-command/earthly-command.md +++ b/docs/earthly-command/earthly-command.md @@ -150,7 +150,7 @@ Overrides the earthly [configuration file](../earthly-config/earthly-config.md), Also available as an env var setting: `EARTHLY_INSTALLATION_NAME=`. -Overrides the Earthly installation name. The installation name is used for the Buildkit Daemon name, the cache volume name, the configuration directory (`~/.`) and for the ports used by Buildkit. Using multiple installation names on the same system allows Earthly to run as multiple isolated instances, each with its own configuration, cache and daemon. Defaults to `earthly`. +Overrides the Earthly installation name. The installation name is used for the BuildKit Daemon name, the cache volume name, the configuration directory (`~/.`) and for the ports used by BuildKit. Using multiple installation names on the same system allows Earthly to run as multiple isolated instances, each with its own configuration, cache and daemon. Defaults to `earthly`. ##### `--ssh-auth-sock ` diff --git a/docs/earthly-config/earthly-config.md b/docs/earthly-config/earthly-config.md index b885388d8a..e521fe3766 100644 --- a/docs/earthly-config/earthly-config.md +++ b/docs/earthly-config/earthly-config.md @@ -57,7 +57,7 @@ earthly config global.cache_size_mb 20000 ### cache_size_mb Specifies the total size of the BuildKit cache, in MB. The BuildKit daemon uses this setting to configure automatic garbage collection of old cache. -Setting this to 0, either explicitly or by omission, will cause buildkit to use its internal default of 10% of the root filesystem. +Setting this to 0, either explicitly or by omission, will cause BuildKit to use its internal default of 10% of the root filesystem. ### cache_size_pct @@ -126,7 +126,7 @@ The number of concurrent converters for speeding up build targets that use block ### buildkit_max_parallelism -The maximum parallelism configured for the buildkit daemon workers. The default is 20. +The maximum parallelism configured for the BuildKit daemon workers. The default is 20. {% hint style='info' %} ##### Note @@ -138,7 +138,7 @@ Set this configuration to a lower value if your machine is resource constrained ### buildkit_additional_args This option allows you to pass additional options to Docker when starting up the Earthly BuildKit daemon. -Note that changes to these values will trigger earthly to restart buildkit on the next run. +Note that changes to these values will trigger earthly to restart BuildKit on the next run. #### Bypass User Namespacing @@ -151,7 +151,7 @@ global: #### Session Timeout -By default, Buildkit will automatically cancel sessions (i.e. individual builds) after 24 hours. +By default, BuildKit will automatically cancel sessions (i.e. individual builds) after 24 hours. This value can be overriden using the following option: ```yaml @@ -165,12 +165,12 @@ This can be useful in cases where long-lived interactive sessions are used. ### buildkit_additional_config This option allows you to pass additional options to BuildKit. -Note that changes to these values will trigger earthly to restart buildkit on the next run. +Note that changes to these values will trigger earthly to restart BuildKit on the next run. #### Additional CA Certificates -Additional CA certificates can be passed in to buildkit. This also requires a corresponding change in `buildkit_additional_args`. +Additional CA certificates can be passed in to BuildKit. This also requires a corresponding change in `buildkit_additional_args`. ```yaml global: diff --git a/docs/examples/examples.md b/docs/examples/examples.md index f9a09a7ea0..7c1a14c03e 100644 --- a/docs/examples/examples.md +++ b/docs/examples/examples.md @@ -127,14 +127,14 @@ Here's a snip from an support request with gitbook: * [Earthfile](https://github.com/earthly/earthly/blob/main/Earthfile) - the root build file -* [buildkitd/Earthfile](https://github.com/earthly/earthly/blob/main/buildkitd/Earthfile) - the build of the Buildkit daemon +* [buildkitd/Earthfile](https://github.com/earthly/earthly/blob/main/buildkitd/Earthfile) - the build of the BuildKit daemon * [AST/parser/Earthfile](https://github.com/earthly/earthly/blob/main/ast/parser/Earthfile) - the build of the parser, which generates .go files * [tests/Earthfile](https://github.com/earthly/earthly/blob/main/tests/Earthfile) - system and smoke tests * [earthfile-grammar/Earthfile](https://github.com/earthly/earthfile-grammar/blob/main/Earthfile) - the build of the VS Code extension --> * [Earthfile](https://tinyurl.com/yt3d3cx6) - the root build file -* [buildkitd/Earthfile](https://tinyurl.com/yvnpuru7) - the build of the Buildkit daemon +* [buildkitd/Earthfile](https://tinyurl.com/yvnpuru7) - the build of the BuildKit daemon * [AST/parser/Earthfile](https://tinyurl.com/2k3u4vty) - the build of the parser, which generates .go files * [tests/Earthfile](https://tinyurl.com/2p8ws579) - system and smoke tests * [earthfile-grammar/Earthfile](https://tinyurl.com/2vyjprt6) - the build of the VS Code extension diff --git a/docs/guides/auth.md b/docs/guides/auth.md index 06821cf076..36bb8cdea4 100644 --- a/docs/guides/auth.md +++ b/docs/guides/auth.md @@ -155,14 +155,14 @@ Note that patterns are evaluated from the top to the bottom, subgroup specific c You can run earthly with `--verbose`, which will provide debugging messages to help understand how a remote earthly reference is transformed into a git URL for cloning. -You can additionally enable low-level git debugging in buildkit, by adding the following to your `~/.earthly/config.yml`: +You can additionally enable low-level git debugging in BuildKit, by adding the following to your `~/.earthly/config.yml`: ```yaml global: buildkit_additional_args: [ '-e', 'BUILDKIT_DEBUG_GIT=1' ] ``` -The buildkit logs can be displayed with `docker logs earthly-buildkitd`. +The BuildKit logs can be displayed with `docker logs earthly-buildkitd`. ## Docker authentication @@ -189,6 +189,12 @@ You can see examples of configuring Docker to use these, and working with Earthl * [Pushing and Pulling Images with GCP Artifact Registry](./registries/gcp-artifact-registry.md) * [Pushing and Pulling Images with Azure ACR](./registries/azure-acr.md) +## Cloud Providers + +Currently Earthly provides a built-in way to easily authenticate to AWS during a build. + +* [Accessing AWS resources](./cloud-providers/aws.md) + ## See also * The [earthly command reference](../earthly-command/earthly-command.md) diff --git a/docs/guides/best-practices.md b/docs/guides/best-practices.md index 31736a3b86..a7b3ca688b 100644 --- a/docs/guides/best-practices.md +++ b/docs/guides/best-practices.md @@ -304,7 +304,7 @@ In certain cases, it may be desirable to execute certain targets on the host mac Suppose we wanted the following target to be executed on against the host's Docker daemon: ```Dockerfile -FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 +FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 WORKDIR /app COPY docker-compose.yml ./ WITH DOCKER --compose docker-compose.yml \ @@ -335,7 +335,7 @@ ARG run_locally=false IF [ "$run_locally" = "true" ] LOCALLY ELSE - FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 WORKDIR /app COPY docker-compose.yml ./ END @@ -553,7 +553,7 @@ The `--load` instruction will inform Earthly that the two targets depend on each When referencing an external image in the body of a `WITH DOCKER` block, it is important to declare it via `WITH DOCKER --pull`, for a few reasons: -* The image will be cached as part of buildkit, allowing for faster builds. This is especially important as `WITH DOCKER` wipes the state of the Docker daemon (including its cache) after every run. +* The image will be cached as part of BuildKit, allowing for faster builds. This is especially important as `WITH DOCKER` wipes the state of the Docker daemon (including its cache) after every run. * The Daemon within `WITH DOCKER` is not logged into registries. Your local Docker login config is not propagated to the daemon. This means that you may run into issues when trying to pull images from private registries, but also, DockerHub rate limiting may prevent you from pulling images consistently from public repositories. ```Dockerfile @@ -926,7 +926,7 @@ https://github.com/earthly/earthly/blob/main/Earthfile was changed to https://ti * [`ast/parser`](https://github.com/earthly/earthly/tree/main/ast/parser) - Earthfile contains the logic for generating Go source code based on an ANTLR grammar. * [`ast/parser/tests`](https://github.com/earthly/earthly/tree/main/ast/tests) - Earthfile contains logic for running AST-specific tests. -* [`buildkitd`](https://github.com/earthly/earthly/tree/main/buildkitd) - Earthfile contains the logic for building the Earthly buildkit image. +* [`buildkitd`](https://github.com/earthly/earthly/tree/main/buildkitd) - Earthfile contains the logic for building the Earthly BuildKit image. * [`tests`](https://github.com/earthly/earthly/tree/main/tests) - Earthfile contains logic for executing e2e tests. * [`release/**/`](https://github.com/earthly/earthly/tree/main/release) - Multiple Earthfiles contain logic used for the release of Earthly. * [The main Earthfile](https://tinyurl.com/yt3d3cx6) - ties everything together, referencing the various targets across the sub-directories. @@ -1062,7 +1062,7 @@ The best supported option, however, is to use the `earthly/dind` image, if possi ```Dockerfile # Best - if possible integration-test: - FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 COPY docker-compose.yml ./ WITH DOCKER --compose docker-compose.yml RUN ... diff --git a/docs/guides/cloud-providers/aws.md b/docs/guides/cloud-providers/aws.md new file mode 100644 index 0000000000..37d0a4721c --- /dev/null +++ b/docs/guides/cloud-providers/aws.md @@ -0,0 +1,33 @@ +# Accessing AWS resources + +## Introduction + +It is common for builds to be able to access AWS resources (For example, one might want to upload artifacts to S3). +Earthly provides two ways to easily authenticate to AWS in order to access resources. + +## Authentication Methods + +### Local Environment Credentials + +Earthly is able to access AWS credentials from the host. +The credentials might be available via environment variables or your `~/.aws` directory. + +To use these credentials simply use `RUN --aws in your command`. + +For example: +```dockerfile +VERSION --run-with-aws 0.8 + +aws: + FROM amazon/aws-cli + RUN --aws aws s3 ls +``` + +For more information, see [here](../../earthfile/earthfile.md#--aws-experimental). + +### OIDC (OpenID Connect) + +OIDC in useful in cases where credentials are not available in the host (e.g. CI system) +and/or when authentication requires MFA (multi-factor authentication). + +For more information on how to set up & authenticate to AWS via OIDC, see [here](../../cloud/oidc.md). diff --git a/docs/guides/docker-in-earthly.md b/docs/guides/docker-in-earthly.md index c405944211..06f12c4315 100644 --- a/docs/guides/docker-in-earthly.md +++ b/docs/guides/docker-in-earthly.md @@ -11,7 +11,7 @@ Here is a quick example of running a `hello-world` docker container via `docker ```Dockerfile hello: - FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 WITH DOCKER --pull hello-world RUN docker run hello-world END @@ -19,7 +19,7 @@ hello: Let's break it down. -`FROM earthly/dind:alpine-3.19-docker-25.0.2-r0` inherits from an Earthly-supported docker-in-docker (dind) image. This is recommended, because `WITH DOCKER` requires all the Docker binaries (not just the client) to be present in the build environment. +`FROM earthly/dind:alpine-3.19-docker-25.0.5-r0` inherits from an Earthly-supported docker-in-docker (dind) image. This is recommended, because `WITH DOCKER` requires all the Docker binaries (not just the client) to be present in the build environment. `WITH DOCKER ... END` starts a Docker daemon for the purpose of running Docker commands against it. At the end of the execution, this also terminates the daemon and permanently deletes all of its data (e.g. daemon cached images). @@ -38,7 +38,7 @@ build: SAVE IMAGE my-image:latest smoke-test: - FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 WITH DOCKER --load test:latest=+build RUN docker run test:latest FROM earthly/dind:alpine @@ -54,7 +54,7 @@ smoke-test: It is possible to run `docker-compose` via `WITH DOCKER`, either explicitly, simply by running the `docker-compose` tool, or implicitly, via the `--compose` flag. The `--compose` flag allows you to specify a Docker compose stack that needs to be brought up before the execution of the `RUN` command. For example: ```Dockerfile -FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 +FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 COPY docker-compose.yml ./ WITH DOCKER \ --compose docker-compose.yml \ @@ -68,7 +68,7 @@ Using the `--compose` flag has the added benefit that any images needed by the c ## Performance -It's recommended to use the `earthly/dind:alpine-3.19-docker-25.0.2-r0` image for running docker-in-docker. See the best-practices' section on using [with docker](../guides/best-practices.md#use-earthly-dind) for more details. +It's recommended to use the `earthly/dind:alpine-3.19-docker-25.0.5-r0` image for running docker-in-docker. See the best-practices' section on using [with docker](../guides/best-practices.md#use-earthly-dind) for more details. In cases when using `earthly/dind` is not possible, Earthly will attempt to install Docker in the image you have chosen. This has the drawback of not being able to use cache efficiently and is not recommended for performance reasons. @@ -102,7 +102,7 @@ The current implementation of Docker in Earthly has a number of limitations: ... END ``` -* It is recommended that the target containing the `WITH DOCKER` clause inherits from a supported Docker-in-Docker (dind) image such as `earthly/dind:alpine-3.19-docker-25.0.2-r0` or `earthly/dind:ubuntu-23.04-docker-24.0.5-1`. If your build requires the use of an alternative environment as part of a test (e.g. to run commands like `sbt test` or `go test` together with a docker-compose stack), consider placing the test itself in a Docker image, then loading that image via `--load` and running the test as a Docker container. +* It is recommended that the target containing the `WITH DOCKER` clause inherits from a supported Docker-in-Docker (dind) image such as `earthly/dind:alpine-3.19-docker-25.0.5-r0` or `earthly/dind:ubuntu-23.04-docker-25.0.2-1`. If your build requires the use of an alternative environment as part of a test (e.g. to run commands like `sbt test` or `go test` together with a docker-compose stack), consider placing the test itself in a Docker image, then loading that image via `--load` and running the test as a Docker container. * If you do not use an officially supported Docker-in-Docker image, Earthly will attempt to install Docker in whatever image you have chosen. This has the drawback of not being able to use cache efficiently and is not recommended for performance reasons. * To maximize the use of cache, all external images used should be declared via the options `--pull` or `--compose`. Even though commands such as `docker run` automatically pull an image if it is not found locally, it will do so every single time the `WITH DOCKER` clause is executed, due to Docker caching not being preserved between runs. Pre-declaring the images ensures that they are properly cached by Earthly to minimize unnecessary redownloads. * `docker build` cannot be used to build Dockerfiles. However, the Earthly command `FROM DOCKERFILE` can be used instead. See [alternative to docker build](#alternative-to-docker-build) below. diff --git a/docs/guides/importing.md b/docs/guides/importing.md index 76579777eb..2bfae6ebf6 100644 --- a/docs/guides/importing.md +++ b/docs/guides/importing.md @@ -96,7 +96,7 @@ Here are some examples: * `+build` * `./js+deps` -* `github.com/earthly/earthly:v0.8.7+earthly` +* `github.com/earthly/earthly:v0.8.13+earthly` * `my-import+build` ## Artifact reference @@ -110,7 +110,7 @@ Here are some examples: * `+build/my-artifact` * `+build/some/artifact/deep/in/a/dir` * `./js+build/dist` -* `github.com/earthly/earthly:v0.8.7+earthly/earthly` +* `github.com/earthly/earthly:v0.8.13+earthly/earthly` * `my-import+build/my-artifact` ## Image reference @@ -131,7 +131,7 @@ Here are some examples: * `+COMPILE` * `./js+NPM_INSTALL` -* `github.com/earthly/earthly:v0.8.7+DOWNLOAD_DIND` +* `github.com/earthly/earthly:v0.8.13+DOWNLOAD_DIND` * `my-import+COMPILE` For more information on functions, see the [Functions Guide](./functions.md). @@ -176,7 +176,7 @@ Another form of a Earthfile reference is the remote form. In this form, the reci |----|----|----|----| | `///path/in/project[:some-tag]` | `///path/in/project[:some-tag]+` | `///path/in/project[:some-tag]+/` | `///path/in/project[:some-tag]+` | | `github.com/earthly/earthly/buildkitd` | `github.com/earthly/earthly/buildkitd+build` | `github.com/earthly/earthly/buildkitd+build/out.bin` | `github.com/earthly/earthly/buildkitd+COMPILE` | -| `github.com/earthly/earthly:v0.8.7` | `github.com/earthly/earthly:v0.8.7+build` | `github.com/earthly/earthly:v0.8.7+build/out.bin` | `github.com/earthly/earthly:v0.8.7+COMPILE` | +| `github.com/earthly/earthly:v0.8.13` | `github.com/earthly/earthly:v0.8.13+build` | `github.com/earthly/earthly:v0.8.13+build/out.bin` | `github.com/earthly/earthly:v0.8.13+COMPILE` | ### Import reference @@ -186,7 +186,7 @@ Finally, the last form of Earthfile referencing is an import reference. Import r |----|----|----|----|----| | `IMPORT AS ` | `` | `+` | `+/` | `+` | | `IMPORT github.com/earthly/earthly/buildkitd` | `buildkitd` | `buildkitd+build` | `buildkitd+build/out.bin` | `buildkitd+COMPILE` | -| `IMPORT github.com/earthly/earthly:v0.8.7` | `earthly` | `earthly+build` | `earthly+build/out.bin` | `earthly+COMPILE` | +| `IMPORT github.com/earthly/earthly:v0.8.13` | `earthly` | `earthly+build` | `earthly+build/out.bin` | `earthly+COMPILE` | Here is an example in an Earthfile: diff --git a/docs/guides/integration.md b/docs/guides/integration.md index e20fe8a950..e2dcc2af86 100644 --- a/docs/guides/integration.md +++ b/docs/guides/integration.md @@ -133,7 +133,7 @@ We start with a simple Earthfile that can build and create a docker image for ou We start from an appropriate docker image and set up a working directory. ``` Dockerfile VERSION 0.8 -FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 +FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 WORKDIR /scala-example RUN apk add openjdk11 bash wget postgresql-client ``` @@ -331,7 +331,7 @@ all: =========================== SUCCESS =========================== ``` -There we have it, a reproducible integration process. If you have questions about the example, [ask](https://gitter.im/earthly-room/community) +There we have it, a reproducible integration process. If you have questions about the example, [ask](https://earthly.dev/slack). ## See also * [Docker In Earthly](./docker-in-earthly.md) diff --git a/docs/guides/podman.md b/docs/guides/podman.md index bc27b7ac08..0286e2ac5b 100644 --- a/docs/guides/podman.md +++ b/docs/guides/podman.md @@ -99,7 +99,7 @@ graphDriverName: overlay # or something similar ### Mac: docker-credential-desktop: executable file not found in $PATH This error typically occurs when switching from docker desktop to podman without docker installed. -There may be a lingering configuration file that will be read by the attachable used to authenticate calls to buildkit. +There may be a lingering configuration file that will be read by the attachable used to authenticate calls to BuildKit. To fix this issue, try removing or renaming the `~/.docker/config.json` file. diff --git a/docs/guides/registries/aws-ecr.md b/docs/guides/registries/aws-ecr.md index 18331603bb..ace8458eb1 100644 --- a/docs/guides/registries/aws-ecr.md +++ b/docs/guides/registries/aws-ecr.md @@ -96,7 +96,7 @@ Loaded image: .dkr.ecr..amazonaws.com/hello-earthly:with Using this credential helper; you can also pull images without any special handling in an Earthfile: ``` -FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 +FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 run: WITH DOCKER --pull .dkr.ecr..amazonaws.com/hello-earthly:with-love @@ -109,12 +109,12 @@ And here is how you would run it: ``` ❯ earthly -P +run buildkitd | Found buildkit daemon as docker container (earthly-buildkitd) - earthly/dind:alpine-3.19-docker-25.0.2-r0 | --> Load metadata linux/amd64 + earthly/dind:alpine-3.19-docker-25.0.5-r0 | --> Load metadata linux/amd64 4/hello-earthly:with-love | --> Load metadata linux/amd64 4/hello-earthly:with-love | --> DOCKER PULL .dkr.ecr..amazonaws.com/hello-earthly:with-love 4/hello-earthly:with-love | [██████████] resolve .dkr.ecr..amazonaws.com/hello-earthly:with-love@sha256:9ab4df74dafa2a71d71e39e1af133d110186698c78554ab000159cfa92081de4 ... 100% - +base | --> FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 - +base | [██████████] resolve docker.io/earthly/dind:alpine-3.19-docker-25.0.2-r0@sha256:2cef4089960efe028de40721749e3ec6eba9f471562bf10681de729287bd78fb ... 100% + +base | --> FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 + +base | [██████████] resolve docker.io/earthly/dind:alpine-3.19-docker-25.0.5-r0@sha256:2cef4089960efe028de40721749e3ec6eba9f471562bf10681de729287bd78fb ... 100% +run | *cached* --> WITH DOCKER (install deps) +run | *cached* --> WITH DOCKER RUN docker run .dkr.ecr..amazonaws.com/hello-earthly:with-love output | --> exporting outputs diff --git a/docs/guides/registries/self-signed.md b/docs/guides/registries/self-signed.md index c12068d2c8..ee81ef0889 100644 --- a/docs/guides/registries/self-signed.md +++ b/docs/guides/registries/self-signed.md @@ -59,7 +59,7 @@ build: ##### Note The `http` and `insecure` settings are typically mutually exclusive. Setting `insecure=true` should only be used when the registry is https and is configured with an insecure certificate. -Setting `http=true` is only for the case where a standard http-based registry is used (i.e. no SSL encryption). If both are set buildkit will attempt to connect to the registry using either http (port 80), or https (port 443). +Setting `http=true` is only for the case where a standard http-based registry is used (i.e. no SSL encryption). If both are set BuildKit will attempt to connect to the registry using either http (port 80), or https (port 443). {% endhint %} diff --git a/docs/guides/target-ref.md b/docs/guides/target-ref.md index 172acc06fb..bc383f68b0 100644 --- a/docs/guides/target-ref.md +++ b/docs/guides/target-ref.md @@ -20,7 +20,7 @@ Here are some examples: * `+build` * `./js+deps` -* `github.com/earthly/earthly:v0.8.6+earthly` +* `github.com/earthly/earthly:v0.8.13+earthly` ## Artifact reference @@ -33,7 +33,7 @@ Here are some examples: * `+build/my-artifact` * `+build/some/artifact/deep/in/a/dir` * `./js+build/dist` -* `github.com/earthly/earthly:v0.8.6+earthly/earthly` +* `github.com/earthly/earthly:v0.8.13+earthly/earthly` ## Image reference @@ -53,7 +53,7 @@ Here are some examples: * `+COMPILE` * `./js+NPM_INSTALL` -* `github.com/earthly/earthly:v0.8.6+DOWNLOAD_DIND` +* `github.com/earthly/earthly:v0.8.13+DOWNLOAD_DIND` For more information on functions, see the [functions guide](./functions.md). @@ -93,7 +93,7 @@ Another form of a project reference is the remote form. In this form, the recipe |----|----|----|----| | `///path/in/project[:some-tag]` | `///path/in/project[:some-tag]+` | `///path/in/project[:some-tag]+/` | `///path/in/project[:some-tag]+` | | `github.com/earthly/earthly/buildkitd` | `github.com/earthly/earthly/buildkitd+build` | `github.com/earthly/earthly/buildkitd+build/out.bin` | `github.com/earthly/earthly/buildkitd+COMPILE` | -| `github.com/earthly/earthly:v0.8.6` | `github.com/earthly/earthly:v0.8.6+build` | `github.com/earthly/earthly:v0.8.6+build/out.bin` | `github.com/earthly/earthly:v0.8.6+COMPILE` | +| `github.com/earthly/earthly:v0.8.13` | `github.com/earthly/earthly:v0.8.13+build` | `github.com/earthly/earthly:v0.8.13+build/out.bin` | `github.com/earthly/earthly:v0.8.13+COMPILE` | ### Import reference @@ -103,7 +103,7 @@ Finally, the last form of project referencing is an import reference. Import ref |----|----|----|----|----| | `IMPORT AS ` | `` | `+` | `+/` | `+` | | `IMPORT github.com/earthly/earthly/buildkitd` | `buildkitd` | `buildkitd+build` | `buildkitd+build/out.bin` | `buildkitd+COMPILE` | -| `IMPORT github.com/earthly/earthly:v0.8.6` | `earthly` | `earthly+build` | `earthly+build/out.bin` | `earthly+COMPILE` | +| `IMPORT github.com/earthly/earthly:v0.8.13` | `earthly` | `earthly+build` | `earthly+build/out.bin` | `earthly+COMPILE` | Here is an example in an Earthfile: diff --git a/docs/remote-runners.md b/docs/remote-runners.md index cfbfa25808..817dadd283 100644 --- a/docs/remote-runners.md +++ b/docs/remote-runners.md @@ -28,17 +28,19 @@ To get started with free remote runners managed by Earthly, check out [Earthly S To get started with self-hosted runners, see the [Self-Hosted Satellites Guide](cloud/satellites/self-hosted.md). -If your use case cannot tolerate a cloud-based control plane, then self-hosting a remote Buildkit is the best approach. Remote Buildkit has less features, is less secure, and is more difficult to deploy than Self-Hosted Satellites (see diagram below for comparison). To get started self-hosting Buildkit, see the [remote buildkit page](ci-integration/remote-buildkit.md). +If your use case cannot tolerate a cloud-based control plane, then self-hosting a remote BuildKit is the best approach. Remote BuildKit has less features, is less secure, and is more difficult to deploy than Self-Hosted Satellites (see diagram below for comparison). To get started self-hosting BuildKit, see the [remote BuildKit page](ci-integration/remote-buildkit.md). ### Types of Remote Runners Below is a comparison of the different features available with each kind of remote runner. -| Feature | Cloud Satellites | Self-Hosted Satellites | Remote Buildkit | -| --- | --- | --- | --- | -| Managed By | Earthly | You | You | -| Cache Persistence | ✅ Yes | 🟡 Needs configuration | 🟡 Needs configuration | -| Cloud Control-Plane | ✅ Yes | ✅ Yes | ❌ No | -| Managed TLS Certificates | ✅ Yes | ✅ Yes | ❌ No | -| Auto-Sleep | ✅ Yes | ❌ No | ❌ No | -| Auto-Updates | ✅ Yes | ❌ No | ❌ No | +| Feature | Cloud Satellites | Self-Hosted Satellites | Remote BuildKit | +|-----------------------------------------------------------------------------|------------------|------------------------|------------------------| +| Managed By | Earthly | You | You | +| Cache Persistence | ✅ Yes | 🟡 Needs configuration | 🟡 Needs configuration | +| Cloud Control-Plane | ✅ Yes | ✅ Yes | ❌ No | +| Managed TLS Certificates | ✅ Yes | ✅ Yes | ❌ No | +| Auto-Sleep | ✅ Yes | ❌ No | ❌ No | +| Auto-Updates | ✅ Yes | ❌ No | ❌ No | +| [GitHub Actions integration](cloud/satellites/gha-runners.md) | ✅ Yes | ✅ Yes | ❌ No | + diff --git a/earthfile2llb/converter.go b/earthfile2llb/converter.go index 90bf152a78..0d9f0c95e1 100644 --- a/earthfile2llb/converter.go +++ b/earthfile2llb/converter.go @@ -43,6 +43,7 @@ import ( "github.com/earthly/earthly/util/llbutil/llbfactory" "github.com/earthly/earthly/util/llbutil/pllb" "github.com/earthly/earthly/util/llbutil/secretprovider" + "github.com/earthly/earthly/util/oidcutil" "github.com/earthly/earthly/util/platutil" "github.com/earthly/earthly/util/shell" "github.com/earthly/earthly/util/stringutil" @@ -50,6 +51,7 @@ import ( "github.com/earthly/earthly/util/vertexmeta" "github.com/earthly/earthly/variables" "github.com/earthly/earthly/variables/reserved" + "github.com/google/uuid" "github.com/moby/buildkit/client/llb" dockerimage "github.com/moby/buildkit/exporter/containerimage/image" "github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb" @@ -646,6 +648,7 @@ type ConvertRunOpts struct { InteractiveKeep bool InteractiveSaveFiles []debuggercommon.SaveFilesSettings WithAWSCredentials bool + OIDCInfo *oidcutil.AWSOIDCInfo RawOutput bool // Internal. @@ -2218,7 +2221,7 @@ func (c *Converter) internalRun(ctx context.Context, opts ConvertRunOpts) (pllb. } // AWS credential import. if opts.WithAWSCredentials { - awsRunOpts, awsEnvs, err := c.awsSecrets(ctx) + awsRunOpts, awsEnvs, err := c.awsSecrets(ctx, opts.OIDCInfo) if err != nil { return pllb.State{}, err } @@ -2304,6 +2307,10 @@ func (c *Converter) internalRun(ctx context.Context, opts ConvertRunOpts) (pllb. // Shell and debugger wrap. prependDebugger := !opts.Locally finalArgs = opts.shellWrap(finalArgs, extraEnvVars, opts.WithShell, prependDebugger, isInteractive) + if opts.NoCache { + // llb.IgnoreCache is not always enough; we will force a different cache key as a work-around + finalArgs = append(finalArgs, "#"+uuid.NewString()) + } if opts.Locally { // buildkit-hack in order to run locally, we prepend the command with a magic UUID. finalArgs = append( @@ -2405,19 +2412,24 @@ func (c *Converter) internalRun(ctx context.Context, opts ConvertRunOpts) (pllb. } } -func (c *Converter) awsSecrets(ctx context.Context) ([]llb.RunOption, []string, error) { +func (c *Converter) awsSecrets(ctx context.Context, oidcInfo *oidcutil.AWSOIDCInfo) ([]llb.RunOption, []string, error) { var ( runOpts = []llb.RunOption{} extraEnvs = []string{} ) + //set additional params in case oidc is in play + var setOIDCInfo func(values url.Values) // no-op by default + if oidcInfo != nil { + setOIDCInfo = secretprovider.SetURLValuesFunc(oidcInfo) + } // Add LLB secrets for each of the typical secrets which will then be // sourced from the environment during lookup. for _, secretName := range secretprovider.AWSCredentials { secretPath := path.Join("/run/secrets", secretName) secretOpts := []llb.SecretOption{ - llb.SecretID(c.secretID(secretName)), + llb.SecretID(c.secretID(secretName, setOIDCInfo)), llb.SecretFileOpt(0, 0, 0444), } runOpts = append(runOpts, llb.AddSecret(secretPath, secretOpts...)) @@ -2433,7 +2445,7 @@ func (c *Converter) awsSecrets(ctx context.Context) ([]llb.RunOption, []string, // version, name, org, and project. The version value informs the secret // providers how to handle the secret name and whether to use the new // project-based secrets endpoints. -func (c *Converter) secretID(name string) string { +func (c *Converter) secretID(name string, opts ...func(values url.Values)) string { v := url.Values{} v.Set("name", name) if c.ftrs.UseProjectSecrets { @@ -2443,6 +2455,11 @@ func (c *Converter) secretID(name string) string { } else { v.Set("v", "0") } + for _, opt := range opts { + if opt != nil { + opt(v) + } + } return v.Encode() } diff --git a/earthfile2llb/interpreter.go b/earthfile2llb/interpreter.go index 78a57ab7df..60d6bb1ade 100644 --- a/earthfile2llb/interpreter.go +++ b/earthfile2llb/interpreter.go @@ -22,11 +22,13 @@ import ( "github.com/earthly/earthly/domain" "github.com/earthly/earthly/internal/version" "github.com/earthly/earthly/util/flagutil" + "github.com/earthly/earthly/util/oidcutil" "github.com/earthly/earthly/util/platutil" "github.com/earthly/earthly/util/shell" "github.com/earthly/earthly/variables" "github.com/docker/go-connections/nat" + "github.com/google/uuid" "github.com/jessevdk/go-flags" "github.com/pkg/errors" ) @@ -35,6 +37,9 @@ const maxCommandRenameWarnings = 3 var errCannotAsync = errors.New("cannot run async operation") +// use as default to differentiate between an un specified string flag and a specified flag with empty value +var defaultZeroStringFlag = uuid.NewString() + // Interpreter interprets Earthly AST's into calls to the converter. type Interpreter struct { converter *Converter @@ -682,7 +687,7 @@ func (i *Interpreter) handleRun(ctx context.Context, cmd spec.Command) error { if len(cmd.Args) < 1 { return i.errorf(cmd.SourceLocation, "not enough arguments for RUN") } - opts := commandflag.RunOpts{} + opts := commandflag.RunOpts{OIDC: defaultZeroStringFlag} args, err := flagutil.ParseArgsWithValueModifierCleaned("RUN", &opts, flagutil.GetArgsCopy(cmd), i.flagValModifierFuncWithContext(ctx)) if err != nil { return i.wrapError(err, cmd.SourceLocation, "invalid RUN arguments %v", cmd.Args) @@ -731,6 +736,11 @@ func (i *Interpreter) handleRun(ctx context.Context, cmd spec.Command) error { return i.errorf(cmd.SourceLocation, "RUN --aws requires the --run-with-aws feature flag") } + awsOIDCInfo, err := i.handleOIDC(ctx, &cmd, &opts) + if err != nil { + return err + } + if opts.RawOutput && !i.converter.opt.Features.RawOutput { return i.errorf(cmd.SourceLocation, "RUN --raw-output requires the --raw-output feature flag") } @@ -756,6 +766,7 @@ func (i *Interpreter) handleRun(ctx context.Context, cmd spec.Command) error { InteractiveKeep: opts.InteractiveKeep, InteractiveSaveFiles: i.interactiveSaveFiles, WithAWSCredentials: opts.WithAWS, + OIDCInfo: awsOIDCInfo, RawOutput: opts.RawOutput, } err = i.converter.Run(ctx, opts) @@ -807,6 +818,34 @@ func (i *Interpreter) handleRun(ctx context.Context, cmd spec.Command) error { return nil } +// handleOIDC parse the oidc string value into a struct +// Returns error if the value cannot be parsed of if the feature flag is not set +func (i *Interpreter) handleOIDC(ctx context.Context, cmd *spec.Command, opts *commandflag.RunOpts) (*oidcutil.AWSOIDCInfo, error) { + if opts.OIDC == defaultZeroStringFlag { + // oidc is not in use, set it to empty string just in case + opts.OIDC = "" + return nil, nil + } + if !i.converter.opt.Features.RunWithAWSOIDC { + return nil, i.errorf(cmd.SourceLocation, "RUN --aws-oidc requires the --run-with-aws-oidc feature flag") + } + if !opts.WithAWS { + return nil, i.errorf(cmd.SourceLocation, "RUN --oidc also requires the --aws RUN flag") + } + expanded, err := i.expandArgs(ctx, opts.OIDC, false, false) + if err != nil { + return nil, i.errorf(cmd.SourceLocation, "failed to expand oidc arg in RUN: %s", opts.OIDC) + } + opts.OIDC = expanded + // we currently only support oidc for AWS + awsInfo, err := oidcutil.ParseAWSOIDCInfo(opts.OIDC) + if err != nil { + return nil, i.errorf(cmd.SourceLocation, "invalid value for oidc flag: %v", err) + } + + return awsInfo, nil +} + func (i *Interpreter) handleFromDockerfile(ctx context.Context, cmd spec.Command) error { if i.pushOnlyAllowed { return i.pushOnlyErr(cmd.SourceLocation) diff --git a/earthly-next b/earthly-next new file mode 100644 index 0000000000..f7a6e86492 --- /dev/null +++ b/earthly-next @@ -0,0 +1 @@ +88ecf5d6f17aa02643b80697f5251a2b2c1538e9 diff --git a/examples/grpc/Earthfile b/examples/grpc/Earthfile index 542038f7c6..47fdf9b86e 100644 --- a/examples/grpc/Earthfile +++ b/examples/grpc/Earthfile @@ -35,7 +35,7 @@ WORKDIR /example-grpc # the returned value is "salmon". test: - FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 WITH DOCKER \ --load kvserver:latest=github.com/earthly/earthly-example-proto-server:main+kvserver-docker \ --load kv-py-client:latest=github.com/earthly/earthly-example-proto-python-client:main+kvclient-docker \ diff --git a/examples/integration-test/Earthfile b/examples/integration-test/Earthfile index 2c04b4e246..1631369e53 100644 --- a/examples/integration-test/Earthfile +++ b/examples/integration-test/Earthfile @@ -1,5 +1,5 @@ VERSION 0.8 -FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 +FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 WORKDIR /scala-example RUN apk add openjdk11 bash wget postgresql-client diff --git a/examples/mkdocs/Pipfile.lock b/examples/mkdocs/Pipfile.lock index 57154f4e2e..340025f71b 100644 --- a/examples/mkdocs/Pipfile.lock +++ b/examples/mkdocs/Pipfile.lock @@ -18,107 +18,107 @@ "default": { "certifi": { "hashes": [ - "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082", - "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9" + "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f", + "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1" ], "markers": "python_version >= '3.6'", - "version": "==2023.7.22" + "version": "==2024.2.2" }, "charset-normalizer": { "hashes": [ - "sha256:02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843", - "sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786", - "sha256:03680bb39035fbcffe828eae9c3f8afc0428c91d38e7d61aa992ef7a59fb120e", - "sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8", - "sha256:09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4", - "sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa", - "sha256:1063da2c85b95f2d1a430f1c33b55c9c17ffaf5e612e10aeaad641c55a9e2b9d", - "sha256:12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82", - "sha256:153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7", - "sha256:15b26ddf78d57f1d143bdf32e820fd8935d36abe8a25eb9ec0b5a71c82eb3895", - "sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d", - "sha256:187d18082694a29005ba2944c882344b6748d5be69e3a89bf3cc9d878e548d5a", - "sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382", - "sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678", - "sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b", - "sha256:24817cb02cbef7cd499f7c9a2735286b4782bd47a5b3516a0e84c50eab44b98e", - "sha256:249c6470a2b60935bafd1d1d13cd613f8cd8388d53461c67397ee6a0f5dce741", - "sha256:24a91a981f185721542a0b7c92e9054b7ab4fea0508a795846bc5b0abf8118d4", - "sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596", - "sha256:250c9eb0f4600361dd80d46112213dff2286231d92d3e52af1e5a6083d10cad9", - "sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69", - "sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c", - "sha256:2f4a0033ce9a76e391542c182f0d48d084855b5fcba5010f707c8e8c34663d77", - "sha256:30a85aed0b864ac88309b7d94be09f6046c834ef60762a8833b660139cfbad13", - "sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459", - "sha256:3ae38d325b512f63f8da31f826e6cb6c367336f95e418137286ba362925c877e", - "sha256:3b447982ad46348c02cb90d230b75ac34e9886273df3a93eec0539308a6296d7", - "sha256:3debd1150027933210c2fc321527c2299118aa929c2f5a0a80ab6953e3bd1908", - "sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a", - "sha256:468d2a840567b13a590e67dd276c570f8de00ed767ecc611994c301d0f8c014f", - "sha256:4cc152c5dd831641e995764f9f0b6589519f6f5123258ccaca8c6d34572fefa8", - "sha256:542da1178c1c6af8873e143910e2269add130a299c9106eef2594e15dae5e482", - "sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d", - "sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d", - "sha256:588245972aca710b5b68802c8cad9edaa98589b1b42ad2b53accd6910dad3545", - "sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34", - "sha256:619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86", - "sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6", - "sha256:67b8cc9574bb518ec76dc8e705d4c39ae78bb96237cb533edac149352c1f39fe", - "sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e", - "sha256:70f1d09c0d7748b73290b29219e854b3207aea922f839437870d8cc2168e31cc", - "sha256:750b446b2ffce1739e8578576092179160f6d26bd5e23eb1789c4d64d5af7dc7", - "sha256:7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd", - "sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c", - "sha256:7f5d10bae5d78e4551b7be7a9b29643a95aded9d0f602aa2ba584f0388e7a557", - "sha256:805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a", - "sha256:81bf654678e575403736b85ba3a7867e31c2c30a69bc57fe88e3ace52fb17b89", - "sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078", - "sha256:85a32721ddde63c9df9ebb0d2045b9691d9750cb139c161c80e500d210f5e26e", - "sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4", - "sha256:86f63face3a527284f7bb8a9d4f78988e3c06823f7bea2bd6f0e0e9298ca0403", - "sha256:8eaf82f0eccd1505cf39a45a6bd0a8cf1c70dcfc30dba338207a969d91b965c0", - "sha256:93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89", - "sha256:96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115", - "sha256:9cf3126b85822c4e53aa28c7ec9869b924d6fcfb76e77a45c44b83d91afd74f9", - "sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05", - "sha256:a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a", - "sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec", - "sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56", - "sha256:aae32c93e0f64469f74ccc730a7cb21c7610af3a775157e50bbd38f816536b38", - "sha256:aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479", - "sha256:abecce40dfebbfa6abf8e324e1860092eeca6f7375c8c4e655a8afb61af58f2c", - "sha256:abf0d9f45ea5fb95051c8bfe43cb40cda383772f7e5023a83cc481ca2604d74e", - "sha256:ac71b2977fb90c35d41c9453116e283fac47bb9096ad917b8819ca8b943abecd", - "sha256:ada214c6fa40f8d800e575de6b91a40d0548139e5dc457d2ebb61470abf50186", - "sha256:b09719a17a2301178fac4470d54b1680b18a5048b481cb8890e1ef820cb80455", - "sha256:b1121de0e9d6e6ca08289583d7491e7fcb18a439305b34a30b20d8215922d43c", - "sha256:b3b2316b25644b23b54a6f6401074cebcecd1244c0b8e80111c9a3f1c8e83d65", - "sha256:b3d9b48ee6e3967b7901c052b670c7dda6deb812c309439adaffdec55c6d7b78", - "sha256:b5bcf60a228acae568e9911f410f9d9e0d43197d030ae5799e20dca8df588287", - "sha256:b8f3307af845803fb0b060ab76cf6dd3a13adc15b6b451f54281d25911eb92df", - "sha256:c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43", - "sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1", - "sha256:c5a74c359b2d47d26cdbbc7845e9662d6b08a1e915eb015d044729e92e7050b7", - "sha256:c71f16da1ed8949774ef79f4a0260d28b83b3a50c6576f8f4f0288d109777989", - "sha256:d47ecf253780c90ee181d4d871cd655a789da937454045b17b5798da9393901a", - "sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63", - "sha256:d97d85fa63f315a8bdaba2af9a6a686e0eceab77b3089af45133252618e70884", - "sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649", - "sha256:dc45229747b67ffc441b3de2f3ae5e62877a282ea828a5bdb67883c4ee4a8810", - "sha256:e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828", - "sha256:e39c7eb31e3f5b1f88caff88bcff1b7f8334975b46f6ac6e9fc725d829bc35d4", - "sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2", - "sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd", - "sha256:edfe077ab09442d4ef3c52cb1f9dab89bff02f4524afc0acf2d46be17dc479f5", - "sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe", - "sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293", - "sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e", - "sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e", - "sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8" + "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027", + "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087", + "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786", + "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8", + "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09", + "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185", + "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574", + "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e", + "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519", + "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898", + "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269", + "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3", + "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f", + "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6", + "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8", + "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a", + "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73", + "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", + "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714", + "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2", + "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc", + "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce", + "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d", + "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e", + "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6", + "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269", + "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96", + "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d", + "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a", + "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4", + "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77", + "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d", + "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0", + "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed", + "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068", + "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac", + "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25", + "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8", + "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab", + "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26", + "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2", + "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db", + "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f", + "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5", + "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99", + "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c", + "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d", + "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811", + "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa", + "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a", + "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03", + "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b", + "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04", + "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c", + "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001", + "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458", + "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389", + "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99", + "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985", + "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537", + "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238", + "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f", + "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d", + "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796", + "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a", + "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143", + "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8", + "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c", + "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5", + "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5", + "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711", + "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4", + "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6", + "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c", + "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7", + "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4", + "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b", + "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae", + "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12", + "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c", + "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae", + "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8", + "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887", + "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b", + "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4", + "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f", + "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5", + "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33", + "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", + "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561" ], "markers": "python_full_version >= '3.7.0'", - "version": "==3.3.0" + "version": "==3.3.2" }, "click": { "hashes": [ @@ -148,7 +148,6 @@ "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc", "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0" ], - "index": "pypi", "markers": "python_version >= '3.5'", "version": "==3.7" }, @@ -484,11 +483,12 @@ }, "requests": { "hashes": [ - "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", - "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1" + "sha256:f2c3881dddb70d056c5bd7600a4fae312b2a300e39be6a118d30b90bd27262b5", + "sha256:fa5490319474c82ef1d2c9bc459d3652e3ae4ef4c4ebdd18a21145a47ca4b6b8" ], - "markers": "python_version >= '3.7'", - "version": "==2.31.0" + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==2.32.0" }, "six": { "hashes": [ @@ -500,12 +500,11 @@ }, "urllib3": { "hashes": [ - "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84", - "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e" + "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d", + "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19" ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==2.0.7" + "markers": "python_version >= '3.8'", + "version": "==2.2.1" }, "watchdog": { "hashes": [ diff --git a/examples/next-js-netlify/package-lock.json b/examples/next-js-netlify/package-lock.json index b7fbbb7478..12a4c5cd85 100644 --- a/examples/next-js-netlify/package-lock.json +++ b/examples/next-js-netlify/package-lock.json @@ -17,14 +17,14 @@ } }, "node_modules/@next/env": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/env/-/env-13.5.4.tgz", - "integrity": "sha512-LGegJkMvRNw90WWphGJ3RMHMVplYcOfRWf2Be3td3sUa+1AaxmsYyANsA+znrGCBjXJNi4XAQlSoEfUxs/4kIQ==" + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.3.tgz", + "integrity": "sha512-W7fd7IbkfmeeY2gXrzJYDx8D2lWKbVoTIj1o1ScPHNzvp30s1AuoEFSdr39bC5sjxJaxTtq3OTCZboNp0lNWHA==" }, "node_modules/@next/swc-darwin-arm64": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-13.5.4.tgz", - "integrity": "sha512-Df8SHuXgF1p+aonBMcDPEsaahNo2TCwuie7VXED4FVyECvdXfRT9unapm54NssV9tF3OQFKBFOdlje4T43VO0w==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.3.tgz", + "integrity": "sha512-3pEYo/RaGqPP0YzwnlmPN2puaF2WMLM3apt5jLW2fFdXD9+pqcoTzRk+iZsf8ta7+quAe4Q6Ms0nR0SFGFdS1A==", "cpu": [ "arm64" ], @@ -37,9 +37,9 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-13.5.4.tgz", - "integrity": "sha512-siPuUwO45PnNRMeZnSa8n/Lye5ZX93IJom9wQRB5DEOdFrw0JjOMu1GINB8jAEdwa7Vdyn1oJ2xGNaQpdQQ9Pw==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.3.tgz", + "integrity": "sha512-6adp7waE6P1TYFSXpY366xwsOnEXM+y1kgRpjSRVI2CBDOcbRjsJ67Z6EgKIqWIue52d2q/Mx8g9MszARj8IEA==", "cpu": [ "x64" ], @@ -52,9 +52,9 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-13.5.4.tgz", - "integrity": "sha512-l/k/fvRP/zmB2jkFMfefmFkyZbDkYW0mRM/LB+tH5u9pB98WsHXC0WvDHlGCYp3CH/jlkJPL7gN8nkTQVrQ/2w==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.3.tgz", + "integrity": "sha512-cuzCE/1G0ZSnTAHJPUT1rPgQx1w5tzSX7POXSLaS7w2nIUJUD+e25QoXD/hMfxbsT9rslEXugWypJMILBj/QsA==", "cpu": [ "arm64" ], @@ -67,9 +67,9 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-13.5.4.tgz", - "integrity": "sha512-YYGb7SlLkI+XqfQa8VPErljb7k9nUnhhRrVaOdfJNCaQnHBcvbT7cx/UjDQLdleJcfyg1Hkn5YSSIeVfjgmkTg==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.3.tgz", + "integrity": "sha512-0D4/oMM2Y9Ta3nGuCcQN8jjJjmDPYpHX9OJzqk42NZGJocU2MqhBq5tWkJrUQOQY9N+In9xOdymzapM09GeiZw==", "cpu": [ "arm64" ], @@ -82,9 +82,9 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-13.5.4.tgz", - "integrity": "sha512-uE61vyUSClnCH18YHjA8tE1prr/PBFlBFhxBZis4XBRJoR+txAky5d7gGNUIbQ8sZZ7LVkSVgm/5Fc7mwXmRAg==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.3.tgz", + "integrity": "sha512-ENPiNnBNDInBLyUU5ii8PMQh+4XLr4pG51tOp6aJ9xqFQ2iRI6IH0Ds2yJkAzNV1CfyagcyzPfROMViS2wOZ9w==", "cpu": [ "x64" ], @@ -97,9 +97,9 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-13.5.4.tgz", - "integrity": "sha512-qVEKFYML/GvJSy9CfYqAdUexA6M5AklYcQCW+8JECmkQHGoPxCf04iMh7CPR7wkHyWWK+XLt4Ja7hhsPJtSnhg==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.3.tgz", + "integrity": "sha512-BTAbq0LnCbF5MtoM7I/9UeUu/8ZBY0i8SFjUMCbPDOLv+un67e2JgyN4pmgfXBwy/I+RHu8q+k+MCkDN6P9ViQ==", "cpu": [ "x64" ], @@ -112,9 +112,9 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-13.5.4.tgz", - "integrity": "sha512-mDSQfqxAlfpeZOLPxLymZkX0hYF3juN57W6vFHTvwKlnHfmh12Pt7hPIRLYIShk8uYRsKPtMTth/EzpwRI+u8w==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.3.tgz", + "integrity": "sha512-AEHIw/dhAMLNFJFJIJIyOFDzrzI5bAjI9J26gbO5xhAKHYTZ9Or04BesFPXiAYXDNdrwTP2dQceYA4dL1geu8A==", "cpu": [ "arm64" ], @@ -127,9 +127,9 @@ } }, "node_modules/@next/swc-win32-ia32-msvc": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-13.5.4.tgz", - "integrity": "sha512-aoqAT2XIekIWoriwzOmGFAvTtVY5O7JjV21giozBTP5c6uZhpvTWRbmHXbmsjZqY4HnEZQRXWkSAppsIBweKqw==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.3.tgz", + "integrity": "sha512-vga40n1q6aYb0CLrM+eEmisfKCR45ixQYXuBXxOOmmoV8sYST9k7E3US32FsY+CkkF7NtzdcebiFT4CHuMSyZw==", "cpu": [ "ia32" ], @@ -142,9 +142,9 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-13.5.4.tgz", - "integrity": "sha512-cyRvlAxwlddlqeB9xtPSfNSCRy8BOa4wtMo0IuI9P7Y0XT2qpDrpFKRyZ7kUngZis59mPVla5k8X1oOJ8RxDYg==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.3.tgz", + "integrity": "sha512-Q1/zm43RWynxrO7lW4ehciQVj+5ePBhOK+/K2P7pLFX3JaJ/IZVC69SHidrmZSOkqz7ECIOhhy7XhAFG4JYyHA==", "cpu": [ "x64" ], @@ -156,11 +156,17 @@ "node": ">= 10" } }, + "node_modules/@swc/counter": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz", + "integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==" + }, "node_modules/@swc/helpers": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.2.tgz", - "integrity": "sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw==", + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.5.tgz", + "integrity": "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A==", "dependencies": { + "@swc/counter": "^0.1.3", "tslib": "^2.4.0" } }, @@ -214,9 +220,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001412", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001412.tgz", - "integrity": "sha512-+TeEIee1gS5bYOiuf+PS/kp2mrXic37Hl66VY6EAfxasIk5fELTktK2oOezYed12H8w7jt3s512PpulQidPjwA==", + "version": "1.0.30001617", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001617.tgz", + "integrity": "sha512-mLyjzNI9I+Pix8zwcrpxEbGlfqOkF9kM3ptzmKNw5tizSyYwMe+nGLTqMK9cO+0E+Bh6TsBxNAaHWEM8xwSsmA==", "funding": [ { "type": "opencollective", @@ -225,6 +231,10 @@ { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" } ] }, @@ -239,11 +249,6 @@ "integrity": "sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw==", "dev": true }, - "node_modules/glob-to-regexp": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", - "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==" - }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", @@ -283,37 +288,38 @@ } }, "node_modules/next": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/next/-/next-13.5.4.tgz", - "integrity": "sha512-+93un5S779gho8y9ASQhb/bTkQF17FNQOtXLKAj3lsNgltEcF0C5PMLLncDmH+8X1EnJH1kbqAERa29nRXqhjA==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/next/-/next-14.2.3.tgz", + "integrity": "sha512-dowFkFTR8v79NPJO4QsBUtxv0g9BrS/phluVpMAt2ku7H+cbcBJlopXjkWlwxrk/xGqMemr7JkGPGemPrLLX7A==", "dependencies": { - "@next/env": "13.5.4", - "@swc/helpers": "0.5.2", + "@next/env": "14.2.3", + "@swc/helpers": "0.5.5", "busboy": "1.6.0", - "caniuse-lite": "^1.0.30001406", + "caniuse-lite": "^1.0.30001579", + "graceful-fs": "^4.2.11", "postcss": "8.4.31", - "styled-jsx": "5.1.1", - "watchpack": "2.4.0" + "styled-jsx": "5.1.1" }, "bin": { "next": "dist/bin/next" }, "engines": { - "node": ">=16.14.0" + "node": ">=18.17.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "13.5.4", - "@next/swc-darwin-x64": "13.5.4", - "@next/swc-linux-arm64-gnu": "13.5.4", - "@next/swc-linux-arm64-musl": "13.5.4", - "@next/swc-linux-x64-gnu": "13.5.4", - "@next/swc-linux-x64-musl": "13.5.4", - "@next/swc-win32-arm64-msvc": "13.5.4", - "@next/swc-win32-ia32-msvc": "13.5.4", - "@next/swc-win32-x64-msvc": "13.5.4" + "@next/swc-darwin-arm64": "14.2.3", + "@next/swc-darwin-x64": "14.2.3", + "@next/swc-linux-arm64-gnu": "14.2.3", + "@next/swc-linux-arm64-musl": "14.2.3", + "@next/swc-linux-x64-gnu": "14.2.3", + "@next/swc-linux-x64-musl": "14.2.3", + "@next/swc-win32-arm64-msvc": "14.2.3", + "@next/swc-win32-ia32-msvc": "14.2.3", + "@next/swc-win32-x64-msvc": "14.2.3" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", + "@playwright/test": "^1.41.2", "react": "^18.2.0", "react-dom": "^18.2.0", "sass": "^1.3.0" @@ -322,6 +328,9 @@ "@opentelemetry/api": { "optional": true }, + "@playwright/test": { + "optional": true + }, "sass": { "optional": true } @@ -445,85 +454,79 @@ "engines": { "node": ">=4.2.0" } - }, - "node_modules/watchpack": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz", - "integrity": "sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==", - "dependencies": { - "glob-to-regexp": "^0.4.1", - "graceful-fs": "^4.1.2" - }, - "engines": { - "node": ">=10.13.0" - } } }, "dependencies": { "@next/env": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/env/-/env-13.5.4.tgz", - "integrity": "sha512-LGegJkMvRNw90WWphGJ3RMHMVplYcOfRWf2Be3td3sUa+1AaxmsYyANsA+znrGCBjXJNi4XAQlSoEfUxs/4kIQ==" + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.3.tgz", + "integrity": "sha512-W7fd7IbkfmeeY2gXrzJYDx8D2lWKbVoTIj1o1ScPHNzvp30s1AuoEFSdr39bC5sjxJaxTtq3OTCZboNp0lNWHA==" }, "@next/swc-darwin-arm64": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-13.5.4.tgz", - "integrity": "sha512-Df8SHuXgF1p+aonBMcDPEsaahNo2TCwuie7VXED4FVyECvdXfRT9unapm54NssV9tF3OQFKBFOdlje4T43VO0w==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.3.tgz", + "integrity": "sha512-3pEYo/RaGqPP0YzwnlmPN2puaF2WMLM3apt5jLW2fFdXD9+pqcoTzRk+iZsf8ta7+quAe4Q6Ms0nR0SFGFdS1A==", "optional": true }, "@next/swc-darwin-x64": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-13.5.4.tgz", - "integrity": "sha512-siPuUwO45PnNRMeZnSa8n/Lye5ZX93IJom9wQRB5DEOdFrw0JjOMu1GINB8jAEdwa7Vdyn1oJ2xGNaQpdQQ9Pw==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.3.tgz", + "integrity": "sha512-6adp7waE6P1TYFSXpY366xwsOnEXM+y1kgRpjSRVI2CBDOcbRjsJ67Z6EgKIqWIue52d2q/Mx8g9MszARj8IEA==", "optional": true }, "@next/swc-linux-arm64-gnu": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-13.5.4.tgz", - "integrity": "sha512-l/k/fvRP/zmB2jkFMfefmFkyZbDkYW0mRM/LB+tH5u9pB98WsHXC0WvDHlGCYp3CH/jlkJPL7gN8nkTQVrQ/2w==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.3.tgz", + "integrity": "sha512-cuzCE/1G0ZSnTAHJPUT1rPgQx1w5tzSX7POXSLaS7w2nIUJUD+e25QoXD/hMfxbsT9rslEXugWypJMILBj/QsA==", "optional": true }, "@next/swc-linux-arm64-musl": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-13.5.4.tgz", - "integrity": "sha512-YYGb7SlLkI+XqfQa8VPErljb7k9nUnhhRrVaOdfJNCaQnHBcvbT7cx/UjDQLdleJcfyg1Hkn5YSSIeVfjgmkTg==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.3.tgz", + "integrity": "sha512-0D4/oMM2Y9Ta3nGuCcQN8jjJjmDPYpHX9OJzqk42NZGJocU2MqhBq5tWkJrUQOQY9N+In9xOdymzapM09GeiZw==", "optional": true }, "@next/swc-linux-x64-gnu": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-13.5.4.tgz", - "integrity": "sha512-uE61vyUSClnCH18YHjA8tE1prr/PBFlBFhxBZis4XBRJoR+txAky5d7gGNUIbQ8sZZ7LVkSVgm/5Fc7mwXmRAg==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.3.tgz", + "integrity": "sha512-ENPiNnBNDInBLyUU5ii8PMQh+4XLr4pG51tOp6aJ9xqFQ2iRI6IH0Ds2yJkAzNV1CfyagcyzPfROMViS2wOZ9w==", "optional": true }, "@next/swc-linux-x64-musl": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-13.5.4.tgz", - "integrity": "sha512-qVEKFYML/GvJSy9CfYqAdUexA6M5AklYcQCW+8JECmkQHGoPxCf04iMh7CPR7wkHyWWK+XLt4Ja7hhsPJtSnhg==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.3.tgz", + "integrity": "sha512-BTAbq0LnCbF5MtoM7I/9UeUu/8ZBY0i8SFjUMCbPDOLv+un67e2JgyN4pmgfXBwy/I+RHu8q+k+MCkDN6P9ViQ==", "optional": true }, "@next/swc-win32-arm64-msvc": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-13.5.4.tgz", - "integrity": "sha512-mDSQfqxAlfpeZOLPxLymZkX0hYF3juN57W6vFHTvwKlnHfmh12Pt7hPIRLYIShk8uYRsKPtMTth/EzpwRI+u8w==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.3.tgz", + "integrity": "sha512-AEHIw/dhAMLNFJFJIJIyOFDzrzI5bAjI9J26gbO5xhAKHYTZ9Or04BesFPXiAYXDNdrwTP2dQceYA4dL1geu8A==", "optional": true }, "@next/swc-win32-ia32-msvc": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-13.5.4.tgz", - "integrity": "sha512-aoqAT2XIekIWoriwzOmGFAvTtVY5O7JjV21giozBTP5c6uZhpvTWRbmHXbmsjZqY4HnEZQRXWkSAppsIBweKqw==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.3.tgz", + "integrity": "sha512-vga40n1q6aYb0CLrM+eEmisfKCR45ixQYXuBXxOOmmoV8sYST9k7E3US32FsY+CkkF7NtzdcebiFT4CHuMSyZw==", "optional": true }, "@next/swc-win32-x64-msvc": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-13.5.4.tgz", - "integrity": "sha512-cyRvlAxwlddlqeB9xtPSfNSCRy8BOa4wtMo0IuI9P7Y0XT2qpDrpFKRyZ7kUngZis59mPVla5k8X1oOJ8RxDYg==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.3.tgz", + "integrity": "sha512-Q1/zm43RWynxrO7lW4ehciQVj+5ePBhOK+/K2P7pLFX3JaJ/IZVC69SHidrmZSOkqz7ECIOhhy7XhAFG4JYyHA==", "optional": true }, + "@swc/counter": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz", + "integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==" + }, "@swc/helpers": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.2.tgz", - "integrity": "sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw==", + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.5.tgz", + "integrity": "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A==", "requires": { + "@swc/counter": "^0.1.3", "tslib": "^2.4.0" } }, @@ -574,9 +577,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001412", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001412.tgz", - "integrity": "sha512-+TeEIee1gS5bYOiuf+PS/kp2mrXic37Hl66VY6EAfxasIk5fELTktK2oOezYed12H8w7jt3s512PpulQidPjwA==" + "version": "1.0.30001617", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001617.tgz", + "integrity": "sha512-mLyjzNI9I+Pix8zwcrpxEbGlfqOkF9kM3ptzmKNw5tizSyYwMe+nGLTqMK9cO+0E+Bh6TsBxNAaHWEM8xwSsmA==" }, "client-only": { "version": "0.0.1", @@ -589,11 +592,6 @@ "integrity": "sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw==", "dev": true }, - "glob-to-regexp": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", - "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==" - }, "graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", @@ -618,26 +616,26 @@ "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==" }, "next": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/next/-/next-13.5.4.tgz", - "integrity": "sha512-+93un5S779gho8y9ASQhb/bTkQF17FNQOtXLKAj3lsNgltEcF0C5PMLLncDmH+8X1EnJH1kbqAERa29nRXqhjA==", + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/next/-/next-14.2.3.tgz", + "integrity": "sha512-dowFkFTR8v79NPJO4QsBUtxv0g9BrS/phluVpMAt2ku7H+cbcBJlopXjkWlwxrk/xGqMemr7JkGPGemPrLLX7A==", "requires": { - "@next/env": "13.5.4", - "@next/swc-darwin-arm64": "13.5.4", - "@next/swc-darwin-x64": "13.5.4", - "@next/swc-linux-arm64-gnu": "13.5.4", - "@next/swc-linux-arm64-musl": "13.5.4", - "@next/swc-linux-x64-gnu": "13.5.4", - "@next/swc-linux-x64-musl": "13.5.4", - "@next/swc-win32-arm64-msvc": "13.5.4", - "@next/swc-win32-ia32-msvc": "13.5.4", - "@next/swc-win32-x64-msvc": "13.5.4", - "@swc/helpers": "0.5.2", + "@next/env": "14.2.3", + "@next/swc-darwin-arm64": "14.2.3", + "@next/swc-darwin-x64": "14.2.3", + "@next/swc-linux-arm64-gnu": "14.2.3", + "@next/swc-linux-arm64-musl": "14.2.3", + "@next/swc-linux-x64-gnu": "14.2.3", + "@next/swc-linux-x64-musl": "14.2.3", + "@next/swc-win32-arm64-msvc": "14.2.3", + "@next/swc-win32-ia32-msvc": "14.2.3", + "@next/swc-win32-x64-msvc": "14.2.3", + "@swc/helpers": "0.5.5", "busboy": "1.6.0", - "caniuse-lite": "^1.0.30001406", + "caniuse-lite": "^1.0.30001579", + "graceful-fs": "^4.2.11", "postcss": "8.4.31", - "styled-jsx": "5.1.1", - "watchpack": "2.4.0" + "styled-jsx": "5.1.1" } }, "picocolors": { @@ -708,15 +706,6 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.8.4.tgz", "integrity": "sha512-QCh+85mCy+h0IGff8r5XWzOVSbBO+KfeYrMQh7NJ58QujwcE22u+NUSmUxqF+un70P9GXKxa2HCNiTTMJknyjQ==", "dev": true - }, - "watchpack": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz", - "integrity": "sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==", - "requires": { - "glob-to-regexp": "^0.4.1", - "graceful-fs": "^4.1.2" - } } } } diff --git a/examples/react/Earthfile b/examples/react/Earthfile index 10a6e4cb26..077968a7e3 100644 --- a/examples/react/Earthfile +++ b/examples/react/Earthfile @@ -16,7 +16,7 @@ docker: SAVE IMAGE --push earthly/examples:react run: - FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 WITH DOCKER --load app:test=+docker RUN docker run --rm -p 3000:80 app:test END diff --git a/examples/ruby-on-rails/Gemfile.lock b/examples/ruby-on-rails/Gemfile.lock index 7a3428918a..23450f836a 100644 --- a/examples/ruby-on-rails/Gemfile.lock +++ b/examples/ruby-on-rails/Gemfile.lock @@ -82,10 +82,10 @@ GEM mini_mime (>= 0.1.1) method_source (1.0.0) mini_mime (1.0.2) - mini_portile2 (2.8.5) + mini_portile2 (2.8.6) minitest (5.17.0) nio4r (2.7.0) - nokogiri (1.16.2) + nokogiri (1.16.5) mini_portile2 (~> 2.8.2) racc (~> 1.4) public_suffix (4.0.7) diff --git a/examples/tutorial/go/part6/Earthfile b/examples/tutorial/go/part6/Earthfile index 7ac90c2446..dfcd13c591 100644 --- a/examples/tutorial/go/part6/Earthfile +++ b/examples/tutorial/go/part6/Earthfile @@ -29,7 +29,7 @@ test-setup: SAVE IMAGE test:latest integration-tests: - FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 COPY docker-compose.yml ./ WITH DOCKER --compose docker-compose.yml --load tests:latest=+test-setup RUN docker run --network=default_go/part6_default tests:latest | grep "ok" diff --git a/examples/tutorial/java/part6/Earthfile b/examples/tutorial/java/part6/Earthfile index 03b36af885..4795d0fcd7 100644 --- a/examples/tutorial/java/part6/Earthfile +++ b/examples/tutorial/java/part6/Earthfile @@ -23,7 +23,7 @@ docker: SAVE IMAGE java-example:$tag integration-tests: - FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 COPY ./docker-compose.yml . RUN apk update RUN apk add postgresql-client diff --git a/examples/tutorial/js/part6/Earthfile b/examples/tutorial/js/part6/Earthfile index 01c08ec103..e7e146c60f 100644 --- a/examples/tutorial/js/part6/Earthfile +++ b/examples/tutorial/js/part6/Earthfile @@ -44,7 +44,7 @@ api-docker: SAVE IMAGE js-api:$tag integration-tests: - FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 RUN apk add curl WITH DOCKER \ --load app:latest=+app-docker \ diff --git a/examples/tutorial/python/part6/Earthfile b/examples/tutorial/python/part6/Earthfile index bfe37c7412..296a29220a 100644 --- a/examples/tutorial/python/part6/Earthfile +++ b/examples/tutorial/python/part6/Earthfile @@ -12,7 +12,7 @@ build: SAVE ARTIFACT src /src integration-tests: - FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 + FROM earthly/dind:alpine-3.19-docker-25.0.5-r0 COPY ./docker-compose.yml . COPY ./tests ./tests RUN apk update diff --git a/features/features.go b/features/features.go index 94a3457fe8..d8366e3034 100644 --- a/features/features.go +++ b/features/features.go @@ -78,6 +78,7 @@ type Features struct { GitAuthorEmailNameArgs bool `long:"git-author-email-name-args" description:"includes EARTHLY_GIT_AUTHOR_EMAIL and EARTHLY_GIT_AUTHOR_NAME builtin ARGs"` AllowWithoutEarthlyLabels bool `long:"allow-without-earthly-labels" description:"Allow the usage of --without-earthly-labels in SAVE IMAGE"` DockerCache bool `long:"docker-cache" description:"enable the WITH DOCKER --cache-id option"` + RunWithAWSOIDC bool `long:"run-with-aws-oidc" description:"make AWS credentials via OIDC provider available to RUN commands"` // version numbers Major int diff --git a/go.mod b/go.mod index fd8554d001..dc7b477dcd 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/alexcb/binarystream v0.0.0-20231130184431-f2f7a7543c6d github.com/aws/aws-sdk-go-v2 v1.26.1 github.com/aws/aws-sdk-go-v2/config v1.18.16 + github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0 github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230227212328-9f4511cd144a github.com/containerd/containerd v1.7.8 github.com/containerd/go-runc v1.1.0 @@ -18,9 +19,9 @@ require ( github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.5.0 github.com/dustin/go-humanize v1.0.1 - github.com/earthly/cloud-api v1.0.1-0.20240419182846-287b8dd9616f + github.com/earthly/cloud-api v1.0.1-0.20240516231256-26ec57717150 github.com/earthly/earthly/ast v0.0.0-00010101000000-000000000000 - github.com/earthly/earthly/util/deltautil v0.0.0-20231221211955-0fd4ae2cc257 + github.com/earthly/earthly/util/deltautil v0.0.0-20240507235053-335389ed3e2a github.com/elastic/go-sysinfo v1.9.0 github.com/fatih/color v1.16.0 github.com/gofrs/flock v0.8.1 @@ -37,6 +38,7 @@ require ( github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.20 github.com/mitchellh/hashstructure/v2 v2.0.2 + github.com/mitchellh/mapstructure v1.5.0 github.com/moby/buildkit v0.8.2-0.20210129065303-6b9ea0c202cf github.com/moby/patternmatcher v0.6.0 github.com/opencontainers/go-digest v1.0.0 @@ -71,8 +73,8 @@ require ( github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230219212500-1f9a474cc2dc // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.16 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.24 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31 // indirect github.com/aws/aws-sdk-go-v2/service/ecr v1.24.3 // indirect github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.15.1 // indirect @@ -149,6 +151,6 @@ replace ( github.com/jdxcode/netrc => github.com/mikejholly/netrc v0.0.0-20221121193719-a154cb29ec2a github.com/jessevdk/go-flags => github.com/alexcb/go-flags v0.0.0-20210722203016-f11d7ecb5ee5 - github.com/moby/buildkit => github.com/earthly/buildkit v0.0.0-20240419223558-594835b598b9 + github.com/moby/buildkit => github.com/earthly/buildkit v0.0.0-20240515200521-531b303aa8ec github.com/tonistiigi/fsutil => github.com/earthly/fsutil v0.0.0-20231030221755-644b08355b65 ) diff --git a/go.sum b/go.sum index c21fd4bc91..540c48a2a6 100644 --- a/go.sum +++ b/go.sum @@ -85,16 +85,18 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.24/go.mod h1:neYVaeKr5eT7Bzw github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28/go.mod h1:3lwChorpIM/BhImY/hy+Z6jekmN92cXGPI1QJasVPYY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30/go.mod h1:LUBAO3zNXQjoONBKn/kR1y0Q4cj/D02Ts0uHYjcCQLM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 h1:8GVZIR0y6JRIUNSYI1xAMF4HDfV8H/bOsZ/8AD/uY5Q= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8/go.mod h1:rwBfu0SoUkBUZndVgPZKAD9Y2JigaZtRP68unRiYToQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22/go.mod h1:EqK7gVrIGAHyZItrD1D8B0ilgwMD1GiWAmbU4u/JHNk= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.24/go.mod h1:gAuCezX/gob6BSMbItsSlMb6WZGV7K2+fWOvk8xBSto= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 h1:ZE2ds/qeBkhk3yqYvS3CDCFNvd9ir5hMjlVStLZWrvM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8/go.mod h1:/lAPPymDYL023+TS6DJmjuL42nxix2AvEvfjqOBRODk= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29/go.mod h1:TwuqRBGzxjQJIwH16/fOZodwXt2Zxa9/cwJC5ke4j7s= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31 h1:hf+Vhp5WtTdcSdE+yEcUz8L73sAzN0R+0jQv+Z51/mI= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31/go.mod h1:5zUjguZfG5qjhG9/wqmuyHRyUftl2B5Cp6NNxNC6kRA= +github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0 h1:Ap5tOJfeAH1hO2UQc3X3uMlwP7uryFeZXMvZCXIlLSE= +github.com/aws/aws-sdk-go-v2/service/cloudformation v1.50.0/go.mod h1:/v2KYdCW4BaHKayenaWEXOOdxItIwEA3oU0XzuQY3F0= github.com/aws/aws-sdk-go-v2/service/ecr v1.18.2/go.mod h1:53xgmccefO+AwKsxVKuTh2vo/IDOkeMWNpmDuhZH1Vc= github.com/aws/aws-sdk-go-v2/service/ecr v1.24.3 h1:+sbyLjtAq0Xg9ZOQ2mBibklsGUyX6I2OfRTDsha9uU4= github.com/aws/aws-sdk-go-v2/service/ecr v1.24.3/go.mod h1:/m9MiYl5Ds0cZqy/bbeSUWxKLwTarGugjXxSgiXNQFc= @@ -190,10 +192,10 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/earthly/buildkit v0.0.0-20240419223558-594835b598b9 h1:yN1+9T45k+wyzD2Jaf0xSc7bvE0nx09WH2oLb49+vk4= -github.com/earthly/buildkit v0.0.0-20240419223558-594835b598b9/go.mod h1:1/yAC8A0Tu94Bdmv07gaG1pFBp+CetVwO7oB3qvZXUc= -github.com/earthly/cloud-api v1.0.1-0.20240419182846-287b8dd9616f h1:nNXnxnK4+fB++j/aj8jMqPbA4uh4OEKZr1jq1Ub3adI= -github.com/earthly/cloud-api v1.0.1-0.20240419182846-287b8dd9616f/go.mod h1:rU/tYJ7GFBjdKAITV2heDbez++glpGSbtJaZcp73rNI= +github.com/earthly/buildkit v0.0.0-20240515200521-531b303aa8ec h1:vf6x0fPOWKakjH3n2N1O9Tg5j1HDIJsC3Kkgmuko2U0= +github.com/earthly/buildkit v0.0.0-20240515200521-531b303aa8ec/go.mod h1:1/yAC8A0Tu94Bdmv07gaG1pFBp+CetVwO7oB3qvZXUc= +github.com/earthly/cloud-api v1.0.1-0.20240516231256-26ec57717150 h1:hHdgFB5BE+OAFee+CpassZP2GOkdRt5YyhsFmJJVtc8= +github.com/earthly/cloud-api v1.0.1-0.20240516231256-26ec57717150/go.mod h1:rU/tYJ7GFBjdKAITV2heDbez++glpGSbtJaZcp73rNI= github.com/earthly/fsutil v0.0.0-20231030221755-644b08355b65 h1:6oyWHoxHXwcTt4EqmMw6361scIV87uEAB1N42+VpIwk= github.com/earthly/fsutil v0.0.0-20231030221755-644b08355b65/go.mod h1:9kMVqMyQ/Sx2df5LtnGG+nbrmiZzCS7V6gjW3oGHsvI= github.com/elastic/go-sysinfo v1.9.0 h1:usICqY/Nw4Mpn9f4LdtpFrKxXroJDe81GaxxUlCckIo= @@ -354,6 +356,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= diff --git a/logbus/formatter/formatter.go b/logbus/formatter/formatter.go index 13c67f27df..e0e9a97a5a 100644 --- a/logbus/formatter/formatter.go +++ b/logbus/formatter/formatter.go @@ -18,6 +18,7 @@ import ( "github.com/earthly/earthly/util/deltautil" "github.com/earthly/earthly/util/execstatssummary" "github.com/earthly/earthly/util/progressbar" + "github.com/earthly/earthly/util/stringutil" "github.com/hashicorp/go-multierror" "github.com/mattn/go-isatty" "github.com/pkg/errors" @@ -483,21 +484,23 @@ func (f *Formatter) printGHAFailure() { // Extract the error from first line of the error message lines := strings.Split(failure.GetErrorMessage(), "\n") - var message string + var singleLineMessage string if len(lines) > 1 { - message = strings.Join(strings.Fields(strings.Join(lines[1:], " ")), " ") + singleLineMessage = strings.Join(strings.Fields(strings.Join(lines[1:], " ")), " ") } else { - message = strings.Join(strings.Fields(lines[0]), " ") + singleLineMessage = strings.Join(strings.Fields(lines[0]), " ") } + singleLineMessage = stringutil.ScrubANSICodes(singleLineMessage) // Print GHA Error with line info if available if cm != nil && cm.SourceLocation != nil && cm.SourceLocation.File != "" && cm.SourceLocation.StartLine > 0 { - c.PrintGHAError(message, conslogging.WithGHASourceLocation(cm.SourceLocation.File, cm.SourceLocation.StartLine, cm.SourceLocation.StartColumn)) + c.PrintGHAError(singleLineMessage, conslogging.WithGHASourceLocation(cm.SourceLocation.File, cm.SourceLocation.StartLine, cm.SourceLocation.StartColumn)) } else { - c.PrintGHAError(message) + c.PrintGHAError(singleLineMessage) } - + fullErrorMessage := stringutil.ScrubANSICodes(failure.GetErrorMessage()) + output := stringutil.ScrubANSICodes(string(failure.GetOutput())) //GHA Summary markdown markdown := fmt.Sprintf(` # ❌ Build Failure ❌ @@ -513,7 +516,7 @@ func (f *Formatter) printGHAFailure() { ~~~ %s ~~~ -`, failure.GetErrorMessage(), string(failure.GetOutput())) +`, fullErrorMessage, output) c.PrintGHASummary(markdown) } diff --git a/outmon/solvermon.go b/outmon/solvermon.go deleted file mode 100644 index 1a9a13ef01..0000000000 --- a/outmon/solvermon.go +++ /dev/null @@ -1,420 +0,0 @@ -package outmon - -import ( - "context" - "fmt" - "sort" - "strings" - "sync" - "time" - - "github.com/dustin/go-humanize" - "github.com/earthly/earthly/conslogging" - "github.com/earthly/earthly/logbus/formatter" - "github.com/earthly/earthly/util/buildkitutil" - "github.com/earthly/earthly/util/vertexmeta" - "github.com/moby/buildkit/client" - "github.com/opencontainers/go-digest" -) - -const ( - durationBetweenSha256ProgressUpdate = 5 * time.Second - durationBetweenProgressUpdate = 3 * time.Second - durationBetweenProgressUpdateIfSame = 5 * time.Millisecond - durationBetweenOpenLineUpdate = time.Second - durationBetweenNoOutputUpdates = 5 * time.Second - durationBetweenNoOutputUpdatesNoAnsi = 60 * time.Second - tailErrorBufferSizeBytes = 80 * 1024 // About as much as 1024 lines of 80 chars each. -) - -// SolverMonitor is an object that monitors for status updates from a buildkit solve -// and prints them to the console. -type SolverMonitor struct { - msgMu sync.Mutex - console conslogging.ConsoleLogger - verbose bool - disableNoOutputUpdates bool - vertices map[digest.Digest]*vertexMonitor - saltSeen map[string]bool - lastVertexOutput *vertexMonitor - lastOutputWasProgress bool - lastOutputWasNoOutputUpdate bool - timingTable map[timingKey]time.Duration - startTime time.Time - noOutputTicker *time.Ticker - noOutputTick time.Duration - errVertex *vertexMonitor - - mu sync.Mutex - ongoing bool -} - -type timingKey struct { - targetStr string - targetBrackets string - salt string -} - -// NewSolverMonitor retuns a new solver monitor. -func NewSolverMonitor(console conslogging.ConsoleLogger, verbose bool, disableNoOutputUpdates bool) *SolverMonitor { - noOutputTick := durationBetweenNoOutputUpdatesNoAnsi - if ansiSupported { - noOutputTick = durationBetweenNoOutputUpdates - } - return &SolverMonitor{ - console: console, - verbose: verbose, - disableNoOutputUpdates: disableNoOutputUpdates, - vertices: make(map[digest.Digest]*vertexMonitor), - saltSeen: make(map[string]bool), - timingTable: make(map[timingKey]time.Duration), - startTime: time.Now(), - noOutputTicker: time.NewTicker(noOutputTick), - noOutputTick: noOutputTick, - } -} - -// MonitorProgress consumes progress messages from a solve statue channel and prints them to the console. -func (sm *SolverMonitor) MonitorProgress(ctx context.Context, ch chan *client.SolveStatus, phaseText string, sideRun bool, bkClient *client.Client) (string, error) { - if !sideRun { - sm.mu.Lock() - sm.ongoing = true - sm.mu.Unlock() - } -Loop: - for { - select { - case ss, ok := <-ch: - if !ok { - break Loop - } - err := sm.processStatus(ss) - if err != nil { - return "", err - } - case <-sm.noOutputTicker.C: - err := sm.processNoOutputTick(ctx, bkClient) - if err != nil { - return "", err - } - } - } - failedVertexOutput := "" - if !sideRun { - sm.msgMu.Lock() - if sm.errVertex != nil { - if sm.errVertex.tailOutput != nil { - failedVertexOutput = string(sm.errVertex.tailOutput.Bytes()) - } - sm.reprintFailure(sm.errVertex, phaseText) - } - sm.msgMu.Unlock() - sm.mu.Lock() - sm.ongoing = false - sm.mu.Unlock() - sm.PrintTiming() - sm.noOutputTicker.Stop() - } - return failedVertexOutput, nil -} - -func (sm *SolverMonitor) processStatus(ss *client.SolveStatus) error { - sm.msgMu.Lock() - defer sm.msgMu.Unlock() - for _, vertex := range ss.Vertexes { - vm, ok := sm.vertices[vertex.Digest] - if !ok { - meta, operation := vertexmeta.ParseFromVertexPrefix(vertex.Name) - vm = &vertexMonitor{ - vertex: vertex, - meta: meta, - operation: operation, - console: sm.console.WithPrefixAndSalt(meta.TargetName, meta.Salt()), - lastPercentage: make(map[string]int), - lastProgress: make(map[string]time.Time), - } - if vm.meta.Local { - vm.console = vm.console.WithLocal(true) - } - sm.vertices[vertex.Digest] = vm - } - vm.vertex = vertex - if !vm.headerPrinted && - ((!vm.meta.Internal && (vertex.Cached || vertex.Started != nil)) || vertex.Error != "") { - sm.printHeader(vm) - sm.noOutputTicker.Reset(sm.noOutputTick) - } - if vertex.Error != "" { - if strings.Contains(vertex.Error, "context canceled") { - if !vm.meta.Internal { - vm.console.Printf("WARN: Canceled\n") - vm.isCanceled = true - sm.noOutputTicker.Reset(sm.noOutputTick) - } - } else { - vm.isError = vm.printError() - if sm.errVertex == nil && vm.isError { - sm.errVertex = vm - } - sm.noOutputTicker.Reset(sm.noOutputTick) - } - } - if sm.verbose { - vm.printTimingInfo() - sm.recordTiming(vm, vertex) - sm.noOutputTicker.Reset(sm.noOutputTick) - } - - vm.reportStatusToConsole() - vm.reportResultToConsole() - } - for _, vs := range ss.Statuses { - vm, ok := sm.vertices[vs.Vertex] - if !ok || vm.meta.Internal { - // No logging for internal operations. - continue - } - progress := int(0) - if vs.Total != 0 { - progress = int(100.0 * float32(vs.Current) / float32(vs.Total)) - } - if vs.Completed != nil { - progress = 100 - } - sm.printProgress(vm, vs.ID, progress) - sm.noOutputTicker.Reset(sm.noOutputTick) - } - for _, logLine := range ss.Logs { - vm, ok := sm.vertices[logLine.Vertex] - if !ok || vm.meta.Internal { - // No logging for internal operations. - continue - } - if !vm.headerPrinted { - sm.printHeader(vm) - } - if logLine.Stream == formatter.BuildkitStatsStream { - // --exec-stats requires logbus to be enabled - continue - } - err := sm.printOutput(vm, logLine.Data) - if err != nil { - return err - } - sm.noOutputTicker.Reset(sm.noOutputTick) - } - return nil -} - -func (sm *SolverMonitor) processNoOutputTick(ctx context.Context, bkClient *client.Client) error { - ongoingCons := sm.console.WithPrefix("ongoing") - sm.msgMu.Lock() - defer sm.msgMu.Unlock() - if sm.disableNoOutputUpdates { - return nil - } - ongoingBuilder := []string{} - if sm.lastOutputWasNoOutputUpdate { - // Overwrite previous line if the previous update was also a no-output update. - ongoingBuilder = append(ongoingBuilder, string(ansiUp)) - } - ongoing := []string{} - now := time.Now() - for _, vm := range sm.vertices { - if !vm.isOngoing() { - continue - } - if vm.meta.Interactive { - // Don't print ongoing updates when an interactive session is ongoing. - return nil - } - - col := vm.console.PrefixColor() - relTime := humanize.RelTime(*vm.vertex.Started, now, "ago", "from now") - ongoing = append(ongoing, fmt.Sprintf("%s (%s)", col.Sprintf("%s", vm.meta.TargetName), relTime)) - } - var ongoingStr string - warn := false - defer func() { - // Note: This part assumes that we are still under lock. - ongoingBuilder = append(ongoingBuilder, ongoingStr, string(ansiEraseRestLine)) - outStr := strings.Join(ongoingBuilder, "") - if warn { - ongoingCons.Warnf("%s\n", outStr) - } else { - ongoingCons.Printf("%s\n", outStr) - } - sm.lastOutputWasProgress = false - sm.lastOutputWasNoOutputUpdate = true - }() - - if len(ongoing) != 0 { - sort.Strings(ongoing) // not entirely correct, but makes the ordering consistent - if len(ongoing) > 2 { - ongoingStr = fmt.Sprintf("%s and %d others", strings.Join(ongoing[:2], ", "), len(ongoing)-2) - } else { - ongoingStr = strings.Join(ongoing, ", ") - } - return nil - } - - // Nothing running, but also no output taking place. We're just sitting, - // waiting for buildkit to make progress. Let's check if buildkit is - // overwhelmed and report accordingly. - workers, err := bkClient.ListWorkers(ctx) - if err != nil { - ongoingStr = fmt.Sprintf("error getting buildkit worker info: %v", err) - warn = true - return nil // no need to crash - } - numOtherSessions := -1 // default to unknown (since Info is a newer call and might not be implemented) - if info, err := bkClient.Info(ctx); err == nil { - numOtherSessions = info.NumSessions - 1 // don't count current session - } - - if len(workers) == 0 { - ongoingStr = "error getting buildkit worker info: no workers" - warn = true - return nil // no need to crash - } - workerInfo := workers[0] - load := workerInfo.ParallelismCurrent + workerInfo.ParallelismWaiting - switch { - case workerInfo.ParallelismWaiting > 5: - ongoingStr = fmt.Sprintf( - "Waiting... Buildkit is currently under heavy load (%s)", buildkitutil.FormatUtilization(numOtherSessions, load, workerInfo.ParallelismMax)) - warn = true - case workerInfo.ParallelismWaiting > 0: - ongoingStr = fmt.Sprintf( - "Waiting... Buildkit is currently under significant load (%s)", buildkitutil.FormatUtilization(numOtherSessions, load, workerInfo.ParallelismMax)) - default: - ongoingStr = fmt.Sprintf( - "Waiting on Buildkit... (%s)", buildkitutil.FormatUtilization(numOtherSessions, load, workerInfo.ParallelismMax)) - } - if workerInfo.GCAnalytics.CurrentStartTime != nil { - d := now.Sub(*workerInfo.GCAnalytics.CurrentStartTime).Round(time.Second) - if d <= 5*time.Minute { - ongoingStr += fmt.Sprintf(" GC (%v ago)", d) - } else { - ongoingStr += fmt.Sprintf(" GC is slow (%v ago)", d) - warn = true - } - } - if workerInfo.GCAnalytics.AllTimeMaxDuration > 5*time.Minute { - ongoingStr += fmt.Sprintf( - " GCs historically slow (max %v)", - workerInfo.GCAnalytics.AllTimeMaxDuration.Round(time.Second)) - warn = true - } - return nil -} - -func (sm *SolverMonitor) printOutput(vm *vertexMonitor, data []byte) error { - sameAsLast := (sm.lastVertexOutput == vm && !sm.lastOutputWasProgress) - sm.lastVertexOutput = vm - sm.lastOutputWasProgress = false - sm.lastOutputWasNoOutputUpdate = false - return vm.printOutput(data, sameAsLast) -} - -func (sm *SolverMonitor) printProgress(vm *vertexMonitor, id string, progress int) { - if vm.shouldPrintProgress(id, progress, sm.verbose, sm.lastOutputWasProgress) { - if !vm.headerPrinted { - sm.printHeader(vm) - } - vm.printProgress(id, progress, sm.verbose, sm.lastOutputWasProgress) - sm.lastOutputWasProgress = (progress != 100) - sm.lastOutputWasNoOutputUpdate = false - } -} - -func (sm *SolverMonitor) printHeader(vm *vertexMonitor) { - seen := sm.saltSeen[vm.meta.Salt()] - if !seen { - sm.saltSeen[vm.meta.Salt()] = true - } - vm.printHeader(sm.verbose) - sm.lastOutputWasProgress = false - sm.lastOutputWasNoOutputUpdate = false -} - -func (sm *SolverMonitor) recordTiming(vm *vertexMonitor, vertex *client.Vertex) { - if vertex.Started == nil || vertex.Completed == nil { - return - } - dur := vertex.Completed.Sub(*vertex.Started) - if dur == 0 { - return - } - key := timingKey{ - targetStr: vm.meta.TargetName, - targetBrackets: vm.meta.OverridingArgsString(), - salt: vm.meta.Salt(), - } - sm.timingTable[key] += dur -} - -// PrintTiming prints the accumulated timing information. -func (sm *SolverMonitor) PrintTiming() { - if !sm.verbose { - return - } - sm.console. - WithMetadataMode(true). - Printf("Summary of timing information\n" + - "Note that the times do not include the expansion of commands like BUILD, FROM, COPY (artifact).") - var total time.Duration - type durationAndKey struct { - dur time.Duration - key timingKey - } - durs := make([]durationAndKey, 0, len(sm.timingTable)) - for key, dur := range sm.timingTable { - durs = append(durs, durationAndKey{ - dur: dur, - key: key, - }) - total += dur - } - sort.Slice(durs, func(i, j int) bool { - return durs[i].dur > durs[j].dur - }) - for _, d := range durs { - sm.console. - WithPrefixAndSalt(d.key.targetStr, d.key.salt). - WithMetadataMode(true). - Printf("(%s) %s\n", d.key.targetBrackets, d.dur) - } - sm.console. - WithMetadataMode(true). - Printf("===============================================================\n") - sm.console. - WithMetadataMode(true). - Printf("Total \t%s\n", total) - sm.console. - WithMetadataMode(true). - Printf("Total (real)\t%s\n", time.Since(sm.startTime)) -} - -func (sm *SolverMonitor) reprintFailure(errVertex *vertexMonitor, phaseText string) { - sm.lastOutputWasProgress = false - sm.lastOutputWasNoOutputUpdate = false - sm.console.PrintFailure(phaseText) - sm.console.Warnf("Repeating the output of the command that caused the failure\n") - errVertex.console = errVertex.console.WithFailed(true) - errVertex.printHeader(sm.verbose) - if errVertex.tailOutput != nil { - isTruncated := (errVertex.tailOutput.TotalWritten() > errVertex.tailOutput.Size()) - if errVertex.tailOutput.TotalWritten() == 0 { - errVertex.console.Printf("[no output]\n") - } else { - if isTruncated { - errVertex.console.Printf("[...]\n") - } - errVertex.console.PrintBytes(errVertex.tailOutput.Bytes()) - } - } else { - errVertex.console.Printf("[no output]\n") - } - errVertex.printError() -} diff --git a/outmon/vertexmon.go b/outmon/vertexmon.go deleted file mode 100644 index 0b37cb16f1..0000000000 --- a/outmon/vertexmon.go +++ /dev/null @@ -1,269 +0,0 @@ -package outmon - -import ( - "bytes" - "fmt" - "math" - "os" - "regexp" - "strconv" - "strings" - "time" - - "github.com/earthly/earthly/conslogging" - "github.com/earthly/earthly/util/circbuf" - "github.com/earthly/earthly/util/progressbar" - "github.com/earthly/earthly/util/vertexmeta" - "github.com/mattn/go-isatty" - "github.com/moby/buildkit/client" - "github.com/pkg/errors" -) - -type vertexMonitor struct { - vertex *client.Vertex - meta *vertexmeta.VertexMeta - operation string - lastProgress map[string]time.Time - lastPercentage map[string]int - console conslogging.ConsoleLogger - headerPrinted bool - isError bool - isCanceled bool - tailOutput *circbuf.Buffer - // Line of output that has not yet been terminated with a \n. - openLine []byte - lastOpenLineUpdate time.Time - lastOpenLineSkipped bool -} - -func (vm *vertexMonitor) printHeader(verbose bool) { - vm.headerPrinted = true - if vm.operation == "" { - return - } - c := vm.console - var metaParts []string - if vm.meta.NonDefaultPlatform && vm.meta.Platform != "" { - metaParts = append(metaParts, vm.meta.Platform) - } - if vm.meta.OverridingArgs != nil { - metaParts = append(metaParts, vm.meta.OverridingArgsString()) - } - if verbose && len(vm.meta.Secrets) != 0 { - metaParts = append(metaParts, vm.meta.SecretsString()) - } - if len(metaParts) > 0 { - c.WithMetadataMode(true).Printf("%s\n", strings.Join(metaParts, " | ")) - } - out := []string{} - out = append(out, "-->") - out = append(out, vm.operation) - if vm.vertex.Cached { - c = c.WithCached(true) - } - c.Printf("%s\n", strings.Join(out, " ")) -} - -var internalProgress = map[string]bool{ - "exporting manifest": true, - "sending tarballs": true, - "exporting config": true, - "exporting layers": true, - "copying files": true, -} - -const esc = 27 - -var ansiUp = []byte(fmt.Sprintf("%c[A", esc)) -var ansiEraseRestLine = []byte(fmt.Sprintf("%c[K", esc)) -var ansiSupported = os.Getenv("TERM") != "dumb" && - (isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd())) - -func (vm *vertexMonitor) printOutput(output []byte, sameAsLast bool) error { - if vm.tailOutput == nil { - var err error - vm.tailOutput, err = circbuf.NewBuffer(tailErrorBufferSizeBytes) - if err != nil { - return errors.Wrap(err, "allocate buffer for output") - } - } - // Use the raw output for the tail buffer. - _, err := vm.tailOutput.Write(output) - if err != nil { - return errors.Wrap(err, "write to in-memory output buffer") - } - printOutput := make([]byte, 0, len(vm.openLine)+len(output)+10) - if bytes.HasPrefix(output, []byte{'\n'}) && len(vm.openLine) > 0 && !vm.lastOpenLineSkipped { - // Optimization for cases where ansi control sequences are not supported: - // if the output starts with a \n, then treat the open line as closed and - // just keep going after that. - vm.openLine = nil - output = output[1:] - vm.lastOpenLineUpdate = time.Time{} - } - if sameAsLast && len(vm.openLine) > 0 { - // Prettiness optimization: if there is an open line and the previous print out - // was of the same vertex, then use ANSI control sequence to go up one line and - // keep writing there. - printOutput = append(printOutput, ansiUp...) - } - // Prepend the open line to the output. - printOutput = append(printOutput, vm.openLine...) - printOutput = append(printOutput, output...) - // Look for the last \n to update the open line. - lastNewLine := bytes.LastIndexByte(printOutput, '\n') - if lastNewLine != -1 { - // Ends up being empty slice if output ends in \n. - vm.openLine = printOutput[(lastNewLine + 1):] - // A \n exists - reset the open line timer. - vm.lastOpenLineUpdate = time.Time{} - } else { - // No \n found - update vm.openLine to append the new output. - vm.openLine = append(vm.openLine, output...) - } - if !bytes.HasSuffix(printOutput, []byte{'\n'}) { - if vm.lastOpenLineUpdate.Add(durationBetweenOpenLineUpdate).After(time.Now()) { - // Skip printing if trying to update the same line too frequently. - vm.lastOpenLineSkipped = true - return nil - } - vm.lastOpenLineUpdate = time.Now() - // If output doesn't terminate in \n, add our own. - printOutput = append(printOutput, '\n') - } - vm.lastOpenLineSkipped = false - vm.console.PrintBytes(printOutput) - return nil -} - -func (vm *vertexMonitor) shouldPrintProgress(id string, percent int, verbose bool, sameAsLast bool) bool { - if !vm.headerPrinted { - return false - } - if !verbose && !ansiSupported { - for prefix := range internalProgress { - if strings.HasPrefix(id, prefix) { - return false - } - } - } - minDelta := durationBetweenProgressUpdate - if sameAsLast && ansiSupported { - minDelta = durationBetweenProgressUpdateIfSame - } else if strings.HasPrefix(id, "sha256:") || strings.HasPrefix(id, "extracting sha256:") { - // These progress updates are a bit more annoying - do them more rarely. - minDelta = durationBetweenSha256ProgressUpdate - } - now := time.Now() - lastProgress := vm.lastProgress[id] - lastPercentage := -1 - lastPercentageStored, ok := vm.lastPercentage[id] - if ok { - lastPercentage = lastPercentageStored - } - if now.Sub(lastProgress) < minDelta && percent < 100 { - return false - } - if lastPercentage == percent { - return false - } - vm.lastProgress[id] = now - vm.lastPercentage[id] = percent - return true -} - -func (vm *vertexMonitor) printProgress(id string, progress int, verbose bool, sameAsLast bool) { - builder := make([]string, 0, 2) - if sameAsLast { - // Overwrite previous line if this update is for the same thing as the previous one. - builder = append(builder, string(ansiUp)) - } - progressBar := progressbar.ProgressBar(progress, 10) - builder = append(builder, fmt.Sprintf("[%s] %3d%% %s%s\n", progressBar, progress, id, string(ansiEraseRestLine))) - vm.console.PrintBytes([]byte(strings.Join(builder, ""))) -} - -var reErrExitCode = regexp.MustCompile(`^process (".*") did not complete successfully: exit code: ([0-9]+)$`) -var reErrNotFound = regexp.MustCompile(`^failed to calculate checksum of ref ([^ ]*): (.*)$`) - -func (vm *vertexMonitor) printError() bool { - isFatal := false - errString := vm.vertex.Error - indentOp := strings.Join(strings.Split(vm.operation, "\n"), "\n ") - internalStr := "" - if vm.meta.Internal { - internalStr = " internal" - } - switch { - case reErrExitCode.MatchString(errString): - m := reErrExitCode.FindStringSubmatch(errString) - - // Ignore the Error, default case will print it as a string using the source, so we won't miss any data. - exitCode, _ := strconv.ParseUint(m[2], 10, 32) - - switch exitCode { - case math.MaxUint32: - errString = fmt.Sprintf(""+ - " The%s command\n"+ - " %s\n"+ - " was terminated because the build system ran out of memory.\n"+ - " If you are using a satellite or other remote buildkit, it is the remote system that ran out of memory.", - internalStr, indentOp) - default: - errString = fmt.Sprintf(""+ - " The%s command\n"+ - " %s\n"+ - " did not complete successfully. Exit code %s", - internalStr, indentOp, m[2]) - } - - isFatal = true - case reErrNotFound.MatchString(errString): - m := reErrNotFound.FindStringSubmatch(errString) - errString = fmt.Sprintf(""+ - " The%s command\n"+ - " %s\n"+ - " failed: %s", - internalStr, indentOp, m[2]) - isFatal = true - case errString == "no active sessions": - errString = "Canceled" - default: - errString = fmt.Sprintf( - "The%s command '%s' failed: %s", internalStr, vm.operation, errString) - } - slString := "" - if vm.meta.SourceLocation != nil { - slString = fmt.Sprintf( - " %s:%d:%d", - vm.meta.SourceLocation.File, vm.meta.SourceLocation.StartLine, - vm.meta.SourceLocation.StartColumn) - } - if isFatal { - vm.console.Warnf("ERROR%s\n%s\n", slString, errString) - } else { - vm.console.Printf("WARN%s: %s\n", slString, errString) - } - vm.console.VerbosePrintf("Overriding args used: %s\n", vm.meta.OverridingArgsString()) - return isFatal -} - -func (vm *vertexMonitor) printTimingInfo() { - if vm.vertex.Started == nil || vm.vertex.Completed == nil { - return - } - vm.console.WithMetadataMode(true). - Printf("Completed in %s\n", vm.vertex.Completed.Sub(*vm.vertex.Started)) -} - -func (vm *vertexMonitor) isOngoing() bool { - return vm.vertex.Started != nil && vm.vertex.Completed == nil && !vm.isError -} - -func (vm *vertexMonitor) reportStatusToConsole() { - vm.console.MarkBundleBuilderStatus(vm.vertex.Started != nil, vm.vertex.Completed != nil, vm.isCanceled) -} - -func (vm *vertexMonitor) reportResultToConsole() { - vm.console.MarkBundleBuilderResult(vm.isError, vm.isCanceled) -} diff --git a/release/Earthfile b/release/Earthfile index b8f3e21caf..73fa73b05e 100644 --- a/release/Earthfile +++ b/release/Earthfile @@ -40,6 +40,17 @@ perform-release-dockerhub: ARG DOCKERHUB_USER="earthly" ARG DOCKERHUB_IMG="earthly" ARG DOCKERHUB_BUILDKIT_IMG="buildkitd" + BUILD +perform-release-earthly-dockerhub + BUILD +perform-release-buildkitd-dockerhub + +perform-release-earthly-dockerhub: + ARG --required RELEASE_TAG + ARG PUSH_LATEST_TAG="false" + ARG PUSH_PRERELEASE_TAG="false" + ARG DOCKERHUB_USER="earthly" + ARG DOCKERHUB_IMG="earthly" + ARG DOCKERHUB_BUILDKIT_IMG="buildkitd" + ARG BUILDKIT_PROJECT BUILD \ --platform=linux/amd64 \ --platform=linux/arm64 \ @@ -48,14 +59,25 @@ perform-release-dockerhub: --DOCKERHUB_USER="$DOCKERHUB_USER" \ --DOCKERHUB_IMG="$DOCKERHUB_IMG" \ --PUSH_LATEST_TAG="$PUSH_LATEST_TAG" \ - --PUSH_PRERELEASE_TAG="$PUSH_PRERELEASE_TAG" + --PUSH_PRERELEASE_TAG="$PUSH_PRERELEASE_TAG" \ + --BUILDKIT_PROJECT="$BUILDKIT_PROJECT" + +perform-release-buildkitd-dockerhub: + ARG --required RELEASE_TAG + ARG PUSH_LATEST_TAG="false" + ARG PUSH_PRERELEASE_TAG="false" + ARG DOCKERHUB_USER="earthly" + ARG DOCKERHUB_IMG="earthly" + ARG DOCKERHUB_BUILDKIT_IMG="buildkitd" + ARG BUILDKIT_PROJECT BUILD \ --platform=linux/amd64 \ --platform=linux/arm64 \ ../buildkitd+buildkitd \ --TAG="$RELEASE_TAG" \ --DOCKERHUB_USER="$DOCKERHUB_USER" \ - --DOCKERHUB_BUILDKIT_IMG="$DOCKERHUB_BUILDKIT_IMG" + --DOCKERHUB_BUILDKIT_IMG="$DOCKERHUB_BUILDKIT_IMG" \ + --BUILDKIT_PROJECT="$BUILDKIT_PROJECT" release-notes: FROM ..+changelog-parser diff --git a/release/README.md b/release/README.md index ed49fc7fcf..0578b8ccc7 100644 --- a/release/README.md +++ b/release/README.md @@ -18,7 +18,7 @@ git checkout main && git pull ``` * Update the CHANGELOG.md with the corresponding release notes and open a PR - * Use a comparison such as https://github.com/earthly/earthly/compare/v0.8.6...main (replace the versions in the URL with the previously released version) or a tool such as `gitk` (aka `git-gui`) to see which PRs will go into this release. + * Use a comparison such as https://github.com/earthly/earthly/compare/v0.8.13...main (replace the versions in the URL with the previously released version) or a tool such as `gitk` (aka `git-gui`) to see which PRs will go into this release. * Make sure that main build is green for all platforms (check build status for the latest commit on GitHub). * Make sure the following build status are green: | Platform | Status | @@ -109,13 +109,13 @@ We currently have syntax highlighting for the following: 1. [emacs](https://github.com/earthly/earthly-emacs) -#### VSCode + Github +#### VSCode + GitHub Release instructions can be found in the [project repo](https://github.com/earthly/earthfile-grammar#how-to-release). #### Intellij -Intellij pulls its syntax highlighting from the [same repo used by VSCODE + Github](https://github.com/earthly/earthfile-grammar) and so should be released after to keep up to date. +Intellij pulls its syntax highlighting from the [same repo used by VSCODE + GitHub](https://github.com/earthly/earthfile-grammar) and so should be released after to keep up to date. 1. Go to the [repo](https://github.com/earthly/earthfile-grammar) 1. Make relevant changes to the branches + test in this order: diff --git a/release/release.sh b/release/release.sh index 0bbe4799e2..75601853ed 100755 --- a/release/release.sh +++ b/release/release.sh @@ -71,6 +71,7 @@ export GITHUB_SECRET_PATH=$GITHUB_SECRET_PATH export PRERELEASE=${PRERELEASE:-true} export SKIP_CHANGELOG_DATE_TEST=${SKIP_CHANGELOG_DATE_TEST:-false} export S3_BUCKET=${S3_BUCKET:-production-pkg} +export EARTHLY_STAGING=${EARTHLY_STAGING:-false} if [ "$PRERELEASE" != "false" ] && [ "$PRERELEASE" != "true" ]; then @@ -133,14 +134,39 @@ if [ "$PRERELEASE" = "true" ] || [ "$PRODUCTION_RELEASE" != "true" ]; then PUSH_LATEST_TAG="false" fi -"$earthly" --push --build-arg DOCKERHUB_USER --build-arg DOCKERHUB_IMG --build-arg DOCKERHUB_BUILDKIT_IMG --build-arg RELEASE_TAG +release-dockerhub --PUSH_PRERELEASE_TAG="$PRERELEASE" --PUSH_LATEST_TAG="$PUSH_LATEST_TAG" -"$earthly" --push --build-arg GITHUB_USER --build-arg EARTHLY_REPO --build-arg BREW_REPO --build-arg DOCKERHUB_USER --build-arg DOCKERHUB_BUILDKIT_IMG --build-arg RELEASE_TAG --build-arg SKIP_CHANGELOG_DATE_TEST --build-arg PRERELEASE $GITHUB_SECRET_PATH_BUILD_ARG +release-github +GITHUB_PRERELEASE="$PRERELEASE" +if [ "$EARTHLY_STAGING" = "true" ]; then + # special case to ensure https://github.com/earthly/earthly-staging/releases/latest/download/earthly-linux-amd64 is kept up to date + GITHUB_PRERELEASE="false" + + # make sure we aren't accidentally doing a regular release + if [ "$PUSH_LATEST_TAG" = "true" ]; then + echo "something is wrong; PUSH_LATEST_TAG should be false" + exit 1 + fi +fi + +earthlynext="$(cat ../earthly-next)" +if [[ ! "$earthlynext" =~ ^[a-zA-Z0-9]{40}$ ]]; then + echo "../earthly-next does not contain a valid git commit hash; got $earthlynext" + exit 1 +fi +echo "earthlynext is $earthlynext" + +"$earthly" --push --build-arg DOCKERHUB_USER --build-arg DOCKERHUB_IMG --build-arg DOCKERHUB_BUILDKIT_IMG +release-dockerhub --PUSH_PRERELEASE_TAG="$PRERELEASE" --PUSH_LATEST_TAG="$PUSH_LATEST_TAG" --RELEASE_TAG="$RELEASE_TAG" +"$earthly" --push --build-arg DOCKERHUB_USER --build-arg DOCKERHUB_IMG --build-arg DOCKERHUB_BUILDKIT_IMG +release-dockerhub --PUSH_PRERELEASE_TAG="false" --PUSH_LATEST_TAG="false" --RELEASE_TAG="$RELEASE_TAG-ticktock" --BUILDKIT_PROJECT=github.com/earthly/buildkit:$earthlynext +"$earthly" --push --build-arg GITHUB_USER --build-arg EARTHLY_REPO --build-arg BREW_REPO --build-arg DOCKERHUB_USER --build-arg DOCKERHUB_BUILDKIT_IMG --build-arg RELEASE_TAG --build-arg SKIP_CHANGELOG_DATE_TEST $GITHUB_SECRET_PATH_BUILD_ARG +release-github --PRERELEASE="$GITHUB_PRERELEASE" if [ "$PRERELEASE" != "false" ]; then echo "exiting due to PRERELEASE=$PRERELEASE" exit 0 fi +if [ "$EARTHLY_STAGING" = "true" ]; then + echo "exiting due to EARTHLY_STAGING=$EARTHLY_STAGING" + exit 0 +fi + echo "homebrew release with gu=$GITHUB_USER; er=$EARTHLY_REPO; br=$BREW_REPO; du=$DOCKERHUB_USER; rt=$RELEASE_TAG" "$earthly" --push --build-arg GITHUB_USER --build-arg EARTHLY_REPO --build-arg BREW_REPO --build-arg DOCKERHUB_USER --build-arg RELEASE_TAG $GITHUB_SECRET_PATH_BUILD_ARG +release-homebrew diff --git a/scripts/tests/backwards-compatability.sh b/scripts/tests/backwards-compatability.sh new file mode 100755 index 0000000000..ef4a37f0a0 --- /dev/null +++ b/scripts/tests/backwards-compatability.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -xeu + +# used to start earthly-integration-buildkitd +earthly="${earthly:=earthly}" +if [ -f "$earthly" ]; then + earthly=$(realpath "$earthly") +fi + +# used for testing backwards compatability issues +crustly="${crustly:=earthly-v0.8.0}" +if [ -f "$crustly" ]; then + crustly=$(realpath "$crustly") +fi + +# change directory to script location +cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" + +current_git_sha="$(git rev-parse HEAD)" + +if "$("$crustly" --version)" | grep "$current_git_sha" >/dev/null; then + echo "ERROR: $crustly was built using the current git sha $current_git_sha" + exit 1 +fi + +echo "running tests with earthly=$earthly for bootstrapping and crustly=$crustly for cli" +echo "earthly=$("$earthly" --version)" +echo "crustly=$("$crustly" --version)" +frontend="${frontend:-$(which docker || which podman)}" +test -n "$frontend" || (>&2 echo "Error: frontend is empty" && exit 1) +echo "using frontend=$frontend" + +PATH="$(realpath ../acbtest):$PATH" + +# prevent the self-update of earthly from running (this ensures no bogus data is printed to stdout, +# which would mess with the secrets data being fetched) +date +%s > /tmp/last-earthly-prerelease-check + +set +x # dont remove or the token will be leaked +if [ -z "${EARTHLY_TOKEN:-}" ]; then + echo "using EARTHLY_TOKEN from earthly secrets" + EARTHLY_TOKEN="$(earthly secrets --org earthly-technologies --project core get earthly-token-for-satellite-tests)" + export EARTHLY_TOKEN +fi +test -n "$EARTHLY_TOKEN" || (echo "error: EARTHLY_TOKEN is not set" && exit 1) +set -x + +EARTHLY_INSTALLATION_NAME="earthly-integration" +export EARTHLY_INSTALLATION_NAME +rm -rf "$HOME/.earthly.integration/" + +echo "$earthly" +# ensure earthly login works (and print out who gets logged in) +"$earthly" account login + +# start buildkitd container +"$earthly" bootstrap + +# start a build using an older version of the earthly cli +"$crustly" --no-buildkit-update -P ../../tests/with-docker+all + +# validate buildkitd container was compiled using the current branch +buildkitd_earthly_version="$(docker logs earthly-integration-buildkitd |& grep -o 'EARTHLY_GIT_HASH=[a-z0-9]*')" +acbtest "$buildkitd_earthly_version" = "EARTHLY_GIT_HASH=$current_git_sha" + +echo "=== All tests have passed ===" diff --git a/tests/Earthfile b/tests/Earthfile index 0af6209e47..f7bf96103f 100644 --- a/tests/Earthfile +++ b/tests/Earthfile @@ -215,7 +215,8 @@ ga-no-qemu-group11: ga-no-qemu-group12: BUILD --pass-args ./warn-if-not-logged-in+test BUILD --pass-args ./with-docker-validate-labels+all - + BUILD --pass-args +test-aws-oidc + BUILD --pass-args +run-no-cache-save-artifact ga-no-qemu-slow: BUILD +server @@ -254,6 +255,7 @@ tests-that-require-earthly-technologies-account-access: BUILD --pass-args ./account+test BUILD --pass-args ./registry-command+test BUILD --pass-args ./web+test + BUILD --pass-args ./oidc+test # tests that only run on linux amd64 # Note: this target is used to validate the USERPLATFORM user arg, @@ -1059,6 +1061,26 @@ run-no-cache: DO +RUN_EARTHLY --earthfile=run-no-cache.earth --use_tmpfs=false --target=+test-from \ --grep_flags="-v" --output_contains="\\\*cached\\\* --> .*motd2 \\\.\\\//" +run-no-cache-save-artifact: + # based on https://github.com/earthly/earthly/issues/2593#issuecomment-1777657111 + COPY run-no-cache-save-artifact.earth Earthfile + RUN touch some-file + LET numruns=10 + RUN echo "#!/bin/sh +set -ex +for x in \$(seq 1 $numruns); do + earthly --config \$earthly_config --artifact '+build/*' test/\$x/ +done +" >/tmp/test-earthly-script && chmod +x /tmp/test-earthly-script + ENV EARTHLY_EXEC_CMD="/tmp/test-earthly-script" + RUN --privileged \ + --entrypoint \ + --mount=type=tmpfs,target=/tmp/earthly \ + -- -P + # test that all output files are different + RUN cat test/*/dep1 + RUN acbtest "$(cat test/*/dep1 | sort | uniq | wc -l)" = "$numruns" + save-artifact-after-push: # test that save after push only outputs files before the RUN --push DO +RUN_EARTHLY --earthfile=save-artifact-after-push.earth --target=+test @@ -1624,6 +1646,17 @@ test-aws-flag-envs: RUN cat earthly.output | acbgrep "AWS_REGION=us-west-1" RUN cat earthly.output | acbgrep "AWS_SECRET_ACCESS_KEY=aws-secret-key" +# test-aws-oidc tests expected errors for misusing of oidc flag +# for happy path test go to /tests/oidc +test-aws-oidc: + DO +RUN_EARTHLY --earthfile=aws-flag.earth --target=+oidc --should_fail=true --output_contains="RUN --aws-oidc requires the --run-with-aws-oidc feature flag" + # enable flag + RUN sed -i "1s/VERSION \(.*\)/VERSION --run-with-aws-oidc \1/" Earthfile + # empty oidc flag + DO +RUN_EARTHLY --target=+oidc --should_fail=true --output_contains="role-arn must be specified" + # invalid oidc flag + DO +RUN_EARTHLY --target=+oidc --extra_args="--build-arg OIDC=\"foo=bar\"" --should_fail=true --output_contains="invalid value for oidc flag: 1 error(s) decoding" + test-aws-flag-configs: RUN mkdir -p /root/.aws RUN echo "[default] diff --git a/tests/aws-flag.earth b/tests/aws-flag.earth index 94e80f5e9e..b31d5b8956 100644 --- a/tests/aws-flag.earth +++ b/tests/aws-flag.earth @@ -1,6 +1,12 @@ VERSION --run-with-aws 0.8 +PROJECT earthly-technologies/core + FROM alpine basic: RUN --aws env | grep AWS + +oidc: + ARG OIDC="" + RUN --aws --oidc=$OIDC echo this should not succeed diff --git a/tests/oidc/Earthfile b/tests/oidc/Earthfile new file mode 100644 index 0000000000..b0ab8823bd --- /dev/null +++ b/tests/oidc/Earthfile @@ -0,0 +1,39 @@ +VERSION 0.8 +PROJECT earthly-technologies/core + +FROM --pass-args ..+base + +IMPORT .. AS tests + +WORKDIR /test + +test: + BUILD +test-aws + +test-aws: + BUILD +test-aws-success + BUILD +test-aws-failure + +test-aws-base: + COPY test-aws.sh . + COPY aws.earth Earthfile + ENV EARTHLY_EXEC_CMD=/test/test-aws.sh + +test-aws-success: + FROM +test-aws-base + # happy path + ARG ROLE_ARN="arn:aws:iam::404851345508:role/oidc-ci-test" + RUN --no-cache --secret OIDC_USER_TOKEN=test-oidc-user/token \ + --mount=type=tmpfs,target=/tmp/earthly \ + --privileged \ + --entrypoint + +test-aws-failure: + FROM +test-aws-base + # expect failure when arn role does not exist/not set + ARG ROLE_ARN="arn:aws:iam::123456789012:role/does-not-exist" + RUN --secret OIDC_USER_TOKEN=test-oidc-user/token \ + --mount=type=tmpfs,target=/tmp/earthly \ + --privileged \ + --entrypoint \ + 2>&1 |grep "Help: make sure the role \"arn:aws:iam::123456789012:role/does-not-exist\" has a valid trust policy configured in AWS" diff --git a/tests/oidc/aws.earth b/tests/oidc/aws.earth new file mode 100644 index 0000000000..e25967b268 --- /dev/null +++ b/tests/oidc/aws.earth @@ -0,0 +1,15 @@ +VERSION --run-with-aws --run-with-aws-oidc 0.8 + +PROJECT other-service+oidc-ci-test/my-project + +oidc: + FROM alpine + # set baseline - expects no envs with AWS_ prefix + LET expected=0 + RUN export result=$(env |grep AWS_ |wc -l); \ + test $result -eq $expected || (echo "expected \$expected env vars for AWS but got $result" && exit 1) + SET expected=4 + ARG --required ROLE_ARN + LET OIDC="role-arn=$ROLE_ARN,session-name=earthly-ci-test-session,region=us-west-2" + RUN --aws --oidc=$OIDC export result=$(env |grep AWS_ |wc -l); \ + test $result -eq $expected || (echo "expected $expected env vars for AWS but got $result" && exit 1) diff --git a/tests/oidc/test-aws.sh b/tests/oidc/test-aws.sh new file mode 100755 index 0000000000..fd9bd26827 --- /dev/null +++ b/tests/oidc/test-aws.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eo pipefail # DONT add a set -x or you will leak the key + +acbtest -n "$OIDC_USER_TOKEN" +acbtest -n "$ROLE_ARN" +acbtest -n "$earthly_config" # set by earthly-entrypoint.sh + + +echo "== it should login to user with token ==" +EARTHLY_TOKEN="$OIDC_USER_TOKEN" earthly account login 2>&1 | acbgrep 'Logged in as "other-service+oidc-ci-test@earthly.dev" using token auth' + +echo "== it should access aws via oidc ==" +earthly --config "$earthly_config" +oidc --ROLE_ARN="$ROLE_ARN" diff --git a/tests/run-no-cache-save-artifact.earth b/tests/run-no-cache-save-artifact.earth new file mode 100644 index 0000000000..0ce4839fb3 --- /dev/null +++ b/tests/run-no-cache-save-artifact.earth @@ -0,0 +1,20 @@ +VERSION 0.8 + +FROM alpine:3.18 +WORKDIR /work-der-work + +build: + COPY +dep2/out dep2 + COPY +dep1/out dep1 + RUN rm dep2 + SAVE ARTIFACT dep1 + +dep1: + COPY some-file . # required to trigger bug + COPY +dep2/out dep2 + RUN cp dep2 out + SAVE ARTIFACT out + +dep2: + RUN --no-cache head -c 20 /dev/urandom | base64 > out + SAVE ARTIFACT out diff --git a/util/containerutil/frontend.go b/util/containerutil/frontend.go index cab0aa8adc..9ec0fe8baf 100644 --- a/util/containerutil/frontend.go +++ b/util/containerutil/frontend.go @@ -44,8 +44,8 @@ type FrontendConfig struct { LocalRegistryHostFileValue string - InstallationName string - DefaultPort int + LocalContainerName string + DefaultPort int Console conslogging.ConsoleLogger } diff --git a/util/containerutil/settings_test.go b/util/containerutil/settings_test.go index 9f3991bff3..eda567ddf0 100644 --- a/util/containerutil/settings_test.go +++ b/util/containerutil/settings_test.go @@ -36,7 +36,7 @@ func TestBuildArgMatrix(t *testing.T) { }, noopArgs, results{ - buildkit: "docker-container://test-buildkitd", + buildkit: "docker-container://test", localRegistry: "", }, }, @@ -122,7 +122,7 @@ func TestBuildArgMatrix(t *testing.T) { logger = logger.WithWriter(&logs) frontend, err := NewStubFrontend(ctx, &FrontendConfig{ - InstallationName: "test-stub", + LocalContainerName: "test-stub", }) assert.NoError(t, err) @@ -133,7 +133,7 @@ func TestBuildArgMatrix(t *testing.T) { BuildkitHostCLIValue: tt.args.buildkit, BuildkitHostFileValue: tt.config.BuildkitHost, LocalRegistryHostFileValue: tt.config.LocalRegistryHost, - InstallationName: "test", + LocalContainerName: "test", DefaultPort: 8372, Console: logger, }) @@ -189,7 +189,7 @@ func TestBuildArgMatrixValidationFailures(t *testing.T) { logger = logger.WithWriter(&logs) frontend, err := NewStubFrontend(ctx, &FrontendConfig{ - InstallationName: "test-stub", + LocalContainerName: "test-stub", }) assert.NoError(t, err) @@ -200,7 +200,7 @@ func TestBuildArgMatrixValidationFailures(t *testing.T) { BuildkitHostFileValue: tt.config.BuildkitHost, LocalRegistryHostFileValue: tt.config.LocalRegistryHost, Console: logger, - InstallationName: "test", + LocalContainerName: "test", DefaultPort: 8372, }) assert.ErrorIs(t, err, tt.expected) @@ -290,7 +290,7 @@ func TestBuildArgMatrixValidationNonIssues(t *testing.T) { logger = logger.WithWriter(&logs) frontend, err := NewStubFrontend(ctx, &FrontendConfig{ - InstallationName: "test-stub", + LocalContainerName: "test-stub", }) assert.NoError(t, err) @@ -301,7 +301,7 @@ func TestBuildArgMatrixValidationNonIssues(t *testing.T) { BuildkitHostFileValue: tt.config.BuildkitHost, LocalRegistryHostFileValue: tt.config.LocalRegistryHost, Console: logger, - InstallationName: "test", + LocalContainerName: "test", DefaultPort: 8372, }) assert.NoError(t, err) diff --git a/util/containerutil/shell_shared.go b/util/containerutil/shell_shared.go index 1ca9dab1b9..0b62653ae4 100644 --- a/util/containerutil/shell_shared.go +++ b/util/containerutil/shell_shared.go @@ -358,7 +358,7 @@ func (sf *shellFrontend) setupAndValidateAddresses(feType string, cfg *FrontendC calculatedBuildkitHost = cfg.BuildkitHostFileValue } else { var err error - calculatedBuildkitHost, err = DefaultAddressForSetting(feType, cfg.InstallationName, cfg.DefaultPort) + calculatedBuildkitHost, err = DefaultAddressForSetting(feType, cfg.LocalContainerName, cfg.DefaultPort) if err != nil { return nil, errors.Wrap(err, "could not validate default address") } @@ -394,16 +394,16 @@ func (sf *shellFrontend) setupAndValidateAddresses(feType string, cfg *FrontendC } // DefaultAddressForSetting returns an address (signifying the desired/default transport) for a given frontend specified by setting. -func DefaultAddressForSetting(setting string, installationName string, defaultPort int) (string, error) { +func DefaultAddressForSetting(setting string, localContainerName string, defaultPort int) (string, error) { switch setting { case FrontendDockerShell: - return fmt.Sprintf(DockerAddressFmt, installationName), nil + return DockerSchemePrefix + localContainerName, nil case FrontendPodmanShell: return fmt.Sprintf(TCPAddressFmt, defaultPort), nil // Right now, podman only works over TCP. There are weird errors when trying to use the provided helper from buildkit. case FrontendStub: - return fmt.Sprintf(DockerAddressFmt, installationName), nil // Maintain old behavior + return DockerSchemePrefix + localContainerName, nil // Maintain old behavior } return "", fmt.Errorf("no default buildkit address for %s", setting) diff --git a/util/containerutil/types.go b/util/containerutil/types.go index b2acdcf5ed..c05c2d59f5 100644 --- a/util/containerutil/types.go +++ b/util/containerutil/types.go @@ -194,12 +194,8 @@ const ( // TCPAddressFmt is the address at which the daemon is available when using TCP. TCPAddressFmt = "tcp://127.0.0.1:%d" - // DockerAddressFmt is the address at which the daemon is available when using a Docker Container directly - DockerAddressFmt = "docker-container://%s-buildkitd" - - // PodmanAddressFmt is the address at which the daemon is available when using a Podman Container directly. - // Currently unused due to image export issues - PodmanAddressFmt = "podman-container://%s-buildkitd" + // DockerSchemePrefix is used to construct the buildkit address for local docker-based connections + DockerSchemePrefix = "docker-container://" // SatelliteAddress is the remote address when using a Satellite to execute your builds remotely. SatelliteAddress = "tcp://satellite.earthly.dev:8372" diff --git a/util/deltautil/go.mod b/util/deltautil/go.mod index 9b310513dc..dcd0a82a57 100644 --- a/util/deltautil/go.mod +++ b/util/deltautil/go.mod @@ -3,7 +3,7 @@ module github.com/earthly/earthly/util/deltautil go 1.21 require ( - github.com/earthly/cloud-api v1.0.1-0.20240216175649-9c937bc41efb + github.com/earthly/cloud-api v1.0.1-0.20240508215807-a958f373126f google.golang.org/protobuf v1.33.0 ) diff --git a/util/deltautil/go.sum b/util/deltautil/go.sum index a6350b1048..0a469fb4b7 100644 --- a/util/deltautil/go.sum +++ b/util/deltautil/go.sum @@ -1,5 +1,5 @@ -github.com/earthly/cloud-api v1.0.1-0.20240216175649-9c937bc41efb h1:lYdDcDqqEQ7QNNU3BPr8Pfs8M8caOfa8ctZvByNTc1Y= -github.com/earthly/cloud-api v1.0.1-0.20240216175649-9c937bc41efb/go.mod h1:rU/tYJ7GFBjdKAITV2heDbez++glpGSbtJaZcp73rNI= +github.com/earthly/cloud-api v1.0.1-0.20240508215807-a958f373126f h1:8CXT0MQ7dQrtm/IwVIexffosImh4ht0WUiWQAt0UoeQ= +github.com/earthly/cloud-api v1.0.1-0.20240508215807-a958f373126f/go.mod h1:rU/tYJ7GFBjdKAITV2heDbez++glpGSbtJaZcp73rNI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= diff --git a/util/fsutilprogress/progress.go b/util/fsutilprogress/progress.go index 6ad8e71508..eec9ef4a45 100644 --- a/util/fsutilprogress/progress.go +++ b/util/fsutilprogress/progress.go @@ -89,7 +89,7 @@ func (s *progressCallback) Verbose(relPath string, status fsutil.VerboseProgress } s.console.Printf("sent %s (%s)%s\n", humanize.Bytes(uint64(s.bytesSent)), puralize(s.numSent, "file"), transferRate) } else { - s.console.Printf("sent %s)\n", puralize(s.numStats, "file stat")) + s.console.Printf("sent %s\n", puralize(s.numStats, "file stat")) } if s.numReceived > 0 { var transferRate string diff --git a/util/llbutil/secretprovider/aws_creds.go b/util/llbutil/secretprovider/aws_creds.go index dfb84dce57..5a16af8d3b 100644 --- a/util/llbutil/secretprovider/aws_creds.go +++ b/util/llbutil/secretprovider/aws_creds.go @@ -4,11 +4,20 @@ import ( "context" "net/url" "strings" + "sync" + "time" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/config" "github.com/moby/buildkit/session/secrets" + "github.com/moby/buildkit/util/grpcerrors" "github.com/pkg/errors" + "google.golang.org/grpc/codes" + + "github.com/earthly/earthly/cloud" + "github.com/earthly/earthly/util/hint" + "github.com/earthly/earthly/util/oidcutil" ) // Internal reserved credentials names used to acquire the equivalent values @@ -18,6 +27,11 @@ const ( awsSecretKey = "aws:secret_key" awsSessionToken = "aws:session_token" awsRegion = "aws:region" + + roleARNURLParam = "role-arn" + regionURLParam = "region" + sessionDurationURLParam = "session-duration" + sessionNameURLParam = "session-name" ) // AWSCredentials contains the basic set of credentials that users will need to @@ -36,6 +50,9 @@ var awsEnvNames = map[string]string{ awsRegion: "AWS_REGION", } +var oidcCredsProviderCache = make(map[string]*aws.Config) +var oidcCredsProviderCacheMU sync.Mutex + // AWSEnvName converts and internal AWS secret name to the equivalent official // environmental variable. func AWSEnvName(name string) (string, bool) { @@ -43,12 +60,16 @@ func AWSEnvName(name string) (string, bool) { return envName, ok } -// AWSCredentialProvider can load AWS settings from the environment. -type AWSCredentialProvider struct{} +// AWSCredentialProvider can load AWS settings from the environment or oidc provider +type AWSCredentialProvider struct { + client *cloud.Client +} // NewAWSCredentialProvider creates and returns a credential provider for AWS. -func NewAWSCredentialProvider() *AWSCredentialProvider { - return &AWSCredentialProvider{} +func NewAWSCredentialProvider(client *cloud.Client) *AWSCredentialProvider { + return &AWSCredentialProvider{ + client: client, + } } // GetSecret attempts to find an AWS secret from either the environment or a local config file. @@ -60,21 +81,23 @@ func (c *AWSCredentialProvider) GetSecret(ctx context.Context, name string) ([]b } secretName := q.Get("name") + orgName := q.Get("org") + projectName := q.Get("project") // This provider only deals with secrets prefixed with "aws:". if !strings.HasPrefix(secretName, "aws:") { return nil, secrets.ErrNotFound } - // Note: results of this call are cached. - cfg, err := config.LoadDefaultConfig(ctx, config.WithDefaultsMode(aws.DefaultsModeStandard)) + oidcInfo := oidcInfoFromValues(q) + cfg, err := getCFG(ctx, orgName, projectName, oidcInfo, c.client) if err != nil { - return nil, errors.Wrap(err, "failed to load AWS config") + return nil, err } - creds, err := cfg.Credentials.Retrieve(ctx) - if err != nil { - return nil, errors.Wrap(err, "failed to load AWS credentials") + + if err = handleError(err, oidcInfo.RoleARNString(), cfg.Region, orgName, projectName); err != nil { + return nil, err } var val string @@ -92,10 +115,134 @@ func (c *AWSCredentialProvider) GetSecret(ctx context.Context, name string) ([]b return nil, errors.Errorf("unexpected secret: %s", secretName) } - if val == "" { + if val == "" && secretName != awsRegion { + // the region may be provided by a separate arg/env if it's not provided in the local env or oidc configuration // Use a custom error here as not to fall back on other secret providers. return nil, errors.Errorf("AWS setting %s not found in environment", secretName) } return []byte(val), nil } + +func oidcInfoFromValues(values url.Values) *oidcutil.AWSOIDCInfo { + roleARN := values.Get(roleARNURLParam) + if roleARN == "" { // no arn implies oidc is not in play + return nil + } + region := values.Get(regionURLParam) + sessionDuration := values.Get(sessionDurationURLParam) + // the values are pre validated in the interperter + parsedARN, _ := arn.Parse(roleARN) + var duration *time.Duration + if sessionDuration != "" { + durVal, _ := time.ParseDuration(sessionDuration) + duration = &durVal + } + return &oidcutil.AWSOIDCInfo{ + RoleARN: &parsedARN, + Region: region, + SessionDuration: duration, + SessionName: values.Get(sessionNameURLParam), + } +} + +type oidcCredentialsProvider struct { + client *cloud.Client + cache *aws.Credentials + oidcInfo *oidcutil.AWSOIDCInfo + orgName string + projectName string + cacheMU sync.Mutex +} + +func (p *oidcCredentialsProvider) Retrieve(ctx context.Context) (aws.Credentials, error) { + if p.cache != nil { + return *p.cache, nil + } + p.cacheMU.Lock() + defer p.cacheMU.Unlock() + if p.cache != nil { + return *p.cache, nil + } + res, err := p.client.GetAWSCredentials(ctx, p.oidcInfo.SessionName, p.oidcInfo.RoleARN.String(), p.orgName, p.projectName, p.oidcInfo.Region, p.oidcInfo.SessionDuration) + if err != nil { + return aws.Credentials{}, err + } + p.cache = &aws.Credentials{ + AccessKeyID: res.GetCredentials().GetAccessKeyId(), + SecretAccessKey: res.GetCredentials().GetSecretAccessKey(), + SessionToken: res.GetCredentials().GetSessionToken(), + CanExpire: true, + Expires: res.GetCredentials().GetExpiry().AsTime().UTC(), + } + return *p.cache, nil +} + +// getCFG returns a configuration that can provide credentials and region +// The cfg is either host environment based (e.g. ~/.aws) or oidc based. +// When based on oidc, it would get a session token from the cloud and cache the result. +// Caching is done so that next calls of GetSecret would get the rest of the matching credentials keys +func getCFG(ctx context.Context, orgName string, projectName string, oidcInfo *oidcutil.AWSOIDCInfo, client *cloud.Client) (aws.Config, error) { + if oidcInfo == nil { + // no oidc info implies getting the secrets from the host environment + // Note: results of this call are cached. + cfg, err := config.LoadDefaultConfig(ctx, config.WithDefaultsMode(aws.DefaultsModeStandard)) + if err != nil { + return aws.Config{}, errors.Wrap(err, "failed to load AWS config") + } + return cfg, nil + } + // check if we already have a config for the specified oidc info + key := oidcInfo.String() + if cfg, ok := oidcCredsProviderCache[key]; ok { + return *cfg, nil + } + // check one more time, this time with a lock + oidcCredsProviderCacheMU.Lock() + defer oidcCredsProviderCacheMU.Unlock() + if cfg, ok := oidcCredsProviderCache[key]; ok { + return *cfg, nil + } + cfg := &aws.Config{ + Region: oidcInfo.Region, + Credentials: &oidcCredentialsProvider{ + client: client, + oidcInfo: oidcInfo, + orgName: orgName, + projectName: projectName, + }, + } + oidcCredsProviderCache[key] = cfg + return *cfg, nil +} + +// SetURLValuesFunc returs a function that takes url.Values and sets oidc values. +// This is used by SecretID() to be able to identify secrets from this provider +func SetURLValuesFunc(awsInfo *oidcutil.AWSOIDCInfo) func(values url.Values) { + return func(values url.Values) { + values.Set(sessionNameURLParam, awsInfo.SessionName) + values.Set(roleARNURLParam, awsInfo.RoleARN.String()) + values.Set(regionURLParam, awsInfo.Region) + if awsInfo.SessionDuration != nil { + values.Set(sessionDurationURLParam, awsInfo.SessionDuration.String()) + } + } +} + +func handleError(err error, roleARN, region, orgName, projectName string) error { + if err == nil { + return nil + } + if grpcErr, ok := grpcerrors.AsGRPCStatus(err); ok { + switch grpcErr.Code() { + case codes.InvalidArgument: + if strings.Contains(grpcErr.Message(), "could not be found") { + return hint.Wrapf(err, `do the org %q and project %q exist`, orgName, projectName) + } + return hint.Wrapf(err, `is %q a valid AWS region?`, region) + case codes.PermissionDenied, codes.FailedPrecondition: + return hint.Wrapf(err, `make sure the role %q has a valid trust policy configured in AWS`, roleARN) + } + } + return errors.Wrap(err, "failed to load AWS credentials") +} diff --git a/util/oidcutil/aws.go b/util/oidcutil/aws.go new file mode 100644 index 0000000000..c5568e5a86 --- /dev/null +++ b/util/oidcutil/aws.go @@ -0,0 +1,152 @@ +package oidcutil + +import ( + "errors" + "fmt" + "reflect" + "slices" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/earthly/earthly/util/parseutil" + "github.com/mitchellh/mapstructure" +) + +type AWSOIDCInfo struct { + RoleARN *arn.ARN `mapstructure:"role-arn"` + SessionName string `mapstructure:"session-name"` + Region string `mapstructure:"region"` + SessionDuration *time.Duration `mapstructure:"session-duration"` +} + +var requiredFields = []string{"role-arn", "session-name"} +var decodeCFGTemplate = mapstructure.DecoderConfig{ + DecodeHook: mapstructure.ComposeDecodeHookFunc( + timeDurationValidationsHookFunc(func(input time.Duration) error { + if input.Seconds() < 900 || input.Seconds() > 43200 { + return errors.New("duration must be between 900s and 43200s") + } + return nil + }), + stringToARN(func(input *arn.ARN) error { + if input.Service != "iam" { + return fmt.Errorf(`aws service ("%s") must be "iam"`, input.Service) + } + if !strings.HasPrefix(input.Resource, "role/") { + return fmt.Errorf(`resource ("%s") must be an aws role"`, input.Resource) + } + return nil + }), + ), + WeaklyTypedInput: true, +} + +func newDecodeCFG(result interface{}, metadata *mapstructure.Metadata, template mapstructure.DecoderConfig) *mapstructure.DecoderConfig { + res := template + res.Result = result + res.Metadata = metadata + return &res +} +func (oi *AWSOIDCInfo) String() string { + if oi == nil { + return "" + } + sb := strings.Builder{} + if oi.SessionName != "" { + sb.WriteString(fmt.Sprintf("session-name=%s", oi.SessionName)) + } + if oi.RoleARN != nil { + sb.WriteString(fmt.Sprintf(",role-arn=%s", oi.RoleARN.String())) + } + if oi.Region != "" { + sb.WriteString(fmt.Sprintf(",region=%s", oi.Region)) + } + if oi.SessionDuration != nil { + sb.WriteString(fmt.Sprintf(",session-duration=%s", oi.SessionDuration.String())) + } + return strings.TrimPrefix(sb.String(), ",") +} + +func (oi *AWSOIDCInfo) RoleARNString() string { + if oi != nil && oi.RoleARN != nil { + return oi.RoleARN.String() + } + return "" +} + +// ParseAWSOIDCInfo takes a string that represents a list of oidc key/value pairs and returns it +// in the form of a *AWSOIDCInfo. The function errors if the string is invalid, including unexpected keys and/or values. +func ParseAWSOIDCInfo(oidcInfo string) (*AWSOIDCInfo, error) { + m, err := parseutil.StringToMap(oidcInfo) + if err != nil { + return nil, fmt.Errorf("oidc info is invalid: %w", err) + } + info := &AWSOIDCInfo{} + metadata := &mapstructure.Metadata{} + decodeCFG := newDecodeCFG(info, metadata, decodeCFGTemplate) + decoder, _ := mapstructure.NewDecoder(decodeCFG) + if err := decoder.Decode(m); err != nil { + return nil, err + } + if len(metadata.Unused) > 0 { + return nil, &mapstructure.Error{Errors: []string{fmt.Sprintf("key(s) [%s] are invalid", strings.Join(metadata.Unused, ","))}} + } + for _, f := range requiredFields { + if slices.Contains(metadata.Unset, f) { + return nil, &mapstructure.Error{Errors: []string{fmt.Sprintf("%s must be specified", f)}} + } + } + return info, nil +} + +func stringToARN(validators ...func(input *arn.ARN) error) mapstructure.DecodeHookFunc { + return func( + f reflect.Type, + t reflect.Type, + data interface{}) (interface{}, error) { + if f.Kind() != reflect.String { + return data, nil + } + if t != reflect.TypeOf(arn.ARN{}) { + return data, nil + } + + res, err := arn.Parse(data.(string)) + if err != nil { + return nil, err + } + for _, validator := range validators { + if err := validator(&res); err != nil { + return nil, err + } + } + return &res, nil + } +} + +func timeDurationValidationsHookFunc(validators ...func(input time.Duration) error) mapstructure.DecodeHookFunc { + return func( + f reflect.Type, + t reflect.Type, + data interface{}) (interface{}, error) { + if f.Kind() != reflect.String { + return data, nil + } + if t != reflect.TypeOf(time.Duration(5)) { + return data, nil + } + + // Convert it by parsing + parsed, err := time.ParseDuration(data.(string)) + if err != nil { + return nil, err + } + for _, validator := range validators { + if err := validator(parsed); err != nil { + return nil, err + } + } + return parsed, nil + } +} diff --git a/util/oidcutil/aws_test.go b/util/oidcutil/aws_test.go new file mode 100644 index 0000000000..a1474a047b --- /dev/null +++ b/util/oidcutil/aws_test.go @@ -0,0 +1,215 @@ +package oidcutil + +import ( + "errors" + "fmt" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/mitchellh/mapstructure" + "github.com/stretchr/testify/assert" +) + +func TestAWSOIDCInfoString(t *testing.T) { + tests := map[string]struct { + subject *AWSOIDCInfo + expected string + }{ + "happy path - nil": {}, + "happy path - when everything is set": { + subject: &AWSOIDCInfo{ + RoleARN: &arn.ARN{ + Service: "iam", + Region: "us-east-1", + Resource: "role/123", + }, + Region: "us-west-2", + SessionDuration: aws.Duration(time.Second), + SessionName: "my-session", + }, + expected: "session-name=my-session,role-arn=arn::iam:us-east-1::role/123,region=us-west-2,session-duration=1s", + }, + "happy path - no role-arn": { + subject: &AWSOIDCInfo{ + Region: "us-west-2", + SessionDuration: aws.Duration(time.Second), + SessionName: "my-session", + }, + expected: "session-name=my-session,region=us-west-2,session-duration=1s", + }, + "happy path - no region": { + subject: &AWSOIDCInfo{ + RoleARN: &arn.ARN{ + Service: "iam", + Region: "us-east-1", + Resource: "role/123", + }, + SessionDuration: aws.Duration(time.Second), + SessionName: "my-session", + }, + expected: "session-name=my-session,role-arn=arn::iam:us-east-1::role/123,session-duration=1s", + }, + "happy path - no session duration": { + subject: &AWSOIDCInfo{ + RoleARN: &arn.ARN{ + Service: "iam", + Region: "us-east-1", + Resource: "role/123", + }, + Region: "us-west-2", + SessionName: "my-session", + }, + expected: "session-name=my-session,role-arn=arn::iam:us-east-1::role/123,region=us-west-2", + }, + "happy path - no session name": { + subject: &AWSOIDCInfo{ + RoleARN: &arn.ARN{ + Service: "iam", + Region: "us-east-1", + Resource: "role/123", + }, + Region: "us-west-2", + SessionDuration: aws.Duration(time.Second), + }, + expected: "role-arn=arn::iam:us-east-1::role/123,region=us-west-2,session-duration=1s", + }, + } + + for name, tc := range tests { + name, tc := name, tc + t.Run(name, func(t *testing.T) { + t.Parallel() + res := tc.subject.String() + assert.Equal(t, tc.expected, res) + }) + } +} + +func TestAWSOIDCInfoRoleARNString(t *testing.T) { + tests := map[string]struct { + subject *AWSOIDCInfo + expected string + }{ + "struct is nil": {}, + "role arn is nil": { + subject: &AWSOIDCInfo{}, + }, + "role arn is not nil": { + subject: &AWSOIDCInfo{ + RoleARN: &arn.ARN{ + Service: "iam", + Region: "us-east-1", + Resource: "role/123", + }, + }, + expected: "arn::iam:us-east-1::role/123", + }, + } + + for name, tc := range tests { + name, tc := name, tc + t.Run(name, func(t *testing.T) { + t.Parallel() + res := tc.subject.RoleARNString() + assert.Equal(t, tc.expected, res) + }) + } +} + +func TestParseAWSOIDCInfo(t *testing.T) { + tests := map[string]struct { + input string + expected *AWSOIDCInfo + expectedErr error + }{ + "error when string is invalid": { + input: "invalid string", + expectedErr: fmt.Errorf("oidc info is invalid: %w", errors.New("key/value must be set with =")), + }, + "error when duration is invalid": { + input: "session-duration=invalid", + expectedErr: &mapstructure.Error{Errors: []string{`error decoding 'session-duration': time: invalid duration "invalid"`}}, + }, + "error when duration is less than 900s": { + input: "role-arn=arn::iam::123:role/456,session-duration=899s", + expectedErr: &mapstructure.Error{Errors: []string{`error decoding 'session-duration': duration must be between 900s and 43200s`}}, + }, + "error when duration is more than 43200s": { + input: "role-arn=arn::iam::123:role/456,session-duration=43201s", + expectedErr: &mapstructure.Error{Errors: []string{`error decoding 'session-duration': duration must be between 900s and 43200s`}}, + }, + "error when session name is missing": { + input: "role-arn=arn::iam::123:role/456,region=us-west-2,session-duration=900s", + expectedErr: &mapstructure.Error{Errors: []string{"session-name must be specified"}}, + }, + "error when role arn is missing": { + input: "session-duration=902s", + expectedErr: &mapstructure.Error{Errors: []string{"role-arn must be specified"}}, + }, + "error when role arn is invalid": { + input: "role-arn=invalid", + expectedErr: &mapstructure.Error{Errors: []string{`error decoding 'role-arn': arn: invalid prefix`}}, + }, + "error when role arn is not iam service": { + input: "role-arn=arn::kinesis:us-east-1::role/123", + expectedErr: &mapstructure.Error{Errors: []string{`error decoding 'role-arn': aws service ("kinesis") must be "iam"`}}, + }, + "error when role arn resource is not a role": { + input: "role-arn=arn::iam:us-east-1::user/123", + expectedErr: &mapstructure.Error{Errors: []string{`error decoding 'role-arn': resource ("user/123") must be an aws role"`}}, + }, + "error when using unrecognized keys": { + input: "role-arn=arn::iam:us-east-1::role/123,session-name=my-session,foo=bar", + expectedErr: &mapstructure.Error{Errors: []string{"key(s) [foo] are invalid"}}, + }, + "happy path": { + input: "role-arn=arn::iam::123:role/456,region=us-west-2,session-duration=900s,session-name=my-session", + expected: &AWSOIDCInfo{ + RoleARN: &arn.ARN{ + Service: "iam", + AccountID: "123", + Resource: "role/456", + }, + Region: "us-west-2", + SessionDuration: aws.Duration(time.Second * 900), + SessionName: "my-session", + }, + }, + "happy path - no region": { + input: "role-arn=arn::iam::123:role/456,session-duration=900s,session-name=my-session", + expected: &AWSOIDCInfo{ + RoleARN: &arn.ARN{ + Service: "iam", + AccountID: "123", + Resource: "role/456", + }, + SessionDuration: aws.Duration(time.Second * 900), + SessionName: "my-session", + }, + }, + "happy path - no session duration": { + input: "role-arn=arn::iam::123:role/456,region=us-west-2,session-name=my-session", + expected: &AWSOIDCInfo{ + RoleARN: &arn.ARN{ + Service: "iam", + AccountID: "123", + Resource: "role/456", + }, + Region: "us-west-2", + SessionName: "my-session", + }, + }, + } + + for name, tc := range tests { + name, tc := name, tc + t.Run(name, func(t *testing.T) { + t.Parallel() + res, err := ParseAWSOIDCInfo(tc.input) + assert.Equal(t, tc.expectedErr, err) + assert.Equal(t, tc.expected, res) + }) + } +} diff --git a/util/parseutil/map.go b/util/parseutil/map.go new file mode 100644 index 0000000000..b36c0c074b --- /dev/null +++ b/util/parseutil/map.go @@ -0,0 +1,24 @@ +package parseutil + +import ( + "errors" + "strings" +) + +// StringToMap expects to get a string in the form of key1=val1,key2=val2,... +// and returns a map with the keys and values +func StringToMap(str string) (map[string]string, error) { + pairs := strings.Split(str, ",") + kvp := make(map[string]string, len(pairs)) + for _, pair := range pairs { + if strings.TrimSpace(pair) == "" { + continue + } + k, v, ok := strings.Cut(pair, "=") + if !ok { + return nil, errors.New("key/value must be set with =") + } + kvp[strings.TrimSpace(k)] = strings.TrimSpace(v) + } + return kvp, nil +} diff --git a/util/parseutil/map_test.go b/util/parseutil/map_test.go new file mode 100644 index 0000000000..41d1675804 --- /dev/null +++ b/util/parseutil/map_test.go @@ -0,0 +1,49 @@ +package parseutil + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStringToMap(t *testing.T) { + tests := map[string]struct { + input string + expected map[string]string + expectedErr error + }{ + "happy path": { + input: "key1=val1,key2= val2 , key3 =val3 ,", + expected: map[string]string{ + "key1": "val1", + "key2": "val2", + "key3": "val3", + }, + }, + "happy path - empty map": { + input: " ", + expected: map[string]string{}, + }, + "happy path - single value": { + input: "key1=val1", + expected: map[string]string{ + "key1": "val1", + }, + }, + "error when no equal sign": { + input: "key1=val1,key2 val2 , key3 =val3 ,", + expectedErr: errors.New("key/value must be set with ="), + }, + } + + for name, tc := range tests { + name, tc := name, tc + t.Run(name, func(t *testing.T) { + t.Parallel() + res, err := StringToMap(tc.input) + assert.Equal(t, tc.expectedErr, err) + assert.Equal(t, tc.expected, res) + }) + } +} diff --git a/util/stringutil/scrub.go b/util/stringutil/scrub.go index b5dc584206..a000bda0a7 100644 --- a/util/stringutil/scrub.go +++ b/util/stringutil/scrub.go @@ -29,3 +29,10 @@ func ScrubCredentialsAll(s string) string { } return strings.Join(ret, " ") } + +// ScrubANSICodes removes ANSI escape codes from a string. +func ScrubANSICodes(input string) string { + re := regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`) + cleanedString := re.ReplaceAllString(input, "") + return cleanedString +} diff --git a/util/stringutil/scrub_test.go b/util/stringutil/scrub_test.go index a0af76cb8c..b327cc2b48 100644 --- a/util/stringutil/scrub_test.go +++ b/util/stringutil/scrub_test.go @@ -18,3 +18,8 @@ func TestScrubInline(t *testing.T) { s := ScrubCredentialsAll("Here is a URL: https://user:password@github.com/org/repo.git") Equal(t, "Here is a URL: https://user:xxxxx@github.com/org/repo.git", s) } + +func TestANSICodes(t *testing.T) { + s := ScrubANSICodes("\033[0;32mCommand succeeded.\033[0m") + Equal(t, "Command succeeded.", s) +}