Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Migrate to composite GHA actions (#11718)
This migrates from reusable workflows to composite GHA actions. This has the following advantages: 1) We can split them into smaller, easier to reason about chunks 2) We can reuse them more freely between each other and workflows 3) They don't complicate the job names 4) In theory they'll be easier to test in isolation. While composite actions can't be unit-tested, we can easily break them up into nodejs or docker actions that can be. As a proof of concept, some of our non-Bazel tests are migrated to GHA as well (CMake + Composer) Closes #11718 COPYBARA_INTEGRATE_REVIEW=#11718 from protocolbuffers:gha-actions 5403307 PiperOrigin-RevId: 506375417
- Loading branch information
1 parent
037f06d
commit 2b4e426
Showing
14 changed files
with
350 additions
and
162 deletions.
There are no files selected for viewing
Validating CODEOWNERS rules …
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| name: 'Docker Bazel Run' | ||
| description: 'Run a Bazel-based docker image for Protobuf CI testing' | ||
| inputs: | ||
| credentials: | ||
| required: true | ||
| description: "The GCP credentials to use for reading the docker image" | ||
| type: string | ||
| image: | ||
| required: false | ||
| default: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 | ||
| description: "The docker image to use" | ||
| type: string | ||
| bazel-cache: | ||
| required: false | ||
| description: > | ||
| A unique path for the Bazel cache. This will trigger the generation | ||
| of a BAZEL_CACHE environment variable inside the container that provides | ||
| the appropriate flags for any bazel command. | ||
| type: string | ||
| bazel: | ||
| required: false | ||
| description: "The Bazel command to run" | ||
| type: string | ||
| bash: | ||
| required: false | ||
| description: "A bash command to run. $BAZEL_FLAGS will be available to use for bazel runs." | ||
| type: string | ||
|
|
||
| runs: | ||
| using: 'composite' | ||
| steps: | ||
| - name: Authenticate | ||
| id: auth | ||
| uses: ./.github/actions/internal/docker-auth | ||
| with: | ||
| credentials: ${{ inputs.credentials }} | ||
|
|
||
| - name: Validate inputs | ||
| if: ${{ inputs.bash && inputs.bazel}} | ||
| shell: bash | ||
| run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1 | ||
|
|
||
| - name: Initialize Bazel flags | ||
| shell: bash | ||
| run: echo "BAZEL_FLAGS=--keep_going --test_output=errors --test_timeout=600" >> $GITHUB_ENV | ||
|
|
||
| - name: Configure Bazel caching | ||
| shell: bash | ||
| # Skip bazel cache for local act runs due to issue with credential files | ||
| # and nested docker images | ||
| if: ${{ inputs.bazel-cache && !github.event.act_local_test }} | ||
| run: > | ||
| echo "BAZEL_FLAGS=$BAZEL_FLAGS | ||
| --google_credentials=/workspace/$(basename ${{ steps.auth.outputs.credentials-file }}) | ||
| --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV | ||
| - name: Configure Bazel cache writing | ||
| shell: bash | ||
| # External PRs should never write to our caches. | ||
| if: ${{ github.event_name != 'pull_request_target' && inputs.bazel-cache && !github.event.act_local_test }} | ||
| run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --remote_upload_local_results" >> $GITHUB_ENV | ||
|
|
||
| - name: Run Bash Docker | ||
| uses: ./.github/actions/internal/docker-run | ||
| if: ${{ inputs.bash }} | ||
| with: | ||
| image: ${{ inputs.image }} | ||
| run-flags: --entrypoint "/bin/bash" | ||
| command: -l -c "${{ inputs.bash }}" | ||
|
|
||
| - name: Run Bazel Docker | ||
| uses: ./.github/actions/internal/docker-run | ||
| if: ${{ !inputs.bash }} | ||
| with: | ||
| image: ${{ inputs.image }} | ||
| command: ${{ inputs.bazel }} $BAZEL_FLAGS |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| name: 'Authenticate Docker' | ||
| description: 'Authenticate a workflow for Protobuf CI testing' | ||
| inputs: | ||
| credentials: | ||
| required: true | ||
| description: "The GCP credentials to use for reading the docker image" | ||
| type: string | ||
|
|
||
| outputs: | ||
| credentials-file: | ||
| description: "Credentials file generated for GCP" | ||
| value: ${{ steps.auth.outputs.credentials_file_path }} | ||
|
|
||
| runs: | ||
| using: 'composite' | ||
| steps: | ||
| - name: Authenticate to Google Cloud | ||
| id: auth | ||
| uses: google-github-actions/auth@ef5d53e30bbcd8d0836f4288f5e50ff3e086997d # v1.0.0 | ||
| with: | ||
| credentials_json: ${{ inputs.credentials }} | ||
| - name: Set up Cloud SDK | ||
| uses: google-github-actions/setup-gcloud@d51b5346f85640ec2aa2fa057354d2b82c2fcbce # v1.0.1 | ||
| - name: Use gcloud CLI | ||
| shell: bash | ||
| run: gcloud info | ||
| - name: Authenticate for GAR use | ||
| shell: bash | ||
| run: gcloud auth configure-docker -q us-docker.pkg.dev |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| name: 'Run Docker' | ||
| description: 'Run a docker image for Protobuf CI testing' | ||
| inputs: | ||
| image: | ||
| required: true | ||
| description: "The docker image to use" | ||
| type: string | ||
| command: | ||
| required: true | ||
| description: "A raw docker command to run" | ||
| type: string | ||
| run-flags: | ||
| required: false | ||
| description: "Additional flags to pass to docker run" | ||
| type: string | ||
|
|
||
| # WARNING: loading from cache appears to be slower than pull! | ||
| docker-cache: | ||
| required: false | ||
| description: "Enabled caching of pulled docker images." | ||
|
|
||
| runs: | ||
| using: 'composite' | ||
| steps: | ||
| - name: Setup QEMU for possible emulation | ||
| uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0 | ||
|
|
||
| - name: Check docker cache | ||
| if: ${{ inputs.docker-cache }} | ||
| id: check-docker-cache | ||
| uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 | ||
| with: | ||
| path: ci/docker/ | ||
| key: ${{ inputs.image }} | ||
|
|
||
| - name: Pull and store if cache miss | ||
| shell: bash | ||
| if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit != 'true' }} | ||
| run: > | ||
| time docker pull -q ${{ inputs.image }} && | ||
| mkdir -p ci/docker/$(dirname ${{ inputs.image }}) && | ||
| time docker image save ${{ inputs.image }} --output ./ci/docker/${{ inputs.image }}.tar | ||
| - name: Use the cached image on cache hit | ||
| shell: bash | ||
| if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit == 'true' }} | ||
| run: time docker image load --input ./ci/docker/${{ inputs.image }}.tar | ||
|
|
||
| - name: Pull fresh docker image | ||
| shell: bash | ||
| if: ${{ !inputs.docker-cache }} | ||
| run: time docker pull -q ${{ inputs.image }} | ||
|
|
||
| - name: Run docker | ||
| shell: bash | ||
| run: > | ||
| time docker run ${{ inputs.run-flags}} -v${{ github.workspace }}:/workspace | ||
| ${{ inputs.image }} ${{ inputs.command }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| name: 'Docker Non-Bazel Run' | ||
| description: 'Run a docker image for Protobuf CI testing with a non-Bazel build system' | ||
| inputs: | ||
| credentials: | ||
| required: true | ||
| description: "The GCP credentials to use for reading the docker image" | ||
| type: string | ||
| command: | ||
| required: true | ||
| description: A command to run in the docker image | ||
| image: | ||
| required: false | ||
| default: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 | ||
| description: "The docker image to use" | ||
| type: string | ||
|
|
||
| runs: | ||
| using: 'composite' | ||
| steps: | ||
| - name: Update stale files using Bazel | ||
| uses: ./.github/actions/bazel-docker | ||
| with: | ||
| image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 | ||
| credentials: ${{ inputs.credentials }} | ||
| bazel-cache: regenerate-stale-files | ||
| bash: ./regenerate_stale_files.sh $BAZEL_FLAGS | ||
|
|
||
| - name: Run Docker | ||
| uses: ./.github/actions/internal/docker-run | ||
| with: | ||
| image: ${{ inputs.image }} | ||
| command: ${{ inputs.command }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| name: PHP Tests | ||
|
|
||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| safe-checkout: | ||
| required: true | ||
| description: "The SHA key for the commit we want to run over" | ||
| type: string | ||
|
|
||
| jobs: | ||
| linux: | ||
| strategy: | ||
| fail-fast: false # Don't cancel all jobs if one fails. | ||
| matrix: | ||
| include: | ||
| - name: 7.3 Debug | ||
| version: 7.3.28-dbg | ||
| command: composer test \&\& composer test_c | ||
| - name: 7.4 Debug | ||
| version: 7.4.18-dbg | ||
| command: composer test \&\& composer test_c | ||
| - name: 8.0 Optimized | ||
| version: 8.0.5 | ||
| command: composer test \&\& composer test_c | ||
| - name: 8.0 Debug | ||
| version: 8.0.5-dbg | ||
| command: composer test \&\& composer test_c | ||
| - name: 8.0 Memory Leak | ||
| version: 8.0.5-dbg | ||
| # Run specialized memory leak & multirequest tests. | ||
| command: composer test_c \&\& tests/multirequest.sh \&\& tests/memory_leak_test.sh | ||
| - name: 8.0 Valgrind | ||
| version: 8.0.5-dbg | ||
| command: composer test_valgrind | ||
|
|
||
| name: Linux PHP ${{ matrix.name}} | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout pending changes | ||
| uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 | ||
| with: | ||
| submodules: recursive | ||
| ref: ${{ inputs.safe-checkout }} | ||
| - name: Run tests | ||
| uses: ./.github/actions/non-bazel-docker | ||
| with: | ||
| image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-6e95c0e221e4bd52e3b4dc1398c6336985196931 | ||
| credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} | ||
| command: ${{ matrix.command }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.