From ccad823f75a60d2bed6b01fde8b5c3d2b62f4be6 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Wed, 11 Jun 2025 13:00:05 +0200 Subject: [PATCH 1/5] Fix #908: Add reusable workflow to publish ingestion jobs --- actions/ingestion-job-publish/action.yaml | 119 ++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 actions/ingestion-job-publish/action.yaml diff --git a/actions/ingestion-job-publish/action.yaml b/actions/ingestion-job-publish/action.yaml new file mode 100644 index 00000000..57cc2927 --- /dev/null +++ b/actions/ingestion-job-publish/action.yaml @@ -0,0 +1,119 @@ +name: Build and Publish Docker Container +description: | + Build the ingestion job container image and push it to Google Artifact Registry (GAR). + Make sure the repository was added to the Terraform configuration on STAGE and PROD. + See https://github.com/mozilla-it/webservices-infra/blob/e3403053/remote-settings/tf/prod/main.tf#L56-L78 + + This workflow is meant to be used on main branch and on version tags like this: + + ```yaml + on: + pull_request: + branches: + - main + push: + branches: + - main + tags: + - v[0-9]+.[0-9]+.[0-9]+ + + jobs: + build-and-publish: + uses: mozilla/remote-settings/actions/ingestion-job-publish@main + with: + realm: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && 'prod' || 'nonprod' }} + publish: ${{ github.event_name != 'pull_request' }} + ``` + +on: + workflow_call: + inputs: + realm: + description: "Realm (nonprod or prod) to use for the GCP project and GAR repository." + default: nonprod + type: string + options: + - nonprod + - prod + required: true + publish: + description: "Whether to publish the image to GAR. Set to false for testing purposes." + default: true + type: boolean + + outputs: + image: + description: "The image tags that were built and pushed to GAR." + value: ${{ steps.meta.outputs.tags }} + digest: + description: "The digest of the pushed image" + value: ${{ steps.meta.outputs.digest }} + +env: + BUILDX_NO_DEFAULT_ATTESTATIONS: 1 # Reduce warnings from Docker Buildx + GAR_LOCATION: us + GCP_PROJECT_ID: moz-fx-remote-settings-${{ inputs.realm }} + GAR_REPOSITORY: ingestion-cronjob-${{ github.event.repository.name }} + GAR_IMAGE_NAME: ${{ github.event.repository.name }} + +jobs: + build-and-publish: + runs-on: ubuntu-latest + outputs: + image: ${{ steps.meta.outputs.tags }} + steps: + - name: Check out the repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch everything (tags) + fetch-tags: true + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.GAR_REPOSITORY }}/${{ env.GAR_IMAGE_NAME }} + # https://github.com/marketplace/actions/docker-metadata-action#tags-input + tags: | + type=sha,prefix=,enable={{ inputs.publish }} + type=semver,pattern={{raw}},enable=${{ inputs.publish }} + type=raw,value=latest,enable={{ inputs.publish }} + + - name: Set Service Account from Github repository + id: gcp-service-account + run: | + # Match how the service account is created in our Terraform code + # https://github.com/mozilla-it/webservices-infra/blob/main/remote-settings/tf/modules/remote_settings_infra/ingestion_jobs.tf + # Remove dashes and "remotesettings" from the repository name + cleaned_string=$(echo "${{ github.event.repository.name }}" | sed 's/-//g' | sed 's/remotesettings//g') + # Trim to 30 characters here too. + result=$(echo "ingest-job-${cleaned_string}" | cut -c 1-30) + echo "email=${result}@${{ env.GCP_PROJECT_ID }}.iam.gserviceaccount.com" >> "$GITHUB_OUTPUT" + + - id: gcp_auth + name: Log into GCP + uses: google-github-actions/auth@v2 + with: + token_format: access_token + service_account: ${{ steps.gcp-service-account.outputs.email }} + workload_identity_provider: ${{ vars.GCPV2_GITHUB_WORKLOAD_IDENTITY_PROVIDER }} + + - name: Login to GAR + uses: docker/login-action@v3 + with: + registry: ${{ env.GAR_LOCATION }}-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.gcp_auth.outputs.access_token }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.publish }} + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha,buildkit=true + cache-to: type=gha,mode=max,buildkit=true From 3a25e95f962a27f164f7ea9f7345f43bfebe6885 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Wed, 11 Jun 2025 17:54:34 +0200 Subject: [PATCH 2/5] Move reusable action to workflows folder --- .../action.yaml => .github/workflows/ingestion-job-publish.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename actions/ingestion-job-publish/action.yaml => .github/workflows/ingestion-job-publish.yaml (100%) diff --git a/actions/ingestion-job-publish/action.yaml b/.github/workflows/ingestion-job-publish.yaml similarity index 100% rename from actions/ingestion-job-publish/action.yaml rename to .github/workflows/ingestion-job-publish.yaml From 7a3303828ddeb69bc7822f4a2aa379b2af498e99 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Wed, 11 Jun 2025 19:24:54 +0200 Subject: [PATCH 3/5] Fix syntax errors --- .github/workflows/ingestion-job-publish.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ingestion-job-publish.yaml b/.github/workflows/ingestion-job-publish.yaml index 57cc2927..fc93f51b 100644 --- a/.github/workflows/ingestion-job-publish.yaml +++ b/.github/workflows/ingestion-job-publish.yaml @@ -21,8 +21,8 @@ description: | build-and-publish: uses: mozilla/remote-settings/actions/ingestion-job-publish@main with: - realm: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && 'prod' || 'nonprod' }} - publish: ${{ github.event_name != 'pull_request' }} + realm: $\{{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && 'prod' || 'nonprod' }} + publish: $\{{ github.event_name != 'pull_request' }} ``` on: @@ -32,9 +32,6 @@ on: description: "Realm (nonprod or prod) to use for the GCP project and GAR repository." default: nonprod type: string - options: - - nonprod - - prod required: true publish: description: "Whether to publish the image to GAR. Set to false for testing purposes." From c96baba7635c04e00d44d5611babfa4eaa6f5f33 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Thu, 12 Jun 2025 08:44:44 +0200 Subject: [PATCH 4/5] Fix lint --- .github/workflows/ingestion-job-publish.yaml | 61 ++++++++------------ 1 file changed, 25 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ingestion-job-publish.yaml b/.github/workflows/ingestion-job-publish.yaml index fc93f51b..34c8fc27 100644 --- a/.github/workflows/ingestion-job-publish.yaml +++ b/.github/workflows/ingestion-job-publish.yaml @@ -1,36 +1,35 @@ +# Build the ingestion job container image and push it to Google Artifact Registry (GAR). +# Make sure the repository was added to the Terraform configuration on STAGE and PROD. +# See https://github.com/mozilla-it/webservices-infra/blob/e3403053/remote-settings/tf/prod/main.tf#L56-L78 +# +# This workflow is meant to be used on main branch and on version tags like this: +# +# ```yaml +# on: +# pull_request: +# branches: +# - main +# push: +# branches: +# - main +# tags: +# - v[0-9]+.[0-9]+.[0-9]+ +# +# jobs: +# build-and-publish: +# uses: mozilla/remote-settings/actions/ingestion-job-publish@main +# with: +# realm: $\{{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && 'prod' || 'nonprod' }} +# publish: $\{{ github.event_name != 'pull_request' }} +# ``` +# name: Build and Publish Docker Container -description: | - Build the ingestion job container image and push it to Google Artifact Registry (GAR). - Make sure the repository was added to the Terraform configuration on STAGE and PROD. - See https://github.com/mozilla-it/webservices-infra/blob/e3403053/remote-settings/tf/prod/main.tf#L56-L78 - - This workflow is meant to be used on main branch and on version tags like this: - - ```yaml - on: - pull_request: - branches: - - main - push: - branches: - - main - tags: - - v[0-9]+.[0-9]+.[0-9]+ - - jobs: - build-and-publish: - uses: mozilla/remote-settings/actions/ingestion-job-publish@main - with: - realm: $\{{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && 'prod' || 'nonprod' }} - publish: $\{{ github.event_name != 'pull_request' }} - ``` on: workflow_call: inputs: realm: description: "Realm (nonprod or prod) to use for the GCP project and GAR repository." - default: nonprod type: string required: true publish: @@ -38,14 +37,6 @@ on: default: true type: boolean - outputs: - image: - description: "The image tags that were built and pushed to GAR." - value: ${{ steps.meta.outputs.tags }} - digest: - description: "The digest of the pushed image" - value: ${{ steps.meta.outputs.digest }} - env: BUILDX_NO_DEFAULT_ATTESTATIONS: 1 # Reduce warnings from Docker Buildx GAR_LOCATION: us @@ -56,8 +47,6 @@ env: jobs: build-and-publish: runs-on: ubuntu-latest - outputs: - image: ${{ steps.meta.outputs.tags }} steps: - name: Check out the repo uses: actions/checkout@v4 From a384fd400903786d50cc03e7cbf72b3b1482c014 Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Thu, 12 Jun 2025 09:07:10 +0200 Subject: [PATCH 5/5] Fix enable tags --- .github/workflows/ingestion-job-publish.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ingestion-job-publish.yaml b/.github/workflows/ingestion-job-publish.yaml index 34c8fc27..af2297ae 100644 --- a/.github/workflows/ingestion-job-publish.yaml +++ b/.github/workflows/ingestion-job-publish.yaml @@ -62,9 +62,9 @@ jobs: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.GAR_REPOSITORY }}/${{ env.GAR_IMAGE_NAME }} # https://github.com/marketplace/actions/docker-metadata-action#tags-input tags: | - type=sha,prefix=,enable={{ inputs.publish }} + type=sha,prefix=,enable=${{ inputs.publish }} type=semver,pattern={{raw}},enable=${{ inputs.publish }} - type=raw,value=latest,enable={{ inputs.publish }} + type=raw,value=latest,enable=${{ inputs.publish }} - name: Set Service Account from Github repository id: gcp-service-account