From 6b32f473515d3545579f74e593a85c89a02b201b Mon Sep 17 00:00:00 2001 From: Brian Muse Date: Wed, 5 Apr 2023 13:45:39 -0700 Subject: [PATCH] Update the existing Build Images workflow to build all FBPCS images and run on pull requests. (#2213) Summary: Pull Request resolved: https://github.com/facebookresearch/fbpcs/pull/2213 ## Context We have seen some issues slip through recently that have caused our B&R system to be unstable. Right now, the only testing that we do on diffs is to see if the onedocker image builds. This is insufficient to ensure that changes don't break our B&R system. The new flow for the "Build, Test, and Publish the FBPCS Docker Images" workflow will be to build ~~and test~~ every pull request or push to main ~~with the basic, non-GraphAPI end to end tests~~. For pull requests, it will block the landing of the diff until the ~~tests~~ builds pass. For merges to main, it will create a task and assign it to the person who pushed the commit. This will provide 2 benefits: 1. For pull requests, this adds a higher bar of testing at Diff time. This should block potential bugs from slipping through to block our release flow 2. For merges to main, this will make the tasks more accurate. Since it's running on every commit, it will know which commit likely broke the tests. Right now, it only runs for each bundle from conveyor which might include multiple changes and may not assign the task to the right developer. ## Workflow Design The Build, Test, and Publish the FBPCS Docker Images workflow follows these steps: 1. Build the Coordinator Image 2. In parallel, build the EMP Games and Data Processing images 1. When those are done, build the bundled OneDocker image 3. ~~Once all images are built successfully, run E2E tests~~ {F898944897} ## Updates for V15 I've removed the E2E testing because there is a tag conflict with the RC OneDocker image and will require more effort to include E2E testing. For now, this is still usefull as it makes the builds more like the production release process and includes the coordinator image. ## This Diff This diff updates the already existing workflow that would build images on pushes to main to also build on pull requests and to test the images with the local (non-GraphAPI) E2E tests. Differential Revision: D44229635 fbshipit-source-id: 3742ec2a4abb4f969aaa277f09d1615578269096 --- .github/workflows/build_binary_image.yml | 31 ++++--------- .github/workflows/build_fbpcs_images.yml | 56 +++++++++++++++++++++-- .github/workflows/coordinator-publish.yml | 46 +++++++++++++------ .github/workflows/docker-publish.yml | 4 +- docker/onedocker/test/Dockerfile.ubuntu | 6 ++- 5 files changed, 98 insertions(+), 45 deletions(-) diff --git a/.github/workflows/build_binary_image.yml b/.github/workflows/build_binary_image.yml index 3a660aa0c..bfe696921 100644 --- a/.github/workflows/build_binary_image.yml +++ b/.github/workflows/build_binary_image.yml @@ -31,7 +31,7 @@ on: env: DISTRO: ubuntu REGISTRY: ghcr.io - RC_REGISTRY_IMAGE_URL: ghcr.io/${{ github.repository }}/rc/ + RC_REGISTRY_URL: ghcr.io/${{ github.repository }}/rc/ jobs: build_image: @@ -44,17 +44,14 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Get Docker Image Name + id: get_docker_image_name + run: | + echo "image_tag_name=$(echo ${{ inputs.image_to_build }} | tr '_' '-')" >> $GITHUB_OUTPUT + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-${{ inputs.image_to_build }}-${{ github.ref_name }} - restore-keys: | - ${{ runner.os }}-${{ inputs.image_to_build }} - - name: Pull FBPCF Image run: docker pull ghcr.io/facebookresearch/fbpcf/ubuntu:${{ inputs.fbpcf_version }} @@ -71,22 +68,12 @@ jobs: context: . file: docker/${{ inputs.image_to_build }}/Dockerfile.ubuntu tags: | - ${{ env.RC_REGISTRY_IMAGE_URL }}${{ inputs.image_to_build }}:${{ github.sha }} + ${{ env.RC_REGISTRY_URL }}${{ steps.get_docker_image_name.outputs.image_tag_name }}:${{ github.sha }} build-args: | tag=latest os_release=20.04 fbpcf_image=ghcr.io/facebookresearch/fbpcf/ubuntu:${{ inputs.fbpcf_version }} push: ${{ inputs.push_to_registry }} load: ${{ inputs.load_image_locally }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - - # This ugly bit is necessary or else our cache will grow forever - # until it hits GitHub's limit of 5GB. - # Temp fix: T135482742 - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache + cache-from: type=gha,scope=${{ steps.get_docker_image_name.outputs.image_tag_name }}-${{ github.ref_name }} + cache-to: type=gha,scope=${{ steps.get_docker_image_name.outputs.image_tag_name }}-${{ github.ref_name }},mode=max diff --git a/.github/workflows/build_fbpcs_images.yml b/.github/workflows/build_fbpcs_images.yml index fc479764c..fc1418289 100644 --- a/.github/workflows/build_fbpcs_images.yml +++ b/.github/workflows/build_fbpcs_images.yml @@ -1,12 +1,18 @@ -name: Build and Publish Data Processing and EMP Games Docker Images +name: Build, Test, and Publish the FBPCS Docker Images on: push: branches: [ main ] + pull_request: + branches: [ main ] env: - FBPCF_VERSION: 2.1.144 # Please also update line 25 (FBPCF_VERSION) in .github/workflows/docker-publish.yml + FBPCF_VERSION: 2.1.144 # Please also update line 25 in .github/workflows/docker-publish.yml + PID_VERSION: 0.0.8 # Please also update line 26 in .github/workflows/docker-publish.yml REGISTRY: ghcr.io + LOCAL_IMAGE_NAME: fbpcs/onedocker/test + RC_REGISTRY_URL: ghcr.io/${{ github.repository }}/rc + RC_REGISTRY_IMAGE_NAME: ghcr.io/${{ github.repository }}/rc/onedocker jobs: output_version: @@ -18,6 +24,13 @@ jobs: - id: set_version run: echo "version=${{ env.FBPCF_VERSION }}" >> $GITHUB_OUTPUT + build_coordinator_image: + name: Build the Coordinator image + uses: ./.github/workflows/coordinator-publish.yml + with: + new_tag: ${{ github.sha }} + push_to_registry: ${{ github.event_name != pull_request || github.event.pull_request.head.repo.full_name == github.repository }} + build_and_publish_data_processing_image: name: Build and Publish Data Processing Image needs: output_version @@ -26,7 +39,7 @@ jobs: image_to_build: data_processing tag: latest fbpcf_version: ${{needs.output_version.outputs.fbpcf_version}} - push_to_registry: true + push_to_registry: ${{ github.event_name != pull_request || github.event.pull_request.head.repo.full_name == github.repository }} build_and_publish_emp_games_image: name: Build and Publish EMP Games Image @@ -36,4 +49,39 @@ jobs: image_to_build: emp_games tag: latest fbpcf_version: ${{needs.output_version.outputs.fbpcf_version}} - push_to_registry: true + push_to_registry: ${{ github.event_name != pull_request || github.event.pull_request.head.repo.full_name == github.repository }} + + build_test_onedocker_image: + name: Build the bundled test version of the onedocker image + needs: [build_and_publish_data_processing_image, build_and_publish_emp_games_image] + runs-on: ubuntu-latest + if: github.event_name != pull_request || github.event.pull_request.head.repo.full_name == github.repository + steps: + - uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build Image + uses: docker/build-push-action@v3 + with: + context: . + file: docker/onedocker/test/Dockerfile.ubuntu + tags: | + ${{ env.RC_REGISTRY_IMAGE_NAME }}:${{ github.sha }} + build-args: | + tag=${{ github.sha }} + os_release=20.04 + private_id_tag=${{ env.PID_VERSION }} + repository_url=${{ env.RC_REGISTRY_URL }} + push: true + pull: true + cache-from: type=gha,scope=onedocker-test-${{ github.ref_name }} + cache-to: type=gha,scope=onedocker-test-${{ github.ref_name }},mode=max diff --git a/.github/workflows/coordinator-publish.yml b/.github/workflows/coordinator-publish.yml index 29a820f15..809f619c9 100644 --- a/.github/workflows/coordinator-publish.yml +++ b/.github/workflows/coordinator-publish.yml @@ -3,18 +3,36 @@ name: Publish Coordinator Image on: workflow_dispatch: inputs: - name: - description: "Manually running this workflow to build a coordinator image" - default: "Run" new_tag: description: "The new tag of the docker image" required: false type: string default: latest-build + push_to_registry: + description: "Whether or not to push the image to the GitHub Container registry" + required: false + type: boolean + default: true tracker_hash: description: "[Internal usage] Used for tracking workflow job status within Meta infra" required: false - type: str + type: string + workflow_call: + inputs: + new_tag: + description: "The new tag of the docker image" + required: false + type: string + default: latest-build + push_to_registry: + description: "Whether or not to push the image to the GitHub Container registry" + required: false + type: boolean + default: true + tracker_hash: + description: "[Internal usage] Used for tracking workflow job status within Meta infra" + required: false + type: string env: DISTRO: ubuntu @@ -31,33 +49,31 @@ jobs: packages: write steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Print Tracker Hash - run: echo ${{ github.event.inputs.tracker_hash }} + run: echo ${{ inputs.tracker_hash }} - name: Build image run: | - docker build --build-arg FBPCS_BUNDLE_ID=${{ github.event.inputs.new_tag }} -f ./fbpcs/Dockerfile -t ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} . + docker build --build-arg FBPCS_BUNDLE_ID=${{ inputs.new_tag }} -f ./fbpcs/Dockerfile -t ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} . # Tests will be added here - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set output - id: vars - run: echo ::set-output name=ref::${GITHUB_REF##*/} - - name: Tag docker image run: | - docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.sha }} - docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ steps.vars.outputs.ref }} - docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} + docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.sha }} + ${{ ! contains(github.ref, 'refs/pull') }} && docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.ref_name }} + docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ inputs.new_tag }} - name: Push image to registry + if: inputs.push_to_registry == true run: | docker push --all-tags ${{ env.REGISTRY_IMAGE_NAME }} diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index db81ca75e..19ba367f8 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -22,8 +22,8 @@ env: PL_CONTAINER_NAME: e2e_pl_container PA_CONTAINER_NAME: e2e_pa_container TIME_RANGE: 24 hours - FBPCF_VERSION: 2.1.144 # Please also update line 8 in .github/workflows/build_fbpcs_images.yml - PID_VERSION: 0.0.9 + FBPCF_VERSION: 2.1.144 # Please also update line 10 in .github/workflows/build_fbpcs_images.yml + PID_VERSION: 0.0.8 # Please also update line 11 in .github/workflows/build_fbpcs_images.yml jobs: ### Build and publish rc/onedocker image diff --git a/docker/onedocker/test/Dockerfile.ubuntu b/docker/onedocker/test/Dockerfile.ubuntu index d9ac024ad..c7e5f0fcd 100644 --- a/docker/onedocker/test/Dockerfile.ubuntu +++ b/docker/onedocker/test/Dockerfile.ubuntu @@ -5,8 +5,10 @@ ARG os_release="20.04" ARG tag="latest" ARG private_id_tag="latest" -FROM fbpcs/data-processing:${tag} as data_processing -FROM fbpcs/emp-games:${tag} as emp_games +# Use the local repository as default +ARG repository_url="fbpcs" +FROM ${repository_url}/data-processing:${tag} as data_processing +FROM ${repository_url}/emp-games:${tag} as emp_games FROM ghcr.io/facebookresearch/private-id:${private_id_tag} as private_id FROM ubuntu:${os_release}