diff --git a/.github/workflows/build_binary_image.yml b/.github/workflows/build_binary_image.yml index 3a660aa0c..bfe696921 100644 --- a/.github/workflows/build_binary_image.yml +++ b/.github/workflows/build_binary_image.yml @@ -31,7 +31,7 @@ on: env: DISTRO: ubuntu REGISTRY: ghcr.io - RC_REGISTRY_IMAGE_URL: ghcr.io/${{ github.repository }}/rc/ + RC_REGISTRY_URL: ghcr.io/${{ github.repository }}/rc/ jobs: build_image: @@ -44,17 +44,14 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Get Docker Image Name + id: get_docker_image_name + run: | + echo "image_tag_name=$(echo ${{ inputs.image_to_build }} | tr '_' '-')" >> $GITHUB_OUTPUT + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-${{ inputs.image_to_build }}-${{ github.ref_name }} - restore-keys: | - ${{ runner.os }}-${{ inputs.image_to_build }} - - name: Pull FBPCF Image run: docker pull ghcr.io/facebookresearch/fbpcf/ubuntu:${{ inputs.fbpcf_version }} @@ -71,22 +68,12 @@ jobs: context: . file: docker/${{ inputs.image_to_build }}/Dockerfile.ubuntu tags: | - ${{ env.RC_REGISTRY_IMAGE_URL }}${{ inputs.image_to_build }}:${{ github.sha }} + ${{ env.RC_REGISTRY_URL }}${{ steps.get_docker_image_name.outputs.image_tag_name }}:${{ github.sha }} build-args: | tag=latest os_release=20.04 fbpcf_image=ghcr.io/facebookresearch/fbpcf/ubuntu:${{ inputs.fbpcf_version }} push: ${{ inputs.push_to_registry }} load: ${{ inputs.load_image_locally }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - - # This ugly bit is necessary or else our cache will grow forever - # until it hits GitHub's limit of 5GB. - # Temp fix: T135482742 - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache + cache-from: type=gha,scope=${{ steps.get_docker_image_name.outputs.image_tag_name }}-${{ github.ref_name }} + cache-to: type=gha,scope=${{ steps.get_docker_image_name.outputs.image_tag_name }}-${{ github.ref_name }},mode=max diff --git a/.github/workflows/build_fbpcs_images.yml b/.github/workflows/build_fbpcs_images.yml index fc479764c..fc1418289 100644 --- a/.github/workflows/build_fbpcs_images.yml +++ b/.github/workflows/build_fbpcs_images.yml @@ -1,12 +1,18 @@ -name: Build and Publish Data Processing and EMP Games Docker Images +name: Build, Test, and Publish the FBPCS Docker Images on: push: branches: [ main ] + pull_request: + branches: [ main ] env: - FBPCF_VERSION: 2.1.144 # Please also update line 25 (FBPCF_VERSION) in .github/workflows/docker-publish.yml + FBPCF_VERSION: 2.1.144 # Please also update line 25 in .github/workflows/docker-publish.yml + PID_VERSION: 0.0.8 # Please also update line 26 in .github/workflows/docker-publish.yml REGISTRY: ghcr.io + LOCAL_IMAGE_NAME: fbpcs/onedocker/test + RC_REGISTRY_URL: ghcr.io/${{ github.repository }}/rc + RC_REGISTRY_IMAGE_NAME: ghcr.io/${{ github.repository }}/rc/onedocker jobs: output_version: @@ -18,6 +24,13 @@ jobs: - id: set_version run: echo "version=${{ env.FBPCF_VERSION }}" >> $GITHUB_OUTPUT + build_coordinator_image: + name: Build the Coordinator image + uses: ./.github/workflows/coordinator-publish.yml + with: + new_tag: ${{ github.sha }} + push_to_registry: ${{ github.event_name != pull_request || github.event.pull_request.head.repo.full_name == github.repository }} + build_and_publish_data_processing_image: name: Build and Publish Data Processing Image needs: output_version @@ -26,7 +39,7 @@ jobs: image_to_build: data_processing tag: latest fbpcf_version: ${{needs.output_version.outputs.fbpcf_version}} - push_to_registry: true + push_to_registry: ${{ github.event_name != pull_request || github.event.pull_request.head.repo.full_name == github.repository }} build_and_publish_emp_games_image: name: Build and Publish EMP Games Image @@ -36,4 +49,39 @@ jobs: image_to_build: emp_games tag: latest fbpcf_version: ${{needs.output_version.outputs.fbpcf_version}} - push_to_registry: true + push_to_registry: ${{ github.event_name != pull_request || github.event.pull_request.head.repo.full_name == github.repository }} + + build_test_onedocker_image: + name: Build the bundled test version of the onedocker image + needs: [build_and_publish_data_processing_image, build_and_publish_emp_games_image] + runs-on: ubuntu-latest + if: github.event_name != pull_request || github.event.pull_request.head.repo.full_name == github.repository + steps: + - uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build Image + uses: docker/build-push-action@v3 + with: + context: . + file: docker/onedocker/test/Dockerfile.ubuntu + tags: | + ${{ env.RC_REGISTRY_IMAGE_NAME }}:${{ github.sha }} + build-args: | + tag=${{ github.sha }} + os_release=20.04 + private_id_tag=${{ env.PID_VERSION }} + repository_url=${{ env.RC_REGISTRY_URL }} + push: true + pull: true + cache-from: type=gha,scope=onedocker-test-${{ github.ref_name }} + cache-to: type=gha,scope=onedocker-test-${{ github.ref_name }},mode=max diff --git a/.github/workflows/coordinator-publish.yml b/.github/workflows/coordinator-publish.yml index 29a820f15..809f619c9 100644 --- a/.github/workflows/coordinator-publish.yml +++ b/.github/workflows/coordinator-publish.yml @@ -3,18 +3,36 @@ name: Publish Coordinator Image on: workflow_dispatch: inputs: - name: - description: "Manually running this workflow to build a coordinator image" - default: "Run" new_tag: description: "The new tag of the docker image" required: false type: string default: latest-build + push_to_registry: + description: "Whether or not to push the image to the GitHub Container registry" + required: false + type: boolean + default: true tracker_hash: description: "[Internal usage] Used for tracking workflow job status within Meta infra" required: false - type: str + type: string + workflow_call: + inputs: + new_tag: + description: "The new tag of the docker image" + required: false + type: string + default: latest-build + push_to_registry: + description: "Whether or not to push the image to the GitHub Container registry" + required: false + type: boolean + default: true + tracker_hash: + description: "[Internal usage] Used for tracking workflow job status within Meta infra" + required: false + type: string env: DISTRO: ubuntu @@ -31,33 +49,31 @@ jobs: packages: write steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Print Tracker Hash - run: echo ${{ github.event.inputs.tracker_hash }} + run: echo ${{ inputs.tracker_hash }} - name: Build image run: | - docker build --build-arg FBPCS_BUNDLE_ID=${{ github.event.inputs.new_tag }} -f ./fbpcs/Dockerfile -t ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} . + docker build --build-arg FBPCS_BUNDLE_ID=${{ inputs.new_tag }} -f ./fbpcs/Dockerfile -t ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} . # Tests will be added here - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set output - id: vars - run: echo ::set-output name=ref::${GITHUB_REF##*/} - - name: Tag docker image run: | - docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.sha }} - docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ steps.vars.outputs.ref }} - docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.event.inputs.new_tag }} + docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.sha }} + ${{ ! contains(github.ref, 'refs/pull') }} && docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ github.ref_name }} + docker tag ${{ env.LOCAL_IMAGE_NAME }}:${{ inputs.new_tag }} ${{ env.REGISTRY_IMAGE_NAME }}:${{ inputs.new_tag }} - name: Push image to registry + if: inputs.push_to_registry == true run: | docker push --all-tags ${{ env.REGISTRY_IMAGE_NAME }} diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index db81ca75e..19ba367f8 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -22,8 +22,8 @@ env: PL_CONTAINER_NAME: e2e_pl_container PA_CONTAINER_NAME: e2e_pa_container TIME_RANGE: 24 hours - FBPCF_VERSION: 2.1.144 # Please also update line 8 in .github/workflows/build_fbpcs_images.yml - PID_VERSION: 0.0.9 + FBPCF_VERSION: 2.1.144 # Please also update line 10 in .github/workflows/build_fbpcs_images.yml + PID_VERSION: 0.0.8 # Please also update line 11 in .github/workflows/build_fbpcs_images.yml jobs: ### Build and publish rc/onedocker image diff --git a/docker/onedocker/test/Dockerfile.ubuntu b/docker/onedocker/test/Dockerfile.ubuntu index d9ac024ad..c7e5f0fcd 100644 --- a/docker/onedocker/test/Dockerfile.ubuntu +++ b/docker/onedocker/test/Dockerfile.ubuntu @@ -5,8 +5,10 @@ ARG os_release="20.04" ARG tag="latest" ARG private_id_tag="latest" -FROM fbpcs/data-processing:${tag} as data_processing -FROM fbpcs/emp-games:${tag} as emp_games +# Use the local repository as default +ARG repository_url="fbpcs" +FROM ${repository_url}/data-processing:${tag} as data_processing +FROM ${repository_url}/emp-games:${tag} as emp_games FROM ghcr.io/facebookresearch/private-id:${private_id_tag} as private_id FROM ubuntu:${os_release}