From 63ce3e489bd5521941b19f3a577961e8dad11963 Mon Sep 17 00:00:00 2001 From: Andrew Dye Date: Thu, 3 Nov 2022 10:41:21 -0700 Subject: [PATCH 01/10] Build arm64 images Signed-off-by: Andrew Dye --- .github/workflows/pythonpublish.yml | 75 ++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 22 deletions(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index b2619aaa52..40e1a7671f 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -1,6 +1,9 @@ name: Publish Python Package on: + # TODO: remove, used for testing manually + workflow_dispatch: + release: types: [published] @@ -65,31 +68,59 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: "0" - - name: Build & Push Flytekit Python${{ matrix.python-version }} Docker Image to Github Registry - uses: whoan/docker-build-with-cache-action@v5 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + - name: Login to GitHub Container Registry + if: ${{ github.event_name == 'release' }} + uses: docker/login-action@v1 with: - # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - image_name: ${{ github.repository_owner }}/flytekit - image_tag: py${{ matrix.python-version }}-latest,py${{ matrix.python-version }}-${{ github.sha }},py${{ matrix.python-version }}-${{ needs.deploy.outputs.version }} - push_git_tag: true - push_image_and_stages: true registry: ghcr.io - build_extra_args: "--compress=true --build-arg=VERSION=${{ needs.deploy.outputs.version }} --build-arg=DOCKER_IMAGE=ghcr.io/flyteorg/flytekit:py${{ matrix.python-version }}-${{ needs.deploy.outputs.version }} --build-arg=PYTHON_VERSION=${{ matrix.python-version }}" + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # username: "${{ secrets.FLYTE_BOT_USERNAME }}" + # password: "${{ secrets.FLYTE_BOT_PAT }}" + - name: Prepare Flytekit Image Names + id: flytekit-names + uses: docker/metadata-action@v3 + with: + images: | + ghcr.io/${{ github.repository_owner }}/flytekit + tags: | + py${{ matrix.python-version }}-latest + py${{ matrix.python-version }}-${{ github.sha }} + py${{ matrix.python-version }}-${{ needs.deploy.outputs.version }} + - name: Build & Push Flytekit Python${{ matrix.python-version }} Docker Image to Github Registry + uses: docker/build-push-action@v2 + with: context: . - dockerfile: Dockerfile + platforms: linux/arm64, linux/amd64 + push: ${{ github.event_name == 'release' }} + tags: ${{ steps.flytekit-names.outputs.tags }} + build-args: "--compress=true --build-arg=VERSION=${{ needs.deploy.outputs.version }} --build-arg=DOCKER_IMAGE=ghcr.io/flyteorg/flytekit:py${{ matrix.python-version }}-${{ needs.deploy.outputs.version }} --build-arg=PYTHON_VERSION=${{ matrix.python-version }}" + file: Dockerfile + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Prepare SQLAlchemy Image Names + id: sqlalchemy-names + uses: docker/metadata-action@v3 + with: + images: | + ghcr.io/${{ github.repository_owner }}/flytekit + tags: | + py${{ matrix.python-version }}-sqlalchemy-latest + py${{ matrix.python-version }}-sqlalchemy-${{ github.sha }} + py${{ matrix.python-version }}-sqlalchemy-${{ needs.deploy.outputs.version }} - name: Push SQLAlchemy Image to GitHub Registry - uses: whoan/docker-build-with-cache-action@v5 + uses: docker/build-push-action@v2 with: - # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - image_name: ${{ github.repository_owner }}/flytekit - image_tag: py${{ matrix.python-version }}-sqlalchemy-latest,py${{ matrix.python-version }}-sqlalchemy-${{ github.sha }},py${{ matrix.python-version }}-sqlalchemy-${{ needs.deploy.outputs.version }} - push_git_tag: true - push_image_and_stages: true - registry: ghcr.io - build_extra_args: "--compress=true --build-arg=VERSION=${{ needs.deploy.outputs.version }}" context: "./plugins/flytekit-sqlalchemy/" - dockerfile: Dockerfile.py${{ matrix.python-version }} + platforms: linux/arm64, linux/amd64 + push: ${{ github.event_name == 'release' }} + tags: ${{ steps.sqlalchemy-names.outputs.tags }} + build-args: "--compress=true --build-arg=VERSION=${{ needs.deploy.outputs.version }}" + file: Dockerfile.py${{ matrix.python-version }} + cache-from: type=gha + cache-to: type=gha,mode=max From 15c1a18114fdc1be64c14e347cbf8d842306eb19 Mon Sep 17 00:00:00 2001 From: Andrew Dye Date: Thu, 3 Nov 2022 10:49:09 -0700 Subject: [PATCH 02/10] Comment out build/plugins Signed-off-by: Andrew Dye --- .github/workflows/pythonpublish.yml | 50 ++++++++++++++--------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 40e1a7671f..336b38f6ca 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -30,31 +30,31 @@ jobs: echo "::set-output name=version::$VERSION" VERSION=$VERSION make update_version shell: bash - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* - - name: Autobump plugin version - run: | - # from refs/tags/v1.2.3 get 1.2.3 - VERSION=$(echo $GITHUB_REF | sed 's#.*/v##') - VERSION=$VERSION make -C plugins update_all_versions - shell: bash - - name: Build all Plugins and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - make -C plugins build_all_plugins - make -C plugins publish_all_plugins - # Added sleep because PYPI take some time in publish - - name: Sleep for 180 seconds - uses: jakejarvis/wait-action@master - with: - time: '180s' + # - name: Build and publish + # env: + # TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + # TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + # run: | + # python setup.py sdist bdist_wheel + # twine upload dist/* + # - name: Autobump plugin version + # run: | + # # from refs/tags/v1.2.3 get 1.2.3 + # VERSION=$(echo $GITHUB_REF | sed 's#.*/v##') + # VERSION=$VERSION make -C plugins update_all_versions + # shell: bash + # - name: Build all Plugins and publish + # env: + # TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + # TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + # run: | + # make -C plugins build_all_plugins + # make -C plugins publish_all_plugins + # # Added sleep because PYPI take some time in publish + # - name: Sleep for 180 seconds + # uses: jakejarvis/wait-action@master + # with: + # time: '180s' outputs: version: ${{ steps.bump.outputs.version }} From 2a1b1e8b1cab582de25d5dc2d6d3a57446f32961 Mon Sep 17 00:00:00 2001 From: Andrew Dye Date: Thu, 3 Nov 2022 11:02:16 -0700 Subject: [PATCH 03/10] Fix build args Signed-off-by: Andrew Dye --- .github/workflows/pythonpublish.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 336b38f6ca..012290e5cf 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -99,10 +99,14 @@ jobs: platforms: linux/arm64, linux/amd64 push: ${{ github.event_name == 'release' }} tags: ${{ steps.flytekit-names.outputs.tags }} - build-args: "--compress=true --build-arg=VERSION=${{ needs.deploy.outputs.version }} --build-arg=DOCKER_IMAGE=ghcr.io/flyteorg/flytekit:py${{ matrix.python-version }}-${{ needs.deploy.outputs.version }} --build-arg=PYTHON_VERSION=${{ matrix.python-version }}" + build-args: | + VERSION=${{ needs.deploy.outputs.version }} + DOCKER_IMAGE=ghcr.io/flyteorg/flytekit:py${{ matrix.python-version }}-${{ needs.deploy.outputs.version }} + PYTHON_VERSION=${{ matrix.python-version }}" file: Dockerfile cache-from: type=gha cache-to: type=gha,mode=max + # TODO: compression=gzip? - name: Prepare SQLAlchemy Image Names id: sqlalchemy-names uses: docker/metadata-action@v3 @@ -120,7 +124,9 @@ jobs: platforms: linux/arm64, linux/amd64 push: ${{ github.event_name == 'release' }} tags: ${{ steps.sqlalchemy-names.outputs.tags }} - build-args: "--compress=true --build-arg=VERSION=${{ needs.deploy.outputs.version }}" + build-args: | + VERSION=${{ needs.deploy.outputs.version }} file: Dockerfile.py${{ matrix.python-version }} cache-from: type=gha cache-to: type=gha,mode=max + # TODO: compression=gzip? From 2e68b6516069037e5aa1f4beb5694523ed16048a Mon Sep 17 00:00:00 2001 From: Andrew Dye Date: Thu, 3 Nov 2022 11:07:46 -0700 Subject: [PATCH 04/10] Fix build args extra quotation Signed-off-by: Andrew Dye --- .github/workflows/pythonpublish.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 012290e5cf..f793354d8c 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -102,11 +102,10 @@ jobs: build-args: | VERSION=${{ needs.deploy.outputs.version }} DOCKER_IMAGE=ghcr.io/flyteorg/flytekit:py${{ matrix.python-version }}-${{ needs.deploy.outputs.version }} - PYTHON_VERSION=${{ matrix.python-version }}" + PYTHON_VERSION=${{ matrix.python-version }} file: Dockerfile cache-from: type=gha cache-to: type=gha,mode=max - # TODO: compression=gzip? - name: Prepare SQLAlchemy Image Names id: sqlalchemy-names uses: docker/metadata-action@v3 @@ -129,4 +128,3 @@ jobs: file: Dockerfile.py${{ matrix.python-version }} cache-from: type=gha cache-to: type=gha,mode=max - # TODO: compression=gzip? From bc77f947b491395014689eeb9207a4a712bcd171 Mon Sep 17 00:00:00 2001 From: Andrew Dye Date: Thu, 3 Nov 2022 11:17:00 -0700 Subject: [PATCH 05/10] Hardcode existing version Signed-off-by: Andrew Dye --- .github/workflows/pythonpublish.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index f793354d8c..cd323fdf94 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -24,10 +24,11 @@ jobs: pip install setuptools wheel twine - name: Autobump version id: bump + # TODO: remove, hardcoding version run: | # from refs/tags/v1.2.3 get 1.2.3 VERSION=$(echo $GITHUB_REF | sed 's#.*/v##') - echo "::set-output name=version::$VERSION" + echo "::set-output name=version::1.2.3" VERSION=$VERSION make update_version shell: bash # - name: Build and publish @@ -101,7 +102,7 @@ jobs: tags: ${{ steps.flytekit-names.outputs.tags }} build-args: | VERSION=${{ needs.deploy.outputs.version }} - DOCKER_IMAGE=ghcr.io/flyteorg/flytekit:py${{ matrix.python-version }}-${{ needs.deploy.outputs.version }} + DOCKER_IMAGE=ghcr.io/${{ github.repository_owner }}/flytekit:py${{ matrix.python-version }}-${{ needs.deploy.outputs.version }} PYTHON_VERSION=${{ matrix.python-version }} file: Dockerfile cache-from: type=gha From e3c71cd8c2fa3529f9df3606dd04ef6a46f3d275 Mon Sep 17 00:00:00 2001 From: Andrew Dye Date: Thu, 3 Nov 2022 15:45:05 -0700 Subject: [PATCH 06/10] Build phik wheel Signed-off-by: Andrew Dye --- Dockerfile | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index aa7f9a7a4e..c70a9949a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,11 @@ ARG PYTHON_VERSION +FROM python:${PYTHON_VERSION}-slim-buster AS builder + +RUN apt-get update && apt-get install build-essential -y + +RUN mkdir /root/wheels/ +RUN pip wheel --no-deps -w /root/wheels/ phik + FROM python:${PYTHON_VERSION}-slim-buster MAINTAINER Flyte Team @@ -10,11 +17,16 @@ ENV PYTHONPATH /root ARG VERSION ARG DOCKER_IMAGE +RUN mkdir /root/wheels/ +COPY --from=builder /root/wheels/ /root/wheels/ + # Pod tasks should be exposed in the default image -RUN pip install -U flytekit==$VERSION \ - flytekitplugins-pod==$VERSION \ - flytekitplugins-deck-standard==$VERSION \ - flytekitplugins-data-fsspec==$VERSION \ - scikit-learn +RUN pip install -U /root/wheels/*.whl flytekit==$VERSION \ + flytekitplugins-pod==$VERSION \ + flytekitplugins-deck-standard==$VERSION \ + flytekitplugins-data-fsspec==$VERSION \ + scikit-learn + +RUN rm -rf /root/wheels/ ENV FLYTE_INTERNAL_IMAGE "$DOCKER_IMAGE" From 193362c3db547f97550348326009e5269175e69e Mon Sep 17 00:00:00 2001 From: Andrew Dye Date: Thu, 3 Nov 2022 18:52:31 -0700 Subject: [PATCH 07/10] Include build essentials in image Signed-off-by: Andrew Dye --- Dockerfile | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index c70a9949a2..c1ee9c349c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,4 @@ ARG PYTHON_VERSION -FROM python:${PYTHON_VERSION}-slim-buster AS builder - -RUN apt-get update && apt-get install build-essential -y - -RUN mkdir /root/wheels/ -RUN pip wheel --no-deps -w /root/wheels/ phik - FROM python:${PYTHON_VERSION}-slim-buster MAINTAINER Flyte Team @@ -17,16 +10,13 @@ ENV PYTHONPATH /root ARG VERSION ARG DOCKER_IMAGE -RUN mkdir /root/wheels/ -COPY --from=builder /root/wheels/ /root/wheels/ +RUN apt-get update && apt-get install build-essential -y # Pod tasks should be exposed in the default image -RUN pip install -U /root/wheels/*.whl flytekit==$VERSION \ +RUN pip install -U flytekit==$VERSION \ flytekitplugins-pod==$VERSION \ flytekitplugins-deck-standard==$VERSION \ flytekitplugins-data-fsspec==$VERSION \ scikit-learn -RUN rm -rf /root/wheels/ - ENV FLYTE_INTERNAL_IMAGE "$DOCKER_IMAGE" From df521055c10700e9eeee8584d6e9fdf216676951 Mon Sep 17 00:00:00 2001 From: Andrew Dye Date: Thu, 3 Nov 2022 19:15:45 -0700 Subject: [PATCH 08/10] Fix sqlalchemy Dockerfile path Signed-off-by: Andrew Dye --- .github/workflows/pythonpublish.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index cd323fdf94..0b23168f01 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -1,9 +1,6 @@ name: Publish Python Package on: - # TODO: remove, used for testing manually - workflow_dispatch: - release: types: [published] @@ -24,10 +21,10 @@ jobs: pip install setuptools wheel twine - name: Autobump version id: bump - # TODO: remove, hardcoding version run: | # from refs/tags/v1.2.3 get 1.2.3 VERSION=$(echo $GITHUB_REF | sed 's#.*/v##') + # echo "::set-output name=version::$VERSION" echo "::set-output name=version::1.2.3" VERSION=$VERSION make update_version shell: bash @@ -126,6 +123,6 @@ jobs: tags: ${{ steps.sqlalchemy-names.outputs.tags }} build-args: | VERSION=${{ needs.deploy.outputs.version }} - file: Dockerfile.py${{ matrix.python-version }} + file: ./plugins/flytekit-sqlalchemy/Dockerfile.py${{ matrix.python-version }} cache-from: type=gha cache-to: type=gha,mode=max From e6ebd8064655f6257efe4ff1fa0f6df8ba896b22 Mon Sep 17 00:00:00 2001 From: Andrew Dye Date: Thu, 3 Nov 2022 20:10:11 -0700 Subject: [PATCH 09/10] Revert hacks for testing Signed-off-by: Andrew Dye --- .github/workflows/pythonpublish.yml | 59 ++++++++++++++--------------- 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 0b23168f01..097d82323e 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -24,35 +24,34 @@ jobs: run: | # from refs/tags/v1.2.3 get 1.2.3 VERSION=$(echo $GITHUB_REF | sed 's#.*/v##') - # echo "::set-output name=version::$VERSION" - echo "::set-output name=version::1.2.3" + echo "::set-output name=version::$VERSION" VERSION=$VERSION make update_version shell: bash - # - name: Build and publish - # env: - # TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - # TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - # run: | - # python setup.py sdist bdist_wheel - # twine upload dist/* - # - name: Autobump plugin version - # run: | - # # from refs/tags/v1.2.3 get 1.2.3 - # VERSION=$(echo $GITHUB_REF | sed 's#.*/v##') - # VERSION=$VERSION make -C plugins update_all_versions - # shell: bash - # - name: Build all Plugins and publish - # env: - # TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - # TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - # run: | - # make -C plugins build_all_plugins - # make -C plugins publish_all_plugins - # # Added sleep because PYPI take some time in publish - # - name: Sleep for 180 seconds - # uses: jakejarvis/wait-action@master - # with: - # time: '180s' + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* + - name: Autobump plugin version + run: | + # from refs/tags/v1.2.3 get 1.2.3 + VERSION=$(echo $GITHUB_REF | sed 's#.*/v##') + VERSION=$VERSION make -C plugins update_all_versions + shell: bash + - name: Build all Plugins and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + make -C plugins build_all_plugins + make -C plugins publish_all_plugins + # Added sleep because PYPI take some time in publish + - name: Sleep for 180 seconds + uses: jakejarvis/wait-action@master + with: + time: '180s' outputs: version: ${{ steps.bump.outputs.version }} @@ -76,10 +75,8 @@ jobs: uses: docker/login-action@v1 with: registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # username: "${{ secrets.FLYTE_BOT_USERNAME }}" - # password: "${{ secrets.FLYTE_BOT_PAT }}" + username: "${{ secrets.FLYTE_BOT_USERNAME }}" + password: "${{ secrets.FLYTE_BOT_PAT }}" - name: Prepare Flytekit Image Names id: flytekit-names uses: docker/metadata-action@v3 From b397a0d0556a2625346b8b48a9757e69e7e87dee Mon Sep 17 00:00:00 2001 From: Andrew Dye Date: Fri, 4 Nov 2022 07:42:29 -0700 Subject: [PATCH 10/10] Install fsspec for aws and gcp Signed-off-by: Andrew Dye --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c1ee9c349c..82f4fe5366 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,8 @@ RUN apt-get update && apt-get install build-essential -y RUN pip install -U flytekit==$VERSION \ flytekitplugins-pod==$VERSION \ flytekitplugins-deck-standard==$VERSION \ - flytekitplugins-data-fsspec==$VERSION \ + flytekitplugins-data-fsspec[aws]==$VERSION \ + flytekitplugins-data-fsspec[gcp]==$VERSION \ scikit-learn ENV FLYTE_INTERNAL_IMAGE "$DOCKER_IMAGE"