From bf393fdc56bc5ba14fc286d3e852124edfcdcbb4 Mon Sep 17 00:00:00 2001 From: Jared Casey Date: Fri, 11 Oct 2024 00:19:34 -0600 Subject: [PATCH] PYCO-17: Finalize Publishing Changes ======= * Update CI to publish to PyPI * Update CI to verify published package --- .github/workflows/publish.yml | 41 ++++-- .github/workflows/verify_release.yml | 193 +++++++++++++++++++++++---- 2 files changed, 198 insertions(+), 36 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4227c5c..41af437 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -201,10 +201,12 @@ jobs: && needs.output-publish-params.result == 'success' && needs.output-publish-params.outputs.publish_test_pypi == 'true' runs-on: ubuntu-22.04 - environment: publish - # permissions: - # # This permission is required for pypi's "trusted publisher" feature - # id-token: write + environment: + name: publish + url: https://pypi.org/p/couchbase-columnar + permissions: + # This permission is required for pypi's "trusted publisher" feature + id-token: write steps: - uses: actions/download-artifact@v4 if: ${{ inputs.wheels_run_id == '' }} @@ -244,6 +246,10 @@ jobs: run: | ls -alh dist twine check dist/* + - name: Publish package distributions to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ validate-test-pypi: @@ -270,7 +276,12 @@ jobs: && needs.output-publish-params.result == 'success' && needs.output-publish-params.outputs.publish_pypi == 'true' runs-on: ubuntu-22.04 - environment: publish + environment: + name: publish + url: https://pypi.org/p/couchbase-columnar + permissions: + # This permission is required for pypi's "trusted publisher" feature + id-token: write steps: - uses: actions/download-artifact@v4 if: ${{ inputs.wheels_run_id == '' }} @@ -310,6 +321,8 @@ jobs: run: | ls -alh dist twine check dist/* + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 validate-pypi: @@ -330,6 +343,9 @@ jobs: name: Publish SDK API docs needs: [upload-api-docs, validate-pypi] environment: publish + # permissions: + # id-token: write + # contents: read if: | always() && (needs.build-wheels.result == 'success' || needs.build-wheels.result == 'skipped') @@ -342,7 +358,16 @@ jobs: uses: actions/download-artifact@v4 with: name: pycbcc-api-docs - - name: Publish API docs to S3 + # - name: Setup AWS Credentials + # uses: aws-actions/configure-aws-credentials@v3 + # with: + # role-to-assume: arn:aws:iam::786014483886:role/SDK_GHA + # aws-region: us-west-1 + - name: Upload API docs to S3 run: | - export SPHINX_DIR=${{ github.workspace }}/html - ls -alh $SPHINX_DIR + ls -alh ./html + export S3_URL="s3://docs.couchbase.com/sdk-api/columnar-python-client-$SDK_VERSION/" + echo "publish command: aws s3 cp ./html $S3_URL --recursive --acl public-read" + # aws s3 cp ./html s3://docs.couchbase.com/sdk-api/columnar-python-client-$SDK_VERSION/ --recursive --acl public-read + env: + SDK_VERSION: ${{ inputs.version }} diff --git a/.github/workflows/verify_release.yml b/.github/workflows/verify_release.yml index fdae31d..1771411 100644 --- a/.github/workflows/verify_release.yml +++ b/.github/workflows/verify_release.yml @@ -40,6 +40,10 @@ on: type: string +env: + DEFAULT_PYTHON: "3.8" + + jobs: validate-input: runs-on: ubuntu-22.04 @@ -82,8 +86,6 @@ jobs: test-setup: runs-on: ubuntu-22.04 - outputs: - install-cmd: ${{ inputs.packaging_index == 'TEST_PYPI' && steps.test-pypi-cmd.outputs.install_cmd && steps.pypi-cmd.outputs.install_cmd }} needs: validate-input steps: - name: Setup Python @@ -105,24 +107,13 @@ jobs: mv test_config.ini pycbcc_test env: CI_SCRIPTS_URL: "https://raw.githubusercontent.com/couchbaselabs/sdkbuild-jenkinsfiles/master/python" - - name: Create install command - id: test-pypi-cmd - if: ${{ inputs.packaging_index == 'TEST_PYPI'}} - run: | - export INSTALL_CMD='-i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple couchbase==$SDK_VERSION' - echo "INSTALL_CMD=$INSTALL_CMD" - echo "install_cmd=$INSTALL_CMD" >> $GITHUB_OUTPUT - env: - SDK_VERSION: ${{ inputs.version }} - - name: Create install command - id: pypi-cmd - if: ${{ inputs.packaging_index == 'PYPI'}} + - name: Download quick test script run: | - export=INSTALL_CMD='couchbase==$SDK_VERSION' - echo "INSTALL_CMD=$INSTALL_CMD" - echo "install_cmd=$INSTALL_CMD" >> $GITHUB_OUTPUT + cd pycbcc_test + curl -o columnar_async_quick_test.py ${CI_SCRIPTS_URL}/release_smoke_tests/columnar_async_quick_test.py + curl -o columnar_quick_test.py ${CI_SCRIPTS_URL}/release_smoke_tests/columnar_quick_test.py env: - SDK_VERSION: ${{ inputs.version }} + CI_SCRIPTS_URL: "https://raw.githubusercontent.com/couchbaselabs/sdkbuild-jenkinsfiles/master/python" - name: Upload test setup uses: actions/upload-artifact@v4 with: @@ -151,26 +142,172 @@ jobs: with: name: pycbcc-test-setup path: pycbcc - - name: Download quick test script - run: | - cd pycbcc - curl -o columnar_async_quick_test.py ${CI_SCRIPTS_URL}/release_smoke_tests/columnar_async_quick_test.py - curl -o columnar_quick_test.py ${CI_SCRIPTS_URL}/release_smoke_tests/columnar_quick_test.py - env: - CI_SCRIPTS_URL: "https://raw.githubusercontent.com/couchbaselabs/sdkbuild-jenkinsfiles/master/python" - - name: Run verification tests in docker + - name: Run test PyPI verification tests in docker + if: ${{ inputs.packaging_index == 'TEST_PYPI'}} + uses: addnab/docker-run-action@v3 + with: + image: python:${{ matrix.python-version }}-slim-bullseye + options: >- + --platform linux/${{ matrix.arch == 'aarch64' && 'arm64' || 'amd64'}} + -v ${{ github.workspace }}/pycbcc:/pycbcc + -e SDK_VERSION=${{ inputs.version }} + run: | + env + python -m pip install --upgrade pip setuptools wheel + python -m pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple couchbase-columnar==$SDK_VERSION -v -v -v + python -m pip list + cd pycbcc + python columnar_quick_test.py + python columnar_async_quick_test.py + - name: Run PyPI verification tests in docker + if: ${{ inputs.packaging_index == 'PYPI'}} uses: addnab/docker-run-action@v3 with: image: python:${{ matrix.python-version }}-slim-bullseye options: >- --platform linux/${{ matrix.arch == 'aarch64' && 'arm64' || 'amd64'}} -v ${{ github.workspace }}/pycbcc:/pycbcc - -e INSTALL_CMD=${{ needs.test-setup.outputs.install-cmd }} + -e SDK_VERSION=${{ inputs.version }} run: | env python -m pip install --upgrade pip setuptools wheel - python -m pip install $INSTALL_CMD -v -v -v + python -m pip install couchbase-columnar==$SDK_VERSION -v -v -v python -m pip list cd pycbcc python columnar_quick_test.py python columnar_async_quick_test.py + + verify-musllinux: + needs: test-setup + runs-on: ${{ matrix.os }} + container: + image: python:${{ matrix.python-version }}-alpine3.19 + strategy: + fail-fast: false + matrix: + os: ['ubuntu-22.04'] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + arch: ['x86_64'] + steps: + - name: Download test setup + uses: actions/download-artifact@v4 + with: + name: pycbcc-test-setup + path: pycbcc + - name: Confirm Python version + run: python -c "import sys; print(sys.version)" + - name: Run test PyPI verification tests + if: ${{ inputs.packaging_index == 'TEST_PYPI'}} + run: | + env + python -m pip install --upgrade pip setuptools wheel + python -m pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple couchbase-columnar==$SDK_VERSION -v -v -v + python -m pip list + cd pycbcc + python columnar_quick_test.py + python columnar_async_quick_test.py + env: + SDK_VERSION: ${{ inputs.version }} + - name: Run PyPI verification tests + if: ${{ inputs.packaging_index == 'PYPI'}} + run: | + env + python -m pip install --upgrade pip setuptools wheel + python -m pip install couchbase-columnar==$SDK_VERSION -v -v -v + python -m pip list + cd pycbcc + python columnar_quick_test.py + python columnar_async_quick_test.py + env: + SDK_VERSION: ${{ inputs.version }} + + verify-macos: + needs: test-setup + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["macos-13", "macos-14"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + steps: + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Confirm Python version + run: python -c "import sys; print(sys.version)" + - name: Download test setup + uses: actions/download-artifact@v4 + with: + name: pycbcc-test-setup + path: pycbcc + - name: Run test PyPI verification tests + if: ${{ inputs.packaging_index == 'TEST_PYPI'}} + run: | + env + python -m pip install --upgrade pip setuptools wheel + python -m pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple couchbase-columnar==$SDK_VERSION -v -v -v + python -m pip list + cd pycbcc + python columnar_quick_test.py + python columnar_async_quick_test.py + env: + SDK_VERSION: ${{ inputs.version }} + - name: Run PyPI verification tests + if: ${{ inputs.packaging_index == 'PYPI'}} + run: | + env + python -m pip install --upgrade pip setuptools wheel + python -m pip install couchbase-columnar==$SDK_VERSION -v -v -v + python -m pip list + cd pycbcc + python columnar_quick_test.py + python columnar_async_quick_test.py + env: + SDK_VERSION: ${{ inputs.version }} + + verify-windows: + needs: test-setup + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["windows-latest"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + arch: ["AMD64"] + steps: + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Confirm Python version + run: python -c "import sys; print(sys.version)" + - name: Download test setup + uses: actions/download-artifact@v4 + with: + name: pycbcc-test-setup + path: pycbcc + - name: Run test PyPI verification tests + if: ${{ inputs.packaging_index == 'TEST_PYPI'}} + run: | + env + python -m pip install --upgrade pip setuptools wheel + python -m pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple couchbase-columnar==$env:SDK_VERSION -v -v -v + python -m pip list + cd pycbcc + python columnar_quick_test.py + python columnar_async_quick_test.py + env: + SDK_VERSION: ${{ inputs.version }} + - name: Run PyPI verification tests + if: ${{ inputs.packaging_index == 'PYPI'}} + run: | + env + python -m pip install --upgrade pip setuptools wheel + python -m pip install couchbase-columnar==$env:SDK_VERSION -v -v -v + python -m pip list + cd pycbcc + python columnar_quick_test.py + python columnar_async_quick_test.py + env: + SDK_VERSION: ${{ inputs.version }}