Parallel Wheel Builds (#1024)

* Fix import issue in tests * Parallelize wheel building and add muslinux support --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
newrelic · Jan 4, 2024 · 1483f73 · 1483f73
1 parent 3962f54
commit 1483f73
Show file tree

Hide file tree

Showing 2 changed files with 120 additions and 27 deletions.
diff --git a/.github/workflows/deploy-python.yml b/.github/workflows/deploy-python.yml
@@ -20,18 +20,121 @@ on:
       - published
 
 jobs:
-  deploy-linux:
+  build-linux-py3:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        wheel:
+          - cp37-manylinux
+          - cp37-musllinux
+          - cp38-manylinux
+          - cp38-musllinux
+          - cp39-manylinux
+          - cp39-musllinux
+          - cp310-manylinux
+          - cp310-musllinux
+          - cp311-manylinux
+          - cp311-musllinux
+          - cp312-manylinux
+          - cp312-musllinux
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           persist-credentials: false
           fetch-depth: 0
 
       - name: Setup QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
+      - name: Build Wheels
+        uses: pypa/cibuildwheel@v2.16.2
+        env:
+          CIBW_PLATFORM: linux
+          CIBW_BUILD: "${{ matrix.wheel }}*"
+          CIBW_ARCHS_LINUX: x86_64 aarch64
+          CIBW_ENVIRONMENT: "LD_LIBRARY_PATH=/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:/opt/rh/devtoolset-8/root/usr/lib64/dyninst:/opt/rh/devtoolset-8/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib"
+          CIBW_TEST_REQUIRES: pytest
+          CIBW_TEST_COMMAND: "PYTHONPATH={project}/tests pytest {project}/tests/agent_unittests -vx"
+
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v4.0.0
+        with:
+          name: ${{ github.job }}-${{ matrix.wheel }}
+          path: ./wheelhouse/*.whl
+          retention-days: 1
+
+  build-linux-py2:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Build Wheels
+        uses: pypa/cibuildwheel@v1.12.0
+        env:
+          CIBW_PLATFORM: linux
+          CIBW_BUILD: cp27-manylinux_x86_64
+          CIBW_ARCHS_LINUX: x86_64
+          CIBW_ENVIRONMENT: "LD_LIBRARY_PATH=/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:/opt/rh/devtoolset-8/root/usr/lib64/dyninst:/opt/rh/devtoolset-8/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib"
+          CIBW_TEST_REQUIRES: pytest==4.6.11
+          CIBW_TEST_COMMAND: "PYTHONPATH={project}/tests pytest {project}/tests/agent_unittests -vx"
+
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v4.0.0
+        with:
+          name: ${{ github.job }}
+          path: ./wheelhouse/*.whl
+          retention-days: 1
+
+  build-sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Install Dependencies
+        run: |
+          pip install -U pip
+          pip install -U setuptools
+
+      - name: Build Source Package
+        run: |
+          python setup.py sdist
+
+      - name: Prepare MD5 Hash File
+        run: |
+          tarball="$(python setup.py --fullname).tar.gz"
+          md5_file=${tarball}.md5
+          openssl md5 -binary dist/${tarball} | xxd -p | tr -d '\n' > dist/${md5_file}
+
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v4.0.0
+        with:
+          name: ${{ github.job }}-sdist
+          path: |
+            ./dist/*.tar.gz
+            ./dist/*.tar.gz.md5
+          retention-days: 1
+
+  deploy:
+    runs-on: ubuntu-latest
+
+    needs: 
+      - build-linux-py3
+      - build-linux-py2
+      - build-sdist
+
+    steps:
       - uses: actions/setup-python@v2
         with:
           python-version: "3.x"
@@ -42,32 +145,22 @@ jobs:
           pip install -U pip
           pip install -U wheel setuptools twine
 
-      - name: Build Source Package
-        run: python setup.py sdist
-
-      - name: Build Manylinux Wheels (Python 2)
-        uses: pypa/cibuildwheel@v1.12.0
-        env:
-          CIBW_PLATFORM: linux
-          CIBW_BUILD: cp27-manylinux_x86_64
-          CIBW_ARCHS: x86_64
-          CIBW_ENVIRONMENT: "LD_LIBRARY_PATH=/opt/rh/=vtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:/opt/rh/devtoolset-8/root/usr/lib64/dyninst:/opt/rh/devtoolset-8/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib"
+      - name: Download Artifacts
+        uses: actions/download-artifact@v4.1.0
+        with:
+          path: ./artifacts/
 
-      - name: Build Manylinux Wheels (Python 3)
-        uses: pypa/cibuildwheel@v2.11.1
-        env:
-          CIBW_PLATFORM: linux
-          CIBW_BUILD: cp37-manylinux* cp38-manylinux* cp39-manylinux* cp310-manylinux* cp311-manylinux*
-          CIBW_ARCHS: x86_64 aarch64
-          CIBW_ENVIRONMENT: "LD_LIBRARY_PATH=/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:/opt/rh/devtoolset-8/root/usr/lib64/dyninst:/opt/rh/devtoolset-8/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib"
+      - name: Unpack Artifacts
+        run: |
+          mkdir -p dist/
+          mv artifacts/**/*{.whl,.tar.gz,.tar.gz.md5} dist/
 
       - name: Upload Package to S3
         run: |
           tarball="$(python setup.py --fullname).tar.gz"
-          md5_file=$(mktemp)
-          openssl md5 -binary dist/$tarball | xxd -p | tr -d '\n' > $md5_file
-          aws s3 cp $md5_file $S3_DST/${tarball}.md5
-          aws s3 cp dist/$tarball $S3_DST/$tarball
+          md5_file=${tarball}.md5
+          aws s3 cp dist/${md5_file} $S3_DST/${md5_file}
+          aws s3 cp dist/${tarball} $S3_DST/${tarball}
         env:
           S3_DST: s3://nr-downloads-main/python_agent/release
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -76,7 +169,7 @@ jobs:
 
       - name: Upload Package to PyPI
         run: |
-          twine upload --non-interactive dist/*.tar.gz wheelhouse/*-manylinux*.whl
+          twine upload --non-interactive dist/*.tar.gz dist/*.whl
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
diff --git a/tests/agent_unittests/test_package_version_utils.py b/tests/agent_unittests/test_package_version_utils.py
@@ -16,7 +16,6 @@
 import warnings
 
 import pytest
-import six
 from testing_support.validators.validate_function_called import validate_function_called
 
 from newrelic.common.package_version_utils import (
@@ -26,6 +25,7 @@
     get_package_version,
     get_package_version_tuple,
 )
+from newrelic.packages import six
 
 # Notes:
 # importlib.metadata was a provisional addition to the std library in PY38 and PY39
@@ -143,7 +143,7 @@ def test_deprecation_warning_suppression(monkeypatch, recwarn):
     monkeypatch.setattr(pytest, "__getattr__", _getattr_deprecation_warning, raising=False)
 
     assert get_package_version("pytest") == "3.2.1"
-    
+
     assert not recwarn.list, "Warnings not suppressed."