MuslLinux Wheels (#480)

.github/workflows/ci.yml

@@ -7,7 +7,7 @@ on:
       - 'docs'
 
 env:
-  BUILDER_VERSION: v0.9.46
+  BUILDER_VERSION: v0.9.47
   BUILDER_SOURCE: releases
   BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
   PACKAGE_NAME: aws-crt-python
@@ -63,6 +63,30 @@ jobs:
         aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
         ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python
 
+  musllinux-1-1:
+    runs-on: ubuntu-22.04 # latest
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - x64
+          - aarch64
+        python:
+          - cp37-cp37m
+          - cp38-cp38
+          - cp39-cp39
+          - cp310-cp310
+          - cp311-cp311
+          - cp312-cp312
+    steps:
+    # Only aarch64 needs this, but it doesn't hurt anything
+    - name: Install qemu/docker
+      run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+    - name: Build ${{ env.PACKAGE_NAME }}
+      run: |
+        aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
+        ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python
 
   raspberry:
     runs-on: ubuntu-20.04 # latest

builder.json

@@ -7,6 +7,10 @@
         "manylinux": {
             "_comment": "Use existing compiler on manylinux. These are the images we use for release. We want to be sure things work with the defaults.",
             "needs_compiler": false
+        },
+        "musllinux": {
+            "_comment": "Use existing compiler on musllinux. These are the images we use for release. We want to be sure things work with the defaults.",
+            "needs_compiler": false
         }
     },
     "targets": {

continuous-delivery/build-wheels-manylinux2014-aarch64.sh

@@ -19,6 +19,9 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp310*.whl
 /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl
 
+# Don't need to build wheels for Python 3.12 and later.
+# The 3.11 wheel uses the stable ABI, so it works with newer versions too.
+
 rm dist/*.whl
 cp -rv wheelhouse/* dist/
 

continuous-delivery/build-wheels-manylinux2014-x86_64.sh

@@ -19,6 +19,9 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp310*.whl
 /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl
 
+# Don't need to build wheels for Python 3.12 and later.
+# The 3.11 wheel uses the stable ABI, so it works with newer versions too.
+
 rm dist/*.whl
 cp -rv wheelhouse/* dist/
 

continuous-delivery/build-wheels-musllinux-1-1-aarch64-jenkins.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+#run build-wheels script in musllinux_1_1 docker image
+set -ex
+
+DOCKER_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/aws-crt-musllinux-1-1-aarch64:latest
+
+$(aws --region us-east-1 ecr get-login --no-include-email)
+
+docker pull $DOCKER_IMAGE
+
+# NOTE: run as current user to avoid git "dubious ownership" error,
+# and so that output artifacts don't belong to "root"
+docker run --rm \
+    --mount type=bind,source=`pwd`,target=/aws-crt-python \
+    --user "$(id -u):$(id -g)" \
+    --workdir /aws-crt-python \
+    --entrypoint /bin/bash \
+    $DOCKER_IMAGE \
+    continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh

continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+#assumes image based on musllinux_1_1
+set -ex
+
+/opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py
+
+/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp37*.whl
+
+/opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp38*.whl
+
+/opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp39*.whl
+
+/opt/python/cp310-cp310/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp310*.whl
+
+/opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp311*.whl
+
+# Don't need to build wheels for Python 3.12 and later.
+# The 3.11 wheel uses the stable ABI, so it works with newer versions too.
+
+rm dist/*.whl
+cp -rv wheelhouse/* dist/
+
+#now you just need to run twine (that's in a different script)

continuous-delivery/build-wheels-musllinux-1-1-x86_64-jenkins.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+#run build-wheels script in musllinux_1_1 docker image
+set -ex
+
+DOCKER_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/aws-crt-musllinux-1-1-x64:latest
+
+$(aws --region us-east-1 ecr get-login --no-include-email)
+
+docker pull $DOCKER_IMAGE
+
+# NOTE: run as current user to avoid git "dubious ownership" error,
+# and so that output artifacts don't belong to "root"
+docker run --rm \
+    --mount type=bind,source=`pwd`,target=/aws-crt-python \
+    --user "$(id -u):$(id -g)" \
+    --workdir /aws-crt-python \
+    --entrypoint /bin/bash \
+    $DOCKER_IMAGE \
+    continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh

continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+#assumes image based on musllinux_1_1 + extras (pip)
+set -ex
+
+/opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py
+
+/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp37*.whl
+
+/opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp38*.whl
+
+/opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp39*.whl
+
+/opt/python/cp310-cp310/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp310*.whl
+
+/opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp311*.whl
+
+# Don't need to build wheels for Python 3.12 and later.
+# The 3.11 wheel uses the stable ABI, so it works with newer versions too.
+
+rm dist/*.whl
+cp -rv wheelhouse/* dist/
+
+#now you just need to run twine (that's in a different script)

continuous-delivery/build-wheels-osx.sh

@@ -11,4 +11,7 @@ set -ex
 /Library/Frameworks/Python.framework/Versions/3.10/bin/python3 setup.py sdist bdist_wheel
 /Library/Frameworks/Python.framework/Versions/3.11/bin/python3 setup.py sdist bdist_wheel
 
+# Don't need to build wheels for Python 3.12 and later.
+# The 3.11 wheel uses the stable ABI, so it works with newer versions too.
+
 #now you just need to run twine (that's in a different script)