Python 3.11 Abi3 wheels (#479)

Co-authored-by: Michael Graeb <graebm@amazon.com>

.github/workflows/ci.yml

@@ -7,7 +7,7 @@ on:
       - 'docs'
 
 env:
-  BUILDER_VERSION: v0.9.45
+  BUILDER_VERSION: v0.9.47
   BUILDER_SOURCE: releases
   BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
   PACKAGE_NAME: aws-crt-python
@@ -52,6 +52,7 @@ jobs:
           - cp39-cp39
           - cp310-cp310
           - cp311-cp311
+          - cp312-cp312
     steps:
     # Only aarch64 needs this, but it doesn't hurt anything
     - name: Install qemu/docker
@@ -62,6 +63,30 @@ jobs:
         aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
         ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python
 
+  musllinux-1-1:
+    runs-on: ubuntu-22.04 # latest
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - x64
+          - aarch64
+        python:
+          - cp37-cp37m
+          - cp38-cp38
+          - cp39-cp39
+          - cp310-cp310
+          - cp311-cp311
+          - cp312-cp312
+    steps:
+    # Only aarch64 needs this, but it doesn't hurt anything
+    - name: Install qemu/docker
+      run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+    - name: Build ${{ env.PACKAGE_NAME }}
+      run: |
+        aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
+        ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python
 
   raspberry:
     runs-on: ubuntu-20.04 # latest
@@ -82,7 +107,7 @@ jobs:
 
 
   linux-compat:
-    runs-on: ubuntu-20.04 # latest
+    runs-on: ubuntu-22.04 # latest
     strategy:
       matrix:
         image:
@@ -99,7 +124,7 @@ jobs:
 
 
   linux-compiler-compat:
-    runs-on: ubuntu-20.04 # latest
+    runs-on: ubuntu-22.04 # latest
     strategy:
       matrix:
         compiler:
@@ -156,7 +181,7 @@ jobs:
 
 
   osx:
-    runs-on: macos-11 # latest
+    runs-on: macos-13 # latest
     steps:
     - name: Build ${{ env.PACKAGE_NAME }} + consumers
       run: |
@@ -211,7 +236,7 @@ jobs:
 
   # check that tests requiring custom env-vars or AWS credentials are simply skipped
   tests-ok-without-env-vars:
-    runs-on: ubuntu-20.04 # latest
+    runs-on: ubuntu-22.04 # latest
     steps:
       - uses: actions/checkout@v3
         with:
@@ -228,7 +253,7 @@ jobs:
           python3 -m unittest discover --failfast --verbose
 
   package-source:
-    runs-on: ubuntu-20.04 # latest
+    runs-on: ubuntu-22.04 # latest
     steps:
       - uses: actions/checkout@v3
         with:
@@ -242,7 +267,7 @@ jobs:
 
   # check that docs can still build
   check-docs:
-    runs-on: ubuntu-20.04 # latest
+    runs-on: ubuntu-22.04 # latest
     steps:
       - uses: actions/checkout@v3
         with:
@@ -254,7 +279,7 @@ jobs:
           ./scripts/make-docs.py
 
   check-submodules:
-    runs-on: ubuntu-20.04 # latest
+    runs-on: ubuntu-22.04 # latest
     steps:
     - name: Checkout Source
       uses: actions/checkout@v3

builder.json

@@ -7,6 +7,10 @@
         "manylinux": {
             "_comment": "Use existing compiler on manylinux. These are the images we use for release. We want to be sure things work with the defaults.",
             "needs_compiler": false
+        },
+        "musllinux": {
+            "_comment": "Use existing compiler on musllinux. These are the images we use for release. We want to be sure things work with the defaults.",
+            "needs_compiler": false
         }
     },
     "targets": {

continuous-delivery/build-wheels-manylinux2014-aarch64.sh

@@ -19,6 +19,9 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp310*.whl
 /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl
 
+# Don't need to build wheels for Python 3.12 and later.
+# The 3.11 wheel uses the stable ABI, so it works with newer versions too.
+
 rm dist/*.whl
 cp -rv wheelhouse/* dist/
 

continuous-delivery/build-wheels-manylinux2014-x86_64.sh

@@ -19,6 +19,9 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp310*.whl
 /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
 auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl
 
+# Don't need to build wheels for Python 3.12 and later.
+# The 3.11 wheel uses the stable ABI, so it works with newer versions too.
+
 rm dist/*.whl
 cp -rv wheelhouse/* dist/
 

continuous-delivery/build-wheels-musllinux-1-1-aarch64-jenkins.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+#run build-wheels script in musllinux_1_1 docker image
+set -ex
+
+DOCKER_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/aws-crt-musllinux-1-1-aarch64:latest
+
+$(aws --region us-east-1 ecr get-login --no-include-email)
+
+docker pull $DOCKER_IMAGE
+
+# NOTE: run as current user to avoid git "dubious ownership" error,
+# and so that output artifacts don't belong to "root"
+docker run --rm \
+    --mount type=bind,source=`pwd`,target=/aws-crt-python \
+    --user "$(id -u):$(id -g)" \
+    --workdir /aws-crt-python \
+    --entrypoint /bin/bash \
+    $DOCKER_IMAGE \
+    continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh

continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+#assumes image based on musllinux_1_1
+set -ex
+
+/opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py
+
+/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp37*.whl
+
+/opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp38*.whl
+
+/opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp39*.whl
+
+/opt/python/cp310-cp310/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp310*.whl
+
+/opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp311*.whl
+
+# Don't need to build wheels for Python 3.12 and later.
+# The 3.11 wheel uses the stable ABI, so it works with newer versions too.
+
+rm dist/*.whl
+cp -rv wheelhouse/* dist/
+
+#now you just need to run twine (that's in a different script)

continuous-delivery/build-wheels-musllinux-1-1-x86_64-jenkins.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+#run build-wheels script in musllinux_1_1 docker image
+set -ex
+
+DOCKER_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/aws-crt-musllinux-1-1-x64:latest
+
+$(aws --region us-east-1 ecr get-login --no-include-email)
+
+docker pull $DOCKER_IMAGE
+
+# NOTE: run as current user to avoid git "dubious ownership" error,
+# and so that output artifacts don't belong to "root"
+docker run --rm \
+    --mount type=bind,source=`pwd`,target=/aws-crt-python \
+    --user "$(id -u):$(id -g)" \
+    --workdir /aws-crt-python \
+    --entrypoint /bin/bash \
+    $DOCKER_IMAGE \
+    continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh

continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+#assumes image based on musllinux_1_1 + extras (pip)
+set -ex
+
+/opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py
+
+/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp37*.whl
+
+/opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp38*.whl
+
+/opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp39*.whl
+
+/opt/python/cp310-cp310/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp310*.whl
+
+/opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel
+auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp311*.whl
+
+# Don't need to build wheels for Python 3.12 and later.
+# The 3.11 wheel uses the stable ABI, so it works with newer versions too.
+
+rm dist/*.whl
+cp -rv wheelhouse/* dist/
+
+#now you just need to run twine (that's in a different script)

continuous-delivery/build-wheels-osx.sh

@@ -11,4 +11,7 @@ set -ex
 /Library/Frameworks/Python.framework/Versions/3.10/bin/python3 setup.py sdist bdist_wheel
 /Library/Frameworks/Python.framework/Versions/3.11/bin/python3 setup.py sdist bdist_wheel
 
+# Don't need to build wheels for Python 3.12 and later.
+# The 3.11 wheel uses the stable ABI, so it works with newer versions too.
+
 #now you just need to run twine (that's in a different script)

setup.py

@@ -14,10 +14,11 @@
 import subprocess
 import sys
 import sysconfig
+from wheel.bdist_wheel import bdist_wheel
 
 
 def is_64bit():
-    return sys.maxsize > 2**32
+    return sys.maxsize > 2 ** 32
 
 
 def is_32bit():
@@ -152,7 +153,6 @@ def __init__(self, name, extra_cmake_args=[], libname=None):
 AWS_LIBS.append(AwsLib('aws-c-mqtt'))
 AWS_LIBS.append(AwsLib('aws-c-s3'))
 
-
 PROJECT_DIR = os.path.dirname(os.path.realpath(__file__))
 
 VERSION_RE = re.compile(r""".*__version__ = ["'](.*?)['"]""", re.S)
@@ -283,11 +283,23 @@ def run(self):
         super().run()
 
 
+class bdist_wheel_abi3(bdist_wheel):
+    def get_tag(self):
+        python, abi, plat = super().get_tag()
+        if python.startswith("cp") and sys.version_info >= (3, 11):
+            # on CPython, our wheels are abi3 and compatible back to 3.11
+            return "cp311", "abi3", plat
+
+        return python, abi, plat
+
+
 def awscrt_ext():
     # fetch the CFLAGS/LDFLAGS from env
     extra_compile_args = os.environ.get('CFLAGS', '').split()
     extra_link_args = os.environ.get('LDFLAGS', '').split()
     extra_objects = []
+    define_macros = []
+    py_limited_api = False
 
     libraries = [x.libname for x in AWS_LIBS]
 
@@ -350,14 +362,20 @@ def awscrt_ext():
             if not is_macos_universal2():
                 extra_link_args += ['-Wl,-fatal_warnings']
 
+    if sys.version_info >= (3, 11):
+        define_macros.append(('Py_LIMITED_API', '0x030B0000'))
+        py_limited_api = True
+
     return setuptools.Extension(
         '_awscrt',
         language='c',
         libraries=libraries,
         sources=glob.glob('source/*.c'),
         extra_compile_args=extra_compile_args,
         extra_link_args=extra_link_args,
-        extra_objects=extra_objects
+        extra_objects=extra_objects,
+        define_macros=define_macros,
+        py_limited_api=py_limited_api,
     )
 
 
@@ -392,6 +410,6 @@ def _load_version():
     ],
     python_requires='>=3.7',
     ext_modules=[awscrt_ext()],
-    cmdclass={'build_ext': awscrt_build_ext},
+    cmdclass={'build_ext': awscrt_build_ext, "bdist_wheel": bdist_wheel_abi3},
     test_suite='test',
 )

source/auth_credentials.c

@@ -513,7 +513,7 @@ PyObject *aws_py_credentials_provider_new_chain(PyObject *self, PyObject *args)
     if (!providers_pyseq) {
         goto done;
     }
-    size_t provider_count = (size_t)PySequence_Fast_GET_SIZE(providers_pyseq);
+    size_t provider_count = (size_t)PySequence_Size(providers_pyseq);
     if (provider_count == 0) {
         PyErr_SetString(PyExc_ValueError, "Must supply at least one AwsCredentialsProvider.");
         goto done;
@@ -526,8 +526,9 @@ PyObject *aws_py_credentials_provider_new_chain(PyObject *self, PyObject *args)
     }
 
     for (size_t i = 0; i < provider_count; ++i) {
-        PyObject *provider_py = PySequence_Fast_GET_ITEM(providers_pyseq, i);
+        PyObject *provider_py = PySequence_GetItem(providers_pyseq, i); /* new reference */
         providers_carray[i] = aws_py_get_credentials_provider(provider_py);
+        Py_XDECREF(provider_py);
         if (!providers_carray[i]) {
             goto done;
         }
@@ -724,7 +725,7 @@ PyObject *aws_py_credentials_provider_new_cognito(PyObject *self, PyObject *args
             goto done;
         }
 
-        logins_count = (size_t)PySequence_Fast_GET_SIZE(logins_pyseq);
+        logins_count = (size_t)PySequence_Size(logins_pyseq);
         if (logins_count > 0) {
 
             logins_carray =
@@ -735,7 +736,7 @@ PyObject *aws_py_credentials_provider_new_cognito(PyObject *self, PyObject *args
             }
 
             for (size_t i = 0; i < logins_count; ++i) {
-                PyObject *login_tuple_py = PySequence_Fast_GET_ITEM(logins_pyseq, i);
+                PyObject *login_tuple_py = PySequence_GetItem(logins_pyseq, i); /* New reference */
                 struct aws_cognito_identity_provider_token_pair *login_entry = &logins_carray[i];
                 AWS_ZERO_STRUCT(*login_entry);
 
@@ -750,8 +751,10 @@ PyObject *aws_py_credentials_provider_new_cognito(PyObject *self, PyObject *args
                         PyExc_TypeError,
                         "cognito credentials provider: logins[%zu] is invalid, should be type (str, str)",
                         i);
+                    Py_XDECREF(login_tuple_py);
                     goto done;
                 }
+                Py_XDECREF(login_tuple_py);
             }
         }
     }