From 7431e2d562f622a782b89c4ef8f22de5e710e236 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Wed, 14 Jun 2023 13:04:56 -0700 Subject: [PATCH] Python 3.11 Abi3 wheels (#479) Co-authored-by: Michael Graeb --- .github/workflows/ci.yml | 41 ++++++++++++--- builder.json | 4 ++ .../build-wheels-manylinux2014-aarch64.sh | 3 ++ .../build-wheels-manylinux2014-x86_64.sh | 3 ++ ...ld-wheels-musllinux-1-1-aarch64-jenkins.sh | 19 +++++++ .../build-wheels-musllinux-1-1-aarch64.sh | 28 +++++++++++ ...ild-wheels-musllinux-1-1-x86_64-jenkins.sh | 19 +++++++ .../build-wheels-musllinux-1-1-x86_64.sh | 28 +++++++++++ continuous-delivery/build-wheels-osx.sh | 3 ++ crt/aws-c-common | 2 +- crt/aws-c-http | 2 +- crt/aws-c-io | 2 +- crt/aws-c-mqtt | 2 +- crt/aws-c-s3 | 2 +- crt/aws-c-sdkutils | 2 +- crt/aws-checksums | 2 +- crt/s2n | 2 +- setup.py | 26 ++++++++-- source/auth_credentials.c | 11 ++-- source/event_stream_headers.c | 11 ++-- source/http_headers.c | 50 +++++++++++-------- source/http_stream.c | 2 +- source/module.c | 15 ++---- source/mqtt5_client.c | 40 ++++++++++----- source/mqtt_client_connection.c | 30 +++++++---- source/s3_meta_request.c | 2 +- source/websocket.c | 10 ++-- test/test_http_client.py | 7 ++- 28 files changed, 274 insertions(+), 94 deletions(-) create mode 100755 continuous-delivery/build-wheels-musllinux-1-1-aarch64-jenkins.sh create mode 100755 continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh create mode 100755 continuous-delivery/build-wheels-musllinux-1-1-x86_64-jenkins.sh create mode 100755 continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b7de1565..7f158d02f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: - 'docs' env: - BUILDER_VERSION: v0.9.45 + BUILDER_VERSION: v0.9.47 BUILDER_SOURCE: releases BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-python @@ -52,6 +52,7 @@ jobs: - cp39-cp39 - cp310-cp310 - cp311-cp311 + - cp312-cp312 steps: # Only aarch64 needs this, but it doesn't hurt anything - name: Install qemu/docker @@ -62,6 +63,30 @@ jobs: aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + musllinux-1-1: + runs-on: ubuntu-22.04 # latest + strategy: + fail-fast: false + matrix: + image: + - x64 + - aarch64 + python: + - cp37-cp37m + - cp38-cp38 + - cp39-cp39 + - cp310-cp310 + - cp311-cp311 + - cp312-cp312 + steps: + # Only aarch64 needs this, but it doesn't hurt anything + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python raspberry: runs-on: ubuntu-20.04 # latest @@ -82,7 +107,7 @@ jobs: linux-compat: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-22.04 # latest strategy: matrix: image: @@ -99,7 +124,7 @@ jobs: linux-compiler-compat: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-22.04 # latest strategy: matrix: compiler: @@ -156,7 +181,7 @@ jobs: osx: - runs-on: macos-11 # latest + runs-on: macos-13 # latest steps: - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | @@ -211,7 +236,7 @@ jobs: # check that tests requiring custom env-vars or AWS credentials are simply skipped tests-ok-without-env-vars: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-22.04 # latest steps: - uses: actions/checkout@v3 with: @@ -228,7 +253,7 @@ jobs: python3 -m unittest discover --failfast --verbose package-source: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-22.04 # latest steps: - uses: actions/checkout@v3 with: @@ -242,7 +267,7 @@ jobs: # check that docs can still build check-docs: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-22.04 # latest steps: - uses: actions/checkout@v3 with: @@ -254,7 +279,7 @@ jobs: ./scripts/make-docs.py check-submodules: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-22.04 # latest steps: - name: Checkout Source uses: actions/checkout@v3 diff --git a/builder.json b/builder.json index 1668a5ad9..c2e186c31 100644 --- a/builder.json +++ b/builder.json @@ -7,6 +7,10 @@ "manylinux": { "_comment": "Use existing compiler on manylinux. These are the images we use for release. We want to be sure things work with the defaults.", "needs_compiler": false + }, + "musllinux": { + "_comment": "Use existing compiler on musllinux. These are the images we use for release. We want to be sure things work with the defaults.", + "needs_compiler": false } }, "targets": { diff --git a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh index 753249370..a140b7d67 100755 --- a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh @@ -19,6 +19,9 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl +# Don't need to build wheels for Python 3.12 and later. +# The 3.11 wheel uses the stable ABI, so it works with newer versions too. + rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh index 8d3b874c9..41ce9fdef 100755 --- a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh @@ -19,6 +19,9 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl +# Don't need to build wheels for Python 3.12 and later. +# The 3.11 wheel uses the stable ABI, so it works with newer versions too. + rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-musllinux-1-1-aarch64-jenkins.sh b/continuous-delivery/build-wheels-musllinux-1-1-aarch64-jenkins.sh new file mode 100755 index 000000000..2cefaed70 --- /dev/null +++ b/continuous-delivery/build-wheels-musllinux-1-1-aarch64-jenkins.sh @@ -0,0 +1,19 @@ +#!/bin/bash +#run build-wheels script in musllinux_1_1 docker image +set -ex + +DOCKER_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/aws-crt-musllinux-1-1-aarch64:latest + +$(aws --region us-east-1 ecr get-login --no-include-email) + +docker pull $DOCKER_IMAGE + +# NOTE: run as current user to avoid git "dubious ownership" error, +# and so that output artifacts don't belong to "root" +docker run --rm \ + --mount type=bind,source=`pwd`,target=/aws-crt-python \ + --user "$(id -u):$(id -g)" \ + --workdir /aws-crt-python \ + --entrypoint /bin/bash \ + $DOCKER_IMAGE \ + continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh diff --git a/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh b/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh new file mode 100755 index 000000000..0cbc1a196 --- /dev/null +++ b/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh @@ -0,0 +1,28 @@ +#!/bin/bash +#assumes image based on musllinux_1_1 +set -ex + +/opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py + +/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp37*.whl + +/opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp38*.whl + +/opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp39*.whl + +/opt/python/cp310-cp310/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp310*.whl + +/opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp311*.whl + +# Don't need to build wheels for Python 3.12 and later. +# The 3.11 wheel uses the stable ABI, so it works with newer versions too. + +rm dist/*.whl +cp -rv wheelhouse/* dist/ + +#now you just need to run twine (that's in a different script) diff --git a/continuous-delivery/build-wheels-musllinux-1-1-x86_64-jenkins.sh b/continuous-delivery/build-wheels-musllinux-1-1-x86_64-jenkins.sh new file mode 100755 index 000000000..00d6e5259 --- /dev/null +++ b/continuous-delivery/build-wheels-musllinux-1-1-x86_64-jenkins.sh @@ -0,0 +1,19 @@ +#!/bin/bash +#run build-wheels script in musllinux_1_1 docker image +set -ex + +DOCKER_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/aws-crt-musllinux-1-1-x64:latest + +$(aws --region us-east-1 ecr get-login --no-include-email) + +docker pull $DOCKER_IMAGE + +# NOTE: run as current user to avoid git "dubious ownership" error, +# and so that output artifacts don't belong to "root" +docker run --rm \ + --mount type=bind,source=`pwd`,target=/aws-crt-python \ + --user "$(id -u):$(id -g)" \ + --workdir /aws-crt-python \ + --entrypoint /bin/bash \ + $DOCKER_IMAGE \ + continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh diff --git a/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh b/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh new file mode 100755 index 000000000..6637d95ac --- /dev/null +++ b/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh @@ -0,0 +1,28 @@ +#!/bin/bash +#assumes image based on musllinux_1_1 + extras (pip) +set -ex + +/opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py + +/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp37*.whl + +/opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp38*.whl + +/opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp39*.whl + +/opt/python/cp310-cp310/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp310*.whl + +/opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp311*.whl + +# Don't need to build wheels for Python 3.12 and later. +# The 3.11 wheel uses the stable ABI, so it works with newer versions too. + +rm dist/*.whl +cp -rv wheelhouse/* dist/ + +#now you just need to run twine (that's in a different script) diff --git a/continuous-delivery/build-wheels-osx.sh b/continuous-delivery/build-wheels-osx.sh index 18b4b993d..a3a92bc63 100755 --- a/continuous-delivery/build-wheels-osx.sh +++ b/continuous-delivery/build-wheels-osx.sh @@ -11,4 +11,7 @@ set -ex /Library/Frameworks/Python.framework/Versions/3.10/bin/python3 setup.py sdist bdist_wheel /Library/Frameworks/Python.framework/Versions/3.11/bin/python3 setup.py sdist bdist_wheel +# Don't need to build wheels for Python 3.12 and later. +# The 3.11 wheel uses the stable ABI, so it works with newer versions too. + #now you just need to run twine (that's in a different script) diff --git a/crt/aws-c-common b/crt/aws-c-common index 7036fcfd2..9f36d0718 160000 --- a/crt/aws-c-common +++ b/crt/aws-c-common @@ -1 +1 @@ -Subproject commit 7036fcfd2949a65bda0af1f09cc9862c4c39088b +Subproject commit 9f36d07189435db27f746c10e39ba58511fadd0d diff --git a/crt/aws-c-http b/crt/aws-c-http index ac18abb48..27efc273f 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit ac18abb489c1baeafb2fb9dec0c6d223601ad590 +Subproject commit 27efc273f228306031c7345de8acaeee956db765 diff --git a/crt/aws-c-io b/crt/aws-c-io index 118a57ecf..af3af9967 160000 --- a/crt/aws-c-io +++ b/crt/aws-c-io @@ -1 +1 @@ -Subproject commit 118a57ecf0f88e047bb3e75227e08adb5782074c +Subproject commit af3af9967ff0f093520228b0eb4d55fc0e36b391 diff --git a/crt/aws-c-mqtt b/crt/aws-c-mqtt index b672bff09..8e40355e0 160000 --- a/crt/aws-c-mqtt +++ b/crt/aws-c-mqtt @@ -1 +1 @@ -Subproject commit b672bff0907603987bc93dae946c4f121c80e14c +Subproject commit 8e40355e03b456bb7b4bf0cd37c5d878afa2476f diff --git a/crt/aws-c-s3 b/crt/aws-c-s3 index 6203b2305..f14d92f58 160000 --- a/crt/aws-c-s3 +++ b/crt/aws-c-s3 @@ -1 +1 @@ -Subproject commit 6203b23057e40fcc73f44e3425e4942cbcf66308 +Subproject commit f14d92f581161686dad97c82aa994862da195f96 diff --git a/crt/aws-c-sdkutils b/crt/aws-c-sdkutils index 812761fdb..df511a1f2 160000 --- a/crt/aws-c-sdkutils +++ b/crt/aws-c-sdkutils @@ -1 +1 @@ -Subproject commit 812761fdbf791f77cb358212cefade9cc16974e7 +Subproject commit df511a1f2518279eb5721ab5fca6bc816efc6b32 diff --git a/crt/aws-checksums b/crt/aws-checksums index 1aa903d52..a5b0e7f00 160000 --- a/crt/aws-checksums +++ b/crt/aws-checksums @@ -1 +1 @@ -Subproject commit 1aa903d5204ed665010204d8521e56e092932e32 +Subproject commit a5b0e7f00be4240f77d3b6e090c8bed3c9c2e536 diff --git a/crt/s2n b/crt/s2n index 9b7b1f334..b9c4d60de 160000 --- a/crt/s2n +++ b/crt/s2n @@ -1 +1 @@ -Subproject commit 9b7b1f334a4ab33397174e505955231ff77b3f85 +Subproject commit b9c4d60de49193cea31f0acfbddc47cc66d41d2c diff --git a/setup.py b/setup.py index 36a8cdc44..0b106b2c4 100644 --- a/setup.py +++ b/setup.py @@ -14,10 +14,11 @@ import subprocess import sys import sysconfig +from wheel.bdist_wheel import bdist_wheel def is_64bit(): - return sys.maxsize > 2**32 + return sys.maxsize > 2 ** 32 def is_32bit(): @@ -152,7 +153,6 @@ def __init__(self, name, extra_cmake_args=[], libname=None): AWS_LIBS.append(AwsLib('aws-c-mqtt')) AWS_LIBS.append(AwsLib('aws-c-s3')) - PROJECT_DIR = os.path.dirname(os.path.realpath(__file__)) VERSION_RE = re.compile(r""".*__version__ = ["'](.*?)['"]""", re.S) @@ -283,11 +283,23 @@ def run(self): super().run() +class bdist_wheel_abi3(bdist_wheel): + def get_tag(self): + python, abi, plat = super().get_tag() + if python.startswith("cp") and sys.version_info >= (3, 11): + # on CPython, our wheels are abi3 and compatible back to 3.11 + return "cp311", "abi3", plat + + return python, abi, plat + + def awscrt_ext(): # fetch the CFLAGS/LDFLAGS from env extra_compile_args = os.environ.get('CFLAGS', '').split() extra_link_args = os.environ.get('LDFLAGS', '').split() extra_objects = [] + define_macros = [] + py_limited_api = False libraries = [x.libname for x in AWS_LIBS] @@ -350,6 +362,10 @@ def awscrt_ext(): if not is_macos_universal2(): extra_link_args += ['-Wl,-fatal_warnings'] + if sys.version_info >= (3, 11): + define_macros.append(('Py_LIMITED_API', '0x030B0000')) + py_limited_api = True + return setuptools.Extension( '_awscrt', language='c', @@ -357,7 +373,9 @@ def awscrt_ext(): sources=glob.glob('source/*.c'), extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, - extra_objects=extra_objects + extra_objects=extra_objects, + define_macros=define_macros, + py_limited_api=py_limited_api, ) @@ -392,6 +410,6 @@ def _load_version(): ], python_requires='>=3.7', ext_modules=[awscrt_ext()], - cmdclass={'build_ext': awscrt_build_ext}, + cmdclass={'build_ext': awscrt_build_ext, "bdist_wheel": bdist_wheel_abi3}, test_suite='test', ) diff --git a/source/auth_credentials.c b/source/auth_credentials.c index 80bca197c..66b1afa8e 100644 --- a/source/auth_credentials.c +++ b/source/auth_credentials.c @@ -513,7 +513,7 @@ PyObject *aws_py_credentials_provider_new_chain(PyObject *self, PyObject *args) if (!providers_pyseq) { goto done; } - size_t provider_count = (size_t)PySequence_Fast_GET_SIZE(providers_pyseq); + size_t provider_count = (size_t)PySequence_Size(providers_pyseq); if (provider_count == 0) { PyErr_SetString(PyExc_ValueError, "Must supply at least one AwsCredentialsProvider."); goto done; @@ -526,8 +526,9 @@ PyObject *aws_py_credentials_provider_new_chain(PyObject *self, PyObject *args) } for (size_t i = 0; i < provider_count; ++i) { - PyObject *provider_py = PySequence_Fast_GET_ITEM(providers_pyseq, i); + PyObject *provider_py = PySequence_GetItem(providers_pyseq, i); /* new reference */ providers_carray[i] = aws_py_get_credentials_provider(provider_py); + Py_XDECREF(provider_py); if (!providers_carray[i]) { goto done; } @@ -724,7 +725,7 @@ PyObject *aws_py_credentials_provider_new_cognito(PyObject *self, PyObject *args goto done; } - logins_count = (size_t)PySequence_Fast_GET_SIZE(logins_pyseq); + logins_count = (size_t)PySequence_Size(logins_pyseq); if (logins_count > 0) { logins_carray = @@ -735,7 +736,7 @@ PyObject *aws_py_credentials_provider_new_cognito(PyObject *self, PyObject *args } for (size_t i = 0; i < logins_count; ++i) { - PyObject *login_tuple_py = PySequence_Fast_GET_ITEM(logins_pyseq, i); + PyObject *login_tuple_py = PySequence_GetItem(logins_pyseq, i); /* New reference */ struct aws_cognito_identity_provider_token_pair *login_entry = &logins_carray[i]; AWS_ZERO_STRUCT(*login_entry); @@ -750,8 +751,10 @@ PyObject *aws_py_credentials_provider_new_cognito(PyObject *self, PyObject *args PyExc_TypeError, "cognito credentials provider: logins[%zu] is invalid, should be type (str, str)", i); + Py_XDECREF(login_tuple_py); goto done; } + Py_XDECREF(login_tuple_py); } } } diff --git a/source/event_stream_headers.c b/source/event_stream_headers.c index f8ae4ef41..cfb1ac33f 100644 --- a/source/event_stream_headers.c +++ b/source/event_stream_headers.c @@ -166,19 +166,20 @@ bool aws_py_event_stream_native_headers_init(struct aws_array_list *native_heade bool success = false; PyObject *sequence_py = NULL; - sequence_py = PySequence_Fast(headers_py, "Expected sequence of Headers"); /* New reference */ + sequence_py = PySequence_Fast(headers_py, "Expected sequence of Headers"); if (!sequence_py) { goto done; } - const Py_ssize_t count = PySequence_Fast_GET_SIZE(sequence_py); + const Py_ssize_t count = PySequence_Size(sequence_py); for (Py_ssize_t i = 0; i < count; ++i) { - /* Borrowed reference, don't need to decref */ - PyObject *header_py = PySequence_Fast_GET_ITEM(sequence_py, i); + PyObject *header_py = PySequence_GetItem(sequence_py, i); /* New Reference */ if (!s_add_native_header(native_headers, header_py)) { + Py_XDECREF(header_py); goto done; } + Py_XDECREF(header_py); } success = true; @@ -270,7 +271,7 @@ PyObject *aws_py_event_stream_python_headers_create( goto error; } - PyList_SET_ITEM(list_py, i, tuple_py); /* steals reference to tuple */ + PyList_SetItem(list_py, i, tuple_py); /* steals reference to tuple */ } return list_py; diff --git a/source/http_headers.c b/source/http_headers.c index 187cd7c0e..66974f9cf 100644 --- a/source/http_headers.c +++ b/source/http_headers.c @@ -85,6 +85,29 @@ PyObject *aws_py_http_headers_add(PyObject *self, PyObject *args) { Py_RETURN_NONE; } +static bool s_py_http_headers_add_pair(PyObject *py_pair, struct aws_http_headers *headers) { + + const char *type_errmsg = "List of (name,value) pairs expected."; + if (!PyTuple_Check(py_pair) || PyTuple_Size(py_pair) != 2) { + PyErr_SetString(PyExc_TypeError, type_errmsg); + return false; + } + + struct aws_byte_cursor name = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 0) /* Borrowed reference */); + struct aws_byte_cursor value = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 1) /* Borrowed reference */); + if (!name.ptr || !value.ptr) { + PyErr_SetString(PyExc_TypeError, type_errmsg); + return false; + } + + if (aws_http_headers_add(headers, name, value)) { + PyErr_SetAwsLastError(); + return false; + } + + return true; +} + PyObject *aws_py_http_headers_add_pairs(PyObject *self, PyObject *args) { PyObject *py_pairs; S_HEADERS_METHOD_START("O", &py_pairs); @@ -96,25 +119,12 @@ PyObject *aws_py_http_headers_add_pairs(PyObject *self, PyObject *args) { return NULL; } - const Py_ssize_t count = PySequence_Fast_GET_SIZE(py_sequence); + const Py_ssize_t count = PySequence_Size(py_pairs); for (Py_ssize_t i = 0; i < count; ++i) { - /* XYZ_GET_ITEM() calls returns borrowed references */ - PyObject *py_pair = PySequence_Fast_GET_ITEM(py_sequence, i); - - if (!PyTuple_Check(py_pair) || PyTuple_GET_SIZE(py_pair) != 2) { - PyErr_SetString(PyExc_TypeError, type_errmsg); - goto done; - } - - struct aws_byte_cursor name = aws_byte_cursor_from_pyunicode(PyTuple_GET_ITEM(py_pair, 0)); - struct aws_byte_cursor value = aws_byte_cursor_from_pyunicode(PyTuple_GET_ITEM(py_pair, 1)); - if (!name.ptr || !value.ptr) { - PyErr_SetString(PyExc_TypeError, type_errmsg); - goto done; - } - - if (aws_http_headers_add(headers, name, value)) { - PyErr_SetAwsLastError(); + PyObject *py_pair = PySequence_GetItem(py_sequence, i); /* New Reference */ + bool success = s_py_http_headers_add_pair(py_pair, headers); + Py_DECREF(py_pair); + if (!success) { goto done; } } @@ -175,8 +185,8 @@ static PyObject *s_py_tuple_from_header(struct aws_http_header header) { goto error; } - PyTuple_SET_ITEM(py_pair, 0, py_name); - PyTuple_SET_ITEM(py_pair, 1, py_value); + PyTuple_SetItem(py_pair, 0, py_name); /* Steals a reference */ + PyTuple_SetItem(py_pair, 1, py_value); /* Steals a reference */ return py_pair; error: diff --git a/source/http_stream.c b/source/http_stream.c index 18ed406eb..bf7021862 100644 --- a/source/http_stream.c +++ b/source/http_stream.c @@ -121,7 +121,7 @@ static int s_on_incoming_header_block_done( goto done; } - PyList_SET_ITEM(header_list, i, tuple); /* steals reference to tuple */ + PyList_SetItem(header_list, i, tuple); /* steals reference to tuple */ } /* TODO: handle informational and trailing headers */ diff --git a/source/module.c b/source/module.c index ca8a0d986..f120daaa7 100644 --- a/source/module.c +++ b/source/module.c @@ -531,20 +531,12 @@ void *aws_py_get_binding(PyObject *obj, const char *capsule_name, const char *cl PyObject *py_binding = PyObject_GetAttrString(obj, "_binding"); /* new reference */ if (!py_binding) { - return PyErr_Format( - PyExc_TypeError, - "Expected valid '%s', received '%s' (no '_binding' attribute)", - class_name, - Py_TYPE(obj)->tp_name); + return PyErr_Format(PyExc_TypeError, "Expected valid '%s' (no '_binding' attribute)", class_name); } void *binding = NULL; if (!PyCapsule_CheckExact(py_binding)) { - PyErr_Format( - PyExc_TypeError, - "Expected valid '%s', received '%s' ('_binding' attribute is not a capsule)", - class_name, - Py_TYPE(obj)->tp_name); + PyErr_Format(PyExc_TypeError, "Expected valid '%s' ('_binding' attribute is not a capsule)", class_name); goto done; } @@ -552,9 +544,8 @@ void *aws_py_get_binding(PyObject *obj, const char *capsule_name, const char *cl if (!binding) { PyErr_Format( PyExc_TypeError, - "Expected valid '%s', received '%s' ('_binding' attribute does not contain '%s')", + "Expected valid '%s' ('_binding' attribute does not contain '%s')", class_name, - Py_TYPE(obj)->tp_name, capsule_name); goto done; } diff --git a/source/mqtt5_client.c b/source/mqtt5_client.c index e07eb914a..c43c90c24 100644 --- a/source/mqtt5_client.c +++ b/source/mqtt5_client.c @@ -200,7 +200,7 @@ static PyObject *s_aws_set_user_properties_to_PyObject( Py_XDECREF(user_properties_list); return NULL; } - PyList_SET_ITEM(user_properties_list, i, tuple); /* Steals reference to tuple */ + PyList_SetItem(user_properties_list, i, tuple); /* Steals reference to tuple */ } return user_properties_list; } @@ -237,8 +237,10 @@ static void s_on_publish_received(const struct aws_mqtt5_packet_publish_view *pu } for (size_t i = 0; i < subscription_identifier_count; ++i) { - PyList_SET_ITEM( - subscription_identifier_list, i, PyLong_FromLongLong(publish_packet->subscription_identifiers[i])); + PyList_SetItem( + subscription_identifier_list, + i, + PyLong_FromLongLong(publish_packet->subscription_identifiers[i])); /* Steals a reference */ } user_properties_list = s_aws_set_user_properties_to_PyObject(publish_packet->user_properties, user_property_count); @@ -1629,7 +1631,7 @@ static void s_on_subscribe_complete_fn( } for (size_t i = 0; i < reason_codes_count; ++i) { - PyList_SET_ITEM(reason_codes_list, i, PyLong_FromLong(suback->reason_codes[i])); + PyList_SetItem(reason_codes_list, i, PyLong_FromLong(suback->reason_codes[i])); /* Steals a reference */ } } @@ -1860,7 +1862,7 @@ static void s_on_unsubscribe_complete_fn( } for (size_t i = 0; i < reason_codes_count; ++i) { - PyList_SET_ITEM(reason_codes_list, i, PyLong_FromLong(unsuback->reason_codes[i])); + PyList_SetItem(reason_codes_list, i, PyLong_FromLong(unsuback->reason_codes[i])); /* Steals a reference */ } } @@ -2030,23 +2032,35 @@ PyObject *aws_py_mqtt5_client_get_stats(PyObject *self, PyObject *args) { goto done; } - PyTuple_SET_ITEM(result, 0, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_count)); - if (PyTuple_GET_ITEM(result, 0) == NULL) { + PyTuple_SetItem( + result, + 0, + PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_count)); /* Steals a reference */ + if (PyTuple_GetItem(result, 0) == NULL) { /* Borrowed reference */ goto done; } - PyTuple_SET_ITEM(result, 1, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_size)); - if (PyTuple_GET_ITEM(result, 1) == NULL) { + PyTuple_SetItem( + result, + 1, + PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_size)); /* Steals a reference */ + if (PyTuple_GetItem(result, 1) == NULL) { /* Borrowed reference */ goto done; } - PyTuple_SET_ITEM(result, 2, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_count)); - if (PyTuple_GET_ITEM(result, 2) == NULL) { + PyTuple_SetItem( + result, + 2, + PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_count)); /* Steals a reference */ + if (PyTuple_GetItem(result, 2) == NULL) { /* Borrowed reference */ goto done; } - PyTuple_SET_ITEM(result, 3, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_size)); - if (PyTuple_GET_ITEM(result, 3) == NULL) { + PyTuple_SetItem( + result, + 3, + PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_size)); /* Steals a reference */ + if (PyTuple_GetItem(result, 3) == NULL) { /* Borrowed reference */ goto done; } diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index 8a78615e5..526157f29 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -1180,7 +1180,7 @@ static void s_suback_multi_callback( goto done_prepping_args; } - PyList_SET_ITEM(topic_qos_list, i, tuple); /* Steals reference to tuple */ + PyList_SetItem(topic_qos_list, i, tuple); /* Steals reference to tuple */ } done_prepping_args:; @@ -1329,23 +1329,35 @@ PyObject *aws_py_mqtt_client_connection_get_stats(PyObject *self, PyObject *args goto done; } - PyTuple_SET_ITEM(result, 0, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_count)); - if (PyTuple_GET_ITEM(result, 0) == NULL) { + PyTuple_SetItem( + result, + 0, + PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_count)); /* Steals a reference */ + if (PyTuple_GetItem(result, 0) == NULL) { /* Borrowed reference */ goto done; } - PyTuple_SET_ITEM(result, 1, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_size)); - if (PyTuple_GET_ITEM(result, 1) == NULL) { + PyTuple_SetItem( + result, + 1, + PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_size)); /* Steals a reference */ + if (PyTuple_GetItem(result, 1) == NULL) { /* Borrowed reference */ goto done; } - PyTuple_SET_ITEM(result, 2, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_count)); - if (PyTuple_GET_ITEM(result, 2) == NULL) { + PyTuple_SetItem( + result, + 2, + PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_count)); /* Steals a reference */ + if (PyTuple_GetItem(result, 2) == NULL) { /* Borrowed reference */ goto done; } - PyTuple_SET_ITEM(result, 3, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_size)); - if (PyTuple_GET_ITEM(result, 3) == NULL) { + PyTuple_SetItem( + result, + 3, + PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_size)); /* Steals a reference */ + if (PyTuple_GetItem(result, 3) == NULL) { /* Borrowed reference */ goto done; } diff --git a/source/s3_meta_request.c b/source/s3_meta_request.c index 88a7c19d7..f7ffa56bd 100644 --- a/source/s3_meta_request.c +++ b/source/s3_meta_request.c @@ -69,7 +69,7 @@ static PyObject *s_get_py_headers(const struct aws_http_headers *headers) { if (!tuple) { goto error; } - PyList_SET_ITEM(header_list, i, tuple); /* steals reference to tuple */ + PyList_SetItem(header_list, i, tuple); /* steals reference to tuple */ } return header_list; error: diff --git a/source/websocket.c b/source/websocket.c index 2b056ee8a..95f9d8e6c 100644 --- a/source/websocket.c +++ b/source/websocket.c @@ -211,13 +211,13 @@ static void s_websocket_on_connection_setup( PyObject *name_py = PyUnicode_FromAwsByteCursor(&header_i->name); AWS_FATAL_ASSERT(name_py && "header name wrangling failed"); - PyTuple_SET_ITEM(tuple_py, 0, name_py); + PyTuple_SetItem(tuple_py, 0, name_py); /* Steals a reference */ PyObject *value_py = PyUnicode_FromAwsByteCursor(&header_i->value); AWS_FATAL_ASSERT(value_py && "header value wrangling failed"); - PyTuple_SET_ITEM(tuple_py, 1, value_py); + PyTuple_SetItem(tuple_py, 1, value_py); /* Steals a reference */ - PyList_SET_ITEM(headers_py, i, tuple_py); + PyList_SetItem(headers_py, i, tuple_py); /* Steals a reference */ } } @@ -580,13 +580,13 @@ PyObject *aws_py_websocket_create_handshake_request(PyObject *self, PyObject *ar if (!request_binding_py) { goto cleanup; } - PyTuple_SET_ITEM(tuple_py, 0, request_binding_py); /* steals reference to request_binding_py */ + PyTuple_SetItem(tuple_py, 0, request_binding_py); /* steals reference to request_binding_py */ PyObject *headers_binding_py = aws_py_http_headers_new_from_native(aws_http_message_get_headers(request)); if (!headers_binding_py) { goto cleanup; } - PyTuple_SET_ITEM(tuple_py, 1, headers_binding_py); /* steals reference to headers_binding_py */ + PyTuple_SetItem(tuple_py, 1, headers_binding_py); /* steals reference to headers_binding_py */ /* Success! */ success = true; diff --git a/test/test_http_client.py b/test/test_http_client.py index 5af87b647..a9cb4593d 100644 --- a/test/test_http_client.py +++ b/test/test_http_client.py @@ -57,10 +57,9 @@ def _start_server(self, secure, http_1_0=False): self.server = HTTPServer((self.hostname, 0), TestRequestHandler) if secure: - self.server.socket = ssl.wrap_socket(self.server.socket, - keyfile="test/resources/unittest.key", - certfile='test/resources/unittest.crt', - server_side=True) + context = ssl.SSLContext() + context.load_cert_chain(certfile='test/resources/unittest.crt', keyfile="test/resources/unittest.key") + self.server.socket = context.wrap_socket(self.server.socket, server_side=True) self.port = self.server.server_address[1] # put requests are stored in this dict