From cc4f18470dfaa4ea6c7d3dd531ff47f8b5ba14a2 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 22 May 2023 14:57:16 -0700 Subject: [PATCH 01/29] add wheels for python 3.12 and update ci --- .github/workflows/ci.yml | 17 +++++++++-------- .../build-wheels-manylinux2014-aarch64.sh | 3 +++ .../build-wheels-manylinux2014-x86_64.sh | 3 +++ continuous-delivery/build-wheels-osx.sh | 1 + continuous-delivery/build-wheels-win32.bat | 1 + continuous-delivery/build-wheels-win64.bat | 1 + 6 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b7de1565..dae3c8635 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,7 @@ jobs: - cp39-cp39 - cp310-cp310 - cp311-cp311 + - cp312-cp312 steps: # Only aarch64 needs this, but it doesn't hurt anything - name: Install qemu/docker @@ -82,7 +83,7 @@ jobs: linux-compat: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-22.04 # latest strategy: matrix: image: @@ -99,7 +100,7 @@ jobs: linux-compiler-compat: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-22.04 # latest strategy: matrix: compiler: @@ -121,7 +122,7 @@ jobs: ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }} use-system-libcrypto: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-22.04 # latest steps: - name: Build ${{ env.PACKAGE_NAME }} env: @@ -156,7 +157,7 @@ jobs: osx: - runs-on: macos-11 # latest + runs-on: macos-13 # latest steps: - name: Build ${{ env.PACKAGE_NAME }} + consumers run: | @@ -211,7 +212,7 @@ jobs: # check that tests requiring custom env-vars or AWS credentials are simply skipped tests-ok-without-env-vars: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-22.04 # latest steps: - uses: actions/checkout@v3 with: @@ -228,7 +229,7 @@ jobs: python3 -m unittest discover --failfast --verbose package-source: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-22.04 # latest steps: - uses: actions/checkout@v3 with: @@ -242,7 +243,7 @@ jobs: # check that docs can still build check-docs: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-22.04 # latest steps: - uses: actions/checkout@v3 with: @@ -254,7 +255,7 @@ jobs: ./scripts/make-docs.py check-submodules: - runs-on: ubuntu-20.04 # latest + runs-on: ubuntu-22.04 # latest steps: - name: Checkout Source uses: actions/checkout@v3 diff --git a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh index 753249370..afa4a8dd8 100755 --- a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh @@ -19,6 +19,9 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl +/opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp312*.whl + rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh index 8d3b874c9..db1436715 100755 --- a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh @@ -19,6 +19,9 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl +/opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp312*.whl + rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-osx.sh b/continuous-delivery/build-wheels-osx.sh index 18b4b993d..55470e5bc 100755 --- a/continuous-delivery/build-wheels-osx.sh +++ b/continuous-delivery/build-wheels-osx.sh @@ -10,5 +10,6 @@ set -ex /Library/Frameworks/Python.framework/Versions/3.9/bin/python3 setup.py sdist bdist_wheel /Library/Frameworks/Python.framework/Versions/3.10/bin/python3 setup.py sdist bdist_wheel /Library/Frameworks/Python.framework/Versions/3.11/bin/python3 setup.py sdist bdist_wheel +/Library/Frameworks/Python.framework/Versions/3.12/bin/python3 setup.py sdist bdist_wheel #now you just need to run twine (that's in a different script) diff --git a/continuous-delivery/build-wheels-win32.bat b/continuous-delivery/build-wheels-win32.bat index 6a3e53977..15a5a2725 100644 --- a/continuous-delivery/build-wheels-win32.bat +++ b/continuous-delivery/build-wheels-win32.bat @@ -6,6 +6,7 @@ "C:\Program Files (x86)\Python39-32\python.exe" setup.py sdist bdist_wheel || goto error "C:\Program Files (x86)\Python310-32\python.exe" setup.py sdist bdist_wheel || goto error "C:\Program Files (x86)\Python311-32\python.exe" setup.py sdist bdist_wheel || goto error +"C:\Program Files (x86)\Python312-32\python.exe" setup.py sdist bdist_wheel || goto error goto :EOF diff --git a/continuous-delivery/build-wheels-win64.bat b/continuous-delivery/build-wheels-win64.bat index 5a862e6e1..f3e644d4f 100644 --- a/continuous-delivery/build-wheels-win64.bat +++ b/continuous-delivery/build-wheels-win64.bat @@ -5,6 +5,7 @@ "C:\Program Files\Python39\python.exe" setup.py sdist bdist_wheel || goto error "C:\Program Files\Python310\python.exe" setup.py sdist bdist_wheel || goto error "C:\Program Files\Python311\python.exe" setup.py sdist bdist_wheel || goto error +"C:\Program Files\Python312\python.exe" setup.py sdist bdist_wheel || goto error goto :EOF From 62bbc386aba5066cf93b35ffade5067c01b11b4b Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Wed, 7 Jun 2023 13:30:20 -0700 Subject: [PATCH 02/29] fix version --- continuous-delivery/build-wheels-manylinux2014-aarch64.sh | 2 +- continuous-delivery/build-wheels-manylinux2014-x86_64.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh index afa4a8dd8..0eb1e74e9 100755 --- a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh @@ -19,7 +19,7 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl -/opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel +/opt/python/cp312-cp312/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp312*.whl rm dist/*.whl diff --git a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh index db1436715..1f0b32c4a 100755 --- a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh @@ -19,7 +19,7 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl -/opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel +/opt/python/cp312-cp312/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp312*.whl rm dist/*.whl From c0c261f2b64287eacf75bb6cd57f6843252ac2a2 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Wed, 7 Jun 2023 13:47:22 -0700 Subject: [PATCH 03/29] fix wrap_socket --- test/test_http_client.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/test_http_client.py b/test/test_http_client.py index 5af87b647..a9cb4593d 100644 --- a/test/test_http_client.py +++ b/test/test_http_client.py @@ -57,10 +57,9 @@ def _start_server(self, secure, http_1_0=False): self.server = HTTPServer((self.hostname, 0), TestRequestHandler) if secure: - self.server.socket = ssl.wrap_socket(self.server.socket, - keyfile="test/resources/unittest.key", - certfile='test/resources/unittest.crt', - server_side=True) + context = ssl.SSLContext() + context.load_cert_chain(certfile='test/resources/unittest.crt', keyfile="test/resources/unittest.key") + self.server.socket = context.wrap_socket(self.server.socket, server_side=True) self.port = self.server.server_address[1] # put requests are stored in this dict From db0ed315bff18909e08a5ad393da7331732f7aca Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 8 Jun 2023 10:48:24 -0700 Subject: [PATCH 04/29] update api for stable 311 api --- setup.py | 29 ++++++++++++++++++++++++++--- source/auth_credentials.c | 8 ++++---- source/event_stream_headers.c | 6 +++--- source/http_headers.c | 14 +++++++------- source/http_stream.c | 2 +- source/module.c | 13 +++++-------- source/mqtt5_client.c | 24 ++++++++++++------------ source/mqtt_client_connection.c | 18 +++++++++--------- source/s3_meta_request.c | 2 +- source/websocket.c | 10 +++++----- 10 files changed, 73 insertions(+), 53 deletions(-) diff --git a/setup.py b/setup.py index 36a8cdc44..cd895c8dd 100644 --- a/setup.py +++ b/setup.py @@ -14,10 +14,11 @@ import subprocess import sys import sysconfig +from wheel.bdist_wheel import bdist_wheel def is_64bit(): - return sys.maxsize > 2**32 + return sys.maxsize > 2 ** 32 def is_32bit(): @@ -152,7 +153,6 @@ def __init__(self, name, extra_cmake_args=[], libname=None): AWS_LIBS.append(AwsLib('aws-c-mqtt')) AWS_LIBS.append(AwsLib('aws-c-s3')) - PROJECT_DIR = os.path.dirname(os.path.realpath(__file__)) VERSION_RE = re.compile(r""".*__version__ = ["'](.*?)['"]""", re.S) @@ -283,6 +283,16 @@ def run(self): super().run() +class bdist_wheel_abi3(bdist_wheel): + def get_tag(self): + python, abi, plat = super().get_tag() + if python.startswith("cp") and sys.version_info >= (3, 11): + # on CPython, our wheels are abi3 and compatible back to 3.11 + return "cp311", "abi3", plat + + return python, abi, plat + + def awscrt_ext(): # fetch the CFLAGS/LDFLAGS from env extra_compile_args = os.environ.get('CFLAGS', '').split() @@ -350,6 +360,19 @@ def awscrt_ext(): if not is_macos_universal2(): extra_link_args += ['-Wl,-fatal_warnings'] + if sys.version_info >= (3, 11): + return setuptools.Extension( + '_awscrt', + language='c', + libraries=libraries, + sources=glob.glob('source/*.c'), + extra_compile_args=extra_compile_args, + extra_link_args=extra_link_args, + extra_objects=extra_objects, + define_macros=[('Py_LIMITED_API', '0x030B0000')], + py_limited_api=True, + ) + return setuptools.Extension( '_awscrt', language='c', @@ -392,6 +415,6 @@ def _load_version(): ], python_requires='>=3.7', ext_modules=[awscrt_ext()], - cmdclass={'build_ext': awscrt_build_ext}, + cmdclass={'build_ext': awscrt_build_ext, "bdist_wheel": bdist_wheel_abi3}, test_suite='test', ) diff --git a/source/auth_credentials.c b/source/auth_credentials.c index 80bca197c..ba7a9aab0 100644 --- a/source/auth_credentials.c +++ b/source/auth_credentials.c @@ -513,7 +513,7 @@ PyObject *aws_py_credentials_provider_new_chain(PyObject *self, PyObject *args) if (!providers_pyseq) { goto done; } - size_t provider_count = (size_t)PySequence_Fast_GET_SIZE(providers_pyseq); + size_t provider_count = (size_t)PySequence_Size(providers_pyseq); if (provider_count == 0) { PyErr_SetString(PyExc_ValueError, "Must supply at least one AwsCredentialsProvider."); goto done; @@ -526,7 +526,7 @@ PyObject *aws_py_credentials_provider_new_chain(PyObject *self, PyObject *args) } for (size_t i = 0; i < provider_count; ++i) { - PyObject *provider_py = PySequence_Fast_GET_ITEM(providers_pyseq, i); + PyObject *provider_py = PySequence_GetItem(providers_pyseq, i); providers_carray[i] = aws_py_get_credentials_provider(provider_py); if (!providers_carray[i]) { goto done; @@ -724,7 +724,7 @@ PyObject *aws_py_credentials_provider_new_cognito(PyObject *self, PyObject *args goto done; } - logins_count = (size_t)PySequence_Fast_GET_SIZE(logins_pyseq); + logins_count = (size_t)PySequence_Size(logins_pyseq); if (logins_count > 0) { logins_carray = @@ -735,7 +735,7 @@ PyObject *aws_py_credentials_provider_new_cognito(PyObject *self, PyObject *args } for (size_t i = 0; i < logins_count; ++i) { - PyObject *login_tuple_py = PySequence_Fast_GET_ITEM(logins_pyseq, i); + PyObject *login_tuple_py = PySequence_GetItem(logins_pyseq, i); struct aws_cognito_identity_provider_token_pair *login_entry = &logins_carray[i]; AWS_ZERO_STRUCT(*login_entry); diff --git a/source/event_stream_headers.c b/source/event_stream_headers.c index f8ae4ef41..1335bde20 100644 --- a/source/event_stream_headers.c +++ b/source/event_stream_headers.c @@ -171,10 +171,10 @@ bool aws_py_event_stream_native_headers_init(struct aws_array_list *native_heade goto done; } - const Py_ssize_t count = PySequence_Fast_GET_SIZE(sequence_py); + const Py_ssize_t count = PySequence_Size(sequence_py); for (Py_ssize_t i = 0; i < count; ++i) { /* Borrowed reference, don't need to decref */ - PyObject *header_py = PySequence_Fast_GET_ITEM(sequence_py, i); + PyObject *header_py = PySequence_GetItem(sequence_py, i); if (!s_add_native_header(native_headers, header_py)) { goto done; @@ -270,7 +270,7 @@ PyObject *aws_py_event_stream_python_headers_create( goto error; } - PyList_SET_ITEM(list_py, i, tuple_py); /* steals reference to tuple */ + PyList_SetItem(list_py, i, tuple_py); /* steals reference to tuple */ } return list_py; diff --git a/source/http_headers.c b/source/http_headers.c index 187cd7c0e..344904010 100644 --- a/source/http_headers.c +++ b/source/http_headers.c @@ -96,18 +96,18 @@ PyObject *aws_py_http_headers_add_pairs(PyObject *self, PyObject *args) { return NULL; } - const Py_ssize_t count = PySequence_Fast_GET_SIZE(py_sequence); + const Py_ssize_t count = PySequence_Size(py_sequence); for (Py_ssize_t i = 0; i < count; ++i) { /* XYZ_GET_ITEM() calls returns borrowed references */ - PyObject *py_pair = PySequence_Fast_GET_ITEM(py_sequence, i); + PyObject *py_pair = PySequence_GetItem(py_sequence, i); - if (!PyTuple_Check(py_pair) || PyTuple_GET_SIZE(py_pair) != 2) { + if (!PyTuple_Check(py_pair) || PyTuple_Size(py_pair) != 2) { PyErr_SetString(PyExc_TypeError, type_errmsg); goto done; } - struct aws_byte_cursor name = aws_byte_cursor_from_pyunicode(PyTuple_GET_ITEM(py_pair, 0)); - struct aws_byte_cursor value = aws_byte_cursor_from_pyunicode(PyTuple_GET_ITEM(py_pair, 1)); + struct aws_byte_cursor name = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 0)); + struct aws_byte_cursor value = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 1)); if (!name.ptr || !value.ptr) { PyErr_SetString(PyExc_TypeError, type_errmsg); goto done; @@ -175,8 +175,8 @@ static PyObject *s_py_tuple_from_header(struct aws_http_header header) { goto error; } - PyTuple_SET_ITEM(py_pair, 0, py_name); - PyTuple_SET_ITEM(py_pair, 1, py_value); + PyTuple_SetItem(py_pair, 0, py_name); + PyTuple_SetItem(py_pair, 1, py_value); return py_pair; error: diff --git a/source/http_stream.c b/source/http_stream.c index 18ed406eb..bf7021862 100644 --- a/source/http_stream.c +++ b/source/http_stream.c @@ -121,7 +121,7 @@ static int s_on_incoming_header_block_done( goto done; } - PyList_SET_ITEM(header_list, i, tuple); /* steals reference to tuple */ + PyList_SetItem(header_list, i, tuple); /* steals reference to tuple */ } /* TODO: handle informational and trailing headers */ diff --git a/source/module.c b/source/module.c index ca8a0d986..41d7fef1b 100644 --- a/source/module.c +++ b/source/module.c @@ -533,18 +533,16 @@ void *aws_py_get_binding(PyObject *obj, const char *capsule_name, const char *cl if (!py_binding) { return PyErr_Format( PyExc_TypeError, - "Expected valid '%s', received '%s' (no '_binding' attribute)", - class_name, - Py_TYPE(obj)->tp_name); + "Expected valid '%s' (no '_binding' attribute)", + class_name); } void *binding = NULL; if (!PyCapsule_CheckExact(py_binding)) { PyErr_Format( PyExc_TypeError, - "Expected valid '%s', received '%s' ('_binding' attribute is not a capsule)", - class_name, - Py_TYPE(obj)->tp_name); + "Expected valid '%s' ('_binding' attribute is not a capsule)", + class_name); goto done; } @@ -552,9 +550,8 @@ void *aws_py_get_binding(PyObject *obj, const char *capsule_name, const char *cl if (!binding) { PyErr_Format( PyExc_TypeError, - "Expected valid '%s', received '%s' ('_binding' attribute does not contain '%s')", + "Expected valid '%s' ('_binding' attribute does not contain '%s')", class_name, - Py_TYPE(obj)->tp_name, capsule_name); goto done; } diff --git a/source/mqtt5_client.c b/source/mqtt5_client.c index e07eb914a..69bc736f6 100644 --- a/source/mqtt5_client.c +++ b/source/mqtt5_client.c @@ -200,7 +200,7 @@ static PyObject *s_aws_set_user_properties_to_PyObject( Py_XDECREF(user_properties_list); return NULL; } - PyList_SET_ITEM(user_properties_list, i, tuple); /* Steals reference to tuple */ + PyList_SetItem(user_properties_list, i, tuple); /* Steals reference to tuple */ } return user_properties_list; } @@ -237,7 +237,7 @@ static void s_on_publish_received(const struct aws_mqtt5_packet_publish_view *pu } for (size_t i = 0; i < subscription_identifier_count; ++i) { - PyList_SET_ITEM( + PyList_SetItem( subscription_identifier_list, i, PyLong_FromLongLong(publish_packet->subscription_identifiers[i])); } @@ -1629,7 +1629,7 @@ static void s_on_subscribe_complete_fn( } for (size_t i = 0; i < reason_codes_count; ++i) { - PyList_SET_ITEM(reason_codes_list, i, PyLong_FromLong(suback->reason_codes[i])); + PyList_SetItem(reason_codes_list, i, PyLong_FromLong(suback->reason_codes[i])); } } @@ -1860,7 +1860,7 @@ static void s_on_unsubscribe_complete_fn( } for (size_t i = 0; i < reason_codes_count; ++i) { - PyList_SET_ITEM(reason_codes_list, i, PyLong_FromLong(unsuback->reason_codes[i])); + PyList_SetItem(reason_codes_list, i, PyLong_FromLong(unsuback->reason_codes[i])); } } @@ -2030,23 +2030,23 @@ PyObject *aws_py_mqtt5_client_get_stats(PyObject *self, PyObject *args) { goto done; } - PyTuple_SET_ITEM(result, 0, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_count)); - if (PyTuple_GET_ITEM(result, 0) == NULL) { + PyTuple_SetItem(result, 0, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_count)); + if (PyTuple_GetItem(result, 0) == NULL) { goto done; } - PyTuple_SET_ITEM(result, 1, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_size)); - if (PyTuple_GET_ITEM(result, 1) == NULL) { + PyTuple_SetItem(result, 1, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_size)); + if (PyTuple_GetItem(result, 1) == NULL) { goto done; } - PyTuple_SET_ITEM(result, 2, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_count)); - if (PyTuple_GET_ITEM(result, 2) == NULL) { + PyTuple_SetItem(result, 2, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_count)); + if (PyTuple_GetItem(result, 2) == NULL) { goto done; } - PyTuple_SET_ITEM(result, 3, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_size)); - if (PyTuple_GET_ITEM(result, 3) == NULL) { + PyTuple_SetItem(result, 3, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_size)); + if (PyTuple_GetItem(result, 3) == NULL) { goto done; } diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index bd06d290e..2e1120885 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -1063,7 +1063,7 @@ static void s_suback_multi_callback( goto done_prepping_args; } - PyList_SET_ITEM(topic_qos_list, i, tuple); /* Steals reference to tuple */ + PyList_SetItem(topic_qos_list, i, tuple); /* Steals reference to tuple */ } done_prepping_args:; @@ -1202,23 +1202,23 @@ PyObject *aws_py_mqtt_client_connection_get_stats(PyObject *self, PyObject *args goto done; } - PyTuple_SET_ITEM(result, 0, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_count)); - if (PyTuple_GET_ITEM(result, 0) == NULL) { + PyTuple_SetItem(result, 0, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_count)); + if (PyTuple_GetItem(result, 0) == NULL) { goto done; } - PyTuple_SET_ITEM(result, 1, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_size)); - if (PyTuple_GET_ITEM(result, 1) == NULL) { + PyTuple_SetItem(result, 1, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_size)); + if (PyTuple_GetItem(result, 1) == NULL) { goto done; } - PyTuple_SET_ITEM(result, 2, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_count)); - if (PyTuple_GET_ITEM(result, 2) == NULL) { + PyTuple_SetItem(result, 2, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_count)); + if (PyTuple_GetItem(result, 2) == NULL) { goto done; } - PyTuple_SET_ITEM(result, 3, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_size)); - if (PyTuple_GET_ITEM(result, 3) == NULL) { + PyTuple_SetItem(result, 3, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_size)); + if (PyTuple_GetItem(result, 3) == NULL) { goto done; } diff --git a/source/s3_meta_request.c b/source/s3_meta_request.c index 88a7c19d7..f7ffa56bd 100644 --- a/source/s3_meta_request.c +++ b/source/s3_meta_request.c @@ -69,7 +69,7 @@ static PyObject *s_get_py_headers(const struct aws_http_headers *headers) { if (!tuple) { goto error; } - PyList_SET_ITEM(header_list, i, tuple); /* steals reference to tuple */ + PyList_SetItem(header_list, i, tuple); /* steals reference to tuple */ } return header_list; error: diff --git a/source/websocket.c b/source/websocket.c index 2b056ee8a..2d96f92da 100644 --- a/source/websocket.c +++ b/source/websocket.c @@ -211,13 +211,13 @@ static void s_websocket_on_connection_setup( PyObject *name_py = PyUnicode_FromAwsByteCursor(&header_i->name); AWS_FATAL_ASSERT(name_py && "header name wrangling failed"); - PyTuple_SET_ITEM(tuple_py, 0, name_py); + PyTuple_SetItem(tuple_py, 0, name_py); PyObject *value_py = PyUnicode_FromAwsByteCursor(&header_i->value); AWS_FATAL_ASSERT(value_py && "header value wrangling failed"); - PyTuple_SET_ITEM(tuple_py, 1, value_py); + PyTuple_SetItem(tuple_py, 1, value_py); - PyList_SET_ITEM(headers_py, i, tuple_py); + PyList_SetItem(headers_py, i, tuple_py); } } @@ -580,13 +580,13 @@ PyObject *aws_py_websocket_create_handshake_request(PyObject *self, PyObject *ar if (!request_binding_py) { goto cleanup; } - PyTuple_SET_ITEM(tuple_py, 0, request_binding_py); /* steals reference to request_binding_py */ + PyTuple_SetItem(tuple_py, 0, request_binding_py); /* steals reference to request_binding_py */ PyObject *headers_binding_py = aws_py_http_headers_new_from_native(aws_http_message_get_headers(request)); if (!headers_binding_py) { goto cleanup; } - PyTuple_SET_ITEM(tuple_py, 1, headers_binding_py); /* steals reference to headers_binding_py */ + PyTuple_SetItem(tuple_py, 1, headers_binding_py); /* steals reference to headers_binding_py */ /* Success! */ success = true; From f4511ff5b45115b84783c25783e92199d78fc571 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 8 Jun 2023 10:49:31 -0700 Subject: [PATCH 05/29] remove 312 wheel --- continuous-delivery/build-wheels-manylinux2014-aarch64.sh | 3 --- continuous-delivery/build-wheels-manylinux2014-x86_64.sh | 3 --- continuous-delivery/build-wheels-osx.sh | 1 - continuous-delivery/build-wheels-win32.bat | 1 - continuous-delivery/build-wheels-win64.bat | 1 - 5 files changed, 9 deletions(-) diff --git a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh index 0eb1e74e9..753249370 100755 --- a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh @@ -19,9 +19,6 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl -/opt/python/cp312-cp312/bin/python setup.py sdist bdist_wheel -auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp312*.whl - rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh index 1f0b32c4a..8d3b874c9 100755 --- a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh @@ -19,9 +19,6 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl -/opt/python/cp312-cp312/bin/python setup.py sdist bdist_wheel -auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp312*.whl - rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-osx.sh b/continuous-delivery/build-wheels-osx.sh index 55470e5bc..18b4b993d 100755 --- a/continuous-delivery/build-wheels-osx.sh +++ b/continuous-delivery/build-wheels-osx.sh @@ -10,6 +10,5 @@ set -ex /Library/Frameworks/Python.framework/Versions/3.9/bin/python3 setup.py sdist bdist_wheel /Library/Frameworks/Python.framework/Versions/3.10/bin/python3 setup.py sdist bdist_wheel /Library/Frameworks/Python.framework/Versions/3.11/bin/python3 setup.py sdist bdist_wheel -/Library/Frameworks/Python.framework/Versions/3.12/bin/python3 setup.py sdist bdist_wheel #now you just need to run twine (that's in a different script) diff --git a/continuous-delivery/build-wheels-win32.bat b/continuous-delivery/build-wheels-win32.bat index 15a5a2725..6a3e53977 100644 --- a/continuous-delivery/build-wheels-win32.bat +++ b/continuous-delivery/build-wheels-win32.bat @@ -6,7 +6,6 @@ "C:\Program Files (x86)\Python39-32\python.exe" setup.py sdist bdist_wheel || goto error "C:\Program Files (x86)\Python310-32\python.exe" setup.py sdist bdist_wheel || goto error "C:\Program Files (x86)\Python311-32\python.exe" setup.py sdist bdist_wheel || goto error -"C:\Program Files (x86)\Python312-32\python.exe" setup.py sdist bdist_wheel || goto error goto :EOF diff --git a/continuous-delivery/build-wheels-win64.bat b/continuous-delivery/build-wheels-win64.bat index f3e644d4f..5a862e6e1 100644 --- a/continuous-delivery/build-wheels-win64.bat +++ b/continuous-delivery/build-wheels-win64.bat @@ -5,7 +5,6 @@ "C:\Program Files\Python39\python.exe" setup.py sdist bdist_wheel || goto error "C:\Program Files\Python310\python.exe" setup.py sdist bdist_wheel || goto error "C:\Program Files\Python311\python.exe" setup.py sdist bdist_wheel || goto error -"C:\Program Files\Python312\python.exe" setup.py sdist bdist_wheel || goto error goto :EOF From 5e65e02da6651652d3b67d3a9655daaaa18d8c99 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 8 Jun 2023 10:50:55 -0700 Subject: [PATCH 06/29] lint fix --- source/module.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/source/module.c b/source/module.c index 41d7fef1b..f120daaa7 100644 --- a/source/module.c +++ b/source/module.c @@ -531,18 +531,12 @@ void *aws_py_get_binding(PyObject *obj, const char *capsule_name, const char *cl PyObject *py_binding = PyObject_GetAttrString(obj, "_binding"); /* new reference */ if (!py_binding) { - return PyErr_Format( - PyExc_TypeError, - "Expected valid '%s' (no '_binding' attribute)", - class_name); + return PyErr_Format(PyExc_TypeError, "Expected valid '%s' (no '_binding' attribute)", class_name); } void *binding = NULL; if (!PyCapsule_CheckExact(py_binding)) { - PyErr_Format( - PyExc_TypeError, - "Expected valid '%s' ('_binding' attribute is not a capsule)", - class_name); + PyErr_Format(PyExc_TypeError, "Expected valid '%s' ('_binding' attribute is not a capsule)", class_name); goto done; } From 66bd959a0e508fe57838c4573b172eee3d41bcec Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 8 Jun 2023 11:21:43 -0700 Subject: [PATCH 07/29] fix mem leaks --- source/auth_credentials.c | 2 ++ source/event_stream_headers.c | 1 + source/http_headers.c | 1 + 3 files changed, 4 insertions(+) diff --git a/source/auth_credentials.c b/source/auth_credentials.c index ba7a9aab0..1dab1dfca 100644 --- a/source/auth_credentials.c +++ b/source/auth_credentials.c @@ -531,6 +531,7 @@ PyObject *aws_py_credentials_provider_new_chain(PyObject *self, PyObject *args) if (!providers_carray[i]) { goto done; } + Py_XDECREF(provider_py); } struct credentials_provider_binding *binding; @@ -752,6 +753,7 @@ PyObject *aws_py_credentials_provider_new_cognito(PyObject *self, PyObject *args i); goto done; } + Py_XDECREF(login_tuple_py); } } } diff --git a/source/event_stream_headers.c b/source/event_stream_headers.c index 1335bde20..d929bbf97 100644 --- a/source/event_stream_headers.c +++ b/source/event_stream_headers.c @@ -179,6 +179,7 @@ bool aws_py_event_stream_native_headers_init(struct aws_array_list *native_heade if (!s_add_native_header(native_headers, header_py)) { goto done; } + Py_XDECREF(header_py); } success = true; diff --git a/source/http_headers.c b/source/http_headers.c index 344904010..6a9efc555 100644 --- a/source/http_headers.c +++ b/source/http_headers.c @@ -108,6 +108,7 @@ PyObject *aws_py_http_headers_add_pairs(PyObject *self, PyObject *args) { struct aws_byte_cursor name = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 0)); struct aws_byte_cursor value = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 1)); + Py_DECREF(py_pair); if (!name.ptr || !value.ptr) { PyErr_SetString(PyExc_TypeError, type_errmsg); goto done; From 6042562951615768b32503161dd11ad419a7487b Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 8 Jun 2023 12:42:48 -0700 Subject: [PATCH 08/29] update builder --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dae3c8635..805466ff9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,8 +7,8 @@ on: - 'docs' env: - BUILDER_VERSION: v0.9.45 - BUILDER_SOURCE: releases + BUILDER_VERSION: python-312 + BUILDER_SOURCE: channels BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-python LINUX_BASE_IMAGE: ubuntu-18-x64 From 70a039b9237372f877b7bef98c5e4acde03e2b0c Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 8 Jun 2023 12:46:00 -0700 Subject: [PATCH 09/29] revert ubuntu --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 805466ff9..02218878a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,8 +7,8 @@ on: - 'docs' env: - BUILDER_VERSION: python-312 - BUILDER_SOURCE: channels + BUILDER_VERSION: v0.9.45 + BUILDER_SOURCE: releases BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-python LINUX_BASE_IMAGE: ubuntu-18-x64 @@ -122,7 +122,7 @@ jobs: ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }} use-system-libcrypto: - runs-on: ubuntu-22.04 # latest + runs-on: ubuntu-20.04 # latest steps: - name: Build ${{ env.PACKAGE_NAME }} env: From 3785f8f2f2bc16cc9fe0928557c81f14884374b0 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 8 Jun 2023 13:20:24 -0700 Subject: [PATCH 10/29] latest submodule --- crt/aws-c-common | 2 +- crt/aws-c-http | 2 +- crt/aws-c-io | 2 +- crt/aws-c-mqtt | 2 +- crt/aws-c-s3 | 2 +- crt/aws-c-sdkutils | 2 +- crt/aws-checksums | 2 +- crt/s2n | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crt/aws-c-common b/crt/aws-c-common index 7036fcfd2..9f36d0718 160000 --- a/crt/aws-c-common +++ b/crt/aws-c-common @@ -1 +1 @@ -Subproject commit 7036fcfd2949a65bda0af1f09cc9862c4c39088b +Subproject commit 9f36d07189435db27f746c10e39ba58511fadd0d diff --git a/crt/aws-c-http b/crt/aws-c-http index ac18abb48..27efc273f 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit ac18abb489c1baeafb2fb9dec0c6d223601ad590 +Subproject commit 27efc273f228306031c7345de8acaeee956db765 diff --git a/crt/aws-c-io b/crt/aws-c-io index 118a57ecf..af3af9967 160000 --- a/crt/aws-c-io +++ b/crt/aws-c-io @@ -1 +1 @@ -Subproject commit 118a57ecf0f88e047bb3e75227e08adb5782074c +Subproject commit af3af9967ff0f093520228b0eb4d55fc0e36b391 diff --git a/crt/aws-c-mqtt b/crt/aws-c-mqtt index b672bff09..8e40355e0 160000 --- a/crt/aws-c-mqtt +++ b/crt/aws-c-mqtt @@ -1 +1 @@ -Subproject commit b672bff0907603987bc93dae946c4f121c80e14c +Subproject commit 8e40355e03b456bb7b4bf0cd37c5d878afa2476f diff --git a/crt/aws-c-s3 b/crt/aws-c-s3 index 6203b2305..f14d92f58 160000 --- a/crt/aws-c-s3 +++ b/crt/aws-c-s3 @@ -1 +1 @@ -Subproject commit 6203b23057e40fcc73f44e3425e4942cbcf66308 +Subproject commit f14d92f581161686dad97c82aa994862da195f96 diff --git a/crt/aws-c-sdkutils b/crt/aws-c-sdkutils index 812761fdb..df511a1f2 160000 --- a/crt/aws-c-sdkutils +++ b/crt/aws-c-sdkutils @@ -1 +1 @@ -Subproject commit 812761fdbf791f77cb358212cefade9cc16974e7 +Subproject commit df511a1f2518279eb5721ab5fca6bc816efc6b32 diff --git a/crt/aws-checksums b/crt/aws-checksums index 1aa903d52..a5b0e7f00 160000 --- a/crt/aws-checksums +++ b/crt/aws-checksums @@ -1 +1 @@ -Subproject commit 1aa903d5204ed665010204d8521e56e092932e32 +Subproject commit a5b0e7f00be4240f77d3b6e090c8bed3c9c2e536 diff --git a/crt/s2n b/crt/s2n index 9b7b1f334..b9c4d60de 160000 --- a/crt/s2n +++ b/crt/s2n @@ -1 +1 @@ -Subproject commit 9b7b1f334a4ab33397174e505955231ff77b3f85 +Subproject commit b9c4d60de49193cea31f0acfbddc47cc66d41d2c From 3796a83c03d3319133602245d509679498f33650 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Fri, 9 Jun 2023 12:58:36 -0700 Subject: [PATCH 11/29] Better DecRef --- setup.py | 19 ++++++--------- source/auth_credentials.c | 7 +++--- source/event_stream_headers.c | 6 ++--- source/http_headers.c | 45 +++++++++++++++++++++-------------- 4 files changed, 41 insertions(+), 36 deletions(-) diff --git a/setup.py b/setup.py index cd895c8dd..0214304cf 100644 --- a/setup.py +++ b/setup.py @@ -298,6 +298,8 @@ def awscrt_ext(): extra_compile_args = os.environ.get('CFLAGS', '').split() extra_link_args = os.environ.get('LDFLAGS', '').split() extra_objects = [] + macros = [] + py_limited_api = False libraries = [x.libname for x in AWS_LIBS] @@ -361,17 +363,8 @@ def awscrt_ext(): extra_link_args += ['-Wl,-fatal_warnings'] if sys.version_info >= (3, 11): - return setuptools.Extension( - '_awscrt', - language='c', - libraries=libraries, - sources=glob.glob('source/*.c'), - extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args, - extra_objects=extra_objects, - define_macros=[('Py_LIMITED_API', '0x030B0000')], - py_limited_api=True, - ) + macros = [('Py_LIMITED_API', '0x030B0000')] + py_limited_api = True return setuptools.Extension( '_awscrt', @@ -380,7 +373,9 @@ def awscrt_ext(): sources=glob.glob('source/*.c'), extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, - extra_objects=extra_objects + extra_objects=extra_objects, + define_macros=macros, + py_limited_api=py_limited_api, ) diff --git a/source/auth_credentials.c b/source/auth_credentials.c index 1dab1dfca..66b1afa8e 100644 --- a/source/auth_credentials.c +++ b/source/auth_credentials.c @@ -526,12 +526,12 @@ PyObject *aws_py_credentials_provider_new_chain(PyObject *self, PyObject *args) } for (size_t i = 0; i < provider_count; ++i) { - PyObject *provider_py = PySequence_GetItem(providers_pyseq, i); + PyObject *provider_py = PySequence_GetItem(providers_pyseq, i); /* new reference */ providers_carray[i] = aws_py_get_credentials_provider(provider_py); + Py_XDECREF(provider_py); if (!providers_carray[i]) { goto done; } - Py_XDECREF(provider_py); } struct credentials_provider_binding *binding; @@ -736,7 +736,7 @@ PyObject *aws_py_credentials_provider_new_cognito(PyObject *self, PyObject *args } for (size_t i = 0; i < logins_count; ++i) { - PyObject *login_tuple_py = PySequence_GetItem(logins_pyseq, i); + PyObject *login_tuple_py = PySequence_GetItem(logins_pyseq, i); /* New reference */ struct aws_cognito_identity_provider_token_pair *login_entry = &logins_carray[i]; AWS_ZERO_STRUCT(*login_entry); @@ -751,6 +751,7 @@ PyObject *aws_py_credentials_provider_new_cognito(PyObject *self, PyObject *args PyExc_TypeError, "cognito credentials provider: logins[%zu] is invalid, should be type (str, str)", i); + Py_XDECREF(login_tuple_py); goto done; } Py_XDECREF(login_tuple_py); diff --git a/source/event_stream_headers.c b/source/event_stream_headers.c index d929bbf97..cfb1ac33f 100644 --- a/source/event_stream_headers.c +++ b/source/event_stream_headers.c @@ -166,17 +166,17 @@ bool aws_py_event_stream_native_headers_init(struct aws_array_list *native_heade bool success = false; PyObject *sequence_py = NULL; - sequence_py = PySequence_Fast(headers_py, "Expected sequence of Headers"); /* New reference */ + sequence_py = PySequence_Fast(headers_py, "Expected sequence of Headers"); if (!sequence_py) { goto done; } const Py_ssize_t count = PySequence_Size(sequence_py); for (Py_ssize_t i = 0; i < count; ++i) { - /* Borrowed reference, don't need to decref */ - PyObject *header_py = PySequence_GetItem(sequence_py, i); + PyObject *header_py = PySequence_GetItem(sequence_py, i); /* New Reference */ if (!s_add_native_header(native_headers, header_py)) { + Py_XDECREF(header_py); goto done; } Py_XDECREF(header_py); diff --git a/source/http_headers.c b/source/http_headers.c index 6a9efc555..2646de796 100644 --- a/source/http_headers.c +++ b/source/http_headers.c @@ -85,6 +85,29 @@ PyObject *aws_py_http_headers_add(PyObject *self, PyObject *args) { Py_RETURN_NONE; } +static int s_http_headers_add_pair(PyObject *py_pair, struct aws_http_headers *headers) { + + const char *type_errmsg = "List of (name,value) pairs expected."; + if (!PyTuple_Check(py_pair) || PyTuple_Size(py_pair) != 2) { + PyErr_SetString(PyExc_TypeError, type_errmsg); + return AWS_OP_ERR; + } + + struct aws_byte_cursor name = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 0)); + struct aws_byte_cursor value = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 1)); + if (!name.ptr || !value.ptr) { + PyErr_SetString(PyExc_TypeError, type_errmsg); + return AWS_OP_ERR; + } + + if (aws_http_headers_add(headers, name, value)) { + PyErr_SetAwsLastError(); + return AWS_OP_ERR; + } + + return AWS_OP_SUCCESS; +} + PyObject *aws_py_http_headers_add_pairs(PyObject *self, PyObject *args) { PyObject *py_pairs; S_HEADERS_METHOD_START("O", &py_pairs); @@ -96,26 +119,12 @@ PyObject *aws_py_http_headers_add_pairs(PyObject *self, PyObject *args) { return NULL; } - const Py_ssize_t count = PySequence_Size(py_sequence); + const Py_ssize_t count = PySequence_Size(py_pairs); for (Py_ssize_t i = 0; i < count; ++i) { - /* XYZ_GET_ITEM() calls returns borrowed references */ - PyObject *py_pair = PySequence_GetItem(py_sequence, i); - - if (!PyTuple_Check(py_pair) || PyTuple_Size(py_pair) != 2) { - PyErr_SetString(PyExc_TypeError, type_errmsg); - goto done; - } - - struct aws_byte_cursor name = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 0)); - struct aws_byte_cursor value = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 1)); + PyObject *py_pair = PySequence_GetItem(py_sequence, i); /* New Reference */ + int error = s_http_headers_add_pair(py_pair, headers); Py_DECREF(py_pair); - if (!name.ptr || !value.ptr) { - PyErr_SetString(PyExc_TypeError, type_errmsg); - goto done; - } - - if (aws_http_headers_add(headers, name, value)) { - PyErr_SetAwsLastError(); + if(error) { goto done; } } From 232e5566f162c065851379369f18f134ba7088d9 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 12 Jun 2023 08:33:02 -0700 Subject: [PATCH 12/29] lint fix --- source/http_headers.c | 2 +- source/mqtt5_client.c | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/source/http_headers.c b/source/http_headers.c index 2646de796..642da8257 100644 --- a/source/http_headers.c +++ b/source/http_headers.c @@ -124,7 +124,7 @@ PyObject *aws_py_http_headers_add_pairs(PyObject *self, PyObject *args) { PyObject *py_pair = PySequence_GetItem(py_sequence, i); /* New Reference */ int error = s_http_headers_add_pair(py_pair, headers); Py_DECREF(py_pair); - if(error) { + if (error) { goto done; } } diff --git a/source/mqtt5_client.c b/source/mqtt5_client.c index 69bc736f6..2e37ca6c2 100644 --- a/source/mqtt5_client.c +++ b/source/mqtt5_client.c @@ -261,7 +261,9 @@ static void s_on_publish_received(const struct aws_mqtt5_packet_publish_view *pu /* i */ (int)(publish_packet->payload_format ? *publish_packet->payload_format : 0), /* O */ publish_packet->message_expiry_interval_seconds ? Py_True : Py_False, /* I */ - (unsigned int)(publish_packet->message_expiry_interval_seconds ? *publish_packet->message_expiry_interval_seconds : 0), + (unsigned int)(publish_packet->message_expiry_interval_seconds + ? *publish_packet->message_expiry_interval_seconds + : 0), /* O */ publish_packet->topic_alias ? Py_True : Py_False, /* H */ (unsigned short)(publish_packet->topic_alias ? *publish_packet->topic_alias : 0), /* s */ publish_packet->response_topic ? publish_packet->response_topic->ptr : NULL, @@ -529,7 +531,9 @@ static void s_lifecycle_event_disconnection( /* i */ (int)(disconnect ? disconnect->reason_code : 0), /* O */ (disconnect && disconnect->session_expiry_interval_seconds) ? Py_True : Py_False, /* I */ - (unsigned int)((disconnect && disconnect->session_expiry_interval_seconds) ? *disconnect->session_expiry_interval_seconds : 0), + (unsigned int)((disconnect && disconnect->session_expiry_interval_seconds) + ? *disconnect->session_expiry_interval_seconds + : 0), /* s */ (disconnect && disconnect->reason_string) ? disconnect->reason_string->ptr : NULL, /* # */ (disconnect && disconnect->reason_string) ? disconnect->reason_string->len : 0, /* O */ user_property_count > 0 ? user_properties_list : Py_None, From 6998ab6dbd269d2dbef38af340c230b5de5c592d Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 12 Jun 2023 08:42:32 -0700 Subject: [PATCH 13/29] Adds steal reference comment --- setup.py | 2 +- source/http_headers.c | 4 ++-- source/mqtt5_client.c | 16 ++++++++-------- source/websocket.c | 6 +++--- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index 0214304cf..cb57aa658 100644 --- a/setup.py +++ b/setup.py @@ -363,7 +363,7 @@ def awscrt_ext(): extra_link_args += ['-Wl,-fatal_warnings'] if sys.version_info >= (3, 11): - macros = [('Py_LIMITED_API', '0x030B0000')] + macros.append('Py_LIMITED_API', '0x030B0000') py_limited_api = True return setuptools.Extension( diff --git a/source/http_headers.c b/source/http_headers.c index 642da8257..55c1c4a43 100644 --- a/source/http_headers.c +++ b/source/http_headers.c @@ -185,8 +185,8 @@ static PyObject *s_py_tuple_from_header(struct aws_http_header header) { goto error; } - PyTuple_SetItem(py_pair, 0, py_name); - PyTuple_SetItem(py_pair, 1, py_value); + PyTuple_SetItem(py_pair, 0, py_name); /* Steals a reference */ + PyTuple_SetItem(py_pair, 1, py_value); /* Steals a reference */ return py_pair; error: diff --git a/source/mqtt5_client.c b/source/mqtt5_client.c index 2e37ca6c2..399ca7a58 100644 --- a/source/mqtt5_client.c +++ b/source/mqtt5_client.c @@ -2034,23 +2034,23 @@ PyObject *aws_py_mqtt5_client_get_stats(PyObject *self, PyObject *args) { goto done; } - PyTuple_SetItem(result, 0, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_count)); - if (PyTuple_GetItem(result, 0) == NULL) { + PyTuple_SetItem(result, 0, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_count)); /* Steals a reference */ + if (PyTuple_GetItem(result, 0) == NULL) { /* Steals a reference */ goto done; } - PyTuple_SetItem(result, 1, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_size)); - if (PyTuple_GetItem(result, 1) == NULL) { + PyTuple_SetItem(result, 1, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_size)); /* Steals a reference */ + if (PyTuple_GetItem(result, 1) == NULL) { /* Steals a reference */ goto done; } - PyTuple_SetItem(result, 2, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_count)); - if (PyTuple_GetItem(result, 2) == NULL) { + PyTuple_SetItem(result, 2, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_count)); /* Steals a reference */ + if (PyTuple_GetItem(result, 2) == NULL) { /* Steals a reference */ goto done; } - PyTuple_SetItem(result, 3, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_size)); - if (PyTuple_GetItem(result, 3) == NULL) { + PyTuple_SetItem(result, 3, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_size)); /* Steals a reference */ + if (PyTuple_GetItem(result, 3) == NULL) { /* Steals a reference */ goto done; } diff --git a/source/websocket.c b/source/websocket.c index 2d96f92da..95f9d8e6c 100644 --- a/source/websocket.c +++ b/source/websocket.c @@ -211,13 +211,13 @@ static void s_websocket_on_connection_setup( PyObject *name_py = PyUnicode_FromAwsByteCursor(&header_i->name); AWS_FATAL_ASSERT(name_py && "header name wrangling failed"); - PyTuple_SetItem(tuple_py, 0, name_py); + PyTuple_SetItem(tuple_py, 0, name_py); /* Steals a reference */ PyObject *value_py = PyUnicode_FromAwsByteCursor(&header_i->value); AWS_FATAL_ASSERT(value_py && "header value wrangling failed"); - PyTuple_SetItem(tuple_py, 1, value_py); + PyTuple_SetItem(tuple_py, 1, value_py); /* Steals a reference */ - PyList_SetItem(headers_py, i, tuple_py); + PyList_SetItem(headers_py, i, tuple_py); /* Steals a reference */ } } From b87e90999d2f9ef8402a63c278e145289ba99f1a Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 12 Jun 2023 08:45:19 -0700 Subject: [PATCH 14/29] add comments --- source/http_headers.c | 6 +++--- source/mqtt5_client.c | 36 +++++++++++++++++++++++---------- source/mqtt_client_connection.c | 28 +++++++++++++++++-------- 3 files changed, 48 insertions(+), 22 deletions(-) diff --git a/source/http_headers.c b/source/http_headers.c index 55c1c4a43..effa79f29 100644 --- a/source/http_headers.c +++ b/source/http_headers.c @@ -93,8 +93,8 @@ static int s_http_headers_add_pair(PyObject *py_pair, struct aws_http_headers *h return AWS_OP_ERR; } - struct aws_byte_cursor name = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 0)); - struct aws_byte_cursor value = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 1)); + struct aws_byte_cursor name = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 0) /* Steals a reference */); + struct aws_byte_cursor value = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 1) /* Steals a reference */); if (!name.ptr || !value.ptr) { PyErr_SetString(PyExc_TypeError, type_errmsg); return AWS_OP_ERR; @@ -185,7 +185,7 @@ static PyObject *s_py_tuple_from_header(struct aws_http_header header) { goto error; } - PyTuple_SetItem(py_pair, 0, py_name); /* Steals a reference */ + PyTuple_SetItem(py_pair, 0, py_name); /* Steals a reference */ PyTuple_SetItem(py_pair, 1, py_value); /* Steals a reference */ return py_pair; diff --git a/source/mqtt5_client.c b/source/mqtt5_client.c index 399ca7a58..42132c6c6 100644 --- a/source/mqtt5_client.c +++ b/source/mqtt5_client.c @@ -238,7 +238,9 @@ static void s_on_publish_received(const struct aws_mqtt5_packet_publish_view *pu for (size_t i = 0; i < subscription_identifier_count; ++i) { PyList_SetItem( - subscription_identifier_list, i, PyLong_FromLongLong(publish_packet->subscription_identifiers[i])); + subscription_identifier_list, + i, + PyLong_FromLongLong(publish_packet->subscription_identifiers[i])); /* Steals a reference */ } user_properties_list = s_aws_set_user_properties_to_PyObject(publish_packet->user_properties, user_property_count); @@ -1633,7 +1635,7 @@ static void s_on_subscribe_complete_fn( } for (size_t i = 0; i < reason_codes_count; ++i) { - PyList_SetItem(reason_codes_list, i, PyLong_FromLong(suback->reason_codes[i])); + PyList_SetItem(reason_codes_list, i, PyLong_FromLong(suback->reason_codes[i])); /* Steals a reference */ } } @@ -1864,7 +1866,7 @@ static void s_on_unsubscribe_complete_fn( } for (size_t i = 0; i < reason_codes_count; ++i) { - PyList_SetItem(reason_codes_list, i, PyLong_FromLong(unsuback->reason_codes[i])); + PyList_SetItem(reason_codes_list, i, PyLong_FromLong(unsuback->reason_codes[i])); /* Steals a reference */ } } @@ -2034,23 +2036,35 @@ PyObject *aws_py_mqtt5_client_get_stats(PyObject *self, PyObject *args) { goto done; } - PyTuple_SetItem(result, 0, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_count)); /* Steals a reference */ - if (PyTuple_GetItem(result, 0) == NULL) { /* Steals a reference */ + PyTuple_SetItem( + result, + 0, + PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_count)); /* Steals a reference */ + if (PyTuple_GetItem(result, 0) == NULL) { /* Steals a reference */ goto done; } - PyTuple_SetItem(result, 1, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_size)); /* Steals a reference */ - if (PyTuple_GetItem(result, 1) == NULL) { /* Steals a reference */ + PyTuple_SetItem( + result, + 1, + PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_size)); /* Steals a reference */ + if (PyTuple_GetItem(result, 1) == NULL) { /* Steals a reference */ goto done; } - PyTuple_SetItem(result, 2, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_count)); /* Steals a reference */ - if (PyTuple_GetItem(result, 2) == NULL) { /* Steals a reference */ + PyTuple_SetItem( + result, + 2, + PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_count)); /* Steals a reference */ + if (PyTuple_GetItem(result, 2) == NULL) { /* Steals a reference */ goto done; } - PyTuple_SetItem(result, 3, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_size)); /* Steals a reference */ - if (PyTuple_GetItem(result, 3) == NULL) { /* Steals a reference */ + PyTuple_SetItem( + result, + 3, + PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_size)); /* Steals a reference */ + if (PyTuple_GetItem(result, 3) == NULL) { /* Steals a reference */ goto done; } diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index 707ef6cb7..a201b2c4a 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -1329,23 +1329,35 @@ PyObject *aws_py_mqtt_client_connection_get_stats(PyObject *self, PyObject *args goto done; } - PyTuple_SetItem(result, 0, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_count)); - if (PyTuple_GetItem(result, 0) == NULL) { + PyTuple_SetItem( + result, + 0, + PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_count)); /* Steals a reference */ + if (PyTuple_GetItem(result, 0) == NULL) { /* Steals a reference */ goto done; } - PyTuple_SetItem(result, 1, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_size)); - if (PyTuple_GetItem(result, 1) == NULL) { + PyTuple_SetItem( + result, + 1, + PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_size)); /* Steals a reference */ + if (PyTuple_GetItem(result, 1) == NULL) { /* Steals a reference */ goto done; } - PyTuple_SetItem(result, 2, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_count)); - if (PyTuple_GetItem(result, 2) == NULL) { + PyTuple_SetItem( + result, + 2, + PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_count)); /* Steals a reference */ + if (PyTuple_GetItem(result, 2) == NULL) { /* Steals a reference */ goto done; } - PyTuple_SetItem(result, 3, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_size)); - if (PyTuple_GetItem(result, 3) == NULL) { + PyTuple_SetItem( + result, + 3, + PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_size)); /* Steals a reference */ + if (PyTuple_GetItem(result, 3) == NULL) { /* Steals a reference */ goto done; } From 1ca818e809a033d8b71343caeb33bad5256711c4 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 12 Jun 2023 08:58:02 -0700 Subject: [PATCH 15/29] fix append function --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cb57aa658..a678adf4a 100644 --- a/setup.py +++ b/setup.py @@ -363,7 +363,7 @@ def awscrt_ext(): extra_link_args += ['-Wl,-fatal_warnings'] if sys.version_info >= (3, 11): - macros.append('Py_LIMITED_API', '0x030B0000') + macros.append(('Py_LIMITED_API', '0x030B0000')) py_limited_api = True return setuptools.Extension( From 1260521baafb600cd45ac8c07b46bc25ca7ec249 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 12 Jun 2023 09:16:56 -0700 Subject: [PATCH 16/29] lint fix --- source/mqtt5_client.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/source/mqtt5_client.c b/source/mqtt5_client.c index 42132c6c6..a5a5463b1 100644 --- a/source/mqtt5_client.c +++ b/source/mqtt5_client.c @@ -263,9 +263,7 @@ static void s_on_publish_received(const struct aws_mqtt5_packet_publish_view *pu /* i */ (int)(publish_packet->payload_format ? *publish_packet->payload_format : 0), /* O */ publish_packet->message_expiry_interval_seconds ? Py_True : Py_False, /* I */ - (unsigned int)(publish_packet->message_expiry_interval_seconds - ? *publish_packet->message_expiry_interval_seconds - : 0), + (unsigned int)(publish_packet->message_expiry_interval_seconds ? *publish_packet->message_expiry_interval_seconds : 0), /* O */ publish_packet->topic_alias ? Py_True : Py_False, /* H */ (unsigned short)(publish_packet->topic_alias ? *publish_packet->topic_alias : 0), /* s */ publish_packet->response_topic ? publish_packet->response_topic->ptr : NULL, @@ -533,9 +531,7 @@ static void s_lifecycle_event_disconnection( /* i */ (int)(disconnect ? disconnect->reason_code : 0), /* O */ (disconnect && disconnect->session_expiry_interval_seconds) ? Py_True : Py_False, /* I */ - (unsigned int)((disconnect && disconnect->session_expiry_interval_seconds) - ? *disconnect->session_expiry_interval_seconds - : 0), + (unsigned int)((disconnect && disconnect->session_expiry_interval_seconds) ? *disconnect->session_expiry_interval_seconds : 0), /* s */ (disconnect && disconnect->reason_string) ? disconnect->reason_string->ptr : NULL, /* # */ (disconnect && disconnect->reason_string) ? disconnect->reason_string->len : 0, /* O */ user_property_count > 0 ? user_properties_list : Py_None, From b4cd04b7a5748faf5f050f862ef814c6975a4a85 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 12 Jun 2023 10:19:49 -0700 Subject: [PATCH 17/29] Update source/http_headers.c Co-authored-by: Michael Graeb --- source/http_headers.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/http_headers.c b/source/http_headers.c index effa79f29..4e8901191 100644 --- a/source/http_headers.c +++ b/source/http_headers.c @@ -93,8 +93,8 @@ static int s_http_headers_add_pair(PyObject *py_pair, struct aws_http_headers *h return AWS_OP_ERR; } - struct aws_byte_cursor name = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 0) /* Steals a reference */); - struct aws_byte_cursor value = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 1) /* Steals a reference */); + struct aws_byte_cursor name = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 0) /* Borrowed reference */); + struct aws_byte_cursor value = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 1) /* Borrowed reference */); if (!name.ptr || !value.ptr) { PyErr_SetString(PyExc_TypeError, type_errmsg); return AWS_OP_ERR; From 5540a0db0752bae9c877c0df77f74c37b6870a7a Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 12 Jun 2023 10:20:23 -0700 Subject: [PATCH 18/29] Update source/mqtt5_client.c Co-authored-by: Michael Graeb --- source/mqtt5_client.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/mqtt5_client.c b/source/mqtt5_client.c index a5a5463b1..9a3f9a9a6 100644 --- a/source/mqtt5_client.c +++ b/source/mqtt5_client.c @@ -2036,7 +2036,7 @@ PyObject *aws_py_mqtt5_client_get_stats(PyObject *self, PyObject *args) { result, 0, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_count)); /* Steals a reference */ - if (PyTuple_GetItem(result, 0) == NULL) { /* Steals a reference */ + if (PyTuple_GetItem(result, 0) == NULL) { /* Borrowed reference */ goto done; } From 0e85cd62221d32b341dec2a36f47b16ba18fb449 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 12 Jun 2023 10:20:35 -0700 Subject: [PATCH 19/29] Update source/mqtt5_client.c Co-authored-by: Michael Graeb --- source/mqtt5_client.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/mqtt5_client.c b/source/mqtt5_client.c index 9a3f9a9a6..9a3d65057 100644 --- a/source/mqtt5_client.c +++ b/source/mqtt5_client.c @@ -2044,7 +2044,7 @@ PyObject *aws_py_mqtt5_client_get_stats(PyObject *self, PyObject *args) { result, 1, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_size)); /* Steals a reference */ - if (PyTuple_GetItem(result, 1) == NULL) { /* Steals a reference */ + if (PyTuple_GetItem(result, 1) == NULL) { /* Borrowed reference */ goto done; } From 884289e7a400c3c528a5811d77ebffcd12b3d5de Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 12 Jun 2023 10:20:44 -0700 Subject: [PATCH 20/29] Update source/mqtt_client_connection.c Co-authored-by: Michael Graeb --- source/mqtt_client_connection.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index a201b2c4a..d0c0d87bd 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -1357,7 +1357,7 @@ PyObject *aws_py_mqtt_client_connection_get_stats(PyObject *self, PyObject *args result, 3, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_size)); /* Steals a reference */ - if (PyTuple_GetItem(result, 3) == NULL) { /* Steals a reference */ + if (PyTuple_GetItem(result, 3) == NULL) { /* Borrowed reference */ goto done; } From 61270a6dafb0740c76c3c9532a1d5cc5d82bac4c Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 12 Jun 2023 10:20:54 -0700 Subject: [PATCH 21/29] Update source/mqtt_client_connection.c Co-authored-by: Michael Graeb --- source/mqtt_client_connection.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index d0c0d87bd..c4794f637 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -1349,7 +1349,7 @@ PyObject *aws_py_mqtt_client_connection_get_stats(PyObject *self, PyObject *args result, 2, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_count)); /* Steals a reference */ - if (PyTuple_GetItem(result, 2) == NULL) { /* Steals a reference */ + if (PyTuple_GetItem(result, 2) == NULL) { /* Borrowed reference */ goto done; } From bf8e02a4766bc5d910c07e1cf53caa3364a70817 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 12 Jun 2023 10:21:00 -0700 Subject: [PATCH 22/29] Update source/mqtt_client_connection.c Co-authored-by: Michael Graeb --- source/mqtt_client_connection.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index c4794f637..dd5ba9c93 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -1341,7 +1341,7 @@ PyObject *aws_py_mqtt_client_connection_get_stats(PyObject *self, PyObject *args result, 1, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_size)); /* Steals a reference */ - if (PyTuple_GetItem(result, 1) == NULL) { /* Steals a reference */ + if (PyTuple_GetItem(result, 1) == NULL) { /* Borrowed reference */ goto done; } From b59045db31a1611ec7730232526aa2071dee7e7c Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 12 Jun 2023 10:21:07 -0700 Subject: [PATCH 23/29] Update source/mqtt_client_connection.c Co-authored-by: Michael Graeb --- source/mqtt_client_connection.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index dd5ba9c93..526157f29 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -1333,7 +1333,7 @@ PyObject *aws_py_mqtt_client_connection_get_stats(PyObject *self, PyObject *args result, 0, PyLong_FromUnsignedLongLong((unsigned long long)stats.incomplete_operation_count)); /* Steals a reference */ - if (PyTuple_GetItem(result, 0) == NULL) { /* Steals a reference */ + if (PyTuple_GetItem(result, 0) == NULL) { /* Borrowed reference */ goto done; } From 4015a94414bd579b1d119997652277cb49f3117d Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 12 Jun 2023 10:21:13 -0700 Subject: [PATCH 24/29] Update source/mqtt5_client.c Co-authored-by: Michael Graeb --- source/mqtt5_client.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/mqtt5_client.c b/source/mqtt5_client.c index 9a3d65057..5a17291b5 100644 --- a/source/mqtt5_client.c +++ b/source/mqtt5_client.c @@ -2060,7 +2060,7 @@ PyObject *aws_py_mqtt5_client_get_stats(PyObject *self, PyObject *args) { result, 3, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_size)); /* Steals a reference */ - if (PyTuple_GetItem(result, 3) == NULL) { /* Steals a reference */ + if (PyTuple_GetItem(result, 3) == NULL) { /* Borrowed reference */ goto done; } From 8ff4f03715764bc33d7153685f1403672c4bb516 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 12 Jun 2023 10:21:24 -0700 Subject: [PATCH 25/29] Update source/mqtt5_client.c Co-authored-by: Michael Graeb --- source/mqtt5_client.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/mqtt5_client.c b/source/mqtt5_client.c index 5a17291b5..c43c90c24 100644 --- a/source/mqtt5_client.c +++ b/source/mqtt5_client.c @@ -2052,7 +2052,7 @@ PyObject *aws_py_mqtt5_client_get_stats(PyObject *self, PyObject *args) { result, 2, PyLong_FromUnsignedLongLong((unsigned long long)stats.unacked_operation_count)); /* Steals a reference */ - if (PyTuple_GetItem(result, 2) == NULL) { /* Steals a reference */ + if (PyTuple_GetItem(result, 2) == NULL) { /* Borrowed reference */ goto done; } From 25509002e07659319d3f72763242580ba798da61 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 12 Jun 2023 11:11:09 -0700 Subject: [PATCH 26/29] PR feedback --- setup.py | 6 +++--- source/http_headers.c | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index a678adf4a..0b106b2c4 100644 --- a/setup.py +++ b/setup.py @@ -298,7 +298,7 @@ def awscrt_ext(): extra_compile_args = os.environ.get('CFLAGS', '').split() extra_link_args = os.environ.get('LDFLAGS', '').split() extra_objects = [] - macros = [] + define_macros = [] py_limited_api = False libraries = [x.libname for x in AWS_LIBS] @@ -363,7 +363,7 @@ def awscrt_ext(): extra_link_args += ['-Wl,-fatal_warnings'] if sys.version_info >= (3, 11): - macros.append(('Py_LIMITED_API', '0x030B0000')) + define_macros.append(('Py_LIMITED_API', '0x030B0000')) py_limited_api = True return setuptools.Extension( @@ -374,7 +374,7 @@ def awscrt_ext(): extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, extra_objects=extra_objects, - define_macros=macros, + define_macros=define_macros, py_limited_api=py_limited_api, ) diff --git a/source/http_headers.c b/source/http_headers.c index 4e8901191..fb6bfd822 100644 --- a/source/http_headers.c +++ b/source/http_headers.c @@ -85,27 +85,27 @@ PyObject *aws_py_http_headers_add(PyObject *self, PyObject *args) { Py_RETURN_NONE; } -static int s_http_headers_add_pair(PyObject *py_pair, struct aws_http_headers *headers) { +static bool s_py_http_headers_add_pair(PyObject *py_pair, struct aws_http_headers *headers) { const char *type_errmsg = "List of (name,value) pairs expected."; if (!PyTuple_Check(py_pair) || PyTuple_Size(py_pair) != 2) { PyErr_SetString(PyExc_TypeError, type_errmsg); - return AWS_OP_ERR; + return false; } struct aws_byte_cursor name = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 0) /* Borrowed reference */); struct aws_byte_cursor value = aws_byte_cursor_from_pyunicode(PyTuple_GetItem(py_pair, 1) /* Borrowed reference */); if (!name.ptr || !value.ptr) { PyErr_SetString(PyExc_TypeError, type_errmsg); - return AWS_OP_ERR; + return false; } if (aws_http_headers_add(headers, name, value)) { PyErr_SetAwsLastError(); - return AWS_OP_ERR; + return false; } - return AWS_OP_SUCCESS; + return true; } PyObject *aws_py_http_headers_add_pairs(PyObject *self, PyObject *args) { @@ -122,9 +122,9 @@ PyObject *aws_py_http_headers_add_pairs(PyObject *self, PyObject *args) { const Py_ssize_t count = PySequence_Size(py_pairs); for (Py_ssize_t i = 0; i < count; ++i) { PyObject *py_pair = PySequence_GetItem(py_sequence, i); /* New Reference */ - int error = s_http_headers_add_pair(py_pair, headers); + bool result = s_py_http_headers_add_pair(py_pair, headers); Py_DECREF(py_pair); - if (error) { + if (!result) { goto done; } } From ad411405cf7dd5312bf24de6ceb8ce8bb86b70cd Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 12 Jun 2023 11:45:15 -0700 Subject: [PATCH 27/29] update builder --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02218878a..ec4a37e85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: - 'docs' env: - BUILDER_VERSION: v0.9.45 + BUILDER_VERSION: v0.9.46 BUILDER_SOURCE: releases BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-python From b9e356ba6d61242265e4f5cdace755f3b6aa644a Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 12 Jun 2023 13:14:59 -0700 Subject: [PATCH 28/29] update variable name --- source/http_headers.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/http_headers.c b/source/http_headers.c index fb6bfd822..66974f9cf 100644 --- a/source/http_headers.c +++ b/source/http_headers.c @@ -122,9 +122,9 @@ PyObject *aws_py_http_headers_add_pairs(PyObject *self, PyObject *args) { const Py_ssize_t count = PySequence_Size(py_pairs); for (Py_ssize_t i = 0; i < count; ++i) { PyObject *py_pair = PySequence_GetItem(py_sequence, i); /* New Reference */ - bool result = s_py_http_headers_add_pair(py_pair, headers); + bool success = s_py_http_headers_add_pair(py_pair, headers); Py_DECREF(py_pair); - if (!result) { + if (!success) { goto done; } } From 19df8c3eabdaba008cc20bde1aebaf72d9364fe4 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Wed, 14 Jun 2023 11:12:26 -0700 Subject: [PATCH 29/29] MuslLinux Wheels (#480) --- .github/workflows/ci.yml | 26 ++++++++++++++++- builder.json | 4 +++ .../build-wheels-manylinux2014-aarch64.sh | 3 ++ .../build-wheels-manylinux2014-x86_64.sh | 3 ++ ...ld-wheels-musllinux-1-1-aarch64-jenkins.sh | 19 +++++++++++++ .../build-wheels-musllinux-1-1-aarch64.sh | 28 +++++++++++++++++++ ...ild-wheels-musllinux-1-1-x86_64-jenkins.sh | 19 +++++++++++++ .../build-wheels-musllinux-1-1-x86_64.sh | 28 +++++++++++++++++++ continuous-delivery/build-wheels-osx.sh | 3 ++ 9 files changed, 132 insertions(+), 1 deletion(-) create mode 100755 continuous-delivery/build-wheels-musllinux-1-1-aarch64-jenkins.sh create mode 100755 continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh create mode 100755 continuous-delivery/build-wheels-musllinux-1-1-x86_64-jenkins.sh create mode 100755 continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec4a37e85..7f158d02f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: - 'docs' env: - BUILDER_VERSION: v0.9.46 + BUILDER_VERSION: v0.9.47 BUILDER_SOURCE: releases BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-python @@ -63,6 +63,30 @@ jobs: aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + musllinux-1-1: + runs-on: ubuntu-22.04 # latest + strategy: + fail-fast: false + matrix: + image: + - x64 + - aarch64 + python: + - cp37-cp37m + - cp38-cp38 + - cp39-cp39 + - cp310-cp310 + - cp311-cp311 + - cp312-cp312 + steps: + # Only aarch64 needs this, but it doesn't hurt anything + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python raspberry: runs-on: ubuntu-20.04 # latest diff --git a/builder.json b/builder.json index 1668a5ad9..c2e186c31 100644 --- a/builder.json +++ b/builder.json @@ -7,6 +7,10 @@ "manylinux": { "_comment": "Use existing compiler on manylinux. These are the images we use for release. We want to be sure things work with the defaults.", "needs_compiler": false + }, + "musllinux": { + "_comment": "Use existing compiler on musllinux. These are the images we use for release. We want to be sure things work with the defaults.", + "needs_compiler": false } }, "targets": { diff --git a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh index 753249370..a140b7d67 100755 --- a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh @@ -19,6 +19,9 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl +# Don't need to build wheels for Python 3.12 and later. +# The 3.11 wheel uses the stable ABI, so it works with newer versions too. + rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh index 8d3b874c9..41ce9fdef 100755 --- a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh @@ -19,6 +19,9 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl +# Don't need to build wheels for Python 3.12 and later. +# The 3.11 wheel uses the stable ABI, so it works with newer versions too. + rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-musllinux-1-1-aarch64-jenkins.sh b/continuous-delivery/build-wheels-musllinux-1-1-aarch64-jenkins.sh new file mode 100755 index 000000000..2cefaed70 --- /dev/null +++ b/continuous-delivery/build-wheels-musllinux-1-1-aarch64-jenkins.sh @@ -0,0 +1,19 @@ +#!/bin/bash +#run build-wheels script in musllinux_1_1 docker image +set -ex + +DOCKER_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/aws-crt-musllinux-1-1-aarch64:latest + +$(aws --region us-east-1 ecr get-login --no-include-email) + +docker pull $DOCKER_IMAGE + +# NOTE: run as current user to avoid git "dubious ownership" error, +# and so that output artifacts don't belong to "root" +docker run --rm \ + --mount type=bind,source=`pwd`,target=/aws-crt-python \ + --user "$(id -u):$(id -g)" \ + --workdir /aws-crt-python \ + --entrypoint /bin/bash \ + $DOCKER_IMAGE \ + continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh diff --git a/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh b/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh new file mode 100755 index 000000000..0cbc1a196 --- /dev/null +++ b/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh @@ -0,0 +1,28 @@ +#!/bin/bash +#assumes image based on musllinux_1_1 +set -ex + +/opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py + +/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp37*.whl + +/opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp38*.whl + +/opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp39*.whl + +/opt/python/cp310-cp310/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp310*.whl + +/opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp311*.whl + +# Don't need to build wheels for Python 3.12 and later. +# The 3.11 wheel uses the stable ABI, so it works with newer versions too. + +rm dist/*.whl +cp -rv wheelhouse/* dist/ + +#now you just need to run twine (that's in a different script) diff --git a/continuous-delivery/build-wheels-musllinux-1-1-x86_64-jenkins.sh b/continuous-delivery/build-wheels-musllinux-1-1-x86_64-jenkins.sh new file mode 100755 index 000000000..00d6e5259 --- /dev/null +++ b/continuous-delivery/build-wheels-musllinux-1-1-x86_64-jenkins.sh @@ -0,0 +1,19 @@ +#!/bin/bash +#run build-wheels script in musllinux_1_1 docker image +set -ex + +DOCKER_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/aws-crt-musllinux-1-1-x64:latest + +$(aws --region us-east-1 ecr get-login --no-include-email) + +docker pull $DOCKER_IMAGE + +# NOTE: run as current user to avoid git "dubious ownership" error, +# and so that output artifacts don't belong to "root" +docker run --rm \ + --mount type=bind,source=`pwd`,target=/aws-crt-python \ + --user "$(id -u):$(id -g)" \ + --workdir /aws-crt-python \ + --entrypoint /bin/bash \ + $DOCKER_IMAGE \ + continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh diff --git a/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh b/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh new file mode 100755 index 000000000..6637d95ac --- /dev/null +++ b/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh @@ -0,0 +1,28 @@ +#!/bin/bash +#assumes image based on musllinux_1_1 + extras (pip) +set -ex + +/opt/python/cp39-cp39/bin/python ./continuous-delivery/update-version.py + +/opt/python/cp37-cp37m/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp37*.whl + +/opt/python/cp38-cp38/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp38*.whl + +/opt/python/cp39-cp39/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp39*.whl + +/opt/python/cp310-cp310/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp310*.whl + +/opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp311*.whl + +# Don't need to build wheels for Python 3.12 and later. +# The 3.11 wheel uses the stable ABI, so it works with newer versions too. + +rm dist/*.whl +cp -rv wheelhouse/* dist/ + +#now you just need to run twine (that's in a different script) diff --git a/continuous-delivery/build-wheels-osx.sh b/continuous-delivery/build-wheels-osx.sh index 18b4b993d..a3a92bc63 100755 --- a/continuous-delivery/build-wheels-osx.sh +++ b/continuous-delivery/build-wheels-osx.sh @@ -11,4 +11,7 @@ set -ex /Library/Frameworks/Python.framework/Versions/3.10/bin/python3 setup.py sdist bdist_wheel /Library/Frameworks/Python.framework/Versions/3.11/bin/python3 setup.py sdist bdist_wheel +# Don't need to build wheels for Python 3.12 and later. +# The 3.11 wheel uses the stable ABI, so it works with newer versions too. + #now you just need to run twine (that's in a different script)