From 6360266431c090684a4112346f949890e32cb083 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 20 Apr 2023 12:15:51 +0800 Subject: [PATCH 1/2] Add the CI to verify Python 3.7 and the fastavro dependency ### Motivation There is no workflow to verify the fastavro dependency works well for all supported Python versions, so the `[avro]` component was broken and not found before. See https://github.com/apache/pulsar-client-python/issues/67 ### Modifications - In all workflows, install the wheel with the `[avro]` component and verify `pulsar.schema` can be imported successfully. - Add workflows for the lowest Python version (3.7) we supported. --- .github/workflows/ci-pr-validation.yaml | 7 ++++++- pkg/mac/build-mac-wheels.sh | 8 ++++---- pkg/test-wheel.sh | 6 ++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index aae85514..320fa6e8 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -93,6 +93,7 @@ jobs: - {name: 'manylinux2014', py_suffix: ''} - {name: 'manylinux_musl', py_suffix: '-alpine'} python: + - {version: '3.7', spec: 'cp37-cp37m'} - {version: '3.11', spec: 'cp311-cp311'} cpu: - {arch: 'x86_64', platform: 'x86_64'} @@ -141,6 +142,7 @@ jobs: strategy: matrix: py: + - {version: '3.7', version_long: '3.7.16'} - {version: '3.11', version_long: '3.11.1'} steps: @@ -187,6 +189,7 @@ jobs: fail-fast: false matrix: python: + - version: '3.7' - version: '3.11' steps: @@ -227,8 +230,10 @@ jobs: cmake --build build --config Release --target install python -m pip install wheel python setup.py bdist_wheel - python -m pip install ./dist/*.whl + WHEEL=$(ls dist/*.whl) + python -m pip install "$WHEEL[avro]" python -c 'import pulsar; c = pulsar.Client("pulsar://localhost:6650"); c.close()' + python -c 'from pulsar.schema import *; s = String(); print(s.schema_info(""));' check-completion: name: Check Completion diff --git a/pkg/mac/build-mac-wheels.sh b/pkg/mac/build-mac-wheels.sh index 917bfecf..b0756f99 100755 --- a/pkg/mac/build-mac-wheels.sh +++ b/pkg/mac/build-mac-wheels.sh @@ -69,7 +69,7 @@ $PY_EXE setup.py bdist_wheel PY_SPEC=$(echo $PYTHON_VERSION | sed 's/\.//g') cd /tmp -$PIP_EXE install --no-dependencies --force-reinstall \ - $ROOT_DIR/dist/pulsar_client-${PYTHON_CLIENT_VERSION}-cp$PY_SPEC-*-macosx_10_15_${PIP_TAG}.whl -$PY_EXE -c 'import pulsar' - +WHEEL=$(ls $ROOT_DIR/dist/pulsar_client-${PYTHON_CLIENT_VERSION}-cp$PY_SPEC-*-macosx_10_15_${PIP_TAG}.whl) +$PIP_EXE install --no-dependencies --force-reinstall "$WHEEL[avro]" +$PY_EXE -c 'import pulsar; c = pulsar.Client("pulsar://localhost:6650"); c.close()' +$PY_EXE -c 'from pulsar.schema import *; s = String(); print(s.schema_info(""));' diff --git a/pkg/test-wheel.sh b/pkg/test-wheel.sh index d19b2802..16198a77 100755 --- a/pkg/test-wheel.sh +++ b/pkg/test-wheel.sh @@ -22,7 +22,9 @@ set -e -x cd / -pip3 install /pulsar-client-python/wheelhouse/pulsar_client-*.whl +WHEEL=$(ls /pulsar-client-python/wheelhouse/pulsar_client-*.whl) +pip3 install "$WHEEL[avro]" # Load the wheel to ensure there are no linking problems -python3 -c 'import pulsar' +python3 -c 'import pulsar; c = pulsar.Client("pulsar://localhost:6650"); c.close()' +python3 -c 'from pulsar.schema import *; s = String(); print(s.schema_info(""));' From f4e6cbb5d82d3695bad6cced165bf46bee432f92 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 20 Apr 2023 19:56:49 +0800 Subject: [PATCH 2/2] Build Python with the SSL support --- pkg/mac/build-dependencies.sh | 53 ++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/pkg/mac/build-dependencies.sh b/pkg/mac/build-dependencies.sh index 84f48dc0..918d468d 100755 --- a/pkg/mac/build-dependencies.sh +++ b/pkg/mac/build-dependencies.sh @@ -71,32 +71,6 @@ else echo "Using cached ZLib" fi -############################################################################### -if [ ! -f Python-${PYTHON_VERSION_LONG}/.done ]; then - echo "Building Python $PYTHON_VERSION_LONG" - curl -O -L https://www.python.org/ftp/python/${PYTHON_VERSION_LONG}/Python-${PYTHON_VERSION_LONG}.tgz - tar xfz Python-${PYTHON_VERSION_LONG}.tgz - - pushd Python-${PYTHON_VERSION_LONG} - if [ $PYTHON_VERSION = '3.7' ]; then - patch -p1 < ${ROOT_DIR}/pkg/mac/python-3.7.patch - fi - - CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET} -I${PREFIX}/include ${PY_CFLAGS}" \ - LDFLAGS=" ${PY_CFLAGS} -L${PREFIX}/lib" \ - ./configure --prefix=$PREFIX --enable-shared --enable-universalsdk --with-universal-archs=universal2 - make -j16 - make install - - curl -O -L https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl - $PREFIX/bin/pip3 install wheel-*.whl - - touch .done - popd -else - echo "Using cached Python $PYTHON_VERSION_LONG" -fi - ############################################################################### OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then @@ -135,6 +109,33 @@ else echo "Using cached OpenSSL" fi +############################################################################### +if [ ! -f Python-${PYTHON_VERSION_LONG}/.done ]; then + echo "Building Python $PYTHON_VERSION_LONG" + curl -O -L https://www.python.org/ftp/python/${PYTHON_VERSION_LONG}/Python-${PYTHON_VERSION_LONG}.tgz + tar xfz Python-${PYTHON_VERSION_LONG}.tgz + + pushd Python-${PYTHON_VERSION_LONG} + if [ $PYTHON_VERSION = '3.7' ]; then + patch -p1 < ${ROOT_DIR}/pkg/mac/python-3.7.patch + fi + + CFLAGS="-fPIC -O3 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET} -I${PREFIX}/include ${PY_CFLAGS}" \ + LDFLAGS=" ${PY_CFLAGS} -L${PREFIX}/lib" \ + ./configure --prefix=$PREFIX --enable-shared --enable-universalsdk --with-universal-archs=universal2 \ + --with-openssl=$PREFIX + make -j16 + make install + + curl -O -L https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl + $PREFIX/bin/pip3 install wheel-*.whl + + touch .done + popd +else + echo "Using cached Python $PYTHON_VERSION_LONG" +fi + ############################################################################### BOOST_VERSION_=${BOOST_VERSION//./_} if [ ! -f boost/.done ]; then