From ea29f3d52fe2260756c340302e2e73081cfc9400 Mon Sep 17 00:00:00 2001 From: Henry Tang Date: Thu, 26 Mar 2020 21:52:16 +0800 Subject: [PATCH 1/3] Fix manylinux1 (#146) Remove use of six in setup.py Use cmake to build rabbitmq-c --- Makefile | 4 ++-- build-manylinux1-wheels.sh | 13 +++++++++---- setup.py | 37 ++++++++++++++++++++++++++++++++----- 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index bb073c0..e7538a1 100644 --- a/Makefile +++ b/Makefile @@ -20,10 +20,10 @@ rabbitmq-c: submodules rabbitmq-clean: - -(cd $(RABBIT_DIR) && make clean) + -(cd $(RABBIT_DIR) && make clean || echo "warning... rabbitmq-clean failed") rabbitmq-distclean: - -(cd $(RABBIT_DIR) && make distclean) + -(cd $(RABBIT_DIR) && make distclean || echo "warning... rabbitmq-distclean failed") clean-build: -rm -rf build diff --git a/build-manylinux1-wheels.sh b/build-manylinux1-wheels.sh index c0a3901..664075c 100755 --- a/build-manylinux1-wheels.sh +++ b/build-manylinux1-wheels.sh @@ -2,13 +2,12 @@ # Script modified from https://github.com/pypa/python-manylinux-demo set -e -x -# Install a system package required by our library -yum install -y librabbitmq-devel make librabbitmq python-devel gcc automake - +# Install system packages required by our library +yum install -y cmake openssl-devel gcc automake # Compile wheels for PYBIN in /opt/python/*/bin; do - #${PYBIN}/pip install -r /workspace/dev-requirements.txt + (cd /workspace && ${PYBIN}/python setup.py install) ${PYBIN}/pip wheel /workspace/ -w wheelhouse/ done @@ -20,6 +19,12 @@ done # Install packages and test for PYBIN in /opt/python/*/bin/; do + # vine 5.0.0a1 breaks python2 + # https://github.com/celery/vine/issues/34 + if [[ "$PYBIN" == *"python/cp2"* ]]; then + ${PYBIN}/pip install -f "vine==1.3.0" + fi + ${PYBIN}/pip install librabbitmq -f /workspace/wheelhouse ${PYBIN}/python -c "import librabbitmq" #(cd $HOME; ${PYBIN}/nosetests pymanylinuxdemo) diff --git a/setup.py b/setup.py index d91ffce..6985565 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,10 @@ def append_env(L, e): sys.argv[1:] = unprocessed incdirs.append(LRMQSRC()) + if find_cmake() != "": + incdirs.append(LRMQDIST('build', 'librabbitmq')) + + PyC_files = map(PYCP, [ 'connection.c', ]) @@ -107,6 +111,7 @@ def run(self): here = os.path.abspath(os.getcwd()) config = sysconfig.get_config_vars() make = find_make() + cmake = find_cmake() try: vars = {'ld': config['LDFLAGS'], @@ -128,21 +133,34 @@ def run(self): if os.path.isfile('Makefile'): os.system(' '.join([make, 'submodules'])) else: - os.system(' '.join(['git', 'clone', '-b', 'v0.8.0', + os.system(' '.join(['git', 'clone', '-b', 'v0.8.0', 'https://github.com/alanxz/rabbitmq-c.git', 'rabbitmq-c'])) os.chdir(LRMQDIST()) - if not os.path.isfile('configure'): + if cmake == "" and not os.path.isfile('configure'): print('- autoreconf') os.system('autoreconf -i') - if not os.path.isfile('config.h'): + if cmake == "" and not os.path.isfile('config.h'): print('- configure rabbitmq-c...') if os.system('/bin/sh configure --disable-tools \ --disable-docs --disable-dependency-tracking'): return + + if cmake: + print('- cmake rabbitmq-c...') + if os.system('mkdir -p build'): + return + + os.chdir('build') + if os.system(cmake + ' ..'): + return + + if os.system(make + ' rabbitmq rabbitmq-static'): + return + finally: os.environ.update(restore) finally: @@ -164,7 +182,16 @@ def find_make(alt=('gmake', 'gnumake', 'make', 'nmake')): return make -if six.PY2: +def find_cmake(): + for path in os.environ['PATH'].split(':'): + make = os.path.join(path, 'cmake') + if os.path.isfile(make): + return make + + return "" + + +if sys.version_info[0] < 3: with open(os.path.join(BASE_PATH, 'README.rst'), 'U') as f: long_description = f.read() else: @@ -243,7 +270,7 @@ def find_make(alt=('gmake', 'gnumake', 'make', 'nmake')): license='MPL', description='AMQP Client using the rabbitmq-c library.', long_description=long_description, - test_suite="tests", + test_suite="tests", zip_safe=False, packages=packages, cmdclass=cmdclass, From 0cc779cd5bd61fa39bed33a2cdfca7ba83d96f5d Mon Sep 17 00:00:00 2001 From: Henry Tang Date: Fri, 27 Mar 2020 15:24:36 +0800 Subject: [PATCH 2/3] Ensure a fresh build of rabbitmq-c before building wheels Fix vine install --- build-manylinux1-wheels.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/build-manylinux1-wheels.sh b/build-manylinux1-wheels.sh index 664075c..b111482 100755 --- a/build-manylinux1-wheels.sh +++ b/build-manylinux1-wheels.sh @@ -5,6 +5,11 @@ set -e -x # Install system packages required by our library yum install -y cmake openssl-devel gcc automake +# Ensure a fresh build of rabbitmq-c. +(cd /workspace && make submodules) +rm -rf /workspace/rabbitmq-c/build +(cd /workspace && make rabbitmq-c) + # Compile wheels for PYBIN in /opt/python/*/bin; do (cd /workspace && ${PYBIN}/python setup.py install) @@ -22,7 +27,7 @@ for PYBIN in /opt/python/*/bin/; do # vine 5.0.0a1 breaks python2 # https://github.com/celery/vine/issues/34 if [[ "$PYBIN" == *"python/cp2"* ]]; then - ${PYBIN}/pip install -f "vine==1.3.0" + ${PYBIN}/pip install --force-reinstall "vine==1.3.0" fi ${PYBIN}/pip install librabbitmq -f /workspace/wheelhouse From 7c0466e1c1a5820a2f9c52ec0314de0d3354e7d8 Mon Sep 17 00:00:00 2001 From: Henry Tang Date: Fri, 27 Mar 2020 17:07:17 +0800 Subject: [PATCH 3/3] Skip cp27-cp27mu builds (due to undefined symbol: PyUnicodeUCS2_AsASCIIString) --- Makefile | 1 + build-manylinux1-wheels.sh | 22 ++++++++++++++-------- setup.py | 5 +++-- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index e7538a1..0615af3 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ rabbitmq-c: submodules rabbitmq-clean: + -(rm -rf $(RABBIT_DIR)/build || true) -(cd $(RABBIT_DIR) && make clean || echo "warning... rabbitmq-clean failed") rabbitmq-distclean: diff --git a/build-manylinux1-wheels.sh b/build-manylinux1-wheels.sh index b111482..9c534de 100755 --- a/build-manylinux1-wheels.sh +++ b/build-manylinux1-wheels.sh @@ -5,29 +5,35 @@ set -e -x # Install system packages required by our library yum install -y cmake openssl-devel gcc automake -# Ensure a fresh build of rabbitmq-c. -(cd /workspace && make submodules) -rm -rf /workspace/rabbitmq-c/build -(cd /workspace && make rabbitmq-c) - # Compile wheels for PYBIN in /opt/python/*/bin; do + # cp27-cp27mu builds fail with: undefined symbol: PyUnicodeUCS2_AsASCIIString + if [[ "${PYBIN}" == *"cp27-cp27mu"* ]]; then + continue + fi + + # Ensure a fresh build of rabbitmq-c. + rm -rf /workspace/rabbitmq-c/build (cd /workspace && ${PYBIN}/python setup.py install) ${PYBIN}/pip wheel /workspace/ -w wheelhouse/ done # Bundle external shared libraries into the wheels -#ls wheelhouse/* for whl in wheelhouse/*linux*.whl; do auditwheel repair $whl -w /workspace/wheelhouse/ done # Install packages and test for PYBIN in /opt/python/*/bin/; do - # vine 5.0.0a1 breaks python2 + if [[ "${PYBIN}" == *"cp27-cp27mu"* ]]; then + continue + fi + + # amqp 5.0.0a1 and vine 5.0.0a1 breaks python2 # https://github.com/celery/vine/issues/34 - if [[ "$PYBIN" == *"python/cp2"* ]]; then + if [[ "${PYBIN}" == *"python/cp2"* ]]; then ${PYBIN}/pip install --force-reinstall "vine==1.3.0" + ${PYBIN}/pip install --force-reinstall "amqp==2.5.2" fi ${PYBIN}/pip install librabbitmq -f /workspace/wheelhouse diff --git a/setup.py b/setup.py index 6985565..e3fee76 100644 --- a/setup.py +++ b/setup.py @@ -155,8 +155,9 @@ def run(self): return os.chdir('build') - if os.system(cmake + ' ..'): - return + if not os.path.isfile('Makefile'): + if os.system(cmake + ' ..'): + return if os.system(make + ' rabbitmq rabbitmq-static'): return