Add Python 3.7.0b5

Fixes #1395
mozilla · Jun 5, 2018 · d08ef84 · d08ef84
1 parent ad6a070
commit d08ef84
Show file tree

Hide file tree

Showing 8 changed files with 124 additions and 14 deletions.
diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml
@@ -1,10 +1,12 @@
 python:
   packages_trusty:
     apt: 'make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev'
+  packages_trusty_py37: # TODO: Extra packages to be able to run our build of scipy, until official manylinux cp37m are available
+    apt: 'libopenblas-base liblapack3'
   packages_stretch:
     apt: 'make build-essential libssl1.0-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev'
   brew:
-    setup: 'install_local_homebrew "python-ds-test" && install_pkg_local_homebrew "sox"'
+    setup: 'install_local_homebrew "python-ds-test" && install_pkg_local_homebrew "sox" && install_pkg_local_homebrew "readline" && install_pkg_local_homebrew "openssl" && install_pkg_local_homebrew "pkg-config"'
     env: 'export EXTRA_ENV="PATH=$TASKCLUSTER_TASK_DIR/python-ds-test.brew/bin/:$PATH"'
 nodejs:
   packages_trusty:

diff --git a/taskcluster/test-python_37-darwin-amd64-opt.yml b/taskcluster/test-python_37-darwin-amd64-opt.yml
@@ -0,0 +1,13 @@
+build:
+  template_file: test-darwin-opt-base.tyml
+  dependencies:
+    - "darwin-amd64-cpu-opt"
+    - "test-training_upstream-linux-amd64-py27mu-opt"
+  system_setup:
+    >
+      ${python.brew.setup} && ${python.brew.env}
+  args:
+    tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tc-python-tests.sh 3.7.0b5:m"
+  metadata:
+    name: "DeepSpeech OSX AMD64 CPU Python v3.7 tests"
+    description: "Testing DeepSpeech for OSX/AMD64 on Python v3.7.0b5:m, CPU only, optimized version"
diff --git a/taskcluster/test-python_37-linux-amd64-aot_test-opt.yml b/taskcluster/test-python_37-linux-amd64-aot_test-opt.yml
@@ -0,0 +1,13 @@
+build:
+  template_file: test-linux-opt-base.tyml
+  dependencies:
+    - "linux-amd64-cpu-aot_test-opt"
+    - "test-training_upstream-linux-amd64-py27mu-opt"
+  system_setup:
+    >
+      apt-get -qq -y install ${python.packages_trusty.apt} ${python.packages_trusty_py37.apt}
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 3.7.0b5:m --aot"
+  metadata:
+    name: "DeepSpeech Linux AMD64 CPU Python v3.7 tests AOT (test)"
+    description: "Testing DeepSpeech for Linux/AMD64 on Python v3.7, AOT Model (test), CPU only, optimized version"
diff --git a/taskcluster/test-python_37-linux-amd64-opt.yml b/taskcluster/test-python_37-linux-amd64-opt.yml
@@ -0,0 +1,13 @@
+build:
+  template_file: test-linux-opt-base.tyml
+  dependencies:
+    - "linux-amd64-cpu-opt"
+    - "test-training_upstream-linux-amd64-py27mu-opt"
+  system_setup:
+    >
+      apt-get -qq -y install ${python.packages_trusty.apt} ${python.packages_trusty_py37.apt}
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests.sh 3.7.0b5:m"
+  metadata:
+    name: "DeepSpeech Linux AMD64 CPU Python v3.7 tests"
+    description: "Testing DeepSpeech for Linux/AMD64 on Python v3.7, CPU only, optimized version"
diff --git a/taskcluster/test-python_37-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-python_37-linux-amd64-prod_pbmodel-opt.yml
@@ -0,0 +1,13 @@
+build:
+  template_file: test-linux-opt-base.tyml
+  dependencies:
+    - "linux-amd64-cpu-opt"
+    - "test-training_upstream-linux-amd64-py27mu-opt"
+  system_setup:
+    >
+      apt-get -qq -y install ${python.packages_trusty.apt} ${python.packages_trusty_py37.apt}
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-python-tests-prod.sh 3.7.0b5:m"
+  metadata:
+    name: "DeepSpeech Linux AMD64 CPU Python v3.7 prod tests"
+    description: "Testing DeepSpeech for Linux/AMD64 on Python v3.7 on prod model, CPU only, optimized version"
diff --git a/tc-python-tests-prod.sh b/tc-python-tests-prod.sh
@@ -41,16 +41,18 @@ download_data
 install_pyenv "${PYENV_ROOT}"
 install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv"
 
+maybe_ssl102_py37 ${pyver}
+
 PYENV_NAME=deepspeech-test
-PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv install ${pyver}
+LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv install ${pyver}
 pyenv virtualenv ${pyver} ${PYENV_NAME}
 source ${PYENV_ROOT}/versions/${pyver}/envs/${PYENV_NAME}/bin/activate
 
 platform=$(python -c 'import sys; import platform; plat = platform.system().lower(); arch = platform.machine().lower(); plat = "manylinux1" if plat == "linux" and arch == "x86_64" else plat; plat = "macosx_10_10" if plat == "darwin" else plat; sys.stdout.write("%s_%s" % (plat, platform.machine()));')
 whl_ds_version="$(python -c 'from pkg_resources import parse_version; print(parse_version("'${DS_VERSION}'"))')"
 deepspeech_pkg="deepspeech-${whl_ds_version}-cp${pyver_pkg}-cp${pyver_pkg}${py_unicode_type}-${platform}.whl"
 
-pip install --only-binary :all: --upgrade ${DEEPSPEECH_ARTIFACTS_ROOT}/${deepspeech_pkg} | cat
+LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${DEEPSPEECH_ARTIFACTS_ROOT}/${deepspeech_pkg} | cat
 
 run_prod_inference_tests
 

diff --git a/tc-python-tests.sh b/tc-python-tests.sh
@@ -36,8 +36,10 @@ download_data
 install_pyenv "${PYENV_ROOT}"
 install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv"
 
+maybe_ssl102_py37 ${pyver}
+
 PYENV_NAME=deepspeech-test
-PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv install ${pyver}
+LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv install ${pyver}
 pyenv virtualenv ${pyver} ${PYENV_NAME}
 source ${PYENV_ROOT}/versions/${pyver}/envs/${PYENV_NAME}/bin/activate
 
@@ -50,7 +52,7 @@ if [ "${aot_model}" = "--aot" ]; then
 else
     deepspeech_pkg_url=${DEEPSPEECH_ARTIFACTS_ROOT}/${deepspeech_pkg}
 fi
-pip install --only-binary :all: --upgrade ${deepspeech_pkg_url} | cat
+LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat
 
 run_all_inference_tests
 

diff --git a/tc-tests-utils.sh b/tc-tests-utils.sh
@@ -44,7 +44,7 @@ model_name="$(basename "${model_source}")"
 model_name_mmap="$(basename -s ".pb" "${model_source}").pbmm"
 model_source_mmap="$(dirname "${model_source}")/${model_name_mmap}"
 
-SUPPORTED_PYTHON_VERSIONS=${SUPPORTED_PYTHON_VERSIONS:-2.7.14:ucs2 2.7.14:ucs4 3.4.8:ucs4 3.5.5:ucs4 3.6.4:ucs4}
+SUPPORTED_PYTHON_VERSIONS=${SUPPORTED_PYTHON_VERSIONS:-2.7.14:ucs2 2.7.14:ucs4 3.4.8:ucs4 3.5.5:ucs4 3.6.4:ucs4 3.7.0b5:ucs4}
 SUPPORTED_NODEJS_VERSIONS=${SUPPORTED_NODEJS_VERSIONS:-4.9.1 5.12.0 6.14.1 7.10.1 8.11.1 9.11.1 10.3.0}
 
 # This verify exact inference result
@@ -348,7 +348,7 @@ install_pyenv()
 
   git clone --quiet https://github.com/pyenv/pyenv.git ${PYENV_ROOT}
   pushd ${PYENV_ROOT}
-    git checkout --quiet a8e207f330509b12724454b1dd38dcc31193212f
+    git checkout --quiet 8eeddaebc1153313010724f81917a96d62dc4e2a
   popd
   eval "$(pyenv init -)"
 }
@@ -492,6 +492,50 @@ do_deepspeech_binary_build()
     deepspeech
 }
 
+# Hack to extract Ubuntu's 16.04 libssl 1.0.2 packages and use them during the
+# local build of Python.
+#
+# Avoid (risky) upgrade of base system, allowing to keep one task build that
+# builds all the python packages
+maybe_ssl102_py37()
+{
+    pyver=$1
+
+    unset PY37_OPENSSL
+    unset PY37_LDPATH
+    unset PY37_SOURCE_PACKAGE
+
+    case "${pyver}" in
+        3.7*)
+            if [ "${OS}" = "Linux" ]; then
+                PY37_OPENSSL_DIR=${DS_ROOT_TASK}/ssl-xenial
+                mkdir -p ${PY37_OPENSSL_DIR}
+                wget -P ${TASKCLUSTER_TMP_DIR} \
+                        http://${TASKCLUSTER_WORKER_GROUP}.ec2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl-dev_1.0.2g-1ubuntu4.12_amd64.deb \
+                        http://${TASKCLUSTER_WORKER_GROUP}.ec2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.0.0_1.0.2g-1ubuntu4.12_amd64.deb
+
+                for deb in ${TASKCLUSTER_TMP_DIR}/libssl*.deb; do
+                    dpkg -x ${deb} ${PY37_OPENSSL_DIR}
+                done;
+
+                # Python configure expects things to be under lib/
+                mv ${PY37_OPENSSL_DIR}/usr/include/x86_64-linux-gnu/openssl/opensslconf.h ${PY37_OPENSSL_DIR}/usr/include/openssl/
+                mv ${PY37_OPENSSL_DIR}/lib/x86_64-linux-gnu/lib* ${PY37_OPENSSL_DIR}/usr/lib/
+                mv ${PY37_OPENSSL_DIR}/usr/lib/x86_64-linux-gnu/* ${PY37_OPENSSL_DIR}/usr/lib/
+                ln -sfn libcrypto.so.1.0.0 ${PY37_OPENSSL_DIR}/usr/lib/libcrypto.so
+                ln -sfn libssl.so.1.0.0 ${PY37_OPENSSL_DIR}/usr/lib/libssl.so
+
+                export PY37_OPENSSL="--with-openssl=${PY37_OPENSSL_DIR}/usr"
+                export PY37_LDPATH="${PY37_OPENSSL_DIR}/usr/lib/"
+            fi;
+
+            # TODO: Temporary workaround because no py37m numpy wheel and we
+            # want to be able to test that.
+            export PY37_SOURCE_PACKAGE="--extra-index-url https://lissyx.github.io/deepspeech-python-wheels/"
+        ;;
+    esac
+}
+
 do_deepspeech_python_build()
 {
   rename_to_gpu=$1
@@ -515,16 +559,24 @@ do_deepspeech_python_build()
     pyver=$(echo "${pyver_conf}" | cut -d':' -f1)
     pyconf=$(echo "${pyver_conf}" | cut -d':' -f2)
 
-    PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf}" pyenv install ${pyver}
+    maybe_ssl102_py37 ${pyver}
+
+    LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" pyenv install ${pyver}
+
     pyenv virtualenv ${pyver} deepspeech
     source ${PYENV_ROOT}/versions/${pyver}/envs/deepspeech/bin/activate
 
-    EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" EXTRA_LDFLAGS="${EXTRA_LOCAL_LDFLAGS}" EXTRA_LIBS="${EXTRA_LOCAL_LIBS}" make -C native_client/ \
-      TARGET=${SYSTEM_TARGET} \
-      RASPBIAN=${SYSTEM_RASPBIAN} \
-      TFDIR=${DS_TFDIR} \
-      SETUP_FLAGS="${SETUP_FLAGS}" \
-      bindings-clean bindings
+    # Set LD path because python ssl might require it
+    LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \
+    EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" \
+    EXTRA_LDFLAGS="${EXTRA_LOCAL_LDFLAGS}" \
+    EXTRA_LIBS="${EXTRA_LOCAL_LIBS}" \
+    make -C native_client/ \
+        TARGET=${SYSTEM_TARGET} \
+        RASPBIAN=${SYSTEM_RASPBIAN} \
+        TFDIR=${DS_TFDIR} \
+        SETUP_FLAGS="${SETUP_FLAGS}" \
+        bindings-clean bindings
 
     cp native_client/dist/*.whl wheels