Python 3.7 macOS Support (#2079)

* For macOS and Linux, update all necessary dependencies, other than mxnet. * Install MXNet 1.5.0 only for Python 3.7 on AVX2 supported macOS. * Prevent stackoverflow (inifinte recursion) for TC image resize. * Updated panda dtypes. * Don't clobber symlinks in deps/local/include. * Object Detection use new version of MXNet. * Sound Classifier use new version of MXNet. * Update what versions of Python we say we support. * Make clear Python 3.7 is supported on macOS only * Remove unneeded additional depedencies for scenario tests. * Add pyton 3.7 to docker configs. * Add '--docker-python3.7' option to scenario tests. * Make run_cpp_test.py python 3 compatible. * Add logging to error case. * Use ignore_stale_grad and even_split to ignore MXNet warnings surfaced as exceptions. * Minor unit test refactor. * GitLab CI: Add Python 3.7 for macOS. * GitLab CI: commented out another 10.13 jobs. * Fix Travis CI.
apple · Aug 7, 2019 · ad9c748 · ad9c748
1 parent c49db67
commit ad9c748
Show file tree

Hide file tree

Showing 21 changed files with 192 additions and 110 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -128,6 +128,22 @@ build_wheel_mac_10_14_python36:
     paths:
       - target/
 
+build_wheel_mac_10_14_python37:
+  tags:
+    - macos10.14
+  only:
+    variables:
+      - $TC_HAS_MACOS_RUNNERS == "1"
+  stage: build
+  script:
+    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.7/bin/virtualenv"
+    - source scripts/use_ccache.sh
+    - bash scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number=$CI_PIPELINE_ID --num_procs=4
+  artifacts:
+    expire_in: 1 day
+    paths:
+      - target/
+
 build_dylib_macos_10_14:
   tags:
     - macos10.14
@@ -283,9 +299,29 @@ test_python_linux_python36:
     paths:
       - pytest.*
 
-test_python_mac_python27_10_13:
+# This coremltools issues needs to be resolved before we can run on macOS 10.13
+# https://github.com/apple/coremltools/issues/408
+#test_python_mac_python27_10_13:
+#  tags:
+#    - macos10.13
+#  only:
+#    variables:
+#      - $TC_HAS_MACOS_RUNNERS == "1"
+#  stage: test
+#  dependencies:
+#    - build_wheel_mac_10_14_python27
+#  script:
+#    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/2.7/bin/virtualenv"
+#    - bash scripts/test_wheel.sh
+#  artifacts:
+#    when: always
+#    expire_in: 2 weeks
+#    paths:
+#      - pytest.*
+
+test_python_mac_python27_10_14:
   tags:
-    - macos10.13
+    - macos10.14
   only:
     variables:
       - $TC_HAS_MACOS_RUNNERS == "1"
@@ -301,53 +337,53 @@ test_python_mac_python27_10_13:
     paths:
       - pytest.*
 
-test_python_mac_python27_10_14:
+test_python_mac_python35_10_14:
   tags:
     - macos10.14
   only:
     variables:
       - $TC_HAS_MACOS_RUNNERS == "1"
   stage: test
   dependencies:
-    - build_wheel_mac_10_14_python27
+    - build_wheel_mac_10_14_python35
   script:
-    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/2.7/bin/virtualenv"
+    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.5/bin/virtualenv"
     - bash scripts/test_wheel.sh
   artifacts:
     when: always
     expire_in: 2 weeks
     paths:
       - pytest.*
 
-test_python_mac_python35_10_14:
+test_python_mac_python36_10_14:
   tags:
     - macos10.14
   only:
     variables:
       - $TC_HAS_MACOS_RUNNERS == "1"
   stage: test
   dependencies:
-    - build_wheel_mac_10_14_python35
+    - build_wheel_mac_10_14_python36
   script:
-    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.5/bin/virtualenv"
+    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.6/bin/virtualenv"
     - bash scripts/test_wheel.sh
   artifacts:
     when: always
     expire_in: 2 weeks
     paths:
       - pytest.*
 
-test_python_mac_python36_10_14:
+test_python_mac_python37_10_14:
   tags:
     - macos10.14
   only:
     variables:
       - $TC_HAS_MACOS_RUNNERS == "1"
   stage: test
   dependencies:
-    - build_wheel_mac_10_14_python36
+    - build_wheel_mac_10_14_python37
   script:
-    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.6/bin/virtualenv"
+    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.7/bin/virtualenv"
     - bash scripts/test_wheel.sh
   artifacts:
     when: always
@@ -406,19 +442,21 @@ scenario_tests_linux_python36:
     - cd scenario-tests
     - ./run_scenario_tests.sh --docker-python3.6 ../target/turicreate-*.whl
 
-scenario_tests_mac_python27_10_13:
-  tags:
-    - macos10.13
-  only:
-    variables:
-      - $TC_HAS_MACOS_RUNNERS == "1"
-  stage: test
-  dependencies:
-    - build_wheel_mac_10_14_python27
-  script:
-    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/2.7/bin/virtualenv"
-    - cd scenario-tests
-    - ./run_scenario_tests.sh ../target/turicreate-*.whl
+# This coremltools issues needs to be resolved before we can run on macOS 10.13
+# https://github.com/apple/coremltools/issues/408
+#scenario_tests_mac_python27_10_13:
+#  tags:
+#    - macos10.13
+#  only:
+#    variables:
+#      - $TC_HAS_MACOS_RUNNERS == "1"
+#  stage: test
+#  dependencies:
+#    - build_wheel_mac_10_14_python27
+#  script:
+#    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/2.7/bin/virtualenv"
+#    - cd scenario-tests
+#    - ./run_scenario_tests.sh ../target/turicreate-*.whl
 
 scenario_tests_mac_python27_10_14:
   tags:
@@ -450,6 +488,7 @@ collect_artifacts:
     - build_wheel_mac_10_14_python27
     - build_wheel_mac_10_14_python35
     - build_wheel_mac_10_14_python36
+    - build_wheel_mac_10_14_python37
   script:
     - mv release/src/deployment/*.dylib target/
     - rmdir -p release || find release

diff --git a/README.md b/README.md
@@ -79,7 +79,7 @@ System Requirements
 
 Turi Create requires:
 
-* Python 2.7, 3.5, 3.6
+* Python 2.7, 3.5, 3.6, 3.7 (macOS only)
 * x86\_64 architecture
 * At least 4 GB of RAM
 
@@ -109,8 +109,8 @@ source ~/venv/bin/activate
 ```
 Alternatively, if you are using [Anaconda](https://www.anaconda.com/what-is-anaconda/), you may use its virtual environment:
 ```shell
-conda create -n venv python=2.7 anaconda
-source activate venv
+conda create -n virtual_environment_name anaconda
+conda activate virtual_environment_name
 ```
 
 To install `Turi Create` within your virtual environment:

diff --git a/scenario-tests/additional_requirements.txt b/scenario-tests/additional_requirements.txt
@@ -1,8 +0,0 @@
-beautifulsoup4
-certifi==2015.04.28
-nltk==3.2
-pyscreenshot==0.4
-python-swiftclient
-python-keystoneclient
-testfixtures==4.10.0
-h5py==2.7.1

diff --git a/scenario-tests/run_scenario_tests.sh b/scenario-tests/run_scenario_tests.sh
@@ -39,6 +39,8 @@ function print_help {
   echo
   echo "  --docker-python3.6       Use docker to test on Python 3.6 in Ubuntu 18.04."
   echo
+  echo "  --docker-python3.7       Use docker to test on Python 3.7 in Ubuntu 18.04."
+  echo
   exit 0
 } # end of print help
 
@@ -55,6 +57,7 @@ while [ $# -gt 0 ]
     --docker-python2.7)     USE_DOCKER=1;DOCKER_PYTHON=2.7;;
     --docker-python3.5)     USE_DOCKER=1;DOCKER_PYTHON=3.5;;
     --docker-python3.6)     USE_DOCKER=1;DOCKER_PYTHON=3.6;;
+    --docker-python3.7)     USE_DOCKER=1;DOCKER_PYTHON=3.7;;
     --help)                 print_help ;;
     *) TC_WHEEL_UNDER_TEST=$1 ;;
   esac
@@ -86,7 +89,7 @@ if [[ -n "${USE_DOCKER}" ]]; then
       -e "VIRTUALENV=virtualenv --python=python${DOCKER_PYTHON}" \
       ${TC_BUILD_IMAGE_1404} \
       /build/scenario-tests/run_scenario_tests.sh $TC_WHEEL_UNDER_TEST
-  elif [[ "${DOCKER_PYTHON}" == "3.6" ]]; then
+  elif [[ "${DOCKER_PYTHON}" == "3.6" ]] || [[ "${DOCKER_PYTHON}" == "3.7" ]]; then
     docker run --rm -m=4g \
       --mount type=bind,source=$WORKSPACE,target=/build,consistency=delegated \
       -e "VIRTUALENV=virtualenv --python=python${DOCKER_PYTHON}" \

diff --git a/scripts/install_python_toolchain.sh b/scripts/install_python_toolchain.sh
@@ -36,20 +36,13 @@ function linux_patch_sigfpe_handler {
 }
 
 $PIP install --upgrade "pip>=8.1"
-
-# numpy needs to be installed before everything else so that installing
-# numba (a dependency of resampy) doesn't fail on Python 3.5. This can
-# be removed once numba publishes a Python 3.5 wheel for their most
-# recent version.
-$PIP install numpy==1.16.4
-
 $PIP install -r scripts/requirements.txt
 
 mkdir -p deps/local/lib
 mkdir -p deps/local/include
 
 pushd deps/local/include
-for f in `ls ../../env/include/$PYTHON_FULL_NAME/*`; do  
+for f in `ls -d ../../env/include/$PYTHON_FULL_NAME/*`; do
   ln -Ffs $f
 done
 popd

diff --git a/scripts/make_wheel.sh b/scripts/make_wheel.sh
@@ -44,6 +44,8 @@ print_help() {
   echo
   echo "  --docker-python3.6       Use docker to build for Python 3.6 in Ubuntu 10.04 with GCC 4.8."
   echo
+  echo "  --docker-python3.7       Use docker to build for Python 3.7 in Ubuntu 10.04 with GCC 4.8."
+  echo
   echo "  --num_procs=n            Specify the number of proceses to run in parallel."
   echo
   echo "  --target-dir=[dir]       The directory where the wheel and associated files are put."
@@ -70,6 +72,7 @@ while [ $# -gt 0 ]
     --docker-python2.7)     USE_DOCKER=1;DOCKER_PYTHON=2.7;;
     --docker-python3.5)     USE_DOCKER=1;DOCKER_PYTHON=3.5;;
     --docker-python3.6)     USE_DOCKER=1;DOCKER_PYTHON=3.6;;
+    --docker-python3.7)     USE_DOCKER=1;DOCKER_PYTHON=3.7;;
     --help)                 print_help ;;
     *) unknown_option $1 ;;
   esac

diff --git a/scripts/requirements.txt b/scripts/requirements.txt
@@ -1,26 +1,26 @@
-coremltools==2.1.0
+coremltools==3.0b3
 scipy==1.2.1
 numpy==1.16.4
-cython==0.29.12
+cython==0.29.10
 argparse==1.2.1
-decorator==4.1.2
-mock==2.0.0
-pytest==3.2.5
-pandas==0.22.0
-pillow==5.0.0
+decorator==4.4.0
+mock==3.0.5
+pytest==4.6.3
+pandas==0.24.2
+pillow==6.0.0
 prettytable==0.7.2
-pytz==2016.3
+pytz==2019.1
 resampy==0.2.1
 requests>=2.9.1
-scikit-learn==0.17.1
-six==1.10.0
-statsmodels==0.8.0
+scikit-learn==0.20.3
+six==1.12.0
+statsmodels==0.9.0
 wheel==0.29.0
-mxnet==1.1.0
+mxnet==1.1.0; sys_platform != 'darwin'
 UISoup==2.5.7
-pyobjc==4.2.2; sys_platform == 'darwin'
-future==0.16.0
-pyOpenSSL
-ndg-httpsclient
-pyasn1
-hypothesis==3.76.0
+pyobjc==5.2; sys_platform == 'darwin'
+future==0.17.1
+pyOpenSSL==19.0.0
+ndg-httpsclient==0.5.1
+pyasn1==0.4.5
+hypothesis==4.24.3
diff --git a/scripts/run_cpp_tests.py b/scripts/run_cpp_tests.py
@@ -162,17 +162,17 @@ def run_in_docker(cmd, workdir='/build'):
     line = ctest_process.stdout.readline()
     if len(line) == 0:
       break
-    sys.stdout.write(line)
+    sys.stdout.write(line.decode())
     sys.stdout.flush()
     lines.append(line)
 
   out, err = ctest_process.communicate()
 
   if args.cache:
     # go through all the tests and see if we have a "Passed" line matching it
-    for i in xrange(len(tests)):
+    for i in range(len(tests)):
       for line in lines:
-        if ('Passed' in line) and ((" " + runtests[i] + " ") in line):
+        if ('Passed' in line.decode()) and ((" " + runtests[i] + " ") in line.decode()):
           # pass!
           cache.add(new_tests[tests[i]])
 

diff --git a/scripts/test_wheel.sh b/scripts/test_wheel.sh
@@ -27,6 +27,7 @@ print_help() {
   echo
   echo "  --docker-python3.6       Use docker to test on Python 3.6 in Ubuntu 18.04."
   echo
+  echo "  --docker-python3.7       Use docker to test on Python 3.7 in Ubuntu 18.04."
   exit 1
 } # end of print help
 
@@ -37,6 +38,7 @@ while [ $# -gt 0 ]
     --docker-python2.7)     USE_DOCKER=1;DOCKER_PYTHON=2.7;;
     --docker-python3.5)     USE_DOCKER=1;DOCKER_PYTHON=3.5;;
     --docker-python3.6)     USE_DOCKER=1;DOCKER_PYTHON=3.6;;
+    --docker-python3.7)     USE_DOCKER=1;DOCKER_PYTHON=3.7;;
     --help)                 print_help ;;
     *) unknown_option $1 ;;
   esac
@@ -61,7 +63,7 @@ if [[ -n "${USE_DOCKER}" ]]; then
       -e "VIRTUALENV=virtualenv --python=python${DOCKER_PYTHON}" \
       ${TC_BUILD_IMAGE_1404} \
       /build/scripts/test_wheel.sh
-  elif [[ "${DOCKER_PYTHON}" == "3.6" ]]; then
+  elif [[ "${DOCKER_PYTHON}" == "3.6" ]] || [[ "${DOCKER_PYTHON}" == "3.7" ]]; then
     docker run --rm -m=8g \
       --mount type=bind,source=$WORKSPACE,target=/build,consistency=delegated \
       -e "VIRTUALENV=virtualenv --python=python${DOCKER_PYTHON}" \

diff --git a/src/python/README.rst b/src/python/README.rst
@@ -18,6 +18,7 @@ System Requirements
 -------------------
 
 -  Python 2.7, 3.5, or 3.6
+-  Python 3.7 macOS only
 -  x86_64 architecture
 
 Installation