Python 3.9 support -- minimal TuriCreate version only (#3417)

* Update docker images to include Python 3.9

* Update dependency version to those that support Python 3.9

* Add Python 3.9 option to build and test scripts

* Add GitLab CI jobs for building wheels for Python 3.9 and testing only the minimal wheel on Linux and macOS

* Remove Ubuntu docker 12 image. It's not being used

* Docker build script updates

* Fix Python 3.6 panda requirement

* Don't need resampy dependency minimal TuriCreate

* Don't pin TensorFlow version for Python 3.9

* Pin llvmlite dependency for Python 3.9

* Can't enforce numba requirement for Python 3.9

* There is no way to smoke test the Python 3.9 wheel on CentOS 6

* Bump version number

* propogate skip_smoke_test arg to docker

.gitlab-ci.yml

@@ -128,6 +128,27 @@ build_wheel_linux_python38:
     paths:
       - target/
 
+build_wheel_linux_python39:
+  tags:
+    - docker
+  services:
+    - docker:19.03-dind
+  image: docker:19.03.8
+  stage: build
+  variables:
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_DRIVER: overlay2
+    DOCKER_TLS_CERTDIR: ""
+  script:
+    - apk add bash
+    - mkdir -p /etc/ssl/certs/
+    # Smoke test must be skipped. A Python 3.9 numba wheel can not be installed on Centos 6 (manylinux2014 vs manylinux2010).
+    - bash -e scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number=$CI_PIPELINE_ID --num_procs=4 --docker-python3.9 --skip_smoke_test
+  artifacts:
+    expire_in: 1 day
+    paths:
+      - target/
+
 build_wheel_mac_10_15_python27:
   tags:
     - macos10.15
@@ -208,6 +229,22 @@ build_wheel_mac_10_15_python38:
     paths:
       - target/
 
+build_wheel_mac_10_15_python39:
+  tags:
+    - macos10.15
+  only:
+    variables:
+      - $TC_HAS_MACOS_RUNNERS == "1"
+  stage: build
+  script:
+    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.9/bin/virtualenv"
+    - source scripts/use_ccache.sh
+    - bash -e scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number=$CI_PIPELINE_ID --num_procs=4
+  artifacts:
+    expire_in: 1 day
+    paths:
+      - target/
+
 build_dylib_macos_10_15:
   tags:
     - macos10.15
@@ -503,6 +540,28 @@ test_python_linux_python38:
     paths:
       - pytest.*
 
+test_python_linux_python39_minimal:
+  tags:
+    - docker
+  services:
+    - docker:19.03-dind
+  image: docker:19.03.8
+  stage: test
+  variables:
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_DRIVER: overlay2
+    DOCKER_TLS_CERTDIR: ""
+  dependencies:
+    - build_wheel_linux_python39
+  script:
+    - apk add bash
+    - bash -e scripts/test_wheel.sh --docker-python3.9 --minimal
+  artifacts:
+    when: always
+    expire_in: 2 weeks
+    paths:
+      - pytest.*
+
 test_python_mac_python27_10_13:
   tags:
     - macos10.13
@@ -629,6 +688,24 @@ test_python_mac_python38_10_15:
     paths:
       - pytest.*
 
+test_python_mac_python39_10_15_minimal:
+  tags:
+    - macos10.15
+  only:
+    variables:
+      - $TC_HAS_MACOS_RUNNERS == "1"
+  stage: test
+  dependencies:
+    - build_wheel_mac_10_15_python39
+  script:
+    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.9/bin/virtualenv"
+    - bash -e scripts/test_wheel.sh --minimal
+  artifacts:
+    when: always
+    expire_in: 2 weeks
+    paths:
+      - pytest.*
+
 test_python_mac_python38_10_14_minimal:
   tags:
     - macos10.14
@@ -745,11 +822,13 @@ collect_artifacts:
     - build_wheel_linux_python36
     - build_wheel_linux_python37
     - build_wheel_linux_python38
+    - build_wheel_linux_python39
     - build_wheel_mac_10_15_python27
     - build_wheel_mac_10_15_python35
     - build_wheel_mac_10_15_python36
     - build_wheel_mac_10_15_python37
     - build_wheel_mac_10_15_python38
+    - build_wheel_mac_10_15_python39
     - build_documentation
   script:
     - find release/src/deployment/ -name '*.dylib' -exec mv {} target/ \;

scripts/Dockerfile-CentOS-6

@@ -144,22 +144,36 @@ RUN ./configure --prefix=/usr/local --enable-unicode=ucs4 --enable-shared --enab
     ldconfig && \
     rm -rf /src/Python-3.8.5*
 
+# Install Python 3.9 from source
+WORKDIR /src
+RUN curl -O https://www.python.org/ftp/python/3.9.6/Python-3.9.6.tgz
+RUN tar xf Python-3.9.6.tgz
+WORKDIR /src/Python-3.9.6
+RUN ./configure --prefix=/usr/local --enable-unicode=ucs4 --enable-shared --enable-loadable-sqlite-extensions && \
+    make -j4 --quiet && \
+    make install && \
+    ldconfig && \
+    rm -rf /src/Python-3.9.5*
+
 ENV CC="/usr/local/bin/gcc"
 ENV CXX="/usr/local/bin/g++"
 
 # Install pip and virtualenv
 WORKDIR /src
-RUN curl -O https://bootstrap.pypa.io/get-pip.py
+RUN curl -O https://bootstrap.pypa.io/pip/2.7/get-pip.py
 RUN python2.7 get-pip.py
 RUN pip2.7 install virtualenv
 RUN python3.5 get-pip.py
 RUN pip3.5 install virtualenv
+RUN curl -O https://bootstrap.pypa.io/get-pip.py
 RUN python3.6 get-pip.py
 RUN pip3.6 install virtualenv
 RUN python3.7 get-pip.py
 RUN pip3.7 install virtualenv
 RUN python3.8 get-pip.py
 RUN pip3.8 install virtualenv
+RUN python3.9 get-pip.py
+RUN pip3.9 install virtualenv
 RUN rm -rf /src/get-pip.py
 
 ENV LD="/usr/local/lib64"

scripts/Dockerfile-Ubuntu-14.04

@@ -18,7 +18,7 @@ RUN apt-get -y upgrade
 RUN apt-get -y install python2.7 python3.5 libgomp1
 
 # Install pip and virtualenv
-ADD https://bootstrap.pypa.io/get-pip.py /src/get-pip.py
+ADD https://bootstrap.pypa.io/pip/2.7/get-pip.py /src/get-pip.py
 WORKDIR /src
 RUN python2.7 get-pip.py
 RUN pip2.7 install virtualenv

scripts/Dockerfile-Ubuntu-18.04

@@ -22,6 +22,10 @@ RUN apt-get -y install software-properties-common
 RUN add-apt-repository -y ppa:deadsnakes/ppa
 RUN apt-get -y install python3.8
 
+# Install Python 3.9
+RUN apt-get update
+RUN apt-get -y install python3.9 python3.9-distutils
+
 # Install build-essential (needed by numpy, which tries to
 # build itself from source on 3.6 and above)
 RUN apt-get -y install build-essential libpython3.6-dev libpython3.7-dev
@@ -36,3 +40,5 @@ RUN python3.7 get-pip.py
 RUN pip3.7 install virtualenv
 RUN python3.8 get-pip.py
 RUN pip3.8 install virtualenv
+RUN python3.9 get-pip.py
+RUN pip3.9 install virtualenv

scripts/create_docker_images.sh

@@ -13,7 +13,6 @@ cd ${WORKSPACE}
 
 # The build image version that will be used for testing
 TC_BUILD_IMAGE_CENTOS6=$(bash $WORKSPACE/scripts/get_docker_image.sh --centos=6)
-TC_BUILD_IMAGE_1204=$(bash $WORKSPACE/scripts/get_docker_image.sh --ubuntu=12.04)
 TC_BUILD_IMAGE_1404=$(bash $WORKSPACE/scripts/get_docker_image.sh --ubuntu=14.04)
 TC_BUILD_IMAGE_1804=$(bash $WORKSPACE/scripts/get_docker_image.sh --ubuntu=18.04)
 
@@ -41,23 +40,18 @@ done
 if [[ -n $PERSIST_TO_REPO_DIR ]]; then
   mkdir -p .docker_images
   (test -f .docker_images/image-centos6.gz && docker load -i .docker_images/image-centos6.gz) || true
-  (test -f .docker_images/image-12.04.gz && docker load -i .docker_images/image-12.04.gz) || true
   (test -f .docker_images/image-14.04.gz && docker load -i .docker_images/image-14.04.gz) || true
   (test -f .docker_images/image-18.04.gz && docker load -i .docker_images/image-18.04.gz) || true
 fi
 
 # Load images from registry if possible
 (docker pull -q ${TC_BUILD_IMAGE_CENTOS6}) || true
-(docker pull -q ${TC_BUILD_IMAGE_1204}) || true
 (docker pull -q ${TC_BUILD_IMAGE_1404}) || true
 (docker pull ${TC_BUILD_IMAGE_1804}) || true
 
 (docker image ls ${TC_BUILD_IMAGE_CENTOS6} | grep turicreate/build-image) || \
 cat scripts/Dockerfile-CentOS-6 | docker build -t ${TC_BUILD_IMAGE_CENTOS6} -
 
-(docker image ls ${TC_BUILD_IMAGE_1204} | grep turicreate/build-image) || \
-cat scripts/Dockerfile-Ubuntu-12.04 | docker build -t ${TC_BUILD_IMAGE_1204} -
-
 (docker image ls ${TC_BUILD_IMAGE_1404} | grep turicreate/build-image) || \
 cat scripts/Dockerfile-Ubuntu-14.04 | docker build -t ${TC_BUILD_IMAGE_1404} -
 
@@ -69,7 +63,6 @@ cat scripts/Dockerfile-Ubuntu-18.04 | docker build -t ${TC_BUILD_IMAGE_1804} -
 #  primarily intended for CI systems that create a new VM for each run, but can cache the filesystem.)
 if [[ -n $PERSIST_TO_REPO_DIR ]]; then
   docker save ${TC_BUILD_IMAGE_CENTOS6} | gzip -c > .docker_images/image-centos6.gz
-  docker save ${TC_BUILD_IMAGE_1204} | gzip -c > .docker_images/image-12.04.gz
   docker save ${TC_BUILD_IMAGE_1404} | gzip -c > .docker_images/image-14.04.gz
   docker save ${TC_BUILD_IMAGE_1804} | gzip -c > .docker_images/image-18.04.gz
 fi

scripts/get_docker_image.sh

@@ -37,7 +37,7 @@ if [[ -z "${UBUNTU_VERSION}" && -z "${CENTOS_VERSION}" ]]; then
 fi
 
 # The build image version that will be used for building
-TC_BUILD_IMAGE_VERSION="1.0.7"
+TC_BUILD_IMAGE_VERSION="1.0.8"
 
 if [[ -z "${CENTOS_VERSION}" ]]; then
   # The base image name - using Gitlab CI registry

scripts/make_wheel.sh

@@ -46,6 +46,8 @@ print_help() {
   echo
   echo "  --docker-python3.8       Use docker to build for Python 3.8 in CentOS 6 with Clang 8."
   echo
+  echo "  --docker-python3.9       Use docker to build for Python 3.9 in CentOS 6 with Clang 8."
+  echo
   echo "  --num_procs=n            Specify the number of proceses to run in parallel."
   echo
   echo "  --target-dir=[dir]       The directory where the wheel and associated files are put."
@@ -73,6 +75,7 @@ while [ $# -gt 0 ]
     --docker-python3.6)     USE_DOCKER=1;DOCKER_PYTHON=3.6;;
     --docker-python3.7)     USE_DOCKER=1;DOCKER_PYTHON=3.7;;
     --docker-python3.8)     USE_DOCKER=1;DOCKER_PYTHON=3.8;;
+    --docker-python3.9)     USE_DOCKER=1;DOCKER_PYTHON=3.9;;
     --help)                 print_help ;;
     *) unknown_option $1 ;;
   esac
@@ -111,6 +114,9 @@ if [[ -n "${USE_DOCKER}" ]]; then
   if [[ -n $SKIP_CPP_TEST ]]; then
     make_wheel_args="$make_wheel_args --skip_cpp_test"
   fi
+  if [[ -n $SKIP_SMOKE_TEST ]]; then
+    make_wheel_args="$make_wheel_args --skip_smoke_test"
+  fi
   if [[ "$build_type" == "debug" ]]; then
     make_wheel_args="$make_wheel_args --debug"
   fi

scripts/requirements-minimal.txt

@@ -1,11 +1,17 @@
 numpy==1.16.4; python_version < '3.6'
-numpy==1.19.1; python_version >= '3.6'
+numpy==1.19.1; python_version >= '3.6' and python_version < '3.9'
+numpy==1.21.1; python_version == '3.9'
 
 cython==0.29.21
 decorator==4.4.0
+
 pandas==0.24.2; python_version <= '3.6'
-pandas==1.0.5; python_version > '3.6'
-pillow==6.2.2
+pandas==1.0.5; python_version > '3.6' and python_version < '3.9'
+pandas==1.3.1; python_version == '3.9'
+
+pillow==6.2.2; python_version < '3.9'
+pillow==8.3.1; python_version == '3.9'
+
 prettytable==0.7.2
 pytz==2019.1
 future==0.17.1

scripts/requirements.txt

@@ -2,25 +2,29 @@
 
 ### extra for full pkg ###
 
-coremltools==4.0b4
+coremltools==5.0b2
 
 scipy==1.2.1; python_version < '3.8'
 scipy==1.5.1; python_version == '3.8'
+scipy==1.7.1; python_version == '3.9'
 
-resampy==0.2.1
+resampy==0.2.1; python_version < '3.9'
 
 scikit-learn==0.20.3; python_version < '3.8'
 scikit-learn==0.23.1; python_version == '3.8'
+scikit-learn==0.24.2; python_version == '3.9'
 
 statsmodels==0.9.0; python_version < '3.8'
 statsmodels==0.11.1; python_version == '3.8'
+statsmodels==0.12.2; python_version == '3.9'
 
-tensorflow==2.0.2; sys_platform == 'darwin' and python_version != '2.7' and python_version != '3.8'
+tensorflow==2.0.2; sys_platform == 'darwin' and python_version != '2.7' and python_version != '3.8' and python_version != '3.9'
 
 UISoup==2.5.7
 
-llvmlite==0.33.0; python_version >= '3.6'
 llvmlite==0.31.0; python_version < '3.6'
+llvmlite==0.33.0; python_version >= '3.6' and python_version < '3.9'
+llvmlite==0.36.0; python_version == '3.9'
 
 ########## >=  #########
 

scripts/test_wheel.sh

@@ -31,6 +31,8 @@ print_help() {
   echo "  --docker-python3.7       Use docker to test on Python 3.7 in Ubuntu 18.04."
   echo
   echo "  --docker-python3.8       Use docker to test on Python 3.8 in Ubuntu 18.04."
+  ehco
+  echo "  --docker-python3.9       Use docker to test on Python 3.9 in Ubuntu 18.04."
   echo
   echo "  --minimal                test the turicreate+minimal package"
   echo
@@ -48,6 +50,7 @@ while [ $# -gt 0 ]
     --docker-python3.6)     USE_DOCKER=1;DOCKER_PYTHON=3.6;;
     --docker-python3.7)     USE_DOCKER=1;DOCKER_PYTHON=3.7;;
     --docker-python3.8)     USE_DOCKER=1;DOCKER_PYTHON=3.8;;
+    --docker-python3.9)     USE_DOCKER=1;DOCKER_PYTHON=3.9;;
     --minimal)              USE_MINIMAL=1;;
     --help)                 print_help ;;
     *) unknown_option "$1" ;;
@@ -84,7 +87,7 @@ if [[ -n "${USE_DOCKER}" ]]; then
       --env-file "${envlist}" \
       "${TC_BUILD_IMAGE_1404}" \
       /build/scripts/test_wheel.sh ${docker_extra_args}
-  elif [[ "${DOCKER_PYTHON}" == "3.6" ]] || [[ "${DOCKER_PYTHON}" == "3.7" ]] || [[ "${DOCKER_PYTHON}" == "3.8" ]]; then
+  elif [[ "${DOCKER_PYTHON}" == "3.6" ]] || [[ "${DOCKER_PYTHON}" == "3.7" ]] || [[ "${DOCKER_PYTHON}" == "3.8" ]] || [[ "${DOCKER_PYTHON}" == "3.9" ]]; then
     docker run --rm -m=8g \
       --mount type=bind,source="$WORKSPACE",target=/build,consistency=delegated \
       --env-file "${envlist}" \

src/model_server/lib/version_number.hpp

@@ -3,4 +3,4 @@
  * Use of this source code is governed by a BSD-3-clause license that can
  * be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
  */
-#define __UNITY_VERSION__ "6.4.1"//#{{VERSION_STRING}}
+#define __UNITY_VERSION__ "6.4.2"//#{{VERSION_STRING}}