ARROW-10224: [Python][Packaging] Build, test, and support Python 3.9

dev/release/verify-release-candidate.sh

@@ -625,7 +625,7 @@ check_python_imports() {
 }
 
 test_linux_wheels() {
-  local py_arches="3.5m 3.6m 3.7m 3.8"
+  local py_arches="3.5m 3.6m 3.7m 3.8 3.9"
   local manylinuxes="1 2010 2014"
 
   for py_arch in ${py_arches}; do
@@ -649,7 +649,7 @@ test_linux_wheels() {
 }
 
 test_macos_wheels() {
-  local py_arches="3.5m 3.6m 3.7m 3.8"
+  local py_arches="3.5m 3.6m 3.7m 3.8 3.9"
 
   for py_arch in ${py_arches}; do
     local env=_verify_wheel-${py_arch}

dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.9.____cpython.yaml

@@ -0,0 +1,66 @@
+BUILD:
+- aarch64-conda_cos7-linux-gnu
+aws_sdk_cpp:
+- 1.8.54
+boost_cpp:
+- 1.74.0
+bzip2:
+- '1'
+c_compiler:
+- gcc
+c_compiler_version:
+- '7.5'
+cdt_arch:
+- aarch64
+cdt_name:
+- cos7
+channel_sources:
+- conda-forge
+channel_targets:
+- conda-forge main
+cuda_compiler_version:
+- None
+cxx_compiler:
+- gxx
+cxx_compiler_version:
+- '7.5'
+docker_image:
+- condaforge/linux-anvil-aarch64
+gflags:
+- '2.2'
+glog:
+- 0.4.0
+grpc_cpp:
+- '1.30'
+libprotobuf:
+- '3.13'
+lz4_c:
+- 1.9.2
+numpy:
+- '1.16'
+orc:
+- 1.6.4
+pin_run_as_build:
+  boost-cpp:
+    max_pin: x.x.x
+  bzip2:
+    max_pin: x
+  lz4-c:
+    max_pin: x.x.x
+  python:
+    min_pin: x.x
+    max_pin: x.x
+  zlib:
+    max_pin: x.x
+python:
+- 3.9.* *_cpython
+re2:
+- 2020.08.01
+snappy:
+- '1'
+target_platform:
+- linux-aarch64
+zlib:
+- '1.2'
+zstd:
+- '1.4'

dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.9.____cpython.yaml

@@ -0,0 +1,65 @@
+aws_sdk_cpp:
+- 1.8.54
+boost_cpp:
+- 1.74.0
+bzip2:
+- '1'
+c_compiler:
+- gcc
+c_compiler_version:
+- '7'
+channel_sources:
+- conda-forge,defaults
+channel_targets:
+- conda-forge main
+cuda_compiler:
+- nvcc
+cuda_compiler_version:
+- '9.2'
+cxx_compiler:
+- gxx
+cxx_compiler_version:
+- '7'
+docker_image:
+- condaforge/linux-anvil-cuda:9.2
+gflags:
+- '2.2'
+glog:
+- 0.4.0
+grpc_cpp:
+- '1.30'
+libprotobuf:
+- '3.13'
+lz4_c:
+- 1.9.2
+numpy:
+- '1.16'
+orc:
+- 1.6.4
+pin_run_as_build:
+  boost-cpp:
+    max_pin: x.x.x
+  bzip2:
+    max_pin: x
+  lz4-c:
+    max_pin: x.x.x
+  python:
+    min_pin: x.x
+    max_pin: x.x
+  zlib:
+    max_pin: x.x
+python:
+- 3.9.* *_cpython
+re2:
+- 2020.08.01
+snappy:
+- '1'
+target_platform:
+- linux-64
+zip_keys:
+- - cuda_compiler_version
+  - docker_image
+zlib:
+- '1.2'
+zstd:
+- '1.4'

dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.9.____cpython.yaml

@@ -0,0 +1,65 @@
+aws_sdk_cpp:
+- 1.8.54
+boost_cpp:
+- 1.74.0
+bzip2:
+- '1'
+c_compiler:
+- gcc
+c_compiler_version:
+- '7'
+channel_sources:
+- conda-forge,defaults
+channel_targets:
+- conda-forge main
+cuda_compiler:
+- nvcc
+cuda_compiler_version:
+- None
+cxx_compiler:
+- gxx
+cxx_compiler_version:
+- '7'
+docker_image:
+- condaforge/linux-anvil-comp7
+gflags:
+- '2.2'
+glog:
+- 0.4.0
+grpc_cpp:
+- '1.30'
+libprotobuf:
+- '3.13'
+lz4_c:
+- 1.9.2
+numpy:
+- '1.16'
+orc:
+- 1.6.4
+pin_run_as_build:
+  boost-cpp:
+    max_pin: x.x.x
+  bzip2:
+    max_pin: x
+  lz4-c:
+    max_pin: x.x.x
+  python:
+    min_pin: x.x
+    max_pin: x.x
+  zlib:
+    max_pin: x.x
+python:
+- 3.9.* *_cpython
+re2:
+- 2020.08.01
+snappy:
+- '1'
+target_platform:
+- linux-64
+zip_keys:
+- - cuda_compiler_version
+  - docker_image
+zlib:
+- '1.2'
+zstd:
+- '1.4'

dev/tasks/conda-recipes/.ci_support/osx_python3.9.____cpython.yaml

@@ -0,0 +1,62 @@
+MACOSX_DEPLOYMENT_TARGET:
+- '10.9'
+aws_sdk_cpp:
+- 1.8.54
+boost_cpp:
+- 1.74.0
+bzip2:
+- '1'
+c_compiler:
+- clang
+c_compiler_version:
+- '10'
+channel_sources:
+- conda-forge,defaults
+channel_targets:
+- conda-forge main
+cuda_compiler_version:
+- None
+cxx_compiler:
+- clangxx
+cxx_compiler_version:
+- '10'
+gflags:
+- '2.2'
+glog:
+- 0.4.0
+grpc_cpp:
+- '1.30'
+libprotobuf:
+- '3.13'
+lz4_c:
+- 1.9.2
+macos_machine:
+- x86_64-apple-darwin13.4.0
+numpy:
+- '1.16'
+orc:
+- 1.6.4
+pin_run_as_build:
+  boost-cpp:
+    max_pin: x.x.x
+  bzip2:
+    max_pin: x
+  lz4-c:
+    max_pin: x.x.x
+  python:
+    min_pin: x.x
+    max_pin: x.x
+  zlib:
+    max_pin: x.x
+python:
+- 3.9.* *_cpython
+re2:
+- 2020.08.01
+snappy:
+- '1'
+target_platform:
+- osx-64
+zlib:
+- '1.2'
+zstd:
+- '1.4'

dev/tasks/conda-recipes/.ci_support/win_python3.9.____cpython.yaml

@@ -0,0 +1,52 @@
+aws_sdk_cpp:
+- 1.8.54
+boost_cpp:
+- 1.74.0
+bzip2:
+- '1'
+c_compiler:
+- vs2017
+channel_sources:
+- conda-forge,defaults
+channel_targets:
+- conda-forge main
+cuda_compiler_version:
+- None
+cxx_compiler:
+- vs2017
+gflags:
+- '2.2'
+glog:
+- 0.4.0
+grpc_cpp:
+- '1.30'
+libprotobuf:
+- '3.13'
+lz4_c:
+- 1.9.2
+numpy:
+- '1.16'
+pin_run_as_build:
+  boost-cpp:
+    max_pin: x.x.x
+  bzip2:
+    max_pin: x
+  lz4-c:
+    max_pin: x.x.x
+  python:
+    min_pin: x.x
+    max_pin: x.x
+  zlib:
+    max_pin: x.x
+python:
+- 3.9.* *_cpython
+re2:
+- 2020.08.01
+snappy:
+- '1'
+target_platform:
+- win-64
+zlib:
+- '1.2'
+zstd:
+- '1.4'

dev/tasks/docker-tests/github.linux.yml

@@ -42,7 +42,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.8
+          python-version: 3.9
       - name: Setup Archery
         run: pip install -e arrow/dev/archery[docker]
       - name: Execute Docker Build

dev/tasks/homebrew-formulae/apache-arrow.rb

@@ -16,7 +16,7 @@ class ApacheArrow < Formula
   depends_on "numpy"
   depends_on "openssl@1.1"
   depends_on "protobuf"
-  depends_on "python@3.8"
+  depends_on "python@3.9"
   depends_on "rapidjson"
   depends_on "snappy"
   depends_on "thrift"
@@ -40,7 +40,7 @@ def install
       -DARROW_WITH_SNAPPY=ON
       -DARROW_WITH_BROTLI=ON
       -DARROW_INSTALL_NAME_RPATH=OFF
-      -DPYTHON_EXECUTABLE=#{Formula["python@3.8"].bin/"python3"}
+      -DPYTHON_EXECUTABLE=#{Formula["python@3.9"].bin/"python3"}
     ]
     # Re-enable -DARROW_S3=ON and add back aws-sdk-cpp to depends_on in ARROW-6437
 

dev/tasks/python-wheels/travis.osx.yml

@@ -31,7 +31,7 @@ addons:
       - git
       - openssl@1.1
       - protobuf
-      - python@3.8
+      - python@3.9
       - thrift
       - wget
     update: false
@@ -52,7 +52,7 @@ env:
 
 before_install:
   - git clone https://github.com/matthew-brett/multibuild
-  - git -C multibuild checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa
+  - git -C multibuild checkout 8882150df6529658700b66bec124dfb77eefca26
 
   - git clone --no-checkout {{ arrow.remote }} arrow
   - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
@@ -84,8 +84,8 @@ install:
   - deactivate
 
   # crossbow dependencies for deployment
-  - /usr/local/opt/python@3.8/libexec/bin/pip install click ruamel.yaml setuptools_scm github3.py toolz requests[security]
-  - /usr/local/opt/python@3.8/libexec/bin/python arrow/dev/tasks/crossbow.py
+  - /usr/local/opt/python@3.9/libexec/bin/pip install click ruamel.yaml setuptools_scm github3.py toolz requests[security]
+  - /usr/local/opt/python@3.9/libexec/bin/python arrow/dev/tasks/crossbow.py
       --queue-path $(pwd)
       --queue-remote {{ queue_remote_url }}
       upload-artifacts

dev/tasks/python-wheels/win-build.bat

@@ -110,6 +110,6 @@ set ARROW_TEST_DATA=%ARROW_SRC%\testing\data
 %PYTHON_INTERPRETER% -c "import pyarrow.dataset" || exit /B
 
 @rem run the python tests, but disable the cython because there is a linking
-@rem issue on python 3.8
+@rem issue on python 3.9
 set PYARROW_TEST_CYTHON=OFF
 %PYTHON_INTERPRETER% -m pytest -rs --pyargs pyarrow || exit /B