Add support for Python 3.11

ci/Jenkinsfile.groovy

@@ -1,6 +1,6 @@
 #!/usr/bin/groovy
 //------------------------------------------------------------------------------
-// Copyright 2018-2020 H2O.ai
+// Copyright 2018-2022 H2O.ai
 //
 // Permission is hereby granted, free of charge, to any person obtaining a
 // copy of this software and associated documentation files (the "Software"),
@@ -256,12 +256,14 @@ ansiColor('xterm') {
                                             /opt/python/cp38-cp38/bin/python3.8 ci/ext.py wheel --audit && \
                                             /opt/python/cp39-cp39/bin/python3.9 ci/ext.py wheel --audit && \
                                             /opt/python/cp310-cp310/bin/python3.10 ci/ext.py wheel --audit && \
+                                            /opt/python/cp311-cp311/bin/python3.11 ci/ext.py wheel --audit && \
                                             echo '===== Py3.8 Debug =====' && unzip -p dist/*debug*.whl datatable/_build_info.py && \
                                             mv dist/*debug*.whl . && \
                                             echo '===== Py3.7 =====' && unzip -p dist/*cp37*.whl datatable/_build_info.py && \
                                             echo '===== Py3.8 =====' && unzip -p dist/*cp38*.whl datatable/_build_info.py && \
                                             echo '===== Py3.9 =====' && unzip -p dist/*cp39*.whl datatable/_build_info.py && \
                                             echo '===== Py3.10 =====' && unzip -p dist/*cp310*.whl datatable/_build_info.py && \
+                                            echo '===== Py3.11 =====' && unzip -p dist/*cp311*.whl datatable/_build_info.py && \
                                             mv *debug*.whl dist/ && \
                                             ls -la dist"
                                 """
@@ -296,10 +298,14 @@ ansiColor('xterm') {
                                         source /Users/jenkins/datatable_envs/py310/bin/activate
                                         python ci/ext.py wheel
                                         deactivate
+                                        source /Users/jenkins/datatable_envs/py311/bin/activate
+                                        python ci/ext.py wheel
+                                        deactivate
                                         echo '===== Py3.7 =====' && unzip -p dist/*cp37*.whl datatable/_build_info.py
                                         echo '===== Py3.8 =====' && unzip -p dist/*cp38*.whl datatable/_build_info.py
                                         echo '===== Py3.9 =====' && unzip -p dist/*cp39*.whl datatable/_build_info.py
                                         echo '===== Py3.10 =====' && unzip -p dist/*cp310*.whl datatable/_build_info.py
+                                        echo '===== Py3.11 =====' && unzip -p dist/*cp311*.whl datatable/_build_info.py
                                         ls dist
                                     """
                                     stash name: 'x86_64-macos-wheels', includes: "dist/*.whl"
@@ -340,12 +346,14 @@ ansiColor('xterm') {
                                                 /opt/python/cp38-cp38/bin/python3.8 ci/ext.py wheel --audit && \
                                                 /opt/python/cp39-cp39/bin/python3.9 ci/ext.py wheel --audit && \
                                                 /opt/python/cp310-cp310/bin/python3.10 ci/ext.py wheel --audit && \
+                                                /opt/python/cp311-cp311/bin/python3.11 ci/ext.py wheel --audit && \
                                                 echo '===== Py3.8 Debug =====' && unzip -p dist/*debug*.whl datatable/_build_info.py && \
                                                 mv dist/*debug*.whl . && \
                                                 echo '===== Py3.7 =====' && unzip -p dist/*cp37*.whl datatable/_build_info.py && \
                                                 echo '===== Py3.8 =====' && unzip -p dist/*cp38*.whl datatable/_build_info.py && \
                                                 echo '===== Py3.9 =====' && unzip -p dist/*cp39*.whl datatable/_build_info.py && \
                                                 echo '===== Py3.10 =====' && unzip -p dist/*cp310*.whl datatable/_build_info.py && \
+                                                echo '===== Py3.11 =====' && unzip -p dist/*cp311*.whl datatable/_build_info.py && \
                                                 mv *debug*.whl dist/ && \
                                                 ls -la dist"
                                     """
@@ -435,6 +443,20 @@ ansiColor('xterm') {
                             }
                         }
                     }) <<
+                    namedStage('Test x86_64-manylinux-py311', { stageName, stageDir ->
+                        node(NODE_LINUX) {
+                            buildSummary.stageWithSummary(stageName, stageDir) {
+                                cleanWs()
+                                dumpInfo()
+                                dir(stageDir) {
+                                    unstash 'datatable-sources'
+                                    unstash 'x86_64-manylinux-wheels'
+                                    test_in_docker("x86_64-manylinux-py311", "311",
+                                                   DOCKER_IMAGE_X86_64_MANYLINUX)
+                                }
+                            }
+                        }
+                    }) <<
                     namedStage('Test ppc64le-manylinux-py37', doPpcTests, { stageName, stageDir ->
                         node(NODE_PPC) {
                             buildSummary.stageWithSummary(stageName, stageDir) {
@@ -505,6 +527,20 @@ ansiColor('xterm') {
                             }
                         }
                     }) <<
+                    namedStage('Test ppc64le-manylinux-py311', doPpcTests && doPy38Tests, { stageName, stageDir ->
+                        node(NODE_PPC) {
+                            buildSummary.stageWithSummary(stageName, stageDir) {
+                                cleanWs()
+                                dumpInfo()
+                                dir(stageDir) {
+                                    unstash 'datatable-sources'
+                                    unstash 'ppc64le-manylinux-wheels'
+                                    test_in_docker("ppc64le-manylinux-py311", "311",
+                                                   DOCKER_IMAGE_PPC64LE_MANYLINUX)
+                                }
+                            }
+                        }
+                    }) <<
                     namedStage('Test x86_64-macos-py37', { stageName, stageDir ->
                         node(NODE_MACOS) {
                             buildSummary.stageWithSummary(stageName, stageDir) {
@@ -792,6 +828,7 @@ def get_python_for_docker(String pyver, String image) {
         if (pyver == "38") return "/opt/python/cp38-cp38/bin/python3.8"
         if (pyver == "39") return "/opt/python/cp39-cp39/bin/python3.9"
         if (pyver == "310") return "/opt/python/cp310-cp310/bin/python3.10"
+        if (pyver == "311") return "/opt/python/cp311-cp311/bin/python3.11"
     }
     throw new Exception("Unknown python ${pyver} for docker ${image}")
 }

ci/appveyor.yml

@@ -1,7 +1,8 @@
 image:
   - Visual Studio 2019
-  - macos-monterey
   - Ubuntu
+  # Uncomment the line below when python 3.11 is added to macOS image
+  # - macos-monterey
 
 init:
   # Uncomment the line below to enable RDP access on AppVeyor
@@ -82,18 +83,18 @@ build_script:
 
 
     # =======================================================================
-    #   Build and test wheel for Python 3.10
+    #   Build and test wheel for Python 3.11
     # =======================================================================
 
-    source $HOME/venv3.10/bin/activate
+    source $HOME/venv3.11/bin/activate
 
     python -V
 
     python ci/ext.py wheel
 
-    DT_WHEEL=`ls dist/*-cp310-*.whl`
+    DT_WHEEL=`ls dist/*-cp311-*.whl`
 
-    echo "----- _build_info.py for Python 3.10 ------------------------------"
+    echo "----- _build_info.py for Python 3.11 ------------------------------"
 
     cat src/datatable/_build_info.py
 
@@ -351,3 +352,37 @@ build_script:
 
     python -m pip uninstall -y $DT_WHEEL
 
+
+
+    # =======================================================================
+    #   Build and test wheel for Python 3.11
+    # =======================================================================
+
+    $env:PATH = "C:/Python311-x64;C:/Python311-x64/Scripts;$DEFAULT_PATH"
+
+    python -V
+
+    python ci/ext.py wheel
+
+    $DT_WHEEL = ls dist/*-cp311-*.whl
+
+    echo "DT_WHEEL = $DT_WHEEL"
+
+    echo "----- _build_info.py for Python 3.11 ------------------------------"
+
+    cat src/datatable/_build_info.py
+
+    echo "------------------------------------------------------------------"
+
+    python -m pip install --upgrade pip
+
+    python -m pip install $DT_WHEEL
+
+    python -m pip install pytest docutils pandas pyarrow
+
+    python -m pytest -ra --maxfail=10 -Werror -vv --showlocals ./tests/
+
+    if(!$?) { Exit $LASTEXITCODE }
+
+    python -m pip uninstall -y $DT_WHEEL
+

ci/ext.py

@@ -337,6 +337,7 @@ def build_extension(cmd, verbosity=3):
                 "-Weverything",
                 "-Wno-c++98-compat-pedantic",
                 "-Wno-c99-extensions",
+                "-Wno-disabled-macro-expansion",
                 "-Wno-exit-time-destructors",
                 "-Wno-float-equal",
                 "-Wno-global-constructors",

docs/releases/v1.1.0.rst

@@ -200,6 +200,8 @@
 
     -[enh] Added support for Python `3.10`. [#3210]
 
+    -[enh] Added support for Python `3.11`. [#3374]
+
     -[enh] Parameter ``force=True`` in function :func:`rbind()` (or method
         :meth:`dt.Frame.rbind()`) will now allow combining columns
         of incompatible types. [#3062]

src/core/_dt.h

@@ -30,10 +30,6 @@
 #include <utility>      // std::move, std::pair
 #include <vector>       // std::vector
 
-typedef struct _object     PyObject;
-typedef struct _typeobject PyTypeObject;
-typedef struct bufferinfo  Py_buffer;
-
 class Buffer;
 class Column;
 class ColumnJayData;

src/core/buffer.cc

@@ -24,7 +24,6 @@
 #include <cstring>                            // std::strerror, std::memcpy
 #include <mutex>                              // std::mutex, std::lock_guard
 #include "buffer.h"
-#include "lib/pythoncapi/pythoncapi_compat.h" // Py_SET_REFCNT()
 #include "mmm.h"                              // MemoryMapWorker, MemoryMapManager
 #include "python/pybuffer.h"                  // py::buffer
 #include "utils/alloc.h"                      // dt::malloc, dt::realloc

src/core/call_logger.h

@@ -22,6 +22,7 @@
 #ifndef dt_CALL_LOGGER_h
 #define dt_CALL_LOGGER_h
 #include "_dt.h"
+#include "python/python.h"
 namespace dt {
 
 

src/core/column/const_na.cc

@@ -19,7 +19,6 @@
 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 // IN THE SOFTWARE.
 //------------------------------------------------------------------------------
-#include "lib/pythoncapi/pythoncapi_compat.h" // Py_SET_REFCNT()
 #include "column/const.h"
 #include "column/sentinel_fw.h"
 #include "column/sentinel_str.h"