Skip to content

Commit

Permalink
Add support for Python 3.11 (#3387)
Browse files Browse the repository at this point in the history 
- make codebase compatible with Python 3.11 changes;
- only use the required adopted  code from `python/pythoncapi-compat`;
- remove obsolete code for the older Python versions;
- switch to Python 3.11 on AppVeyor for Windows and linux;
- add Python 3.11 support to Jenkins;
 
Closes #3374
  • Loading branch information
@oleksiyskononenko
oleksiyskononenko committed Dec 1, 2022
1 parent 7e8f19f commit 6e80543
Show file tree
Hide file tree
Showing 18 changed files with 131 additions and 492 deletions.
39 changes: 38 additions & 1 deletion ci/Jenkinsfile.groovy
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/groovy
//------------------------------------------------------------------------------
// Copyright 2018-2020 H2O.ai
// Copyright 2018-2022 H2O.ai
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
Expand Down Expand Up @@ -256,12 +256,14 @@ ansiColor('xterm') {
/opt/python/cp38-cp38/bin/python3.8 ci/ext.py wheel --audit && \
/opt/python/cp39-cp39/bin/python3.9 ci/ext.py wheel --audit && \
/opt/python/cp310-cp310/bin/python3.10 ci/ext.py wheel --audit && \
/opt/python/cp311-cp311/bin/python3.11 ci/ext.py wheel --audit && \
echo '===== Py3.8 Debug =====' && unzip -p dist/*debug*.whl datatable/_build_info.py && \
mv dist/*debug*.whl . && \
echo '===== Py3.7 =====' && unzip -p dist/*cp37*.whl datatable/_build_info.py && \
echo '===== Py3.8 =====' && unzip -p dist/*cp38*.whl datatable/_build_info.py && \
echo '===== Py3.9 =====' && unzip -p dist/*cp39*.whl datatable/_build_info.py && \
echo '===== Py3.10 =====' && unzip -p dist/*cp310*.whl datatable/_build_info.py && \
echo '===== Py3.11 =====' && unzip -p dist/*cp311*.whl datatable/_build_info.py && \
mv *debug*.whl dist/ && \
ls -la dist"
"""
Expand Down Expand Up @@ -296,10 +298,14 @@ ansiColor('xterm') {
source /Users/jenkins/datatable_envs/py310/bin/activate
python ci/ext.py wheel
deactivate
source /Users/jenkins/datatable_envs/py311/bin/activate
python ci/ext.py wheel
deactivate
echo '===== Py3.7 =====' && unzip -p dist/*cp37*.whl datatable/_build_info.py
echo '===== Py3.8 =====' && unzip -p dist/*cp38*.whl datatable/_build_info.py
echo '===== Py3.9 =====' && unzip -p dist/*cp39*.whl datatable/_build_info.py
echo '===== Py3.10 =====' && unzip -p dist/*cp310*.whl datatable/_build_info.py
echo '===== Py3.11 =====' && unzip -p dist/*cp311*.whl datatable/_build_info.py
ls dist
"""
stash name: 'x86_64-macos-wheels', includes: "dist/*.whl"
Expand Down Expand Up @@ -340,12 +346,14 @@ ansiColor('xterm') {
/opt/python/cp38-cp38/bin/python3.8 ci/ext.py wheel --audit && \
/opt/python/cp39-cp39/bin/python3.9 ci/ext.py wheel --audit && \
/opt/python/cp310-cp310/bin/python3.10 ci/ext.py wheel --audit && \
/opt/python/cp311-cp311/bin/python3.11 ci/ext.py wheel --audit && \
echo '===== Py3.8 Debug =====' && unzip -p dist/*debug*.whl datatable/_build_info.py && \
mv dist/*debug*.whl . && \
echo '===== Py3.7 =====' && unzip -p dist/*cp37*.whl datatable/_build_info.py && \
echo '===== Py3.8 =====' && unzip -p dist/*cp38*.whl datatable/_build_info.py && \
echo '===== Py3.9 =====' && unzip -p dist/*cp39*.whl datatable/_build_info.py && \
echo '===== Py3.10 =====' && unzip -p dist/*cp310*.whl datatable/_build_info.py && \
echo '===== Py3.11 =====' && unzip -p dist/*cp311*.whl datatable/_build_info.py && \
mv *debug*.whl dist/ && \
ls -la dist"
"""
Expand Down Expand Up @@ -435,6 +443,20 @@ ansiColor('xterm') {
}
}
}) <<
namedStage('Test x86_64-manylinux-py311', { stageName, stageDir ->
node(NODE_LINUX) {
buildSummary.stageWithSummary(stageName, stageDir) {
cleanWs()
dumpInfo()
dir(stageDir) {
unstash 'datatable-sources'
unstash 'x86_64-manylinux-wheels'
test_in_docker("x86_64-manylinux-py311", "311",
DOCKER_IMAGE_X86_64_MANYLINUX)
}
}
}
}) <<
namedStage('Test ppc64le-manylinux-py37', doPpcTests, { stageName, stageDir ->
node(NODE_PPC) {
buildSummary.stageWithSummary(stageName, stageDir) {
Expand Down Expand Up @@ -505,6 +527,20 @@ ansiColor('xterm') {
}
}
}) <<
namedStage('Test ppc64le-manylinux-py311', doPpcTests && doPy38Tests, { stageName, stageDir ->
node(NODE_PPC) {
buildSummary.stageWithSummary(stageName, stageDir) {
cleanWs()
dumpInfo()
dir(stageDir) {
unstash 'datatable-sources'
unstash 'ppc64le-manylinux-wheels'
test_in_docker("ppc64le-manylinux-py311", "311",
DOCKER_IMAGE_PPC64LE_MANYLINUX)
}
}
}
}) <<
namedStage('Test x86_64-macos-py37', { stageName, stageDir ->
node(NODE_MACOS) {
buildSummary.stageWithSummary(stageName, stageDir) {
Expand Down Expand Up @@ -792,6 +828,7 @@ def get_python_for_docker(String pyver, String image) {
if (pyver == "38") return "/opt/python/cp38-cp38/bin/python3.8"
if (pyver == "39") return "/opt/python/cp39-cp39/bin/python3.9"
if (pyver == "310") return "/opt/python/cp310-cp310/bin/python3.10"
if (pyver == "311") return "/opt/python/cp311-cp311/bin/python3.11"
}
throw new Exception("Unknown python ${pyver} for docker ${image}")
}
Expand Down
45 changes: 40 additions & 5 deletions ci/appveyor.yml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
image:
- Visual Studio 2019
- macos-monterey
- Ubuntu
# Uncomment the line below when python 3.11 is added to macOS image
# - macos-monterey

init:
# Uncomment the line below to enable RDP access on AppVeyor
Expand Down Expand Up @@ -82,18 +83,18 @@ build_script:
# =======================================================================
# Build and test wheel for Python 3.10
# Build and test wheel for Python 3.11
# =======================================================================
source $HOME/venv3.10/bin/activate
source $HOME/venv3.11/bin/activate
python -V
python ci/ext.py wheel
DT_WHEEL=`ls dist/*-cp310-*.whl`
DT_WHEEL=`ls dist/*-cp311-*.whl`
echo "----- _build_info.py for Python 3.10 ------------------------------"
echo "----- _build_info.py for Python 3.11 ------------------------------"
cat src/datatable/_build_info.py
Expand Down Expand Up @@ -351,3 +352,37 @@ build_script:
python -m pip uninstall -y $DT_WHEEL
# =======================================================================
# Build and test wheel for Python 3.11
# =======================================================================
$env:PATH = "C:/Python311-x64;C:/Python311-x64/Scripts;$DEFAULT_PATH"
python -V
python ci/ext.py wheel
$DT_WHEEL = ls dist/*-cp311-*.whl
echo "DT_WHEEL = $DT_WHEEL"
echo "----- _build_info.py for Python 3.11 ------------------------------"
cat src/datatable/_build_info.py
echo "------------------------------------------------------------------"
python -m pip install --upgrade pip
python -m pip install $DT_WHEEL
python -m pip install pytest docutils pandas pyarrow
python -m pytest -ra --maxfail=10 -Werror -vv --showlocals ./tests/
if(!$?) { Exit $LASTEXITCODE }
python -m pip uninstall -y $DT_WHEEL
1 change: 1 addition & 0 deletions ci/ext.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,7 @@ def build_extension(cmd, verbosity=3):
"-Weverything",
"-Wno-c++98-compat-pedantic",
"-Wno-c99-extensions",
"-Wno-disabled-macro-expansion",
"-Wno-exit-time-destructors",
"-Wno-float-equal",
"-Wno-global-constructors",
Expand Down
2 changes: 2 additions & 0 deletions docs/releases/v1.1.0.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,8 @@

-[enh] Added support for Python `3.10`. [#3210]

-[enh] Added support for Python `3.11`. [#3374]

-[enh] Parameter ``force=True`` in function :func:`rbind()` (or method
:meth:`dt.Frame.rbind()`) will now allow combining columns
of incompatible types. [#3062]
Expand Down
4 changes: 0 additions & 4 deletions src/core/_dt.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,6 @@
#include <utility> // std::move, std::pair
#include <vector> // std::vector

typedef struct _object PyObject;
typedef struct _typeobject PyTypeObject;
typedef struct bufferinfo Py_buffer;

class Buffer;
class Column;
class ColumnJayData;
Expand Down
1 change: 0 additions & 1 deletion src/core/buffer.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
#include <cstring> // std::strerror, std::memcpy
#include <mutex> // std::mutex, std::lock_guard
#include "buffer.h"
#include "lib/pythoncapi/pythoncapi_compat.h" // Py_SET_REFCNT()
#include "mmm.h" // MemoryMapWorker, MemoryMapManager
#include "python/pybuffer.h" // py::buffer
#include "utils/alloc.h" // dt::malloc, dt::realloc
Expand Down
1 change: 1 addition & 0 deletions src/core/call_logger.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#ifndef dt_CALL_LOGGER_h
#define dt_CALL_LOGGER_h
#include "_dt.h"
#include "python/python.h"
namespace dt {


Expand Down
1 change: 0 additions & 1 deletion src/core/column/const_na.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.
//------------------------------------------------------------------------------
#include "lib/pythoncapi/pythoncapi_compat.h" // Py_SET_REFCNT()
#include "column/const.h"
#include "column/sentinel_fw.h"
#include "column/sentinel_str.h"
Expand Down

0 comments on commit 6e80543

Please sign in to comment.