Python 3.12 support

[Turakar]

I am interested in the state of Python 3.12 support, and want to push it forward. However, I did not find much existing information on it. It seems like there are no open issues and PRs on it, yet.

So what is the current state of this and how can we advance it?

I did not work with easybuild until today.

[Turakar]

Ok, so I did some first steps here. I can now install Python 3.12.1 using EasyBuild, but it is a bit hacky.

Python-3.12.1-GCCcore-13.2.0.eb

name = 'Python'
version = '3.12.1'

homepage = 'https://python.org/'
description = """Python is a programming language that lets you work more quickly and integrate your systems
 more effectively."""

toolchain = {'name': 'GCCcore', 'version': '13.2.0'}
toolchainopts = {'pic': True}

source_urls = ['https://www.python.org/ftp/%(namelower)s/%(version)s/']
sources = [SOURCE_TGZ]
checksums = ['d01ec6a33bc10009b09c17da95cc2759af5a580a7316b3a446eb4190e13f97b2']
patches = ['Python-3.12.1-ncurses-v3.patch']

builddependencies = [
    ('UnZip', '6.0'),
    ('pkgconf', '2.1.0'),
]

# Dependencies checked 2024-01-17
dependencies = [
    ('binutils', '2.40'),  # Should match toolchain
    ('bzip2', '1.0.8'),  # required for bz2 package in Python stdlib
    ('zlib', '1.3'),
    ('ncurses', '6.4'),
    ('libreadline', '8.2'),
    ('SQLite', '3.45.0'),
    ('XZ', '5.4.5'),
    ('libffi', '3.4.4'),
    ('OpenSSL', '1.1', '', SYSTEM),
]

install_pip = True

exts_default_options = {
    'download_dep_fail': True,
    'sanity_pip_check': True,
    'source_urls': [PYPI_SOURCE],
    'use_pip': True,
}

exts_list = [
    ('flit_core', '3.9.0', {
        'checksums': ['72ad266176c4a3fcfab5f2930d76896059851240570ce9a98733b658cb786eba'],
    }),
    ('wheel', '0.42.0', {
        'checksums': ['c45be39f7882c9d34243236f2d63cbd58039e360f85d0913425fbd7ceea617a8'],
    }),
    ('setuptools', '69.0.3', {
        'checksums': ['be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78'],
    }),
    ('pip', '23.3.2', {
        'checksums': ['7fd9972f96db22c8077a1ee2691b172c8089b17a5652a44494a9ecb0d78f9149'],
    }),
]

moduleclass = 'lang'

Python-3.12.1-ncurses-v3.patch

diff -ruN a/configure b/configure
--- a/configure	2024-01-17 16:53:44.864748925 +0100
+++ b/configure	2024-01-17 17:02:43.414498302 +0100
@@ -25787,7 +25787,7 @@
 if test "x$ac_cv_lib_ncursesw_initscr" = xyes
 then :
 
-          printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h
+          printf "%s\n" "#undef HAVE_NCURSESW" >>confdefs.h
 
           have_curses=ncursesw
           CURSES_CFLAGS=${CURSES_CFLAGS-""}
@@ -25851,7 +25851,7 @@
 if test "x$ac_cv_lib_ncursesw_initscr" = xyes
 then :
 
-          printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h
+          printf "%s\n" "#undef HAVE_NCURSESW" >>confdefs.h
 
           have_curses=ncursesw
           CURSES_CFLAGS=${CURSES_CFLAGS-""}
@@ -25873,7 +25873,7 @@
         { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5
 printf "%s\n" "yes" >&6; }
 
-      printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h
+      printf "%s\n" "#undef HAVE_NCURSESW" >>confdefs.h
 
       have_curses=ncursesw
 
@@ -26082,7 +26082,7 @@
 if test "$have_curses" != no -a "$ac_sys_system" = "Darwin"; then
 
   as_fn_append CURSES_CFLAGS " -D_XOPEN_SOURCE_EXTENDED=1"
-  printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h
+  printf "%s\n" "#undef HAVE_NCURSESW" >>confdefs.h
 
 fi
 

pkgconf-2.1.0-GCCcore-13.2.0.eb

easyblock = 'ConfigureMake'

name = 'pkgconf'
version = '2.1.0'

homepage = 'https://github.com/pkgconf/pkgconf'

description = """pkgconf is a program which helps to configure compiler and linker flags for development libraries.
 It is similar to pkg-config from freedesktop.org."""

toolchain = {'name': 'GCCcore', 'version': '13.2.0'}

source_urls = ['https://distfiles.ariadne.space/pkgconf/']
sources = [SOURCELOWER_TAR_GZ]
checksums = ['ab41b88c34fcae7f7ef29df663c90927e1b129a098af51f7d755d2ef44303cf4']

builddependencies = [('binutils', '2.40')]

postinstallcmds = ["cd %(installdir)s/bin && ln -s pkgconf pkg-config"]

sanity_check_paths = {
    'files': ['bin/pkg-config', 'bin/pkgconf'],
    'dirs': [],
}

sanity_check_commands = [
    "pkg-config --help",
    "pkgconf --help",
]

moduleclass = 'devel'

SQLite-3.45.0-GCCcore-13.2.0.eb

easyblock = 'ConfigureMake'

name = 'SQLite'
version = '3.45.0'
local_filename_version = '3450000'

homepage = 'https://www.sqlite.org/'
description = "SQLite: SQL Database Engine in a C Library"

toolchain = {'name': 'GCCcore', 'version': '13.2.0'}
toolchainopts = {'pic': True}

source_urls = ['https://www.sqlite.org/2024/']
sources = ['%%(namelower)s-autoconf-%s.tar.gz' % (local_filename_version)]
checksums = ['72887d57a1d8f89f52be38ef84a6353ce8c3ed55ada7864eb944abd9a495e436']

builddependencies = [
    ('binutils', '2.40'),
]
dependencies = [
    ('libreadline', '8.2'),
    ('Tcl', '8.6.13'),
]

# enable additional APIs that provide access to meta-data about tables and queries
# needed for GDAL when it used as a dep for QGIS
buildopts = 'CC="$CC" CFLAGS="$CFLAGS -DSQLITE_ENABLE_COLUMN_METADATA"'

sanity_check_paths = {
    'files': ['bin/sqlite3', 'include/sqlite3ext.h', 'include/sqlite3.h',
              'lib/libsqlite3.a', 'lib/libsqlite3.%s' % SHLIB_EXT],
    'dirs': ['lib/pkgconfig'],
}

sanity_check_commands = [
    'sqlite3 --version | grep ^%(version)s',
]

moduleclass = 'devel'

XZ-5.4.5-GCCcore-13.2.0.eb

easyblock = 'ConfigureMake'

name = 'XZ'
version = '5.4.5'

homepage = 'https://tukaani.org/xz/'
description = "xz: XZ utilities"

toolchain = {'name': 'GCCcore', 'version': '13.2.0'}

source_urls = ['https://tukaani.org/xz/']
sources = [SOURCELOWER_TAR_BZ2]
checksums = ['8ccf5fff868c006f29522e386fb4c6a1b66463fbca65a4cfc3c4bd596e895e79']

builddependencies = [
    # use gettext built with system toolchain as build dep to avoid cyclic dependency (XZ -> gettext -> libxml2 -> XZ)
    ('gettext', '0.22', '', SYSTEM),
    ('binutils', '2.40'),
]

# may become useful in non-x86 archs
# configopts = ' --disable-assembler '

sanity_check_paths = {
    'files': ['bin/lzmainfo', 'bin/unxz', 'bin/xz'],
    'dirs': []
}

sanity_check_commands = [
    "xz --help",
    "unxz --help",
]

moduleclass = 'tools'

zlib-1.3-GCCcore-13.2.0.eb

easyblock = 'ConfigureMake'

name = 'zlib'
version = '1.3'

homepage = 'https://www.zlib.net/'
description = """zlib is designed to be a free, general-purpose, legally unencumbered -- that is,
 not covered by any patents -- lossless data-compression library for use on virtually any
 computer hardware and operating system."""

toolchain = {'name': 'GCCcore', 'version': '13.2.0'}
toolchainopts = {'pic': True}

source_urls = ['https://zlib.net/fossils']
sources = [SOURCELOWER_TAR_GZ]
checksums = ['ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e']

# patches = ['zlib-%(version)s_fix-CC-logic-in-configure.patch']

# use same binutils version that was used when building GCC toolchain
builddependencies = [('binutils', '2.40')]

sanity_check_paths = {
    'files': ['include/zconf.h', 'include/zlib.h', 'lib/libz.a', 'lib/libz.%s' % SHLIB_EXT],
    'dirs': [],
}

moduleclass = 'lib'

There are some issues. First, I was unable to create a -bare installation because the Python Easyblock expects setuptools, but Python 3.12 does not seem to ship it by default anymore. The second issue is about ncurses.

Python 3.12 completely changed the corresponding configuration part in configure.ac, which now configures it to use ncursesw, the wide-character variant of ncurses. This seems to be caused by the fact that EasyBuild's ncurses package actually contains the library ncursesw. However, as include flags we only pass <root>/include, and not <root>/include/ncursesw, which is not recognized by Python's configure. As such, configure enables wide-character support, but misses the corresponding headers, which leads to implicit function declaration errors.

My current fix is quite hacky - as I was not able to edit configure.ac with a patch and then run autoreconf, I instead opted to patch configure by replacing all #define HAVE_NCURSESW 1 by a #undef HAVE_NCURSESW.

In my opinion, it would be nicer to actually enable wide-character support in ncurses and then pass the proper include flags, but I am not sure on how to do this, so help would be appreciated.

[casparvl]

Just leaving a breadcrumb here, related to Python 3.12 support (but unrelated to your earlier issues @Turakar) : from @jngrad I learned that unittest in Python-3.12 behaves differently with respect to return codes for skipped tests. Skipped tests used to return 0, now they return 5. Especially combined with e.g. OpenMPI 5 (which will emit an error whenever any non-zero exit code is returned) this caused issues for him (see this bug report).

I'm assuming we will hit issues with tests that use unittest suddenly failing because of this. If people hit that, I'm hoping they find this comment so that they know where it might be coming from... :)

[branfosj]

Python/3.12 is available in #20579, with #20405 to add the .pc files for ncurses to have Python correctly detect the right part of ncurses.