From c2e6f81e4ab2a2cf2127ddb1c875278e862b13ff Mon Sep 17 00:00:00 2001 From: Jeffrey Bosboom Date: Sat, 20 Sep 2025 00:54:57 -0700 Subject: [PATCH 1/3] Implement os.statx --- Doc/library/os.rst | 131 +++++ Doc/library/stat.rst | 19 + .../pycore_global_objects_fini_generated.h | 2 + Include/internal/pycore_global_strings.h | 2 + .../internal/pycore_runtime_init_generated.h | 2 + .../internal/pycore_unicodeobject_generated.h | 8 + Lib/os.py | 6 + Lib/stat.py | 15 + Lib/test/test_os.py | 107 +++- Lib/test/test_posix.py | 30 +- Misc/ACKS | 1 + ...5-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst | 2 + Modules/clinic/posixmodule.c.h | 141 ++++- Modules/posixmodule.c | 505 ++++++++++++++++-- configure | 6 + configure.ac | 2 +- pyconfig.h.in | 3 + 17 files changed, 918 insertions(+), 64 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index b7fa365166d608..90cd5587f48318 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3383,6 +3383,137 @@ features: Added the :attr:`st_birthtime` member on Windows. +.. function:: statx(path, mask, *, dir_fd=None, follow_symlinks=True, sync=None) + + Get the status of a file or file descriptor by performing a :c:func:`!statx` + system call on the given path. *path* may be specified as either a string or + bytes -- directly or indirectly through the :class:`PathLike` interface -- + or as an open file descriptor. *mask* is a combination of the module-level + :const:`STATX_* ` constants specifying the information to + retrieve. Returns a :class:`statx_result` object whose + :attr:`~os.statx_result.stx_mask` attribute specifies the information + actually retrieved (which may differ from *mask*). + + The optional parameter *sync* controls the freshness of the returned + information. ``sync=True`` requests that the kernel return up-to-date + information, even when doing so is expensive (for example, requiring a + round trip to the server for a file on a network filesystem). + ``sync=False`` requests that the kernel return cached information if + available. ``sync=None`` expresses no preference, in which case the kernel + will return information as fresh as :func:`~os.stat` does. + + This function supports :ref:`specifying a file descriptor `, + :ref:`paths relative to directory descriptors `, and + :ref:`not following symlinks `. + + .. seealso:: The :manpage:`statx(2)` man page. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: next + + +.. class:: statx_result + + Object whose attributes correspond roughly to the members of the + :c:struct:`!statx` structure. It is used for the result of :func:`os.statx`. + :class:`!statx_result` has all of the attributes of :class:`stat_result` + available on Linux, but is not a subclass of :class:`stat_result` nor a + tuple. :class:`!statx_result` has the following additional attributes: + + .. attribute:: stx_mask + + Bitmask of :const:`STATX_* ` constants specifying the + information retrieved, which may differ from what was requested depending + on the filesystem, filesystem type, and kernel version. All attributes + of this class are accessible regardless of the value of + :attr:`!stx_mask`, and they may have useful fictitious values. For + example, for a file on a network filesystem, :const:`STATX_UID` and + :const:`STATX_GID` may be unset because file ownership on the server is + based on an external user database, but :attr:`!st_uid` and + :attr:`!st_gid` may contain the IDs of the local user who controls the + mount. + + .. attribute:: stx_attributes_mask + + Bitmask of :const:`!STATX_ATTR_* ` constants + specifying the attributes bits supported for this file. + + .. attribute:: stx_attributes + + Bitmask of :const:`!STATX_ATTR_* ` constants + specifying the attributes of this file. + + .. attribute:: stx_mnt_id + + Mount ID. + + .. attribute:: stx_dio_mem_align + + Direct I/O memory buffer alignment requirement. + + .. attribute:: stx_dio_offset_align + + Direct I/O file offset alignment requirement. + + .. attribute:: stx_subvol + + Subvolume ID. + + .. attribute:: stx_atomic_write_unit_min + + Minimum size for direct I/O with torn-write protection. + + .. attribute:: stx_atomic_write_unit_max + + Maximum size for direct I/O with torn-write protection. + + .. attribute:: stx_atomic_write_segments_max + + Maximum iovecs for direct I/O with torn-write protection. + + .. attribute:: stx_dio_read_offset_align + + Direct I/O file offset alignment requirement for reads. + + .. attribute:: stx_atomic_write_unit_max_opt + + Maximum optimized size for direct I/O with torn-write protection. + + .. seealso:: The :manpage:`statx(2)` man page. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: next + + +.. data:: STATX_TYPE + STATX_MODE + STATX_NLINK + STATX_UID + STATX_GID + STATX_ATIME + STATX_MTIME + STATX_CTIME + STATX_INO + STATX_SIZE + STATX_BLOCKS + STATX_BASIC_STATS + STATX_BTIME + STATX_MNT_ID + STATX_DIOALIGN + STATX_MNT_ID_UNIQUE + STATX_SUBVOL + STATX_WRITE_ATOMIC + STATX_DIO_READ_ALIGN + + Bitflags for use as the *mask* parameter to :func:`os.statx`. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: next + + .. function:: statvfs(path) Perform a :c:func:`!statvfs` system call on the given path. The return value is diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index 8434b2e8c75cf4..1cbec3ab847c5f 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -493,3 +493,22 @@ constants, but are not an exhaustive list. IO_REPARSE_TAG_APPEXECLINK .. versionadded:: 3.8 + +On Linux, the following file attribute constants are available for use when +testing bits in the :attr:`~os.statx_result.stx_attributes` and +:attr:`~os.statx_result.stx_attributes_mask` members returned by +:func:`os.statx`. See the :manpage:`statx(2)` man page for more detail on the +meaning of these constants. + +.. data:: STATX_ATTR_COMPRESSED + STATX_ATTR_IMMUTABLE + STATX_ATTR_APPEND + STATX_ATTR_NODUMP + STATX_ATTR_ENCRYPTED + STATX_ATTR_AUTOMOUNT + STATX_ATTR_MOUNT_ROOT + STATX_ATTR_VERITY + STATX_ATTR_DAX + STATX_ATTR_WRITE_ATOMIC + + .. versionadded:: next diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index f393537141c076..266f65c1e5dbe8 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1099,6 +1099,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(loop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(manual_reset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mapping)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mask)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(match)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_length)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxdigits)); @@ -1293,6 +1294,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sub_key)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(subcalls)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(symmetric_difference_update)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sync)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tabsize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tag)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(target)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index f4fde6142b9e82..14342fccc8c208 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -590,6 +590,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(loop) STRUCT_FOR_ID(manual_reset) STRUCT_FOR_ID(mapping) + STRUCT_FOR_ID(mask) STRUCT_FOR_ID(match) STRUCT_FOR_ID(max_length) STRUCT_FOR_ID(maxdigits) @@ -784,6 +785,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(sub_key) STRUCT_FOR_ID(subcalls) STRUCT_FOR_ID(symmetric_difference_update) + STRUCT_FOR_ID(sync) STRUCT_FOR_ID(tabsize) STRUCT_FOR_ID(tag) STRUCT_FOR_ID(target) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 5c0ec7dd547115..6636b22dc1eca9 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1097,6 +1097,7 @@ extern "C" { INIT_ID(loop), \ INIT_ID(manual_reset), \ INIT_ID(mapping), \ + INIT_ID(mask), \ INIT_ID(match), \ INIT_ID(max_length), \ INIT_ID(maxdigits), \ @@ -1291,6 +1292,7 @@ extern "C" { INIT_ID(sub_key), \ INIT_ID(subcalls), \ INIT_ID(symmetric_difference_update), \ + INIT_ID(sync), \ INIT_ID(tabsize), \ INIT_ID(tag), \ INIT_ID(target), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 1a7f1c13c6dd16..04d9a0448a3605 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -2148,6 +2148,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(mask); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(match); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2924,6 +2928,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(sync); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tabsize); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/os.py b/Lib/os.py index 710d6f8cfcdf74..09c9e47f419771 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -131,6 +131,8 @@ def _add(str, fn): _add("HAVE_UNLINKAT", "unlink") _add("HAVE_UNLINKAT", "rmdir") _add("HAVE_UTIMENSAT", "utime") + if _exists("statx"): + _set.add(statx) supports_dir_fd = _set _set = set() @@ -152,6 +154,8 @@ def _add(str, fn): _add("HAVE_FPATHCONF", "pathconf") if _exists("statvfs") and _exists("fstatvfs"): # mac os x10.3 _add("HAVE_FSTATVFS", "statvfs") + if _exists("statx"): + _set.add(statx) supports_fd = _set _set = set() @@ -190,6 +194,8 @@ def _add(str, fn): _add("HAVE_FSTATAT", "stat") _add("HAVE_UTIMENSAT", "utime") _add("MS_WINDOWS", "stat") + if _exists("statx"): + _set.add(statx) supports_follow_symlinks = _set del _set diff --git a/Lib/stat.py b/Lib/stat.py index 1b4ed1ebc940ef..ab1b25b9d6351c 100644 --- a/Lib/stat.py +++ b/Lib/stat.py @@ -200,6 +200,21 @@ def filemode(mode): FILE_ATTRIBUTE_VIRTUAL = 65536 +# Linux STATX_ATTR constants for interpreting os.statx()'s +# "stx_attributes" and "stx_attributes_mask" members + +STATX_ATTR_COMPRESSED = 0x00000004 +STATX_ATTR_IMMUTABLE = 0x00000010 +STATX_ATTR_APPEND = 0x00000020 +STATX_ATTR_NODUMP = 0x00000040 +STATX_ATTR_ENCRYPTED = 0x00000800 +STATX_ATTR_AUTOMOUNT = 0x00001000 +STATX_ATTR_MOUNT_ROOT = 0x00002000 +STATX_ATTR_VERITY = 0x00100000 +STATX_ATTR_DAX = 0x00200000 +STATX_ATTR_WRITE_ATOMIC = 0x00400000 + + # If available, use C implementation try: from _stat import * diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index cd15aa10f16de8..1e86d159350bd2 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -640,6 +640,14 @@ def setUp(self): self.addCleanup(os_helper.unlink, self.fname) create_file(self.fname, b"ABC") + def check_timestamp_agreement(self, result, names): + # Make sure that the st_?time and st_?time_ns fields roughly agree + # (they should always agree up to around tens-of-microseconds) + for name in names: + floaty = int(getattr(result, name) * 100000) + nanosecondy = getattr(result, name + "_ns") // 10000 + self.assertAlmostEqual(floaty, nanosecondy, delta=2, msg=name) + def check_stat_attributes(self, fname): result = os.stat(fname) @@ -660,21 +668,15 @@ def trunc(x): return x result[getattr(stat, name)]) self.assertIn(attr, members) - # Make sure that the st_?time and st_?time_ns fields roughly agree - # (they should always agree up to around tens-of-microseconds) - for name in 'st_atime st_mtime st_ctime'.split(): - floaty = int(getattr(result, name) * 100000) - nanosecondy = getattr(result, name + "_ns") // 10000 - self.assertAlmostEqual(floaty, nanosecondy, delta=2) - - # Ensure both birthtime and birthtime_ns roughly agree, if present + time_attributes = ['st_atime', 'st_mtime', 'st_ctime'] try: - floaty = int(result.st_birthtime * 100000) - nanosecondy = result.st_birthtime_ns // 10000 + result.st_birthtime + result.st_birthtime_ns except AttributeError: pass else: - self.assertAlmostEqual(floaty, nanosecondy, delta=2) + time_attributes.append('st_birthtime') + self.check_timestamp_agreement(result, time_attributes) try: result[200] @@ -735,6 +737,89 @@ def test_stat_result_pickle(self): unpickled = pickle.loads(p) self.assertEqual(result, unpickled) + def check_statx_attributes(self, fname): + maximal_mask = 0 + for name in dir(os): + if name.startswith('STATX_'): + maximal_mask |= getattr(os, name) + result = os.statx(self.fname, maximal_mask) + + time_attributes = ('st_atime', 'st_mtime', 'st_ctime', 'st_birthtime') + self.check_timestamp_agreement(result, time_attributes) + + # Check that valid attributes match os.stat. + requirements = ( + ('st_mode', os.STATX_TYPE | os.STATX_MODE), + ('st_nlink', os.STATX_NLINK), + ('st_uid', os.STATX_UID), + ('st_gid', os.STATX_GID), + ('st_atime', os.STATX_ATIME), + ('st_atime_ns', os.STATX_ATIME), + ('st_mtime', os.STATX_MTIME), + ('st_mtime_ns', os.STATX_MTIME), + ('st_ctime', os.STATX_CTIME), + ('st_ctime_ns', os.STATX_CTIME), + ('st_ino', os.STATX_INO), + ('st_size', os.STATX_SIZE), + ('st_blocks', os.STATX_BLOCKS), + ('st_birthtime', os.STATX_BTIME), + ('st_birthtime_ns', os.STATX_BTIME), + # unconditionally valid members + ('st_blksize', 0), + ('st_rdev', 0), + ('st_dev', 0), + ) + basic_result = os.stat(self.fname) + for name, bits in requirements: + if result.stx_mask & bits == bits and hasattr(basic_result, name): + x = getattr(result, name) + b = getattr(basic_result, name) + if isinstance(x, float): + self.assertAlmostEqual(x, b, msg=name) + else: + self.assertEqual(x, b, msg=name) + + self.assertEqual(result.stx_rdev_major, os.major(result.st_rdev)) + self.assertEqual(result.stx_rdev_minor, os.minor(result.st_rdev)) + self.assertEqual(result.stx_dev_major, os.major(result.st_dev)) + self.assertEqual(result.stx_dev_minor, os.minor(result.st_dev)) + + members = [name for name in dir(result) + if name.startswith('st_') or name.startswith('stx_')] + for name in members: + try: + setattr(result, name, 1) + self.fail("No exception raised") + except AttributeError: + pass + + self.assertEqual(result.stx_attributes & result.stx_attributes_mask, + result.stx_attributes) + + @unittest.skipUnless(hasattr(os, 'statx'), 'test needs os.statx()') + def test_statx_attributes(self): + self.check_statx_attributes(self.fname) + + @unittest.skipUnless(hasattr(os, 'statx'), 'test needs os.statx()') + def test_statx_attributes_bytes(self): + try: + fname = self.fname.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + self.skipTest("cannot encode %a for the filesystem" % self.fname) + self.check_statx_attributes(fname) + + @unittest.skipUnless(hasattr(os, 'statx'), 'test needs os.statx()') + def test_statx_attributes_pathlike(self): + self.check_statx_attributes(FakePath(self.fname)) + + @unittest.skipUnless(hasattr(os, 'statx'), 'test needs os.statx()') + def test_statx_sync(self): + # Test sync= kwarg parsing. (We can't predict if or how the result + # will change.) + for sync in (False, True): + with self.subTest(sync=sync): + os.statx(self.fname, os.STATX_BASIC_STATS, sync=sync) + @unittest.skipUnless(hasattr(os, 'statvfs'), 'test needs os.statvfs()') def test_statvfs_attributes(self): result = os.statvfs(self.fname) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index ab3d128d08ab47..9b6320f8dcad78 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1629,33 +1629,41 @@ def test_chown_dir_fd(self): with self.prepare_file() as (dir_fd, name, fullname): posix.chown(name, os.getuid(), os.getgid(), dir_fd=dir_fd) - @unittest.skipUnless(os.stat in os.supports_dir_fd, "test needs dir_fd support in os.stat()") - def test_stat_dir_fd(self): + def check_statlike_dir_fd(self, func): with self.prepare() as (dir_fd, name, fullname): with open(fullname, 'w') as outfile: outfile.write("testline\n") self.addCleanup(posix.unlink, fullname) - s1 = posix.stat(fullname) - s2 = posix.stat(name, dir_fd=dir_fd) - self.assertEqual(s1, s2) - s2 = posix.stat(fullname, dir_fd=None) - self.assertEqual(s1, s2) + s1 = func(fullname) + s2 = func(name, dir_fd=dir_fd) + self.assertEqual((s1.st_dev, s1.st_ino), (s2.st_dev, s2.st_ino)) + s2 = func(fullname, dir_fd=None) + self.assertEqual((s1.st_dev, s1.st_ino), (s2.st_dev, s2.st_ino)) self.assertRaisesRegex(TypeError, 'should be integer or None, not', - posix.stat, name, dir_fd=posix.getcwd()) + func, name, dir_fd=posix.getcwd()) self.assertRaisesRegex(TypeError, 'should be integer or None, not', - posix.stat, name, dir_fd=float(dir_fd)) + func, name, dir_fd=float(dir_fd)) self.assertRaises(OverflowError, - posix.stat, name, dir_fd=10**20) + func, name, dir_fd=10**20) for fd in False, True: with self.assertWarnsRegex(RuntimeWarning, 'bool is used as a file descriptor') as cm: with self.assertRaises(OSError): - posix.stat('nonexisting', dir_fd=fd) + func('nonexisting', dir_fd=fd) self.assertEqual(cm.filename, __file__) + @unittest.skipUnless(os.stat in os.supports_dir_fd, "test needs dir_fd support in os.stat()") + def test_stat_dir_fd(self): + self.check_statlike_dir_fd(posix.stat) + + @unittest.skipUnless(hasattr(posix, 'statx'), "test needs os.statx()") + def test_statx_dir_fd(self): + func = lambda path, **kwargs: posix.statx(path, os.STATX_INO, **kwargs) + self.check_statlike_dir_fd(func) + @unittest.skipUnless(os.utime in os.supports_dir_fd, "test needs dir_fd support in os.utime()") def test_utime_dir_fd(self): with self.prepare_file() as (dir_fd, name, fullname): diff --git a/Misc/ACKS b/Misc/ACKS index c54a27bbc8eb0b..a747c16deee5a8 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -210,6 +210,7 @@ Médéric Boquien Matias Bordese Jonas Borgström Jurjen Bos +Jeffrey Bosboom Peter Bosch Dan Boswell Eric Bouck diff --git a/Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst b/Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst new file mode 100644 index 00000000000000..7229a361147ee2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst @@ -0,0 +1,2 @@ +Implement :func:`os.statx` on Linux kernel versions 4.11 and later with +glibc versions 2.28 and later. Contributed by Jeffrey Bosboom. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index dddf98d127c15f..f9291d9136328c 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -186,6 +186,141 @@ os_lstat(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw return return_value; } +#if defined(HAVE_STATX) + +PyDoc_STRVAR(os_statx__doc__, +"statx($module, /, path, mask, *, dir_fd=None, follow_symlinks=True,\n" +" sync=None)\n" +"--\n" +"\n" +"Perform a statx system call on the given path.\n" +"\n" +" path\n" +" Path to be examined; can be string, bytes, a path-like object or\n" +" open-file-descriptor int.\n" +" mask\n" +" A bitmask of STATX_* constants defining the requested information.\n" +" dir_fd\n" +" If not None, it should be a file descriptor open to a directory,\n" +" and path should be a relative string; path will then be relative to\n" +" that directory.\n" +" follow_symlinks\n" +" If False, and the last element of the path is a symbolic link,\n" +" statx will examine the symbolic link itself instead of the file\n" +" the link points to.\n" +" sync\n" +" If True, statx will return up-to-date values, even if doing so is\n" +" expensive. If False, statx will return cached values if possible.\n" +" If None, statx lets the operating system decide.\n" +"\n" +"It\'s an error to use dir_fd or follow_symlinks when specifying path as\n" +" an open file descriptor."); + +#define OS_STATX_METHODDEF \ + {"statx", _PyCFunction_CAST(os_statx), METH_FASTCALL|METH_KEYWORDS, os_statx__doc__}, + +static PyObject * +os_statx_impl(PyObject *module, path_t *path, unsigned int mask, int dir_fd, + int follow_symlinks, int sync); + +static PyObject * +os_statx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 5 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(path), &_Py_ID(mask), &_Py_ID(dir_fd), &_Py_ID(follow_symlinks), &_Py_ID(sync), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", "mask", "dir_fd", "follow_symlinks", "sync", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "statx", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[5]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + path_t path = PATH_T_INITIALIZE_P("statx", "path", 0, 0, 0, 1); + unsigned int mask; + int dir_fd = DEFAULT_DIR_FD; + int follow_symlinks = 1; + int sync = -1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!path_converter(args[0], &path)) { + goto exit; + } + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &mask, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[2]) { + if (!dir_fd_converter(args[2], &dir_fd)) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (args[3]) { + follow_symlinks = PyObject_IsTrue(args[3]); + if (follow_symlinks < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (!optional_bool_converter(args[4], &sync)) { + goto exit; + } +skip_optional_kwonly: + return_value = os_statx_impl(module, &path, mask, dir_fd, follow_symlinks, sync); + +exit: + /* Cleanup for path */ + path_cleanup(&path); + + return return_value; +} + +#endif /* defined(HAVE_STATX) */ + PyDoc_STRVAR(os_access__doc__, "access($module, /, path, mode, *, dir_fd=None, effective_ids=False,\n" " follow_symlinks=True)\n" @@ -12771,6 +12906,10 @@ os__emscripten_log(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #endif /* defined(__EMSCRIPTEN__) */ +#ifndef OS_STATX_METHODDEF + #define OS_STATX_METHODDEF +#endif /* !defined(OS_STATX_METHODDEF) */ + #ifndef OS_TTYNAME_METHODDEF #define OS_TTYNAME_METHODDEF #endif /* !defined(OS_TTYNAME_METHODDEF) */ @@ -13446,4 +13585,4 @@ os__emscripten_log(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #ifndef OS__EMSCRIPTEN_LOG_METHODDEF #define OS__EMSCRIPTEN_LOG_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ -/*[clinic end generated code: output=b5b370c499174f85 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1572887ef174fb7b input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6da90dc95addce..802b992cf3d0bf 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -40,6 +40,7 @@ // --- System includes ------------------------------------------------------ +#include // offsetof() #include // ctermid() #include // system() @@ -408,6 +409,33 @@ extern char *ctermid_r(char *); # define STRUCT_STAT struct stat #endif +#ifdef HAVE_STATX +# pragma weak statx +/* provide constants introduced later than statx itself */ +# ifndef STATX_MNT_ID +# define STATX_MNT_ID 0x00001000U +# endif +# ifndef STATX_DIOALIGN +# define STATX_DIOALIGN 0x00002000U +# endif +# ifndef STATX_MNT_ID_UNIQUE +# define STATX_MNT_ID_UNIQUE 0x00004000U +# endif +# ifndef STATX_SUBVOL +# define STATX_SUBVOL 0x00008000U +# endif +# ifndef STATX_WRITE_ATOMIC +# define STATX_WRITE_ATOMIC 0x00010000U +# endif +# ifndef STATX_DIO_READ_ALIGN +# define STATX_DIO_READ_ALIGN 0x00020000U +# endif +# define _Py_STATX_KNOWN (STATX_BASIC_STATS | STATX_BTIME | STATX_MNT_ID | \ + STATX_DIOALIGN | STATX_MNT_ID_UNIQUE | \ + STATX_SUBVOL | STATX_WRITE_ATOMIC | \ + STATX_DIO_READ_ALIGN) +#endif /* HAVE_STATX */ + #if !defined(EX_OK) && defined(EXIT_SUCCESS) # define EX_OK EXIT_SUCCESS @@ -1159,6 +1187,9 @@ typedef struct { #endif newfunc statresult_new_orig; PyObject *StatResultType; +#ifdef HAVE_STATX + PyObject *StatxResultType; +#endif PyObject *StatVFSResultType; PyObject *TerminalSizeType; PyObject *TimesResultType; @@ -2539,6 +2570,9 @@ _posix_clear(PyObject *module) Py_CLEAR(state->SchedParamType); #endif Py_CLEAR(state->StatResultType); +#ifdef HAVE_STATX + Py_CLEAR(state->StatxResultType); +#endif Py_CLEAR(state->StatVFSResultType); Py_CLEAR(state->TerminalSizeType); Py_CLEAR(state->TimesResultType); @@ -2564,6 +2598,9 @@ _posix_traverse(PyObject *module, visitproc visit, void *arg) Py_VISIT(state->SchedParamType); #endif Py_VISIT(state->StatResultType); +#ifdef HAVE_STATX + Py_VISIT(state->StatxResultType); +#endif Py_VISIT(state->StatVFSResultType); Py_VISIT(state->TerminalSizeType); Py_VISIT(state->TimesResultType); @@ -2584,12 +2621,46 @@ _posix_free(void *module) _posix_clear((PyObject *)module); } + +#define SEC_TO_NS (1000000000LL) +static PyObject * +nanosecond_timestamp(_posixstate *state, time_t sec, unsigned long nsec) { + /* 1677-09-21 00:12:44 to 2262-04-11 23:47:15 UTC inclusive */ + if ((LLONG_MIN/SEC_TO_NS) <= sec && sec <= (LLONG_MAX/SEC_TO_NS - 1)) { + return PyLong_FromLongLong(sec * SEC_TO_NS + nsec); + } + else { + PyObject *s_in_ns = NULL; + PyObject *s = _PyLong_FromTime_t(sec); + PyObject *ns_fractional = PyLong_FromUnsignedLong(nsec); + if (s == NULL || ns_fractional == NULL) { + goto exit; + } + + s_in_ns = PyNumber_Multiply(s, state->billion); + if (s_in_ns == NULL) { + goto exit; + } + + PyObject *ns_total = PyNumber_Add(s_in_ns, ns_fractional); + if (ns_total == NULL) { + goto exit; + } + return ns_total; + + exit: + Py_XDECREF(s); + Py_XDECREF(ns_fractional); + Py_XDECREF(s_in_ns); + return NULL; + } +} + static int fill_time(_posixstate *state, PyObject *v, int s_index, int f_index, int ns_index, time_t sec, unsigned long nsec) { assert(!PyErr_Occurred()); -#define SEC_TO_NS (1000000000LL) assert(nsec < SEC_TO_NS); if (s_index >= 0) { @@ -2608,50 +2679,18 @@ fill_time(_posixstate *state, PyObject *v, int s_index, int f_index, PyStructSequence_SET_ITEM(v, f_index, float_s); } - int res = -1; if (ns_index >= 0) { - /* 1677-09-21 00:12:44 to 2262-04-11 23:47:15 UTC inclusive */ - if ((LLONG_MIN/SEC_TO_NS) <= sec && sec <= (LLONG_MAX/SEC_TO_NS - 1)) { - PyObject *ns_total = PyLong_FromLongLong(sec * SEC_TO_NS + nsec); - if (ns_total == NULL) { - return -1; - } - PyStructSequence_SET_ITEM(v, ns_index, ns_total); - assert(!PyErr_Occurred()); - res = 0; - } - else { - PyObject *s_in_ns = NULL; - PyObject *ns_total = NULL; - PyObject *s = _PyLong_FromTime_t(sec); - PyObject *ns_fractional = PyLong_FromUnsignedLong(nsec); - if (s == NULL || ns_fractional == NULL) { - goto exit; - } - - s_in_ns = PyNumber_Multiply(s, state->billion); - if (s_in_ns == NULL) { - goto exit; - } - - ns_total = PyNumber_Add(s_in_ns, ns_fractional); - if (ns_total == NULL) { - goto exit; - } - PyStructSequence_SET_ITEM(v, ns_index, ns_total); - assert(!PyErr_Occurred()); - res = 0; - - exit: - Py_XDECREF(s); - Py_XDECREF(ns_fractional); - Py_XDECREF(s_in_ns); + PyObject *ns_total = nanosecond_timestamp(state, sec, nsec); + if (ns_total == NULL) { + return -1; } + PyStructSequence_SET_ITEM(v, ns_index, ns_total); } - return res; - #undef SEC_TO_NS + assert(!PyErr_Occurred()); + return 0; } +#undef SEC_TO_NS #ifdef MS_WINDOWS static PyObject* @@ -3277,6 +3316,339 @@ os_lstat_impl(PyObject *module, path_t *path, int dir_fd) } +#ifdef HAVE_STATX +typedef struct { + PyObject_HEAD + struct statx stx; + double atime_sec, btime_sec, ctime_sec, mtime_sec; + dev_t rdev, dev; +} statx_result; + +#define M(attr, type, offset, doc) \ + {attr, type, offset, Py_READONLY, PyDoc_STR(doc)} +#define MO(attr, type, offset, doc) \ + M(#attr, type, offsetof(statx_result, stx) + offset, doc) +#define MM(attr, type, member, doc) \ + M(#attr, type, offsetof(statx_result, stx.stx_##member), doc) +#define MX(attr, type, member, doc) \ + M(#attr, type, offsetof(statx_result, member), doc) + +static PyMemberDef statx_result_members[] = { + MM(stx_mask, Py_T_UINT, mask, "member validity mask"), + MM(st_blksize, Py_T_UINT, blksize, "blocksize for filesystem I/O"), + MM(stx_attributes, Py_T_ULONGLONG, attributes, "Linux inode attribute bits"), + MM(st_nlink, Py_T_UINT, nlink, "number of hard links"), + MM(st_uid, Py_T_UINT, uid, "user ID of owner"), + MM(st_gid, Py_T_UINT, gid, "group ID of owner"), + MM(st_mode, Py_T_USHORT, mode, "protection bits"), + MM(st_ino, Py_T_ULONGLONG, ino, "inode"), + MM(st_size, Py_T_ULONGLONG, size, "total size, in bytes"), + MM(st_blocks, Py_T_ULONGLONG, blocks, "number of blocks allocated"), + MM(stx_attributes_mask, Py_T_ULONGLONG, attributes_mask, + "Linux inode attribute bits supported for this file"), + MX(st_atime, Py_T_DOUBLE, atime_sec, "time of last access"), + MX(st_birthtime, Py_T_DOUBLE, btime_sec, "time of creation"), + MX(st_ctime, Py_T_DOUBLE, ctime_sec, "time of last change"), + MX(st_mtime, Py_T_DOUBLE, mtime_sec, "time of last modification"), + MM(stx_rdev_major, Py_T_UINT, rdev_major, "represented device major number"), + MM(stx_rdev_minor, Py_T_UINT, rdev_minor, "represented device minor number"), + MX(st_rdev, Py_T_ULONGLONG, rdev, "device type (if inode device)"), + MM(stx_dev_major, Py_T_UINT, dev_major, "containing device major number"), + MM(stx_dev_minor, Py_T_UINT, dev_minor, "containing device minor number"), + MX(st_dev, Py_T_ULONGLONG, dev, "device"), + /* We may be building against old kernel API headers that do not have the + names of these members, so access them by offset. The reserved space in + struct statx was originally defined as arrays of u64, so later members + of other types must use getters to avoid a strict aliasing violation. */ + MO(stx_mnt_id, Py_T_ULONGLONG, 144, "mount ID"), + MO(stx_subvol, Py_T_ULONGLONG, 160, "subvolume ID"), + {NULL}, +}; + +#undef MX +#undef MM +#undef MO +#undef M + +#define DECLARE_GET(name, type, func) \ + static PyObject * \ + statx_result_get_##name(PyObject *op, void *context) { \ + statx_result *self = (statx_result *) op; \ + uint16_t offset = (uintptr_t)context; \ + type val; \ + memcpy(&val, (void *)self + offset, sizeof(val)); \ + return func(val); \ + } +DECLARE_GET(u32, uint32_t, PyLong_FromUInt32) +#undef DECLARE_GET + +static PyObject * +statx_result_get_nsec(PyObject *op, void *context) { + statx_result *self = (statx_result *) op; + uint16_t offset = (uintptr_t)context; + struct statx_timestamp val; + memcpy(&val, (void *)self + offset, sizeof(val)); + _posixstate *state = PyType_GetModuleState(Py_TYPE(op)); + assert(state != NULL); + return nanosecond_timestamp(state, val.tv_sec, val.tv_nsec); +} + +/* The low 16 bits of the context pointer are the offset from the start of + statx_result to the struct statx member. */ +#define OFFSET_CONTEXT(offset) (void *)(offsetof(statx_result, stx) + offset) +#define MEMBER_CONTEXT(name) OFFSET_CONTEXT(offsetof(struct statx, stx_##name)) + +#define G(attr, type, doc, context) \ + {attr, statx_result_get_##type, NULL, PyDoc_STR(doc), context} +#define GM(attr, type, member, doc) \ + G(#attr, type, doc, MEMBER_CONTEXT(member)) +#define GO(attr, type, offset, doc) \ + G(#attr, type, doc, OFFSET_CONTEXT(offset)) + +static PyGetSetDef statx_result_getset[] = { + GM(st_atime_ns, nsec, atime, "time of last access in nanoseconds"), + GM(st_birthtime_ns, nsec, btime, "time of creation in nanoseconds"), + GM(st_ctime_ns, nsec, ctime, "time of last change in nanoseconds"), + GM(st_mtime_ns, nsec, mtime, "time of last modification in nanoseconds"), + GO(stx_dio_mem_align, u32, 152, "direct I/O memory buffer alignment"), + GO(stx_dio_offset_align, u32, 156, "direct I/O file offset alignment"), + GO(stx_atomic_write_unit_min, u32, 168, + "minimum size for direct I/O with torn-write protection"), + GO(stx_atomic_write_unit_max, u32, 172, + "maximum size for direct I/O with torn-write protection"), + GO(stx_atomic_write_segments_max, u32, 176, + "maximum iovecs for direct I/O with torn-write protection"), + GO(stx_dio_read_offset_align, u32, 180, + "direct I/O file offset alignment for reads"), + GO(stx_atomic_write_unit_max_opt, u32, 184, + "maximum optimized size for direct I/O with torn-write protection"), + {NULL}, +}; + +#undef GO +#undef GOC +#undef GM +#undef GMC +#undef G +#undef MEMBER_CONTEXT +#undef MEMBER_CACHE_CONTEXT +#undef OFFSET_CONTEXT +#undef OFFSET_CACHE_CONTEXT + +static PyObject * +statx_result_repr(PyObject *op) { + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + return NULL; + } +#define WRITE_ASCII(s, n) \ + do { \ + if (PyUnicodeWriter_WriteASCII(writer, s, n) < 0) { \ + goto error; \ + } \ + } while (0) + + WRITE_ASCII("os.statx_result(", -1); + + for (size_t i = 0; i < Py_ARRAY_LENGTH(statx_result_members) - 1; ++i) { + if (i > 0) { + WRITE_ASCII(", ", 2); + } + + PyMemberDef *d = &statx_result_members[i]; + WRITE_ASCII(d->name, -1); + WRITE_ASCII("=", 1); + + PyObject *o = PyMember_GetOne((const char *)op, d); + if (o == NULL) { + goto error; + } + if (PyUnicodeWriter_WriteRepr(writer, o) < 0) { + goto error; + } + Py_DECREF(o); + } + + if (Py_ARRAY_LENGTH(statx_result_members) > 1 + && Py_ARRAY_LENGTH(statx_result_getset) > 1) { + WRITE_ASCII(", ", 2); + } + + for (size_t i = 0; i < Py_ARRAY_LENGTH(statx_result_getset) - 1; ++i) { + if (i > 0) { + WRITE_ASCII(", ", 2); + } + + PyGetSetDef *d = &statx_result_getset[i]; + WRITE_ASCII(d->name, -1); + WRITE_ASCII("=", 1); + + PyObject *o = d->get(op, d->closure); + if (o == NULL) { + goto error; + } + if (PyUnicodeWriter_WriteRepr(writer, o) < 0) { + goto error; + } + Py_DECREF(o); + } + + WRITE_ASCII(")", 1); + return PyUnicodeWriter_Finish(writer); +#undef WRITE_ASCII + +error: + PyUnicodeWriter_Discard(writer); + return NULL; +} + +static int +statx_result_traverse(PyObject *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); + return 0; +} + +static void +statx_result_dealloc(PyObject *op) { + statx_result *self = (statx_result *) op; + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyType_Slot statx_result_slots[] = { + {Py_tp_repr, statx_result_repr}, + {Py_tp_traverse, statx_result_traverse}, + {Py_tp_dealloc, statx_result_dealloc}, + {Py_tp_members, statx_result_members}, + {Py_tp_getset, statx_result_getset}, + {0, NULL}, +}; + +static PyType_Spec statx_result_spec = { + .name = "statx_result", + .basicsize = sizeof(statx_result), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .slots = statx_result_slots, +}; + +static int +optional_bool_converter(PyObject *arg, void *addr) { + int value; + if (arg == Py_None) { + value = -1; + } + else { + value = Py_IsTrue(arg); + if (value < 0) { + return 0; + } + } + *((int *)addr) = value; + return 1; +} + +/*[python input] +class optional_bool_converter(CConverter): + type = 'int' + converter = 'optional_bool_converter' +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=47de85b300eeb19e]*/ + +/*[clinic input] + +os.statx + + path : path_t(allow_fd=True) + Path to be examined; can be string, bytes, a path-like object or + open-file-descriptor int. + + mask: unsigned_int(bitwise=True) + A bitmask of STATX_* constants defining the requested information. + + * + + dir_fd : dir_fd = None + If not None, it should be a file descriptor open to a directory, + and path should be a relative string; path will then be relative to + that directory. + + follow_symlinks: bool = True + If False, and the last element of the path is a symbolic link, + statx will examine the symbolic link itself instead of the file + the link points to. + + sync: optional_bool(c_default='-1') = None + If True, statx will return up-to-date values, even if doing so is + expensive. If False, statx will return cached values if possible. + If None, statx lets the operating system decide. + +Perform a statx system call on the given path. + +It's an error to use dir_fd or follow_symlinks when specifying path as + an open file descriptor. + +[clinic start generated code]*/ + +static PyObject * +os_statx_impl(PyObject *module, path_t *path, unsigned int mask, int dir_fd, + int follow_symlinks, int sync) +/*[clinic end generated code: output=fe385235585f3d07 input=148c4fce440ca53a]*/ +{ + if (path_and_dir_fd_invalid("statx", path, dir_fd) || + dir_fd_and_fd_invalid("statx", dir_fd, path->fd) || + fd_and_follow_symlinks_invalid("statx", path->fd, follow_symlinks)) + return NULL; + + /* Future bits may refer to members beyond the current size of struct + statx, so we need to mask them off to prevent memory corruption. */ + mask &= _Py_STATX_KNOWN; + int flags = AT_NO_AUTOMOUNT | (follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); + if (sync != -1) { + flags |= sync ? AT_STATX_FORCE_SYNC : AT_STATX_DONT_SYNC; + } + + _posixstate *state = get_posix_state(module); + PyTypeObject *tp = (PyTypeObject *)state->StatxResultType; + statx_result *v = (statx_result *)tp->tp_alloc(tp, 0); + if (v == NULL) { + return NULL; + } + + int result; + Py_BEGIN_ALLOW_THREADS + if (path->fd != -1) { + result = statx(path->fd, "", flags | AT_EMPTY_PATH, mask, &v->stx); + } + else { + result = statx(dir_fd, path->narrow, flags, mask, &v->stx); + } + Py_END_ALLOW_THREADS + + if (result != 0) { + Py_DECREF(v); + return path_error(path); + } + + v->atime_sec = ((double)v->stx.stx_atime.tv_sec + + 1e-9 * v->stx.stx_atime.tv_nsec); + v->btime_sec = ((double)v->stx.stx_btime.tv_sec + + 1e-9 * v->stx.stx_btime.tv_nsec); + v->ctime_sec = ((double)v->stx.stx_ctime.tv_sec + + 1e-9 * v->stx.stx_ctime.tv_nsec); + v->mtime_sec = ((double)v->stx.stx_mtime.tv_sec + + 1e-9 * v->stx.stx_mtime.tv_nsec); + v->rdev = makedev(v->stx.stx_rdev_major, v->stx.stx_rdev_minor); + v->dev = makedev(v->stx.stx_dev_major, v->stx.stx_dev_minor); + + assert(!PyErr_Occurred()); + return (PyObject *)v; +} +#endif /* HAVE_STATX */ + + /*[clinic input] os.access -> bool @@ -17025,6 +17397,7 @@ os__emscripten_log_impl(PyObject *module, const char *arg) static PyMethodDef posix_methods[] = { OS_STAT_METHODDEF + OS_STATX_METHODDEF OS_ACCESS_METHODDEF OS_TTYNAME_METHODDEF OS_CHDIR_METHODDEF @@ -17870,6 +18243,30 @@ all_ins(PyObject *m) if (PyModule_Add(m, "NODEV", _PyLong_FromDev(NODEV))) return -1; #endif +#ifdef HAVE_STATX + if (PyModule_AddIntMacro(m, STATX_TYPE)) return -1; + if (PyModule_AddIntMacro(m, STATX_MODE)) return -1; + if (PyModule_AddIntMacro(m, STATX_NLINK)) return -1; + if (PyModule_AddIntMacro(m, STATX_UID)) return -1; + if (PyModule_AddIntMacro(m, STATX_GID)) return -1; + if (PyModule_AddIntMacro(m, STATX_ATIME)) return -1; + if (PyModule_AddIntMacro(m, STATX_MTIME)) return -1; + if (PyModule_AddIntMacro(m, STATX_CTIME)) return -1; + if (PyModule_AddIntMacro(m, STATX_INO)) return -1; + if (PyModule_AddIntMacro(m, STATX_SIZE)) return -1; + if (PyModule_AddIntMacro(m, STATX_BLOCKS)) return -1; + if (PyModule_AddIntMacro(m, STATX_BASIC_STATS)) return -1; + if (PyModule_AddIntMacro(m, STATX_BTIME)) return -1; + if (PyModule_AddIntMacro(m, STATX_MNT_ID)) return -1; + if (PyModule_AddIntMacro(m, STATX_DIOALIGN)) return -1; + if (PyModule_AddIntMacro(m, STATX_MNT_ID_UNIQUE)) return -1; + if (PyModule_AddIntMacro(m, STATX_SUBVOL)) return -1; + if (PyModule_AddIntMacro(m, STATX_WRITE_ATOMIC)) return -1; + if (PyModule_AddIntMacro(m, STATX_DIO_READ_ALIGN)) return -1; + /* STATX_ALL intentionally omitted because it is deprecated */ + /* STATX_ATTR_* constants are in the stat module */ +#endif /* HAVE_STATX */ + #if defined(__APPLE__) if (PyModule_AddIntConstant(m, "_COPYFILE_DATA", COPYFILE_DATA)) return -1; if (PyModule_AddIntConstant(m, "_COPYFILE_STAT", COPYFILE_STAT)) return -1; @@ -18141,6 +18538,34 @@ posixmodule_exec(PyObject *m) } #endif +#ifdef HAVE_STATX + /* We retract os.statx if: + - the weakly-linked statx wrapper function is not available (old libc) + - the wrapper function fails with ENOSYS (libc built without fallback + running on an old kernel) + - the wrapper function fails with EINVAL on sync flags (glibc's + emulation of statx via stat fails in this way) */ + struct statx stx; + if (statx == NULL + || (statx(-1, "/", AT_STATX_DONT_SYNC, 0, &stx) == -1 + && (errno == ENOSYS || errno == EINVAL))) { + PyObject* dct = PyModule_GetDict(m); + if (dct == NULL) { + return -1; + } + if (PyDict_PopString(dct, "statx", NULL) < 0) { + return -1; + } + } + else { + statx_result_spec.name = "os.statx_result"; + state->StatxResultType = PyType_FromModuleAndSpec(m, &statx_result_spec, NULL); + if (PyModule_AddObjectRef(m, "statx_result", state->StatxResultType) < 0) { + return -1; + } + } +#endif + /* Initialize environ dictionary */ if (PyModule_Add(m, "environ", convertenviron()) != 0) { return -1; diff --git a/configure b/configure index ed6befdbced108..16136423e88755 100755 --- a/configure +++ b/configure @@ -20186,6 +20186,12 @@ if test "x$ac_cv_func_splice" = xyes then : printf "%s\n" "#define HAVE_SPLICE 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "statx" "ac_cv_func_statx" +if test "x$ac_cv_func_statx" = xyes +then : + printf "%s\n" "#define HAVE_STATX 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "strftime" "ac_cv_func_strftime" if test "x$ac_cv_func_strftime" = xyes diff --git a/configure.ac b/configure.ac index 5d4c5c43187953..a39d2170fe82e4 100644 --- a/configure.ac +++ b/configure.ac @@ -5252,7 +5252,7 @@ AC_CHECK_FUNCS([ \ setitimer setlocale setpgid setpgrp setpriority setregid setresgid \ setresuid setreuid setsid setuid setvbuf shutdown sigaction sigaltstack \ sigfillset siginterrupt sigpending sigrelse sigtimedwait sigwait \ - sigwaitinfo snprintf splice strftime strlcpy strsignal symlinkat sync \ + sigwaitinfo snprintf splice statx strftime strlcpy strsignal symlinkat sync \ sysconf tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ tmpnam tmpnam_r truncate ttyname_r umask uname unlinkat unlockpt utimensat utimes vfork \ wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \ diff --git a/pyconfig.h.in b/pyconfig.h.in index 60bff4a9f26356..f8793c5c529683 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1282,6 +1282,9 @@ /* Define to 1 if you have the 'statvfs' function. */ #undef HAVE_STATVFS +/* Define to 1 if you have the 'statx' function. */ +#undef HAVE_STATX + /* Define if you have struct stat.st_mtim.tv_nsec */ #undef HAVE_STAT_TV_NSEC From a1110f39ab7f6716ae744031faa1c890dbeeb45d Mon Sep 17 00:00:00 2001 From: Jeffrey Bosboom Date: Mon, 22 Sep 2025 21:10:37 -0700 Subject: [PATCH 2/3] Apply suggestions from code review Co-authored-by: Victor Stinner --- Doc/library/os.rst | 8 ++- Lib/test/test_os.py | 4 +- Lib/test/test_posix.py | 3 +- Modules/posixmodule.c | 122 ++++++++++++++++++++--------------------- 4 files changed, 69 insertions(+), 68 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 90cd5587f48318..72c5f7050e5fc6 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3386,9 +3386,11 @@ features: .. function:: statx(path, mask, *, dir_fd=None, follow_symlinks=True, sync=None) Get the status of a file or file descriptor by performing a :c:func:`!statx` - system call on the given path. *path* may be specified as either a string or - bytes -- directly or indirectly through the :class:`PathLike` interface -- - or as an open file descriptor. *mask* is a combination of the module-level + system call on the given path. + + *path* may be specified as either a string or bytes -- directly or + indirectly through the :class:`PathLike` interface -- or as an open file + descriptor. *mask* is a combination of the module-level :const:`STATX_* ` constants specifying the information to retrieve. Returns a :class:`statx_result` object whose :attr:`~os.statx_result.stx_mask` attribute specifies the information diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 1e86d159350bd2..fecbd9f81973ad 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -644,8 +644,8 @@ def check_timestamp_agreement(self, result, names): # Make sure that the st_?time and st_?time_ns fields roughly agree # (they should always agree up to around tens-of-microseconds) for name in names: - floaty = int(getattr(result, name) * 100000) - nanosecondy = getattr(result, name + "_ns") // 10000 + floaty = int(getattr(result, name) * 100_000) + nanosecondy = getattr(result, name + "_ns") // 10_000 self.assertAlmostEqual(floaty, nanosecondy, delta=2, msg=name) def check_stat_attributes(self, fname): diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 9b6320f8dcad78..dd8c8fa7616c80 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1661,7 +1661,8 @@ def test_stat_dir_fd(self): @unittest.skipUnless(hasattr(posix, 'statx'), "test needs os.statx()") def test_statx_dir_fd(self): - func = lambda path, **kwargs: posix.statx(path, os.STATX_INO, **kwargs) + def func(path, **kwargs): + return posix.statx(path, os.STATX_INO, **kwargs) self.check_statlike_dir_fd(func) @unittest.skipUnless(os.utime in os.supports_dir_fd, "test needs dir_fd support in os.utime()") diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 802b992cf3d0bf..a8b3f640daf5d3 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -410,6 +410,7 @@ extern char *ctermid_r(char *); #endif #ifdef HAVE_STATX +/* until we can assume glibc 2.28 at runtime, we must weakly link */ # pragma weak statx /* provide constants introduced later than statx itself */ # ifndef STATX_MNT_ID @@ -2624,7 +2625,8 @@ _posix_free(void *module) #define SEC_TO_NS (1000000000LL) static PyObject * -nanosecond_timestamp(_posixstate *state, time_t sec, unsigned long nsec) { +stat_nanosecond_timestamp(_posixstate *state, time_t sec, unsigned long nsec) +{ /* 1677-09-21 00:12:44 to 2262-04-11 23:47:15 UTC inclusive */ if ((LLONG_MIN/SEC_TO_NS) <= sec && sec <= (LLONG_MAX/SEC_TO_NS - 1)) { return PyLong_FromLongLong(sec * SEC_TO_NS + nsec); @@ -2680,7 +2682,7 @@ fill_time(_posixstate *state, PyObject *v, int s_index, int f_index, } if (ns_index >= 0) { - PyObject *ns_total = nanosecond_timestamp(state, sec, nsec); + PyObject *ns_total = stat_nanosecond_timestamp(state, sec, nsec); if (ns_total == NULL) { return -1; } @@ -3322,18 +3324,18 @@ typedef struct { struct statx stx; double atime_sec, btime_sec, ctime_sec, mtime_sec; dev_t rdev, dev; -} statx_result; +} Py_statx_result; #define M(attr, type, offset, doc) \ {attr, type, offset, Py_READONLY, PyDoc_STR(doc)} #define MO(attr, type, offset, doc) \ - M(#attr, type, offsetof(statx_result, stx) + offset, doc) + M(#attr, type, offsetof(Py_statx_result, stx) + offset, doc) #define MM(attr, type, member, doc) \ - M(#attr, type, offsetof(statx_result, stx.stx_##member), doc) + M(#attr, type, offsetof(Py_statx_result, stx.stx_##member), doc) #define MX(attr, type, member, doc) \ - M(#attr, type, offsetof(statx_result, member), doc) + M(#attr, type, offsetof(Py_statx_result, member), doc) -static PyMemberDef statx_result_members[] = { +static PyMemberDef pystatx_result_members[] = { MM(stx_mask, Py_T_UINT, mask, "member validity mask"), MM(st_blksize, Py_T_UINT, blksize, "blocksize for filesystem I/O"), MM(stx_attributes, Py_T_ULONGLONG, attributes, "Linux inode attribute bits"), @@ -3370,42 +3372,40 @@ static PyMemberDef statx_result_members[] = { #undef MO #undef M -#define DECLARE_GET(name, type, func) \ - static PyObject * \ - statx_result_get_##name(PyObject *op, void *context) { \ - statx_result *self = (statx_result *) op; \ - uint16_t offset = (uintptr_t)context; \ - type val; \ - memcpy(&val, (void *)self + offset, sizeof(val)); \ - return func(val); \ - } -DECLARE_GET(u32, uint32_t, PyLong_FromUInt32) -#undef DECLARE_GET +static PyObject * +pystatx_result_get_u32(PyObject *op, void *context) { + Py_statx_result *self = (Py_statx_result *) op; + uint16_t offset = (uintptr_t)context; + uint32_t val; + memcpy(&val, (void *)self + offset, sizeof(val)); + return PyLong_FromUInt32(val); +} static PyObject * -statx_result_get_nsec(PyObject *op, void *context) { - statx_result *self = (statx_result *) op; +pystatx_result_get_nsec(PyObject *op, void *context) +{ + Py_statx_result *self = (Py_statx_result *) op; uint16_t offset = (uintptr_t)context; struct statx_timestamp val; memcpy(&val, (void *)self + offset, sizeof(val)); _posixstate *state = PyType_GetModuleState(Py_TYPE(op)); assert(state != NULL); - return nanosecond_timestamp(state, val.tv_sec, val.tv_nsec); + return stat_nanosecond_timestamp(state, val.tv_sec, val.tv_nsec); } /* The low 16 bits of the context pointer are the offset from the start of - statx_result to the struct statx member. */ -#define OFFSET_CONTEXT(offset) (void *)(offsetof(statx_result, stx) + offset) + Py_statx_result to the struct statx member. */ +#define OFFSET_CONTEXT(offset) (void *)(offsetof(Py_statx_result, stx) + offset) #define MEMBER_CONTEXT(name) OFFSET_CONTEXT(offsetof(struct statx, stx_##name)) #define G(attr, type, doc, context) \ - {attr, statx_result_get_##type, NULL, PyDoc_STR(doc), context} + {attr, pystatx_result_get_##type, NULL, PyDoc_STR(doc), context} #define GM(attr, type, member, doc) \ G(#attr, type, doc, MEMBER_CONTEXT(member)) #define GO(attr, type, offset, doc) \ G(#attr, type, doc, OFFSET_CONTEXT(offset)) -static PyGetSetDef statx_result_getset[] = { +static PyGetSetDef pystatx_result_getset[] = { GM(st_atime_ns, nsec, atime, "time of last access in nanoseconds"), GM(st_birthtime_ns, nsec, btime, "time of creation in nanoseconds"), GM(st_ctime_ns, nsec, ctime, "time of last change in nanoseconds"), @@ -3426,74 +3426,72 @@ static PyGetSetDef statx_result_getset[] = { }; #undef GO -#undef GOC #undef GM -#undef GMC #undef G #undef MEMBER_CONTEXT -#undef MEMBER_CACHE_CONTEXT #undef OFFSET_CONTEXT -#undef OFFSET_CACHE_CONTEXT static PyObject * -statx_result_repr(PyObject *op) { +pystatx_result_repr(PyObject *op) { PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); if (writer == NULL) { return NULL; } -#define WRITE_ASCII(s, n) \ +#define WRITE_ASCII(s) \ do { \ - if (PyUnicodeWriter_WriteASCII(writer, s, n) < 0) { \ + if (PyUnicodeWriter_WriteASCII(writer, s, strlen(s)) < 0) { \ goto error; \ } \ } while (0) - WRITE_ASCII("os.statx_result(", -1); + WRITE_ASCII("os.statx_result("); - for (size_t i = 0; i < Py_ARRAY_LENGTH(statx_result_members) - 1; ++i) { + for (size_t i = 0; i < Py_ARRAY_LENGTH(pystatx_result_members) - 1; ++i) { if (i > 0) { - WRITE_ASCII(", ", 2); + WRITE_ASCII(", "); } - PyMemberDef *d = &statx_result_members[i]; - WRITE_ASCII(d->name, -1); - WRITE_ASCII("=", 1); + PyMemberDef *d = &pystatx_result_members[i]; + WRITE_ASCII(d->name); + WRITE_ASCII("="); PyObject *o = PyMember_GetOne((const char *)op, d); if (o == NULL) { goto error; } if (PyUnicodeWriter_WriteRepr(writer, o) < 0) { + Py_DECREF(o); goto error; } Py_DECREF(o); } - if (Py_ARRAY_LENGTH(statx_result_members) > 1 - && Py_ARRAY_LENGTH(statx_result_getset) > 1) { - WRITE_ASCII(", ", 2); + if (Py_ARRAY_LENGTH(pystatx_result_members) > 1 + && Py_ARRAY_LENGTH(pystatx_result_getset) > 1) { + WRITE_ASCII(", "); } - for (size_t i = 0; i < Py_ARRAY_LENGTH(statx_result_getset) - 1; ++i) { + for (size_t i = 0; i < Py_ARRAY_LENGTH(pystatx_result_getset) - 1; ++i) { if (i > 0) { - WRITE_ASCII(", ", 2); + WRITE_ASCII(", "); } - PyGetSetDef *d = &statx_result_getset[i]; - WRITE_ASCII(d->name, -1); - WRITE_ASCII("=", 1); + PyGetSetDef *d = &pystatx_result_getset[i]; + WRITE_ASCII(d->name); + WRITE_ASCII("="); PyObject *o = d->get(op, d->closure); if (o == NULL) { goto error; } if (PyUnicodeWriter_WriteRepr(writer, o) < 0) { + Py_DECREF(o); goto error; } Py_DECREF(o); } - WRITE_ASCII(")", 1); + WRITE_ASCII(")"); return PyUnicodeWriter_Finish(writer); #undef WRITE_ASCII @@ -3503,35 +3501,35 @@ statx_result_repr(PyObject *op) { } static int -statx_result_traverse(PyObject *self, visitproc visit, void *arg) { +pystatx_result_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); return 0; } static void -statx_result_dealloc(PyObject *op) { - statx_result *self = (statx_result *) op; +pystatx_result_dealloc(PyObject *op) { + Py_statx_result *self = (Py_statx_result *) op; PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); tp->tp_free(self); Py_DECREF(tp); } -static PyType_Slot statx_result_slots[] = { - {Py_tp_repr, statx_result_repr}, - {Py_tp_traverse, statx_result_traverse}, - {Py_tp_dealloc, statx_result_dealloc}, - {Py_tp_members, statx_result_members}, - {Py_tp_getset, statx_result_getset}, +static PyType_Slot pystatx_result_slots[] = { + {Py_tp_repr, pystatx_result_repr}, + {Py_tp_traverse, pystatx_result_traverse}, + {Py_tp_dealloc, pystatx_result_dealloc}, + {Py_tp_members, pystatx_result_members}, + {Py_tp_getset, pystatx_result_getset}, {0, NULL}, }; -static PyType_Spec statx_result_spec = { +static PyType_Spec pystatx_result_spec = { .name = "statx_result", - .basicsize = sizeof(statx_result), + .basicsize = sizeof(Py_statx_result), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION, - .slots = statx_result_slots, + .slots = pystatx_result_slots, }; static int @@ -3612,7 +3610,7 @@ os_statx_impl(PyObject *module, path_t *path, unsigned int mask, int dir_fd, _posixstate *state = get_posix_state(module); PyTypeObject *tp = (PyTypeObject *)state->StatxResultType; - statx_result *v = (statx_result *)tp->tp_alloc(tp, 0); + Py_statx_result *v = (Py_statx_result *)tp->tp_alloc(tp, 0); if (v == NULL) { return NULL; } @@ -18558,8 +18556,8 @@ posixmodule_exec(PyObject *m) } } else { - statx_result_spec.name = "os.statx_result"; - state->StatxResultType = PyType_FromModuleAndSpec(m, &statx_result_spec, NULL); + pystatx_result_spec.name = "os.statx_result"; + state->StatxResultType = PyType_FromModuleAndSpec(m, &pystatx_result_spec, NULL); if (PyModule_AddObjectRef(m, "statx_result", state->StatxResultType) < 0) { return -1; } From 51ef6cc044582c5a2af7cdb2a926224e506e784d Mon Sep 17 00:00:00 2001 From: Jeffrey Bosboom Date: Tue, 23 Sep 2025 21:51:06 -0700 Subject: [PATCH 3/3] Apply suggestions from code review Co-authored-by: Victor Stinner --- Doc/library/os.rst | 57 ++++++++++++--- .../pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - .../internal/pycore_runtime_init_generated.h | 1 - .../internal/pycore_unicodeobject_generated.h | 4 -- Lib/test/test_os.py | 8 --- Lib/test/test_posix.py | 57 +++++++++++++-- Modules/clinic/posixmodule.c.h | 43 +++++------ Modules/posixmodule.c | 72 +++++++------------ 9 files changed, 147 insertions(+), 97 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 72c5f7050e5fc6..2c01bdca21baf9 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3383,7 +3383,7 @@ features: Added the :attr:`st_birthtime` member on Windows. -.. function:: statx(path, mask, *, dir_fd=None, follow_symlinks=True, sync=None) +.. function:: statx(path, mask, flags=0, *, dir_fd=None, follow_symlinks=True) Get the status of a file or file descriptor by performing a :c:func:`!statx` system call on the given path. @@ -3392,18 +3392,12 @@ features: indirectly through the :class:`PathLike` interface -- or as an open file descriptor. *mask* is a combination of the module-level :const:`STATX_* ` constants specifying the information to - retrieve. Returns a :class:`statx_result` object whose + retrieve. *flags* is a combination of the module-level + :const:`AT_STATX_* ` constants and/or + :const:`AT_NO_AUTOMOUNT`. Returns a :class:`statx_result` object whose :attr:`~os.statx_result.stx_mask` attribute specifies the information actually retrieved (which may differ from *mask*). - The optional parameter *sync* controls the freshness of the returned - information. ``sync=True`` requests that the kernel return up-to-date - information, even when doing so is expensive (for example, requiring a - round trip to the server for a file on a network filesystem). - ``sync=False`` requests that the kernel return cached information if - available. ``sync=None`` expresses no preference, in which case the kernel - will return information as fresh as :func:`~os.stat` does. - This function supports :ref:`specifying a file descriptor `, :ref:`paths relative to directory descriptors `, and :ref:`not following symlinks `. @@ -3515,6 +3509,49 @@ features: .. versionadded:: next +.. data:: AT_STATX_FORCE_SYNC + + A flag for the :func:`os.statx` function. Requests that the kernel return + up-to-date information even when doing so is expensive (for example, + requiring a round trip to the server for a file on a network filesystem). + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: next + +.. data:: AT_STATX_DONT_SYNC + + A flag for the :func:`os.statx` function. Requests that the kernel return + cached information if possible. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: next + +.. data:: AT_STATX_SYNC_AS_STAT + + A flag for the :func:`os.statx` function. This flag is defined as ``0``, so + it has no effect, but it can be used to explicitly indicate neither + :data:`AT_STATX_FORCE_SYNC` nor :data:`AT_STATX_DONT_SYNC` is being passed. + In the absence of the other two flags, the kernel will generally return + information as fresh as :func:`os.stat` would return. + + .. availability:: Linux >= 4.11 with glibc >= 2.28. + + .. versionadded:: next + + +.. data:: AT_NO_AUTOMOUNT + + If the final component of a path is an automount point, operate on the + automount point instead of performing the automount. (On Linux, + :func:`os.stat`, :func:`os.fstat` and :func:`os.lstat` always behave this + way.) + + .. availability:: Linux. + + .. versionadded:: next + .. function:: statvfs(path) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 266f65c1e5dbe8..f1fdd79ce6b04a 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1294,7 +1294,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sub_key)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(subcalls)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(symmetric_difference_update)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sync)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tabsize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tag)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(target)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 14342fccc8c208..5b17c3479dbc36 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -785,7 +785,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(sub_key) STRUCT_FOR_ID(subcalls) STRUCT_FOR_ID(symmetric_difference_update) - STRUCT_FOR_ID(sync) STRUCT_FOR_ID(tabsize) STRUCT_FOR_ID(tag) STRUCT_FOR_ID(target) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 6636b22dc1eca9..512b5588329380 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1292,7 +1292,6 @@ extern "C" { INIT_ID(sub_key), \ INIT_ID(subcalls), \ INIT_ID(symmetric_difference_update), \ - INIT_ID(sync), \ INIT_ID(tabsize), \ INIT_ID(tag), \ INIT_ID(target), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 04d9a0448a3605..9e98764f816a60 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -2928,10 +2928,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(sync); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(tabsize); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index fecbd9f81973ad..cb915d7fe31630 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -812,14 +812,6 @@ def test_statx_attributes_bytes(self): def test_statx_attributes_pathlike(self): self.check_statx_attributes(FakePath(self.fname)) - @unittest.skipUnless(hasattr(os, 'statx'), 'test needs os.statx()') - def test_statx_sync(self): - # Test sync= kwarg parsing. (We can't predict if or how the result - # will change.) - for sync in (False, True): - with self.subTest(sync=sync): - os.statx(self.fname, os.STATX_BASIC_STATS, sync=sync) - @unittest.skipUnless(hasattr(os, 'statvfs'), 'test needs os.statvfs()') def test_statvfs_attributes(self): result = os.statvfs(self.fname) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index dd8c8fa7616c80..905f0201253951 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -668,22 +668,65 @@ def test_fstat(self): finally: fp.close() - def test_stat(self): - self.assertTrue(posix.stat(os_helper.TESTFN)) - self.assertTrue(posix.stat(os.fsencode(os_helper.TESTFN))) + def check_statlike_path(self, func): + self.assertTrue(func(os_helper.TESTFN)) + self.assertTrue(func(os.fsencode(os_helper.TESTFN))) + self.assertTrue(func(os_helper.FakePath(os_helper.TESTFN))) self.assertRaisesRegex(TypeError, 'should be string, bytes, os.PathLike or integer, not', - posix.stat, bytearray(os.fsencode(os_helper.TESTFN))) + func, bytearray(os.fsencode(os_helper.TESTFN))) self.assertRaisesRegex(TypeError, 'should be string, bytes, os.PathLike or integer, not', - posix.stat, None) + func, None) self.assertRaisesRegex(TypeError, 'should be string, bytes, os.PathLike or integer, not', - posix.stat, list(os_helper.TESTFN)) + func, list(os_helper.TESTFN)) self.assertRaisesRegex(TypeError, 'should be string, bytes, os.PathLike or integer, not', - posix.stat, list(os.fsencode(os_helper.TESTFN))) + func, list(os.fsencode(os_helper.TESTFN))) + + def test_stat(self): + self.check_statlike_path(posix.stat) + + @unittest.skipUnless(hasattr(posix, 'statx'), 'test needs posix.statx()') + def test_statx(self): + def func(path, **kwargs): + return posix.statx(path, posix.STATX_BASIC_STATS, **kwargs) + self.check_statlike_path(func) + + @unittest.skipUnless(hasattr(posix, 'statx'), 'test needs posix.statx()') + def test_statx_flags(self): + # glibc's fallback implementation of statx via the stat family fails + # with EINVAL on the (nonzero) sync flags. If you see this failure, + # update your kernel and/or seccomp syscall filter. + valid_flag_names = ('AT_NO_AUTOMOUNT', 'AT_STATX_SYNC_AS_STAT', + 'AT_STATX_FORCE_SYNC', 'AT_STATX_DONT_SYNC') + for flag_name in valid_flag_names: + flag = getattr(posix, flag_name) + with self.subTest(msg=flag_name, flags=flag): + posix.statx(os_helper.TESTFN, posix.STATX_BASIC_STATS, + flags=flag) + + # These flags are not exposed to Python because their functionality is + # implemented via kwargs instead. + kwarg_equivalent_flags = ( + (0x0100, 'AT_SYMLINK_NOFOLLOW', 'follow_symlinks'), + (0x0400, 'AT_SYMLINK_FOLLOW', 'follow_symlinks'), + (0x1000, 'AT_EMPTY_PATH', 'dir_fd'), + ) + for flag, flag_name, kwarg_name in kwarg_equivalent_flags: + with self.subTest(msg=flag_name, flags=flag): + with self.assertRaisesRegex(ValueError, kwarg_name): + posix.statx(os_helper.TESTFN, posix.STATX_BASIC_STATS, + flags=flag) + + with self.subTest(msg="AT_STATX_FORCE_SYNC | AT_STATX_DONT_SYNC"): + with self.assertRaises(OSError) as ctx: + flags = posix.AT_STATX_FORCE_SYNC | posix.AT_STATX_DONT_SYNC + posix.statx(os_helper.TESTFN, posix.STATX_BASIC_STATS, + flags=flags) + self.assertEqual(ctx.exception.errno, errno.EINVAL) @unittest.skipUnless(hasattr(posix, 'mkfifo'), "don't have mkfifo()") def test_mkfifo(self): diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index f9291d9136328c..0dc500ec4b3fbc 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -189,8 +189,8 @@ os_lstat(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #if defined(HAVE_STATX) PyDoc_STRVAR(os_statx__doc__, -"statx($module, /, path, mask, *, dir_fd=None, follow_symlinks=True,\n" -" sync=None)\n" +"statx($module, /, path, mask, flags=0, *, dir_fd=None,\n" +" follow_symlinks=True)\n" "--\n" "\n" "Perform a statx system call on the given path.\n" @@ -200,6 +200,8 @@ PyDoc_STRVAR(os_statx__doc__, " open-file-descriptor int.\n" " mask\n" " A bitmask of STATX_* constants defining the requested information.\n" +" flags\n" +" A bitmask of AT_NO_AUTOMOUNT and/or AT_STATX_* flags.\n" " dir_fd\n" " If not None, it should be a file descriptor open to a directory,\n" " and path should be a relative string; path will then be relative to\n" @@ -208,10 +210,6 @@ PyDoc_STRVAR(os_statx__doc__, " If False, and the last element of the path is a symbolic link,\n" " statx will examine the symbolic link itself instead of the file\n" " the link points to.\n" -" sync\n" -" If True, statx will return up-to-date values, even if doing so is\n" -" expensive. If False, statx will return cached values if possible.\n" -" If None, statx lets the operating system decide.\n" "\n" "It\'s an error to use dir_fd or follow_symlinks when specifying path as\n" " an open file descriptor."); @@ -220,8 +218,8 @@ PyDoc_STRVAR(os_statx__doc__, {"statx", _PyCFunction_CAST(os_statx), METH_FASTCALL|METH_KEYWORDS, os_statx__doc__}, static PyObject * -os_statx_impl(PyObject *module, path_t *path, unsigned int mask, int dir_fd, - int follow_symlinks, int sync); +os_statx_impl(PyObject *module, path_t *path, unsigned int mask, int flags, + int dir_fd, int follow_symlinks); static PyObject * os_statx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -238,7 +236,7 @@ os_statx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(path), &_Py_ID(mask), &_Py_ID(dir_fd), &_Py_ID(follow_symlinks), &_Py_ID(sync), }, + .ob_item = { &_Py_ID(path), &_Py_ID(mask), &_Py_ID(flags), &_Py_ID(dir_fd), &_Py_ID(follow_symlinks), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -247,7 +245,7 @@ os_statx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"path", "mask", "dir_fd", "follow_symlinks", "sync", NULL}; + static const char * const _keywords[] = {"path", "mask", "flags", "dir_fd", "follow_symlinks", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "statx", @@ -258,12 +256,12 @@ os_statx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; path_t path = PATH_T_INITIALIZE_P("statx", "path", 0, 0, 0, 1); unsigned int mask; + int flags = 0; int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; - int sync = -1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } @@ -287,30 +285,35 @@ os_statx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw } } if (!noptargs) { - goto skip_optional_kwonly; + goto skip_optional_pos; } if (args[2]) { - if (!dir_fd_converter(args[2], &dir_fd)) { + flags = PyLong_AsInt(args[2]); + if (flags == -1 && PyErr_Occurred()) { goto exit; } if (!--noptargs) { - goto skip_optional_kwonly; + goto skip_optional_pos; } } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } if (args[3]) { - follow_symlinks = PyObject_IsTrue(args[3]); - if (follow_symlinks < 0) { + if (!dir_fd_converter(args[3], &dir_fd)) { goto exit; } if (!--noptargs) { goto skip_optional_kwonly; } } - if (!optional_bool_converter(args[4], &sync)) { + follow_symlinks = PyObject_IsTrue(args[4]); + if (follow_symlinks < 0) { goto exit; } skip_optional_kwonly: - return_value = os_statx_impl(module, &path, mask, dir_fd, follow_symlinks, sync); + return_value = os_statx_impl(module, &path, mask, flags, dir_fd, follow_symlinks); exit: /* Cleanup for path */ @@ -13585,4 +13588,4 @@ os__emscripten_log(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #ifndef OS__EMSCRIPTEN_LOG_METHODDEF #define OS__EMSCRIPTEN_LOG_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ -/*[clinic end generated code: output=1572887ef174fb7b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d5a13014cfc9a617 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a8b3f640daf5d3..f608bc83ac2c15 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3532,29 +3532,6 @@ static PyType_Spec pystatx_result_spec = { .slots = pystatx_result_slots, }; -static int -optional_bool_converter(PyObject *arg, void *addr) { - int value; - if (arg == Py_None) { - value = -1; - } - else { - value = Py_IsTrue(arg); - if (value < 0) { - return 0; - } - } - *((int *)addr) = value; - return 1; -} - -/*[python input] -class optional_bool_converter(CConverter): - type = 'int' - converter = 'optional_bool_converter' -[python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=47de85b300eeb19e]*/ - /*[clinic input] os.statx @@ -3566,6 +3543,9 @@ os.statx mask: unsigned_int(bitwise=True) A bitmask of STATX_* constants defining the requested information. + flags: int = 0 + A bitmask of AT_NO_AUTOMOUNT and/or AT_STATX_* flags. + * dir_fd : dir_fd = None @@ -3578,11 +3558,6 @@ os.statx statx will examine the symbolic link itself instead of the file the link points to. - sync: optional_bool(c_default='-1') = None - If True, statx will return up-to-date values, even if doing so is - expensive. If False, statx will return cached values if possible. - If None, statx lets the operating system decide. - Perform a statx system call on the given path. It's an error to use dir_fd or follow_symlinks when specifying path as @@ -3591,22 +3566,31 @@ It's an error to use dir_fd or follow_symlinks when specifying path as [clinic start generated code]*/ static PyObject * -os_statx_impl(PyObject *module, path_t *path, unsigned int mask, int dir_fd, - int follow_symlinks, int sync) -/*[clinic end generated code: output=fe385235585f3d07 input=148c4fce440ca53a]*/ +os_statx_impl(PyObject *module, path_t *path, unsigned int mask, int flags, + int dir_fd, int follow_symlinks) +/*[clinic end generated code: output=e3765979ac6fe15b input=7ebd6e0f93476670]*/ { if (path_and_dir_fd_invalid("statx", path, dir_fd) || dir_fd_and_fd_invalid("statx", dir_fd, path->fd) || fd_and_follow_symlinks_invalid("statx", path->fd, follow_symlinks)) return NULL; + /* reject flags covered by kwargs, but allow unknown flags that may be + future AT_STATX_* extensions */ + if (flags & (AT_SYMLINK_NOFOLLOW | AT_SYMLINK_FOLLOW)) { + PyErr_Format(PyExc_ValueError, + "use follow_symlinks kwarg instead of AT_SYMLINK_* flag"); + return NULL; + } + if (flags & AT_EMPTY_PATH) { + PyErr_Format(PyExc_ValueError, + "use dir_fd kwarg instead of AT_EMPTY_PATH flag"); + return NULL; + } + /* Future bits may refer to members beyond the current size of struct statx, so we need to mask them off to prevent memory corruption. */ mask &= _Py_STATX_KNOWN; - int flags = AT_NO_AUTOMOUNT | (follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); - if (sync != -1) { - flags |= sync ? AT_STATX_FORCE_SYNC : AT_STATX_DONT_SYNC; - } _posixstate *state = get_posix_state(module); PyTypeObject *tp = (PyTypeObject *)state->StatxResultType; @@ -18241,6 +18225,10 @@ all_ins(PyObject *m) if (PyModule_Add(m, "NODEV", _PyLong_FromDev(NODEV))) return -1; #endif +#ifdef AT_NO_AUTOMOUNT + if (PyModule_AddIntMacro(m, AT_NO_AUTOMOUNT)) return -1; +#endif + #ifdef HAVE_STATX if (PyModule_AddIntMacro(m, STATX_TYPE)) return -1; if (PyModule_AddIntMacro(m, STATX_MODE)) return -1; @@ -18262,6 +18250,9 @@ all_ins(PyObject *m) if (PyModule_AddIntMacro(m, STATX_WRITE_ATOMIC)) return -1; if (PyModule_AddIntMacro(m, STATX_DIO_READ_ALIGN)) return -1; /* STATX_ALL intentionally omitted because it is deprecated */ + if (PyModule_AddIntMacro(m, AT_STATX_SYNC_AS_STAT)) return -1; + if (PyModule_AddIntMacro(m, AT_STATX_FORCE_SYNC)) return -1; + if (PyModule_AddIntMacro(m, AT_STATX_DONT_SYNC)) return -1; /* STATX_ATTR_* constants are in the stat module */ #endif /* HAVE_STATX */ @@ -18537,16 +18528,7 @@ posixmodule_exec(PyObject *m) #endif #ifdef HAVE_STATX - /* We retract os.statx if: - - the weakly-linked statx wrapper function is not available (old libc) - - the wrapper function fails with ENOSYS (libc built without fallback - running on an old kernel) - - the wrapper function fails with EINVAL on sync flags (glibc's - emulation of statx via stat fails in this way) */ - struct statx stx; - if (statx == NULL - || (statx(-1, "/", AT_STATX_DONT_SYNC, 0, &stx) == -1 - && (errno == ENOSYS || errno == EINVAL))) { + if (statx == NULL) { PyObject* dct = PyModule_GetDict(m); if (dct == NULL) { return -1;