diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index a82caf86e8018a..c7a13abad888d9 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -203,6 +203,20 @@ To map anonymous memory, -1 should be passed as the fileno along with the length exception was raised on error under Unix. + .. method:: madvise(option[, start[, length]]) + + Send advice *option* to the kernel about the memory region beginning at + *start* and extending *length* bytes. *option* must be one of the + :ref:`MADV_* constants ` available on the system. If + *start* and *length* are omitted, the entire mapping is spanned. On + some systems (including Linux), *start* must be a multiple of the + :const:`PAGESIZE`. + + Availability: Systems with the ``madvise()`` system call. + + .. versionadded:: 3.8 + + .. method:: move(dest, src, count) Copy the *count* bytes starting at offset *src* to the destination index @@ -292,3 +306,38 @@ To map anonymous memory, -1 should be passed as the fileno along with the length position of the file pointer; the file position is advanced by ``1``. If the mmap was created with :const:`ACCESS_READ`, then writing to it will raise a :exc:`TypeError` exception. + +.. _madvise-constants: + +MADV_* Constants +++++++++++++++++ + +.. data:: MADV_NORMAL + MADV_RANDOM + MADV_SEQUENTIAL + MADV_WILLNEED + MADV_DONTNEED + MADV_REMOVE + MADV_DONTFORK + MADV_DOFORK + MADV_HWPOISON + MADV_MERGEABLE + MADV_UNMERGEABLE + MADV_SOFT_OFFLINE + MADV_HUGEPAGE + MADV_NOHUGEPAGE + MADV_DONTDUMP + MADV_DODUMP + MADV_FREE + MADV_NOSYNC + MADV_AUTOSYNC + MADV_NOCORE + MADV_CORE + MADV_PROTECT + + These options can be passed to :meth:`mmap.madvise`. Not every option will + be present on every system. + + Availability: Systems with the madvise() system call. + + .. versionadded:: 3.8 diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 6102d8c357cab0..fd5e6496ba2ed5 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -460,6 +460,15 @@ numbers. (Contributed by Pablo Galindo in :issue:`35606`) Added new function :func:`math.isqrt` for computing integer square roots. (Contributed by Mark Dickinson in :issue:`36887`.) + +mmap +---- + +The :class:`mmap.mmap` class now has an :meth:`~mmap.mmap.madvise` method to +access the ``madvise()`` system call. +(Contributed by Zackery Spytz in :issue:`32941`.) + + os -- diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 495d24ad807730..7b2b100dce0155 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -739,6 +739,25 @@ def test_flush_return_value(self): # See bpo-34754 for details. self.assertRaises(OSError, mm.flush, 1, len(b'python')) + @unittest.skipUnless(hasattr(mmap.mmap, 'madvise'), 'needs madvise') + def test_madvise(self): + size = 8192 + m = mmap.mmap(-1, size) + + with self.assertRaisesRegex(ValueError, "madvise start out of bounds"): + m.madvise(mmap.MADV_NORMAL, size) + with self.assertRaisesRegex(ValueError, "madvise start out of bounds"): + m.madvise(mmap.MADV_NORMAL, -1) + with self.assertRaisesRegex(ValueError, "madvise length invalid"): + m.madvise(mmap.MADV_NORMAL, 0, -1) + with self.assertRaisesRegex(OverflowError, "madvise length too large"): + m.madvise(mmap.MADV_NORMAL, PAGESIZE, sys.maxsize) + self.assertEqual(m.madvise(mmap.MADV_NORMAL), None) + self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE), None) + self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE, size), None) + self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, 2), None) + self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, size), None) + class LargeMmapTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2018-03-20-20-57-00.bpo-32941.9FU0gL.rst b/Misc/NEWS.d/next/Library/2018-03-20-20-57-00.bpo-32941.9FU0gL.rst new file mode 100644 index 00000000000000..f7668aecda6b50 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-20-20-57-00.bpo-32941.9FU0gL.rst @@ -0,0 +1,2 @@ +Allow :class:`mmap.mmap` objects to access the madvise() system call +(through :meth:`mmap.mmap.madvise`). diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index fdd60bbb6eefca..36cbaf9fb8b23d 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -708,11 +708,54 @@ mmap__sizeof__method(mmap_object *self, void *unused) } #endif +#ifdef HAVE_MADVISE +static PyObject * +mmap_madvise_method(mmap_object *self, PyObject *args) +{ + int option; + Py_ssize_t start = 0, length; + + CHECK_VALID(NULL); + length = self->size; + + if (!PyArg_ParseTuple(args, "i|nn:madvise", &option, &start, &length)) { + return NULL; + } + + if (start < 0 || start >= self->size) { + PyErr_SetString(PyExc_ValueError, "madvise start out of bounds"); + return NULL; + } + if (length < 0) { + PyErr_SetString(PyExc_ValueError, "madvise length invalid"); + return NULL; + } + if (PY_SSIZE_T_MAX - start < length) { + PyErr_SetString(PyExc_OverflowError, "madvise length too large"); + return NULL; + } + + if (start + length > self->size) { + length = self->size - start; + } + + if (madvise(self->data + start, length, option) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + Py_RETURN_NONE; +} +#endif // HAVE_MADVISE + static struct PyMethodDef mmap_object_methods[] = { {"close", (PyCFunction) mmap_close_method, METH_NOARGS}, {"find", (PyCFunction) mmap_find_method, METH_VARARGS}, {"rfind", (PyCFunction) mmap_rfind_method, METH_VARARGS}, {"flush", (PyCFunction) mmap_flush_method, METH_VARARGS}, +#ifdef HAVE_MADVISE + {"madvise", (PyCFunction) mmap_madvise_method, METH_VARARGS}, +#endif {"move", (PyCFunction) mmap_move_method, METH_VARARGS}, {"read", (PyCFunction) mmap_read_method, METH_VARARGS}, {"read_byte", (PyCFunction) mmap_read_byte_method, METH_NOARGS}, @@ -1494,5 +1537,80 @@ PyInit_mmap(void) setint(dict, "ACCESS_READ", ACCESS_READ); setint(dict, "ACCESS_WRITE", ACCESS_WRITE); setint(dict, "ACCESS_COPY", ACCESS_COPY); + +#ifdef HAVE_MADVISE + // Conventional advice values +#ifdef MADV_NORMAL + setint(dict, "MADV_NORMAL", MADV_NORMAL); +#endif +#ifdef MADV_RANDOM + setint(dict, "MADV_RANDOM", MADV_RANDOM); +#endif +#ifdef MADV_SEQUENTIAL + setint(dict, "MADV_SEQUENTIAL", MADV_SEQUENTIAL); +#endif +#ifdef MADV_WILLNEED + setint(dict, "MADV_WILLNEED", MADV_WILLNEED); +#endif +#ifdef MADV_DONTNEED + setint(dict, "MADV_DONTNEED", MADV_DONTNEED); +#endif + + // Linux-specific advice values +#ifdef MADV_REMOVE + setint(dict, "MADV_REMOVE", MADV_REMOVE); +#endif +#ifdef MADV_DONTFORK + setint(dict, "MADV_DONTFORK", MADV_DONTFORK); +#endif +#ifdef MADV_DOFORK + setint(dict, "MADV_DOFORK", MADV_DOFORK); +#endif +#ifdef MADV_HWPOISON + setint(dict, "MADV_HWPOISON", MADV_HWPOISON); +#endif +#ifdef MADV_MERGEABLE + setint(dict, "MADV_MERGEABLE", MADV_MERGEABLE); +#endif +#ifdef MADV_UNMERGEABLE + setint(dict, "MADV_UNMERGEABLE", MADV_UNMERGEABLE); +#endif +#ifdef MADV_SOFT_OFFLINE + setint(dict, "MADV_SOFT_OFFLINE", MADV_SOFT_OFFLINE); +#endif +#ifdef MADV_HUGEPAGE + setint(dict, "MADV_HUGEPAGE", MADV_HUGEPAGE); +#endif +#ifdef MADV_NOHUGEPAGE + setint(dict, "MADV_NOHUGEPAGE", MADV_NOHUGEPAGE); +#endif +#ifdef MADV_DONTDUMP + setint(dict, "MADV_DONTDUMP", MADV_DONTDUMP); +#endif +#ifdef MADV_DODUMP + setint(dict, "MADV_DODUMP", MADV_DODUMP); +#endif +#ifdef MADV_FREE // (Also present on FreeBSD and macOS.) + setint(dict, "MADV_FREE", MADV_FREE); +#endif + + // FreeBSD-specific +#ifdef MADV_NOSYNC + setint(dict, "MADV_NOSYNC", MADV_NOSYNC); +#endif +#ifdef MADV_AUTOSYNC + setint(dict, "MADV_AUTOSYNC", MADV_AUTOSYNC); +#endif +#ifdef MADV_NOCORE + setint(dict, "MADV_NOCORE", MADV_NOCORE); +#endif +#ifdef MADV_CORE + setint(dict, "MADV_CORE", MADV_CORE); +#endif +#ifdef MADV_PROTECT + setint(dict, "MADV_PROTECT", MADV_PROTECT); +#endif +#endif // HAVE_MADVISE + return module; } diff --git a/configure b/configure index bc276ac58362bf..e8cbfeb154a543 100755 --- a/configure +++ b/configure @@ -11471,7 +11471,7 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \ getpriority getresuid getresgid getpwent getpwnam_r getpwuid_r getspnam getspent getsid getwd \ if_nameindex \ - initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \ + initgroups kill killpg lchmod lchown lockf linkat lstat lutimes madvise mmap \ memrchr mbrtowc mkdirat mkfifo \ mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ posix_fallocate posix_fadvise posix_spawn posix_spawnp pread preadv preadv2 \ diff --git a/configure.ac b/configure.ac index 5e565191f27de1..c743edfdeb183a 100644 --- a/configure.ac +++ b/configure.ac @@ -3527,7 +3527,7 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \ getpriority getresuid getresgid getpwent getpwnam_r getpwuid_r getspnam getspent getsid getwd \ if_nameindex \ - mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ + madvise mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ posix_fallocate posix_fadvise posix_spawn posix_spawnp pread preadv preadv2 \ pthread_condattr_setclock pthread_init pthread_kill putenv pwrite pwritev pwritev2 \ readlink readlinkat readv realpath renameat \ diff --git a/pyconfig.h.in b/pyconfig.h.in index dd5f2e393be094..cc64355416463b 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -658,6 +658,9 @@ /* Define to 1 if you have the `lutimes' function. */ #undef HAVE_LUTIMES +/* Define to 1 if you have the `madvise' function. */ +#undef HAVE_MADVISE + /* Define this if you have the makedev macro. */ #undef HAVE_MAKEDEV