Skip to content
Permalink
Browse files

bpo-32941: Add madvise() for mmap objects (GH-6172)

Allow mmap objects to access the madvise() system call.
  • Loading branch information...
ZackerySpytz authored and pitrou committed May 27, 2019
1 parent 331a6a5 commit 02db696732c031d9a0265dc9bbf4f5b1fad042b3
@@ -203,6 +203,20 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
exception was raised on error under Unix.


.. method:: madvise(option[, start[, length]])

Send advice *option* to the kernel about the memory region beginning at
*start* and extending *length* bytes. *option* must be one of the
:ref:`MADV_* constants <madvise-constants>` available on the system. If
*start* and *length* are omitted, the entire mapping is spanned. On
some systems (including Linux), *start* must be a multiple of the
:const:`PAGESIZE`.

Availability: Systems with the ``madvise()`` system call.

.. versionadded:: 3.8


.. method:: move(dest, src, count)

Copy the *count* bytes starting at offset *src* to the destination index
@@ -292,3 +306,38 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
position of the file pointer; the file position is advanced by ``1``. If
the mmap was created with :const:`ACCESS_READ`, then writing to it will
raise a :exc:`TypeError` exception.

.. _madvise-constants:

MADV_* Constants
++++++++++++++++

.. data:: MADV_NORMAL
MADV_RANDOM
MADV_SEQUENTIAL
MADV_WILLNEED
MADV_DONTNEED
MADV_REMOVE
MADV_DONTFORK
MADV_DOFORK
MADV_HWPOISON
MADV_MERGEABLE
MADV_UNMERGEABLE
MADV_SOFT_OFFLINE
MADV_HUGEPAGE
MADV_NOHUGEPAGE
MADV_DONTDUMP
MADV_DODUMP
MADV_FREE
MADV_NOSYNC
MADV_AUTOSYNC
MADV_NOCORE
MADV_CORE
MADV_PROTECT

These options can be passed to :meth:`mmap.madvise`. Not every option will
be present on every system.

Availability: Systems with the madvise() system call.

.. versionadded:: 3.8
@@ -460,6 +460,15 @@ numbers. (Contributed by Pablo Galindo in :issue:`35606`)
Added new function :func:`math.isqrt` for computing integer square roots.
(Contributed by Mark Dickinson in :issue:`36887`.)


mmap
----

The :class:`mmap.mmap` class now has an :meth:`~mmap.mmap.madvise` method to
access the ``madvise()`` system call.
(Contributed by Zackery Spytz in :issue:`32941`.)


os
--

@@ -739,6 +739,25 @@ def test_flush_return_value(self):
# See bpo-34754 for details.
self.assertRaises(OSError, mm.flush, 1, len(b'python'))

@unittest.skipUnless(hasattr(mmap.mmap, 'madvise'), 'needs madvise')
def test_madvise(self):
size = 8192
m = mmap.mmap(-1, size)

with self.assertRaisesRegex(ValueError, "madvise start out of bounds"):
m.madvise(mmap.MADV_NORMAL, size)
with self.assertRaisesRegex(ValueError, "madvise start out of bounds"):
m.madvise(mmap.MADV_NORMAL, -1)
with self.assertRaisesRegex(ValueError, "madvise length invalid"):
m.madvise(mmap.MADV_NORMAL, 0, -1)
with self.assertRaisesRegex(OverflowError, "madvise length too large"):
m.madvise(mmap.MADV_NORMAL, PAGESIZE, sys.maxsize)
self.assertEqual(m.madvise(mmap.MADV_NORMAL), None)
self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE), None)
self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE, size), None)
self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, 2), None)
self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, size), None)


class LargeMmapTests(unittest.TestCase):

@@ -0,0 +1,2 @@
Allow :class:`mmap.mmap` objects to access the madvise() system call
(through :meth:`mmap.mmap.madvise`).
@@ -708,11 +708,54 @@ mmap__sizeof__method(mmap_object *self, void *unused)
}
#endif

#ifdef HAVE_MADVISE
static PyObject *
mmap_madvise_method(mmap_object *self, PyObject *args)
{
int option;
Py_ssize_t start = 0, length;

CHECK_VALID(NULL);
length = self->size;

if (!PyArg_ParseTuple(args, "i|nn:madvise", &option, &start, &length)) {
return NULL;
}

if (start < 0 || start >= self->size) {
PyErr_SetString(PyExc_ValueError, "madvise start out of bounds");
return NULL;
}
if (length < 0) {
PyErr_SetString(PyExc_ValueError, "madvise length invalid");
return NULL;
}
if (PY_SSIZE_T_MAX - start < length) {
PyErr_SetString(PyExc_OverflowError, "madvise length too large");
return NULL;
}

if (start + length > self->size) {
length = self->size - start;
}

if (madvise(self->data + start, length, option) != 0) {
PyErr_SetFromErrno(PyExc_OSError);
return NULL;
}

Py_RETURN_NONE;
}
#endif // HAVE_MADVISE

static struct PyMethodDef mmap_object_methods[] = {
{"close", (PyCFunction) mmap_close_method, METH_NOARGS},
{"find", (PyCFunction) mmap_find_method, METH_VARARGS},
{"rfind", (PyCFunction) mmap_rfind_method, METH_VARARGS},
{"flush", (PyCFunction) mmap_flush_method, METH_VARARGS},
#ifdef HAVE_MADVISE
{"madvise", (PyCFunction) mmap_madvise_method, METH_VARARGS},
#endif
{"move", (PyCFunction) mmap_move_method, METH_VARARGS},
{"read", (PyCFunction) mmap_read_method, METH_VARARGS},
{"read_byte", (PyCFunction) mmap_read_byte_method, METH_NOARGS},
@@ -1494,5 +1537,80 @@ PyInit_mmap(void)
setint(dict, "ACCESS_READ", ACCESS_READ);
setint(dict, "ACCESS_WRITE", ACCESS_WRITE);
setint(dict, "ACCESS_COPY", ACCESS_COPY);

#ifdef HAVE_MADVISE
// Conventional advice values
#ifdef MADV_NORMAL
setint(dict, "MADV_NORMAL", MADV_NORMAL);
#endif
#ifdef MADV_RANDOM
setint(dict, "MADV_RANDOM", MADV_RANDOM);
#endif
#ifdef MADV_SEQUENTIAL
setint(dict, "MADV_SEQUENTIAL", MADV_SEQUENTIAL);
#endif
#ifdef MADV_WILLNEED
setint(dict, "MADV_WILLNEED", MADV_WILLNEED);
#endif
#ifdef MADV_DONTNEED
setint(dict, "MADV_DONTNEED", MADV_DONTNEED);
#endif

// Linux-specific advice values
#ifdef MADV_REMOVE
setint(dict, "MADV_REMOVE", MADV_REMOVE);
#endif
#ifdef MADV_DONTFORK
setint(dict, "MADV_DONTFORK", MADV_DONTFORK);
#endif
#ifdef MADV_DOFORK
setint(dict, "MADV_DOFORK", MADV_DOFORK);
#endif
#ifdef MADV_HWPOISON
setint(dict, "MADV_HWPOISON", MADV_HWPOISON);
#endif
#ifdef MADV_MERGEABLE
setint(dict, "MADV_MERGEABLE", MADV_MERGEABLE);
#endif
#ifdef MADV_UNMERGEABLE
setint(dict, "MADV_UNMERGEABLE", MADV_UNMERGEABLE);
#endif
#ifdef MADV_SOFT_OFFLINE
setint(dict, "MADV_SOFT_OFFLINE", MADV_SOFT_OFFLINE);
#endif
#ifdef MADV_HUGEPAGE
setint(dict, "MADV_HUGEPAGE", MADV_HUGEPAGE);
#endif
#ifdef MADV_NOHUGEPAGE
setint(dict, "MADV_NOHUGEPAGE", MADV_NOHUGEPAGE);
#endif
#ifdef MADV_DONTDUMP
setint(dict, "MADV_DONTDUMP", MADV_DONTDUMP);
#endif
#ifdef MADV_DODUMP
setint(dict, "MADV_DODUMP", MADV_DODUMP);
#endif
#ifdef MADV_FREE // (Also present on FreeBSD and macOS.)
setint(dict, "MADV_FREE", MADV_FREE);
#endif

// FreeBSD-specific
#ifdef MADV_NOSYNC
setint(dict, "MADV_NOSYNC", MADV_NOSYNC);
#endif
#ifdef MADV_AUTOSYNC
setint(dict, "MADV_AUTOSYNC", MADV_AUTOSYNC);
#endif
#ifdef MADV_NOCORE
setint(dict, "MADV_NOCORE", MADV_NOCORE);
#endif
#ifdef MADV_CORE
setint(dict, "MADV_CORE", MADV_CORE);
#endif
#ifdef MADV_PROTECT
setint(dict, "MADV_PROTECT", MADV_PROTECT);
#endif
#endif // HAVE_MADVISE

return module;
}
@@ -11471,7 +11471,7 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
getpriority getresuid getresgid getpwent getpwnam_r getpwuid_r getspnam getspent getsid getwd \
if_nameindex \
initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \
initgroups kill killpg lchmod lchown lockf linkat lstat lutimes madvise mmap \
memrchr mbrtowc mkdirat mkfifo \
mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \
posix_fallocate posix_fadvise posix_spawn posix_spawnp pread preadv preadv2 \
@@ -3527,7 +3527,7 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
getpriority getresuid getresgid getpwent getpwnam_r getpwuid_r getspnam getspent getsid getwd \
if_nameindex \
mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \
madvise mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \
posix_fallocate posix_fadvise posix_spawn posix_spawnp pread preadv preadv2 \
pthread_condattr_setclock pthread_init pthread_kill putenv pwrite pwritev pwritev2 \
readlink readlinkat readv realpath renameat \
@@ -658,6 +658,9 @@
/* Define to 1 if you have the `lutimes' function. */
#undef HAVE_LUTIMES

/* Define to 1 if you have the `madvise' function. */
#undef HAVE_MADVISE

/* Define this if you have the makedev macro. */
#undef HAVE_MAKEDEV

0 comments on commit 02db696

Please sign in to comment.
You can’t perform that action at this time.