diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_mmap.py b/graalpython/com.oracle.graal.python.test/src/tests/test_mmap.py index 64296a564c..cb186d71fd 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_mmap.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_mmap.py @@ -37,9 +37,11 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import errno import mmap import os import sys +import tempfile import time import unittest @@ -99,6 +101,27 @@ def test_access_copy_without_map_private_constant(): m.close() +@unittest.skipIf(sys.platform == "win32", "trackfd is Unix-only") +def test_trackfd(): + test_case = unittest.TestCase() + # The emulated POSIX backend reopens the file by path when creating the mapping. + # TemporaryFile unlinks that path immediately on POSIX (GR-29159). + with tempfile.NamedTemporaryFile() as f: + f.write(b"x" * 64) + f.flush() + with mmap.mmap(f.fileno(), 32) as m: + assert m.size() == 64 + with mmap.mmap(f.fileno(), 32, trackfd=False) as m: + f.close() + assert len(m) == 32 + with test_case.assertRaises(OSError) as error: + m.size() + assert error.exception.errno == errno.EBADF + with test_case.assertRaisesRegex(ValueError, "trackfd=False"): + m.resize(16) + assert m[:1] == b"x" + + def test_find(): cases = [ # (size, needle_pos) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java index 6d8e0576ef..bcacdacc8d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java @@ -508,7 +508,18 @@ Return an iterator yielding those items of iterable for which function(item) PAsyncGenAWrappedValue("async_generator_wrapped_value", PythonObject, newBuilder()), PMethod("method", PythonObject, newBuilder().slots(AbstractMethodBuiltins.SLOTS, MethodBuiltins.SLOTS).doc(""" Create a bound instance method object.""")), - PMMap("mmap", PythonObject, newBuilder().publishInModule("mmap").basetype().slots(MMapBuiltins.SLOTS)), + PMMap("mmap", PythonObject, newBuilder().publishInModule("mmap").basetype().slots(MMapBuiltins.SLOTS).doc(""" + Windows: mmap(fileno, length[, tagname[, access[, offset]]]) + + Maps length bytes from the file specified by the file handle fileno. + + Unix: mmap(fileno, length[, flags[, prot[, access[, offset]]]], *, trackfd=True) + + Maps length bytes from the file specified by the file descriptor fileno. If length + is 0, the maximum length of the map is the current size of the file. If trackfd is + false, fileno is not duplicated and size() and resize() are unavailable. + + To map anonymous memory, pass -1 as fileno.""")), PNone("NoneType", PythonObject, newBuilder().slots(NoneBuiltins.SLOTS)), PNotImplemented("NotImplementedType", PythonObject, newBuilder().slots(NotImplementedBuiltins.SLOTS)), PProperty(J_PROPERTY, PythonObject, newBuilder().publishInModule(J_BUILTINS).basetype().slots(PropertyBuiltins.SLOTS).doc(""" diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/mmap/MMapBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/mmap/MMapBuiltins.java index 329605d6dd..0134458d6e 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/mmap/MMapBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/mmap/MMapBuiltins.java @@ -114,6 +114,7 @@ import com.oracle.graal.python.lib.PyLongAsIntNode; import com.oracle.graal.python.lib.PyLongAsLongNode; import com.oracle.graal.python.lib.PyNumberAsSizeNode; +import com.oracle.graal.python.lib.PyObjectIsTrueNode; import com.oracle.graal.python.nodes.ErrorMessages; import com.oracle.graal.python.nodes.PConstructAndRaiseNode; import com.oracle.graal.python.nodes.PGuards; @@ -184,7 +185,7 @@ private static byte[] readBytes(VirtualFrame frame, Node inliningTarget, PMMap s @Slot(value = SlotKind.tp_new, isComplex = true) @SlotSignature(name = "mmap", minNumOfPositionalArgs = 3, parameterNames = {"cls", "fd", "length", "flags", "prot", "access", - "offset"}, keywordOnlyNames = {"tagname"}) + "offset"}, keywordOnlyNames = {"tagname", "trackfd"}) @GenerateNodeFactory // Note: it really should not call fileno on fd as per Python spec @ArgumentClinic(name = "fd", conversion = ClinicConversion.Int) @@ -199,7 +200,7 @@ public abstract static class MMapNode extends PythonClinicBuiltinNode { private static final int ANONYMOUS_FD = -1; - private record MMapArgs(int flags, Object tagname) { + private record MMapArgs(int flags, Object tagname, boolean trackFd) { } @Override @@ -210,7 +211,7 @@ protected ArgumentClinicProvider getArgumentClinic() { // mmap(fileno, length, tagname=None, access=ACCESS_DEFAULT[, offset=0]) @Specialization(guards = "!isIllegal(fd)") static PMMap doFile(VirtualFrame frame, Object clazz, int fd, long lengthIn, Object flagsIn, int protIn, @SuppressWarnings("unused") int accessIn, long offset, - Object tagname, + Object tagname, Object trackFdArg, @Bind Node inliningTarget, @Cached SysModuleBuiltins.AuditNode auditNode, @CachedLibrary("getPosixSupport()") PosixSupportLibrary posixSupport, @@ -218,8 +219,9 @@ static PMMap doFile(VirtualFrame frame, Object clazz, int fd, long lengthIn, Obj @Cached TypeNodes.GetInstanceShape getInstanceShape, @Exclusive @Cached CastToTruffleStringNode castTagnameNode, @Exclusive @Cached PyLongAsIntNode flagsAsIntNode, + @Exclusive @Cached PyObjectIsTrueNode isTrueNode, @Exclusive @Cached PRaiseNode raiseNode) { - MMapArgs mmapArgs = parseFlagsAndTagname(frame, inliningTarget, flagsIn, tagname, castTagnameNode, flagsAsIntNode, raiseNode); + MMapArgs mmapArgs = parseConstructorArgs(frame, inliningTarget, flagsIn, tagname, trackFdArg, castTagnameNode, flagsAsIntNode, isTrueNode, raiseNode); Object mmapTagname = PNone.NONE; if (mmapArgs.tagname() != PNone.NO_VALUE && !isPNone(mmapArgs.tagname())) { try { @@ -291,28 +293,36 @@ static PMMap doFile(VirtualFrame frame, Object clazz, int fd, long lengthIn, Obj } // Fixup the flags if we want to use anonymous map - int dupFd; + int trackedFd; if (fd == ANONYMOUS_FD) { - dupFd = ANONYMOUS_FD; + trackedFd = ANONYMOUS_FD; flags |= MAP_ANONYMOUS.value; // TODO: CPython uses mapping to "/dev/zero" on systems that do not support // MAP_ANONYMOUS, maybe this can be detected and handled by the POSIX layer - } else { + } else if (mmapArgs.trackFd()) { try { - dupFd = posixSupport.dup(posixSupport1, fd); + trackedFd = posixSupport.dup(posixSupport1, fd); } catch (PosixException e) { throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); } + } else { + trackedFd = ANONYMOUS_FD; } Object mmapHandle; try { - mmapHandle = posixSupport.mmap(posixSupport1, length, prot, flags, dupFd, offset, mmapTagname); + mmapHandle = posixSupport.mmap(posixSupport1, length, prot, flags, fd, offset, mmapTagname); } catch (PosixException e) { + if (trackedFd != ANONYMOUS_FD) { + try { + posixSupport.close(posixSupport1, trackedFd); + } catch (PosixException ignored) { + } + } throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); } PythonContext context = PythonContext.get(inliningTarget); - PMMap mmap = PFactory.createMMap(context, clazz, getInstanceShape.execute(clazz), mmapHandle, dupFd, length, access); + PMMap mmap = PFactory.createMMap(context, clazz, getInstanceShape.execute(clazz), mmapHandle, trackedFd, length, access, mmapArgs.trackFd()); if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32 && NativeAccessSupport.isAvailable()) { NativeContext.setLastError(0); } @@ -322,17 +332,18 @@ static PMMap doFile(VirtualFrame frame, Object clazz, int fd, long lengthIn, Obj @Specialization(guards = "isIllegal(fd)") @SuppressWarnings("unused") static PMMap doIllegal(VirtualFrame frame, Object clazz, int fd, long lengthIn, Object flagsIn, int protIn, int accessIn, long offset, - Object tagname, + Object tagname, Object trackFdArg, @Bind Node inliningTarget, @Exclusive @Cached CastToTruffleStringNode castTagnameNode, @Exclusive @Cached PyLongAsIntNode flagsAsIntNode, + @Exclusive @Cached PyObjectIsTrueNode isTrueNode, @Exclusive @Cached PRaiseNode raiseNode) { - parseFlagsAndTagname(frame, inliningTarget, flagsIn, tagname, castTagnameNode, flagsAsIntNode, raiseNode); + parseConstructorArgs(frame, inliningTarget, flagsIn, tagname, trackFdArg, castTagnameNode, flagsAsIntNode, isTrueNode, raiseNode); throw PRaiseNode.raiseStatic(inliningTarget, PythonBuiltinClassType.OSError); } - private static MMapArgs parseFlagsAndTagname(VirtualFrame frame, Node inliningTarget, Object flagsArg, Object tagnameArg, - CastToTruffleStringNode castTagnameNode, PyLongAsIntNode flagsAsIntNode, PRaiseNode raiseNode) { + private static MMapArgs parseConstructorArgs(VirtualFrame frame, Node inliningTarget, Object flagsArg, Object tagnameArg, Object trackFdArg, + CastToTruffleStringNode castTagnameNode, PyLongAsIntNode flagsAsIntNode, PyObjectIsTrueNode isTrueNode, PRaiseNode raiseNode) { Object flags = flagsArg; Object tagname = tagnameArg; if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32 && tagname == PNone.NO_VALUE && flags != PNone.NO_VALUE) { @@ -348,7 +359,11 @@ private static MMapArgs parseFlagsAndTagname(VirtualFrame frame, Node inliningTa if (tagname != PNone.NO_VALUE && PythonLanguage.getPythonOS() != PythonOS.PLATFORM_WIN32) { throw raiseNode.raise(inliningTarget, TypeError, ErrorMessages.GOT_UNEXPECTED_KEYWORD_ARG, "mmap", "tagname"); } - return new MMapArgs(flags == PNone.NO_VALUE ? FLAGS_DEFAULT : flagsAsIntNode.execute(frame, inliningTarget, flags), tagname); + if (trackFdArg != PNone.NO_VALUE && PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32) { + throw raiseNode.raise(inliningTarget, TypeError, ErrorMessages.GOT_UNEXPECTED_KEYWORD_ARG, "mmap", "trackfd"); + } + boolean trackFd = trackFdArg == PNone.NO_VALUE || isTrueNode.execute(frame, trackFdArg); + return new MMapArgs(flags == PNone.NO_VALUE ? FLAGS_DEFAULT : flagsAsIntNode.execute(frame, inliningTarget, flags), tagname, trackFd); } protected static boolean isIllegal(int fd) { @@ -630,8 +645,23 @@ static boolean close(PMMap self) { abstract static class SizeNode extends PythonBuiltinNode { @Specialization - static long size(PMMap self) { - return self.getLength(); + static long size(VirtualFrame frame, PMMap self, + @Bind Node inliningTarget, + @Bind PythonContext context, + @CachedLibrary("context.getPosixSupport()") PosixSupportLibrary posixSupport, + @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode, + @Cached PRaiseNode raiseNode) { + if (self.isClosed()) { + throw raiseNode.raise(inliningTarget, ValueError, MMAP_CLOSED_OR_INVALID); + } + if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32 && self.getFd() == -1) { + return self.getLength(); + } + try { + return posixSupport.fstat(context.getPosixSupport(), self.getFd())[ST_SIZE]; + } catch (PosixException e) { + throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); + } } } @@ -643,6 +673,15 @@ abstract static class ResizeNode extends PythonBuiltinNode { @SuppressWarnings("unused") static long resize(PMMap self, Object n, @Bind Node inliningTarget) { + if (self.isClosed()) { + throw PRaiseNode.raiseStatic(inliningTarget, ValueError, MMAP_CLOSED_OR_INVALID); + } + if (!self.tracksFd()) { + throw PRaiseNode.raiseStatic(inliningTarget, PythonBuiltinClassType.ValueError, ErrorMessages.MMAP_CANNOT_RESIZE_WITH_TRACKFD_FALSE); + } + if (!self.isWriteable() || self.getAccess() == ACCESS_COPY) { + throw PRaiseNode.raiseStatic(inliningTarget, TypeError, ErrorMessages.MMAP_CANNOT_RESIZE_READONLY_OR_COPY_ON_WRITE); + } // TODO: implement resize throw PRaiseNode.raiseStatic(inliningTarget, PythonBuiltinClassType.SystemError, ErrorMessages.RESIZING_NOT_AVAILABLE); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/mmap/PMMap.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/mmap/PMMap.java index cec536ae08..bac81eca36 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/mmap/PMMap.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/mmap/PMMap.java @@ -74,10 +74,10 @@ public final class PMMap extends PythonObject { private long pos; private final int access; - public PMMap(Object pythonClass, Shape instanceShape, PythonContext context, Object handle, int fd, long length, int access) { + public PMMap(Object pythonClass, Shape instanceShape, PythonContext context, Object handle, int fd, long length, int access, boolean trackFd) { super(pythonClass, instanceShape); assert handle != null; - this.ref = new PMMap.MMapRef(this, handle, context.getSharedFinalizer(), fd, length); + this.ref = new PMMap.MMapRef(this, handle, context.getSharedFinalizer(), fd, length, trackFd); this.access = access; } @@ -110,6 +110,14 @@ public long getLength() { return ref.length; } + int getFd() { + return ref.fd; + } + + boolean tracksFd() { + return ref.trackFd; + } + public long getPos() { return pos; } @@ -174,11 +182,13 @@ static class MMapRef extends AsyncHandler.SharedFinalizer.FinalizableReference { final int fd; private final long length; + private final boolean trackFd; - MMapRef(PMMap referent, Object handle, AsyncHandler.SharedFinalizer finalizer, int fd, long length) { + MMapRef(PMMap referent, Object handle, AsyncHandler.SharedFinalizer finalizer, int fd, long length, boolean trackFd) { super(referent, handle, finalizer); this.fd = fd; this.length = length; + this.trackFd = trackFd; } @Override diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java index 5c5331f7a0..aa56ba4931 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java @@ -1081,6 +1081,8 @@ public abstract class ErrorMessages { public static final TruffleString MMAP_S_IS_GREATER_THAN_FILE_SIZE = tsLiteral("mmap %s is greater than file size"); public static final TruffleString TOO_MANY_REMAINING_BYTES_TO_BE_STORED = tsLiteral("There are too many remaining bytes to be stored in a bytes object."); public static final TruffleString MMAP_CANNOT_MODIFY_READONLY_MEMORY = tsLiteral("mmap can't modify a readonly memory map."); + public static final TruffleString MMAP_CANNOT_RESIZE_READONLY_OR_COPY_ON_WRITE = tsLiteral("mmap can't resize a readonly or copy-on-write memory map."); + public static final TruffleString MMAP_CANNOT_RESIZE_WITH_TRACKFD_FALSE = tsLiteral("mmap can't resize with trackfd=False."); public static final TruffleString DATA_OUT_OF_RANGE = tsLiteral("data out of range"); public static final TruffleString MMAP_CLOSED_OR_INVALID = tsLiteral("mmap closed or invalid"); public static final TruffleString MMAP_OBJECT_DOESNT_SUPPORT_ITEM_DELETION = tsLiteral("mmap object doesn't support item deletion"); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/object/PFactory.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/object/PFactory.java index 430084a492..8c9de7cc84 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/object/PFactory.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/object/PFactory.java @@ -1150,8 +1150,8 @@ public static PEncodingMap createEncodingMap(PythonLanguage language, int count2 return new PEncodingMap(PythonBuiltinClassType.PEncodingMap, PythonBuiltinClassType.PEncodingMap.getInstanceShape(language), count2, count3, level1, level23); } - public static PMMap createMMap(PythonContext context, Object cls, Shape shape, Object mmapHandle, int fd, long length, int access) { - return new PMMap(cls, shape, context, mmapHandle, fd, length, access); + public static PMMap createMMap(PythonContext context, Object cls, Shape shape, Object mmapHandle, int fd, long length, int access, boolean trackFd) { + return new PMMap(cls, shape, context, mmapHandle, fd, length, access, trackFd); } public static BZ2Object.BZ2Compressor createBZ2Compressor(PythonLanguage language) {