From 8a2c71d747456bbef30775f8c966e4c02c48f5bc Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Wed, 14 Dec 2016 18:52:38 +0100 Subject: [PATCH 1/5] LUCENE-6989: Unmapping byte buffers: Preview version of the Java 9 b148++ patch --- .../apache/lucene/store/MMapDirectory.java | 113 ++++++++++-------- lucene/tools/junit4/tests.policy | 1 - 2 files changed, 61 insertions(+), 53 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java index be08a1663a6b..d7015c82e267 100644 --- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java @@ -34,6 +34,7 @@ import java.util.Objects; import java.util.concurrent.Future; import java.lang.invoke.MethodHandle; +import java.lang.reflect.Field; import java.lang.reflect.Method; import org.apache.lucene.store.ByteBufferGuard.BufferCleaner; @@ -174,14 +175,13 @@ public MMapDirectory(Path path, LockFactory lockFactory, int maxChunkSize) throw * is closed while another thread is still accessing it (SIGSEGV). *

To enable the hack, the following requirements need to be * fulfilled: The used JVM must be Oracle Java / OpenJDK 8 - * (preliminary support for Java 9 was added with Lucene 6). + * (preliminary support for Java 9 (nocommit: build 148 or later) was added with Lucene 6.4). * In addition, the following permissions need to be granted * to {@code lucene-core.jar} in your * policy file: *

* @throws IllegalArgumentException if {@link #UNMAP_SUPPORTED} * is false and the workaround cannot be enabled. @@ -338,61 +338,70 @@ private IOException convertMapFailedIOException(IOException ioe, String resource @SuppressForbidden(reason = "Needs access to private APIs in DirectBuffer and sun.misc.Cleaner to enable hack") private static Object unmapHackImpl() { final Lookup lookup = lookup(); + Class unmappableBufferClass; + MethodHandle unmapper; try { - final Class directBufferClass = Class.forName("java.nio.DirectByteBuffer"); - - final Method m = directBufferClass.getMethod("cleaner"); - m.setAccessible(true); - MethodHandle directBufferCleanerMethod = lookup.unreflect(m); - Class cleanerClass = directBufferCleanerMethod.type().returnType(); - - final MethodHandle cleanMethod; - if (Runnable.class.isAssignableFrom(cleanerClass)) { - // early Java 9 impl using Runnable (we do the security check early that the Runnable does at runtime): - final SecurityManager sm = System.getSecurityManager(); - if (sm != null) { - sm.checkPackageAccess("jdk.internal.ref"); - } - // cast return value of cleaner() to Runnable: - directBufferCleanerMethod = directBufferCleanerMethod.asType(directBufferCleanerMethod.type().changeReturnType(Runnable.class)); - cleanerClass = Runnable.class; - // lookup run() method on the interface instead of Cleaner: - cleanMethod = lookup.findVirtual(cleanerClass, "run", methodType(void.class)); - } else { - // can be either the old internal "sun.misc.Cleaner" or - // the new Java 9 "java.lang.ref.Cleaner$Cleanable": - cleanMethod = lookup.findVirtual(cleanerClass, "clean", methodType(void.class)); + try { + // *** Unsafe unmapping (Java 9+) *** + final Class unsafeClass = Class.forName("sun.misc.Unsafe"); + // we do not need to check for a specific class, we can call the Unsafe method with any buffer class: + unmappableBufferClass = ByteBuffer.class; + // first check if Unsafe has the right method, otherwise we can give up + // without doing any security critical stuff: + unmapper = lookup.findVirtual(unsafeClass, "invokeCleaner", + methodType(void.class, unmappableBufferClass)); + // fetch the unsafe instance and bind it to the virtual MH: + final Field f = unsafeClass.getDeclaredField("theUnsafe"); + f.setAccessible(true); + final Object theUnsafe = f.get(null); + unmapper = unmapper.bindTo(theUnsafe); + } catch (SecurityException se) { + // rethrow to report errors correctly (we need to catch it here, as we also catch RuntimeException below!): + throw se; + } catch (ReflectiveOperationException | RuntimeException e) { + // *** legacy Java 8 unmapping with sun.misc.Cleaner *** + unmappableBufferClass = Class.forName("java.nio.DirectByteBuffer"); + + final Method m = unmappableBufferClass.getMethod("cleaner"); + m.setAccessible(true); + final MethodHandle directBufferCleanerMethod = lookup.unreflect(m); + final Class cleanerClass = directBufferCleanerMethod.type().returnType(); + + final MethodHandle cleanMethod = lookup.findVirtual(cleanerClass, "clean", methodType(void.class)); + final MethodHandle nonNullTest = lookup.findStatic(Objects.class, "nonNull", methodType(boolean.class, Object.class)) + .asType(methodType(boolean.class, cleanerClass)); + final MethodHandle noop = dropArguments(constant(Void.class, null).asType(methodType(void.class)), 0, cleanerClass); + unmapper = filterReturnValue(directBufferCleanerMethod, guardWithTest(nonNullTest, cleanMethod, noop)) + .asType(methodType(void.class, ByteBuffer.class)); } - - final MethodHandle nonNullTest = lookup.findStatic(Objects.class, "nonNull", methodType(boolean.class, Object.class)) - .asType(methodType(boolean.class, cleanerClass)); - final MethodHandle noop = dropArguments(constant(Void.class, null).asType(methodType(void.class)), 0, cleanerClass); - final MethodHandle unmapper = filterReturnValue(directBufferCleanerMethod, guardWithTest(nonNullTest, cleanMethod, noop)) - .asType(methodType(void.class, ByteBuffer.class)); - - return (BufferCleaner) (String resourceDescription, ByteBuffer buffer) -> { - if (directBufferClass.isInstance(buffer)) { - final Throwable error = AccessController.doPrivileged((PrivilegedAction) () -> { - try { - unmapper.invokeExact(buffer); - return null; - } catch (Throwable t) { - return t; - } - }); - if (error != null) { - throw new IOException("Unable to unmap the mapped buffer: " + resourceDescription, error); - } - } - }; - } catch (SecurityException e) { - return "Unmapping is not supported, because not all required permissions are given to the Lucene JAR file: " + e + - " [Please grant at least the following permissions: RuntimePermission(\"accessClassInPackage.sun.misc\"), " + - "RuntimePermission(\"accessClassInPackage.jdk.internal.ref\"), and " + - "ReflectPermission(\"suppressAccessChecks\")]"; + } catch (SecurityException se) { + return "Unmapping is not supported, because not all required permissions are given to the Lucene JAR file: " + se + + " [Please grant at least the following permissions: RuntimePermission(\"accessClassInPackage.sun.misc\") " + + " and ReflectPermission(\"suppressAccessChecks\")]"; } catch (ReflectiveOperationException | RuntimeException e) { return "Unmapping is not supported on this platform, because internal Java APIs are not compatible to this Lucene version: " + e; } + + final Class directBufferClass0 = unmappableBufferClass; + final MethodHandle unmapper0 = unmapper; + return (BufferCleaner) (String resourceDescription, ByteBuffer buffer) -> { + if (!buffer.isDirect()) { + throw new IllegalArgumentException("Unmapping only works with direct buffers"); + } + if (directBufferClass0.isInstance(buffer)) { + final Throwable error = AccessController.doPrivileged((PrivilegedAction) () -> { + try { + unmapper0.invokeExact(buffer); + return null; + } catch (Throwable t) { + return t; + } + }); + if (error != null) { + throw new IOException("Unable to unmap the mapped buffer: " + resourceDescription, error); + } + } + }; } } diff --git a/lucene/tools/junit4/tests.policy b/lucene/tools/junit4/tests.policy index 2dde5c6f329d..b351b172c268 100644 --- a/lucene/tools/junit4/tests.policy +++ b/lucene/tools/junit4/tests.policy @@ -63,7 +63,6 @@ grant { permission java.lang.RuntimePermission "createClassLoader"; // needed to test unmap hack on platforms that support it permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; - permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.ref"; // needed by cyberneko usage by benchmarks on J9 permission java.lang.RuntimePermission "accessClassInPackage.org.apache.xerces.util"; // needed by jacoco to dump coverage From 71d3a5039dbcca6066cc7f6e3ac2f1564afd12b0 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Wed, 14 Dec 2016 20:17:23 +0100 Subject: [PATCH 2/5] LUCENE-6989: Fix annotation of hack --- lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java index d7015c82e267..701ae3be2f4b 100644 --- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java @@ -335,7 +335,7 @@ private IOException convertMapFailedIOException(IOException ioe, String resource } } - @SuppressForbidden(reason = "Needs access to private APIs in DirectBuffer and sun.misc.Cleaner to enable hack") + @SuppressForbidden(reason = "Needs access to private APIs in DirectBuffer, sun.misc.Cleaner, and sun.misc.Unsafe to enable hack") private static Object unmapHackImpl() { final Lookup lookup = lookup(); Class unmappableBufferClass; From b8fe1ff83fa1243172b2812bd976a6011f671914 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Wed, 14 Dec 2016 20:22:42 +0100 Subject: [PATCH 3/5] LUCENE-6989: Rename variable --- .../core/src/java/org/apache/lucene/store/MMapDirectory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java index 701ae3be2f4b..c7ceeb304d5c 100644 --- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java @@ -382,13 +382,13 @@ private static Object unmapHackImpl() { return "Unmapping is not supported on this platform, because internal Java APIs are not compatible to this Lucene version: " + e; } - final Class directBufferClass0 = unmappableBufferClass; + final Class unmappableBufferClass0 = unmappableBufferClass; final MethodHandle unmapper0 = unmapper; return (BufferCleaner) (String resourceDescription, ByteBuffer buffer) -> { if (!buffer.isDirect()) { throw new IllegalArgumentException("Unmapping only works with direct buffers"); } - if (directBufferClass0.isInstance(buffer)) { + if (unmappableBufferClass0.isInstance(buffer)) { final Throwable error = AccessController.doPrivileged((PrivilegedAction) () -> { try { unmapper0.invokeExact(buffer); From ffc957fdb3c21d110ab23392ed91e74cfc1f169d Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Fri, 16 Dec 2016 22:09:54 +0100 Subject: [PATCH 4/5] LUCENE-6989: Refactor code and add documentation --- .../apache/lucene/store/MMapDirectory.java | 63 +++++++++++-------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java index c7ceeb304d5c..fa2ed8d3592c 100644 --- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java @@ -175,7 +175,7 @@ public MMapDirectory(Path path, LockFactory lockFactory, int maxChunkSize) throw * is closed while another thread is still accessing it (SIGSEGV). *

To enable the hack, the following requirements need to be * fulfilled: The used JVM must be Oracle Java / OpenJDK 8 - * (preliminary support for Java 9 (nocommit: build 148 or later) was added with Lucene 6.4). + * (preliminary support for Java 9 EA build 150+ was added with Lucene 6.4). * In addition, the following permissions need to be granted * to {@code lucene-core.jar} in your * policy file: @@ -338,41 +338,48 @@ private IOException convertMapFailedIOException(IOException ioe, String resource @SuppressForbidden(reason = "Needs access to private APIs in DirectBuffer, sun.misc.Cleaner, and sun.misc.Unsafe to enable hack") private static Object unmapHackImpl() { final Lookup lookup = lookup(); - Class unmappableBufferClass; - MethodHandle unmapper; try { try { - // *** Unsafe unmapping (Java 9+) *** + // *** sun.misc.Unsafe unmapping (Java 9+) *** final Class unsafeClass = Class.forName("sun.misc.Unsafe"); - // we do not need to check for a specific class, we can call the Unsafe method with any buffer class: - unmappableBufferClass = ByteBuffer.class; // first check if Unsafe has the right method, otherwise we can give up // without doing any security critical stuff: - unmapper = lookup.findVirtual(unsafeClass, "invokeCleaner", - methodType(void.class, unmappableBufferClass)); + final MethodHandle unmapper = lookup.findVirtual(unsafeClass, "invokeCleaner", + methodType(void.class, ByteBuffer.class)); // fetch the unsafe instance and bind it to the virtual MH: final Field f = unsafeClass.getDeclaredField("theUnsafe"); f.setAccessible(true); final Object theUnsafe = f.get(null); - unmapper = unmapper.bindTo(theUnsafe); + return newBufferCleaner(ByteBuffer.class, unmapper.bindTo(theUnsafe)); } catch (SecurityException se) { // rethrow to report errors correctly (we need to catch it here, as we also catch RuntimeException below!): throw se; } catch (ReflectiveOperationException | RuntimeException e) { - // *** legacy Java 8 unmapping with sun.misc.Cleaner *** - unmappableBufferClass = Class.forName("java.nio.DirectByteBuffer"); + // *** sun.misc.Cleaner unmapping (Java 8) *** + final Class unmappableBufferClass = Class.forName("java.nio.DirectByteBuffer"); final Method m = unmappableBufferClass.getMethod("cleaner"); m.setAccessible(true); final MethodHandle directBufferCleanerMethod = lookup.unreflect(m); final Class cleanerClass = directBufferCleanerMethod.type().returnType(); + /* "Compile" a MH that basically is equivalent to the following code: + * void unmapper(ByteBuffer byteBuffer) { + * sun.misc.Cleaner cleaner = ((java.nio.DirectByteBuffer) byteBuffer).cleaner(); + * if (Objects.nonNull(cleaner)) { + * cleaner.clean(); + * } else { + * noop(cleaner); // the noop is needed because MethodHandles#guardWithTest always needs ELSE + * } + * } + */ final MethodHandle cleanMethod = lookup.findVirtual(cleanerClass, "clean", methodType(void.class)); final MethodHandle nonNullTest = lookup.findStatic(Objects.class, "nonNull", methodType(boolean.class, Object.class)) .asType(methodType(boolean.class, cleanerClass)); final MethodHandle noop = dropArguments(constant(Void.class, null).asType(methodType(void.class)), 0, cleanerClass); - unmapper = filterReturnValue(directBufferCleanerMethod, guardWithTest(nonNullTest, cleanMethod, noop)) + final MethodHandle unmapper = filterReturnValue(directBufferCleanerMethod, guardWithTest(nonNullTest, cleanMethod, noop)) .asType(methodType(void.class, ByteBuffer.class)); + return newBufferCleaner(unmappableBufferClass, unmapper); } } catch (SecurityException se) { return "Unmapping is not supported, because not all required permissions are given to the Lucene JAR file: " + se + @@ -381,25 +388,27 @@ private static Object unmapHackImpl() { } catch (ReflectiveOperationException | RuntimeException e) { return "Unmapping is not supported on this platform, because internal Java APIs are not compatible to this Lucene version: " + e; } - - final Class unmappableBufferClass0 = unmappableBufferClass; - final MethodHandle unmapper0 = unmapper; - return (BufferCleaner) (String resourceDescription, ByteBuffer buffer) -> { + } + + private static BufferCleaner newBufferCleaner(final Class unmappableBufferClass, final MethodHandle unmapper) { + assert Objects.equals(methodType(void.class, ByteBuffer.class), unmapper.type()); + return (String resourceDescription, ByteBuffer buffer) -> { if (!buffer.isDirect()) { throw new IllegalArgumentException("Unmapping only works with direct buffers"); } - if (unmappableBufferClass0.isInstance(buffer)) { - final Throwable error = AccessController.doPrivileged((PrivilegedAction) () -> { - try { - unmapper0.invokeExact(buffer); - return null; - } catch (Throwable t) { - return t; - } - }); - if (error != null) { - throw new IOException("Unable to unmap the mapped buffer: " + resourceDescription, error); + if (!unmappableBufferClass.isInstance(buffer)) { + throw new IllegalArgumentException("ByteBuffer is not an instance of " + unmappableBufferClass.getName()); + } + final Throwable error = AccessController.doPrivileged((PrivilegedAction) () -> { + try { + unmapper.invokeExact(buffer); + return null; + } catch (Throwable t) { + return t; } + }); + if (error != null) { + throw new IOException("Unable to unmap the mapped buffer: " + resourceDescription, error); } }; } From 64c6f359949b62fe981255516ba2286c0adcc190 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Fri, 16 Dec 2016 22:38:30 +0100 Subject: [PATCH 5/5] LUCENE-6989: Comments and final cleanup --- .../java/org/apache/lucene/store/MMapDirectory.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java index fa2ed8d3592c..0487400c7764 100644 --- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java @@ -356,9 +356,9 @@ private static Object unmapHackImpl() { throw se; } catch (ReflectiveOperationException | RuntimeException e) { // *** sun.misc.Cleaner unmapping (Java 8) *** - final Class unmappableBufferClass = Class.forName("java.nio.DirectByteBuffer"); + final Class directBufferClass = Class.forName("java.nio.DirectByteBuffer"); - final Method m = unmappableBufferClass.getMethod("cleaner"); + final Method m = directBufferClass.getMethod("cleaner"); m.setAccessible(true); final MethodHandle directBufferCleanerMethod = lookup.unreflect(m); final Class cleanerClass = directBufferCleanerMethod.type().returnType(); @@ -379,7 +379,7 @@ private static Object unmapHackImpl() { final MethodHandle noop = dropArguments(constant(Void.class, null).asType(methodType(void.class)), 0, cleanerClass); final MethodHandle unmapper = filterReturnValue(directBufferCleanerMethod, guardWithTest(nonNullTest, cleanMethod, noop)) .asType(methodType(void.class, ByteBuffer.class)); - return newBufferCleaner(unmappableBufferClass, unmapper); + return newBufferCleaner(directBufferClass, unmapper); } } catch (SecurityException se) { return "Unmapping is not supported, because not all required permissions are given to the Lucene JAR file: " + se + @@ -394,10 +394,10 @@ private static BufferCleaner newBufferCleaner(final Class unmappableBufferCla assert Objects.equals(methodType(void.class, ByteBuffer.class), unmapper.type()); return (String resourceDescription, ByteBuffer buffer) -> { if (!buffer.isDirect()) { - throw new IllegalArgumentException("Unmapping only works with direct buffers"); + throw new IllegalArgumentException("unmapping only works with direct buffers"); } if (!unmappableBufferClass.isInstance(buffer)) { - throw new IllegalArgumentException("ByteBuffer is not an instance of " + unmappableBufferClass.getName()); + throw new IllegalArgumentException("buffer is not an instance of " + unmappableBufferClass.getName()); } final Throwable error = AccessController.doPrivileged((PrivilegedAction) () -> { try {