Merge branch 'fix-aarch64-safepatching' into develop

com.oracle.max.vm.ext.maxri/src/com/oracle/max/vm/ext/maxri/MaxTargetMethod.java

@@ -148,16 +148,10 @@ private void init(CiTargetMethod ciTargetMethod, boolean install) {
 
         debugInfo = new DebugInfo(debugInfos, this);
 
-        if (!isHosted()) {
-            if (install) {
-                linkDirectCalls();
-                if (Platform.target().arch.isARM() || Platform.target().arch.isAarch64() || Platform.target().arch.isRISCV64()) {
-                    MaxineVM.maxine_cache_flush(codeStart().toPointer(), code().length);
-                }
-            } else {
-                // the displacement between a call site in the heap and a code cache location may not fit in the offset
-                // operand of a call
-            }
+        if (!isHosted() && install) {
+            linkDirectCalls();
+            // Perform cache maintenance after linking calls to ensure visibility of fixed call-sites.
+            maybeCleanCache();
         }
     }
 

com.oracle.max.vm.ext.t1x/src/com/oracle/max/vm/ext/t1x/T1XTargetMethod.java

@@ -257,13 +257,10 @@ public T1XTargetMethod(T1XCompilation comp, boolean install) {
             }
         }
 
-        if (!MaxineVM.isHosted()) {
-            if (install) {
-                linkDirectCalls();
-                if (Platform.target().arch.isARM() || Platform.target().arch.isAarch64() || Platform.target().arch.isRISCV64()) {
-                    MaxineVM.maxine_cache_flush(codeStart().toPointer(), code().length);
-                }
-            }  // the displacement between a call site in the heap and a code cache location may not fit in the offset operand of a call
+        if (!MaxineVM.isHosted() && install) {
+            linkDirectCalls();
+            // Perform cache maintenance after linking calls to ensure visibility of fixed call-sites.
+            maybeCleanCache();
         }
 
         // if the VM is running, validate freshly generated code

com.oracle.max.vm.native/share/log.h

@@ -75,6 +75,7 @@ extern void log_flush(void);
 
 #define log_ALL 0
 
+#define log_MEMBARRIER (log_ALL || 0)
 #define log_LOADER (log_ALL || 0)
 #define log_TRAP (log_ALL || 0)
 #define log_MONITORS (log_ALL || 0)

com.oracle.max.vm.native/substrate/barrier.c

@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2019, APT Group, School of Computer Science,
+ * The University of Manchester. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+/*
+ * The functions in this module perform the membarrier Linux system call.
+ * This system call causes an inter-processor-interrupt (IPI) to be delivered
+ * to concurrently executing cores on the current system. On Aarch64 platforms
+ * we use that mechanism to synchronise instruction streams on multi-cores.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include "log.h"
+#include "os.h"
+#if os_LINUX
+#include "isa.h"
+#include <unistd.h>
+#include <sys/syscall.h>
+#include <linux/membarrier.h>
+#include <linux/version.h>
+
+
+/*
+ * Pre-processor override for whether to compile in the membarrier system call.
+ * Currently only affects Aarch64. See syscall_membarrier() in this compilation
+ * unit.
+ */
+#ifndef USE_SYS_MEMBARRIER
+# define USE_SYS_MEMBARRIER 1
+#endif
+
+/*
+ * Local membarrier macro in the fashion of the one described in
+ * the membarrier manual page.
+ */
+#define membarrier(cmd, flags) syscall(__NR_membarrier, cmd, flags)
+
+/*
+ * Initialise the system to use the best available barrier.
+ */
+static int membarrier_init(void) __attribute__ ((unused));
+
+/*
+ * Execute the membarrier system call
+ */
+void
+syscall_membarrier()
+{
+#if isa_AARCH64
+# if USE_SYS_MEMBARRIER
+    static volatile int barrier_kind = 0;
+    if (!barrier_kind) {
+        barrier_kind = membarrier_init();
+    }
+    membarrier(barrier_kind, 0);
+# endif /* USE_SYS_MEMBARRIER */
+#else
+    log_exit(1, "membarrier not configured on this platform");
+#endif /* isa_AARCH64 */
+}
+
+static int
+membarrier_init(void)
+{
+    long lv;
+
+    lv = membarrier(MEMBARRIER_CMD_QUERY, 0);
+
+    if (lv <= 0) {
+        log_exit(1, "No barriers available on this platform.");
+    }
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,14,0)
+    /*
+     * Check first for the availability of the expedited barrier that
+     * limits the mask to CPUs running the current process only. This
+     * barrier, if available, has a much lower overhead than the shared
+     * barrier that will interrupt all cores on the system, regardless
+     * of whether they are executing the current process.
+     */
+    if (lv & MEMBARRIER_CMD_PRIVATE_EXPEDITED) {
+        /* Register our intention to use the expedited barrier, always
+         * returns 0 so no need to test the return value.
+         */
+        membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0);
+        if (log_MEMBARRIER) {
+            log_println("Using private expedited barrier");
+        }
+        return MEMBARRIER_CMD_PRIVATE_EXPEDITED;
+    }
+#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(4,14,0) */
+
+    /*
+     * Fallback to the system-wide barrier if available.
+     */
+    if (lv & MEMBARRIER_CMD_SHARED) {
+        if (log_MEMBARRIER) {
+            log_println("Using shared barrier");
+        }
+        return MEMBARRIER_CMD_SHARED;
+    }
+    /* No useable barrier available. */
+    log_exit(1, "No useable barrier on this platform.");
+}
+
+#else
+/* Print an informative message if called on a non-Linux OS. */
+void
+syscall_membarrier()
+{
+    log_exit(1, "membarrier not available on this platform");
+}
+#endif /* os_LINUX */
+

com.oracle.max.vm.native/substrate/substrate.mk

@@ -29,7 +29,7 @@ LIB = jvm
 
 SOURCES = c.c condition.c log.c image.c $(ISA).c jni.c jvm.c maxine.c memory.c mutex.c \
           relocation.c dataio.c snippet.c threads.c threadLocals.c time.c trap.c \
-          virtualMemory.c jnitests.c sync.c signal.c jmm.c jvmti.c
+          virtualMemory.c jnitests.c sync.c signal.c jmm.c jvmti.c barrier.c
 
 TARGETOS ?= $(shell uname -s)
 ifeq ($(TARGETOS),Linux)

com.sun.max/src/com/sun/max/vm/MaxineVM.java

@@ -565,7 +565,13 @@ public static void registerCriticalMethod(CriticalMethod criticalEntryPoint) {
      * @param length the number of instructions * sizeof one instruction
      */
     @C_FUNCTION
-    public static native void maxine_cache_flush(Pointer start, int length);
+    public static native void maxine_cache_flush(Address start, int length);
+
+    /**
+     * Executes a membarrier(2) system call on Linux systems.
+     */
+    @C_FUNCTION
+    public static native void syscall_membarrier();
 
     @C_FUNCTION
     public static native long arithmeticldiv(long x, long y);

com.sun.max/src/com/sun/max/vm/code/CodeEviction.java

@@ -339,7 +339,12 @@ public boolean doTargetMethod(TargetMethod targetMethod) {
                     null : (byte[]) relocate(from, to, targetMethod.scalarLiterals());
                 final Object[] referenceLiterals = targetMethod.referenceLiterals() == null ?
                     null : (Object[]) relocate(from, to, targetMethod.referenceLiterals());
+                /* Two calls to cleanCache() are required. The first ensures that old instructions are cleaned from
+                 * caches and prefetch buffers. The second ensures that the effects of any prefetching from the addresses
+                 * that the code is moved to are discarded. */
+                targetMethod.maybeCleanCache();
                 targetMethod.setCodeArrays(code, codeStart, trampolines, trampolineStart, scalarLiterals, referenceLiterals);
+                targetMethod.maybeCleanCache();
                 cr.setMark(cr.mark().plus(size));
                 CodeManager.runtimeBaselineCodeRegion.add(targetMethod);
                 targetMethod.survivedEviction();

com.sun.max/src/com/sun/max/vm/compiler/target/Adapter.java

@@ -112,6 +112,7 @@ public Adapter(AdapterGenerator generator, String description, int frameSize, by
         final TargetBundleLayout targetBundleLayout = new TargetBundleLayout(0, 0, code.length);
         Code.allocate(targetBundleLayout, this);
         setData(null, null, code);
+        maybeCleanCache();
         setSafepoints(new Safepoints(Safepoints.make(Safepoints.safepointPosForCall(callPos, callSize), callPos, INDIRECT_CALL)), TargetMethod.NO_DIRECT_CALLEES);
     }
 

com.sun.max/src/com/sun/max/vm/compiler/target/Stub.java

@@ -180,9 +180,8 @@ public Stub(Type type, String stubName, int frameSize, byte[] code, int callPos,
         }
         if (!isHosted()) {
             linkDirectCalls();
-            if (platform().target.arch.isARM() || platform().target.arch.isAarch64() || platform().target.arch.isRISCV64()) {
-                MaxineVM.maxine_cache_flush(codeStart().toPointer(), code().length);
-            }
+            // Perform cache maintenance after linking calls to ensure visibility of fixed call-sites.
+            maybeCleanCache();
         }
     }
 
@@ -200,6 +199,7 @@ public Stub(Type type, String name, CiTargetMethod tm) {
         for (CiDebugInfo info : debugInfos) {
             assert info == null;
         }
+        maybeCleanCache();
     }
 
     @Override

com.sun.max/src/com/sun/max/vm/compiler/target/TargetMethod.java

@@ -23,6 +23,7 @@
 
 import static com.sun.max.platform.Platform.*;
 import static com.sun.max.vm.MaxineVM.*;
+import static com.sun.max.vm.VMOptions.addFieldOption;
 import static com.sun.max.vm.compiler.target.Safepoints.*;
 
 import java.io.*;
@@ -95,6 +96,19 @@ public interface CodePosClosure {
         boolean doCodePos(ClassMethodActor method, int bci);
     }
 
+    static boolean UseSystemMembarrier = true;
+
+    static boolean UseNonMandatedSystemMembarrier = true;
+
+    static {
+        addFieldOption("-XX:", "UseSystemMembarrier", TargetMethod.class, "Use the membarrier system call after cache maintenance"
+                + " operations to guarantee instruction stream synchronisation with concurrent threads (Linux).");
+
+        addFieldOption("-XX:", "UseNonMandatedSystemMembarrier", TargetMethod.class, "Use the membarrier system call after cache"
+                + " maintenance operations to guarantee instruction stream synchronisation with concurrent threads (Linux)"
+                + " in cases that aren't mandated by the architecture e.g. patching the address operand in a direct branch (Aarch64).");
+    }
+
     /**
      * The (bytecode) method from which this target method was compiled. This will be {@code null} iff this target
      * method is a {@link Stub} or and {@link Adapter}.
@@ -884,23 +898,30 @@ protected final void setData(byte[] scalarLiterals, Object[] objectLiterals, byt
         if (trampolines != null && trampolines.length > 0) {
             System.arraycopy(trampolines, 0, this.trampolines, 0, this.trampolinesLength());
         }
-        cleanCache();
     }
 
     protected final void setData(byte[] scalarLiterals, Object[] objectLiterals, byte[] codeBuffer) {
         setData(scalarLiterals, objectLiterals, codeBuffer, null);
     }
 
-    public void cleanCache() {
+    /**
+     * For architectures with weak or relaxed memory models.
+     * Concrete sub-classes of TargetMethod should call maybeCleanCache() after installing code and fixing
+     * up any call-sites that need to be patched.
+     */
+    public void maybeCleanCache() {
         if (!MaxineVM.isHosted() && (platform().target.arch.isARM() || platform().target.arch.isAarch64() || platform().target.arch.isRISCV64())) {
-            int codePreAmble = 0;
-            if (scalarLiterals != null) {
-                codePreAmble = scalarLiterals.length;
-            }
-            if (referenceLiterals != null) {
-                codePreAmble = codePreAmble + (referenceLiterals.length * 4);
+            long longSize = size().toLong();
+            int size = (int) longSize;
+            assert size == longSize : "Integer overflow";
+            MaxineVM.maxine_cache_flush(start, size);
+            /*
+             * Aarch64 requires an ISB instruction is executed on concurrently executing CPUs to discard
+             * speculatively pre-fetched addresses from buffers. See B2.2.5 ARM ARM (issue E.a).
+             */
+            if (UseSystemMembarrier && platform().target.arch.isAarch64()) {
+                MaxineVM.syscall_membarrier();
             }
-            MaxineVM.maxine_cache_flush(CodePointer.from(codeStart.minus(codePreAmble)).toPointer(), codeLength() + codePreAmble);
         }
     }
 
@@ -1175,7 +1196,7 @@ public String name() {
 
     /**
      * Fixup a call site in the method. This differs from the above in that the call site is updated before any thread
-     * can see it. Thus there isn't any concurrency between modifying the call site and threads trying to run it.
+     * can execute it. Thus there isn't any concurrency between modifying the call site and threads trying to run it.
      *
      * @param callOffset offset to a call site relative to the start of the code of this target method
      * @param callEntryPoint entry point the call site should call after fixup
@@ -1434,6 +1455,21 @@ public final void unmark() {
         oldStart = Address.zero();
     }
 
+    /**
+     * Query the UseSystemMembarrier runtime option.
+     * @return
+     */
+    public static final boolean useSystemMembarrier() {
+        return UseSystemMembarrier;
+    }
+
+    /**
+     * Query the UseNonMandatedSystemMembarrier runtime option.
+     * @return
+     */
+    public static final boolean useNonMandatedSystemMembarrier() {
+        return UseNonMandatedSystemMembarrier;
+    }
     /**
      * Determines if this method was marked to survive {@linkplain CodeEviction code eviction}.
      */