Bug 1777604 - wasm: Perform a pipeline flush while creating a module …

…object. r=nbp, a=RyanVM

Differential Revision: https://phabricator.services.mozilla.com/D152304

js/src/jit/FlushICache.h

@@ -40,6 +40,33 @@ inline void FlushICache(void* code, size_t size,
 #  error "Unknown architecture!"
 #endif
 
+#if (defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)) ||       \
+    (defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)) || \
+    defined(JS_CODEGEN_LOONG64)
+
+inline void FlushExecutionContext() {
+  // No-op. Execution context is coherent with instruction cache.
+}
+
+#elif defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_WASM32)
+
+inline void FlushExecutionContext() { MOZ_CRASH(); }
+
+#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64)
+
+// ARM and ARM64 must flush the instruction pipeline of the current core
+// before executing newly JIT'ed code. This will remove any stale data from
+// the pipeline that may have referenced invalidated instructions.
+//
+// `FlushICache` will perform this for the thread that compiles the code, but
+// other threads that may execute the code are responsible to call
+// this method.
+extern void FlushExecutionContext();
+
+#else
+#  error "Unknown architecture!"
+#endif
+
 }  // namespace jit
 }  // namespace js
 

js/src/jit/arm/Architecture-arm.cpp

@@ -527,5 +527,15 @@ void FlushICache(void* code, size_t size, bool codeIsThreadLocal) {
 #endif
 }
 
+void FlushExecutionContext() {
+#ifndef JS_SIMULATOR_ARM
+  // Ensure that any instructions already in the pipeline are discarded and
+  // reloaded from the icache.
+  asm volatile("isb\n" : : : "memory");
+#else
+  // We assume the icache flushing routines on other platforms take care of this
+#endif
+}
+
 }  // namespace jit
 }  // namespace js

js/src/jit/arm64/Architecture-arm64.cpp

@@ -127,5 +127,7 @@ bool CanFlushICacheFromBackgroundThreads() {
   return vixl::CPU::CanFlushICacheFromBackgroundThreads();
 }
 
+void FlushExecutionContext() { vixl::CPU::FlushExecutionContext(); }
+
 }  // namespace jit
 }  // namespace js

js/src/jit/arm64/vixl/Cpu-vixl.h

@@ -171,6 +171,10 @@ class CPU {
   // cache on a background thread.
   static bool CanFlushICacheFromBackgroundThreads();
 
+  // Flush the local instruction pipeline, forcing a reload of any instructions
+  // beyond this barrier from the icache.
+  static void FlushExecutionContext();
+
   // Read and interpret the ID registers. This requires
   // CPUFeatures::kIDRegisterEmulation, and therefore cannot be called on
   // non-AArch64 platforms.

js/src/jit/arm64/vixl/MozCpu-vixl.cpp

@@ -294,4 +294,21 @@ void CPU::EnsureIAndDCacheCoherency(void *address, size_t length, bool codeIsThr
 #endif
 }
 
+void CPU::FlushExecutionContext() {
+#if defined(JS_SIMULATOR_ARM64) && defined(JS_CACHE_SIMULATOR_ARM64)
+  // Performing an 'isb' will ensure the current core instruction pipeline is
+  // synchronized with an icache flush executed by another core.
+  using js::jit::SimulatorProcess;
+  js::jit::AutoLockSimulatorCache alsc;
+  Simulator* sim = vixl::Simulator::Current();
+  if (sim) {
+    sim->FlushICache();
+  }
+#elif defined(__aarch64__)
+  // Ensure that any instructions already in the pipeline are discarded and
+  // reloaded from the icache.
+  __asm__ __volatile__("isb\n" : : : "memory");
+#endif
+}
+
 }  // namespace vixl

js/src/wasm/WasmJS.cpp

@@ -31,6 +31,7 @@
 #include "ds/IdValuePair.h"  // js::IdValuePair
 #include "gc/GCContext.h"
 #include "jit/AtomicOperations.h"
+#include "jit/FlushICache.h"
 #include "jit/JitContext.h"
 #include "jit/JitOptions.h"
 #include "jit/Simulator.h"
@@ -1727,6 +1728,16 @@ WasmModuleObject* WasmModuleObject::create(JSContext* cx, const Module& module,
     return nullptr;
   }
 
+  // The pipeline state on some architectures may retain stale instructions
+  // even after we invalidate the instruction cache. There is no generally
+  // available method to broadcast this pipeline flush to all threads after
+  // we've compiled new code, so conservatively perform one here when we're
+  // receiving a module that may have been compiled from another thread.
+  //
+  // The cost of this flush is expected to minimal enough to not be worth
+  // optimizing away in the case the module was compiled on this thread.
+  jit::FlushExecutionContext();
+
   // This accounts for module allocation size (excluding code which is handled
   // separately - see below). This assumes that the size of associated data
   // doesn't change for the life of the WasmModuleObject. The size is counted