[JSC] Make CachedCall further faster

https://bugs.webkit.org/show_bug.cgi?id=264726 rdar://118320685 Reviewed by Michael Saboff. 1. We should not do vm.disallowVMEntryCount for every CachedCall calls, which is really slow. We should do it once. This patch moves vm.disallowVMEntryCount check to the same place to VMEntryScope. 2. Extract stack overflow check as _llint_throw_stack_overflow_error_from_vm_entry and jump to this label. 3. Move some of hot VM fields to the consolidated place. 4. Use paired load / store for VM frame initializations. cpp-to-js-cached-call 21.1540+-0.0364 ^ 15.0915+-0.0090 ^ definitely 1.4017x faster * Source/JavaScriptCore/interpreter/CachedCall.h: (JSC::CachedCall::CachedCall): * Source/JavaScriptCore/interpreter/CallFrame.h: (JSC::CallFrame::noCaller): * Source/JavaScriptCore/interpreter/Interpreter.cpp: (JSC::Interpreter::checkVMEntryPermission): (JSC::Interpreter::executeProgram): (JSC::Interpreter::executeCallImpl): (JSC::Interpreter::executeConstruct): (JSC::Interpreter::executeEval): (JSC::Interpreter::executeModuleProgram): * Source/JavaScriptCore/interpreter/Interpreter.h: * Source/JavaScriptCore/llint/LLIntSlowPaths.cpp: (JSC::LLInt::llint_check_vm_entry_permission): * Source/JavaScriptCore/llint/LowLevelInterpreter32_64.asm: * Source/JavaScriptCore/llint/LowLevelInterpreter64.asm: * Source/JavaScriptCore/runtime/VM.cpp: (JSC::VM::VM): * Source/JavaScriptCore/runtime/VM.h: Canonical link: https://commits.webkit.org/270649@main
WebKit · Nov 13, 2023 · 0125fc7 · 0125fc7
1 parent 893f4b2
commit 0125fc7
Show file tree

Hide file tree

Showing 9 changed files with 107 additions and 85 deletions.
diff --git a/Source/JavaScriptCore/interpreter/CachedCall.h b/Source/JavaScriptCore/interpreter/CachedCall.h
@@ -61,6 +61,12 @@ class CachedCall : public CallLinkInfoBase {
             return;
         }
 
+        if (UNLIKELY(vm.disallowVMEntryCount)) {
+            Interpreter::checkVMEntryPermission();
+            throwStackOverflowError(globalObject, scope);
+            return;
+        }
+
         m_arguments.ensureCapacity(argumentCount);
         if (UNLIKELY(m_arguments.hasOverflowed())) {
             throwOutOfMemoryError(globalObject, scope);

diff --git a/Source/JavaScriptCore/interpreter/CallFrame.h b/Source/JavaScriptCore/interpreter/CallFrame.h
@@ -309,7 +309,7 @@ using JSInstruction = BaseInstruction<JSOpcodeTraits>;
 
         static int offsetFor(size_t argumentCountIncludingThis) { return CallFrameSlot::thisArgument + argumentCountIncludingThis - 1; }
 
-        static CallFrame* noCaller() { return nullptr; }
+        static constexpr CallFrame* noCaller() { return nullptr; }
 
         bool isEmptyTopLevelCallFrameForDebugger() const
         {

diff --git a/Source/JavaScriptCore/interpreter/Interpreter.cpp b/Source/JavaScriptCore/interpreter/Interpreter.cpp
@@ -862,6 +862,13 @@ void Interpreter::notifyDebuggerOfExceptionToBeThrown(VM& vm, JSGlobalObject* gl
     exception->setDidNotifyInspectorOfThrow();
 }
 
+NEVER_INLINE JSValue Interpreter::checkVMEntryPermission()
+{
+    if (Options::crashOnDisallowedVMEntry() || g_jscConfig.vmEntryDisallowed)
+        CRASH_WITH_EXTRA_SECURITY_IMPLICATION_AND_INFO(VMEntryDisallowed, "VM entry disallowed"_s);
+    return jsUndefined();
+}
+
 JSValue Interpreter::executeProgram(const SourceCode& source, JSGlobalObject*, JSObject* thisObj)
 {
     VM& vm = this->vm();
@@ -886,6 +893,9 @@ JSValue Interpreter::executeProgram(const SourceCode& source, JSGlobalObject*, J
     if (UNLIKELY(!vm.isSafeToRecurseSoft()))
         return throwStackOverflowError(globalObject, throwScope);
 
+    if (UNLIKELY(vm.disallowVMEntryCount))
+        return checkVMEntryPermission();
+
     // First check if the "program" is actually just a JSON object. If so,
     // we'll handle the JSON object here. Else, we'll handle real JS code
     // below at failedJSONP.
@@ -1132,6 +1142,9 @@ ALWAYS_INLINE JSValue Interpreter::executeCallImpl(VM& vm, JSObject* function, c
     if (UNLIKELY(!vm.isSafeToRecurseSoft() || args.size() > maxArguments))
         return throwStackOverflowError(globalObject, scope);
 
+    if (UNLIKELY(vm.disallowVMEntryCount))
+        return checkVMEntryPermission();
+
     if (UNLIKELY(vm.traps().needHandling(VMTraps::NonDebuggerAsyncEvents))) {
         if (vm.hasExceptionsAfterHandlingTraps())
             return scope.exception();
@@ -1220,6 +1233,11 @@ JSObject* Interpreter::executeConstruct(JSObject* constructor, const CallData& c
         return nullptr;
     }
 
+    if (UNLIKELY(vm.disallowVMEntryCount)) {
+        checkVMEntryPermission();
+        return globalObject->globalThis();
+    }
+
     if (UNLIKELY(vm.traps().needHandling(VMTraps::NonDebuggerAsyncEvents))) {
         if (vm.hasExceptionsAfterHandlingTraps())
             return nullptr;
@@ -1299,6 +1317,9 @@ JSValue Interpreter::executeEval(EvalExecutable* eval, JSValue thisValue, JSScop
     if (UNLIKELY(!vm.isSafeToRecurseSoft()))
         return throwStackOverflowError(globalObject, throwScope);
 
+    if (UNLIKELY(vm.disallowVMEntryCount))
+        return checkVMEntryPermission();
+
     unsigned numVariables = eval->numVariables();
     unsigned numTopLevelFunctionDecls = eval->numTopLevelFunctionDecls();
     unsigned numFunctionHoistingCandidates = eval->numFunctionHoistingCandidates();
@@ -1487,6 +1508,9 @@ JSValue Interpreter::executeModuleProgram(JSModuleRecord* record, ModuleProgramE
     if (UNLIKELY(!vm.isSafeToRecurseSoft()))
         return throwStackOverflowError(globalObject, throwScope);
 
+    if (UNLIKELY(vm.disallowVMEntryCount))
+        return checkVMEntryPermission();
+
     if (UNLIKELY(vm.traps().needHandling(VMTraps::NonDebuggerAsyncEvents))) {
         if (vm.hasExceptionsAfterHandlingTraps())
             return throwScope.exception();

diff --git a/Source/JavaScriptCore/interpreter/Interpreter.h b/Source/JavaScriptCore/interpreter/Interpreter.h
@@ -158,6 +158,8 @@ using JSOrWasmInstruction = std::variant<const JSInstruction*, const WasmInstruc
 
         void getStackTrace(JSCell* owner, Vector<StackFrame>& results, size_t framesToSkip = 0, size_t maxStackSize = std::numeric_limits<size_t>::max(), JSCell* caller = nullptr);
 
+        static JSValue checkVMEntryPermission();
+
     private:
         enum ExecutionFlag { Normal, InitializeAndReturn };
 

diff --git a/Source/JavaScriptCore/llint/LLIntSlowPaths.cpp b/Source/JavaScriptCore/llint/LLIntSlowPaths.cpp
@@ -2622,10 +2622,7 @@ extern "C" void llint_write_barrier_slow(CallFrame* callFrame, JSCell* cell)
 
 extern "C" UGPRPair llint_check_vm_entry_permission(VM*, ProtoCallFrame*)
 {
-    if (Options::crashOnDisallowedVMEntry() || g_jscConfig.vmEntryDisallowed)
-        CRASH_WITH_EXTRA_SECURITY_IMPLICATION_AND_INFO(VMEntryDisallowed, "VM entry disallowed"_s);
-
-    // Else return, and let doVMEntry return undefined.
+    Interpreter::checkVMEntryPermission();
     return encodeResult(nullptr, nullptr);
 }
 

diff --git a/Source/JavaScriptCore/llint/LowLevelInterpreter32_64.asm b/Source/JavaScriptCore/llint/LowLevelInterpreter32_64.asm
@@ -229,9 +229,6 @@ macro doVMEntry(makeCall)
     # Since we have the guarantee that tX != aY when X != Y, we are safe from
     # aliasing problems with our arguments.
 
-    loadi VM::disallowVMEntryCount[vm], t4
-    btinz t4, .checkVMEntryPermission
-
     if ARMv7
         vmEntryRecord(cfr, t3)
         move t3, sp
@@ -263,7 +260,7 @@ macro doVMEntry(makeCall)
     addp CallFrameHeaderSlots, t4, t4
     lshiftp 3, t4
     subp sp, t4, t3
-    bpa t3, sp, .throwStackOverflow
+    bpa t3, sp, _llint_throw_stack_overflow_error_from_vm_entry
 
     # Ensure that we have enough additional stack capacity for the incoming args,
     # and the frame for the JS code we're executing. We need to do this check
@@ -281,9 +278,9 @@ macro doVMEntry(makeCall)
 .stackCheckFailed:
         move t4, entry
         move t5, vm
-        jmp .throwStackOverflow
+        jmp _llint_throw_stack_overflow_error_from_vm_entry
     else
-        bpb t3, VM::m_softStackLimit[vm], .throwStackOverflow
+        bpb t3, VM::m_softStackLimit[vm], _llint_throw_stack_overflow_error_from_vm_entry
     end
 
 .stackHeightOK:
@@ -351,8 +348,13 @@ macro doVMEntry(makeCall)
     popCalleeSaves()
     functionEpilogue()
     ret
+end
+
+_llint_throw_stack_overflow_error_from_vm_entry:
+    const entry = a0
+    const vm = a1
+    const protoCallFrame = a2
 
-.throwStackOverflow:
     subp 8, sp # Align stack for cCall2() to make a call.
     move vm, a0
     move protoCallFrame, a1
@@ -386,26 +388,6 @@ macro doVMEntry(makeCall)
     functionEpilogue()
     ret
 
-.checkVMEntryPermission:
-    move vm, a0
-    move protoCallFrame, a1
-    cCall2(_llint_check_vm_entry_permission)
-
-    # Tag is stored in r1 and payload is stored in r0 in little-endian architectures.
-    move UndefinedTag, r1
-    move 0, r0
-
-    if ARMv7
-        subp cfr, CalleeRegisterSaveSize, t3
-        move t3, sp
-    else
-        subp cfr, CalleeRegisterSaveSize, sp
-    end
-    popCalleeSaves()
-    functionEpilogue()
-    ret
-end
-
 # a0, a2, t3, t4
 macro makeJavaScriptCall(entry, protoCallFrame, temp1, temp2)
     addp CallerFrameAndPCSize, sp

diff --git a/Source/JavaScriptCore/llint/LowLevelInterpreter64.asm b/Source/JavaScriptCore/llint/LowLevelInterpreter64.asm
@@ -240,22 +240,24 @@ macro doVMEntry(makeCall)
 
     checkStackPointerAlignment(t4, 0xbad0dc01)
 
-    loadi VM::disallowVMEntryCount[vm], t4
-    btinz t4, .checkVMEntryPermission
-
     storep vm, VMEntryRecord::m_vm[sp]
-    loadp VM::topCallFrame[vm], t4
-    storep t4, VMEntryRecord::m_prevTopCallFrame[sp]
-    loadp VM::topEntryFrame[vm], t4
-    storep t4, VMEntryRecord::m_prevTopEntryFrame[sp]
+    if ARM64 or ARM64E
+        loadpairq VM::topCallFrame[vm], t4, t3
+        storepairq t4, t3, VMEntryRecord::m_prevTopCallFrame[sp]
+    else
+        loadp VM::topCallFrame[vm], t4
+        storep t4, VMEntryRecord::m_prevTopCallFrame[sp]
+        loadp VM::topEntryFrame[vm], t4
+        storep t4, VMEntryRecord::m_prevTopEntryFrame[sp]
+    end
     loadp ProtoCallFrame::calleeValue[protoCallFrame], t4
     storep t4, VMEntryRecord::m_callee[sp]
 
     loadi ProtoCallFrame::paddedArgCount[protoCallFrame], t4
     addp CallFrameHeaderSlots, t4, t4
     lshiftp 3, t4
     subp sp, t4, t3
-    bqbeq sp, t3, .throwStackOverflow
+    bqbeq sp, t3, _llint_throw_stack_overflow_error_from_vm_entry
 
     # Ensure that we have enough additional stack capacity for the incoming args,
     # and the frame for the JS code we're executing. We need to do this check
@@ -273,21 +275,31 @@ macro doVMEntry(makeCall)
 .stackCheckFailed:
         move t4, entry
         move t5, vm
-        jmp .throwStackOverflow
+        jmp  _llint_throw_stack_overflow_error_from_vm_entry
+.stackHeightOK:
+        move t3, sp
     else
-        bpb t3, VM::m_softStackLimit[vm], .throwStackOverflow
+        bpb t3, VM::m_softStackLimit[vm],  _llint_throw_stack_overflow_error_from_vm_entry
+        move t3, sp
     end
 
-.stackHeightOK:
-    move t3, sp
-    move (constexpr ProtoCallFrame::numberOfRegisters), t3
-
 .copyHeaderLoop:
     # Copy the CodeBlock/Callee/ArgumentCountIncludingThis/|this| from protoCallFrame into the callee frame.
-    subi 1, t3
-    loadq [protoCallFrame, t3, 8], extraTempReg
-    storeq extraTempReg, CodeBlock[sp, t3, 8]
-    btinz t3, .copyHeaderLoop
+    if ARM64 or ARM64E
+        loadpairq (8 * 0)[protoCallFrame], extraTempReg, t3
+        storepairq extraTempReg, t3, CodeBlock + (8 * 0)[sp]
+        loadpairq (8 * 2)[protoCallFrame], extraTempReg, t3
+        storepairq extraTempReg, t3, CodeBlock + (8 * 2)[sp]
+    else
+        loadq (8 * 0)[protoCallFrame], extraTempReg
+        storeq extraTempReg, CodeBlock + (8 * 0)[sp]
+        loadq (8 * 1)[protoCallFrame], extraTempReg
+        storeq extraTempReg, CodeBlock + (8 * 1)[sp]
+        loadq (8 * 2)[protoCallFrame], extraTempReg
+        storeq extraTempReg, CodeBlock + (8 * 2)[sp]
+        loadq (8 * 3)[protoCallFrame], extraTempReg
+        storeq extraTempReg, CodeBlock + (8 * 3)[sp]
+    end
 
     loadi PayloadOffset + ProtoCallFrame::argCountAndCodeOriginValue[protoCallFrame], t4
     subi 1, t4
@@ -314,11 +326,11 @@ macro doVMEntry(makeCall)
 .copyArgsDone:
     if ARM64 or ARM64E
         move sp, t4
-        storep t4, VM::topCallFrame[vm]
+        storepairq t4, cfr, VM::topCallFrame[vm]
     else
         storep sp, VM::topCallFrame[vm]
+        storep cfr, VM::topEntryFrame[vm]
     end
-    storep cfr, VM::topEntryFrame[vm]
 
     checkStackPointerAlignment(extraTempReg, 0xbad0dc02)
 
@@ -332,18 +344,28 @@ macro doVMEntry(makeCall)
     vmEntryRecord(cfr, t4)
 
     loadp VMEntryRecord::m_vm[t4], vm
-    loadp VMEntryRecord::m_prevTopCallFrame[t4], t2
-    storep t2, VM::topCallFrame[vm]
-    loadp VMEntryRecord::m_prevTopEntryFrame[t4], t2
-    storep t2, VM::topEntryFrame[vm]
+    if ARM64 or ARM64E
+        loadpairq VMEntryRecord::m_prevTopCallFrame[t4], t4, t2
+        storepairq t4, t2, VM::topCallFrame[vm]
+    else
+        loadp VMEntryRecord::m_prevTopCallFrame[t4], t2
+        storep t2, VM::topCallFrame[vm]
+        loadp VMEntryRecord::m_prevTopEntryFrame[t4], t2
+        storep t2, VM::topEntryFrame[vm]
+    end
 
     subp cfr, CalleeRegisterSaveSize, sp
 
     popCalleeSaves()
     functionEpilogue()
     ret
+end
+
+_llint_throw_stack_overflow_error_from_vm_entry:
+    const entry = a0
+    const vm = a1
+    const protoCallFrame = a2
 
-.throwStackOverflow:
     move vm, a0
     move protoCallFrame, a1
     cCall2(_llint_throw_stack_overflow_error)
@@ -363,18 +385,6 @@ macro doVMEntry(makeCall)
     functionEpilogue()
     ret
 
-.checkVMEntryPermission:
-    move vm, a0
-    move protoCallFrame, a1
-    cCall2(_llint_check_vm_entry_permission)
-    move ValueUndefined, r0
-
-    subp cfr, CalleeRegisterSaveSize, sp
-    popCalleeSaves()
-    functionEpilogue()
-    ret
-end
-
 # a0, a2, t3, t4
 macro makeJavaScriptCall(entry, protoCallFrame, temp1, temp2)
     addp 16, sp

diff --git a/Source/JavaScriptCore/runtime/VM.cpp b/Source/JavaScriptCore/runtime/VM.cpp
@@ -211,7 +211,8 @@ void VM::computeCanUseJIT()
 static bool vmCreationShouldCrash = false;
 
 VM::VM(VMType vmType, HeapType heapType, WTF::RunLoop* runLoop, bool* success)
-    : m_identifier(VMIdentifier::generate())
+    : topCallFrame(CallFrame::noCaller())
+    , m_identifier(VMIdentifier::generate())
     , m_apiLock(adoptRef(new JSLock(this)))
     , m_runLoop(runLoop ? *runLoop : WTF::RunLoop::current())
     , m_random(Options::seedOfVMRandomForFuzzer() ? Options::seedOfVMRandomForFuzzer() : cryptographicallyRandomNumber<uint32_t>())
@@ -220,7 +221,6 @@ VM::VM(VMType vmType, HeapType heapType, WTF::RunLoop* runLoop, bool* success)
     , heap(*this, heapType)
     , clientHeap(heap)
     , vmType(vmType)
-    , topCallFrame(CallFrame::noCaller())
     , deferredWorkTimer(DeferredWorkTimer::create(*this))
     , m_atomStringTable(vmType == Default ? Thread::current().atomStringTable() : new AtomStringTable)
     , emptyList(new ArgList)