Cherry-pick b13bf4c. rdar://121083665

    [JSC] Introduce InBy megamorphic ICs
    https://bugs.webkit.org/show_bug.cgi?id=269255
    rdar://121083665

    Reviewed by Justin Michaud.

    This patch adds InBy megamorphic IC in all tiers. This is super similar to GetBy megamorphic IC.
    Similar to GetBy megamorphic IC, we store structure/uid pair and the result. And when prototype objects
    get changed, we bump the epoch (it is already done), and then all cache gets invalidated.

    The only conceptual difference is that we can say `true` for custom accessor gets found. This is not
    possible for GetBy megamorphic IC, but for InBy IC, it is OK since it is only asking whether it exists or not.

    * Source/JavaScriptCore/bytecode/AccessCase.cpp:
    (JSC::AccessCase::create):
    (JSC::AccessCase::guardedByStructureCheckSkippingConstantIdentifierCheck const):
    (JSC::AccessCase::requiresIdentifierNameMatch const):
    (JSC::AccessCase::requiresInt32PropertyCheck const):
    (JSC::AccessCase::forEachDependentCell const):
    (JSC::AccessCase::doesCalls const):
    (JSC::AccessCase::canReplace const):
    (JSC::AccessCase::runWithDowncast):
    (JSC::AccessCase::canBeShared):
    * Source/JavaScriptCore/bytecode/AccessCase.h:
    * Source/JavaScriptCore/bytecode/InByStatus.cpp:
    (JSC::InByStatus::computeFor):
    (JSC::InByStatus::computeForStubInfo):
    (JSC::InByStatus::computeForStubInfoWithoutExitSiteFeedback):
    (JSC::InByStatus::merge):
    (JSC::InByStatus::dump const):
    * Source/JavaScriptCore/bytecode/InByStatus.h:
    * Source/JavaScriptCore/bytecode/InlineCacheCompiler.cpp:
    (JSC::needsScratchFPR):
    (JSC::forInBy):
    (JSC::InlineCacheCompiler::generateWithGuard):
    (JSC::InlineCacheCompiler::generateImpl):
    (JSC::InlineCacheCompiler::regenerate):
    * Source/JavaScriptCore/bytecode/InlineCacheCompiler.h:
    (JSC::canUseMegamorphicInById):
    * Source/JavaScriptCore/bytecode/Repatch.cpp:
    (JSC::repatchInBySlowPathCall):
    (JSC::tryCacheInBy):
    (JSC::repatchInBy):
    * Source/JavaScriptCore/bytecode/Repatch.h:
    * Source/JavaScriptCore/bytecode/StructureStubInfo.cpp:
    (JSC::StructureStubInfo::summary const):
    (JSC::SharedJITStubSet::getMegamorphic const):
    (JSC::SharedJITStubSet::setMegamorphic):
    * Source/JavaScriptCore/bytecode/StructureStubInfo.h:
    * Source/JavaScriptCore/dfg/DFGAbstractInterpreterInlines.h:
    (JSC::DFG::AbstractInterpreter<AbstractStateType>::executeEffects):
    * Source/JavaScriptCore/dfg/DFGByteCodeParser.cpp:
    (JSC::DFG::ByteCodeParser::handleInById):
    (JSC::DFG::ByteCodeParser::parseBlock):
    * Source/JavaScriptCore/dfg/DFGClobberize.h:
    (JSC::DFG::clobberize):
    * Source/JavaScriptCore/dfg/DFGConstantFoldingPhase.cpp:
    (JSC::DFG::ConstantFoldingPhase::foldConstants):
    * Source/JavaScriptCore/dfg/DFGDoesGC.cpp:
    (JSC::DFG::doesGC):
    * Source/JavaScriptCore/dfg/DFGFixupPhase.cpp:
    (JSC::DFG::FixupPhase::fixupNode):
    * Source/JavaScriptCore/dfg/DFGNode.cpp:
    (JSC::DFG::Node::convertToInByIdMaybeMegamorphic):
    * Source/JavaScriptCore/dfg/DFGNode.h:
    (JSC::DFG::Node::hasCacheableIdentifier):
    (JSC::DFG::Node::cacheableIdentifier):
    (JSC::DFG::Node::hasArrayMode):
    (JSC::DFG::Node::convertToInById): Deleted.
    * Source/JavaScriptCore/dfg/DFGNodeType.h:
    * Source/JavaScriptCore/dfg/DFGPredictionPropagationPhase.cpp:
    * Source/JavaScriptCore/dfg/DFGSafeToExecute.h:
    (JSC::DFG::safeToExecute):
    * Source/JavaScriptCore/dfg/DFGSpeculativeJIT.h:
    * Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp:
    (JSC::DFG::SpeculativeJIT::compile):
    * Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp:
    (JSC::DFG::SpeculativeJIT::compile):
    (JSC::DFG::SpeculativeJIT::compileInByIdMegamorphic):
    (JSC::DFG::SpeculativeJIT::compileInByValMegamorphic):
    * Source/JavaScriptCore/dfg/DFGStrengthReductionPhase.cpp:
    (JSC::DFG::StrengthReductionPhase::handleNode):
    * Source/JavaScriptCore/ftl/FTLCapabilities.cpp:
    (JSC::FTL::canCompile):
    * Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp:
    (JSC::FTL::DFG::LowerDFGToB3::compileNode):
    (JSC::FTL::DFG::LowerDFGToB3::compileCompareStrictEq):
    * Source/JavaScriptCore/jit/AssemblyHelpers.cpp:
    (JSC::AssemblyHelpers::hasMegamorphicProperty):
    * Source/JavaScriptCore/jit/AssemblyHelpers.h:
    * Source/JavaScriptCore/jit/JITOperations.cpp:
    (JSC::inByIdMegamorphic):
    (JSC::JSC_DEFINE_JIT_OPERATION):
    (JSC::inByValMegamorphic):
    * Source/JavaScriptCore/jit/JITOperations.h:
    * Source/JavaScriptCore/runtime/MegamorphicCache.cpp:
    (JSC::MegamorphicCache::age):
    (JSC::MegamorphicCache::clearEntries):
    * Source/JavaScriptCore/runtime/MegamorphicCache.h:
    (JSC::MegamorphicCache::HasEntry::offsetOfUid):
    (JSC::MegamorphicCache::HasEntry::offsetOfStructureID):
    (JSC::MegamorphicCache::HasEntry::offsetOfEpoch):
    (JSC::MegamorphicCache::HasEntry::offsetOfResult):
    (JSC::MegamorphicCache::HasEntry::init):
    (JSC::MegamorphicCache::offsetOfHasCachePrimaryEntries):
    (JSC::MegamorphicCache::offsetOfHasCacheSecondaryEntries):
    (JSC::MegamorphicCache::hasCachePrimaryHash):
    (JSC::MegamorphicCache::hasCacheSecondaryHash):
    (JSC::MegamorphicCache::initAsHasHit):
    (JSC::MegamorphicCache::initAsHasMiss):

    Canonical link: https://commits.webkit.org/274687@main

Identifier: 272448.611@safari-7618-branch

Source/JavaScriptCore/bytecode/AccessCase.cpp

@@ -70,6 +70,7 @@ Ref<AccessCase> AccessCase::create(VM& vm, JSCell* owner, AccessType type, Cache
     switch (type) {
     case LoadMegamorphic:
     case StoreMegamorphic:
+    case InMegamorphic:
     case InHit:
     case InMiss:
     case DeleteNonConfigurable:
@@ -88,6 +89,7 @@ Ref<AccessCase> AccessCase::create(VM& vm, JSCell* owner, AccessType type, Cache
     case IndexedProxyObjectLoad:
     case IndexedMegamorphicLoad:
     case IndexedMegamorphicStore:
+    case IndexedMegamorphicIn:
     case IndexedInt32Load:
     case IndexedDoubleLoad:
     case IndexedContiguousLoad:
@@ -365,6 +367,7 @@ bool AccessCase::guardedByStructureCheckSkippingConstantIdentifierCheck() const
     switch (m_type) {
     case LoadMegamorphic:
     case StoreMegamorphic:
+    case InMegamorphic:
     case ArrayLength:
     case StringLength:
     case DirectArgumentsLength:
@@ -379,6 +382,7 @@ bool AccessCase::guardedByStructureCheckSkippingConstantIdentifierCheck() const
     case IndexedProxyObjectLoad:
     case IndexedMegamorphicLoad:
     case IndexedMegamorphicStore:
+    case IndexedMegamorphicIn:
     case IndexedInt32Load:
     case IndexedDoubleLoad:
     case IndexedContiguousLoad:
@@ -484,6 +488,7 @@ bool AccessCase::requiresIdentifierNameMatch() const
     case Load:
     case LoadMegamorphic:
     case StoreMegamorphic:
+    case InMegamorphic:
     // We don't currently have a by_val for these puts, but we do care about the identifier.
     case Transition:
     case Delete:
@@ -518,6 +523,7 @@ bool AccessCase::requiresIdentifierNameMatch() const
     case IndexedProxyObjectLoad:
     case IndexedMegamorphicLoad:
     case IndexedMegamorphicStore:
+    case IndexedMegamorphicIn:
     case IndexedInt32Load:
     case IndexedDoubleLoad:
     case IndexedContiguousLoad:
@@ -603,6 +609,7 @@ bool AccessCase::requiresInt32PropertyCheck() const
     case Load:
     case LoadMegamorphic:
     case StoreMegamorphic:
+    case InMegamorphic:
     case Transition:
     case Delete:
     case DeleteNonConfigurable:
@@ -635,6 +642,7 @@ bool AccessCase::requiresInt32PropertyCheck() const
     case IndexedProxyObjectLoad:
     case IndexedMegamorphicLoad:
     case IndexedMegamorphicStore:
+    case IndexedMegamorphicIn:
         return false;
     case IndexedInt32Load:
     case IndexedDoubleLoad:
@@ -774,6 +782,7 @@ void AccessCase::forEachDependentCell(VM&, const Functor& functor) const
     case Load:
     case LoadMegamorphic:
     case StoreMegamorphic:
+    case InMegamorphic:
     case Transition:
     case Delete:
     case DeleteNonConfigurable:
@@ -792,6 +801,7 @@ void AccessCase::forEachDependentCell(VM&, const Functor& functor) const
     case InstanceOfGeneric:
     case IndexedMegamorphicLoad:
     case IndexedMegamorphicStore:
+    case IndexedMegamorphicIn:
     case IndexedInt32Load:
     case IndexedDoubleLoad:
     case IndexedContiguousLoad:
@@ -899,6 +909,7 @@ bool AccessCase::doesCalls(VM& vm, Vector<JSCell*>* cellsToMarkIfDoesCalls) cons
     case Load:
     case LoadMegamorphic:
     case StoreMegamorphic:
+    case InMegamorphic:
     case Miss:
     case GetGetter:
     case InHit:
@@ -915,6 +926,7 @@ bool AccessCase::doesCalls(VM& vm, Vector<JSCell*>* cellsToMarkIfDoesCalls) cons
     case InstanceOfGeneric:
     case IndexedMegamorphicLoad:
     case IndexedMegamorphicStore:
+    case IndexedMegamorphicIn:
     case IndexedInt32Load:
     case IndexedDoubleLoad:
     case IndexedContiguousLoad:
@@ -1052,8 +1064,10 @@ bool AccessCase::canReplace(const AccessCase& other) const
     switch (type()) {
     case LoadMegamorphic:
     case StoreMegamorphic:
+    case InMegamorphic:
     case IndexedMegamorphicLoad:
     case IndexedMegamorphicStore:
+    case IndexedMegamorphicIn:
     case IndexedInt32Load:
     case IndexedDoubleLoad:
     case IndexedContiguousLoad:
@@ -1290,6 +1304,7 @@ inline void AccessCase::runWithDowncast(const Func& func)
     switch (m_type) {
     case LoadMegamorphic:
     case StoreMegamorphic:
+    case InMegamorphic:
     case Transition:
     case Delete:
     case DeleteNonConfigurable:
@@ -1305,6 +1320,7 @@ inline void AccessCase::runWithDowncast(const Func& func)
     case SetPrivateBrand:
     case IndexedMegamorphicLoad:
     case IndexedMegamorphicStore:
+    case IndexedMegamorphicIn:
     case IndexedInt32Load:
     case IndexedDoubleLoad:
     case IndexedContiguousLoad:
@@ -1458,6 +1474,7 @@ bool AccessCase::canBeShared(const AccessCase& lhs, const AccessCase& rhs)
     case Load:
     case LoadMegamorphic:
     case StoreMegamorphic:
+    case InMegamorphic:
     case Transition:
     case Delete:
     case DeleteNonConfigurable:
@@ -1475,6 +1492,7 @@ bool AccessCase::canBeShared(const AccessCase& lhs, const AccessCase& rhs)
     case SetPrivateBrand:
     case IndexedMegamorphicLoad:
     case IndexedMegamorphicStore:
+    case IndexedMegamorphicIn:
     case IndexedInt32Load:
     case IndexedDoubleLoad:
     case IndexedContiguousLoad:

Source/JavaScriptCore/bytecode/AccessCase.h

@@ -112,6 +112,7 @@ DECLARE_ALLOCATOR_WITH_HEAP_IDENTIFIER(AccessCase);
     macro(IntrinsicGetter) \
     macro(InHit) \
     macro(InMiss) \
+    macro(InMegamorphic) \
     macro(ArrayLength) \
     macro(StringLength) \
     macro(DirectArgumentsLength) \
@@ -202,7 +203,7 @@ DECLARE_ALLOCATOR_WITH_HEAP_IDENTIFIER(AccessCase);
     macro(IndexedResizableTypedArrayFloat64InHit) \
     macro(IndexedStringInHit) \
     macro(IndexedNoIndexingInMiss) \
-
+    macro(IndexedMegamorphicIn) \
 
 class AccessCase : public ThreadSafeRefCounted<AccessCase> {
     WTF_MAKE_FAST_ALLOCATED_WITH_HEAP_IDENTIFIER(AccessCase);
@@ -320,7 +321,6 @@ class AccessCase : public ThreadSafeRefCounted<AccessCase> {
 
     bool requiresIdentifierNameMatch() const;
     bool requiresInt32PropertyCheck() const;
-    bool needsScratchFPR() const;
 
     UniquedStringImpl* uid() const { return m_identifier.uid(); }
     CacheableIdentifier identifier() const { return m_identifier; }

Source/JavaScriptCore/bytecode/InByStatus.cpp

@@ -51,14 +51,14 @@ void InByStatus::shrinkToFit()
 }
 
 #if ENABLE(JIT)
-InByStatus InByStatus::computeFor(CodeBlock* profiledBlock, ICStatusMap& map, BytecodeIndex bytecodeIndex, ExitFlag didExit)
+InByStatus InByStatus::computeFor(CodeBlock* profiledBlock, ICStatusMap& map, BytecodeIndex bytecodeIndex, ExitFlag didExit, CodeOrigin codeOrigin)
 {
     ConcurrentJSLocker locker(profiledBlock->m_lock);
 
     InByStatus result;
 
 #if ENABLE(DFG_JIT)
-    result = computeForStubInfoWithoutExitSiteFeedback(locker, profiledBlock->vm(), map.get(CodeOrigin(bytecodeIndex)).stubInfo);
+    result = computeForStubInfoWithoutExitSiteFeedback(locker, profiledBlock->vm(), map.get(CodeOrigin(bytecodeIndex)).stubInfo, codeOrigin);
 
     if (!result.takesSlowPath() && didExit)
         return InByStatus(TakesSlowPath);
@@ -71,9 +71,9 @@ InByStatus InByStatus::computeFor(CodeBlock* profiledBlock, ICStatusMap& map, By
     return result;
 }
 
-InByStatus InByStatus::computeFor(CodeBlock* profiledBlock, ICStatusMap& map, BytecodeIndex bytecodeIndex)
+InByStatus InByStatus::computeFor(CodeBlock* profiledBlock, ICStatusMap& map, BytecodeIndex bytecodeIndex, CodeOrigin codeOrigin)
 {
-    return computeFor(profiledBlock, map, bytecodeIndex, hasBadCacheExitSite(profiledBlock, bytecodeIndex));
+    return computeFor(profiledBlock, map, bytecodeIndex, hasBadCacheExitSite(profiledBlock, bytecodeIndex), codeOrigin);
 }
 
 InByStatus InByStatus::computeFor(
@@ -88,8 +88,7 @@ InByStatus InByStatus::computeFor(
 
         auto bless = [&] (const InByStatus& result) -> InByStatus {
             if (!context->isInlined(codeOrigin)) {
-                InByStatus baselineResult = computeFor(
-                    profiledBlock, baselineMap, bytecodeIndex, didExit);
+                InByStatus baselineResult = computeFor(profiledBlock, baselineMap, bytecodeIndex, didExit, codeOrigin);
                 baselineResult.merge(result);
                 return baselineResult;
             }
@@ -103,7 +102,7 @@ InByStatus InByStatus::computeFor(
             InByStatus result;
             {
                 ConcurrentJSLocker locker(context->optimizedCodeBlock->m_lock);
-                result = computeForStubInfoWithoutExitSiteFeedback(locker, profiledBlock->vm(), status.stubInfo);
+                result = computeForStubInfoWithoutExitSiteFeedback(locker, profiledBlock->vm(), status.stubInfo, codeOrigin);
             }
             if (result.isSet())
                 return bless(result);
@@ -114,21 +113,21 @@ InByStatus InByStatus::computeFor(
             return bless(*status.inStatus);
     }
 
-    return computeFor(profiledBlock, baselineMap, bytecodeIndex, didExit);
+    return computeFor(profiledBlock, baselineMap, bytecodeIndex, didExit, codeOrigin);
 }
 #endif // ENABLE(JIT)
 
 #if ENABLE(DFG_JIT)
 InByStatus InByStatus::computeForStubInfo(const ConcurrentJSLocker& locker, CodeBlock* profiledBlock, StructureStubInfo* stubInfo, CodeOrigin codeOrigin)
 {
-    InByStatus result = InByStatus::computeForStubInfoWithoutExitSiteFeedback(locker, profiledBlock->vm(), stubInfo);
+    InByStatus result = InByStatus::computeForStubInfoWithoutExitSiteFeedback(locker, profiledBlock->vm(), stubInfo, codeOrigin);
 
     if (!result.takesSlowPath() && hasBadCacheExitSite(profiledBlock, codeOrigin.bytecodeIndex()))
         return InByStatus(TakesSlowPath);
     return result;
 }
 
-InByStatus InByStatus::computeForStubInfoWithoutExitSiteFeedback(const ConcurrentJSLocker&, VM& vm, StructureStubInfo* stubInfo)
+InByStatus InByStatus::computeForStubInfoWithoutExitSiteFeedback(const ConcurrentJSLocker&, VM& vm, StructureStubInfo* stubInfo, CodeOrigin codeOrigin)
 {
     StubInfoSummary summary = StructureStubInfo::summary(vm, stubInfo);
     if (!isInlineable(summary))
@@ -164,6 +163,28 @@ InByStatus InByStatus::computeForStubInfoWithoutExitSiteFeedback(const Concurren
 
     case CacheType::Stub: {
         PolymorphicAccess* list = stubInfo->m_stub.get();
+        if (list->size() == 1) {
+            const AccessCase& access = list->at(0);
+            switch (access.type()) {
+            case AccessCase::InMegamorphic:
+            case AccessCase::IndexedMegamorphicIn: {
+                // Emitting InMegamorphic means that we give up polymorphic IC optimization. So this needs very careful handling.
+                // It is possible that one function can be inlined from the other function, and then it gets limited # of structures.
+                // In this case, continue using IC is better than falling back to megamorphic case. But if the function gets compiled before,
+                // and even optimizing JIT saw the megamorphism, then this is likely that this function continues having megamorphic behavior,
+                // and inlined megamorphic code is faster. Currently, we use InMegamorphic only when the exact same form of CodeOrigin gets
+                // this megamorphic GetById before (same level of inlining etc.). This is very conservative but effective since IC is very fast
+                // when it worked well (but costly if it doesn't work and get megamorphic). Once this cost-benefit tradeoff gets changed (via
+                // handler IC), we can revisit this condition.
+                if (isSameStyledCodeOrigin(stubInfo->codeOrigin, codeOrigin) && !stubInfo->tookSlowPath)
+                    return InByStatus(Megamorphic);
+                break;
+            }
+            default:
+                break;
+            }
+        }
+
         for (unsigned listIndex = 0; listIndex < list->size(); ++listIndex) {
             const AccessCase& access = list->at(listIndex);
             if (access.viaGlobalProxy())
@@ -240,7 +261,18 @@ void InByStatus::merge(const InByStatus& other)
     case NoInformation:
         *this = other;
         return;
-
+
+    case Megamorphic:
+        if (m_state != other.m_state) {
+            if (other.m_state == Simple) {
+                *this = other;
+                return;
+            }
+            *this = InByStatus(TakesSlowPath);
+            return;
+        }
+        return;
+
     case Simple:
         if (other.m_state != Simple) {
             *this = InByStatus(TakesSlowPath);
@@ -314,6 +346,9 @@ void InByStatus::dump(PrintStream& out) const
     case Simple:
         out.print("Simple");
         break;
+    case Megamorphic:
+        out.print("Megamorphic");
+        break;
     case TakesSlowPath:
         out.print("TakesSlowPath");
         break;

Source/JavaScriptCore/bytecode/InByStatus.h

@@ -49,6 +49,8 @@ class InByStatus final {
         // It's cached for a simple access to a known object property with
         // a possible structure chain and a possible specific value.
         Simple,
+        // It's cached for a megamorphic case.
+        Megamorphic,
         // It's known to often take slow path.
         TakesSlowPath,
     };
@@ -80,8 +82,8 @@ class InByStatus final {
         RELEASE_ASSERT_NOT_REACHED();
     }
 
-    static InByStatus computeFor(CodeBlock*, ICStatusMap&, BytecodeIndex);
-    static InByStatus computeFor(CodeBlock*, ICStatusMap&, BytecodeIndex, ExitFlag);
+    static InByStatus computeFor(CodeBlock*, ICStatusMap&, BytecodeIndex, CodeOrigin);
+    static InByStatus computeFor(CodeBlock*, ICStatusMap&, BytecodeIndex, ExitFlag, CodeOrigin);
     static InByStatus computeFor(CodeBlock* baselineBlock, ICStatusMap& baselineMap, ICStatusContextStack&, CodeOrigin);
 
 #if ENABLE(DFG_JIT)
@@ -93,13 +95,14 @@ class InByStatus final {
     bool isSet() const { return m_state != NoInformation; }
     explicit operator bool() const { return isSet(); }
     bool isSimple() const { return m_state == Simple; }
+    bool isMegamorphic() const { return m_state == Megamorphic; }
 
     size_t numVariants() const { return m_variants.size(); }
     const Vector<InByVariant, 1>& variants() const { return m_variants; }
     const InByVariant& at(size_t index) const { return m_variants[index]; }
     const InByVariant& operator[](size_t index) const { return at(index); }
 
-    bool takesSlowPath() const { return m_state == TakesSlowPath; }
+    bool takesSlowPath() const { return m_state == Megamorphic || m_state == TakesSlowPath; }
 
     void merge(const InByStatus&);
 
@@ -116,7 +119,7 @@ class InByStatus final {
 
 private:
 #if ENABLE(DFG_JIT)
-    static InByStatus computeForStubInfoWithoutExitSiteFeedback(const ConcurrentJSLocker&, VM&, StructureStubInfo*);
+    static InByStatus computeForStubInfoWithoutExitSiteFeedback(const ConcurrentJSLocker&, VM&, StructureStubInfo*, CodeOrigin);
 #endif
     bool appendVariant(const InByVariant&);
     void shrinkToFit();