Add simple set of captured names to nested ParseNodeFnc

Each `ParseNodeFnc` has an IdentPtrSet which holds the set of names captured by the function. Captured names are ones which are referenced inside the function but not declared within the function.

Implement this by intercepting name references at PushPidRef and keep the name reference in the set of captured names for the current nested function. Remove any names from the captured name list which do not have dangling references after we've bound the pid ref stack for a nested function. When finished parsing a nested function, add the set of captured names onto the parent functions' captured name set.

Gate the feature via a ParseType flag and allow a forcing config option (`CreateParserState`).

Example captured name set:
```js
let x, y, z;
function foo() {
  function bar() {
    return x;
  }
  function baz(x) {
    return y;
  }
  function bat() {
    {
      let z;
      return z;
    }
  }
}

// captured name sets
// foo: [x, y]
// bar: [x]
// baz: [y]
// bat: []
```

Author: boingoing

lib/Common/ConfigFlagsList.h

@@ -15,6 +15,7 @@ PHASE(All)
         PHASE(DeferEventHandlers)
         PHASE(FunctionSourceInfoParse)
         PHASE(StringTemplateParse)
+        PHASE(CreateParserState)
         PHASE(SkipNestedDeferred)
         PHASE(CacheScopeInfoNames)
         PHASE(ScanAhead)
@@ -427,6 +428,7 @@ PHASE(All)
 #define DEFAULT_CONFIG_PrintLineColumnInfo (false)
 #define DEFAULT_CONFIG_ForceDecommitOnCollect (false)
 #define DEFAULT_CONFIG_ForceDeferParse      (false)
+#define DEFAULT_CONFIG_ForceCreateParserState (false)
 #define DEFAULT_CONFIG_NoDeferParse         (false)
 #define DEFAULT_CONFIG_ForceDynamicProfile  (false)
 #define DEFAULT_CONFIG_ForceExpireOnNonCacheCollect (false)
@@ -1144,6 +1146,7 @@ FLAGNR(Boolean, ForceCleanCacheOnCollect, "Force cleaning of dynamic caches on c
 FLAGNR(Boolean, ForceGCAfterJSONParse, "Force GC to happen after JSON parsing", DEFAULT_CONFIG_ForceGCAfterJSONParse)
 FLAGNR(Boolean, ForceDecommitOnCollect, "Force decommit collect", DEFAULT_CONFIG_ForceDecommitOnCollect)
 FLAGNR(Boolean, ForceDeferParse       , "Defer parsing of all function bodies", DEFAULT_CONFIG_ForceDeferParse)
+FLAGNR(Boolean, ForceCreateParserState , "Force creation of parser state", DEFAULT_CONFIG_ForceCreateParserState)
 FLAGNR(Boolean, ForceDiagnosticsMode  , "Enable diagnostics mode and debug interpreter loop", false)
 FLAGNR(Boolean, ForceGetWriteWatchOOM , "Force GetWriteWatch to go into OOM codepath in HeapBlockMap rescan", false)
 FLAGNR(Boolean, ForcePostLowerGlobOptInstrString, "Force tracking of globopt instr string post lower", DEFAULT_CONFIG_ForcePostLowerGlobOptInstrString)

lib/Parser/Parse.cpp

@@ -4881,7 +4881,6 @@ ParseNode * Parser::ParseFncDecl(ushort flags, LPCOLESTR pNameHint, const bool n
     pnodeFnc->cbMin = this->GetScanner()->IecpMinTok();
     pnodeFnc->functionId = (*m_nextFunctionId)++;
 
-
     // Push new parser state with this new function node
 
     AppendFunctionToScopeList(fDeclaration, pnodeFnc);
@@ -4918,6 +4917,9 @@ ParseNode * Parser::ParseFncDecl(ushort flags, LPCOLESTR pNameHint, const bool n
     IdentPtr pFncNamePid = nullptr;
     bool needScanRCurly = true;
     bool result = ParseFncDeclHelper<buildAST>(pnodeFnc, pNameHint, flags, &funcHasName, fUnaryOrParen, noStmtContext, &needScanRCurly, fModule, &pFncNamePid);
+
+    AddNestedCapturedNames(pnodeFnc);
+
     if (!result)
     {
         Assert(!pnodeFncBlockScope);
@@ -5659,6 +5661,8 @@ bool Parser::ParseFncDeclHelper(ParseNodeFnc * pnodeFnc, LPCOLESTR pNameHint, us
             CHAKRATEL_LANGSTATS_INC_LANGFEATURECOUNT(ES6, StrictModeFunction, m_scriptContext);
         }
 
+        ProcessCapturedNames(pnodeFnc);
+
         if (fDeferred)
         {
             AnalysisAssert(pnodeFnc);
@@ -9029,19 +9033,27 @@ void Parser::TrackAssignment(ParseNodePtr pnodeT, IdentToken* pToken)
 
 PidRefStack* Parser::PushPidRef(IdentPtr pid)
 {
+    ParseNodeFnc* currentFnc = GetCurrentFunctionNode();
+
+    if (this->IsCreatingStateCache())
+    {
+        IdentPtrSet* capturedNames = currentFnc->EnsureCapturedNames(&m_nodeAllocator);
+        capturedNames->AddNew(pid);
+    }
+
     if (PHASE_ON1(Js::ParallelParsePhase))
     {
         // NOTE: the phase check is here to protect perf. See OSG 1020424.
         // In some LS AST-rewrite cases we lose a lot of perf searching the PID ref stack rather
         // than just pushing on the top. This hasn't shown up as a perf issue in non-LS benchmarks.
-        return pid->FindOrAddPidRef(&m_nodeAllocator, GetCurrentBlock()->blockId, GetCurrentFunctionNode()->functionId);
+        return pid->FindOrAddPidRef(&m_nodeAllocator, GetCurrentBlock()->blockId, currentFnc->functionId);
     }
 
     Assert(GetCurrentBlock() != nullptr);
     AssertMsg(pid != nullptr, "PID should be created");
     PidRefStack *ref = pid->GetTopRef(m_nextBlockId - 1);
     int blockId = GetCurrentBlock()->blockId;
-    int funcId = GetCurrentFunctionNode()->functionId;
+    int funcId = currentFnc->functionId;
     if (!ref || (ref->GetScopeId() < blockId))
     {
         ref = Anew(&m_nodeAllocator, PidRefStack);
@@ -13826,6 +13838,59 @@ void ParseNode::Dump()
 }
 #endif
 
+void Parser::AddNestedCapturedNames(ParseNodeFnc* pnodeChildFnc)
+{
+    if (m_currentNodeFunc && this->IsCreatingStateCache() && pnodeChildFnc->HasAnyCapturedNames())
+    {
+        IdentPtrSet* parentCapturedNames = GetCurrentFunctionNode()->EnsureCapturedNames(&m_nodeAllocator);
+        IdentPtrSet* childCaptureNames = pnodeChildFnc->GetCapturedNames();
+
+        auto iter = childCaptureNames->GetIterator();
+
+        while (iter.IsValid())
+        {
+            parentCapturedNames->AddNew(iter.CurrentValue());
+            iter.MoveNext();
+        }
+    }
+}
+
+void Parser::ProcessCapturedNames(ParseNodeFnc* pnodeFnc)
+{
+    if (this->IsCreatingStateCache() && pnodeFnc->HasAnyCapturedNames())
+    {
+        IdentPtrSet* capturedNames = pnodeFnc->GetCapturedNames();
+        auto iter = capturedNames->GetIteratorWithRemovalSupport();
+
+        while (iter.IsValid())
+        {
+            const IdentPtr& pid = iter.CurrentValueReference();
+            PidRefStack* ref = pid->GetTopRef();
+
+            // If the pid has no refs left in our function's scope after binding, we didn't capture it.
+            if (!ref || ref->GetScopeId() < pnodeFnc->pnodeBodyScope->blockId)
+            {
+                iter.RemoveCurrent();
+            }
+            else
+            {
+                OUTPUT_TRACE_DEBUGONLY(Js::CreateParserStatePhase, _u("Function %u captured name \"%s\"\n"), pnodeFnc->functionId, pid->Psz());
+            }
+
+            iter.MoveNext();
+        }
+    }
+}
+
+bool Parser::IsCreatingStateCache()
+{
+    return this->m_parseType == ParseType_StateCache
+#ifdef ENABLE_DEBUG_CONFIG_OPTIONS
+        || CONFIG_FLAG(ForceCreateParserState)
+#endif
+            ;
+}
+
 DeferredFunctionStub * BuildDeferredStubTree(ParseNodeFnc *pnodeFnc, Recycler *recycler)
 {
     Assert(pnodeFnc->nop == knopFncDecl);
@@ -13887,6 +13952,7 @@ DeferredFunctionStub * BuildDeferredStubTree(ParseNodeFnc *pnodeFnc, Recycler *r
         deferredStubs[i].restorePoint = *pnodeFncChild->pRestorePoint;
         deferredStubs[i].deferredStubs = BuildDeferredStubTree(pnodeFncChild, recycler);
         deferredStubs[i].ichMin = pnodeChild->ichMin;
+
         ++i;
         pnodeChild = pnodeFncChild->pnodeNext;
     }

lib/Parser/Parse.h

@@ -36,7 +36,8 @@ enum
 enum ParseType
 {
     ParseType_Upfront,
-    ParseType_Deferred
+    ParseType_Deferred,
+    ParseType_StateCache
 };
 
 enum DestructuringInitializerContext
@@ -117,6 +118,8 @@ class Parser
 
     Js::ScriptContext* GetScriptContext() const { return m_scriptContext; }
 
+    bool IsCreatingStateCache();
+
 #if ENABLE_BACKGROUND_PARSING
     bool IsBackgroundParser() const { return m_isInBackground; }
     bool IsDoingFastScan() const { return m_doingFastScan; }
@@ -317,6 +320,8 @@ class Parser
                 return _u("Upfront");
             case ParseType_Deferred:
                 return _u("Deferred");
+            case ParseType_StateCache:
+                return _u("StateCache");
         }
         Assert(false);
         return NULL;
@@ -684,6 +689,9 @@ class Parser
     ParseNodeSpecialName * ReferenceSpecialName(IdentPtr pid, charcount_t ichMin = 0, charcount_t ichLim = 0, bool createNode = false);
     ParseNodeVar * CreateSpecialVarDeclIfNeeded(ParseNodeFnc * pnodeFnc, IdentPtr pid, bool forceCreate = false);
 
+    void ProcessCapturedNames(ParseNodeFnc* pnodeFnc);
+    void AddNestedCapturedNames(ParseNodeFnc* pnodeChildFnc);
+
     template<const bool backgroundPidRefs>
     void BindPidRefs(BlockInfoStack *blockInfo, uint maxBlockId = (uint)-1);
     void BindPidRefsInScope(IdentPtr pid, Symbol *sym, int blockId, uint maxBlockId = (uint)-1);

lib/Parser/ParserCommon.h

@@ -51,6 +51,7 @@ struct ModuleImportOrExportEntry
 
 typedef SList<ModuleImportOrExportEntry, ArenaAllocator> ModuleImportOrExportEntryList;
 typedef SList<IdentPtr, ArenaAllocator> IdentPtrList;
+typedef JsUtil::BaseHashSet<IdentPtr, ArenaAllocator, PowerOf2SizePolicy> IdentPtrSet;
 
 //
 // Below was moved from scrutil.h to share with chakradiag.

lib/Parser/ptree.cpp

@@ -433,7 +433,7 @@ ParseNodeFnc::ParseNodeFnc(OpCode nop, charcount_t ichMin, charcount_t ichLim)
     this->pRestorePoint = nullptr;
     this->deferredStub = nullptr;
 
-    
+    this->capturedNames = nullptr;
 }
 
 ParseNodeClass::ParseNodeClass(OpCode nop, charcount_t ichMin, charcount_t ichLim)
@@ -605,4 +605,23 @@ ParseNodeSuperCall::ParseNodeSuperCall(OpCode nop, charcount_t ichMin, charcount
     this->isSuperCall = true;
     this->pnodeThis = nullptr;
     this->pnodeNewTarget = nullptr;
-}
+}
+
+IdentPtrSet* ParseNodeFnc::EnsureCapturedNames(ArenaAllocator* alloc)
+{
+    if (this->capturedNames == nullptr)
+    {
+        this->capturedNames = Anew(alloc, IdentPtrSet, alloc);
+    }
+    return this->capturedNames;
+}
+
+IdentPtrSet* ParseNodeFnc::GetCapturedNames()
+{
+    return this->capturedNames;
+}
+
+bool ParseNodeFnc::HasAnyCapturedNames()
+{
+    return this->capturedNames != nullptr && this->capturedNames->Count() != 0;
+}

lib/Parser/ptree.h

@@ -490,6 +490,7 @@ class ParseNodeFnc : public ParseNode
 #endif
     RestorePoint *pRestorePoint;
     DeferredFunctionStub *deferredStub;
+    IdentPtrSet *capturedNames;
     bool canBeDeferred;
     bool isBodyAndParamScopeMerged; // Indicates whether the param scope and the body scope of the function can be merged together or not.
                                     // We cannot merge both scopes together if there is any closure capture or eval is present in the param scope.
@@ -641,6 +642,10 @@ class ParseNodeFnc : public ParseNode
         }
     }
 
+    IdentPtrSet* EnsureCapturedNames(ArenaAllocator* alloc);
+    IdentPtrSet* GetCapturedNames();
+    bool HasAnyCapturedNames();
+
     DISABLE_SELF_CAST(ParseNodeFnc);
 };
 

test/Basics/VerifyParserState.baseline

@@ -0,0 +1,149 @@
+nested function capturing var decl
+CreateParserState:Function 2 captured name "a1"
+
+nested function capturing a let decl
+CreateParserState:Function 2 captured name "a2"
+
+nested function capturing a const decl
+CreateParserState:Function 2 captured name "a3"
+
+nested function capturing non-decl names
+CreateParserState:Function 2 captured name "a4"
+CreateParserState:Function 2 captured name "b4"
+CreateParserState:Function 1 captured name "a4"
+CreateParserState:Function 1 captured name "b4"
+
+nested function referencing a local shadowing formal
+
+nested function capturing a named formal
+CreateParserState:Function 2 captured name "a6"
+
+nested functions capturing global declarations
+CreateParserState:Function 3 captured name "d7"
+CreateParserState:Function 3 captured name "v7"
+CreateParserState:Function 3 captured name "a7"
+CreateParserState:Function 3 captured name "c7"
+CreateParserState:Function 3 captured name "b7"
+CreateParserState:Function 3 captured name "l7"
+CreateParserState:Function 2 captured name "d7"
+CreateParserState:Function 2 captured name "v7"
+CreateParserState:Function 2 captured name "a7"
+CreateParserState:Function 2 captured name "c7"
+CreateParserState:Function 2 captured name "l7"
+CreateParserState:Function 1 captured name "d7"
+CreateParserState:Function 1 captured name "v7"
+CreateParserState:Function 1 captured name "b7"
+CreateParserState:Function 1 captured name "c7"
+CreateParserState:Function 1 captured name "l7"
+
+nested function capturing a class decl
+CreateParserState:Function 3 captured name "class_1"
+CreateParserState:Function 2 captured name "class_1"
+
+object literal methods capturing names
+CreateParserState:Function 1 captured name "v9"
+CreateParserState:Function 1 captured name "o_9"
+CreateParserState:Function 1 captured name "c9"
+CreateParserState:Function 1 captured name "l9"
+CreateParserState:Function 1 captured name "b9"
+CreateParserState:Function 2 captured name "v9"
+CreateParserState:Function 2 captured name "o_9"
+CreateParserState:Function 2 captured name "c9"
+CreateParserState:Function 2 captured name "l9"
+CreateParserState:Function 2 captured name "b9"
+CreateParserState:Function 3 captured name "v9"
+CreateParserState:Function 3 captured name "o_9"
+CreateParserState:Function 3 captured name "c9"
+CreateParserState:Function 3 captured name "l9"
+CreateParserState:Function 3 captured name "b9"
+CreateParserState:Function 4 captured name "v9"
+CreateParserState:Function 4 captured name "o_9"
+CreateParserState:Function 4 captured name "c9"
+CreateParserState:Function 4 captured name "l9"
+CreateParserState:Function 4 captured name "b9"
+CreateParserState:Function 5 captured name "v9"
+CreateParserState:Function 5 captured name "o_9"
+CreateParserState:Function 5 captured name "c9"
+CreateParserState:Function 5 captured name "l9"
+CreateParserState:Function 5 captured name "b9"
+CreateParserState:Function 6 captured name "v9"
+CreateParserState:Function 6 captured name "a9"
+CreateParserState:Function 6 captured name "o_9"
+CreateParserState:Function 6 captured name "c9"
+CreateParserState:Function 6 captured name "l9"
+CreateParserState:Function 6 captured name "b9"
+CreateParserState:Function 7 captured name "v9"
+CreateParserState:Function 7 captured name "o_9"
+CreateParserState:Function 7 captured name "c9"
+CreateParserState:Function 7 captured name "l9"
+CreateParserState:Function 7 captured name "b9"
+
+class members capturing names
+CreateParserState:Function 1 captured name "class_10"
+CreateParserState:Function 1 captured name "c10"
+CreateParserState:Function 1 captured name "l10"
+CreateParserState:Function 1 captured name "b10"
+CreateParserState:Function 1 captured name "v10"
+CreateParserState:Function 2 captured name "class_10"
+CreateParserState:Function 2 captured name "c10"
+CreateParserState:Function 2 captured name "l10"
+CreateParserState:Function 2 captured name "b10"
+CreateParserState:Function 2 captured name "v10"
+CreateParserState:Function 3 captured name "class_10"
+CreateParserState:Function 3 captured name "c10"
+CreateParserState:Function 3 captured name "l10"
+CreateParserState:Function 3 captured name "b10"
+CreateParserState:Function 3 captured name "v10"
+CreateParserState:Function 4 captured name "class_10"
+CreateParserState:Function 4 captured name "c10"
+CreateParserState:Function 4 captured name "l10"
+CreateParserState:Function 4 captured name "b10"
+CreateParserState:Function 4 captured name "v10"
+CreateParserState:Function 5 captured name "class_10"
+CreateParserState:Function 5 captured name "c10"
+CreateParserState:Function 5 captured name "l10"
+CreateParserState:Function 5 captured name "b10"
+CreateParserState:Function 5 captured name "v10"
+CreateParserState:Function 6 captured name "class_10"
+CreateParserState:Function 6 captured name "c10"
+CreateParserState:Function 6 captured name "l10"
+CreateParserState:Function 6 captured name "b10"
+CreateParserState:Function 6 captured name "v10"
+CreateParserState:Function 6 captured name "a10"
+CreateParserState:Function 7 captured name "class_10"
+CreateParserState:Function 7 captured name "c10"
+CreateParserState:Function 7 captured name "l10"
+CreateParserState:Function 7 captured name "b10"
+CreateParserState:Function 7 captured name "v10"
+
+lambda capturing special names
+CreateParserState:Function 2 captured name "arguments"
+CreateParserState:Function 3 captured name "*this*"
+CreateParserState:Function 4 captured name "*new.target*"
+CreateParserState:Function 6 captured name "*superconstructor*"
+CreateParserState:Function 6 captured name "*new.target*"
+CreateParserState:Function 6 captured name "*this*"
+CreateParserState:Function 5 captured name "*superconstructor*"
+CreateParserState:Function 8 captured name "*super*"
+CreateParserState:Function 8 captured name "*this*"
+CreateParserState:Function 1 captured name "*superconstructor*"
+
+nested function with shadowing block-scoped name
+CreateParserState:Function 1 captured name "v12"
+
+nested functions with a few nested captures
+CreateParserState:Function 3 captured name "b13"
+CreateParserState:Function 3 captured name "d13"
+CreateParserState:Function 3 captured name "a13"
+CreateParserState:Function 3 captured name "c13"
+CreateParserState:Function 2 captured name "b13"
+CreateParserState:Function 2 captured name "a13"
+CreateParserState:Function 5 captured name "b13"
+CreateParserState:Function 5 captured name "f13"
+CreateParserState:Function 4 captured name "e13"
+CreateParserState:Function 4 captured name "b13"
+CreateParserState:Function 4 captured name "f13"
+CreateParserState:Function 1 captured name "e13"
+CreateParserState:Function 1 captured name "f13"
+CreateParserState:Function 1 captured name "a13"
+