[MERGE #4105 @boingoing] Support for defer parsing lambda functions

Merge pull request #4105 from boingoing:DeferParseLambda_3

Fix up the parser to enable the defer parse methods to handle lambda functions. All lambdas are allowed to be deferred, even lambdas with compact parameter lists or function bodies.

Should also allow lambda functions to be redeferred, which is really the main reason to do this work.

One of the related changes is in `Parser::Parse`. I changed this function to skip calling `ParseStmtList` in the case where we are defer parsing a single function. In these cases we know that we are going to parse a function so we don't need to start parsing a set of statements, then one statement, then one expression, then one terminal, etc until we finally start parsing the function via `ParseFncDecl`. Instead we can just jump directly to `ParseFncDecl` after setting up the correct flags. By doing this, we can avoid reparsing the lambda parameter list (because we never assume it will be an ordinary expression list) which saves us the headache of bookkeeping the block and function ids.

One other change of note is in `ScopeInfo::SaveSymbolInfo`. We now need to save symbol info for the `arguments` symbol since a lambda may be deferred in a function and lambdas capture the `arguments` value from their parent. I suppose we could move the `arguments` default binding into a regular special symbol at some point in the future.

Author: boingoing

lib/Parser/Parse.cpp

@@ -809,8 +809,8 @@ class Parser
     template<bool buildAST> void ParseExpressionLambdaBody(ParseNodePtr pnodeFnc);
     template<bool buildAST> void UpdateCurrentNodeFunc(ParseNodePtr pnodeFnc, bool fLambda);
     bool FncDeclAllowedWithoutContext(ushort flags);
-    void FinishFncDecl(ParseNodePtr pnodeFnc, LPCOLESTR pNameHint, ParseNodePtr *lastNodeRef, bool skipCurlyBraces = false);
-    void ParseTopLevelDeferredFunc(ParseNodePtr pnodeFnc, ParseNodePtr pnodeFncParent, LPCOLESTR pNameHint);
+    void FinishFncDecl(ParseNodePtr pnodeFnc, LPCOLESTR pNameHint, ParseNodePtr *lastNodeRef, bool fLambda, bool skipCurlyBraces = false);
+    void ParseTopLevelDeferredFunc(ParseNodePtr pnodeFnc, ParseNodePtr pnodeFncParent, LPCOLESTR pNameHint, bool fLambda, bool *pNeedScanRCurly = nullptr);
     void ParseNestedDeferredFunc(ParseNodePtr pnodeFnc, bool fLambda, bool *pNeedScanRCurly, bool *pStrictModeTurnedOn);
     void CheckStrictFormalParameters();
     ParseNodePtr AddArgumentsNodeToVars(ParseNodePtr pnodeFnc);

lib/Parser/Parse.h

@@ -99,6 +99,7 @@ Scanner<EncodingPolicy>::Scanner(Parser* parser, HashTbl *phtbl, Token *ptoken,
     m_tempChBufSecondary.m_pscanner = this;
 
     m_iecpLimTokPrevious = (size_t)-1;
+    m_ichLimTokPrevious = (charcount_t)-1;
 
     this->charClassifier = scriptContext->GetCharClassifier();
 
@@ -1634,6 +1635,7 @@ tokens Scanner<EncodingPolicy>::ScanCore(bool identifyKwds)
     // store the last token
     m_tkPrevious = m_ptoken->tk;
     m_iecpLimTokPrevious = IecpLimTok();    // Introduced for use by lambda parsing to find correct span of expression lambdas
+    m_ichLimTokPrevious = IchLimTok();
 
     if (p >= last)
     {

lib/Parser/Scan.cpp

@@ -531,6 +531,12 @@ class Scanner : public IScanner, public EncodingPolicy
         return m_iecpLimTokPrevious;
     }
 
+    charcount_t IchLimTokPrevious() const
+    {
+        AssertMsg(m_ichLimTokPrevious != (charcount_t)-1, "IchLimTokPrevious() cannot be called before scanning a token");
+        return m_ichLimTokPrevious;
+    }
+
     IdentPtr PidAt(size_t iecpMin, size_t iecpLim);
 
     // Returns the character offset within the stream of the first character on the current line.
@@ -549,6 +555,7 @@ class Scanner : public IScanner, public EncodingPolicy
     void SetCurrentCharacter(charcount_t offset, ULONG lineNumber = 0)
     {
         DebugOnly(m_iecpLimTokPrevious = (size_t)-1);
+        DebugOnly(m_ichLimTokPrevious = (charcount_t)-1);
         size_t length = m_pchLast - m_pchBase;
         if (offset > length) offset = static_cast< charcount_t >(length);
         size_t ibOffset = this->CharacterOffsetToUnitOffset(m_pchBase, m_currentCharacter, m_pchLast, offset);
@@ -713,6 +720,7 @@ class Scanner : public IScanner, public EncodingPolicy
 
     tokens m_tkPrevious;
     size_t m_iecpLimTokPrevious;
+    charcount_t m_ichLimTokPrevious;
 
     Scanner(Parser* parser, HashTbl *phtbl, Token *ptoken, Js::ScriptContext *scriptContext);
     ~Scanner(void);

lib/Parser/Scan.h

@@ -1416,6 +1416,7 @@ namespace Js
         bool IsConstructor() const;
         bool IsGenerator() const;
         bool IsClassConstructor() const;
+        bool IsBaseClassConstructor() const;
         bool IsClassMethod() const;
         bool IsModule() const;
         bool IsWasmFunction() const;
@@ -1722,6 +1723,13 @@ namespace Js
         return GetFunctionInfo()->IsClassConstructor();
     }
 
+    inline bool FunctionProxy::IsBaseClassConstructor() const
+    {
+        Assert(GetFunctionInfo());
+        Assert(GetFunctionInfo()->GetFunctionProxy() == this);
+        return GetFunctionInfo()->GetBaseConstructorKind();
+    }
+
     inline bool FunctionProxy::IsClassMethod() const
     {
         Assert(GetFunctionInfo());

lib/Runtime/Base/FunctionBody.h

@@ -1308,6 +1308,11 @@ FuncInfo * ByteCodeGenerator::StartBindFunction(const char16 *name, uint nameLen
             if (pnode->sxFnc.IsClassConstructor())
             {
                 attributes = (Js::FunctionInfo::Attributes)(attributes | Js::FunctionInfo::Attributes::ClassConstructor);
+
+                if (pnode->sxFnc.IsBaseClassConstructor())
+                {
+                    attributes = (Js::FunctionInfo::Attributes)(attributes | Js::FunctionInfo::Attributes::BaseConstructorKind);
+                }
             }
             else
             {

lib/Runtime/ByteCode/ByteCodeGenerator.cpp

@@ -217,7 +217,7 @@ enum FunctionFlags
     ffIsAnonymous                      = 0x100000,
     ffUsesArgumentsObject              = 0x200000,
     ffDoScopeObjectCreation            = 0x400000,
-    ffIsParamAndBodyScopeMerged        = 0x800000
+    ffIsParamAndBodyScopeMerged        = 0x800000,
 };
 
 // Kinds of constant
@@ -2043,10 +2043,11 @@ class ByteCodeBufferBuilder
                   | FunctionInfo::Attributes::CapturesThis
                   | FunctionInfo::Attributes::Generator
                   | FunctionInfo::Attributes::ClassConstructor
+                  | FunctionInfo::Attributes::BaseConstructorKind
                   | FunctionInfo::Attributes::ClassMethod
                   | FunctionInfo::Attributes::EnclosedByGlobalFunc
                   | FunctionInfo::Attributes::AllowDirectSuper)) == 0,
-                "Only the ErrorOnNew|SuperReference|Lambda|CapturesThis|Generator|ClassConstructor|Async|ClassMember|EnclosedByGlobalFunc|AllowDirectSuper attributes should be set on a serialized function");
+                "Only the ErrorOnNew|SuperReference|Lambda|CapturesThis|Generator|ClassConstructor|BaseConstructorKind|Async|ClassMember|EnclosedByGlobalFunc|AllowDirectSuper attributes should be set on a serialized function");
 
         PrependInt32(builder, _u("Offset Into Source"), sourceDiff);
         if (function->GetNestedCount() > 0)

lib/Runtime/ByteCode/ByteCodeSerializer.cpp

@@ -136,22 +136,22 @@ BOOL FuncInfo::HasDirectSuper() const
 
 BOOL FuncInfo::IsClassMember() const
 {
-    return root->sxFnc.IsClassMember();
+    return this->byteCodeFunction->IsClassMethod();
 }
 
 BOOL FuncInfo::IsLambda() const
 {
-    return root->sxFnc.IsLambda();
+    return this->byteCodeFunction->IsLambda();
 }
 
 BOOL FuncInfo::IsClassConstructor() const
 {
-    return root->sxFnc.IsClassConstructor();
+    return this->byteCodeFunction->IsClassConstructor();
 }
 
 BOOL FuncInfo::IsBaseClassConstructor() const
 {
-    return root->sxFnc.IsBaseClassConstructor();
+    return this->byteCodeFunction->IsBaseClassConstructor();
 }
 
 BOOL FuncInfo::IsDerivedClassConstructor() const

lib/Runtime/ByteCode/FuncInfo.cpp

@@ -11,8 +11,7 @@ namespace Js
     //
     void ScopeInfo::SaveSymbolInfo(Symbol* sym, MapSymbolData* mapSymbolData)
     {
-        // We don't need to create slot for or save "arguments"
-        bool needScopeSlot = !sym->IsArguments() && sym->GetHasNonLocalReference();
+        bool needScopeSlot = sym->GetHasNonLocalReference();
         Js::PropertyId scopeSlot = Constants::NoSlot;
 
         if (sym->GetIsModuleExportStorage())