[1.7>master] [MERGE #4105 @boingoing] Support for defer parsing lambda functions

Merge pull request #4105 from boingoing:DeferParseLambda_3

Fix up the parser to enable the defer parse methods to handle lambda functions. All lambdas are allowed to be deferred, even lambdas with compact parameter lists or function bodies.

Should also allow lambda functions to be redeferred, which is really the main reason to do this work.

One of the related changes is in `Parser::Parse`. I changed this function to skip calling `ParseStmtList` in the case where we are defer parsing a single function. In these cases we know that we are going to parse a function so we don't need to start parsing a set of statements, then one statement, then one expression, then one terminal, etc until we finally start parsing the function via `ParseFncDecl`. Instead we can just jump directly to `ParseFncDecl` after setting up the correct flags. By doing this, we can avoid reparsing the lambda parameter list (because we never assume it will be an ordinary expression list) which saves us the headache of bookkeeping the block and function ids.

One other change of note is in `ScopeInfo::SaveSymbolInfo`. We now need to save symbol info for the `arguments` symbol since a lambda may be deferred in a function and lambdas capture the `arguments` value from their parent. I suppose we could move the `arguments` default binding into a regular special symbol at some point in the future.

Author: boingoing

lib/Parser/Parse.cpp

@@ -811,8 +811,8 @@ class Parser
     template<bool buildAST> void ParseExpressionLambdaBody(ParseNodePtr pnodeFnc);
     template<bool buildAST> void UpdateCurrentNodeFunc(ParseNodePtr pnodeFnc, bool fLambda);
     bool FncDeclAllowedWithoutContext(ushort flags);
-    void FinishFncDecl(ParseNodePtr pnodeFnc, LPCOLESTR pNameHint, ParseNodePtr *lastNodeRef, bool skipCurlyBraces = false);
-    void ParseTopLevelDeferredFunc(ParseNodePtr pnodeFnc, ParseNodePtr pnodeFncParent, LPCOLESTR pNameHint);
+    void FinishFncDecl(ParseNodePtr pnodeFnc, LPCOLESTR pNameHint, ParseNodePtr *lastNodeRef, bool fLambda, bool skipCurlyBraces = false);
+    void ParseTopLevelDeferredFunc(ParseNodePtr pnodeFnc, ParseNodePtr pnodeFncParent, LPCOLESTR pNameHint, bool fLambda, bool *pNeedScanRCurly = nullptr);
     void ParseNestedDeferredFunc(ParseNodePtr pnodeFnc, bool fLambda, bool *pNeedScanRCurly, bool *pStrictModeTurnedOn);
     void CheckStrictFormalParameters();
     ParseNodePtr AddArgumentsNodeToVars(ParseNodePtr pnodeFnc);

lib/Parser/Parse.h

@@ -99,6 +99,7 @@ Scanner<EncodingPolicy>::Scanner(Parser* parser, HashTbl *phtbl, Token *ptoken,
     m_tempChBufSecondary.m_pscanner = this;
 
     m_iecpLimTokPrevious = (size_t)-1;
+    m_ichLimTokPrevious = (charcount_t)-1;
 
     this->charClassifier = scriptContext->GetCharClassifier();
 
@@ -1552,6 +1553,7 @@ tokens Scanner<EncodingPolicy>::ScanCore(bool identifyKwds)
     // store the last token
     m_tkPrevious = m_ptoken->tk;
     m_iecpLimTokPrevious = IecpLimTok();    // Introduced for use by lambda parsing to find correct span of expression lambdas
+    m_ichLimTokPrevious = IchLimTok();
 
     if (p >= last)
     {

lib/Parser/Scan.cpp

@@ -531,6 +531,12 @@ class Scanner : public IScanner, public EncodingPolicy
         return m_iecpLimTokPrevious;
     }
 
+    charcount_t IchLimTokPrevious() const
+    {
+        AssertMsg(m_ichLimTokPrevious != (charcount_t)-1, "IchLimTokPrevious() cannot be called before scanning a token");
+        return m_ichLimTokPrevious;
+    }
+
     IdentPtr PidAt(size_t iecpMin, size_t iecpLim);
 
     // Returns the character offset within the stream of the first character on the current line.
@@ -549,6 +555,7 @@ class Scanner : public IScanner, public EncodingPolicy
     void SetCurrentCharacter(charcount_t offset, ULONG lineNumber = 0)
     {
         DebugOnly(m_iecpLimTokPrevious = (size_t)-1);
+        DebugOnly(m_ichLimTokPrevious = (charcount_t)-1);
         size_t length = m_pchLast - m_pchBase;
         if (offset > length) offset = static_cast< charcount_t >(length);
         size_t ibOffset = this->CharacterOffsetToUnitOffset(m_pchBase, m_currentCharacter, m_pchLast, offset);
@@ -713,6 +720,7 @@ class Scanner : public IScanner, public EncodingPolicy
 
     tokens m_tkPrevious;
     size_t m_iecpLimTokPrevious;
+    charcount_t m_ichLimTokPrevious;
 
     Scanner(Parser* parser, HashTbl *phtbl, Token *ptoken, Js::ScriptContext *scriptContext);
     ~Scanner(void);

lib/Parser/Scan.h

@@ -1227,6 +1227,7 @@ namespace Js
         bool IsConstructor() const;
         bool IsGenerator() const;
         bool IsClassConstructor() const;
+        bool IsBaseClassConstructor() const;
         bool IsClassMethod() const;
         bool IsModule() const;
         bool IsWasmFunction() const;
@@ -1533,6 +1534,13 @@ namespace Js
         return GetFunctionInfo()->IsClassConstructor();
     }
 
+    inline bool FunctionProxy::IsBaseClassConstructor() const
+    {
+        Assert(GetFunctionInfo());
+        Assert(GetFunctionInfo()->GetFunctionProxy() == this);
+        return GetFunctionInfo()->GetBaseConstructorKind();
+    }
+
     inline bool FunctionProxy::IsClassMethod() const
     {
         Assert(GetFunctionInfo());

lib/Runtime/Base/FunctionBody.h

@@ -1308,6 +1308,11 @@ FuncInfo * ByteCodeGenerator::StartBindFunction(const char16 *name, uint nameLen
             if (pnode->sxFnc.IsClassConstructor())
             {
                 attributes = (Js::FunctionInfo::Attributes)(attributes | Js::FunctionInfo::Attributes::ClassConstructor);
+
+                if (pnode->sxFnc.IsBaseClassConstructor())
+                {
+                    attributes = (Js::FunctionInfo::Attributes)(attributes | Js::FunctionInfo::Attributes::BaseConstructorKind);
+                }
             }
             else
             {

lib/Runtime/ByteCode/ByteCodeGenerator.cpp

@@ -223,7 +223,7 @@ enum FunctionFlags
     ffIsAnonymous                      = 0x100000,
     ffUsesArgumentsObject              = 0x200000,
     ffDoScopeObjectCreation            = 0x400000,
-    ffIsParamAndBodyScopeMerged        = 0x800000
+    ffIsParamAndBodyScopeMerged        = 0x800000,
 };
 
 // Kinds of constant
@@ -2064,10 +2064,11 @@ class ByteCodeBufferBuilder
                 | FunctionInfo::Attributes::CapturesThis
                 | FunctionInfo::Attributes::Generator
                 | FunctionInfo::Attributes::ClassConstructor
+                | FunctionInfo::Attributes::BaseConstructorKind
                 | FunctionInfo::Attributes::ClassMethod
                 | FunctionInfo::Attributes::EnclosedByGlobalFunc
                 | FunctionInfo::Attributes::AllowDirectSuper)) == 0,
-            "Only the ErrorOnNew|SuperReference|Lambda|CapturesThis|Generator|ClassConstructor|Async|ClassMember|EnclosedByGlobalFunc|AllowDirectSuper attributes should be set on a serialized function");
+            "Only the ErrorOnNew|SuperReference|Lambda|CapturesThis|Generator|ClassConstructor|BaseConstructorKind|Async|ClassMember|EnclosedByGlobalFunc|AllowDirectSuper attributes should be set on a serialized function");
         if (attributes != FunctionInfo::Attributes::None)
         {
             definedFields.has_attributes = true;

lib/Runtime/ByteCode/ByteCodeSerializer.cpp

@@ -136,22 +136,22 @@ BOOL FuncInfo::HasDirectSuper() const
 
 BOOL FuncInfo::IsClassMember() const
 {
-    return root->sxFnc.IsClassMember();
+    return this->byteCodeFunction->IsClassMethod();
 }
 
 BOOL FuncInfo::IsLambda() const
 {
-    return root->sxFnc.IsLambda();
+    return this->byteCodeFunction->IsLambda();
 }
 
 BOOL FuncInfo::IsClassConstructor() const
 {
-    return root->sxFnc.IsClassConstructor();
+    return this->byteCodeFunction->IsClassConstructor();
 }
 
 BOOL FuncInfo::IsBaseClassConstructor() const
 {
-    return root->sxFnc.IsBaseClassConstructor();
+    return this->byteCodeFunction->IsBaseClassConstructor();
 }
 
 BOOL FuncInfo::IsDerivedClassConstructor() const

lib/Runtime/ByteCode/FuncInfo.cpp

@@ -11,8 +11,7 @@ namespace Js
     //
     void ScopeInfo::SaveSymbolInfo(Symbol* sym, MapSymbolData* mapSymbolData)
     {
-        // We don't need to create slot for or save "arguments"
-        bool needScopeSlot = !sym->IsArguments() && sym->GetHasNonLocalReference();
+        bool needScopeSlot = sym->GetHasNonLocalReference();
         Js::PropertyId scopeSlot = Constants::NoSlot;
 
         if (sym->GetIsModuleExportStorage())

lib/Runtime/ByteCode/ScopeInfo.cpp

@@ -1,2 +1,2 @@
-Lambda functions are not supported.
+lambda functions are not allowed
 Asm.js compilation failed.

test/AsmJs/lambda.baseline

@@ -0,0 +1,115 @@
+//-------------------------------------------------------------------------------------------------------
+// Copyright (C) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
+//-------------------------------------------------------------------------------------------------------
+
+WScript.LoadScriptFile("..\\UnitTestFramework\\UnitTestFramework.js");
+
+var tests = [
+    {
+        name: "Simple lambda function deferral",
+        body: function () {
+            var a = () => { return 123 };
+            assert.areEqual(123, a(), "Lambda with no args but empty parens and body surrounded with curly-braces");
+            
+            var b = (arg) => { return arg; };
+            assert.areEqual(123, b(123), "Lambda with an arg in parens");
+            
+            var c = (arg1, arg2) => { return arg1 + arg2; };
+            assert.areEqual(2, c(1, 1), "Lambda with two args in parens");
+            
+            var d = () => 123
+            assert.areEqual(123, d(), "Lambda with empty arg list and single expression-body");
+            
+            var e = arg => arg
+            assert.areEqual(123, e(123), "Lambda with single arg and single expression-body");
+            
+            var f = arg => { return arg }
+            assert.areEqual(123, f(123), "Lambda with single arg and body in curly-braces");
+            
+            var g = (arg1, arg2) => arg1 + arg2
+            assert.areEqual(2, g(1, 1), "Lambda with two args in parens and single expression body");
+        }
+    },
+    {
+        name: "Global lambda function deferral",
+        body: function () {
+            WScript.LoadScript(`
+                var a = () => { return 123 };
+                assert.areEqual(123, a(), "Lambda with no args but empty parens and body surrounded with curly-braces");
+                
+                var b = (arg) => { return arg; };
+                assert.areEqual(123, b(123), "Lambda with an arg in parens");
+                
+                var c = (arg1, arg2) => { return arg1 + arg2; };
+                assert.areEqual(2, c(1, 1), "Lambda with two args in parens");
+                
+                var d = () => 123
+                assert.areEqual(123, d(), "Lambda with empty arg list and single expression-body");
+                
+                var e = arg => arg
+                assert.areEqual(123, e(123), "Lambda with single arg and single expression-body");
+                
+                var f = arg => { return arg }
+                assert.areEqual(123, f(123), "Lambda with single arg and body in curly-braces");
+                
+                var g = (arg1, arg2) => arg1 + arg2
+                assert.areEqual(2, g(1, 1), "Lambda with two args in parens and single expression body");
+            `);
+        }
+    },
+    {
+        name: "Async lambda function deferral",
+        body: function () {
+            var a = async () => { return 123 };
+            assert.isTrue(a() instanceof Promise, "Lambda with no args but empty parens and body surrounded with curly-braces");
+            
+            var b = async (arg) => { return arg; };
+            assert.isTrue(b() instanceof Promise, "Lambda with an arg in parens");
+            
+            var c = async (arg1, arg2) => { return arg1 + arg2; };
+            assert.isTrue(c() instanceof Promise, "Lambda with two args in parens");
+            
+            var d = async () => 123
+            assert.isTrue(d() instanceof Promise, "Lambda with empty arg list and single expression-body");
+            
+            var e = async arg => arg
+            assert.isTrue(e() instanceof Promise, "Lambda with single arg and single expression-body");
+            
+            var f = async arg => { return arg }
+            assert.isTrue(f() instanceof Promise, "Lambda with single arg and body in curly-braces");
+            
+            var g = async (arg1, arg2) => arg1 + arg2
+            assert.isTrue(g() instanceof Promise, "Lambda with two args in parens and single expression body");
+        }
+    },
+    {
+        name: "Global async lambda function deferral",
+        body: function () {
+            WScript.LoadScript(`
+                var a = async () => { return 123 };
+                assert.isTrue(a() instanceof Promise, "Lambda with no args but empty parens and body surrounded with curly-braces");
+                
+                var b = async (arg) => { return arg; };
+                assert.isTrue(b() instanceof Promise, "Lambda with an arg in parens");
+                
+                var c = async (arg1, arg2) => { return arg1 + arg2; };
+                assert.isTrue(c() instanceof Promise, "Lambda with two args in parens");
+                
+                var d = async () => 123
+                assert.isTrue(d() instanceof Promise, "Lambda with empty arg list and single expression-body");
+                
+                var e = async arg => arg
+                assert.isTrue(e() instanceof Promise, "Lambda with single arg and single expression-body");
+                
+                var f = async arg => { return arg }
+                assert.isTrue(f() instanceof Promise, "Lambda with single arg and body in curly-braces");
+                
+                var g = async (arg1, arg2) => arg1 + arg2
+                assert.isTrue(g() instanceof Promise, "Lambda with two args in parens and single expression body");
+            `);
+        }
+    },
+]
+
+testRunner.runTests(tests, { verbose: WScript.Arguments[0] != "summary" });

test/es6/DeferParseLambda.js

@@ -168,12 +168,10 @@ var tests = [
     {
         name: "Runtime error import statements",
         body: function () {
-            testModuleScript('import foo from "ValidExportStatements.js"; assert.throws(()=>{ foo =12; }, TypeError, "assignment to const");', 'Imported default bindings are constant bindings', false);
-            testModuleScript('import { foo } from "ValidExportStatements.js"; assert.throws(()=>{ foo = 12; }, TypeError, "assignment to const");', 'Imported named bindings are constant bindings', false);
-
-            testModuleScript('import * as foo from "ValidExportStatements.js"; assert.throws(()=>{ foo = 12; }, TypeError, "assignment to const");', 'Namespace import bindings are constant bindings', false);
-
-            testModuleScript('import { foo as foo22 } from "ValidExportStatements.js"; assert.throws(()=>{ foo22 = 12; }, TypeError, "assignment to const");', 'Renamed import bindings are constant bindings', false);
+            testModuleScript('import foo from "ValidExportStatements.js"; try { (() => { foo = 12; })() } catch(e) { assert.areEqual("Assignment to const", e.message); }', 'Imported default bindings are constant bindings', false);
+            testModuleScript('import { foo } from "ValidExportStatements.js"; try { (() => { foo = 12; })() } catch(e) { assert.areEqual("Assignment to const", e.message); }', 'Imported named bindings are constant bindings', false);
+            testModuleScript('import * as foo from "ValidExportStatements.js"; try { (() => { foo = 12; })() } catch(e) { assert.areEqual("Assignment to const", e.message); }', 'Namespace import bindings are constant bindings', false);
+            testModuleScript('import { foo as foo22 } from "ValidExportStatements.js"; try { (() => { foo22 = 12; })() } catch(e) { assert.areEqual("Assignment to const", e.message); }', 'Renamed import bindings are constant bindings', false);
         }
     },
     {