Pl/Sql & JavaScript lexer fixes

java/JavaParser.g4

@@ -466,7 +466,7 @@ methodCall
 expression
     : primary
     | expression bop='.'
-      (IDENTIFIER
+      ( IDENTIFIER
       | methodCall
       | THIS
       | NEW nonWildcardTypeArguments? innerCreator

javascript/CSharpSharwell/JavaScriptBaseLexer.cs

@@ -56,6 +56,7 @@ public bool IsSrictMode()
     /// token in case it resides on the default channel. This recorded token
     /// is used to determine when the lexer could possibly match a regex
     /// literal.
+    /// 
     /// </summary>
     /// <returns>
     /// The next token from the character stream.
@@ -65,26 +66,7 @@ public override IToken NextToken()
         // Get the next token.
         IToken next = base.NextToken();
 
-        if (next.Type == OpenBrace)
-        {
-            _useStrictCurrent = scopeStrictModes.Count > 0 && scopeStrictModes.Peek() ? true : UseStrictDefault;
-            scopeStrictModes.Push(_useStrictCurrent);
-        }
-        else if (next.Type == CloseBrace)
-        {
-            _useStrictCurrent = scopeStrictModes.Count > 0 ? scopeStrictModes.Pop() : UseStrictDefault;
-        }
-        else if (next.Type == StringLiteral &&
-            (_lastToken == null || _lastToken.Type == OpenBrace) &&
-            (next.Text.Substring(1, next.Text.Length - 2)) == "use strict")
-        {
-            if (scopeStrictModes.Count > 0)
-                scopeStrictModes.Pop();
-            _useStrictCurrent = true;
-            scopeStrictModes.Push(_useStrictCurrent);
-        }
-
-        if (next.Channel == Lexer.DefaultTokenChannel)
+        if (next.Channel == DefaultTokenChannel)
         {
             // Keep track of the last token on the default channel.
             _lastToken = next;
@@ -93,10 +75,35 @@ public override IToken NextToken()
         return next;
     }
 
+    protected void ProcessOpenBrace()
+    {
+        _useStrictCurrent = scopeStrictModes.Count > 0 && scopeStrictModes.Peek() ? true : UseStrictDefault;
+        scopeStrictModes.Push(_useStrictCurrent);
+    }
+
+    protected void ProcessCloseBrace()
+    {
+        _useStrictCurrent = scopeStrictModes.Count > 0 ? scopeStrictModes.Pop() : UseStrictDefault;
+    }
+
+    protected void ProcessStringLiteral()
+    {
+        if (_lastToken == null || _lastToken.Type == OpenBrace)
+        {
+            if (Text.Equals("\"use strict\"") || Text.Equals("'use strict'"))
+            {
+                if (scopeStrictModes.Count > 0)
+                    scopeStrictModes.Pop();
+                _useStrictCurrent = true;
+                scopeStrictModes.Push(_useStrictCurrent);
+            }
+        }
+    }
+
     /// <summary>
     /// Returns true if the lexer can match a regex literal.
     /// </summary>
-    protected bool RegexPossible()
+    protected bool IsRegexPossible()
     {
         if (_lastToken == null)
         {

javascript/Java/JavaScriptBaseLexer.java

@@ -56,25 +56,6 @@ public boolean IsSrictMode() {
     public Token nextToken() {
         Token next = super.nextToken();
 
-        if (next.getType() == JavaScriptLexer.OpenBrace)
-        {
-            useStrictCurrent = scopeStrictModes.size() > 0 && scopeStrictModes.peek() ? true : useStrictDefault;
-            scopeStrictModes.push(useStrictCurrent);
-        }
-        else if (next.getType() == JavaScriptLexer.CloseBrace)
-        {
-            useStrictCurrent = scopeStrictModes.size() > 0 ? scopeStrictModes.pop() : useStrictDefault;
-        }
-        else if (next.getType() == JavaScriptLexer.StringLiteral &&
-                (lastToken == null || lastToken.getType() == JavaScriptLexer.OpenBrace) &&
-                (next.getText().substring(1, next.getText().length() - 1)).equals("use strict"))
-        {
-            if (scopeStrictModes.size() > 0)
-                scopeStrictModes.pop();
-            useStrictCurrent = true;
-            scopeStrictModes.push(useStrictCurrent);
-        }
-
         if (next.getChannel() == Token.DEFAULT_CHANNEL) {
             // Keep track of the last token on the default channel.
             this.lastToken = next;
@@ -83,10 +64,36 @@ else if (next.getType() == JavaScriptLexer.StringLiteral &&
         return next;
     }
 
+    protected void ProcessOpenBrace()
+    {
+        useStrictCurrent = scopeStrictModes.size() > 0 && scopeStrictModes.peek() ? true : useStrictDefault;
+        scopeStrictModes.push(useStrictCurrent);
+    }
+
+    protected void ProcessCloseBrace()
+    {
+        useStrictCurrent = scopeStrictModes.size() > 0 ? scopeStrictModes.pop() : useStrictDefault;
+    }
+
+    protected void ProcessStringLiteral()
+    {
+        if (lastToken == null || lastToken.getType() == JavaScriptLexer.OpenBrace)
+        {
+            String text = getText();
+            if (text.equals("\"use strict\"") || text.equals("'use strict'"))
+            {
+                if (scopeStrictModes.size() > 0)
+                    scopeStrictModes.pop();
+                useStrictCurrent = true;
+                scopeStrictModes.push(useStrictCurrent);
+            }
+        }
+    }
+
     /**
      * Returns {@code true} if the lexer can match a regex literal.
      */
-    protected boolean RegexPossible() {
+    protected boolean IsRegexPossible() {
 
         if (this.lastToken == null) {
             // No token has been produced yet: at the start of the input,

javascript/JavaScriptLexer.g4

@@ -28,21 +28,20 @@
  */
 lexer grammar JavaScriptLexer;
 
-options {
-    superClass=JavaScriptBaseLexer;
-}
+channels { ERROR }
 
-RegularExpressionLiteral:       {RegexPossible()}? '/' RegularExpressionBody '/' RegularExpressionFlags;
+options { superClass=JavaScriptBaseLexer; }
 
-/// Line Terminators
-LineTerminator:                 [\r\n\u2028\u2029] -> channel(HIDDEN);
+MultiLineComment:               '/*' .*? '*/'             -> channel(HIDDEN);
+SingleLineComment:              '//' ~[\r\n\u2028\u2029]* -> channel(HIDDEN);
+RegularExpressionLiteral:       '/' RegularExpressionChar+ {IsRegexPossible()}? '/' IdentifierPart*;
 
 OpenBracket:                    '[';
 CloseBracket:                   ']';
 OpenParen:                      '(';
 CloseParen:                     ')';
-OpenBrace:                      '{';
-CloseBrace:                     '}';
+OpenBrace:                      '{' {ProcessOpenBrace();};
+CloseBrace:                     '}' {ProcessCloseBrace();};
 SemiColon:                      ';';
 Comma:                          ',';
 Assign:                         '=';
@@ -107,7 +106,7 @@ DecimalLiteral:                 DecimalIntegerLiteral '.' [0-9]* ExponentPart?
 /// Numeric Literals
 
 HexIntegerLiteral:              '0' [xX] HexDigit+;
-OctalIntegerLiteral:            {!IsSrictMode()}? '0' [0-7]+;
+OctalIntegerLiteral:            '0' [0-7]+ {!IsSrictMode()}?;
 OctalIntegerLiteral2:           '0' [oO] [0-7]+;
 BinaryIntegerLiteral:           '0' [bB] [01]+;
 
@@ -153,36 +152,37 @@ Import:                         'import';
 /// The following tokens are also considered to be FutureReservedWords 
 /// when parsing strict mode
 
-Implements:                     {IsSrictMode()}? 'implements';
-Let:                            {IsSrictMode()}? 'let';
-Private:                        {IsSrictMode()}? 'private';
-Public:                         {IsSrictMode()}? 'public';
-Interface:                      {IsSrictMode()}? 'interface';
-Package:                        {IsSrictMode()}? 'package';
-Protected:                      {IsSrictMode()}? 'protected';
-Static:                         {IsSrictMode()}? 'static';
-Yield:                          {IsSrictMode()}? 'yield';
+Implements:                     'implements' {IsSrictMode()}?;
+Let:                            'let' {IsSrictMode()}?;
+Private:                        'private' {IsSrictMode()}?;
+Public:                         'public' {IsSrictMode()}?;
+Interface:                      'interface' {IsSrictMode()}?;
+Package:                        'package' {IsSrictMode()}?;
+Protected:                      'protected' {IsSrictMode()}?;
+Static:                         'static' {IsSrictMode()}?;
+Yield:                          'yield' {IsSrictMode()}?;
 
 /// Identifier Names and Identifiers
 
 Identifier:                     IdentifierStart IdentifierPart*;
 
 /// String Literals
-StringLiteral:                  '"' DoubleStringCharacter* '"'
-            |                   '\'' SingleStringCharacter* '\''
-            ;
+StringLiteral:                 ('"' DoubleStringCharacter* '"'
+             |                  '\'' SingleStringCharacter* '\'') {ProcessStringLiteral();}
+             ;
 
 TemplateStringLiteral:          '`' ('\\`' | ~'`')* '`';
 
 WhiteSpaces:                    [\t\u000B\u000C\u0020\u00A0]+ -> channel(HIDDEN);
 
+LineTerminator:                 [\r\n\u2028\u2029] -> channel(HIDDEN);
+
 /// Comments
 
-MultiLineComment:               '/*' .*? '*/' -> channel(HIDDEN);
-SingleLineComment:              '//' ~[\r\n\u2028\u2029]* -> channel(HIDDEN);
+
 HtmlComment:                    '<!--' .*? '-->' -> channel(HIDDEN);
 CDataComment:                   '<![CDATA[' .*? ']]>' -> channel(HIDDEN);
-UnexpectedCharacter:            . ;
+UnexpectedCharacter:            . -> channel(ERROR);
 
 // Fragment rules
 
@@ -238,12 +238,7 @@ fragment EscapeCharacter
     ;
 
 fragment LineContinuation
-    : '\\' LineTerminatorSequence 
-    ;
-
-fragment LineTerminatorSequence
-    : '\r\n'
-    | LineTerminator
+    : '\\' [\r\n\u2028\u2029]
     ;
 
 fragment HexDigit
@@ -259,19 +254,19 @@ fragment ExponentPart
     : [eE] [+-]? [0-9]+
     ;
 
-fragment IdentifierStart
-    : UnicodeLetter
-    | [$_]
-    | '\\' UnicodeEscapeSequence
-    ;
-
 fragment IdentifierPart
     : IdentifierStart
     | UnicodeCombiningMark
     | UnicodeDigit
     | UnicodeConnectorPunctuation
-    | ZWNJ
-    | ZWJ
+    | '\u200C'
+    | '\u200D'
+    ;
+
+fragment IdentifierStart
+    : UnicodeLetter
+    | [$_]
+    | '\\' UnicodeEscapeSequence
     ;
 
 fragment UnicodeLetter
@@ -674,47 +669,17 @@ fragment UnicodeConnectorPunctuation
     | [\uFF65]
     ;
 
-fragment ZWNJ
-    : '\u200C'
-    ;
-
-fragment ZWJ
-    : '\u200D'
-    ;
-
-fragment RegularExpressionBody
-    : RegularExpressionFirstChar RegularExpressionChar*
-    ;
-
-fragment RegularExpressionFlags
-    : IdentifierPart*
-    ;
-
-fragment RegularExpressionFirstChar
-    : ~[\r\n\u2028\u2029*\\/[]
-    | RegularExpressionBackslashSequence
-    | RegularExpressionClass
-    ;
-
 fragment RegularExpressionChar
     : ~[\r\n\u2028\u2029\\/[]
     | RegularExpressionBackslashSequence
-    | RegularExpressionClass
-    ;
-
-fragment RegularExpressionNonTerminator
-    : ~[\r\n\u2028\u2029]
-    ;
-
-fragment RegularExpressionBackslashSequence
-    : '\\' RegularExpressionNonTerminator
-    ;
-
-fragment RegularExpressionClass
-    : '[' RegularExpressionClassChar* ']'
+    | '[' RegularExpressionClassChar* ']'
     ;
 
 fragment RegularExpressionClassChar
     : ~[\r\n\u2028\u2029\]\\]
     | RegularExpressionBackslashSequence
     ;
+
+fragment RegularExpressionBackslashSequence
+    : '\\' ~[\r\n\u2028\u2029]
+    ;

javascript/examples/StrictFunctions.js

@@ -1,4 +1,7 @@
 function strict() {
+   /* comment */
+   // comment
+   // comment
   'use strict';
   function nested()
   {

plsql/CSharp/PlSqlBaseLexer.cs

@@ -0,0 +1,15 @@
+using Antlr4.Runtime;
+
+public abstract class PlSqlBaseLexer : Lexer
+{
+    public PlSqlBaseLexer(ICharStream input)
+        : base(input)
+    {
+    }
+
+    protected bool IsNewlineAtPos(int pos)
+    {
+        int la = _input.La(pos);
+        return la == -1 || la == '\n';
+    }
+}

plsql/CSharp/PlSqlBaseParser.cs

@@ -0,0 +1,15 @@
+using Antlr4.Runtime;
+
+public abstract class PlSqlBaseParser : Parser
+{
+    private bool _isVersion12 = true;
+
+    public PlSqlBaseParser(ITokenStream input)
+        : base(input)
+    {
+    }
+
+    public bool isVersion12() => _isVersion12;
+
+    public bool setVersion12(bool value) => _isVersion12 = value;
+}

plsql/Java/PlSqlBaseLexer.java

@@ -0,0 +1,15 @@
+import org.antlr.v4.runtime.*;
+
+public abstract class PlSqlBaseLexer extends Lexer
+{
+    public PlSqlBaseLexer(ICharStream input)
+        : base(input)
+    {
+    }
+
+    protected boolean IsNewlineAtPos(int pos)
+    {
+        int la = _input.LA(pos);
+        return la == -1 || la == '\n';
+    }
+}