A new lexer command: Less

contributors.txt

@@ -290,6 +290,7 @@ YYYY/MM/DD, github id, Full name, email
 2021/01/25, l215884529, Qiheng Liu, 13607681+l215884529@users.noreply.github.com
 2021/02/02, tsotnikov, Taras Sotnikov, taras.sotnikov@gmail.com
 2021/02/10, jirislaby, Jiri Slaby, jirislaby@gmail.com
+2021/02/19, dkleszyk, David Kleszyk, dkleszyk@gmail.com
 2021/02/21, namasikanam, Xingyu Xie, namasikanam@gmail.com
 2021/02/26, ahooper, Andrew Hooper, ahooper at kos dot net
 2021/02/27, khmarbaise, Karl Heinz Marbaise, github@soebes.com

doc/lexer-rules.md

@@ -220,6 +220,7 @@ An alternative can have more than one command separated by commas. Here are the
 
 * skip
 * more
+* less
 * popMode
 * mode( x )
 * pushMode( x )
@@ -271,6 +272,26 @@ TEXT : . -> more ; // collect more text for string
 
 Popping the bottom layer of a mode stack will result in an exception. Switching modes with `mode` changes the current stack top.  More than one `more` is the same as just one and the position does not matter.
 
+### less
+
+The 'less' is similar to the 'more' command; it forces the lexer to get another token without throwing out the current text. However, the 'less' command will also rewind the lexer position to the start of the token before resuming.
+
+As a comparison:
+
+More:
+
+```
+// When the PROC_INSTR mode is entered, the lexer will continue matching after the "Name"
+SPECIAL_OPEN : '<?' Name -> more, pushMode(PROC_INSTR);
+```
+
+Less:
+
+```
+// When the PROC_INSTR mode is entered, the lexer will restart matching at the opening tag ("<?")
+SPECIAL_OPEN : '<?' Name -> less, pushMode(PROC_INSTR);
+```
+
 ### type()
 
 ```

runtime/CSharp/src/Atn/ATNDeserializer.cs

@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+/* Copyright (c) 2012-2021 The ANTLR Project. All rights reserved.
  * Use of this file is governed by the BSD 3-clause license that
  * can be found in the LICENSE.txt file in the project root.
  */
@@ -1284,6 +1284,11 @@ protected internal virtual ILexerAction LexerActionFactory(LexerActionType type,
                     return new LexerCustomAction(data1, data2);
                 }
 
+                case LexerActionType.Less:
+                {
+                    return LexerLessAction.Instance;
+                }
+
                 case LexerActionType.Mode:
                 {
                     return new LexerModeAction(data1);

runtime/CSharp/src/Atn/LexerActionType.cs

@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+/* Copyright (c) 2012-2021 The ANTLR Project. All rights reserved.
  * Use of this file is governed by the BSD 3-clause license that
  * can be found in the LICENSE.txt file in the project root.
  */
@@ -18,6 +18,8 @@ public enum LexerActionType
         PopMode,
         PushMode,
         Skip,
-        Type
+        Type,
+        /// <since>4.9.3</since>
+        Less
     }
 }

runtime/CSharp/src/Atn/LexerLessAction.cs

@@ -0,0 +1,98 @@
+/* Copyright (c) 2012-2021 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+using Antlr4.Runtime;
+using Antlr4.Runtime.Atn;
+using Antlr4.Runtime.Misc;
+using Antlr4.Runtime.Sharpen;
+
+namespace Antlr4.Runtime.Atn
+{
+    /// <summary>
+    /// Implements the
+    /// <c>less</c>
+    /// lexer action by calling
+    /// <see cref="Antlr4.Runtime.Lexer.Less()"/>
+    /// .
+    /// <p>The
+    /// <c>less</c>
+    /// command does not have any parameters, so this action is
+    /// implemented as a singleton instance exposed by
+    /// <see cref="Instance"/>
+    /// .</p>
+    /// </summary>
+    /// <author>David Kleszyk</author>
+    /// <since>4.9.3</since>
+    public sealed class LexerLessAction : ILexerAction
+    {
+        /// <summary>Provides a singleton instance of this parameterless lexer action.</summary>
+        /// <remarks>Provides a singleton instance of this parameterless lexer action.</remarks>
+        public static readonly Antlr4.Runtime.Atn.LexerLessAction Instance = new Antlr4.Runtime.Atn.LexerLessAction();
+
+        /// <summary>
+        /// Constructs the singleton instance of the lexer
+        /// <c>less</c>
+        /// command.
+        /// </summary>
+        private LexerLessAction()
+        {
+        }
+
+        /// <summary><inheritDoc/></summary>
+        /// <returns>
+        /// This method returns
+        /// <see cref="LexerActionType.Less"/>
+        /// .
+        /// </returns>
+        public LexerActionType ActionType
+        {
+            get
+            {
+                return LexerActionType.Less;
+            }
+        }
+
+        /// <summary><inheritDoc/></summary>
+        /// <returns>
+        /// This method returns
+        /// <see langword="false"/>
+        /// .
+        /// </returns>
+        public bool IsPositionDependent
+        {
+            get
+            {
+                return false;
+            }
+        }
+
+        /// <summary>
+        /// <inheritDoc/>
+        /// <p>This action is implemented by calling
+        /// <see cref="Antlr4.Runtime.Lexer.Less()"/>
+        /// .</p>
+        /// </summary>
+        public void Execute(Lexer lexer)
+        {
+            lexer.Less();
+        }
+
+        public override int GetHashCode()
+        {
+            int hash = MurmurHash.Initialize();
+            hash = MurmurHash.Update(hash, (int)(ActionType));
+            return MurmurHash.Finish(hash, 1);
+        }
+
+        public override bool Equals(object obj)
+        {
+            return obj == this;
+        }
+
+        public override string ToString()
+        {
+            return "less";
+        }
+    }
+}

runtime/CSharp/src/Lexer.cs

@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+/* Copyright (c) 2012-2021 The ANTLR Project. All rights reserved.
  * Use of this file is governed by the BSD 3-clause license that
  * can be found in the LICENSE.txt file in the project root.
  */
@@ -192,8 +192,14 @@ public virtual IToken NextToken()
                         {
                             goto outer_continue;
                         }
+                        if (_type == TokenTypes.Less)
+                        {
+                            _input.Seek(_tokenStartCharIndex);
+                            this.Column = _tokenStartColumn;
+                            this.Line = _tokenStartLine;
+                        }
                     }
-                    while (_type == TokenTypes.More);
+                    while (_type == TokenTypes.More || _type == TokenTypes.Less);
                     if (_token == null)
                     {
                         Emit();
@@ -231,6 +237,11 @@ public virtual void More()
             _type = TokenTypes.More;
         }
 
+        public virtual void Less()
+        {
+            _type = TokenTypes.Less;
+        }
+
         public virtual void Mode(int m)
         {
             _mode = m;

runtime/CSharp/src/TokenTypes.cs

@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+/* Copyright (c) 2012-2021 The ANTLR Project. All rights reserved.
  * Use of this file is governed by the BSD 3-clause license that
  * can be found in the LICENSE.txt file in the project root.
  */
@@ -9,5 +9,7 @@ public static class TokenTypes
         public const int More = -2;
 
         public const int Skip = -3;
+
+        public const int Less = -4;
     }
 }

runtime/Cpp/runtime/src/Lexer.cpp

@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+/* Copyright (c) 2012-2021 The ANTLR Project. All rights reserved.
  * Use of this file is governed by the BSD 3-clause license that
  * can be found in the LICENSE.txt file in the project root.
  */
@@ -93,7 +93,12 @@ std::unique_ptr<Token> Lexer::nextToken() {
       if (type == SKIP) {
         goto outerContinue;
       }
-    } while (type == MORE);
+      if (type == LESS) {
+        _input->seek(tokenStartCharIndex);
+        setCharPositionInLine(tokenStartCharPositionInLine);
+        setLine(tokenStartLine);
+      }
+    } while (type == MORE || type == LESS);
     if (token == nullptr) {
       emit();
     }
@@ -109,6 +114,10 @@ void Lexer::more() {
   type = MORE;
 }
 
+void Lexer::less() {
+  type = LESS;
+}
+
 void Lexer::setMode(size_t m) {
   mode = m;
 }

runtime/Cpp/runtime/src/Lexer.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+/* Copyright (c) 2012-2021 The ANTLR Project. All rights reserved.
  * Use of this file is governed by the BSD 3-clause license that
  * can be found in the LICENSE.txt file in the project root.
  */
@@ -22,6 +22,7 @@ namespace antlr4 {
     static constexpr size_t DEFAULT_MODE = 0;
     static constexpr size_t MORE = std::numeric_limits<size_t>::max() - 1;
     static constexpr size_t SKIP = std::numeric_limits<size_t>::max() - 2;
+    static constexpr size_t LESS = std::numeric_limits<size_t>::max() - 3;
 
     static constexpr size_t DEFAULT_TOKEN_CHANNEL = Token::DEFAULT_CHANNEL;
     static constexpr size_t HIDDEN = Token::HIDDEN_CHANNEL;
@@ -32,6 +33,7 @@ namespace antlr4 {
       DEFAULT_MODE = 0,
       MORE = static_cast<size_t>(-2), // std::numeric_limits<size_t>::max() - 1; doesn't work in VS 2013
       SKIP = static_cast<size_t>(-3), // std::numeric_limits<size_t>::max() - 2; doesn't work in VS 2013
+      LESS = static_cast<size_t>(-4), // std::numeric_limits<size_t>::max() - 3; doesn't work in VS 2013
 
       DEFAULT_TOKEN_CHANNEL = Token::DEFAULT_CHANNEL,
       HIDDEN = Token::HIDDEN_CHANNEL,
@@ -104,6 +106,7 @@ namespace antlr4 {
     /// and emits it.
     virtual void skip();
     virtual void more();
+    virtual void less();
     virtual void setMode(size_t m);
     virtual void pushMode(size_t m);
     virtual size_t popMode();

runtime/Cpp/runtime/src/antlr4-runtime.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+/* Copyright (c) 2012-2021 The ANTLR Project. All rights reserved.
  * Use of this file is governed by the BSD 3-clause license that
  * can be found in the LICENSE.txt file in the project root.
  */
@@ -86,6 +86,7 @@
 #include "atn/LexerChannelAction.h"
 #include "atn/LexerCustomAction.h"
 #include "atn/LexerIndexedCustomAction.h"
+#include "atn/LexerLessAction.h"
 #include "atn/LexerModeAction.h"
 #include "atn/LexerMoreAction.h"
 #include "atn/LexerPopModeAction.h"

runtime/Cpp/runtime/src/atn/ATNDeserializer.cpp

@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+/* Copyright (c) 2012-2021 The ANTLR Project. All rights reserved.
  * Use of this file is governed by the BSD 3-clause license that
  * can be found in the LICENSE.txt file in the project root.
  */
@@ -42,6 +42,7 @@
 
 #include "atn/LexerCustomAction.h"
 #include "atn/LexerChannelAction.h"
+#include "atn/LexerLessAction.h"
 #include "atn/LexerModeAction.h"
 #include "atn/LexerMoreAction.h"
 #include "atn/LexerPopModeAction.h"
@@ -731,6 +732,9 @@ Ref<LexerAction> ATNDeserializer::lexerActionFactory(LexerActionType type, int d
     case LexerActionType::CUSTOM:
       return std::make_shared<LexerCustomAction>(data1, data2);
 
+    case LexerActionType::LESS:
+      return LexerLessAction::getInstance();
+
     case LexerActionType::MODE:
       return std::make_shared< LexerModeAction>(data1);
 

runtime/Cpp/runtime/src/atn/ATNSerializer.cpp

@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+/* Copyright (c) 2012-2021 The ANTLR Project. All rights reserved.
  * Use of this file is governed by the BSD 3-clause license that
  * can be found in the LICENSE.txt file in the project root.
  */
@@ -307,6 +307,11 @@ std::vector<size_t> ATNSerializer::serialize() {
           break;
         }
 
+        case LexerActionType::LESS:
+          data.push_back(0);
+          data.push_back(0);
+          break;
+
         case LexerActionType::MODE:
         {
           int mode = std::dynamic_pointer_cast<LexerModeAction>(action)->getMode();

runtime/Cpp/runtime/src/atn/LexerActionType.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+/* Copyright (c) 2012-2021 The ANTLR Project. All rights reserved.
  * Use of this file is governed by the BSD 3-clause license that
  * can be found in the LICENSE.txt file in the project root.
  */
@@ -49,6 +49,12 @@ namespace atn {
     /// The type of a <seealso cref="LexerTypeAction"/> action.
     /// </summary>
     TYPE,
+    /// <summary>
+    /// The type of a <seealso cref="LexerLessAction"/> action.
+    ///
+    /// @since 4.9.3
+    /// </summary>
+    LESS,
   };
 
 } // namespace atn