[cling] MetaLexer: return /* and */ as tokens; added `ReadToEndOf…

…Line()`

This commit includes the following changes to MetaLexer:
- Update `MetaLexer::Lex()` to return `/*` (tok::l_comment), `*/` (tok::r_comment),
and `<` (tok::less) as tokens.

- Added `MetaLexer::ReadToEndOfLine()` function: consume input until `\r` or `\n`.

- Added `MetaLexer::RAII`, a RAII object that saves/restores the current position.

interpreter/cling/include/cling/MetaProcessor/MetaLexer.h

@@ -30,12 +30,15 @@ namespace cling {
       quest_mark, // "?"
       slash,      // "/"
       backslash,  // "\"
+      less,       // "<"
       greater,    // ">"
       ampersand,  // "&"
       hash,       // "#"
       ident,      // (a-zA-Z)[(0-9a-zA-Z)*]
       raw_ident,  // .*^(' '|'\t')
       comment,    // //
+      l_comment,  // "/*"
+      r_comment,  // "*/"
       space,      // (' ' | '\t')*
       constant,   // {0-9}
       at,         // @
@@ -54,6 +57,7 @@ namespace cling {
     mutable unsigned value;
   public:
     Token(const char* Buf = nullptr) { startToken(Buf); }
+    Token(const Token&) = default;
 
     void startToken(const char* Pos = nullptr) {
       kind = tok::unknown;
@@ -70,6 +74,14 @@ namespace cling {
 
     bool isNot(tok::TokenKind K) const { return kind != K; }
     bool is(tok::TokenKind K) const { return kind == K; }
+    bool isClosingBrace() const {
+      return kind == tok::r_square || kind == tok::r_paren || kind == tok::r_brace;
+    }
+    ///\brief Return whether this instance matches a opening brace `K`; assumes
+    /// that `r_xxx` brace follows `l_xxx` on the TokenKind enumeration.
+    bool closesBrace(tok::TokenKind K) const {
+      return isClosingBrace() && (kind == K+1);
+    }
 
     llvm::StringRef getIdent() const;
     llvm::StringRef getIdentNoQuotes() const {
@@ -86,11 +98,24 @@ namespace cling {
     const char* bufferStart;
     const char* curPos;
   public:
+    ///\brief A RAII object that saves the input position and restores it
+    /// on destruction.
+    struct RAII {
+      MetaLexer&  Lex;
+      const char* savedPos;
+      RAII(MetaLexer& L) : Lex(L), savedPos(L.curPos) {}
+      ~RAII() { Lex.curPos = savedPos; }
+    };
+
     MetaLexer(llvm::StringRef input, bool skipWhiteSpace = false);
     void reset(llvm::StringRef Line);
 
     void Lex(Token& Tok);
+    ///\brief Lex a tok::raw_ident token that extends until the next whitespace
+    /// character, i.e. ' ' or '\t'.
     void LexAnyString(Token& Tok);
+    ///\brief Lex until '\r' or '\n' and make `Tok` point to consumed data.
+    void ReadToEndOfLine(Token& Tok, tok::TokenKind K = tok::unknown);
 
     static void LexPunctuator(const char* C, Token& Tok);
     // TODO: Revise. We might not need that.

interpreter/cling/lib/MetaProcessor/MetaLexer.cpp

@@ -57,15 +57,22 @@ namespace cling {
     case '"': case '\'':
       return LexQuotedStringAndAdvance(curPos, Tok);
     case '[': case ']': case '(': case ')': case '{': case '}':
-    case '\\': case ',': case '.': case '!': case '?': case '>':
-    case '&': case '#': case '@': case '*': case ';':
+    case '\\': case ',': case '.': case '!': case '?': case '<': case '>':
+    case '&': case '#': case '@': case ';':
       // INTENTIONAL FALL THROUGHs
       return LexPunctuator(curPos - 1, Tok);
 
     case '/':
+      if (*curPos == '/' || *curPos == '*') {
+        Tok.setKind((*curPos++ == '/') ? tok::comment : tok::l_comment);
+        Tok.setLength(2);
+        return;
+      }
+      return LexPunctuator(curPos - 1, Tok);
+    case '*':
       if (*curPos == '/') {
         ++curPos;
-        Tok.setKind(tok::comment);
+        Tok.setKind(tok::r_comment);
         Tok.setLength(2);
         return;
       }
@@ -106,6 +113,16 @@ namespace cling {
     Tok.setLength(curPos - Tok.getBufStart());
   }
 
+  void MetaLexer::ReadToEndOfLine(Token& Tok, tok::TokenKind K) {
+    Tok.startToken(curPos);
+    while (*curPos != '\r' && *curPos != '\n'
+           && *curPos != '\0')
+      curPos++;
+
+    Tok.setKind(K);
+    Tok.setLength(curPos - Tok.getBufStart());
+  }
+
   void MetaLexer::LexPunctuator(const char* C, Token& Tok) {
     Tok.startToken(C);
     Tok.setLength(1);
@@ -124,8 +141,9 @@ namespace cling {
     case '?'  : Tok.setKind(tok::quest_mark); break;
     case '/'  : Tok.setKind(tok::slash); break;
     case '\\' : Tok.setKind(tok::backslash); break;
+    case '<'  : Tok.setKind(tok::less); break;
     case '>'  : Tok.setKind(tok::greater); break;
-    case '@' : Tok.setKind(tok::at); break;
+    case '@'  : Tok.setKind(tok::at); break;
     case '&'  : Tok.setKind(tok::ampersand); break;
     case '#'  : Tok.setKind(tok::hash); break;
     case '*'  : Tok.setKind(tok::asterik); break;