diff --git a/interpreter/cling/include/cling/MetaProcessor/MetaLexer.h b/interpreter/cling/include/cling/MetaProcessor/MetaLexer.h index 193fb4ed221bf..2ebf3d90b74da 100644 --- a/interpreter/cling/include/cling/MetaProcessor/MetaLexer.h +++ b/interpreter/cling/include/cling/MetaProcessor/MetaLexer.h @@ -30,12 +30,15 @@ namespace cling { quest_mark, // "?" slash, // "/" backslash, // "\" + less, // "<" greater, // ">" ampersand, // "&" hash, // "#" ident, // (a-zA-Z)[(0-9a-zA-Z)*] raw_ident, // .*^(' '|'\t') comment, // // + l_comment, // "/*" + r_comment, // "*/" space, // (' ' | '\t')* constant, // {0-9} at, // @ @@ -54,6 +57,7 @@ namespace cling { mutable unsigned value; public: Token(const char* Buf = nullptr) { startToken(Buf); } + Token(const Token&) = default; void startToken(const char* Pos = nullptr) { kind = tok::unknown; @@ -70,6 +74,14 @@ namespace cling { bool isNot(tok::TokenKind K) const { return kind != K; } bool is(tok::TokenKind K) const { return kind == K; } + bool isClosingBrace() const { + return kind == tok::r_square || kind == tok::r_paren || kind == tok::r_brace; + } + ///\brief Return whether this instance matches a opening brace `K`; assumes + /// that `r_xxx` brace follows `l_xxx` on the TokenKind enumeration. + bool closesBrace(tok::TokenKind K) const { + return isClosingBrace() && (kind == K+1); + } llvm::StringRef getIdent() const; llvm::StringRef getIdentNoQuotes() const { @@ -86,11 +98,24 @@ namespace cling { const char* bufferStart; const char* curPos; public: + ///\brief A RAII object that saves the input position and restores it + /// on destruction. + struct RAII { + MetaLexer& Lex; + const char* savedPos; + RAII(MetaLexer& L) : Lex(L), savedPos(L.curPos) {} + ~RAII() { Lex.curPos = savedPos; } + }; + MetaLexer(llvm::StringRef input, bool skipWhiteSpace = false); void reset(llvm::StringRef Line); void Lex(Token& Tok); + ///\brief Lex a tok::raw_ident token that extends until the next whitespace + /// character, i.e. ' ' or '\t'. void LexAnyString(Token& Tok); + ///\brief Lex until '\r' or '\n' and make `Tok` point to consumed data. + void ReadToEndOfLine(Token& Tok, tok::TokenKind K = tok::unknown); static void LexPunctuator(const char* C, Token& Tok); // TODO: Revise. We might not need that. diff --git a/interpreter/cling/lib/MetaProcessor/MetaLexer.cpp b/interpreter/cling/lib/MetaProcessor/MetaLexer.cpp index 0c0c4bccfa668..ba2ee1b20001b 100644 --- a/interpreter/cling/lib/MetaProcessor/MetaLexer.cpp +++ b/interpreter/cling/lib/MetaProcessor/MetaLexer.cpp @@ -57,15 +57,22 @@ namespace cling { case '"': case '\'': return LexQuotedStringAndAdvance(curPos, Tok); case '[': case ']': case '(': case ')': case '{': case '}': - case '\\': case ',': case '.': case '!': case '?': case '>': - case '&': case '#': case '@': case '*': case ';': + case '\\': case ',': case '.': case '!': case '?': case '<': case '>': + case '&': case '#': case '@': case ';': // INTENTIONAL FALL THROUGHs return LexPunctuator(curPos - 1, Tok); case '/': + if (*curPos == '/' || *curPos == '*') { + Tok.setKind((*curPos++ == '/') ? tok::comment : tok::l_comment); + Tok.setLength(2); + return; + } + return LexPunctuator(curPos - 1, Tok); + case '*': if (*curPos == '/') { ++curPos; - Tok.setKind(tok::comment); + Tok.setKind(tok::r_comment); Tok.setLength(2); return; } @@ -106,6 +113,16 @@ namespace cling { Tok.setLength(curPos - Tok.getBufStart()); } + void MetaLexer::ReadToEndOfLine(Token& Tok, tok::TokenKind K) { + Tok.startToken(curPos); + while (*curPos != '\r' && *curPos != '\n' + && *curPos != '\0') + curPos++; + + Tok.setKind(K); + Tok.setLength(curPos - Tok.getBufStart()); + } + void MetaLexer::LexPunctuator(const char* C, Token& Tok) { Tok.startToken(C); Tok.setLength(1); @@ -124,8 +141,9 @@ namespace cling { case '?' : Tok.setKind(tok::quest_mark); break; case '/' : Tok.setKind(tok::slash); break; case '\\' : Tok.setKind(tok::backslash); break; + case '<' : Tok.setKind(tok::less); break; case '>' : Tok.setKind(tok::greater); break; - case '@' : Tok.setKind(tok::at); break; + case '@' : Tok.setKind(tok::at); break; case '&' : Tok.setKind(tok::ampersand); break; case '#' : Tok.setKind(tok::hash); break; case '*' : Tok.setKind(tok::asterik); break;