Token.h

//===--- Token.h - Token interface ------------------------------*- C++ -*-===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
//  This file defines the Token interface.
//
//===----------------------------------------------------------------------===//

#ifndef SWIFT_TOKEN_H
#define SWIFT_TOKEN_H

#include "swift/Basic/SourceLoc.h"
#include "swift/Basic/LLVM.h"
#include "swift/Syntax/TokenKinds.h"
#include "swift/Config.h"
#include "llvm/ADT/StringRef.h"

namespace swift {

/// Token - This structure provides full information about a lexed token.
/// It is not intended to be space efficient, it is intended to return as much
/// information as possible about each returned token.  This is expected to be
/// compressed into a smaller form if memory footprint is important.
///
class Token {
  /// Kind - The actual flavor of token this is.
  ///
  tok Kind;

  /// \brief Whether this token is the first token on the line.
  unsigned AtStartOfLine : 1;

  /// \brief The length of the comment that precedes the token.
  ///
  /// Hopefully 128 Mib is enough.
  unsigned CommentLength : 27;
  
  /// \brief Whether this token is an escaped `identifier` token.
  unsigned EscapedIdentifier : 1;
  
  /// Modifiers for string literals
  unsigned MultilineString : 1;

  /// Text - The actual string covered by the token in the source buffer.
  StringRef Text;

  StringRef trimComment() const {
    assert(hasComment() && "Has no comment to trim.");
    StringRef Raw(Text.begin() - CommentLength, CommentLength);
    return Raw.trim();
  }

public:
  Token() : Kind(tok::NUM_TOKENS), AtStartOfLine(false), CommentLength(0),
            EscapedIdentifier(false) {}

  Token(tok Kind, StringRef Text, unsigned CommentLength)
          : Kind(Kind), AtStartOfLine(false), CommentLength(CommentLength),
            EscapedIdentifier(false), MultilineString(false),
            Text(Text) {}

  Token(tok Kind, StringRef Text): Token(Kind, Text, 0) {};

  tok getKind() const { return Kind; }
  void setKind(tok K) { Kind = K; }
  void clearCommentLength() { CommentLength = 0; }
  
  /// is/isNot - Predicates to check if this token is a specific kind, as in
  /// "if (Tok.is(tok::l_brace)) {...}".
  bool is(tok K) const { return Kind == K; }
  bool isNot(tok K) const { return Kind != K; }

  // Predicates to check to see if the token is any of a list of tokens.

  bool isAny(tok K1) const {
    return is(K1);
  }
  template <typename ...T>
  bool isAny(tok K1, tok K2, T... K) const {
    if (is(K1))
      return true;
    return isAny(K2, K...);
  }

  // Predicates to check to see if the token is not the same as any of a list.
  template <typename ...T>
  bool isNot(tok K1, T... K) const { return !isAny(K1, K...); }

  bool isBinaryOperator() const {
    return Kind == tok::oper_binary_spaced || Kind == tok::oper_binary_unspaced;
  }
  
  bool isAnyOperator() const {
    return isBinaryOperator() || Kind == tok::oper_postfix ||
           Kind == tok::oper_prefix;
  }
  bool isNotAnyOperator() const {
    return !isAnyOperator();
  }

  bool isEllipsis() const {
    return isAnyOperator() && Text == "...";
  }
  bool isNotEllipsis() const {
    return !isEllipsis();
  }

  /// \brief Determine whether this token occurred at the start of a line.
  bool isAtStartOfLine() const { return AtStartOfLine; }

  /// \brief Set whether this token occurred at the start of a line.
  void setAtStartOfLine(bool value) { AtStartOfLine = value; }
  
  /// \brief True if this token is an escaped identifier token.
  bool isEscapedIdentifier() const { return EscapedIdentifier; }
  /// \brief Set whether this token is an escaped identifier token.
  void setEscapedIdentifier(bool value) {
    assert((!value || Kind == tok::identifier) &&
           "only identifiers can be escaped identifiers");
    EscapedIdentifier = value;
  }
  
  bool isContextualKeyword(StringRef ContextKW) const {
    return is(tok::identifier) && !isEscapedIdentifier() &&
           Text == ContextKW;
  }
  
  /// Return true if this is a contextual keyword that could be the start of a
  /// decl.
  bool isContextualDeclKeyword() const {
    if (isNot(tok::identifier) || isEscapedIdentifier() || Text.empty())
      return false;

    switch (Text[0]) {
    case 'c':
      return Text == "convenience";
    case 'd':
      return Text == "dynamic";
    case 'f':
      return Text == "final";
    case 'i':
      return Text == "indirect" || Text == "infix";
    case 'l':
      return Text == "lazy";
    case 'm':
      return Text == "mutating";
    case 'n':
      return Text == "nonmutating";
    case 'o':
      return Text == "open" || Text == "override" || Text == "optional";
    case 'p':
      return Text == "prefix" || Text == "postfix";
    case 'r':
      return Text == "required";
    case 'u':
      return Text == "unowned";
    case 'w':
      return Text == "weak";
    default:
      return false;
    }
  }

  bool isContextualPunctuator(StringRef ContextPunc) const {
    return isAnyOperator() && Text == ContextPunc;
  }

  /// Determine whether the token can be an argument label.
  ///
  /// This covers all identifiers and keywords except those keywords
  /// used
  bool canBeArgumentLabel() const {
    // Identifiers, escaped identifiers, and '_' can be argument labels.
    if (is(tok::identifier) || isEscapedIdentifier() || is(tok::kw__)) {
      // ... except for '__shared' and '__owned'.
      if (getRawText().equals("__shared") ||
          getRawText().equals("__owned"))
        return false;

      return true;
    }

    // 'let', 'var', and 'inout' cannot be argument labels.
    if (isAny(tok::kw_let, tok::kw_var, tok::kw_inout))
      return false;

    // All other keywords can be argument labels.
    return isKeyword();
  }

  /// True if the token is an identifier or '_'.
  bool isIdentifierOrUnderscore() const {
    return isAny(tok::identifier, tok::kw__);
  }

  /// True if the token is an l_paren token that does not start a new line.
  bool isFollowingLParen() const {
    return !isAtStartOfLine() && Kind == tok::l_paren;
  }
  
  /// True if the token is an l_square token that does not start a new line.
  bool isFollowingLSquare() const {
    return !isAtStartOfLine() && Kind == tok::l_square;
  }

  /// True if the token is any keyword.
  bool isKeyword() const {
    switch (Kind) {
#define KEYWORD(X) case tok::kw_##X: return true;
#include "swift/Syntax/TokenKinds.def"
    default: return false;
    }
  }

  /// True if the token is any literal.
  bool isLiteral() const {
    switch(Kind) {
    case tok::integer_literal:
    case tok::floating_literal:
    case tok::string_literal:
      return true;
    default:
      return false;
    }
  }

  bool isPunctuation() const {
    switch (Kind) {
#define PUNCTUATOR(Name, Str) case tok::Name: return true;
#include "swift/Syntax/TokenKinds.def"
    default: return false;
    }
  }
  
  /// getLoc - Return a source location identifier for the specified
  /// offset in the current file.
  SourceLoc getLoc() const {
    return SourceLoc(llvm::SMLoc::getFromPointer(Text.begin()));
  }

  unsigned getLength() const { return Text.size(); }

  CharSourceRange getRange() const {
    return CharSourceRange(getLoc(), getLength());
  }

  bool hasComment() const {
    return CommentLength != 0;
  }

  CharSourceRange getCommentRange() const {
    if (CommentLength == 0)
      return CharSourceRange(SourceLoc(llvm::SMLoc::getFromPointer(Text.begin())),
                             0);
    auto TrimedComment = trimComment();
    return CharSourceRange(
      SourceLoc(llvm::SMLoc::getFromPointer(TrimedComment.begin())),
      TrimedComment.size());
  }
  
  SourceLoc getCommentStart() const {
    if (CommentLength == 0) return SourceLoc();
    return SourceLoc(llvm::SMLoc::getFromPointer(trimComment().begin()));
  }

  StringRef getRawText() const {
    return Text;
  }

  StringRef getText() const {
    if (EscapedIdentifier) {
      // Strip off the backticks on either side.
      assert(Text.front() == '`' && Text.back() == '`');
      return Text.slice(1, Text.size() - 1);
    }
    return Text;
  }

  void setText(StringRef T) { Text = T; }

  /// \brief Set the token to the specified kind and source range.
  void setToken(tok K, StringRef T, unsigned CommentLength = 0,
                bool MultilineString = false) {
    Kind = K;
    Text = T;
    this->CommentLength = CommentLength;
    EscapedIdentifier = false;
    this->MultilineString = MultilineString;
  }

  bool IsMultilineString() const {
    return MultilineString;
  }
};
  
} // end namespace swift


namespace llvm {
  template <typename T> struct isPodLike;
  template <>
  struct isPodLike<swift::Token> { static const bool value = true; };
} // end namespace llvm

#endif