lexer.cr

require "./token"
require "../exception"
require "string_pool"
require "../warnings"

module Crystal
  class Lexer
    property? doc_enabled : Bool
    property? comments_enabled : Bool
    property? count_whitespace : Bool
    property? wants_raw : Bool
    property? slash_is_regex : Bool
    property? wants_def_or_macro_name : Bool
    getter reader : Char::Reader
    getter token : Token
    property line_number : Int32
    property column_number : Int32
    property wants_symbol : Bool
    @filename : String | VirtualFile | Nil
    @stacked_filename : String | VirtualFile | Nil
    @token_end_location : Location?
    @string_pool : StringPool

    # This is an interface for storing data associated to a heredoc
    module HeredocItem
    end

    # Heredocs pushed when found. Should be processed when encountering a newline
    getter heredocs = [] of {Token::DelimiterState, HeredocItem}

    property macro_expansion_pragmas : Hash(Int32, Array(LocPragma))? = nil

    alias LocPragma = LocSetPragma | LocPushPragma | LocPopPragma

    record LocSetPragma,
      filename : String,
      line_number : Int32,
      column_number : Int32 do
      def run_pragma(lexer)
        lexer.set_location filename, line_number, column_number
      end
    end

    record LocPushPragma do
      def run_pragma(lexer)
        lexer.push_location
      end
    end

    record LocPopPragma do
      def run_pragma(lexer)
        lexer.pop_location
      end
    end

    # Warning settings and all detected warnings.
    property warnings : WarningCollection

    def initialize(string, string_pool : StringPool? = nil, warnings : WarningCollection? = nil)
      @warnings = warnings || WarningCollection.new
      @reader = Char::Reader.new(string)
      @token = Token.new
      @temp_token = Token.new
      @line_number = 1
      @column_number = 1
      @filename = ""
      @wants_regex = true
      @doc_enabled = false
      @comment_is_doc = true
      @comments_enabled = false
      @count_whitespace = false
      @slash_is_regex = true
      @wants_raw = false
      @wants_def_or_macro_name = false
      @wants_symbol = true
      @string_pool = string_pool || StringPool.new

      # When lexing macro tokens, when we encounter `#{` inside
      # a string we push the current delimiter here and reset
      # the current one to nil. The reason is, inside strings
      # we don't want to consider %foo a macro variable, but
      # we do want to do this inside interpolations.
      # We then count curly braces, with @macro_curly_count,
      # until we find the last `}` and then we pop from the stack
      # and get the original delimiter.
      @delimiter_state_stack = [] of Token::DelimiterState
      @macro_curly_count = 0

      @stacked = false
      @stacked_filename = ""
      @stacked_line_number = 1
      @stacked_column_number = 1
    end

    def filename=(filename)
      @filename = filename
    end

    def next_token
      # Check previous token:
      if @token.type.newline? || @token.type.eof?
        # 1) After a newline or at the start of the stream (:EOF), a following comment can be a doc comment
        @comment_is_doc = true
      elsif !@token.type.space?
        # 2) Any non-space token prevents a following comment from being a doc
        # comment.
        @comment_is_doc = false
      end

      reset_token

      # Skip comments
      while current_char == '#'
        start = current_pos

        # Check #<loc:...> pragma comment
        if char_sequence?('<', 'l', 'o', 'c', ':', column_increment: false)
          next_char_no_column_increment
          consume_loc_pragma
        else
          if @doc_enabled && @comment_is_doc
            consume_doc
          elsif @comments_enabled
            return consume_comment(start)
          else
            skip_comment
          end
        end
      end

      start = current_pos

      # Fix location by `macro_expansion_pragmas`.
      if me_pragmas = macro_expansion_pragmas
        # It might happen that the current "start" already passed some
        # location pragmas, so we must consume all of those. For example
        # if one does `@{{...}}` inside a macro, "start" will be one less
        # number than the pragma, and after consuming "@..." if we don't
        # consume the pragmas generated by `{{...}}` we'll be in an
        # incorrect location.
        while me_pragmas.first_key?.try &.<=(start)
          _, pragmas = me_pragmas.shift
          pragmas.each &.run_pragma self
        end
      end

      reset_regex_flags = true

      case current_char
      when '\0'
        @token.type = :EOF
      when ' ', '\t'
        consume_whitespace
        reset_regex_flags = false
      when '\\'
        case next_char
        when '\r', '\n'
          handle_slash_r_slash_n_or_slash_n
          incr_line_number
          @token.passed_backslash_newline = true
          consume_whitespace
          reset_regex_flags = false
        else
          unknown_token
        end
      when '\n'
        next_char :NEWLINE
        incr_line_number
        reset_regex_flags = false
        consume_newlines
      when '\r'
        if next_char == '\n'
          next_char :NEWLINE
          incr_line_number
          reset_regex_flags = false
          consume_newlines
        else
          raise "expected '\\n' after '\\r'"
        end
      when '='
        case next_char
        when '='
          case next_char
          when '='
            next_char :OP_EQ_EQ_EQ
          else
            @token.type = :OP_EQ_EQ
          end
        when '>'
          next_char :OP_EQ_GT
        when '~'
          next_char :OP_EQ_TILDE
        else
          @token.type = :OP_EQ
        end
      when '!'
        case next_char
        when '='
          next_char :OP_BANG_EQ
        when '~'
          next_char :OP_BANG_TILDE
        else
          @token.type = :OP_BANG
        end
      when '<'
        case next_char
        when '='
          case next_char
          when '>'
            next_char :OP_LT_EQ_GT
          else
            @token.type = :OP_LT_EQ
          end
        when '<'
          case next_char
          when '='
            next_char :OP_LT_LT_EQ
          when '-'
            consume_heredoc_start(start)
          else
            @token.type = :OP_LT_LT
          end
        else
          @token.type = :OP_LT
        end
      when '>'
        case next_char
        when '='
          next_char :OP_GT_EQ
        when '>'
          case next_char
          when '='
            next_char :OP_GT_GT_EQ
          else
            @token.type = :OP_GT_GT
          end
        else
          @token.type = :OP_GT
        end
      when '+'
        @token.start = start
        case next_char
        when '='
          next_char :OP_PLUS_EQ
        when '0'..'9'
          scan_number start
        when '+'
          raise "postfix increment is not supported, use `exp += 1`"
        else
          @token.type = :OP_PLUS
        end
      when '-'
        @token.start = start
        case next_char
        when '='
          next_char :OP_MINUS_EQ
        when '>'
          next_char :OP_MINUS_GT
        when '0'..'9'
          scan_number start, negative: true
        when '-'
          raise "postfix decrement is not supported, use `exp -= 1`"
        else
          @token.type = :OP_MINUS
        end
      when '*'
        case next_char
        when '='
          next_char :OP_STAR_EQ
        when '*'
          case next_char
          when '='
            next_char :OP_STAR_STAR_EQ
          else
            @token.type = :OP_STAR_STAR
          end
        else
          @token.type = :OP_STAR
        end
      when '/'
        char = next_char
        if (@wants_def_or_macro_name || !@slash_is_regex) && char == '/'
          case next_char
          when '='
            next_char :OP_SLASH_SLASH_EQ
          else
            @token.type = :OP_SLASH_SLASH
          end
        elsif !@slash_is_regex && char == '='
          next_char :OP_SLASH_EQ
        elsif @wants_def_or_macro_name
          @token.type = :OP_SLASH
        elsif @slash_is_regex
          delimited_pair :regex, '/', '/', start, advance: false
        elsif char.ascii_whitespace? || char == '\0'
          @token.type = :OP_SLASH
        elsif @wants_regex
          delimited_pair :regex, '/', '/', start, advance: false
        else
          @token.type = :OP_SLASH
        end
      when '%'
        if @wants_def_or_macro_name
          next_char :OP_PERCENT
        else
          case next_char
          when '='
            next_char :OP_PERCENT_EQ
          when '(', '[', '{', '<', '|'
            delimited_pair :string, current_char, closing_char, start
          when 'i'
            case peek_next_char
            when '(', '{', '[', '<', '|'
              start_char = next_char
              next_char :SYMBOL_ARRAY_START
              @token.raw = "%i#{start_char}" if @wants_raw
              @token.delimiter_state = Token::DelimiterState.new(:symbol_array, start_char, closing_char(start_char))
            else
              @token.type = :OP_PERCENT
            end
          when 'q'
            case peek_next_char
            when '(', '{', '[', '<', '|'
              next_char
              delimited_pair :string, current_char, closing_char, start, allow_escapes: false
            else
              @token.type = :OP_PERCENT
            end
          when 'Q'
            case peek_next_char
            when '(', '{', '[', '<', '|'
              next_char
              delimited_pair :string, current_char, closing_char, start
            else
              @token.type = :OP_PERCENT
            end
          when 'r'
            case next_char
            when '(', '[', '{', '<', '|'
              delimited_pair :regex, current_char, closing_char, start
            else
              raise "unknown %r char"
            end
          when 'x'
            case next_char
            when '(', '[', '{', '<', '|'
              delimited_pair :command, current_char, closing_char, start
            else
              raise "unknown %x char"
            end
          when 'w'
            case peek_next_char
            when '(', '{', '[', '<', '|'
              start_char = next_char
              next_char :STRING_ARRAY_START
              @token.raw = "%w#{start_char}" if @wants_raw
              @token.delimiter_state = Token::DelimiterState.new(:string_array, start_char, closing_char(start_char))
            else
              @token.type = :OP_PERCENT
            end
          when '}'
            next_char :OP_PERCENT_RCURLY
          else
            @token.type = :OP_PERCENT
          end
        end
      when '(' then next_char :OP_LPAREN
      when ')' then next_char :OP_RPAREN
      when '{'
        case next_char
        when '%'
          next_char :OP_LCURLY_PERCENT
        when '{'
          next_char :OP_LCURLY_LCURLY
        else
          @token.type = :OP_LCURLY
        end
      when '}' then next_char :OP_RCURLY
      when '['
        case next_char
        when ']'
          case next_char
          when '='
            next_char :OP_LSQUARE_RSQUARE_EQ
          when '?'
            next_char :OP_LSQUARE_RSQUARE_QUESTION
          else
            @token.type = :OP_LSQUARE_RSQUARE
          end
        else
          @token.type = :OP_LSQUARE
        end
      when ']' then next_char :OP_RSQUARE
      when ',' then next_char :OP_COMMA
      when '?' then next_char :OP_QUESTION
      when ';'
        reset_regex_flags = false
        next_char :OP_SEMICOLON
      when ':'
        if next_char == ':'
          next_char :OP_COLON_COLON
        elsif @wants_symbol
          consume_symbol
        else
          @token.type = :OP_COLON
        end
      when '~'
        next_char :OP_TILDE
      when '.'
        line = @line_number
        column = @column_number
        case next_char
        when '.'
          case next_char
          when '.'
            next_char :OP_PERIOD_PERIOD_PERIOD
          else
            @token.type = :OP_PERIOD_PERIOD
          end
        when .ascii_number?
          raise ".1 style number literal is not supported, put 0 before dot", line, column
        else
          @token.type = :OP_PERIOD
        end
      when '&'
        case next_char
        when '&'
          case next_char
          when '='
            next_char :OP_AMP_AMP_EQ
          else
            @token.type = :OP_AMP_AMP
          end
        when '='
          next_char :OP_AMP_EQ
        when '+'
          case next_char
          when '='
            next_char :OP_AMP_PLUS_EQ
          else
            @token.type = :OP_AMP_PLUS
          end
        when '-'
          # Check if '>' comes after '&-', making it '&->'.
          # We want to parse that like '&(->...)',
          # so we only return '&' for now.
          if peek_next_char == '>'
            @token.type = :OP_AMP
          else
            case next_char
            when '='
              next_char :OP_AMP_MINUS_EQ
            else
              @token.type = :OP_AMP_MINUS
            end
          end
        when '*'
          case next_char
          when '*'
            next_char :OP_AMP_STAR_STAR
          when '='
            next_char :OP_AMP_STAR_EQ
          else
            @token.type = :OP_AMP_STAR
          end
        else
          @token.type = :OP_AMP
        end
      when '|'
        case next_char
        when '|'
          case next_char
          when '='
            next_char :OP_BAR_BAR_EQ
          else
            @token.type = :OP_BAR_BAR
          end
        when '='
          next_char :OP_BAR_EQ
        else
          @token.type = :OP_BAR
        end
      when '^'
        case next_char
        when '='
          next_char :OP_CARET_EQ
        else
          @token.type = :OP_CARET
        end
      when '\''
        line = @line_number
        column = @column_number
        @token.type = :CHAR
        case char1 = next_char
        when '\\'
          case char2 = next_char
          when '\\'
            @token.value = '\\'
          when '\''
            @token.value = '\''
          when 'a'
            @token.value = '\a'
          when 'b'
            @token.value = '\b'
          when 'e'
            @token.value = '\e'
          when 'f'
            @token.value = '\f'
          when 'n'
            @token.value = '\n'
          when 'r'
            @token.value = '\r'
          when 't'
            @token.value = '\t'
          when 'v'
            @token.value = '\v'
          when 'u'
            value = consume_char_unicode_escape
            @token.value = value.chr
          when '0'
            @token.value = '\0'
          when '\0'
            raise "unterminated char literal", line, column
          else
            raise "invalid char escape sequence '\\#{char2}'", line, column
          end
        when '\''
          raise "invalid empty char literal (did you mean '\\''?)", line, column
        when '\0'
          raise "unterminated char literal", line, column
        else
          @token.value = char1
        end
        if next_char != '\''
          raise "unterminated char literal, use double quotes for strings", line, column
        end
        next_char
        set_token_raw_from_start(start)
      when '`'
        if @wants_def_or_macro_name
          next_char :OP_GRAVE
        else
          delimited_pair :command, '`', '`', start
        end
      when '"'
        delimited_pair :string, '"', '"', start
      when '0'..'9'
        scan_number start
      when '@'
        case next_char
        when '['
          next_char :OP_AT_LSQUARE
        when '@'
          next_char
          consume_variable :CLASS_VAR, start
        else
          consume_variable :INSTANCE_VAR, start
        end
      when '$'
        case next_char
        when '~'
          next_char :OP_DOLLAR_TILDE
        when '?'
          next_char :OP_DOLLAR_QUESTION
        when .ascii_number?
          start = current_pos
          if current_char == '0'
            next_char
          else
            while next_char.ascii_number?
              # Nothing to do
            end
            next_char if current_char == '?'
          end
          @token.type = :GLOBAL_MATCH_DATA_INDEX
          @token.value = string_range_from_pool(start)
        else
          consume_variable :GLOBAL, start
        end
      when 'a'
        case next_char
        when 'b'
          if char_sequence?('s', 't', 'r', 'a', 'c', 't')
            return check_ident_or_keyword(:abstract, start)
          end
        when 'l'
          if char_sequence?('i', 'a', 's')
            return check_ident_or_keyword(:alias, start)
          end
        when 's'
          case peek_next_char
          when 'm'
            next_char
            return check_ident_or_keyword(:asm, start)
          when '?'
            next_char
            next_char :IDENT
            @token.value = :as_question
            return @token
          else
            return check_ident_or_keyword(:as, start)
          end
        when 'n'
          if char_sequence?('n', 'o', 't', 'a', 't', 'i', 'o', 'n')
            return check_ident_or_keyword(:annotation, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'b'
        case next_char
        when 'e'
          if char_sequence?('g', 'i', 'n')
            return check_ident_or_keyword(:begin, start)
          end
        when 'r'
          if char_sequence?('e', 'a', 'k')
            return check_ident_or_keyword(:break, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'c'
        case next_char
        when 'a'
          if char_sequence?('s', 'e')
            return check_ident_or_keyword(:case, start)
          end
        when 'l'
          if char_sequence?('a', 's', 's')
            return check_ident_or_keyword(:class, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'd'
        case next_char
        when 'e'
          if next_char == 'f'
            return check_ident_or_keyword(:def, start)
          end
        when 'o' then return check_ident_or_keyword(:do, start)
        else
          # scan_ident
        end
        scan_ident(start)
      when 'e'
        case next_char
        when 'l'
          case next_char
          when 's'
            case next_char
            when 'e' then return check_ident_or_keyword(:else, start)
            when 'i'
              if next_char == 'f'
                return check_ident_or_keyword(:elsif, start)
              end
            else
              # scan_ident
            end
          else
            # scan_ident
          end
        when 'n'
          case next_char
          when 'd'
            return check_ident_or_keyword(:end, start)
          when 's'
            if char_sequence?('u', 'r', 'e')
              return check_ident_or_keyword(:ensure, start)
            end
          when 'u'
            if next_char == 'm'
              return check_ident_or_keyword(:enum, start)
            end
          else
            # scan_ident
          end
        when 'x'
          if char_sequence?('t', 'e', 'n', 'd')
            return check_ident_or_keyword(:extend, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'f'
        case next_char
        when 'a'
          if char_sequence?('l', 's', 'e')
            return check_ident_or_keyword(:false, start)
          end
        when 'o'
          if next_char == 'r'
            return check_ident_or_keyword(:for, start)
          end
        when 'u'
          if next_char == 'n'
            return check_ident_or_keyword(:fun, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'i'
        case next_char
        when 'f'
          return check_ident_or_keyword(:if, start)
        when 'n'
          if ident_part_or_end?(peek_next_char)
            case next_char
            when 'c'
              if char_sequence?('l', 'u', 'd', 'e')
                return check_ident_or_keyword(:include, start)
              end
            when 's'
              if char_sequence?('t', 'a', 'n', 'c', 'e', '_', 's', 'i', 'z', 'e', 'o', 'f')
                return check_ident_or_keyword(:instance_sizeof, start)
              end
            else
              # scan_ident
            end
          else
            next_char :IDENT
            @token.value = :in
            return @token
          end
        when 's'
          if char_sequence?('_', 'a', '?')
            return check_ident_or_keyword(:is_a_question, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'l'
        case next_char
        when 'i'
          if next_char == 'b'
            return check_ident_or_keyword(:lib, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'm'
        case next_char
        when 'a'
          if char_sequence?('c', 'r', 'o')
            return check_ident_or_keyword(:macro, start)
          end
        when 'o'
          case next_char
          when 'd'
            if char_sequence?('u', 'l', 'e')
              return check_ident_or_keyword(:module, start)
            end
          else
            # scan_ident
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'n'
        case next_char
        when 'e'
          if char_sequence?('x', 't')
            return check_ident_or_keyword(:next, start)
          end
        when 'i'
          case next_char
          when 'l'
            if peek_next_char == '?'
              next_char
              return check_ident_or_keyword(:nil_question, start)
            else
              return check_ident_or_keyword(:nil, start)
            end
          else
            # scan_ident
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'o'
        case next_char
        when 'f'
          if peek_next_char == 'f'
            next_char
            if char_sequence?('s', 'e', 't', 'o', 'f')
              return check_ident_or_keyword(:offsetof, start)
            end
          else
            return check_ident_or_keyword(:of, start)
          end
        when 'u'
          if next_char == 't'
            return check_ident_or_keyword(:out, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'p'
        case next_char
        when 'o'
          if char_sequence?('i', 'n', 't', 'e', 'r', 'o', 'f')
            return check_ident_or_keyword(:pointerof, start)
          end
        when 'r'
          case next_char
          when 'i'
            if char_sequence?('v', 'a', 't', 'e')
              return check_ident_or_keyword(:private, start)
            end
          when 'o'
            if char_sequence?('t', 'e', 'c', 't', 'e', 'd')
              return check_ident_or_keyword(:protected, start)
            end
          else
            # scan_ident
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'r'
        case next_char
        when 'e'
          case next_char
          when 's'
            case next_char
            when 'c'
              if char_sequence?('u', 'e')
                return check_ident_or_keyword(:rescue, start)
              end
            when 'p'
              if char_sequence?('o', 'n', 'd', 's', '_', 't', 'o', '?')
                return check_ident_or_keyword(:responds_to_question, start)
              end
            else
              # scan_ident
            end
          when 't'
            if char_sequence?('u', 'r', 'n')
              return check_ident_or_keyword(:return, start)
            end
          when 'q'
            if char_sequence?('u', 'i', 'r', 'e')
              return check_ident_or_keyword(:require, start)
            end
          else
            # scan_ident
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 's'
        case next_char
        when 'e'
          if next_char == 'l'
            case next_char
            when 'e'
              if char_sequence?('c', 't')
                return check_ident_or_keyword(:select, start)
              end
            when 'f'
              return check_ident_or_keyword(:self, start)
            else
              # scan_ident
            end
          end
        when 'i'
          if char_sequence?('z', 'e', 'o', 'f')
            return check_ident_or_keyword(:sizeof, start)
          end
        when 't'
          if char_sequence?('r', 'u', 'c', 't')
            return check_ident_or_keyword(:struct, start)
          end
        when 'u'
          if char_sequence?('p', 'e', 'r')
            return check_ident_or_keyword(:super, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 't'
        case next_char
        when 'h'
          if char_sequence?('e', 'n')
            return check_ident_or_keyword(:then, start)
          end
        when 'r'
          if char_sequence?('u', 'e')
            return check_ident_or_keyword(:true, start)
          end
        when 'y'
          if char_sequence?('p', 'e')
            if peek_next_char == 'o'
              next_char
              if next_char == 'f'
                return check_ident_or_keyword(:typeof, start)
              end
            else
              return check_ident_or_keyword(:type, start)
            end
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'u'
        if next_char == 'n'
          case next_char
          when 'i'
            case next_char
            when 'o'
              if next_char == 'n'
                return check_ident_or_keyword(:union, start)
              end
            when 'n'
              if char_sequence?('i', 't', 'i', 'a', 'l', 'i', 'z', 'e', 'd')
                return check_ident_or_keyword(:uninitialized, start)
              end
            else
              # scan_ident
            end
          when 'l'
            if char_sequence?('e', 's', 's')
              return check_ident_or_keyword(:unless, start)
            end
          when 't'
            if char_sequence?('i', 'l')
              return check_ident_or_keyword(:until, start)
            end
          else
            # scan_ident
          end
        end
        scan_ident(start)
      when 'v'
        if char_sequence?('e', 'r', 'b', 'a', 't', 'i', 'm')
          return check_ident_or_keyword(:verbatim, start)
        end
        scan_ident(start)
      when 'w'
        case next_char
        when 'h'
          case next_char
          when 'e'
            if next_char == 'n'
              return check_ident_or_keyword(:when, start)
            end
          when 'i'
            if char_sequence?('l', 'e')
              return check_ident_or_keyword(:while, start)
            end
          else
            # scan_ident
          end
        when 'i'
          if char_sequence?('t', 'h')
            return check_ident_or_keyword(:with, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'y'
        if char_sequence?('i', 'e', 'l', 'd')
          return check_ident_or_keyword(:yield, start)
        end
        scan_ident(start)
      when '_'
        case next_char
        when '_'
          case next_char
          when 'D'
            if char_sequence?('I', 'R', '_', '_')
              unless ident_part_or_end?(peek_next_char)
                next_char :MAGIC_DIR
                return @token
              end
            end
          when 'E'
            if char_sequence?('N', 'D', '_', 'L', 'I', 'N', 'E', '_', '_')
              unless ident_part_or_end?(peek_next_char)
                next_char :MAGIC_END_LINE
                return @token
              end
            end
          when 'F'
            if char_sequence?('I', 'L', 'E', '_', '_')
              unless ident_part_or_end?(peek_next_char)
                next_char :MAGIC_FILE
                return @token
              end
            end
          when 'L'
            if char_sequence?('I', 'N', 'E', '_', '_')
              unless ident_part_or_end?(peek_next_char)
                next_char :MAGIC_LINE
                return @token
              end
            end
          else
            # scan_ident
          end
        else
          unless ident_part?(current_char)
            @token.type = :UNDERSCORE
            return @token
          end
        end

        scan_ident(start)
      else
        if current_char.ascii_uppercase?
          while ident_part?(next_char)
            # Nothing to do
          end
          @token.type = :CONST
          @token.value = string_range_from_pool(start)
        elsif ident_start?(current_char)
          next_char
          scan_ident(start)
        else
          unknown_token
        end
      end

      if reset_regex_flags
        @wants_regex = true
        @slash_is_regex = false
      end

      @token
    end

    def token_end_location
      @token_end_location ||= Location.new(@filename, @line_number, @column_number - 1)
    end

    def slash_is_regex!
      @slash_is_regex = true
    end

    def slash_is_not_regex!
      @slash_is_regex = false
    end

    def consume_comment(start_pos)
      skip_comment
      @token.type = :COMMENT
      @token.value = string_range(start_pos)
      @token
    end

    def consume_doc
      # Ignore first whitespace after comment, like in `# some doc`
      next_char if current_char == ' '

      start_pos = current_pos

      skip_comment

      if doc_buffer = @token.doc_buffer
        doc_buffer << '\n'
      else
        @token.doc_buffer = doc_buffer = IO::Memory.new
      end

      doc_buffer.write slice_range(start_pos)
    end

    def skip_comment
      char = current_char
      while !char.in?('\n', '\0')
        char = next_char_no_column_increment
      end
    end

    def consume_whitespace
      start_pos = current_pos
      next_char :SPACE
      while true
        case current_char
        when ' ', '\t'
          next_char
        when '\\'
          if next_char.in?('\r', '\n')
            handle_slash_r_slash_n_or_slash_n
            next_char
            incr_line_number
            @token.passed_backslash_newline = true
          else
            unknown_token
          end
        else
          break
        end
      end
      if @count_whitespace
        @token.value = string_range(start_pos)
      end
    end

    def consume_newlines
      # If there are heredocs we don't freely consume newlines because
      # these will be part of the heredoc string
      return unless @heredocs.empty?

      if @count_whitespace
        return
      end

      while true
        case current_char
        when '\n'
          next_char_no_column_increment
          incr_line_number nil
          @token.doc_buffer = nil
        when '\r'
          if next_char_no_column_increment != '\n'
            raise "expected '\\n' after '\\r'"
          end
          next_char_no_column_increment
          incr_line_number nil
          @token.doc_buffer = nil
        else
          break
        end
      end
    end

    def check_ident_or_keyword(keyword : Keyword, start)
      if ident_part_or_end?(peek_next_char)
        scan_ident(start)
      else
        next_char :IDENT
        @token.value = keyword
      end
      @token
    end

    def scan_ident(start)
      while ident_part?(current_char)
        next_char
      end
      if current_char.in?('?', '!') && peek_next_char != '='
        next_char
      end
      @token.type = :IDENT
      @token.value = string_range_from_pool(start)
      @token
    end

    def next_char_and_symbol(value)
      next_char
      symbol value
    end

    def symbol(value)
      @token.type = :SYMBOL
      @token.value = value
      @token.raw = ":#{value}" if @wants_raw
    end

    macro gen_check_int_fits_in_size(type, method, size, number_size, raw_number_string, start, pos_before_suffix, negative)
      {% if type.stringify.starts_with? "U" %}
        raise "Invalid negative value #{string_range({{start}}, {{pos_before_suffix}})} for {{type}}", @token, (current_pos - {{start}}) if {{negative}}
      {% end %}

      if !@token.value || {{number_size}} > {{size}} || ({{number_size}} == {{size}} && {{raw_number_string}}.to_{{method.id}}? == nil)
        raise_value_doesnt_fit_in "{{type}}", {{start}}, {{pos_before_suffix}}
      end
    end

    def raise_value_doesnt_fit_in(type, start, pos_before_suffix)
      raise "#{string_range(start, pos_before_suffix)} doesn't fit in an #{type}", @token, (current_pos - start)
    end

    def raise_value_doesnt_fit_in(type, start, pos_before_suffix, alternative)
      raise "#{string_range(start, pos_before_suffix)} doesn't fit in an #{type}, try using the suffix #{alternative}", @token, (current_pos - start)
    end

    def warn_large_uint64_literal(start, pos_before_suffix)
      @warnings.add_warning_at(@token.location, "#{string_range(start, pos_before_suffix)} doesn't fit in an Int64, try using the suffix u64 or i128")
    end

    private def scan_number(start, negative = false)
      @token.type = :NUMBER
      base = 10
      number_size = 0
      suffix_size = 0
      is_decimal = false
      is_e_notation = false
      has_underscores = false
      last_is_underscore = false
      pos_after_prefix = start
      pos_before_exponent = nil

      # Consume prefix
      if current_char == '0'
        case next_char
        when 'b'      then base = 2
        when 'o'      then base = 8
        when 'x'      then base = 16
        when '0'..'9' then raise("octal constants should be prefixed with 0o", @token, (current_pos - start))
        when '_'
          raise("octal constants should be prefixed with 0o", @token, (current_pos - start)) if next_char.in?('0'..'9')
          has_underscores = last_is_underscore = true
        end

        # Skip prefix (b, o, x)
        unless base == 10
          next_char
          pos_after_prefix = current_pos
          # Enforce number after prefix (disallow ex. "0x", "0x_1")
          raise("unexpected '_' in number", @token, (current_pos - start)) if current_char == '_'

          digit = String::CHAR_TO_DIGIT[current_char.ord]?
          raise("numeric literal without digits", @token, (current_pos - start)) unless digit && digit.to_u8! < base
        end
      end

      # Consume number
      loop do
        loop do
          digit = String::CHAR_TO_DIGIT[current_char.ord]?
          break unless digit && digit.to_u8! < base

          number_size += 1 unless number_size == 0 && current_char == '0'
          next_char
          last_is_underscore = false
        end

        if pos_before_exponent
          raise("invalid decimal number exponent", @token, (current_pos - start)) unless current_pos > pos_before_exponent
        end

        case current_char
        when '_'
          raise("consecutive underscores in numbers aren't allowed", @token, (current_pos - start)) if last_is_underscore
          has_underscores = last_is_underscore = true
        when '.'
          raise("unexpected '_' in number", @token, (current_pos - start)) if last_is_underscore
          break if is_decimal || base != 10 || !peek_next_char.in?('0'..'9')
          is_decimal = true
        when 'e', 'E'
          last_is_underscore = false
          break if is_e_notation || base != 10
          is_e_notation = is_decimal = true
          next_char if peek_next_char.in?('+', '-')
          raise("unexpected '_' in number", @token, (current_pos - start)) if peek_next_char == '_'
          pos_before_exponent = current_pos + 1
        when 'i', 'u', 'f'
          if current_char == 'f' && base != 10
            case base
            when 2 then raise("binary float literal is not supported", @token, (current_pos - start))
            when 8 then raise("octal float literal is not supported", @token, (current_pos - start))
            end
            break
          end
          before_suffix_pos = current_pos
          @token.number_kind = consume_number_suffix
          next_char
          suffix_size = current_pos - before_suffix_pos
          suffix_size += 1 if last_is_underscore
          break
        else
          raise("trailing '_' in number", @token, (current_pos - start)) if last_is_underscore
          break
        end

        next_char
      end

      set_token_raw_from_start(start)

      # Sanitize string (or convert to decimal unless number is in base 10)
      pos_before_suffix = current_pos - suffix_size
      raw_number_string = string_range(pos_after_prefix, pos_before_suffix)
      if base == 10
        raw_number_string = raw_number_string.delete('_') if has_underscores
        @token.value = raw_number_string
      else
        # The conversion to base 10 is first tried using a UInt64 to circumvent compiler
        # regressions caused by bugs in the platform's UInt128 implementation.
        base10_number_string = raw_number_string.to_u64?(base: base, underscore: true).try &.to_s
        base10_number_string ||= raw_number_string.to_u128?(base: base, underscore: true).try &.to_s
        if base10_number_string
          number_size = base10_number_string.size
          first_byte = @reader.string.byte_at(start).chr
          base10_number_string = first_byte + base10_number_string if first_byte.in?('+', '-')
          @token.value = raw_number_string = base10_number_string
        end
      end

      if is_decimal
        @token.number_kind = NumberKind::F64 if suffix_size == 0
        raise("Invalid suffix #{@token.number_kind} for decimal number", @token, (current_pos - start)) unless @token.number_kind.float?
        return
      end

      # Check or determine suffix
      if suffix_size == 0
        raise_value_doesnt_fit_in(negative ? Int64 : UInt64, start, pos_before_suffix) unless @token.value
        @token.number_kind = case number_size
                             when 0..9   then NumberKind::I32
                             when 10     then raw_number_string.to_i32? ? NumberKind::I32 : NumberKind::I64
                             when 11..18 then NumberKind::I64
                             when 19
                               if raw_number_string.to_i64?
                                 NumberKind::I64
                               elsif negative
                                 raise_value_doesnt_fit_in(Int64, start, pos_before_suffix, "i128")
                               else
                                 warn_large_uint64_literal(start, pos_before_suffix)
                                 NumberKind::U64
                               end
                             when 20
                               raise_value_doesnt_fit_in(Int64, start, pos_before_suffix, "i128") if negative
                               if raw_number_string.to_u64?
                                 warn_large_uint64_literal(start, pos_before_suffix)
                                 NumberKind::U64
                               else
                                 raise_value_doesnt_fit_in(UInt64, start, pos_before_suffix, "i128")
                               end
                             when 21..38
                               raise_value_doesnt_fit_in(negative ? Int64 : UInt64, start, pos_before_suffix, "i128")
                             when 39
                               if raw_number_string.to_i128?
                                 raise_value_doesnt_fit_in(negative ? Int64 : UInt64, start, pos_before_suffix, "i128")
                               elsif raw_number_string.to_u128?
                                 raise_value_doesnt_fit_in(UInt64, start, pos_before_suffix, "u128")
                               else
                                 raise_value_doesnt_fit_in(negative ? Int64 : UInt64, start, pos_before_suffix)
                               end
                             else
                               raise_value_doesnt_fit_in(negative ? Int64 : UInt64, start, pos_before_suffix)
                             end
      else
        case @token.number_kind
        when .i8?   then gen_check_int_fits_in_size(Int8, :i8, 3, number_size, raw_number_string, start, pos_before_suffix, negative)
        when .u8?   then gen_check_int_fits_in_size(UInt8, :u8, 3, number_size, raw_number_string, start, pos_before_suffix, negative)
        when .i16?  then gen_check_int_fits_in_size(Int16, :i16, 5, number_size, raw_number_string, start, pos_before_suffix, negative)
        when .u16?  then gen_check_int_fits_in_size(UInt16, :u16, 5, number_size, raw_number_string, start, pos_before_suffix, negative)
        when .i32?  then gen_check_int_fits_in_size(Int32, :i32, 10, number_size, raw_number_string, start, pos_before_suffix, negative)
        when .u32?  then gen_check_int_fits_in_size(UInt32, :u32, 10, number_size, raw_number_string, start, pos_before_suffix, negative)
        when .i64?  then gen_check_int_fits_in_size(Int64, :i64, 19, number_size, raw_number_string, start, pos_before_suffix, negative)
        when .u64?  then gen_check_int_fits_in_size(UInt64, :u64, 20, number_size, raw_number_string, start, pos_before_suffix, negative)
        when .i128? then gen_check_int_fits_in_size(Int128, :i128, 39, number_size, raw_number_string, start, pos_before_suffix, negative)
        when .u128? then gen_check_int_fits_in_size(UInt128, :u128, 39, number_size, raw_number_string, start, pos_before_suffix, negative)
        end
      end
    end

    private def consume_number_suffix : NumberKind
      case current_char
      when 'i'
        case next_char
        when '8' then return NumberKind::I8
        when '1'
          case next_char
          when '2' then return NumberKind::I128 if next_char == '8'
          when '6' then return NumberKind::I16
          end
        when '3' then return NumberKind::I32 if next_char == '2'
        when '6' then return NumberKind::I64 if next_char == '4'
        end
        raise "invalid int suffix"
      when 'u'
        case next_char
        when '8' then return NumberKind::U8
        when '1'
          case next_char
          when '2' then return NumberKind::U128 if next_char == '8'
          when '6' then return NumberKind::U16
          end
        when '3' then return NumberKind::U32 if next_char == '2'
        when '6' then return NumberKind::U64 if next_char == '4'
        end
        raise "invalid uint suffix"
      when 'f'
        case next_char
        when '3' then return NumberKind::F32 if next_char == '2'
        when '6' then return NumberKind::F64 if next_char == '4'
        end
        raise "invalid float suffix"
      end
      raise "BUG: invalid suffix"
    end

    def next_string_token(delimiter_state)
      reset_token

      @token.line_number = @line_number
      @token.delimiter_state = delimiter_state

      start = current_pos
      string_end = delimiter_state.end
      string_nest = delimiter_state.nest
      string_open_count = delimiter_state.open_count

      # For empty heredocs:
      if @token.type.newline? && delimiter_state.kind.heredoc?
        if check_heredoc_end delimiter_state
          set_token_raw_from_start start
          return @token
        end
      end

      case current_char
      when '\0'
        raise_unterminated_quoted delimiter_state
      when string_end
        next_char
        if string_open_count == 0
          @token.type = :DELIMITER_END
        else
          @token.type = :STRING
          @token.value = string_end.to_s
          @token.delimiter_state = delimiter_state.with_open_count_delta(-1)
        end
      when string_nest
        next_char :STRING
        @token.value = string_nest.to_s
        @token.delimiter_state = delimiter_state.with_open_count_delta(+1)
      when '\\'
        if delimiter_state.allow_escapes
          if delimiter_state.kind.regex?
            char = next_char
            raise_unterminated_quoted delimiter_state if char == '\0'
            next_char :STRING
            if char == '/' || char.ascii_whitespace?
              @token.value = char.to_s
            else
              @token.value = "\\#{char}"
            end
          else
            case char = next_char
            when '\\'
              string_token_escape_value "\\"
            when string_end
              string_token_escape_value string_end.to_s
            when string_nest
              string_token_escape_value string_nest.to_s
            when '#'
              string_token_escape_value "#"
            when 'a'
              string_token_escape_value "\a"
            when 'b'
              string_token_escape_value "\b"
            when 'n'
              string_token_escape_value "\n"
            when 'r'
              string_token_escape_value "\r"
            when 't'
              string_token_escape_value "\t"
            when 'v'
              string_token_escape_value "\v"
            when 'f'
              string_token_escape_value "\f"
            when 'e'
              string_token_escape_value "\e"
            when 'x'
              value = consume_string_hex_escape
              next_char :STRING
              @token.value = String.new(1) do |buffer|
                buffer[0] = value
                {1, 0}
              end
            when 'u'
              value = consume_string_unicode_escape
              next_char :STRING
              @token.value = value
            when '0', '1', '2', '3', '4', '5', '6', '7'
              value = consume_octal_escape(char)
              next_char :STRING
              @token.value = String.new(1) do |buffer|
                buffer[0] = value
                {1, 0}
              end
            when '\r', '\n'
              handle_slash_r_slash_n_or_slash_n
              incr_line_number
              @token.line_number = @line_number

              # Skip until the next non-whitespace char
              while true
                char = next_char
                case char
                when '\0'
                  raise_unterminated_quoted delimiter_state
                when '\n'
                  incr_line_number
                  @token.line_number = @line_number
                when .ascii_whitespace?
                  # Continue
                else
                  break
                end
              end
              next_string_token delimiter_state
            when '\0'
              raise_unterminated_quoted delimiter_state
            else
              @token.type = :STRING
              @token.value = current_char.to_s
              @token.invalid_escape = true
              next_char
            end
          end
        else
          @token.type = :STRING
          @token.value = current_char.to_s
          next_char
        end
      when '#'
        if delimiter_state.allow_escapes
          if peek_next_char == '{'
            next_char
            next_char :INTERPOLATION_START
          else
            next_char :STRING
            @token.value = "#"
          end
        else
          next_char :STRING
          @token.value = "#"
        end
      when '\r', '\n'
        is_slash_r = handle_slash_r_slash_n_or_slash_n
        next_char
        incr_line_number 1
        @token.line_number = @line_number
        @token.column_number = @column_number

        if delimiter_state.kind.heredoc?
          unless check_heredoc_end delimiter_state
            next_string_token_noescape delimiter_state
            @token.value = string_range(start)
          end
        else
          @token.type = :STRING
          @token.value = is_slash_r ? "\r\n" : "\n"
        end
      else
        next_string_token_noescape delimiter_state
        @token.value = string_range(start)
      end

      set_token_raw_from_start(start)

      @token
    end

    def next_string_token_noescape(delimiter_state)
      string_end = delimiter_state.end
      string_nest = delimiter_state.nest

      while !current_char.in?(string_end, string_nest, '\0', '\\', '#', '\r', '\n')
        next_char
      end

      @token.type = :STRING
    end

    def check_heredoc_end(delimiter_state)
      string_end = delimiter_state.end.to_s
      old_pos = current_pos
      old_column = @column_number

      while current_char.in?(' ', '\t')
        next_char
      end

      indent = @column_number - 1

      if string_end.starts_with?(current_char)
        reached_end = false

        string_end.each_char do |c|
          unless c == current_char
            reached_end = false
            break
          end
          next_char
          reached_end = true
        end

        if reached_end &&
           (current_char.in?('\n', '\0') ||
           (current_char == '\r' && peek_next_char == '\n' && next_char))
          @token.type = :DELIMITER_END
          @token.delimiter_state = delimiter_state.with_heredoc_indent(indent)
          return true
        end
      end

      @reader.pos = old_pos
      @column_number = old_column
      @token.column_number = @column_number

      false
    end

    def raise_unterminated_quoted(delimiter_state)
      msg = case delimiter_state.kind
            when .command? then "Unterminated command literal"
            when .regex?   then "Unterminated regular expression"
            when .heredoc?
              "Unterminated heredoc: can't find \"#{delimiter_state.end}\" anywhere before the end of file"
            when .string? then "Unterminated string literal"
            else
              ::raise "unreachable"
            end
      raise msg, @line_number, @column_number
    end

    def next_macro_token(macro_state, skip_whitespace)
      reset_token

      nest = macro_state.nest
      control_nest = macro_state.control_nest
      whitespace = macro_state.whitespace
      delimiter_state = macro_state.delimiter_state
      beginning_of_line = macro_state.beginning_of_line
      comment = macro_state.comment
      heredocs = macro_state.heredocs
      yields = false

      if skip_whitespace
        skip_macro_whitespace
      end

      @token.location = nil
      @token.line_number = @line_number
      @token.column_number = @column_number

      start = current_pos

      if current_char == '\0'
        @token.type = :EOF
        return @token
      end

      if current_char == '\\' && peek_next_char == '{'
        beginning_of_line = false
        next_char
        start = current_pos
        if next_char == '%'
          while (char = next_char_check_line).ascii_whitespace?
          end

          case char
          when 'e'
            if char_sequence?('n', 'd') && !ident_part_or_end?(peek_next_char)
              next_char
              nest -= 1
            end
          when 'f'
            if char_sequence?('o', 'r') && !ident_part_or_end?(peek_next_char)
              next_char
              nest += 1
            end
          when 'i'
            if next_char == 'f' && !ident_part_or_end?(peek_next_char)
              next_char
              nest += 1
            end
          when 'u'
            if char_sequence?('n', 'l', 'e', 's', 's') && !ident_part_or_end?(peek_next_char)
              next_char
              nest += 1
            end
          else
            # no nesting
          end
        end

        @token.type = :MACRO_LITERAL
        @token.value = string_range(start)
        @token.macro_state = Token::MacroState.new(whitespace, nest, control_nest, delimiter_state, beginning_of_line, yields, comment, heredocs)
        set_token_raw_from_start(start)
        return @token
      end

      if current_char == '\\' && peek_next_char == '%'
        beginning_of_line = false
        next_char
        next_char :MACRO_LITERAL
        @token.value = "%"
        @token.macro_state = Token::MacroState.new(whitespace, nest, control_nest, delimiter_state, beginning_of_line, yields, comment, heredocs)
        @token.raw = "%"
        return @token
      end

      if current_char == '{'
        case next_char
        when '{'
          beginning_of_line = false
          next_char :MACRO_EXPRESSION_START
          @token.macro_state = Token::MacroState.new(whitespace, nest, control_nest, delimiter_state, beginning_of_line, yields, comment, heredocs)
          return @token
        when '%'
          beginning_of_line = false
          next_char :MACRO_CONTROL_START
          @token.macro_state = Token::MacroState.new(whitespace, nest, control_nest, delimiter_state, beginning_of_line, yields, comment, heredocs)
          return @token
        else
          # Make sure to decrease the '}' count if inside an interpolation
          @macro_curly_count += 1 if @macro_curly_count > 0
        end
      end

      if comment || (!delimiter_state && current_char == '#')
        comment = true
        char = current_char
        char = next_char if current_char == '#'
        while true
          case char
          when '\n'
            comment = false
            beginning_of_line = true
            whitespace = true
            next_char
            incr_line_number
            @token.line_number = @line_number
            @token.column_number = @column_number
            break
          when '{'
            break
          when '\\'
            if peek_next_char == '{'
              break
            else
              char = next_char
            end
          when '\0'
            raise "unterminated macro"
          else
            char = next_char
          end
        end
        @token.type = :MACRO_LITERAL
        @token.value = string_range(start)
        @token.macro_state = Token::MacroState.new(whitespace, nest, control_nest, delimiter_state, beginning_of_line, yields, comment, heredocs)
        set_token_raw_from_start(start)
        return @token
      end

      if !delimiter_state && current_char == '%' && ident_start?(peek_next_char)
        char = next_char
        if char == 'q' && peek_next_char.in?('(', '<', '[', '{', '|')
          next_char
          delimiter_state = Token::DelimiterState.new(:string, char, closing_char, 1)
          next_char
        elsif char == 'Q' && peek_next_char.in?('(', '<', '[', '{', '|')
          next_char
          delimiter_state = Token::DelimiterState.new(:string, char, closing_char, 1)
          next_char
        elsif char == 'i' && peek_next_char.in?('(', '<', '[', '{', '|')
          next_char
          delimiter_state = Token::DelimiterState.new(:symbol_array, char, closing_char, 1)
          next_char
        elsif char == 'w' && peek_next_char.in?('(', '<', '[', '{', '|')
          next_char
          delimiter_state = Token::DelimiterState.new(:string_array, char, closing_char, 1)
          next_char
        elsif char == 'x' && peek_next_char.in?('(', '<', '[', '{', '|')
          next_char
          delimiter_state = Token::DelimiterState.new(:command, char, closing_char, 1)
          next_char
        elsif char == 'r' && peek_next_char.in?('(', '<', '[', '{', '|')
          next_char
          delimiter_state = Token::DelimiterState.new(:regex, char, closing_char, 1)
          next_char
        else
          start = current_pos
          while ident_part?(char)
            char = next_char
          end
          beginning_of_line = false
          @token.type = :MACRO_VAR
          @token.value = string_range_from_pool(start)
          @token.macro_state = Token::MacroState.new(whitespace, nest, control_nest, delimiter_state, beginning_of_line, yields, comment, heredocs)
          return @token
        end
      end

      if !delimiter_state && current_char == 'e'
        if next_char == 'n'
          beginning_of_line = false
          case next_char
          when 'd'
            if whitespace && !ident_part_or_end?(peek_next_char) && peek_next_char != ':'
              if nest == 0 && control_nest == 0
                next_char :MACRO_END
                @token.macro_state = Token::MacroState.default
                return @token
              else
                nest -= 1
                whitespace = current_char.ascii_whitespace?
                next_char
              end
            end
          when 'u'
            if !delimiter_state && whitespace && next_char == 'm' && !ident_part_or_end?(next_char)
              char = current_char
              nest += 1
              whitespace = true
            end
          else
            # not a keyword
          end
        else
          whitespace = false
          beginning_of_line = false
        end
      end

      char = current_char

      until char.in?('{', '\0') ||
            (char == '\\' && peek_next_char.in?('{', '%')) ||
            (whitespace && !delimiter_state && char == 'e')
        case char
        when '\n'
          incr_line_number 0
          whitespace = true
          beginning_of_line = true
          char = next_char

          if !delimiter_state && heredocs && !heredocs.empty?
            delimiter_state = heredocs.shift
          end

          if delimiter_state && delimiter_state.kind.heredoc? && check_heredoc_end(delimiter_state)
            char = current_char
            delimiter_state = heredocs.try &.shift?
          end

          next
        when '\\'
          char = next_char
          if delimiter_state
            case char
            when delimiter_state.end
              char = next_char
            when '\\'
              char = next_char
            end
            whitespace = false
          else
            whitespace = false
          end
          next
        when '\'', '"'
          if delimiter_state
            delimiter_state = nil if delimiter_state.end == char
          else
            delimiter_state = Token::DelimiterState.new(:string, char, char)
          end
          whitespace = false
        when '%'
          case char = peek_next_char
          when '(', '[', '<', '{', '|'
            next_char
            delimiter_state = Token::DelimiterState.new(:string, char, closing_char, 1)
          else
            whitespace = false
            break if !delimiter_state && ident_start?(char)
          end
        when '#'
          if delimiter_state
            # If it's "#{...", we don't want "#{{{" to parse it as "# {{ {", but as "#{ {{"
            # (macro expression inside a string interpolation)
            if peek_next_char == '{'
              char = next_char

              # We should now consider things that follow as crystal expressions,
              # so we reset the delimiter state but save it in a stack
              @macro_curly_count += 1
              @delimiter_state_stack.push delimiter_state
              delimiter_state = nil
            end
            whitespace = false
          else
            break
          end
        when '}'
          if delimiter_state && delimiter_state.end == '}'
            delimiter_state = delimiter_state.with_open_count_delta(-1)
            if delimiter_state.open_count == 0
              delimiter_state = nil
            end
          elsif @macro_curly_count > 0
            # Once we find the final '}' that closes the interpolation,
            # we are back inside the delimiter
            if @macro_curly_count == 1
              delimiter_state = @delimiter_state_stack.pop
            end
            @macro_curly_count -= 1
          end
        when '<'
          if !delimiter_state && @delimiter_state_stack.empty? && (heredoc_delimiter_state = lookahead { check_heredoc_start })
            heredocs ||= [] of Token::DelimiterState
            heredocs << heredoc_delimiter_state
            char = current_char
            next
          end
        else
          if !delimiter_state && whitespace && lookahead { char == 'y' && char_sequence?('i', 'e', 'l', 'd') && !ident_part_or_end?(peek_next_char) }
            yields = true
            char = current_char
            whitespace = true
            beginning_of_line = false
          elsif !delimiter_state && whitespace && (keyword = lookahead { macro_starts_with_keyword?(beginning_of_line) })
            char = current_char

            nest += 1 unless keyword.abstract_def?
            whitespace = true
            beginning_of_line = false
            next
          else
            char = current_char

            if delimiter_state
              case char
              when delimiter_state.nest
                delimiter_state = delimiter_state.with_open_count_delta(+1)
              when delimiter_state.end
                delimiter_state = delimiter_state.with_open_count_delta(-1)
                if delimiter_state.open_count == 0
                  delimiter_state = nil
                end
              end
            end

            # If an assignment comes, we accept if/unless/while/until as nesting
            if char == '=' && peek_next_char.ascii_whitespace?
              whitespace = false
              beginning_of_line = true
            else
              whitespace = char.ascii_whitespace? || char.in?(';', '(', '[', '{')
              if beginning_of_line && !whitespace
                beginning_of_line = false
              end
            end
          end
        end
        char = next_char
      end

      @token.type = :MACRO_LITERAL
      @token.value = string_range(start)
      @token.macro_state = Token::MacroState.new(whitespace, nest, control_nest, delimiter_state, beginning_of_line, yields, comment, heredocs)
      set_token_raw_from_start(start)

      @token
    end

    def lookahead(preserve_token_on_fail = false, &)
      old_pos, old_line, old_column = current_pos, @line_number, @column_number
      @temp_token.copy_from(@token) if preserve_token_on_fail

      result = yield
      unless result
        self.current_pos, @line_number, @column_number = old_pos, old_line, old_column
        @token.copy_from(@temp_token) if preserve_token_on_fail
      end
      result
    end

    def peek_ahead(&)
      result = uninitialized typeof(yield)
      lookahead(preserve_token_on_fail: true) do
        result = yield
        nil
      end
      result
    end

    def skip_macro_whitespace
      start = current_pos
      while current_char.ascii_whitespace?
        if current_char == '\n'
          incr_line_number 0
        end
        next_char
      end
      if @wants_raw
        string_range(start)
      else
        ""
      end
    end

    enum MacroKeywordState
      AbstractDef
      Other
    end

    def macro_starts_with_keyword?(beginning_of_line) : MacroKeywordState?
      case char = current_char
      when 'a'
        case next_char
        when 'b'
          if char_sequence?('s', 't', 'r', 'a', 'c', 't') && next_char.whitespace?
            case next_char
            when 'd'
              MacroKeywordState::AbstractDef if char_sequence?('e', 'f') && peek_not_ident_part_or_end_next_char
            when 'c'
              MacroKeywordState::Other if char_sequence?('l', 'a', 's', 's') && peek_not_ident_part_or_end_next_char
            when 's'
              MacroKeywordState::Other if char_sequence?('t', 'r', 'u', 'c', 't') && peek_not_ident_part_or_end_next_char
            end
          end
        when 'n'
          MacroKeywordState::Other if char_sequence?('n', 'o', 't', 'a', 't', 'i', 'o', 'n') && peek_not_ident_part_or_end_next_char
        end
      when 'b'
        MacroKeywordState::Other if char_sequence?('e', 'g', 'i', 'n') && peek_not_ident_part_or_end_next_char
      when 'c'
        case next_char
        when 'a'
          MacroKeywordState::Other if char_sequence?('s', 'e') && peek_not_ident_part_or_end_next_char
        when 'l'
          MacroKeywordState::Other if char_sequence?('a', 's', 's') && peek_not_ident_part_or_end_next_char
        end
      when 'd'
        case next_char
        when 'o'
          MacroKeywordState::Other if peek_not_ident_part_or_end_next_char
        when 'e'
          MacroKeywordState::Other if next_char == 'f' && peek_not_ident_part_or_end_next_char
        end
      when 'f'
        MacroKeywordState::Other if char_sequence?('u', 'n') && peek_not_ident_part_or_end_next_char
      when 'i'
        MacroKeywordState::Other if beginning_of_line && next_char == 'f' && peek_not_ident_part_or_end_next_char
      when 'l'
        MacroKeywordState::Other if char_sequence?('i', 'b') && peek_not_ident_part_or_end_next_char
      when 'm'
        case next_char
        when 'a'
          MacroKeywordState::Other if char_sequence?('c', 'r', 'o') && peek_not_ident_part_or_end_next_char
        when 'o'
          MacroKeywordState::Other if char_sequence?('d', 'u', 'l', 'e') && peek_not_ident_part_or_end_next_char
        end
      when 's'
        case next_char
        when 'e'
          MacroKeywordState::Other if char_sequence?('l', 'e', 'c', 't') && !ident_part_or_end?(peek_next_char) && next_char
        when 't'
          MacroKeywordState::Other if char_sequence?('r', 'u', 'c', 't') && !ident_part_or_end?(peek_next_char) && next_char
        end
      when 'u'
        if next_char == 'n'
          case next_char
          when 'i'
            MacroKeywordState::Other if char_sequence?('o', 'n') && peek_not_ident_part_or_end_next_char
          when 'l'
            MacroKeywordState::Other if beginning_of_line && char_sequence?('e', 's', 's') && peek_not_ident_part_or_end_next_char
          when 't'
            MacroKeywordState::Other if beginning_of_line && char_sequence?('i', 'l') && peek_not_ident_part_or_end_next_char
          end
        end
      when 'w'
        MacroKeywordState::Other if beginning_of_line && char_sequence?('h', 'i', 'l', 'e') && peek_not_ident_part_or_end_next_char
      end
    end

    def check_heredoc_start
      return nil unless current_char == '<' && char_sequence?('<', '-')

      has_single_quote = false
      found_closing_single_quote = false

      if next_char == '\''
        has_single_quote = true
        next_char
      end

      return nil unless ident_part?(current_char)

      start_here = current_pos
      end_here = 0

      while true
        char = next_char
        case
        when char == '\r'
          if peek_next_char == '\n'
            end_here = current_pos
            next_char
            break
          else
            return nil
          end
        when char == '\n'
          end_here = current_pos
          break
        when ident_part?(char)
          # ok
        when char == '\0'
          return nil
        when has_single_quote
          if char == '\''
            found_closing_single_quote = true
            end_here = current_pos
            next_char
            break
          else
            # wait until another quote
          end
        else
          end_here = current_pos
          break
        end
      end

      return nil if has_single_quote && !found_closing_single_quote

      here = string_range(start_here, end_here)
      Token::DelimiterState.new(:heredoc, here, here, allow_escapes: !has_single_quote)
    end

    def consume_octal_escape(char)
      value = char - '0'
      count = 1
      while count <= 3 && '0' <= peek_next_char < '8'
        next_char
        value = value * 8 + (current_char - '0')
        count += 1
      end
      if value >= 256
        raise "octal value too big"
      end
      value.to_u8
    end

    def consume_char_unicode_escape
      char = peek_next_char
      if char == '{'
        next_char
        consume_braced_unicode_escape
      else
        consume_non_braced_unicode_escape
      end
    end

    def consume_string_hex_escape
      char = next_char
      high = char.to_i?(16)
      raise "invalid hex escape" unless high

      char = next_char
      low = char.to_i?(16)
      raise "invalid hex escape" unless low

      ((high << 4) | low).to_u8
    end

    def consume_string_unicode_escape
      char = peek_next_char
      if char == '{'
        next_char
        consume_string_unicode_brace_escape
      else
        consume_non_braced_unicode_escape.chr.to_s
      end
    end

    def consume_string_unicode_brace_escape
      String.build do |str|
        while true
          str << consume_braced_unicode_escape(allow_spaces: true).chr
          break unless current_char == ' '
        end
      end
    end

    def consume_non_braced_unicode_escape
      codepoint = 0
      4.times do
        hex_value = char_to_hex(next_char) { expected_hexadecimal_character_in_unicode_escape }
        codepoint = 16 * codepoint + hex_value
      end
      if 0xD800 <= codepoint <= 0xDFFF
        raise "invalid unicode codepoint (surrogate half)"
      end
      codepoint
    end

    def consume_braced_unicode_escape(allow_spaces = false)
      codepoint = 0
      found_curly = false
      found_space = false
      found_digit = false
      char = '\0'

      6.times do
        char = next_char
        case char
        when '}'
          found_curly = true
          break
        when ' '
          if allow_spaces
            found_space = true
            break
          else
            expected_hexadecimal_character_in_unicode_escape
          end
        else
          hex_value = char_to_hex(char) { expected_hexadecimal_character_in_unicode_escape }
          codepoint = 16 * codepoint + hex_value
          found_digit = true
        end
      end

      if !found_digit
        expected_hexadecimal_character_in_unicode_escape
      elsif codepoint > 0x10FFFF
        raise "invalid unicode codepoint (too large)"
      elsif 0xD800 <= codepoint <= 0xDFFF
        raise "invalid unicode codepoint (surrogate half)"
      end

      unless found_space
        unless found_curly
          char = next_char
        end

        unless char == '}'
          raise "expected '}' to close unicode escape"
        end
      end

      codepoint
    end

    def expected_hexadecimal_character_in_unicode_escape
      raise "expected hexadecimal character in unicode escape"
    end

    def string_token_escape_value(value)
      next_char :STRING
      @token.value = value
    end

    def delimited_pair(kind : Token::DelimiterKind, string_nest, string_end, start, allow_escapes = true, advance = true)
      next_char if advance
      @token.type = :DELIMITER_START
      @token.delimiter_state = Token::DelimiterState.new(kind, string_nest, string_end, allow_escapes)
      set_token_raw_from_start(start)
    end

    def next_string_array_token
      while true
        if current_char == '\n'
          next_char
          incr_line_number 1
        elsif current_char.ascii_whitespace?
          next_char
        else
          break
        end
      end

      reset_token

      if current_char == @token.delimiter_state.end
        @token.raw = current_char.to_s if @wants_raw
        next_char :STRING_ARRAY_END
        return @token
      end

      start = current_pos
      sub_start = start
      value = String::Builder.new

      escaped = false
      while true
        case current_char
        when Char::ZERO
          break # raise is handled by parser
        when @token.delimiter_state.end
          unless escaped
            if @token.delimiter_state.open_count == 0
              break
            else
              @token.delimiter_state = @token.delimiter_state.with_open_count_delta(-1)
            end
          end
        when @token.delimiter_state.nest
          unless escaped
            @token.delimiter_state = @token.delimiter_state.with_open_count_delta(+1)
          end
        when .ascii_whitespace?
          break unless escaped
        else
          if escaped
            value << '\\'
          end
        end

        escaped = current_char == '\\'
        if escaped
          value.write @reader.string.to_slice[sub_start, current_pos - sub_start]
          sub_start = current_pos + 1
        end

        next_char
      end

      if start == current_pos
        @token.type = :EOF
        return @token
      end

      value.write @reader.string.to_slice[sub_start, current_pos - sub_start]

      @token.type = :STRING
      @token.value = value.to_s
      set_token_raw_from_start(start)

      @token
    end

    def char_to_hex(char, &)
      if '0' <= char <= '9'
        char - '0'
      elsif 'a' <= char <= 'f'
        10 + (char - 'a')
      elsif 'A' <= char <= 'F'
        10 + (char - 'A')
      else
        yield
      end
    end

    def consume_loc_pragma
      case current_char
      when '"'
        # skip '"'
        next_char_no_column_increment

        filename_pos = current_pos

        while true
          case current_char
          when '"'
            break
          when '\0'
            raise "unexpected end of file in loc pragma"
          else
            next_char_no_column_increment
          end
        end

        incr_column_number (current_pos - filename_pos) + 7 # == "#<loc:\"".size
        filename = string_range(filename_pos)

        # skip '"'
        next_char

        unless current_char == ','
          raise "expected ',' in loc pragma after filename"
        end
        next_char

        line_number = 0
        while true
          case current_char
          when '0'..'9'
            line_number = 10 * line_number + (current_char - '0').to_i
          when ','
            next_char
            break
          else
            raise "expected digit or ',' in loc pragma for line number"
          end
          next_char
        end

        column_number = 0
        while true
          case current_char
          when '0'..'9'
            column_number = 10 * column_number + (current_char - '0').to_i
          when '>'
            next_char
            break
          else
            raise "expected digit or '>' in loc pragma for column_number number"
          end
          next_char
        end

        set_location filename, line_number, column_number
      when 'p'
        # skip 'p'
        next_char_no_column_increment

        case current_char
        when 'o'
          unless char_sequence?('p', '>', column_increment: false)
            raise %(expected #<loc:push>, #<loc:pop> or #<loc:"...">)
          end

          # skip '>'
          next_char_no_column_increment

          incr_column_number 10 # == "#<loc:pop>".size

          pop_location
        when 'u'
          unless char_sequence?('s', 'h', '>', column_increment: false)
            raise %(expected #<loc:push>, #<loc:pop> or #<loc:"...">)
          end

          # skip '>'
          next_char_no_column_increment

          incr_column_number 11 # == "#<loc:push>".size

          @token.line_number = @line_number
          @token.column_number = @column_number
          push_location
        else
          raise %(expected #<loc:push>, #<loc:pop> or #<loc:"...">)
        end
      else
        raise %(expected #<loc:push>, #<loc:pop> or #<loc:"...">)
      end
    end

    private def consume_heredoc_start(start)
      has_single_quote = false
      found_closing_single_quote = false

      if next_char == '\''
        has_single_quote = true
        next_char
      end

      unless ident_part?(current_char)
        raise "heredoc identifier starts with invalid character"
      end

      start_here = current_pos
      end_here = 0

      while true
        char = next_char
        case
        when char == '\r'
          if peek_next_char == '\n'
            end_here = current_pos
            next_char
            break
          else
            raise "expecting '\\n' after '\\r'"
          end
        when char == '\n'
          end_here = current_pos
          break
        when ident_part?(char)
          # ok
        when char == '\0'
          raise "Unexpected EOF on heredoc identifier"
        when has_single_quote
          if char == '\''
            found_closing_single_quote = true
            end_here = current_pos
            next_char
            break
          else
            # wait until another quote
          end
        else
          end_here = current_pos
          break
        end
      end

      if has_single_quote && !found_closing_single_quote
        raise "expecting closing single quote"
      end

      here = string_range(start_here, end_here)

      delimited_pair :heredoc, here, here, start, allow_escapes: !has_single_quote, advance: false
    end

    private def consume_symbol
      case char = current_char
      when ':'
        next_char :OP_COLON_COLON
      when '+'
        next_char_and_symbol "+"
      when '-'
        next_char_and_symbol "-"
      when '*'
        if next_char == '*'
          next_char_and_symbol "**"
        else
          symbol "*"
        end
      when '/'
        case next_char
        when '/'
          next_char_and_symbol "//"
        else
          symbol "/"
        end
      when '='
        case next_char
        when '='
          if next_char == '='
            next_char_and_symbol "==="
          else
            symbol "=="
          end
        when '~'
          next_char_and_symbol "=~"
        else
          unknown_token
        end
      when '!'
        case next_char
        when '='
          next_char_and_symbol "!="
        when '~'
          next_char_and_symbol "!~"
        else
          symbol "!"
        end
      when '<'
        case next_char
        when '='
          if next_char == '>'
            next_char_and_symbol "<=>"
          else
            symbol "<="
          end
        when '<'
          next_char_and_symbol "<<"
        else
          symbol "<"
        end
      when '>'
        case next_char
        when '='
          next_char_and_symbol ">="
        when '>'
          next_char_and_symbol ">>"
        else
          symbol ">"
        end
      when '&'
        case next_char
        when '+'
          next_char_and_symbol "&+"
        when '-'
          next_char_and_symbol "&-"
        when '*'
          case next_char
          when '*'
            next_char_and_symbol "&**"
          else
            symbol "&*"
          end
        else
          symbol "&"
        end
      when '|'
        next_char_and_symbol "|"
      when '^'
        next_char_and_symbol "^"
      when '~'
        next_char_and_symbol "~"
      when '%'
        next_char_and_symbol "%"
      when '['
        if next_char == ']'
          case next_char
          when '='
            next_char_and_symbol "[]="
          when '?'
            next_char_and_symbol "[]?"
          else
            symbol "[]"
          end
        else
          unknown_token
        end
      when '"'
        line = @line_number
        column = @column_number
        start = current_pos + 1
        io = IO::Memory.new
        while true
          char = next_char
          case char
          when '\\'
            case char = next_char
            when 'a'
              io << '\a'
            when 'b'
              io << '\b'
            when 'n'
              io << '\n'
            when 'r'
              io << '\r'
            when 't'
              io << '\t'
            when 'v'
              io << '\v'
            when 'f'
              io << '\f'
            when 'e'
              io << '\e'
            when 'x'
              io.write_byte consume_string_hex_escape
            when 'u'
              io << consume_string_unicode_escape
            when '0', '1', '2', '3', '4', '5', '6', '7'
              io.write_byte consume_octal_escape(char)
            when '\n'
              incr_line_number nil
              io << '\n'
            when '\0'
              raise "unterminated quoted symbol", line, column
            else
              io << char
            end
          when '"'
            break
          when '\0'
            raise "unterminated quoted symbol", line, column
          else
            io << char
          end
        end

        @token.type = :SYMBOL
        @token.value = io.to_s
        next_char
        set_token_raw_from_start(start - 2)
      else
        if ident_start?(char)
          start = current_pos
          while ident_part?(next_char)
            # Nothing to do
          end
          if current_char == '?' || (current_char.in?('!', '=') && peek_next_char != '=')
            next_char
          end
          @token.type = :SYMBOL
          @token.value = string_range_from_pool(start)
          set_token_raw_from_start(start - 1)
        else
          @token.type = :OP_COLON
        end
      end
    end

    private def consume_variable(token_type : Token::Kind, start)
      if ident_start?(current_char)
        while ident_part?(next_char)
          # Nothing to do
        end
        @token.type = token_type
        @token.value = string_range_from_pool(start)
      else
        unknown_token
      end
    end

    def set_location(filename, line_number, column_number)
      @token.filename = @filename = filename
      @token.line_number = @line_number = line_number
      @token.column_number = @column_number = column_number
    end

    def pop_location
      if @stacked
        @stacked = false
        @token.filename = @filename = @stacked_filename
        @token.line_number = @line_number = @stacked_line_number
        @token.column_number = @column_number = @stacked_column_number
      end
    end

    def push_location
      unless @stacked
        @stacked = true
        @stacked_filename, @stacked_line_number, @stacked_column_number = @filename, @line_number, @column_number
      end
    end

    def incr_column_number(d = 1)
      @column_number += d
      @stacked_column_number += d if @stacked
    end

    def incr_line_number(column_number = 1)
      @line_number += 1
      @column_number = column_number if column_number
      if @stacked
        @stacked_line_number += 1
        @stacked_column_number = column_number if column_number
      end
    end

    def next_char_no_column_increment
      char = @reader.next_char
      if error = @reader.error
        ::raise InvalidByteSequenceError.new("Unexpected byte 0x#{error.to_s(16)} at position #{@reader.pos}, malformed UTF-8")
      end
      char
    end

    def next_char
      incr_column_number
      next_char_no_column_increment
    end

    def next_char_check_line
      char = next_char_no_column_increment
      if char == '\n'
        incr_line_number
      else
        incr_column_number
      end
      char
    end

    def next_char(token_type : Token::Kind)
      next_char
      @token.type = token_type
    end

    def reset_token
      @token.value = nil
      @token.line_number = @line_number
      @token.column_number = @column_number
      @token.filename = @filename
      @token.location = nil
      @token.passed_backslash_newline = false
      @token.doc_buffer = nil unless @token.type.space? || @token.type.newline?
      @token.invalid_escape = false
      @token_end_location = nil
    end

    def next_token_skip_space
      next_token
      skip_space
    end

    def next_token_skip_space_or_newline
      next_token
      skip_space_or_newline
    end

    def next_token_skip_statement_end
      next_token
      skip_statement_end
    end

    def next_token_never_a_symbol
      @wants_symbol = false
      next_token.tap { @wants_symbol = true }
    end

    def current_char
      @reader.current_char
    end

    def peek_next_char
      @reader.peek_next_char
    end

    def current_pos
      @reader.pos
    end

    def current_pos=(pos)
      @reader.pos = pos
    end

    def string
      @reader.string
    end

    def string_range(start_pos)
      string_range(start_pos, current_pos)
    end

    def string_range(start_pos, end_pos)
      @reader.string.byte_slice(start_pos, end_pos - start_pos)
    end

    def string_range_from_pool(start_pos)
      string_range_from_pool(start_pos, current_pos)
    end

    def string_range_from_pool(start_pos, end_pos)
      @string_pool.get slice_range(start_pos, end_pos)
    end

    def slice_range(start_pos)
      slice_range(start_pos, current_pos)
    end

    def slice_range(start_pos, end_pos)
      Slice.new(@reader.string.to_unsafe + start_pos, end_pos - start_pos)
    end

    def self.ident_start?(char)
      char.ascii_letter? || char == '_' || char.ord > 0x9F
    end

    def self.ident_part?(char)
      ident_start?(char) || char.ascii_number?
    end

    def self.ident?(name)
      !!name[0]?.try { |char| ident_start?(char) }
    end

    def self.setter?(name)
      ident?(name) && name.ends_with?('=')
    end

    private delegate ident_start?, ident_part?, to: Lexer

    def ident_part_or_end?(char)
      ident_part?(char) || char.in?('?', '!')
    end

    def peek_not_ident_part_or_end_next_char
      !ident_part_or_end?(peek_next_char) && peek_next_char != ':' && next_char
    end

    def closing_char(char = current_char)
      case char
      when '<' then '>'
      when '(' then ')'
      when '[' then ']'
      when '{' then '}'
      else          char
      end
    end

    def skip_space
      while @token.type.space?
        next_token
      end
    end

    def skip_space_or_newline
      while (@token.type.space? || @token.type.newline?)
        next_token
      end
    end

    def skip_statement_end
      while (@token.type.space? || @token.type.newline? || @token.type.op_semicolon?)
        next_token
      end
    end

    def handle_slash_r_slash_n_or_slash_n
      is_slash_r = current_char == '\r'
      if is_slash_r
        if next_char != '\n'
          raise "expecting '\\n' after '\\r'"
        end
      end
      is_slash_r
    end

    private def char_sequence?(*tokens, column_increment : Bool = true)
      tokens.all? do |token|
        token == (column_increment ? next_char : next_char_no_column_increment)
      end
    end

    def unknown_token
      raise "unknown token: #{current_char.inspect}", @line_number, @column_number
    end

    def set_token_raw_from_start(start)
      @token.raw = string_range(start) if @wants_raw
    end

    def raise(message, line_number = @line_number, column_number = @column_number, filename = @filename)
      ::raise Crystal::SyntaxException.new(message, line_number, column_number, filename)
    end

    def raise(message, token : Token, size = nil)
      ::raise Crystal::SyntaxException.new(message, token.line_number, token.column_number, token.filename, size)
    end

    def raise(message, location : Location)
      raise message, location.line_number, location.column_number, location.filename
    end
  end
end