require "./token"
require "../exception"
require "string_pool"

module Crystal
  class Lexer
    property? doc_enabled : Bool
    property? comments_enabled : Bool
    property? count_whitespace : Bool
    property? wants_raw : Bool
    property? slash_is_regex : Bool
    property? wants_def_or_macro_name : Bool
    getter reader : Char::Reader
    getter token : Token
    property line_number : Int32
    property column_number : Int32
    @filename : String | VirtualFile | Nil
    @stacked_filename : String | VirtualFile | Nil
    @token_end_location : Location?
    @string_pool : StringPool

    # This is an interface for storing data associated to a heredoc
    module HeredocItem
    end

    # Heredocs pushed when found. Should be processed when encountering a newline
    getter heredocs = [] of {Token::DelimiterState, HeredocItem}

    property macro_expansion_pragmas : Hash(Int32, Array(LocPragma))? = nil

    alias LocPragma = LocSetPragma | LocPushPragma | LocPopPragma

    record LocSetPragma,
      filename : String,
      line_number : Int32,
      column_number : Int32 do
      def run_pragma(lexer)
        lexer.set_location filename, line_number, column_number
      end
    end

    record LocPushPragma do
      def run_pragma(lexer)
        lexer.push_location
      end
    end

    record LocPopPragma do
      def run_pragma(lexer)
        lexer.pop_location
      end
    end

    def initialize(string, string_pool : StringPool? = nil)
      @reader = Char::Reader.new(string)
      @token = Token.new
      @line_number = 1
      @column_number = 1
      @filename = ""
      @wants_regex = true
      @doc_enabled = false
      @comments_enabled = false
      @count_whitespace = false
      @slash_is_regex = true
      @wants_raw = false
      @wants_def_or_macro_name = false
      @string_pool = string_pool || StringPool.new

      # When lexing macro tokens, when we encounter `#{` inside
      # a string we push the current delimiter here and reset
      # the current one to nil. The reason is, inside strings
      # we don't want to consider %foo a macro variable, but
      # we do want to do this inside interpolations.
      # We then count curly braces, with @macro_curly_count,
      # until we find the last `}` and then we pop from the stack
      # and get the original delimiter.
      @delimiter_state_stack = [] of Token::DelimiterState
      @macro_curly_count = 0

      @stacked = false
      @stacked_filename = ""
      @stacked_line_number = 1
      @stacked_column_number = 1
    end

    def filename=(filename)
      @filename = filename
    end

    def next_token
      reset_token

      # Skip comments
      while current_char == '#'
        start = current_pos

        # Check #<loc:...> pragma comment
        if next_char_no_column_increment == '<' &&
           next_char_no_column_increment == 'l' &&
           next_char_no_column_increment == 'o' &&
           next_char_no_column_increment == 'c' &&
           next_char_no_column_increment == ':'
          next_char_no_column_increment
          consume_loc_pragma
          start = current_pos
        else
          if @doc_enabled
            consume_doc
          elsif @comments_enabled
            return consume_comment(start)
          else
            skip_comment
          end
        end
      end

      start = current_pos

      # Fix location by `macro_expansion_pragmas`.
      if me_pragmas = macro_expansion_pragmas
        # It might happen that the current "start" already passed some
        # location pragmas, so we must consume all of those. For example
        # if one does `@{{...}}` inside a macro, "start" will be one less
        # number than the pragma, and after consuming "@..." if we don't
        # consume the pragmas generated by `{{...}}` we'll be in an
        # incorrect location.
        while me_pragmas.first_key?.try &.<=(start)
          _, pragmas = me_pragmas.shift
          pragmas.each &.run_pragma self
        end
      end

      reset_regex_flags = true

      case current_char
      when '\0'
        @token.type = :EOF
      when ' ', '\t'
        consume_whitespace
        reset_regex_flags = false
      when '\\'
        if next_char == '\n'
          incr_line_number
          @token.passed_backslash_newline = true
          consume_whitespace
          reset_regex_flags = false
        else
          unknown_token
        end
      when '\n'
        @token.type = :NEWLINE
        next_char
        incr_line_number
        reset_regex_flags = false
        consume_newlines
      when '\r'
        if next_char == '\n'
          next_char
          @token.type = :NEWLINE
          incr_line_number
          consume_newlines
        else
          raise "expected '\\n' after '\\r'"
        end
      when '='
        case next_char
        when '='
          case next_char
          when '='
            next_char :"==="
          else
            @token.type = :"=="
          end
        when '>'
          next_char :"=>"
        when '~'
          next_char :"=~"
        else
          @token.type = :"="
        end
      when '!'
        case next_char
        when '='
          next_char :"!="
        when '~'
          next_char :"!~"
        else
          @token.type = :"!"
        end
      when '<'
        case next_char
        when '='
          case next_char
          when '>'
            next_char :"<=>"
          else
            @token.type = :"<="
          end
        when '<'
          case next_char
          when '='
            next_char :"<<="
          when '-'
            here = IO::Memory.new(20)
            has_single_quote = false
            found_closing_single_quote = false

            char = next_char
            start_here = current_pos

            if char == '\''
              has_single_quote = true
              char = next_char
              start_here = current_pos
            end

            unless ident_part?(char)
              raise "heredoc identifier starts with invalid character"
            end

            end_here = 0

            while true
              char = next_char
              case
              when char == '\r'
                if peek_next_char == '\n'
                  end_here = current_pos
                  next_char
                  break
                else
                  raise "expecting '\\n' after '\\r'"
                end
              when char == '\n'
                end_here = current_pos
                break
              when ident_part?(char)
                # ok
              when char == '\0'
                raise "Unexpected EOF on heredoc identifier"
              else
                if char == '\'' && has_single_quote
                  found_closing_single_quote = true
                  end_here = current_pos
                  next_char
                  break
                elsif has_single_quote
                  # wait until another quote
                else
                  end_here = current_pos
                  break
                end
              end
            end

            if has_single_quote && !found_closing_single_quote
              raise "expecting closing single quote"
            end

            here = string_range(start_here, end_here)

            delimited_pair :heredoc, here, here, start, allow_escapes: !has_single_quote, advance: false
          else
            @token.type = :"<<"
          end
        else
          @token.type = :"<"
        end
      when '>'
        case next_char
        when '='
          next_char :">="
        when '>'
          case next_char
          when '='
            next_char :">>="
          else
            @token.type = :">>"
          end
        else
          @token.type = :">"
        end
      when '+'
        @token.start = start
        case next_char
        when '='
          next_char :"+="
        when '0'
          scan_zero_number(start)
        when '1', '2', '3', '4', '5', '6', '7', '8', '9'
          scan_number(start)
        when '+'
          raise "postfix increment is not supported, use `exp += 1`"
        else
          @token.type = :"+"
        end
      when '-'
        @token.start = start
        case next_char
        when '='
          next_char :"-="
        when '>'
          next_char :"->"
        when '0'
          scan_zero_number start, negative: true
        when '1', '2', '3', '4', '5', '6', '7', '8', '9'
          scan_number start, negative: true
        when '-'
          raise "postfix decrement is not supported, use `exp -= 1`"
        else
          @token.type = :"-"
        end
      when '*'
        case next_char
        when '='
          next_char :"*="
        when '*'
          case next_char
          when '='
            next_char :"**="
          else
            @token.type = :"**"
          end
        else
          @token.type = :"*"
        end
      when '/'
        line = @line_number
        column = @column_number
        char = next_char
        if (@wants_def_or_macro_name || !@slash_is_regex) && char == '/'
          case next_char
          when '='
            next_char :"//="
          else
            @token.type = :"//"
          end
        elsif !@slash_is_regex && char == '='
          next_char :"/="
        elsif @wants_def_or_macro_name
          @token.type = :"/"
        elsif @slash_is_regex
          @token.type = :DELIMITER_START
          @token.delimiter_state = Token::DelimiterState.new(:regex, '/', '/')
          @token.raw = "/"
        elsif char.ascii_whitespace? || char == '\0'
          @token.type = :"/"
        elsif @wants_regex
          @token.type = :DELIMITER_START
          @token.delimiter_state = Token::DelimiterState.new(:regex, '/', '/')
          @token.raw = "/"
        else
          @token.type = :"/"
        end
      when '%'
        if @wants_def_or_macro_name
          next_char :"%"
        else
          case next_char
          when '='
            next_char :"%="
          when '(', '[', '{', '<', '|'
            delimited_pair :string, current_char, closing_char, start
          when 'i'
            case peek_next_char
            when '(', '{', '[', '<', '|'
              start_char = next_char
              next_char :SYMBOL_ARRAY_START
              @token.raw = "%i#{start_char}" if @wants_raw
              @token.delimiter_state = Token::DelimiterState.new(:symbol_array, start_char, closing_char(start_char))
            else
              @token.type = :"%"
            end
          when 'q'
            case peek_next_char
            when '(', '{', '[', '<', '|'
              next_char
              delimited_pair :string, current_char, closing_char, start, allow_escapes: false
            else
              @token.type = :"%"
            end
          when 'Q'
            case peek_next_char
            when '(', '{', '[', '<', '|'
              next_char
              delimited_pair :string, current_char, closing_char, start
            else
              @token.type = :"%"
            end
          when 'r'
            case next_char
            when '(', '[', '{', '<', '|'
              delimited_pair :regex, current_char, closing_char, start
            else
              raise "unknown %r char"
            end
          when 'x'
            case next_char
            when '(', '[', '{', '<', '|'
              delimited_pair :command, current_char, closing_char, start
            else
              raise "unknown %x char"
            end
          when 'w'
            case peek_next_char
            when '(', '{', '[', '<', '|'
              start_char = next_char
              next_char :STRING_ARRAY_START
              @token.raw = "%w#{start_char}" if @wants_raw
              @token.delimiter_state = Token::DelimiterState.new(:string_array, start_char, closing_char(start_char))
            else
              @token.type = :"%"
            end
          when '}'
            next_char :"%}"
          else
            @token.type = :"%"
          end
        end
      when '(' then next_char :"("
      when ')' then next_char :")"
      when '{'
        char = next_char
        case char
        when '%'
          next_char :"{%"
        when '{'
          next_char :"{{"
        else
          @token.type = :"{"
        end
      when '}' then next_char :"}"
      when '['
        case next_char
        when ']'
          case next_char
          when '='
            next_char :"[]="
          when '?'
            next_char :"[]?"
          else
            @token.type = :"[]"
          end
        else
          @token.type = :"["
        end
      when ']' then next_char :"]"
      when ',' then next_char :","
      when '?' then next_char :"?"
      when ';'
        reset_regex_flags = false
        next_char :";"
      when ':'
        char = next_char
        case char
        when ':'
          next_char :"::"
        when '+'
          next_char_and_symbol "+"
        when '-'
          next_char_and_symbol "-"
        when '*'
          if next_char == '*'
            next_char_and_symbol "**"
          else
            symbol "*"
          end
        when '/'
          case next_char
          when '/'
            next_char_and_symbol "//"
          else
            symbol "/"
          end
        when '='
          case next_char
          when '='
            if next_char == '='
              next_char_and_symbol "==="
            else
              symbol "=="
            end
          when '~'
            next_char_and_symbol "=~"
          else
            unknown_token
          end
        when '!'
          case next_char
          when '='
            next_char_and_symbol "!="
          when '~'
            next_char_and_symbol "!~"
          else
            symbol "!"
          end
        when '<'
          case next_char
          when '='
            if next_char == '>'
              next_char_and_symbol "<=>"
            else
              symbol "<="
            end
          when '<'
            next_char_and_symbol "<<"
          else
            symbol "<"
          end
        when '>'
          case next_char
          when '='
            next_char_and_symbol ">="
          when '>'
            next_char_and_symbol ">>"
          else
            symbol ">"
          end
        when '&'
          case next_char
          when '+'
            next_char_and_symbol "&+"
          when '-'
            next_char_and_symbol "&-"
          when '*'
            case next_char
            when '*'
              next_char_and_symbol "&**"
            else
              symbol "&*"
            end
          else
            symbol "&"
          end
        when '|'
          next_char_and_symbol "|"
        when '^'
          next_char_and_symbol "^"
        when '~'
          next_char_and_symbol "~"
        when '%'
          next_char_and_symbol "%"
        when '['
          if next_char == ']'
            case next_char
            when '='
              next_char_and_symbol "[]="
            when '?'
              next_char_and_symbol "[]?"
            else
              symbol "[]"
            end
          else
            unknown_token
          end
        when '"'
          line = @line_number
          column = @column_number
          start = current_pos + 1
          io = IO::Memory.new
          while true
            char = next_char
            case char
            when '\\'
              case char = next_char
              when 'a'
                io << '\a'
              when 'b'
                io << '\b'
              when 'n'
                io << '\n'
              when 'r'
                io << '\r'
              when 't'
                io << '\t'
              when 'v'
                io << '\v'
              when 'f'
                io << '\f'
              when 'e'
                io << '\e'
              when 'x'
                io.write_byte consume_string_hex_escape
              when 'u'
                io << consume_string_unicode_escape
              when '0', '1', '2', '3', '4', '5', '6', '7'
                io.write_byte consume_octal_escape(char)
              when '\n'
                incr_line_number nil
                io << '\n'
              when '\0'
                raise "unterminated quoted symbol", line, column
              else
                io << char
              end
            when '"'
              break
            when '\0'
              raise "unterminated quoted symbol", line, column
            else
              io << char
            end
          end

          @token.type = :SYMBOL
          @token.value = io.to_s
          next_char
          set_token_raw_from_start(start - 2)
        else
          if ident_start?(char)
            start = current_pos
            while ident_part?(next_char)
              # Nothing to do
            end
            if current_char == '?' || ((current_char == '!' || current_char == '=') && peek_next_char != '=')
              next_char
            end
            @token.type = :SYMBOL
            @token.value = string_range_from_pool(start)
            set_token_raw_from_start(start - 1)
          else
            @token.type = :":"
          end
        end
      when '~'
        next_char :"~"
      when '.'
        case next_char
        when '.'
          case next_char
          when '.'
            next_char :"..."
          else
            @token.type = :".."
          end
        else
          @token.type = :"."
        end
      when '&'
        case next_char
        when '&'
          case next_char
          when '='
            next_char :"&&="
          else
            @token.type = :"&&"
          end
        when '='
          next_char :"&="
        when '+'
          case next_char
          when '='
            next_char :"&+="
          else
            @token.type = :"&+"
          end
        when '-'
          # Check if '>' comes after '&-', making it '&->'.
          # We want to parse that like '&(->...)',
          # so we only return '&' for now.
          if peek_next_char == '>'
            @token.type = :"&"
          else
            case next_char
            when '='
              next_char :"&-="
            else
              @token.type = :"&-"
            end
          end
        when '*'
          case next_char
          when '*'
            next_char :"&**"
          when '='
            next_char :"&*="
          else
            @token.type = :"&*"
          end
        else
          @token.type = :"&"
        end
      when '|'
        case next_char
        when '|'
          case next_char
          when '='
            next_char :"||="
          else
            @token.type = :"||"
          end
        when '='
          next_char :"|="
        else
          @token.type = :"|"
        end
      when '^'
        case next_char
        when '='
          next_char :"^="
        else
          @token.type = :"^"
        end
      when '\''
        start = current_pos
        line = @line_number
        column = @column_number
        @token.type = :CHAR
        case char1 = next_char
        when '\\'
          case char2 = next_char
          when '\\'
            @token.value = '\\'
          when '\''
            @token.value = '\''
          when 'a'
            @token.value = '\a'
          when 'b'
            @token.value = '\b'
          when 'e'
            @token.value = '\e'
          when 'f'
            @token.value = '\f'
          when 'n'
            @token.value = '\n'
          when 'r'
            @token.value = '\r'
          when 't'
            @token.value = '\t'
          when 'v'
            @token.value = '\v'
          when 'u'
            value = consume_char_unicode_escape
            @token.value = value.chr
          when '0'
            @token.value = '\0'
          when '\0'
            raise "unterminated char literal", line, column
          else
            raise "invalid char escape sequence '\\#{char2}'", line, column
          end
        when '\''
          raise "invalid empty char literal (did you mean '\\''?)", line, column
        when '\0'
          raise "unterminated char literal", line, column
        else
          @token.value = char1
        end
        if next_char != '\''
          raise "unterminated char literal, use double quotes for strings", line, column
        end
        next_char
        set_token_raw_from_start(start)
      when '"', '`'
        delimiter = current_char
        if delimiter == '`' && @wants_def_or_macro_name
          next_char :"`"
        else
          next_char
          @token.type = :DELIMITER_START
          @token.delimiter_state = Token::DelimiterState.new(delimiter == '`' ? :command : :string, delimiter, delimiter)
          set_token_raw_from_start(start)
        end
      when '0'
        scan_zero_number(start)
      when '1', '2', '3', '4', '5', '6', '7', '8', '9'
        scan_number current_pos
      when '@'
        start = current_pos
        case next_char
        when '['
          next_char :"@["
        else
          class_var = false
          if current_char == '@'
            class_var = true
            next_char
          end
          if ident_start?(current_char)
            while ident_part?(next_char)
              # Nothing to do
            end
            @token.type = class_var ? :CLASS_VAR : :INSTANCE_VAR
            @token.value = string_range_from_pool(start)
          else
            unknown_token
          end
        end
      when '$'
        start = current_pos
        next_char
        case current_char
        when '~'
          next_char
          @token.type = :"$~"
        when '?'
          next_char
          @token.type = :"$?"
        when .ascii_number?
          start = current_pos
          char = next_char
          if char == '0'
            char = next_char
          else
            while char.ascii_number?
              char = next_char
            end
            char = next_char if char == '?'
          end
          @token.type = :GLOBAL_MATCH_DATA_INDEX
          @token.value = string_range_from_pool(start)
        else
          if ident_start?(current_char)
            while ident_part?(next_char)
              # Nothing to do
            end
            @token.type = :GLOBAL
            @token.value = string_range_from_pool(start)
          else
            unknown_token
          end
        end
      when 'a'
        case next_char
        when 'b'
          if next_char == 's' && next_char == 't' && next_char == 'r' && next_char == 'a' && next_char == 'c' && next_char == 't'
            return check_ident_or_keyword(:abstract, start)
          end
        when 'l'
          if next_char == 'i' && next_char == 'a' && next_char == 's'
            return check_ident_or_keyword(:alias, start)
          end
        when 's'
          peek = peek_next_char
          case peek
          when 'm'
            next_char
            return check_ident_or_keyword(:asm, start)
          when '?'
            next_char
            next_char
            @token.type = :IDENT
            @token.value = :as?
            return @token
          else
            return check_ident_or_keyword(:as, start)
          end
        when 'n'
          if next_char == 'n' && next_char == 'o' && next_char == 't' && next_char == 'a' && next_char == 't' && next_char == 'i' && next_char == 'o' && next_char == 'n'
            return check_ident_or_keyword(:annotation, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'b'
        case next_char
        when 'e'
          if next_char == 'g' && next_char == 'i' && next_char == 'n'
            return check_ident_or_keyword(:begin, start)
          end
        when 'r'
          if next_char == 'e' && next_char == 'a' && next_char == 'k'
            return check_ident_or_keyword(:break, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'c'
        case next_char
        when 'a'
          if next_char == 's' && next_char == 'e'
            return check_ident_or_keyword(:case, start)
          end
        when 'l'
          if next_char == 'a' && next_char == 's' && next_char == 's'
            return check_ident_or_keyword(:class, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'd'
        case next_char
        when 'e'
          if next_char == 'f'
            return check_ident_or_keyword(:def, start)
          end
        when 'o' then return check_ident_or_keyword(:do, start)
        else
          # scan_ident
        end
        scan_ident(start)
      when 'e'
        case next_char
        when 'l'
          case next_char
          when 's'
            case next_char
            when 'e' then return check_ident_or_keyword(:else, start)
            when 'i'
              if next_char == 'f'
                return check_ident_or_keyword(:elsif, start)
              end
            else
              # scan_ident
            end
          else
            # scan_ident
          end
        when 'n'
          case next_char
          when 'd'
            return check_ident_or_keyword(:end, start)
          when 's'
            if next_char == 'u' && next_char == 'r' && next_char == 'e'
              return check_ident_or_keyword(:ensure, start)
            end
          when 'u'
            if next_char == 'm'
              return check_ident_or_keyword(:enum, start)
            end
          else
            # scan_ident
          end
        when 'x'
          if next_char == 't' && next_char == 'e' && next_char == 'n' && next_char == 'd'
            return check_ident_or_keyword(:extend, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'f'
        case next_char
        when 'a'
          if next_char == 'l' && next_char == 's' && next_char == 'e'
            return check_ident_or_keyword(:false, start)
          end
        when 'o'
          if next_char == 'r'
            return check_ident_or_keyword(:for, start)
          end
        when 'u'
          if next_char == 'n'
            return check_ident_or_keyword(:fun, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'i'
        case next_char
        when 'f'
          return check_ident_or_keyword(:if, start)
        when 'n'
          if ident_part_or_end?(peek_next_char)
            case next_char
            when 'c'
              if next_char == 'l' && next_char == 'u' && next_char == 'd' && next_char == 'e'
                return check_ident_or_keyword(:include, start)
              end
            when 's'
              if next_char == 't' && next_char == 'a' && next_char == 'n' && next_char == 'c' && next_char == 'e' && next_char == '_' && next_char == 's' && next_char == 'i' && next_char == 'z' && next_char == 'e' && next_char == 'o' && next_char == 'f'
                return check_ident_or_keyword(:instance_sizeof, start)
              end
            else
              # scan_ident
            end
          else
            next_char
            @token.type = :IDENT
            @token.value = :in
            return @token
          end
        when 's'
          if next_char == '_' && next_char == 'a' && next_char == '?'
            return check_ident_or_keyword(:is_a?, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'l'
        case next_char
        when 'i'
          if next_char == 'b'
            return check_ident_or_keyword(:lib, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'm'
        case next_char
        when 'a'
          if next_char == 'c' && next_char == 'r' && next_char == 'o'
            return check_ident_or_keyword(:macro, start)
          end
        when 'o'
          case next_char
          when 'd'
            if next_char == 'u' && next_char == 'l' && next_char == 'e'
              return check_ident_or_keyword(:module, start)
            end
          else
            # scan_ident
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'n'
        case next_char
        when 'e'
          if next_char == 'x' && next_char == 't'
            return check_ident_or_keyword(:next, start)
          end
        when 'i'
          case next_char
          when 'l'
            if peek_next_char == '?'
              next_char
              return check_ident_or_keyword(:nil?, start)
            else
              return check_ident_or_keyword(:nil, start)
            end
          else
            # scan_ident
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'o'
        case next_char
        when 'f'
          if peek_next_char == 'f'
            next_char
            if next_char == 's' && next_char == 'e' && next_char == 't' && next_char == 'o' && next_char == 'f'
              return check_ident_or_keyword(:offsetof, start)
            end
          else
            return check_ident_or_keyword(:of, start)
          end
        when 'u'
          if next_char == 't'
            return check_ident_or_keyword(:out, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'p'
        case next_char
        when 'o'
          if next_char == 'i' && next_char == 'n' && next_char == 't' && next_char == 'e' && next_char == 'r' && next_char == 'o' && next_char == 'f'
            return check_ident_or_keyword(:pointerof, start)
          end
        when 'r'
          case next_char
          when 'i'
            if next_char == 'v' && next_char == 'a' && next_char == 't' && next_char == 'e'
              return check_ident_or_keyword(:private, start)
            end
          when 'o'
            if next_char == 't' && next_char == 'e' && next_char == 'c' && next_char == 't' && next_char == 'e' && next_char == 'd'
              return check_ident_or_keyword(:protected, start)
            end
          else
            # scan_ident
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'r'
        case next_char
        when 'e'
          case next_char
          when 's'
            case next_char
            when 'c'
              if next_char == 'u' && next_char == 'e'
                return check_ident_or_keyword(:rescue, start)
              end
            when 'p'
              if next_char == 'o' && next_char == 'n' && next_char == 'd' && next_char == 's' && next_char == '_' && next_char == 't' && next_char == 'o' && next_char == '?'
                return check_ident_or_keyword(:responds_to?, start)
              end
            else
              # scan_ident
            end
          when 't'
            if next_char == 'u' && next_char == 'r' && next_char == 'n'
              return check_ident_or_keyword(:return, start)
            end
          when 'q'
            if next_char == 'u' && next_char == 'i' && next_char == 'r' && next_char == 'e'
              return check_ident_or_keyword(:require, start)
            end
          else
            # scan_ident
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 's'
        case next_char
        when 'e'
          if next_char == 'l'
            case next_char
            when 'e'
              if next_char == 'c' && next_char == 't'
                return check_ident_or_keyword(:select, start)
              end
            when 'f'
              return check_ident_or_keyword(:self, start)
            else
              # scan_ident
            end
          end
        when 'i'
          if next_char == 'z' && next_char == 'e' && next_char == 'o' && next_char == 'f'
            return check_ident_or_keyword(:sizeof, start)
          end
        when 't'
          if next_char == 'r' && next_char == 'u' && next_char == 'c' && next_char == 't'
            return check_ident_or_keyword(:struct, start)
          end
        when 'u'
          if next_char == 'p' && next_char == 'e' && next_char == 'r'
            return check_ident_or_keyword(:super, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 't'
        case next_char
        when 'h'
          if next_char == 'e' && next_char == 'n'
            return check_ident_or_keyword(:then, start)
          end
        when 'r'
          if next_char == 'u' && next_char == 'e'
            return check_ident_or_keyword(:true, start)
          end
        when 'y'
          if next_char == 'p' && next_char == 'e'
            if peek_next_char == 'o'
              next_char
              if next_char == 'f'
                return check_ident_or_keyword(:typeof, start)
              end
            else
              return check_ident_or_keyword(:type, start)
            end
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'u'
        if next_char == 'n'
          case next_char
          when 'i'
            case next_char
            when 'o'
              if next_char == 'n'
                return check_ident_or_keyword(:union, start)
              end
            when 'n'
              if next_char == 'i' && next_char == 't' && next_char == 'i' && next_char == 'a' && next_char == 'l' && next_char == 'i' && next_char == 'z' && next_char == 'e' && next_char == 'd'
                return check_ident_or_keyword(:uninitialized, start)
              end
            else
              # scan_ident
            end
          when 'l'
            if next_char == 'e' && next_char == 's' && next_char == 's'
              return check_ident_or_keyword(:unless, start)
            end
          when 't'
            if next_char == 'i' && next_char == 'l'
              return check_ident_or_keyword(:until, start)
            end
          else
            # scan_ident
          end
        end
        scan_ident(start)
      when 'v'
        if next_char == 'e' && next_char == 'r' && next_char == 'b' && next_char == 'a' && next_char == 't' && next_char == 'i' && next_char == 'm'
          return check_ident_or_keyword(:verbatim, start)
        end
        scan_ident(start)
      when 'w'
        case next_char
        when 'h'
          case next_char
          when 'e'
            if next_char == 'n'
              return check_ident_or_keyword(:when, start)
            end
          when 'i'
            if next_char == 'l' && next_char == 'e'
              return check_ident_or_keyword(:while, start)
            end
          else
            # scan_ident
          end
        when 'i'
          if next_char == 't' && next_char == 'h'
            return check_ident_or_keyword(:with, start)
          end
        else
          # scan_ident
        end
        scan_ident(start)
      when 'y'
        if next_char == 'i' && next_char == 'e' && next_char == 'l' && next_char == 'd'
          return check_ident_or_keyword(:yield, start)
        end
        scan_ident(start)
      when '_'
        case next_char
        when '_'
          case next_char
          when 'D'
            if next_char == 'I' && next_char == 'R' && next_char == '_' && next_char == '_'
              if ident_part_or_end?(peek_next_char)
                scan_ident(start)
              else
                next_char
                @token.type = :__DIR__
                return @token
              end
            end
          when 'E'
            if next_char == 'N' && next_char == 'D' && next_char == '_' && next_char == 'L' && next_char == 'I' && next_char == 'N' && next_char == 'E' && next_char == '_' && next_char == '_'
              if ident_part_or_end?(peek_next_char)
                scan_ident(start)
              else
                next_char
                @token.type = :__END_LINE__
                return @token
              end
            end
          when 'F'
            if next_char == 'I' && next_char == 'L' && next_char == 'E' && next_char == '_' && next_char == '_'
              if ident_part_or_end?(peek_next_char)
                scan_ident(start)
              else
                next_char
                @token.type = :__FILE__
                return @token
              end
            end
          when 'L'
            if next_char == 'I' && next_char == 'N' && next_char == 'E' && next_char == '_' && next_char == '_'
              if ident_part_or_end?(peek_next_char)
                scan_ident(start)
              else
                next_char
                @token.type = :__LINE__
                return @token
              end
            end
          else
            # scan_ident
          end
        else
          unless ident_part?(current_char)
            @token.type = :UNDERSCORE
            return @token
          end
        end

        scan_ident(start)
      else
        if current_char.ascii_uppercase?
          start = current_pos
          while ident_part?(next_char)
            # Nothing to do
          end
          @token.type = :CONST
          @token.value = string_range_from_pool(start)
        elsif current_char.ascii_lowercase? || current_char == '_' || current_char.ord > 0x9F
          next_char
          scan_ident(start)
        else
          unknown_token
        end
      end

      if reset_regex_flags
        @wants_regex = true
        @slash_is_regex = false
      end

      @token
    end

    def token_end_location
      @token_end_location ||= Location.new(@filename, @line_number, @column_number - 1)
    end

    def slash_is_regex!
      @slash_is_regex = true
    end

    def slash_is_not_regex!
      @slash_is_regex = false
    end

    def consume_comment(start_pos)
      skip_comment
      @token.type = :COMMENT
      @token.value = string_range(start_pos)
      @token
    end

    def consume_doc
      char = current_char
      start_pos = current_pos

      # Ignore first whitespace after comment, like in `# some doc`
      if char == ' '
        char = next_char
        start_pos = current_pos
      end

      while char != '\n' && char != '\0'
        char = next_char_no_column_increment
      end

      if doc_buffer = @token.doc_buffer
        doc_buffer << '\n'
      else
        @token.doc_buffer = doc_buffer = IO::Memory.new
      end

      doc_buffer.write slice_range(start_pos)
    end

    def skip_comment
      char = current_char
      while char != '\n' && char != '\0'
        char = next_char_no_column_increment
      end
    end

    def consume_whitespace
      start_pos = current_pos
      @token.type = :SPACE
      next_char
      while true
        case current_char
        when ' ', '\t'
          next_char
        when '\\'
          if next_char == '\n'
            next_char
            incr_line_number
            @token.passed_backslash_newline = true
          else
            unknown_token
          end
        else
          break
        end
      end
      if @count_whitespace
        @token.value = string_range(start_pos)
      end
    end

    def consume_newlines
      # If there are heredocs we don't freely consume newlines because
      # these will be part of the heredoc string
      return unless @heredocs.empty?

      if @count_whitespace
        return
      end

      while true
        case current_char
        when '\n'
          next_char_no_column_increment
          incr_line_number nil
          @token.doc_buffer = nil
        when '\r'
          if next_char_no_column_increment != '\n'
            raise "expected '\\n' after '\\r'"
          end
          next_char_no_column_increment
          incr_line_number nil
          @token.doc_buffer = nil
        else
          break
        end
      end
    end

    def check_ident_or_keyword(symbol, start)
      if ident_part_or_end?(peek_next_char)
        scan_ident(start)
      else
        next_char
        @token.type = :IDENT
        @token.value = symbol
      end
      @token
    end

    def scan_ident(start)
      while ident_part?(current_char)
        next_char
      end
      if (current_char == '?' || current_char == '!') && peek_next_char != '='
        next_char
      end
      @token.type = :IDENT
      @token.value = string_range_from_pool(start)
      @token
    end

    def next_char_and_symbol(value)
      next_char
      symbol value
    end

    def symbol(value)
      @token.type = :SYMBOL
      @token.value = value
      @token.raw = ":#{value}" if @wants_raw
    end

    def scan_number(start, negative = false)
      @token.type = :NUMBER

      has_underscore = false
      is_integer = true
      has_suffix = true
      suffix_size = 0

      while true
        char = next_char
        if char.ascii_number?
          # Nothing to do
        elsif char == '_'
          has_underscore = true
        else
          break
        end
      end

      case current_char
      when '.'
        if peek_next_char.ascii_number?
          is_integer = false

          while true
            char = next_char
            if char.ascii_number?
              # Nothing to do
            elsif char == '_'
              has_underscore = true
            else
              break
            end
          end

          if current_char == 'e' || current_char == 'E'
            next_char

            if current_char == '+' || current_char == '-'
              next_char
            end

            while true
              if current_char.ascii_number?
                # Nothing to do
              elsif current_char == '_'
                has_underscore = true
              else
                break
              end
              next_char
            end
          end

          if current_char == 'f'
            suffix_size = consume_float_suffix
          else
            @token.number_kind = :f64
          end
        else
          @token.number_kind = :i32
          has_suffix = false
        end
      when 'e', 'E'
        is_integer = false
        next_char

        if current_char == '+' || current_char == '-'
          next_char
        end

        while true
          if current_char.ascii_number?
            # Nothing to do
          elsif current_char == '_'
            has_underscore = true
          else
            break
          end
          next_char
        end

        if current_char == 'f'
          suffix_size = consume_float_suffix
        else
          @token.number_kind = :f64
        end
      when 'f'
        is_integer = false
        suffix_size = consume_float_suffix
      when 'i'
        suffix_size = consume_int_suffix
      when 'u'
        suffix_size = consume_uint_suffix
      else
        has_suffix = false
        @token.number_kind = :i32
      end

      end_pos = current_pos - suffix_size

      if end_pos - start == 1
        # For numbers such as 0, 1, 2, 3, etc., we use a string from the poll
        string_value = string_range_from_pool(start, end_pos)
      else
        string_value = string_range(start, end_pos)
      end
      string_value = string_value.delete('_') if has_underscore

      if is_integer
        num_size = string_value.size
        num_size -= 1 if negative

        if has_suffix
          check_integer_literal_fits_in_size string_value, num_size, negative, start
        else
          deduce_integer_kind string_value, num_size, negative, start
        end
      end

      @token.value = string_value
      set_token_raw_from_start(start)
    end

    macro gen_check_int_fits_in_size(type, method, size)
      if num_size >= 20
        raise_value_doesnt_fit_in "{{type}}", string_value, start
      end
      if num_size >= {{size}}
        int_value = absolute_integer_value(string_value, negative)
        max = {{type}}::MAX.{{method}}
        max += 1 if negative

        if int_value > max
          raise_value_doesnt_fit_in "{{type}}", string_value, start
        end
      end
    end

    macro gen_check_uint_fits_in_size(type, size)
      if negative
        raise "Invalid negative value #{string_value} for {{type}}"
      end
      if num_size >= 20
        raise_value_doesnt_fit_in "{{type}}", string_value, start
      end
      if num_size >= {{size}}
        int_value = absolute_integer_value(string_value, negative)
        if int_value > {{type}}::MAX
          raise_value_doesnt_fit_in "{{type}}", string_value, start
        end
      end
    end

    def check_integer_literal_fits_in_size(string_value, num_size, negative, start)
      case @token.number_kind
      when :i8
        gen_check_int_fits_in_size Int8, to_u8, 3
      when :u8
        gen_check_uint_fits_in_size UInt8, 3
      when :i16
        gen_check_int_fits_in_size Int16, to_u16, 5
      when :u16
        gen_check_uint_fits_in_size UInt16, 5
      when :i32
        gen_check_int_fits_in_size Int32, to_u32, 10
      when :u32
        gen_check_uint_fits_in_size UInt32, 10
      when :i64
        gen_check_int_fits_in_size Int64, to_u64, 19
      when :u64
        if negative
          raise "Invalid negative value #{string_value} for UInt64"
        end

        check_value_fits_in_uint64 string_value, num_size, start
      else
        # TODO: handle i128 and u128
      end
    end

    def deduce_integer_kind(string_value, num_size, negative, start)
      if negative
        check_negative_value_fits_in_int64 string_value, num_size, start
      else
        check_value_fits_in_uint64 string_value, num_size, start
      end

      if num_size >= 10
        int_value = absolute_integer_value(string_value, negative)

        int64max = Int64::MAX.to_u64
        int64max += 1 if negative

        int32max = Int32::MAX.to_u32
        int32max += 1 if negative

        if int_value > int64max
          @token.number_kind = :u64
        elsif int_value > int32max
          @token.number_kind = :i64
        end
      end
    end

    def absolute_integer_value(string_value, negative)
      if negative
        string_value[1..-1].to_u64
      else
        string_value.to_u64
      end
    end

    def check_negative_value_fits_in_int64(string_value, num_size, start)
      if num_size > 19
        raise_value_doesnt_fit_in "Int64", string_value, start
      end

      if num_size == 19
        i = 1 # skip '-'
        "9223372036854775808".each_byte do |byte|
          string_byte = string_value.byte_at(i)
          if string_byte > byte
            raise_value_doesnt_fit_in "Int64", string_value, start
          elsif string_byte < byte
            break
          end
          i += 1
        end
      end
    end

    def check_value_fits_in_uint64(string_value, num_size, start)
      if num_size > 20
        raise_value_doesnt_fit_in "UInt64", string_value, start
      end

      if num_size == 20
        i = 0
        "18446744073709551615".each_byte do |byte|
          string_byte = string_value.byte_at(i)
          if string_byte > byte
            raise_value_doesnt_fit_in "UInt64", string_value, start
          elsif string_byte < byte
            break
          end
          i += 1
        end
      end
    end

    def raise_value_doesnt_fit_in(type, string_value, start)
      raise "#{string_value} doesn't fit in an #{type}", @token, (current_pos - start)
    end

    def scan_zero_number(start, negative = false)
      case peek_next_char
      when 'x'
        scan_hex_number(start, negative)
      when 'o'
        scan_octal_number(start, negative)
      when 'b'
        scan_bin_number(start, negative)
      when '.'
        scan_number(start)
      when 'i'
        @token.type = :NUMBER
        @token.value = "0"
        next_char
        consume_int_suffix
        set_token_raw_from_start(start)
      when 'f'
        @token.type = :NUMBER
        @token.value = "0"
        next_char
        consume_float_suffix
        set_token_raw_from_start(start)
      when 'u'
        @token.type = :NUMBER
        @token.value = "0"
        next_char
        consume_uint_suffix
        set_token_raw_from_start(start)
      when '_'
        case peek_next_char
        when 'i'
          @token.type = :NUMBER
          @token.value = "0"
          next_char
          consume_int_suffix
          set_token_raw_from_start(start)
        when 'f'
          @token.type = :NUMBER
          @token.value = "0"
          next_char
          consume_float_suffix
        when 'u'
          @token.type = :NUMBER
          @token.value = "0"
          next_char
          consume_uint_suffix
        else
          scan_number(start)
        end
      else
        if next_char.ascii_number?
          raise "octal constants should be prefixed with 0o"
        else
          finish_scan_prefixed_number 0_u64, false, start
        end
      end
    end

    def scan_bin_number(start, negative)
      next_char

      num = 0_u64
      while true
        case next_char
        when '0'
          num *= 2
        when '1'
          num = num * 2 + 1
        when '_'
          # Nothing
        else
          break
        end
      end

      finish_scan_prefixed_number num, negative, start
    end

    def scan_octal_number(start, negative)
      next_char

      num = 0_u64

      while true
        char = next_char
        if '0' <= char <= '7'
          num = num * 8 + (char - '0')
        elsif char == '_'
        else
          break
        end
      end

      finish_scan_prefixed_number num, negative, start
    end

    def scan_hex_number(start, negative = false)
      next_char

      num = 0_u64
      while true
        char = next_char
        if char == '_'
        else
          hex_value = char_to_hex(char) { nil }
          if hex_value
            num = num * 16 + hex_value
          else
            break
          end
        end
      end

      finish_scan_prefixed_number num, negative, start
    end

    def finish_scan_prefixed_number(num, negative, start)
      if negative
        string_value = (num.to_i64 * -1).to_s
      else
        string_value = num.to_s
      end

      name_size = string_value.size
      name_size -= 1 if negative

      case current_char
      when 'i'
        consume_int_suffix
        check_integer_literal_fits_in_size string_value, name_size, negative, start
      when 'u'
        consume_uint_suffix
        check_integer_literal_fits_in_size string_value, name_size, negative, start
      else
        @token.number_kind = :i32
        deduce_integer_kind string_value, name_size, negative, start
      end

      first_byte = @reader.string.byte_at(start)
      if first_byte === '+'
        string_value = "+#{string_value}"
      elsif first_byte === '-' && num == 0
        string_value = "-0"
      end

      @token.type = :NUMBER
      @token.value = string_value
      set_token_raw_from_start(start)
    end

    def consume_int_suffix
      case next_char
      when '8'
        next_char
        @token.number_kind = :i8
        2
      when '1'
        case next_char
        when '2'
          if next_char == '8'
            next_char
            @token.number_kind = :i128
            4
          else
            raise "invalid int suffix"
          end
        when '6'
          next_char
          @token.number_kind = :i16
          3
        else
          raise "invalid int suffix"
        end
      when '3'
        if next_char == '2'
          next_char
          @token.number_kind = :i32
          3
        else
          raise "invalid int suffix"
        end
      when '6'
        if next_char == '4'
          next_char
          @token.number_kind = :i64
          3
        else
          raise "invalid int suffix"
        end
      else
        raise "invalid int suffix"
      end
    end

    def consume_uint_suffix
      case next_char
      when '8'
        next_char
        @token.number_kind = :u8
        2
      when '1'
        case next_char
        when '2'
          if next_char == '8'
            next_char
            @token.number_kind = :u128
            4
          else
            raise "invalid uint suffix"
          end
        when '6'
          next_char
          @token.number_kind = :u16
          3
        else
          raise "invalid uint suffix"
        end
      when '3'
        if next_char == '2'
          next_char
          @token.number_kind = :u32
          3
        else
          raise "invalid uint suffix"
        end
      when '6'
        if next_char == '4'
          next_char
          @token.number_kind = :u64
          3
        else
          raise "invalid uint suffix"
        end
      else
        raise "invalid uint suffix"
      end
    end

    def consume_float_suffix
      case next_char
      when '3'
        if next_char == '2'
          next_char
          @token.number_kind = :f32
          3
        else
          raise "invalid float suffix"
        end
      when '6'
        if next_char == '4'
          next_char
          @token.number_kind = :f64
          3
        else
          raise "invalid float suffix"
        end
      else
        raise "invalid float suffix"
      end
    end

    def next_string_token(delimiter_state)
      reset_token

      @token.line_number = @line_number
      @token.delimiter_state = delimiter_state

      start = current_pos
      string_end = delimiter_state.end
      string_nest = delimiter_state.nest
      string_open_count = delimiter_state.open_count

      # For empty heredocs:
      if @token.type == :NEWLINE && delimiter_state.kind == :heredoc
        if check_heredoc_end delimiter_state
          set_token_raw_from_start start
          return @token
        end
      end

      case current_char
      when '\0'
        raise_unterminated_quoted delimiter_state
      when string_end
        next_char
        if string_open_count == 0
          @token.type = :DELIMITER_END
        else
          @token.type = :STRING
          @token.value = string_end.to_s
          @token.delimiter_state = delimiter_state.with_open_count_delta(-1)
        end
      when string_nest
        next_char
        @token.type = :STRING
        @token.value = string_nest.to_s
        @token.delimiter_state = delimiter_state.with_open_count_delta(+1)
      when '\\'
        if delimiter_state.allow_escapes
          if delimiter_state.kind == :regex
            char = next_char
            next_char
            @token.type = :STRING
            if string_end == '/' && char == '/'
              @token.value = "/"
            else
              @token.value = "\\#{char}"
            end
          else
            case char = next_char
            when '\\'
              string_token_escape_value "\\"
            when string_end
              string_token_escape_value string_end.to_s
            when string_nest
              string_token_escape_value string_nest.to_s
            when '#'
              string_token_escape_value "#"
            when 'a'
              string_token_escape_value "\a"
            when 'b'
              string_token_escape_value "\b"
            when 'n'
              string_token_escape_value "\n"
            when 'r'
              string_token_escape_value "\r"
            when 't'
              string_token_escape_value "\t"
            when 'v'
              string_token_escape_value "\v"
            when 'f'
              string_token_escape_value "\f"
            when 'e'
              string_token_escape_value "\e"
            when 'x'
              value = consume_string_hex_escape
              next_char
              @token.type = :STRING
              @token.value = String.new(1) do |buffer|
                buffer[0] = value
                {1, 0}
              end
            when 'u'
              value = consume_string_unicode_escape
              next_char
              @token.type = :STRING
              @token.value = value
            when '0', '1', '2', '3', '4', '5', '6', '7'
              value = consume_octal_escape(char)
              next_char
              @token.type = :STRING
              @token.value = String.new(1) do |buffer|
                buffer[0] = value
                {1, 0}
              end
            when '\n'
              incr_line_number
              @token.line_number = @line_number

              # Skip until the next non-whitespace char
              while true
                char = next_char
                case char
                when '\0'
                  raise_unterminated_quoted delimiter_state
                when '\n'
                  incr_line_number
                  @token.line_number = @line_number
                when .ascii_whitespace?
                  # Continue
                else
                  break
                end
              end
              next_string_token delimiter_state
            when '\0'
              raise_unterminated_quoted delimiter_state
            else
              @token.type = :STRING
              @token.value = current_char.to_s
              @token.invalid_escape = true
              next_char
            end
          end
        else
          @token.type = :STRING
          @token.value = current_char.to_s
          next_char
        end
      when '#'
        if delimiter_state.allow_escapes
          if peek_next_char == '{'
            next_char
            next_char
            @token.type = :INTERPOLATION_START
          else
            next_char
            @token.type = :STRING
            @token.value = "#"
          end
        else
          next_char
          @token.type = :STRING
          @token.value = "#"
        end
      when '\r', '\n'
        is_slash_r = current_char == '\r'
        if is_slash_r
          if next_char != '\n'
            raise "expecting '\\n' after '\\r'"
          end
        end

        next_char
        incr_line_number 1
        @token.line_number = @line_number
        @token.column_number = @column_number

        if delimiter_state.kind == :heredoc
          unless check_heredoc_end delimiter_state
            next_string_token_noescape delimiter_state
            @token.value = string_range(start)
          end
        else
          @token.type = :STRING
          @token.value = is_slash_r ? "\r\n" : "\n"
        end
      else
        next_string_token_noescape delimiter_state
        @token.value = string_range(start)
      end

      set_token_raw_from_start(start)

      @token
    end

    def next_string_token_noescape(delimiter_state)
      string_end = delimiter_state.end
      string_nest = delimiter_state.nest

      while current_char != string_end &&
            current_char != string_nest &&
            current_char != '\0' &&
            current_char != '\\' &&
            current_char != '#' &&
            current_char != '\r' &&
            current_char != '\n'
        next_char
      end

      @token.type = :STRING
    end

    def check_heredoc_end(delimiter_state)
      string_end = delimiter_state.end.to_s
      old_pos = current_pos
      old_column = @column_number

      while current_char == ' ' || current_char == '\t'
        next_char
      end

      indent = @column_number - 1

      if string_end.starts_with?(current_char)
        reached_end = false

        string_end.each_char do |c|
          unless c == current_char
            reached_end = false
            break
          end
          next_char
          reached_end = true
        end

        if reached_end &&
           (current_char == '\n' || current_char == '\0' ||
           (current_char == '\r' && peek_next_char == '\n' && next_char))
          @token.type = :DELIMITER_END
          @token.delimiter_state = delimiter_state.with_heredoc_indent(indent)
          return true
        end
      end

      @reader.pos = old_pos
      @column_number = old_column
      @token.column_number = @column_number

      false
    end

    def raise_unterminated_quoted(delimiter_state)
      msg = case delimiter_state.kind
            when :command then "Unterminated command literal"
            when :regex   then "Unterminated regular expression"
            when :heredoc
              "Unterminated heredoc: can't find \"#{delimiter_state.end}\" anywhere before the end of file"
            when :string then "Unterminated string literal"
            else
              ::raise "unreachable"
            end
      raise msg, @line_number, @column_number
    end

    def next_macro_token(macro_state, skip_whitespace)
      reset_token

      nest = macro_state.nest
      control_nest = macro_state.control_nest
      whitespace = macro_state.whitespace
      delimiter_state = macro_state.delimiter_state
      beginning_of_line = macro_state.beginning_of_line
      comment = macro_state.comment
      yields = false

      if skip_whitespace
        skip_macro_whitespace
      end

      @token.location = nil
      @token.line_number = @line_number
      @token.column_number = @column_number

      start = current_pos

      if current_char == '\0'
        @token.type = :EOF
        return @token
      end

      if current_char == '\\' && peek_next_char == '{'
        beginning_of_line = false
        next_char
        start = current_pos
        if next_char == '%'
          while (char = next_char).ascii_whitespace?
          end

          case char
          when 'e'
            if next_char == 'n' && next_char == 'd' && !ident_part_or_end?(peek_next_char)
              next_char
              nest -= 1
            end
          when 'f'
            if next_char == 'o' && next_char == 'r' && !ident_part_or_end?(peek_next_char)
              next_char
              nest += 1
            end
          when 'i'
            if next_char == 'f' && !ident_part_or_end?(peek_next_char)
              next_char
              nest += 1
            end
          else
            # no nesting
          end
        end

        @token.type = :MACRO_LITERAL
        @token.value = string_range(start)
        @token.macro_state = Token::MacroState.new(whitespace, nest, control_nest, delimiter_state, beginning_of_line, yields, comment)
        set_token_raw_from_start(start)
        return @token
      end

      if current_char == '\\' && peek_next_char == '%'
        beginning_of_line = false
        next_char
        next_char
        @token.type = :MACRO_LITERAL
        @token.value = "%"
        @token.macro_state = Token::MacroState.new(whitespace, nest, control_nest, delimiter_state, beginning_of_line, yields, comment)
        @token.raw = "%"
        return @token
      end

      if current_char == '{'
        case next_char
        when '{'
          beginning_of_line = false
          next_char
          @token.type = :MACRO_EXPRESSION_START
          @token.macro_state = Token::MacroState.new(whitespace, nest, control_nest, delimiter_state, beginning_of_line, yields, comment)
          return @token
        when '%'
          beginning_of_line = false
          next_char
          @token.type = :MACRO_CONTROL_START
          @token.macro_state = Token::MacroState.new(whitespace, nest, control_nest, delimiter_state, beginning_of_line, yields, comment)
          return @token
        else
          # Make sure to decrease the '}' count if inside an interpolation
          @macro_curly_count += 1 if @macro_curly_count > 0
        end
      end

      if comment || (!delimiter_state && current_char == '#')
        comment = true
        char = current_char
        char = next_char if current_char == '#'
        while true
          case char
          when '\n'
            comment = false
            beginning_of_line = true
            whitespace = true
            next_char
            incr_line_number
            @token.line_number = @line_number
            @token.column_number = @column_number
            break
          when '{'
            break
          when '\0'
            raise "unterminated macro"
          else
            char = next_char
          end
        end
        @token.type = :MACRO_LITERAL
        @token.value = string_range(start)
        @token.macro_state = Token::MacroState.new(whitespace, nest, control_nest, delimiter_state, beginning_of_line, yields, comment)
        set_token_raw_from_start(start)
        return @token
      end

      if !delimiter_state && current_char == '%' && ident_start?(peek_next_char)
        char = next_char
        if char == 'q' && (peek = peek_next_char) && peek.in?('(', '<', '[', '{', '|')
          next_char
          delimiter_state = Token::DelimiterState.new(:string, char, closing_char, 1)
          next_char
        elsif char == 'r' && (peek = peek_next_char) && peek.in?('(', '<', '[', '{', '|')
          next_char
          delimiter_state = Token::DelimiterState.new(:regex, char, closing_char, 1)
          next_char
        else
          start = current_pos
          while ident_part?(char)
            char = next_char
          end
          beginning_of_line = false
          @token.type = :MACRO_VAR
          @token.value = string_range_from_pool(start)
          @token.macro_state = Token::MacroState.new(whitespace, nest, control_nest, delimiter_state, beginning_of_line, yields, comment)
          return @token
        end
      end

      if !delimiter_state && current_char == 'e'
        if next_char == 'n'
          beginning_of_line = false
          case next_char
          when 'd'
            if whitespace && !ident_part_or_end?(peek_next_char)
              if nest == 0 && control_nest == 0
                next_char
                @token.type = :MACRO_END
                @token.macro_state = Token::MacroState.default
                return @token
              else
                nest -= 1
                whitespace = current_char.ascii_whitespace?
                next_char
              end
            end
          when 'u'
            if !delimiter_state && whitespace && next_char == 'm' && !ident_part_or_end?(next_char)
              char = current_char
              nest += 1
              whitespace = true
            end
          else
            # not a keyword
          end
        else
          whitespace = false
          beginning_of_line = false
        end
      end

      char = current_char

      until char == '{' || char == '\0' || (char == '\\' && ((peek = peek_next_char) == '{' || peek == '%')) || (whitespace && !delimiter_state && char == 'e')
        case char
        when '\n'
          incr_line_number 0
          whitespace = true
          beginning_of_line = true
        when '\\'
          char = next_char
          if delimiter_state
            if char == '"'
              char = next_char
            end
            whitespace = false
          else
            whitespace = false
          end
          next
        when '\'', '"'
          if delimiter_state
            delimiter_state = nil if delimiter_state.end == char
          else
            delimiter_state = Token::DelimiterState.new(:string, char, char)
          end
          whitespace = false
        when '%'
          case char = peek_next_char
          when '(', '[', '<', '{', '|'
            next_char
            delimiter_state = Token::DelimiterState.new(:string, char, closing_char, 1)
          else
            whitespace = false
            break if !delimiter_state && ident_start?(char)
          end
        when '#'
          if delimiter_state
            # If it's "#{...", we don't want "#{{{" to parse it as "# {{ {", but as "#{ {{"
            # (macro expression inside a string interpolation)
            if peek_next_char == '{'
              char = next_char

              # We should now consider things that follow as crystal expressions,
              # so we reset the delimiter state but save it in a stack
              @macro_curly_count += 1
              @delimiter_state_stack.push delimiter_state
              delimiter_state = nil
            end
            whitespace = false
          else
            break
          end
        when '}'
          if delimiter_state && delimiter_state.end == '}'
            delimiter_state = delimiter_state.with_open_count_delta(-1)
            if delimiter_state.open_count == 0
              delimiter_state = nil
            end
          elsif @macro_curly_count > 0
            # Once we find the final '}' that closes the interpolation,
            # we are back inside the delimiter
            if @macro_curly_count == 1
              delimiter_state = @delimiter_state_stack.pop
            end
            @macro_curly_count -= 1
          end
        else
          if !delimiter_state && whitespace && lookahead { char == 'y' && next_char == 'i' && next_char == 'e' && next_char == 'l' && next_char == 'd' && !ident_part_or_end?(peek_next_char) }
            yields = true
            char = current_char
            whitespace = true
            beginning_of_line = false
          elsif !delimiter_state && whitespace && (keyword = lookahead { check_macro_opening_keyword(beginning_of_line) })
            char = current_char

            nest += 1 unless keyword == :abstract_def
            whitespace = true
            beginning_of_line = false
            next
          else
            char = current_char

            if delimiter_state
              case char
              when delimiter_state.nest
                delimiter_state = delimiter_state.with_open_count_delta(+1)
              when delimiter_state.end
                delimiter_state = delimiter_state.with_open_count_delta(-1)
                if delimiter_state.open_count == 0
                  delimiter_state = nil
                end
              else
                # Nothing to do
              end
            end

            # If an assignment comes, we accept if/unless/while/until as nesting
            if char == '=' && peek_next_char.ascii_whitespace?
              whitespace = false
              beginning_of_line = true
            else
              whitespace = char.ascii_whitespace? || char == ';' || char == '(' || char == '[' || char == '{'
              if beginning_of_line && !whitespace
                beginning_of_line = false
              end
            end
          end
        end
        char = next_char
      end

      @token.type = :MACRO_LITERAL
      @token.value = string_range(start)
      @token.macro_state = Token::MacroState.new(whitespace, nest, control_nest, delimiter_state, beginning_of_line, yields, comment)
      set_token_raw_from_start(start)

      @token
    end

    def lookahead
      old_pos = @reader.pos
      old_line_number, old_column_number = @line_number, @column_number

      result = yield
      unless result
        @reader.pos = old_pos
        @line_number, @column_number = old_line_number, old_column_number
      end
      result
    end

    def skip_macro_whitespace
      start = current_pos
      while current_char.ascii_whitespace?
        whitespace = true
        if current_char == '\n'
          incr_line_number 0
          beginning_of_line = true
        end
        next_char
      end
      if @wants_raw
        string_range(start)
      else
        ""
      end
    end

    def check_macro_opening_keyword(beginning_of_line)
      case char = current_char
      when 'a'
        case next_char
        when 'b'
          if next_char == 's' && next_char == 't' && next_char == 'r' && next_char == 'a' && next_char == 'c' && next_char == 't' && next_char.whitespace?
            case next_char
            when 'd'
              next_char == 'e' && next_char == 'f' && peek_not_ident_part_or_end_next_char && :abstract_def
            when 'c'
              next_char == 'l' && next_char == 'a' && next_char == 's' && next_char == 's' && peek_not_ident_part_or_end_next_char && :abstract_class
            when 's'
              next_char == 't' && next_char == 'r' && next_char == 'u' && next_char == 'c' && next_char == 't' && peek_not_ident_part_or_end_next_char && :abstract_struct
            else
              false
            end
          end
        when 'n'
          next_char == 'n' && next_char == 'o' && next_char == 't' && next_char == 'a' && next_char == 't' && next_char == 'i' && next_char == 'o' && next_char == 'n' && peek_not_ident_part_or_end_next_char && :annotation
        else
          false
        end
      when 'b'
        next_char == 'e' && next_char == 'g' && next_char == 'i' && next_char == 'n' && peek_not_ident_part_or_end_next_char && :begin
      when 'c'
        (char = next_char) && (
          (char == 'a' && next_char == 's' && next_char == 'e' && peek_not_ident_part_or_end_next_char && :case) ||
            (char == 'l' && next_char == 'a' && next_char == 's' && next_char == 's' && peek_not_ident_part_or_end_next_char && :class)
        )
      when 'd'
        (char = next_char) &&
          ((char == 'o' && peek_not_ident_part_or_end_next_char && :do) ||
            (char == 'e' && next_char == 'f' && peek_not_ident_part_or_end_next_char && :def))
      when 'f'
        next_char == 'u' && next_char == 'n' && peek_not_ident_part_or_end_next_char && :fun
      when 'i'
        beginning_of_line && next_char == 'f' &&
          (char = next_char) && (!ident_part_or_end?(char) && :if)
      when 'l'
        next_char == 'i' && next_char == 'b' && peek_not_ident_part_or_end_next_char && :lib
      when 'm'
        (char = next_char) && (
          (char == 'a' && next_char == 'c' && next_char == 'r' && next_char == 'o' && peek_not_ident_part_or_end_next_char && :macro) ||
            (char == 'o' && next_char == 'd' && next_char == 'u' && next_char == 'l' && next_char == 'e' && peek_not_ident_part_or_end_next_char && :module)
        )
      when 's'
        next_char == 't' && next_char == 'r' && next_char == 'u' && next_char == 'c' && next_char == 't' && !ident_part_or_end?(peek_next_char) && next_char && :struct
      when 'u'
        next_char == 'n' && (char = next_char) && (
          (char == 'i' && next_char == 'o' && next_char == 'n' && peek_not_ident_part_or_end_next_char && :union) ||
            (beginning_of_line && char == 'l' && next_char == 'e' && next_char == 's' && next_char == 's' && peek_not_ident_part_or_end_next_char && :unless) ||
            (beginning_of_line && char == 't' && next_char == 'i' && next_char == 'l' && peek_not_ident_part_or_end_next_char && :until)
        )
      when 'w'
        beginning_of_line && next_char == 'h' && next_char == 'i' && next_char == 'l' && next_char == 'e' && peek_not_ident_part_or_end_next_char && :while
      else
        false
      end
    end

    def consume_octal_escape(char)
      value = char - '0'
      count = 1
      while count <= 3 && '0' <= peek_next_char < '8'
        next_char
        value = value * 8 + (current_char - '0')
        count += 1
      end
      if value >= 256
        raise "octal value too big"
      end
      value.to_u8
    end

    def consume_char_unicode_escape
      char = peek_next_char
      if char == '{'
        next_char
        consume_braced_unicode_escape
      else
        consume_non_braced_unicode_escape
      end
    end

    def consume_string_hex_escape
      char = next_char
      high = char.to_i?(16)
      raise "invalid hex escape" unless high

      char = next_char
      low = char.to_i?(16)
      raise "invalid hex escape" unless low

      ((high << 4) | low).to_u8
    end

    def consume_string_unicode_escape
      char = peek_next_char
      if char == '{'
        next_char
        consume_string_unicode_brace_escape
      else
        consume_non_braced_unicode_escape.chr.to_s
      end
    end

    def consume_string_unicode_brace_escape
      String.build do |str|
        while true
          str << consume_braced_unicode_escape(allow_spaces: true).chr
          break unless current_char == ' '
        end
      end
    end

    def consume_non_braced_unicode_escape
      codepoint = 0
      4.times do
        hex_value = char_to_hex(next_char) { expected_hexacimal_character_in_unicode_escape }
        codepoint = 16 * codepoint + hex_value
      end
      codepoint
    end

    def consume_braced_unicode_escape(allow_spaces = false)
      codepoint = 0
      found_curly = false
      found_space = false
      found_digit = false
      char = '\0'

      6.times do
        char = next_char
        case char
        when '}'
          found_curly = true
          break
        when ' '
          if allow_spaces
            found_space = true
            break
          else
            expected_hexacimal_character_in_unicode_escape
          end
        else
          hex_value = char_to_hex(char) { expected_hexacimal_character_in_unicode_escape }
          codepoint = 16 * codepoint + hex_value
          found_digit = true
        end
      end

      if !found_digit
        expected_hexacimal_character_in_unicode_escape
      elsif codepoint > 0x10FFFF
        raise "invalid unicode codepoint (too large)"
      end

      unless found_space
        unless found_curly
          char = next_char
        end

        unless char == '}'
          raise "expected '}' to close unicode escape"
        end
      end

      codepoint
    end

    def expected_hexacimal_character_in_unicode_escape
      raise "expected hexadecimal character in unicode escape"
    end

    def string_token_escape_value(value)
      next_char
      @token.type = :STRING
      @token.value = value
    end

    def delimited_pair(kind, string_nest, string_end, start, allow_escapes = true, advance = true)
      next_char if advance
      @token.type = :DELIMITER_START
      @token.delimiter_state = Token::DelimiterState.new(kind, string_nest, string_end, allow_escapes)
      set_token_raw_from_start(start)
    end

    def next_string_array_token
      reset_token

      while true
        if current_char == '\n'
          next_char
          incr_line_number 1
        elsif current_char.ascii_whitespace?
          next_char
        else
          break
        end
      end

      if current_char == @token.delimiter_state.end
        @token.raw = current_char.to_s if @wants_raw
        next_char
        @token.type = :STRING_ARRAY_END
        return @token
      end

      start = current_pos
      sub_start = start
      value = String::Builder.new

      escaped = false
      while true
        case current_char
        when Char::ZERO
          break # raise is handled by parser
        when @token.delimiter_state.end
          unless escaped
            if @token.delimiter_state.open_count == 0
              break
            else
              @token.delimiter_state = @token.delimiter_state.with_open_count_delta(-1)
            end
          end
        when @token.delimiter_state.nest
          unless escaped
            @token.delimiter_state = @token.delimiter_state.with_open_count_delta(+1)
          end
        when .ascii_whitespace?
          break unless escaped
        else
          if escaped
            value << '\\'
          end
        end

        escaped = current_char == '\\'
        if escaped
          value.write @reader.string.to_slice[sub_start, current_pos - sub_start]
          sub_start = current_pos + 1
        end

        next_char
      end

      if start == current_pos
        @token.type = :EOF
        return @token
      end

      value.write @reader.string.to_slice[sub_start, current_pos - sub_start]

      @token.type = :STRING
      @token.value = value.to_s
      set_token_raw_from_start(start)

      @token
    end

    def char_to_hex(char)
      if '0' <= char <= '9'
        char - '0'
      elsif 'a' <= char <= 'f'
        10 + (char - 'a')
      elsif 'A' <= char <= 'F'
        10 + (char - 'A')
      else
        yield
      end
    end

    def consume_loc_pragma
      case current_char
      when '"'
        # skip '"'
        next_char_no_column_increment

        filename_pos = current_pos

        while true
          case current_char
          when '"'
            break
          when '\0'
            raise "unexpected end of file in loc pragma"
          else
            next_char_no_column_increment
          end
        end

        incr_column_number (current_pos - filename_pos) + 7 # == "#<loc:\"".size
        filename = string_range(filename_pos)

        # skip '"'
        next_char

        unless current_char == ','
          raise "expected ',' in loc pragma after filename"
        end
        next_char

        line_number = 0
        while true
          case current_char
          when '0'..'9'
            line_number = 10 * line_number + (current_char - '0').to_i
          when ','
            next_char
            break
          else
            raise "expected digit or ',' in loc pragma for line number"
          end
          next_char
        end

        column_number = 0
        while true
          case current_char
          when '0'..'9'
            column_number = 10 * column_number + (current_char - '0').to_i
          when '>'
            next_char
            break
          else
            raise "expected digit or '>' in loc pragma for column_number number"
          end
          next_char
        end

        set_location filename, line_number, column_number
      when 'p'
        # skip 'p'
        next_char_no_column_increment

        case current_char
        when 'o'
          unless next_char_no_column_increment == 'p' &&
                 next_char_no_column_increment == '>'
            raise %(expected #<loc:push>, #<loc:pop> or #<loc:"...">)
          end

          # skip '>'
          next_char_no_column_increment

          incr_column_number 10 # == "#<loc:pop>".size

          pop_location
        when 'u'
          unless next_char_no_column_increment == 's' &&
                 next_char_no_column_increment == 'h' &&
                 next_char_no_column_increment == '>'
            raise %(expected #<loc:push>, #<loc:pop> or #<loc:"...">)
          end

          # skip '>'
          next_char_no_column_increment

          incr_column_number 11 # == "#<loc:push>".size

          @token.line_number = @line_number
          @token.column_number = @column_number
          push_location
        else
          raise %(expected #<loc:push>, #<loc:pop> or #<loc:"...">)
        end
      else
        raise %(expected #<loc:push>, #<loc:pop> or #<loc:"...">)
      end
    end

    def set_location(filename, line_number, column_number)
      @token.filename = @filename = filename
      @token.line_number = @line_number = line_number
      @token.column_number = @column_number = column_number
    end

    def pop_location
      if @stacked
        @stacked = false
        @token.filename = @filename = @stacked_filename
        @token.line_number = @line_number = @stacked_line_number
        @token.column_number = @column_number = @stacked_column_number
      end
    end

    def push_location
      unless @stacked
        @stacked = true
        @stacked_filename, @stacked_line_number, @stacked_column_number = @filename, @line_number, @column_number
      end
    end

    def incr_column_number(d = 1)
      @column_number += d
      @stacked_column_number += d if @stacked
    end

    def incr_line_number(column_number = 1)
      @line_number += 1
      @column_number = column_number if column_number
      if @stacked
        @stacked_line_number += 1
        @stacked_column_number = column_number if column_number
      end
    end

    def next_char_no_column_increment
      char = @reader.next_char
      if error = @reader.error
        ::raise InvalidByteSequenceError.new("Unexpected byte 0x#{error.to_s(16)} at position #{@reader.pos}, malformed UTF-8")
      end
      char
    end

    def next_char
      incr_column_number
      next_char_no_column_increment
    end

    def next_char_check_line
      char = next_char_no_column_increment
      if char == '\n'
        incr_line_number
      else
        incr_column_number = 1
      end
      char
    end

    def next_char(token_type)
      next_char
      @token.type = token_type
    end

    def reset_token
      @token.value = nil
      @token.line_number = @line_number
      @token.column_number = @column_number
      @token.filename = @filename
      @token.location = nil
      @token.passed_backslash_newline = false
      @token.doc_buffer = nil unless @token.type == :SPACE || @token.type == :NEWLINE
      @token.invalid_escape = false
      @token_end_location = nil
    end

    def next_token_skip_space
      next_token
      skip_space
    end

    def next_token_skip_space_or_newline
      next_token
      skip_space_or_newline
    end

    def next_token_skip_statement_end
      next_token
      skip_statement_end
    end

    def current_char
      @reader.current_char
    end

    def peek_next_char
      @reader.peek_next_char
    end

    def current_pos
      @reader.pos
    end

    def current_pos=(pos)
      @reader.pos = pos
    end

    def string
      @reader.string
    end

    def string_range(start_pos)
      string_range(start_pos, current_pos)
    end

    def string_range(start_pos, end_pos)
      @reader.string.byte_slice(start_pos, end_pos - start_pos)
    end

    def string_range_from_pool(start_pos)
      string_range_from_pool(start_pos, current_pos)
    end

    def string_range_from_pool(start_pos, end_pos)
      @string_pool.get slice_range(start_pos, end_pos)
    end

    def slice_range(start_pos)
      slice_range(start_pos, current_pos)
    end

    def slice_range(start_pos, end_pos)
      Slice.new(@reader.string.to_unsafe + start_pos, end_pos - start_pos)
    end

    def ident_start?(char)
      char.ascii_letter? || char == '_' || char.ord > 0x9F
    end

    def ident_part?(char)
      ident_start?(char) || char.ascii_number?
    end

    def ident_part_or_end?(char)
      ident_part?(char) || char == '?' || char == '!'
    end

    def peek_not_ident_part_or_end_next_char
      !ident_part_or_end?(peek_next_char) && next_char
    end

    def closing_char(char = current_char)
      case char
      when '<' then '>'
      when '(' then ')'
      when '[' then ']'
      when '{' then '}'
      else          char
      end
    end

    def skip_space
      while @token.type == :SPACE
        next_token
      end
    end

    def skip_space_or_newline
      while (@token.type == :SPACE || @token.type == :NEWLINE)
        next_token
      end
    end

    def skip_statement_end
      while (@token.type == :SPACE || @token.type == :NEWLINE || @token.type == :";")
        next_token
      end
    end

    def unknown_token
      raise "unknown token: #{current_char.inspect}", @line_number, @column_number
    end

    def set_token_raw_from_start(start)
      @token.raw = string_range(start) if @wants_raw
    end

    def raise(message, line_number = @line_number, column_number = @column_number, filename = @filename)
      ::raise Crystal::SyntaxException.new(message, line_number, column_number, filename)
    end

    def raise(message, token : Token, size = nil)
      ::raise Crystal::SyntaxException.new(message, token.line_number, token.column_number, token.filename, size)
    end

    def raise(message, location : Location)
      raise message, location.line_number, location.column_number, location.filename
    end
  end
end
