scanner.rb

require 'strscan'

module Journey
  class Scanner
    def initialize
      @ss = nil
    end

    def scan_setup str
      @ss = StringScanner.new str
    end

    def eos?
      @ss.eos?
    end

    def pos
      @ss.pos
    end

    def pre_match
      @ss.pre_match
    end

    def next_token
      return if @ss.eos?

      until token = scan || @ss.eos?; end
      token
    end

    private
    def scan
      case
        # /
      when text = @ss.scan(/\//)
        [:SLASH, text]
      when text = @ss.scan(/\*/)
        [:STAR, text]
      when text = @ss.scan(/\(/)
        [:LPAREN, text]
      when text = @ss.scan(/\)/)
        [:RPAREN, text]
      when text = @ss.scan(/\|/)
        [:OR, text]
      when text = @ss.scan(/\./)
        [:DOT, text]
      when text = @ss.scan(/:\w+/)
        [:SYMBOL, text]
      when text = @ss.scan(/[\w-]+/)
        [:LITERAL, text]
        # any char
      when text = @ss.scan(/./)
        [:LITERAL, text]
      end
    end
  end
end