vidarh / writing-a-compiler-in-ruby

Code from my series on writing a Ruby compiler in Ruby

This URL has Read+Write access

Changed method naming in the asm output; added a comment re: self
vidarh (author)
Sat Feb 28 10:30:09 -0800 2009
commit  cbefdad73ef9d0d430b03c939ea921e24a4fee54
tree    c6c68bf7cfd57e8e0f1740e53ce6213644f9d3d4
parent  386f2801b1921bb6c71a743343d60674bb32d7c7
writing-a-compiler-in-ruby / tokens.rb
100644 100 lines (86 sloc) 1.996 kb
raw blame history 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

require 'operators'
 
module Tokens
 
  class Atom
    def self.expect s
      tmp = ""
      if (c = s.peek) && (c == ?_ || (?a .. ?z).member?(c) || (?A .. ?Z).member?(c))
        tmp += s.get
        
        while (c = s.peek) && ((?a .. ?z).member?(c) || 
                                  (?A .. ?Z).member?(c) || 
                                  (?0 .. ?9).member?(c) || ?_ == c)
          tmp += s.get
        end
      end
      return nil if tmp == ""
      return tmp.to_sym
    end
  end
 
  class Int
    def self.expect s
      tmp = ""
      tmp += s.get if s.peek == ?-
      while (c = s.peek) && (?0 .. ?9).member?(c)
        tmp += s.get
      end
      return nil if tmp == ""
      tmp.to_i
    end
  end
 
  class Quoted
    def self.escaped(s)
      return nil if s.peek == ?"
      if s.expect("\\")
        raised "Unexpected EOF" if !s.peek
        return "\\"+s.get 
      end
      return s.get
    end
 
 
    def self.expect s
      return nil if !s.expect('"')
      buf = ""
      while (e = escaped(s)); buf += e; end
      raise "Unterminated string" if !s.expect('"')
      return buf
    end
  end
 
  class Tokenizer
    def initialize scanner
      @s = scanner
    end
 
    def each
      while t = get
        yield t
      end
    end
 
    def get
      @s.nolfws
      case @s.peek
      when ?"
        return @s.expect(Quoted)
      when ?0 .. ?9
        return @s.expect(Int)
      when ?a .. ?z , ?A .. ?Z
        buf = @s.expect(Atom)
        if (buf == :end || buf == :def) # FIXME: Make this a keyword lookup
          @s.unget(buf.to_s)
          return nil
        end
        return buf
      # Special cases - two character operators:
      when ?= 
        @s.get
        return "==" if @s.peek == ?=
        return "="
      when ?!
        @s.get
        return "!=" if @s.peek == ?=
        return "!"
      when nil
        return nil
      else
        return @s.get if Operators[@s.peek.chr]
        return nil
      end
    end
  end
end