Parser now fully processes tokens

lib/schemer/ast.rb

@@ -24,6 +24,18 @@ def inspect
         "#<Identifier::#{@value}>"
       end
     end
+
+    class QuotedIdentifier
+      attr_reader :value
+
+      def initialize(identifier)
+        @value = identifier
+      end
+
+      def inspect
+        "#<QuotedIdentifier::#{@value}>"
+      end
+    end
 
     class Expression
       attr_reader :proc, :args
@@ -38,6 +50,42 @@ def inspect
       end
     end
 
+    class List
+      attr_reader :elements
+
+      def initialize(elements)
+        @elements = elements
+      end
+
+      def inspect
+        "#<List @elements=#{@elements}>"
+      end
+    end
+
+    class QuotedList
+      attr_reader :elements
+
+      def initialize(elements)
+        @elements = elements
+      end
+
+      def inspect
+        "#<QuotedList @elements=#{@elements}>"
+      end
+    end
+
+    class Vector
+      attr_reader :elements
+
+      def initialize(elements)
+        @elements = elements
+      end
+
+      def inspect
+        "#<Vector @elements=#{@elements}>"
+      end
+    end
+
     class AddOperator
       def inspect
         "#<Operator::Add>"

lib/schemer/lexer.rb

@@ -33,25 +33,26 @@ class Lexer < Parslet::Parser
     rule(:boolean)    { `#` >> (`t` | `f`).as(:boolean) }
     rule(:literal)    { numeric | string | character | boolean }
 
-    rule(:quote)      { `'` >> list.as(:quoted_list) }
-    rule(:vector)     { `#` >> list.as(:vector) }
-    rule(:list)       { lparen >> args >> rparen }
-    rule(:pair)       { lparen >> arg >> space >> dot >> space >> args >> rparen }
+    rule(:list)       { lparen >> args.as(:list) >> rparen }
+    rule(:vector)     { `#` >> lparen >> args.as(:vector) >> rparen }
+    rule(:quoted_list){ `'` >> lparen >> args.as(:quoted_list) >> rparen }
+
+    rule(:pair)       { lparen >> (arg >> space >> dot >> space >> args).as(:pair) >> rparen }
 
     rule(:symbol)     { letter >> (letter | integer | special_symbol).repeat(0) }
     rule(:quoted_symbol) { `'` >> symbol.as(:quoted_identifier) }
 
     rule(:operator)   { [`+`, `-`, `*`, `/`, `>=`, `<=`, `>`, `<`, `=`].inject(:|) }
 
 
-    rule(:arg)        { (symbol.as(:identifier) | quote | literal | quoted_symbol | expression | pair | vector | list) }
-    rule(:args)       { (arg >> space?).repeat.as(:args) }
+    rule(:arg)        { (symbol.as(:identifier) | quoted_list | literal | quoted_symbol | expression | pair | vector | list) }
+    rule(:args)       { (arg >> space?).repeat }
 
     rule(:newline)    { str("\n") }
     rule(:comment)    { `;`.repeat(1,3) >> (`\n`.absnt? >> any).repeat.as(:comment) }
-    rule(:expression) { (lparen >> (symbol.as(:identifier) | operator.as(:operator) | expression).as(:proc) >> (space? >> args).maybe >> rparen).as(:expression) }
+    rule(:expression) { (lparen >> (symbol.as(:identifier) | operator.as(:operator) | expression).as(:proc) >> (space? >> args.as(:args)).maybe >> rparen).as(:expression) }
 
-    rule(:body)       { (quote | expression | comment | space).repeat(0) }
+    rule(:body)       { (expression | comment | space).repeat(0) }
     root :body
 
   end

lib/schemer/parser.rb

@@ -11,6 +11,12 @@ class Parser < Parslet::Transform
     rule(:boolean => simple(:boolean))    { boolean == 't' }
 
     rule(:identifier => simple(:identifier))    { AST::Identifier.new(identifier) }
+    rule(:quoted_identifier => simple(:quoted_identifier))    { AST::QuotedIdentifier.new(quoted_identifier) }
+
+    rule(:list => subtree(:list))    { AST::List.new(list) }
+    rule(:quoted_list => subtree(:quoted_list))    { AST::QuotedList.new(quoted_list) }
+    rule(:vector => subtree(:vector))    { AST::Vector.new(vector) }
+    rule(:pair => subtree(:pair))    { AST::List.new(pair) }
 
     rule(:operator => simple(:operator)) do
       case operator

spec/schemer/lexer_spec.rb

@@ -36,15 +36,15 @@ module Schemer
     its(:literal) { should parse('#\z').as(:char => 'z') }
     its(:literal) { should parse('#t').as(:boolean => 't') }
 
-    its(:quote) { should parse("'(1 2 3)") }
-    its(:quote) { should parse("'()") }
+    its(:quoted_list) { should parse("'(1 2 3)") }
+    its(:quoted_list) { should parse("'()") }
     its(:vector) { should parse("#(1 '(2 4) 3)") }
 
     its(:pair) { should parse('(1 . 2)') }
     its(:pair) { should parse('(1 . (2 3))') }
 
-    its(:list) { should parse('(1 2 3)') }
-    its(:list) { should parse('()') }
+    its(:list) { should parse('(1 2 3)').as(:list => [{:integer => '1'}, {:integer => '2'}, {:integer => '3'}]) }
+    its(:list) { should parse('()').as(:list => []) }
     its(:list) { should parse('( )') }
 
     its(:expression) { should parse('(define some "string" #t)') }
@@ -56,8 +56,8 @@ module Schemer
     its(:expression) { should parse('(lambda (define zara \'zara) (write (eqv? zara \'zara)))') }
     its(:expression) { should(parse("(lambda (define (make-new-set?) '()) (define (make-new-set?) '(2 3)))").as do |output|
       output[:expression][:args].should have(2).expressions
-      output[:expression][:args].first[:expression][:args].should include(:quoted_list => {:args => []})
-      output[:expression][:args].last[:expression][:args].should include(:quoted_list => {:args => [{:integer =>"2"}, {:integer => "3"}]})
+      output[:expression][:args].first[:expression][:args].should include(:quoted_list => [])
+      output[:expression][:args].last[:expression][:args].should include(:quoted_list => [{:integer =>"2"}, {:integer => "3"}])
     end) }
 
     # Regression tests

spec/schemer/parser_spec.rb

@@ -47,10 +47,20 @@ module Schemer
     end
 
     describe "Identifiers" do
-      it "are transformed into Identifiers" do
-        text = "(proc my_identifier)"
-        parsed = subject.apply(lexer.parse text).first.args
-        parsed.first.should be_an(AST::Identifier)
+      describe "Regular identifiers" do
+        it "are transformed into Identifiers" do
+          text = "(proc my_identifier)"
+          parsed = subject.apply(lexer.parse text).first.args
+          parsed.first.should be_an(AST::Identifier)
+        end
+      end
+
+      describe "Quoted identifiers" do
+        it "are transformed into QuotedIdentifiers" do
+          text = "(proc 'quoted_identifier)"
+          parsed = subject.apply(lexer.parse text).first.args
+          parsed.first.should be_an(AST::QuotedIdentifier)
+        end
       end
     end
 
@@ -82,5 +92,42 @@ module Schemer
       end
     end
 
+    describe "Lists" do
+      describe "Regular lists" do
+        it 'are transformed into Lists' do
+          text = "(lambda (1 2 3))"
+          parsed = subject.apply(lexer.parse text).first.args.first
+          parsed.should be_an(AST::List)
+          parsed.elements.should == [1, 2, 3]
+        end
+      end
+      describe "Quoted lists" do
+        it 'are transformed into QuotedLists' do
+          text = "(lambda '(1 2 3))"
+          parsed = subject.apply(lexer.parse text).first.args.first
+          parsed.should be_an(AST::QuotedList)
+          parsed.elements.should == [1, 2, 3]
+        end
+      end
+      describe "Vectors" do
+        it 'are transformed into Vectors' do
+          text = "(lambda #(1 2 3))"
+          parsed = subject.apply(lexer.parse text).first.args.first
+          parsed.should be_an(AST::Vector)
+          parsed.elements.should == [1, 2, 3]
+        end
+      end
+      describe "Pairs" do
+        it 'are combined into Lists' do
+          text = "(lambda (1 . (2 3)))"
+          parsed = subject.apply(lexer.parse text).first.args.first
+          parsed.should be_an(AST::List)
+          parsed.elements.first.should == 1
+          parsed.elements.last.should be_an(AST::List)
+          parsed.elements.last.elements.should == [2, 3]
+        end
+      end
+    end
+
   end
 end