Permalink
Browse files

A bit of lexer cleanup, WIP

  • Loading branch information...
1 parent ce08fca commit 69d326ecef1f5b23354e33155b0e50750d4e7e20 @txus committed Mar 9, 2011
Showing with 39 additions and 20 deletions.
  1. +19 −0 benchmark/lexer_benchmark.rb
  2. +5 −0 benchmark/tests
  3. +3 −0 lib/schemer.rb
  4. +10 −17 lib/schemer/lexer.rb
  5. +2 −3 spec/schemer/lexer_spec.rb
@@ -0,0 +1,19 @@
+$: << 'lib'
+
+require 'bundler'
+Bundler.setup
+
+require 'schemer'
+require 'benchmark'
+
+lexer = Schemer::Lexer.new
+
+n = 100
+
+text = File.read(File.expand_path('../examples/bintree.scm'))
+
+Benchmark.bm do |x|
+ x.report do
+ n.times { lexer.parse text }
+ end
+end
View
@@ -0,0 +1,5 @@
+lexer_benchmark.rb
+
+2011-03-09
+ user system total real
+ 9.140000 0.070000 9.210000 ( 9.206295)
View
@@ -1,10 +1,13 @@
require 'schemer/core_ext/fixnum'
+require 'parslet'
+
require 'schemer/lexer'
require 'schemer/ast'
require 'schemer/parser'
require 'schemer/environment'
require 'schemer/interpreter'
+
module Schemer
end
View
@@ -8,26 +8,18 @@ class Lexer < Parslet::Parser
rule(:lparen) { `(` >> space? }
rule(:rparen) { space? >> `)` }
- rule(:text) { any.repeat }
-
- rule :single_quote_string do
- `'` >> (`''` | `'`.absnt? >> any).repeat.as(:string) >> `'`
- end
-
- rule :double_quote_string do
+ rule :string do
`"` >> (`""` | `"`.absnt? >> any).repeat.as(:string) >> `"`
end
- rule(:string) { double_quote_string }
-
rule(:letter) { match('[a-zA-Z]') }
- rule(:dot) { `.` }
- rule(:special_symbol) { `_` | `-` | `?` | `!` | `*` }
+ rule(:dot) { `.` }
+ rule(:special_symbol) { `_` | `-` | `?` | `!` | `+` | `-` | `*` | `/` | `>=` | `<=` | `>` | `<` | `=` }
rule(:integer) { match('\d').repeat(1) }
rule(:float) { integer.repeat(1) >> dot >> integer.repeat(1) }
- rule(:numeric) { float.as(:float) | integer.as(:integer) }
+ rule(:numeric) { `-`.maybe >> (float.as(:float) | integer.as(:integer)) }
rule(:character) { `#\\` >> letter.as(:char) }
rule(:boolean) { `#` >> (`t` | `f`).as(:boolean) }
@@ -42,16 +34,17 @@ class Lexer < Parslet::Parser
rule(:symbol) { letter >> (letter | integer | special_symbol).repeat(0) }
rule(:quoted_symbol) { `'` >> symbol.as(:quoted_identifier) }
- rule(:operator) { [`+`, `-`, `*`, `/`, `>=`, `<=`, `>`, `<`, `=`].inject(:|) }
+ # rule(:operator) { [`+`, `-`, `*`, `/`, `>=`, `<=`, `>`, `<`, `=`].inject(:|) }
+ rule(:operator) { `+` | `-` | `*` | `/` | `>=` | `<=` | `>` | `<` | `=` }
+ # rule(:operator) { match("[\\+-\\*\\/<>=]") }
- rule(:arg) { (symbol.as(:identifier) | quoted_list | literal | quoted_symbol | expression | pair | vector | list) }
+ rule(:arg) { (symbol.as(:identifier) | quoted_list | literal | quoted_symbol | procedure | pair | vector | list) }
rule(:args) { (arg >> space?).repeat }
- rule(:newline) { str("\n") }
rule(:comment) { `;`.repeat(1,3) >> (`\n`.absnt? >> any).repeat.as(:comment) }
- rule(:expression) { (lparen >> (symbol.as(:identifier) | operator.as(:identifier) | expression).as(:proc) >> (space? >> args.as(:args)).maybe >> rparen).as(:expression) }
+ rule(:procedure) { (lparen >> (symbol.as(:identifier) | operator.as(:identifier) | procedure).as(:proc) >> (space? >> args.as(:args)).maybe >> rparen).as(:procedure) }
- rule(:body) { (expression | comment | space).repeat(0) }
+ rule(:body) { (procedure | space | comment).repeat(0) }
root :body
end
@@ -8,9 +8,6 @@ module Schemer
its(:rparen) { should parse(' )') }
its(:space) { should parse(" \n") }
- its(:text) { should parse(' this should be treated as text!! oh yeah I think it should ') }
- its(:text) { should parse('and without trimmable whitespace!!') }
-
its(:symbol) { should parse('some_symbol') }
its(:symbol) { should parse('s423-ome_symbol') }
its(:symbol) { should parse('s423-ome_symbol?') }
@@ -28,7 +25,9 @@ module Schemer
its(:comment) { should parse(';;; some comment!! "whoo"') }
its(:integer) { should parse('123') }
+ its(:integer) { should parse('-123') }
its(:float) { should parse('99.9') }
+ its(:float) { should parse('-99.9') }
its(:literal) { should parse('123').as(:integer => '123') }
its(:literal) { should parse('99.9').as(:float => '99.9') }

0 comments on commit 69d326e

Please sign in to comment.