Permalink
Browse files

Gherkin lexer with rexical

  • Loading branch information...
1 parent 5a5aee4 commit e48635c6c213f9bbd1285d331a8fe90d360fddc4 @txus txus committed Nov 30, 2011
Showing with 209 additions and 2 deletions.
  1. +1 −1 Rakefile
  2. +1 −1 lib/gherkin/parser.rb
  3. +44 −0 lib/gherkin/parser/gherkin.rex
  4. +120 −0 lib/gherkin/parser/lexer.rb
  5. +43 −0 test/gherkin/lexer_test.rb
View
@@ -15,7 +15,7 @@ task :regenerate do
if has_rex && has_racc
`rex lib/gherkin/parser/gherkin.rex -o lib/gherkin/parser/lexer.rb`
- `racc lib/gherkin/parser/gherkin.y -o lib/gherkin/parser/parser.rb`
+ # `racc lib/gherkin/parser/gherkin.y -o lib/gherkin/parser/parser.rb`
else
puts "You need both Rexical and Racc to do that. Install them by doing:"
puts
View
@@ -1,6 +1,6 @@
# require 'parslet'
require_relative 'parser/lexer'
-require_relative 'parser/parser'
+# require_relative 'parser/parser'
# module Gherkin
# class Parser < Parslet::Parser
@@ -0,0 +1,44 @@
+# Compile with: rex gherkin.rex -o lexer.rb
+
+class Gherkin::Lexer
+
+macro
+ BLANK [\ \t]+
+
+rule
+ # Whitespace
+ {BLANK} # no action
+ \#.*$
+
+ # Literals
+ \n+ { [:NEWLINE, text] }
+
+ # Keywords
+ Feature: { [:FEATURE, text[0..-2]] }
+ Background: { [:BACKGROUND, text[0..-2]] }
+ Scenario: { [:SCENARIO, text[0..-2]] }
+
+ # Tags
+ @\w+ { [:TAG, text[1..-1]] }
+
+ # Step keywords
+ Given { [:GIVEN, text] }
+ When { [:WHEN, text] }
+ Then { [:THEN, text] }
+ And { [:AND, text] }
+ But { [:BUT, text] }
+
+ # Text
+ [^#]*$ { [:TEXT, text.strip] }
+
+inner
+ def run(code)
+ scan_setup(code)
+ tokens = []
+ while token = next_token
+ tokens << token
+ end
+ tokens
+ end
+
+end
View
@@ -0,0 +1,120 @@
+#--
+# DO NOT MODIFY!!!!
+# This file is automatically generated by rex 1.0.5
+# from lexical definition file "lib/gherkin/parser/gherkin.rex".
+#++
+
+require 'racc/parser'
+# Compile with: rex gherkin.rex -o lexer.rb
+
+class Gherkin::Lexer < Racc::Parser
+ require 'strscan'
+
+ class ScanError < StandardError ; end
+
+ attr_reader :lineno
+ attr_reader :filename
+ attr_accessor :state
+
+ def scan_setup(str)
+ @ss = StringScanner.new(str)
+ @lineno = 1
+ @state = nil
+ end
+
+ def action
+ yield
+ end
+
+ def scan_str(str)
+ scan_setup(str)
+ do_parse
+ end
+ alias :scan :scan_str
+
+ def load_file( filename )
+ @filename = filename
+ open(filename, "r") do |f|
+ scan_setup(f.read)
+ end
+ end
+
+ def scan_file( filename )
+ load_file(filename)
+ do_parse
+ end
+
+
+ def next_token
+ return if @ss.eos?
+
+ # skips empty actions
+ until token = _next_token or @ss.eos?; end
+ token
+ end
+
+ def _next_token
+ text = @ss.peek(1)
+ @lineno += 1 if text == "\n"
+ token = case @state
+ when nil
+ case
+ when (text = @ss.scan(/[ \t]+/))
+ ;
+
+ when (text = @ss.scan(/\#.*$/))
+ ;
+
+ when (text = @ss.scan(/\n+/))
+ action { [:NEWLINE, text] }
+
+ when (text = @ss.scan(/Feature:/))
+ action { [:FEATURE, text[0..-2]] }
+
+ when (text = @ss.scan(/Background:/))
+ action { [:BACKGROUND, text[0..-2]] }
+
+ when (text = @ss.scan(/Scenario:/))
+ action { [:SCENARIO, text[0..-2]] }
+
+ when (text = @ss.scan(/@\w+/))
+ action { [:TAG, text[1..-1]] }
+
+ when (text = @ss.scan(/Given/))
+ action { [:GIVEN, text] }
+
+ when (text = @ss.scan(/When/))
+ action { [:WHEN, text] }
+
+ when (text = @ss.scan(/Then/))
+ action { [:THEN, text] }
+
+ when (text = @ss.scan(/And/))
+ action { [:AND, text] }
+
+ when (text = @ss.scan(/But/))
+ action { [:BUT, text] }
+
+ when (text = @ss.scan(/[^#]*$/))
+ action { [:TEXT, text.strip] }
+
+ else
+ text = @ss.string[@ss.pos .. -1]
+ raise ScanError, "can not match: '" + text + "'"
+ end # if
+
+ else
+ raise ScanError, "undefined state: '" + state.to_s + "'"
+ end # case state
+ token
+ end # def _next_token
+
+ def run(code)
+ scan_setup(code)
+ tokens = []
+ while token = next_token
+ tokens << token
+ end
+ tokens
+ end
+end # class
View
@@ -0,0 +1,43 @@
+require 'test_helper'
+
+module Gherkin
+ describe Lexer do
+ let(:lexer) { Lexer.new }
+
+ it 'ignores commments' do
+ lexer.run("# this is a comment").must_equal []
+ end
+
+ it 'parses newlines' do
+ lexer.run("\n\n").must_equal [[:NEWLINE, "\n\n"]]
+ end
+
+ it 'parses text and strips it' do
+ lexer.run(" Arbitrary text ").must_equal [[:TEXT, "Arbitrary text"]]
+ end
+
+ it 'parses tags' do
+ lexer.run("@javascript @wip").must_equal [
+ [:TAG, "javascript"],
+ [:TAG, "wip"],
+ ]
+ end
+
+ describe 'Keywords' do
+ %w(Feature: Background: Scenario:).each do |keyword|
+ it "parses #{keyword}:" do
+ name = keyword[0..-2]
+ lexer.run(keyword).must_equal [[name.upcase.to_sym, name]]
+ end
+ end
+ end
+
+ describe 'Step keywords' do
+ %w(Given When Then And But).each do |keyword|
+ it "parses #{keyword}" do
+ lexer.run(keyword).must_equal [[keyword.upcase.to_sym, keyword]]
+ end
+ end
+ end
+ end
+end

0 comments on commit e48635c

Please sign in to comment.