Browse files

Added support for arbitrary directives to the kpeg grammar.

Added support for a "header" directive to the kpeg code formatter.  The header
directive is placed in the output above all other output.

Switched to minitest from test/unit to support warnings emitted by the kpeg
grammar.

Bugs in the kpeg code generator no longer overwrite files.
  • Loading branch information...
1 parent 8829c66 commit bf8b1c6485aac00390ca9f0759dabae78389676c @drbrain drbrain committed Mar 6, 2012
View
6 bin/kpeg
@@ -119,8 +119,10 @@ end
cg = KPeg::CodeGenerator.new name, grammar
cg.standalone = options[:standalone]
-File.open new_path, "w" do |f|
- f << cg.output
+output = cg.output
+
+open new_path, "w" do |io|
+ io << output
end
puts "Wrote #{name} to #{new_path}"
View
2 kpeg.gemspec
@@ -19,6 +19,6 @@ Gem::Specification.new do |s|
s.test_files = Dir["test/**/*.rb"]
s.bindir = "bin"
s.executables = ["kpeg"]
- s.require_paths = ["lib"]
s.add_development_dependency "rake"
+ s.add_development_dependency "minitest", "~> 2.11"
end
View
15 lib/kpeg/code_generator.rb
@@ -323,8 +323,16 @@ def standalone_region(path)
def output
return @output if @output
+
+ code = []
+
+ if header = @grammar.directives['header']
+ code << header.action.strip
+ code << "\n"
+ end
+
if @standalone
- code = "class #{@name}\n"
+ code << "class #{@name}\n"
unless cp = standalone_region(
File.expand_path("../compiled_parser.rb", __FILE__))
@@ -341,7 +349,7 @@ def output
cp.gsub!(/include Position/, pp)
code << cp << "\n"
else
- code = "require 'kpeg/compiled_parser'\n\n"
+ code << "require 'kpeg/compiled_parser'\n\n"
code << "class #{@name} < KPeg::CompiledParser\n"
end
@@ -417,7 +425,8 @@ def output
end
code << "end\n"
- @output = code
+
+ @output = code.join
end
def make(str)
View
1 lib/kpeg/format.kpeg
@@ -108,6 +108,7 @@ sgl_escape_quote = "\\'" { "'" }
| - "%" var:name - "=" - < /[::A-Za-z0-9_]+/ >
{ @g.add_foreign_grammar(name, text) }
| - "%%" - curly:act { @g.add_setup act }
+ | - "%%" - var:name - curly:act { @g.add_directive name, act }
| - "%%" - var:name - "=" - < (!"\n" .)+ >
{ @g.set_variable(name, text) }
statements = statement (- statements)?
View
137 lib/kpeg/format_parser.rb
@@ -186,41 +186,37 @@ def get_byte
end
def parse(rule=nil)
+ # We invoke the rules indirectly via apply
+ # instead of by just calling them as methods because
+ # if the rules use left recursion, apply needs to
+ # manage that.
+
if !rule
- _root ? true : false
+ apply(:_root)
else
- # This is not shared with code_generator.rb so this can be standalone
method = rule.gsub("-","_hyphen_")
- __send__("_#{method}") ? true : false
+ apply :"_#{method}"
end
end
- class LeftRecursive
- def initialize(detected=false)
- @detected = detected
- end
-
- attr_accessor :detected
- end
-
class MemoEntry
def initialize(ans, pos)
@ans = ans
@pos = pos
- @uses = 1
@result = nil
+ @set = false
+ @left_rec = false
end
- attr_reader :ans, :pos, :uses, :result
-
- def inc!
- @uses += 1
- end
+ attr_reader :ans, :pos, :result, :set
+ attr_accessor :left_rec
def move!(ans, pos, result)
@ans = ans
@pos = pos
@result = result
+ @set = true
+ @left_rec = false
end
end
@@ -248,31 +244,30 @@ def external_invoke(other, rule, *args)
def apply_with_args(rule, *args)
memo_key = [rule, args]
if m = @memoizations[memo_key][@pos]
- m.inc!
-
prev = @pos
@pos = m.pos
- if m.ans.kind_of? LeftRecursive
- m.ans.detected = true
+ if !m.set
+ m.left_rec = true
return nil
end
@result = m.result
return m.ans
else
- lr = LeftRecursive.new(false)
- m = MemoEntry.new(lr, @pos)
+ m = MemoEntry.new(nil, @pos)
@memoizations[memo_key][@pos] = m
start_pos = @pos
ans = __send__ rule, *args
+ lr = m.left_rec
+
m.move! ans, @pos, @result
# Don't bother trying to grow the left recursion
# if it's failing straight away (thus there is no seed)
- if ans and lr.detected
+ if ans and lr
return grow_lr(rule, args, start_pos, m)
else
return ans
@@ -284,31 +279,30 @@ def apply_with_args(rule, *args)
def apply(rule)
if m = @memoizations[rule][@pos]
- m.inc!
-
prev = @pos
@pos = m.pos
- if m.ans.kind_of? LeftRecursive
- m.ans.detected = true
+ if !m.set
+ m.left_rec = true
return nil
end
@result = m.result
return m.ans
else
- lr = LeftRecursive.new(false)
- m = MemoEntry.new(lr, @pos)
+ m = MemoEntry.new(nil, @pos)
@memoizations[rule][@pos] = m
start_pos = @pos
ans = __send__ rule
+ lr = m.left_rec
+
m.move! ans, @pos, @result
# Don't bother trying to grow the left recursion
# if it's failing straight away (thus there is no seed)
- if ans and lr.detected
+ if ans and lr
return grow_lr(rule, nil, start_pos, m)
else
return ans
@@ -2474,7 +2468,7 @@ def _args
return _tmp
end
- # statement = (- var:v "(" args:a ")" - "=" - expression:o { @g.set(v, o, a) } | - var:v - "=" - expression:o { @g.set(v, o) } | - "%" var:name - "=" - < /[::A-Za-z0-9_]+/ > { @g.add_foreign_grammar(name, text) } | - "%%" - curly:act { @g.add_setup act } | - "%%" - var:name - "=" - < (!"\n" .)+ > { @g.set_variable(name, text) })
+ # statement = (- var:v "(" args:a ")" - "=" - expression:o { @g.set(v, o, a) } | - var:v - "=" - expression:o { @g.set(v, o) } | - "%" var:name - "=" - < /[::A-Za-z0-9_]+/ > { @g.add_foreign_grammar(name, text) } | - "%%" - curly:act { @g.add_setup act } | - "%%" - var:name - curly:act { @g.add_directive name, act } | - "%%" - var:name - "=" - < (!"\n" .)+ > { @g.set_variable(name, text) })
def _statement
_save = self.pos
@@ -2701,52 +2695,97 @@ def _statement
self.pos = _save5
break
end
- _tmp = match_string("=")
+ _tmp = apply(:_curly)
+ act = @result
unless _tmp
self.pos = _save5
break
end
- _tmp = apply(:__hyphen_)
+ @result = begin; @g.add_directive name, act ; end
+ _tmp = true
unless _tmp
self.pos = _save5
+ end
+ break
+ end # end sequence
+
+ break if _tmp
+ self.pos = _save
+
+ _save6 = self.pos
+ while true # sequence
+ _tmp = apply(:__hyphen_)
+ unless _tmp
+ self.pos = _save6
+ break
+ end
+ _tmp = match_string("%%")
+ unless _tmp
+ self.pos = _save6
+ break
+ end
+ _tmp = apply(:__hyphen_)
+ unless _tmp
+ self.pos = _save6
+ break
+ end
+ _tmp = apply(:_var)
+ name = @result
+ unless _tmp
+ self.pos = _save6
+ break
+ end
+ _tmp = apply(:__hyphen_)
+ unless _tmp
+ self.pos = _save6
+ break
+ end
+ _tmp = match_string("=")
+ unless _tmp
+ self.pos = _save6
+ break
+ end
+ _tmp = apply(:__hyphen_)
+ unless _tmp
+ self.pos = _save6
break
end
_text_start = self.pos
- _save6 = self.pos
-
_save7 = self.pos
+
+ _save8 = self.pos
while true # sequence
- _save8 = self.pos
+ _save9 = self.pos
_tmp = match_string("\n")
_tmp = _tmp ? nil : true
- self.pos = _save8
+ self.pos = _save9
unless _tmp
- self.pos = _save7
+ self.pos = _save8
break
end
_tmp = get_byte
unless _tmp
- self.pos = _save7
+ self.pos = _save8
end
break
end # end sequence
if _tmp
while true
- _save9 = self.pos
+ _save10 = self.pos
while true # sequence
- _save10 = self.pos
+ _save11 = self.pos
_tmp = match_string("\n")
_tmp = _tmp ? nil : true
- self.pos = _save10
+ self.pos = _save11
unless _tmp
- self.pos = _save9
+ self.pos = _save10
break
end
_tmp = get_byte
unless _tmp
- self.pos = _save9
+ self.pos = _save10
end
break
end # end sequence
@@ -2755,19 +2794,19 @@ def _statement
end
_tmp = true
else
- self.pos = _save6
+ self.pos = _save7
end
if _tmp
text = get_text(_text_start)
end
unless _tmp
- self.pos = _save5
+ self.pos = _save6
break
end
@result = begin; @g.set_variable(name, text) ; end
_tmp = true
unless _tmp
- self.pos = _save5
+ self.pos = _save6
end
break
end # end sequence
@@ -3124,7 +3163,7 @@ def _ast_root
Rules[:_choose_cont] = rule_info("choose_cont", "- \"|\" - values:v { v }")
Rules[:_expression] = rule_info("expression", "(values:v choose_cont+:alts { @g.any(v, *alts) } | values)")
Rules[:_args] = rule_info("args", "(args:a \",\" - var:n - { a + [n] } | - var:n - { [n] })")
- Rules[:_statement] = rule_info("statement", "(- var:v \"(\" args:a \")\" - \"=\" - expression:o { @g.set(v, o, a) } | - var:v - \"=\" - expression:o { @g.set(v, o) } | - \"%\" var:name - \"=\" - < /[::A-Za-z0-9_]+/ > { @g.add_foreign_grammar(name, text) } | - \"%%\" - curly:act { @g.add_setup act } | - \"%%\" - var:name - \"=\" - < (!\"\\n\" .)+ > { @g.set_variable(name, text) })")
+ Rules[:_statement] = rule_info("statement", "(- var:v \"(\" args:a \")\" - \"=\" - expression:o { @g.set(v, o, a) } | - var:v - \"=\" - expression:o { @g.set(v, o) } | - \"%\" var:name - \"=\" - < /[::A-Za-z0-9_]+/ > { @g.add_foreign_grammar(name, text) } | - \"%%\" - curly:act { @g.add_setup act } | - \"%%\" - var:name - curly:act { @g.add_directive name, act } | - \"%%\" - var:name - \"=\" - < (!\"\\n\" .)+ > { @g.set_variable(name, text) })")
Rules[:_statements] = rule_info("statements", "statement (- statements)?")
Rules[:_eof] = rule_info("eof", "!.")
Rules[:_root] = rule_info("root", "statements - eof_comment? eof")
View
10 lib/kpeg/grammar.rb
@@ -634,16 +634,26 @@ def inspect
class Grammar
def initialize
+ @directives = {}
@rules = {}
@rule_order = []
@setup_actions = []
@foreign_grammars = {}
@variables = {}
end
+ attr_reader :directives
attr_reader :rules, :rule_order, :setup_actions, :foreign_grammars
attr_reader :variables
+ def add_directive(name, body)
+ if @directives.include? name
+ warn "directive #{name.inspect} appears more than once"
+ end
+
+ @directives[name] = body
+ end
+
def add_setup(act)
@setup_actions << act
end
View
4 test/test_file_parser_roundtrip.rb
@@ -1,11 +1,11 @@
+require 'minitest/autorun'
require 'kpeg'
require 'kpeg/format_parser'
require 'kpeg/grammar_renderer'
require 'kpeg/code_generator'
require 'stringio'
-require 'test/unit'
-class TestKPegRoundtrip < Test::Unit::TestCase
+class TestKPegRoundtrip < MiniTest::Unit::TestCase
PATH = File.expand_path("../../lib/kpeg/format.kpeg", __FILE__)
def test_roundtrip
data = File.read(PATH)
View
4 test/test_gen_calc.rb
@@ -1,10 +1,10 @@
-require 'test/unit'
+require 'minitest/autorun'
require 'kpeg'
require 'kpeg/format_parser'
require 'kpeg/code_generator'
require 'stringio'
-class TestKPegCodeGenerator < Test::Unit::TestCase
+class TestKPegCodeGenerator < MiniTest::Unit::TestCase
GRAMMAR = <<-'STR'
Stmt = - Expr:e EOL { @answers << e }
| ( !EOL . )* EOL { puts "error" }
View
4 test/test_kpeg.rb
@@ -1,8 +1,8 @@
-require 'test/unit'
+require 'minitest/autorun'
require 'kpeg'
require 'stringio'
-class TestKPeg < Test::Unit::TestCase
+class TestKPeg < MiniTest::Unit::TestCase
def assert_match(m, str)
assert_kind_of KPeg::MatchString, m
assert_equal str, m.string
View
35 test/test_kpeg_code_generator.rb
@@ -1,10 +1,10 @@
# encoding: utf-8
-require 'test/unit'
+require 'minitest/autorun'
require 'kpeg'
require 'kpeg/code_generator'
require 'stringio'
-class TestKPegCodeGenerator < Test::Unit::TestCase
+class TestKPegCodeGenerator < MiniTest::Unit::TestCase
def test_dot
gram = KPeg.grammar do |g|
g.root = g.dot
@@ -1370,6 +1370,37 @@ def test_parse_error
assert_equal 5, code.failing_rule_offset
end
+ def test_directive_header
+ gram = KPeg.grammar do |g|
+ g.root = g.dot
+ g.directives['header'] = g.action("\n# coding: UTF-8\n")
+ end
+
+ str = <<-STR
+# coding: UTF-8
+require 'kpeg/compiled_parser'
+
+class Test < KPeg::CompiledParser
+
+ # root = .
+ def _root
+ _tmp = get_byte
+ set_failed_rule :_root unless _tmp
+ return _tmp
+ end
+
+ Rules = {}
+ Rules[:_root] = rule_info("root", ".")
+end
+ STR
+
+ cg = KPeg::CodeGenerator.new "Test", gram
+
+ assert_equal str, cg.output
+
+ assert cg.parse("hello")
+ end
+
def test_setup_actions
gram = KPeg.grammar do |g|
g.root = g.dot
View
4 test/test_kpeg_compiled_parser.rb
@@ -1,9 +1,9 @@
-require 'test/unit'
+require 'minitest/autorun'
require 'kpeg'
require 'kpeg/compiled_parser'
require 'stringio'
-class TestKPegCompiledParser < Test::Unit::TestCase
+class TestKPegCompiledParser < MiniTest::Unit::TestCase
gram = <<-GRAM
letter = [a-z]
View
49 test/test_kpeg_format.rb
@@ -1,11 +1,11 @@
-require 'test/unit'
+require 'minitest/autorun'
require 'kpeg'
require 'kpeg/format_parser'
require 'kpeg/grammar_renderer'
require 'stringio'
require 'rubygems'
-class TestKPegFormat < Test::Unit::TestCase
+class TestKPegFormat < MiniTest::Unit::TestCase
G = KPeg::Grammar.new
gram = File.read File.expand_path("../../lib/kpeg/format.kpeg", __FILE__)
@@ -386,6 +386,51 @@ def test_comment_span
assert_rule G.seq(G.ref('b'), G.ref("c")), m
end
+ def test_parser_directive
+ m = match <<-GRAMMAR
+%% header {
+# coding: UTF-8
+}
+
+a=b
+ GRAMMAR
+
+ assert_rule G.ref("b"), m
+
+ expected = {
+ "header" => KPeg::Action.new("\n# coding: UTF-8\n")
+ }
+
+ assert_equal expected, m.directives
+ end
+
+ def test_parser_directive_duplicate
+ m = nil
+
+ out, err = capture_io do
+ m = match <<-GRAMMAR
+%% header {
+# coding: UTF-8
+}
+
+a=b
+
+%% header {
+# coding: ISO-8859-1
+}
+ GRAMMAR
+ end
+
+ assert_empty out
+ assert_equal "directive \"header\" appears more than once\n", err
+
+ expected = {
+ "header" => KPeg::Action.new("\n# coding: ISO-8859-1\n")
+ }
+
+ assert_equal expected, m.directives
+ end
+
def test_parser_setup
m = match "%% { def initialize; end }\na=b"
assert_rule G.ref("b"), m
View
4 test/test_kpeg_grammar_renderer.rb
@@ -1,9 +1,9 @@
-require 'test/unit'
+require 'minitest/autorun'
require 'kpeg'
require 'kpeg/grammar_renderer'
require 'stringio'
-class TestKPegGrammarRenderer < Test::Unit::TestCase
+class TestKPegGrammarRenderer < MiniTest::Unit::TestCase
def test_escape
str = "hello\nbob"
assert_equal 'hello\nbob', KPeg::GrammarRenderer.escape(str)
View
4 test/test_left_recursion.rb
@@ -1,10 +1,10 @@
-require 'test/unit'
+require 'minitest/autorun'
require 'kpeg'
require 'kpeg/format_parser'
require 'kpeg/code_generator'
require 'stringio'
-class TestKPegLeftRecursion < Test::Unit::TestCase
+class TestKPegLeftRecursion < MiniTest::Unit::TestCase
GRAMMAR = <<-'STR'
name = name:n "[]" { [:array, n] }

0 comments on commit bf8b1c6

Please sign in to comment.