Permalink
Browse files

The big rename and cleanup.

* peg_meta_gen no longer necessary
* peg_gen -> neotoma
* peg_meta -> neotoma_parse
* peg -> neotoma_peg
  • Loading branch information...
1 parent 8652541 commit cbf275a7b638f3c517cbc5b637844bf270b54e59 @seancribbs committed Nov 13, 2009
View
@@ -16,8 +16,8 @@ priv:
ebin_tests:
mkdir ebin_tests
-priv/peg_includes.erl: priv src/peg.erl
- cat src/peg.erl | grep -v "^%" | grep -v "^-" > priv/peg_includes.erl
+priv/peg_includes.erl: priv src/neotoma_peg.erl
+ cat src/neotoma_peg.erl | grep -v "^%" | grep -v "^-" > priv/peg_includes.erl
src_src: ebin src/neotoma.app priv/peg_includes.erl
cd src;erl -pz ../ebin -make
@@ -33,5 +33,5 @@ clean:
rm -rf ebin_tests
bootstrap: src_src
- ${ERL} -pz ebin -b start_sasl -noshell -s init stop -eval 'peg_gen:bootstrap().'
- cd src;erl -pz ../ebin -make
+ ${ERL} -pz ebin -b start_sasl -noshell -s init stop -eval 'neotoma:bootstrap().'
+ cd src;erl -pz ../ebin -make
View
@@ -1,2 +1,2 @@
-{[peg, peg_gen, peg_meta, peg_meta_gen],
+{[neotoma_peg, neotoma, neotoma_parse],
[{outdir, "../ebin"}, {i, "../../"}, debug_info]}.
View
@@ -1,8 +1,8 @@
{application, neotoma,
[
{description, "PEG/Packrat toolkit and parser-generator."},
- {vsn, "1.1"},
- {modules, [peg, peg_transform, peg_itransform, peg_gen, peg_meta, peg_meta_gen]},
+ {vsn, "1.3"},
+ {modules, [neotoma, neotoma_parse, neotoma_peg]},
{applications, [kernel, stdlib]}
]
}.
@@ -1,4 +1,4 @@
--module(peg_gen).
+-module(neotoma).
-author("Sean Cribbs <seancribbs@gmail.com>").
-export([file/1, file/2, bootstrap/0]).
@@ -69,7 +69,7 @@ generate_module_attrs(ModName, Root) ->
" release_memo(), Result.\n"].
parse_grammar(InputFile) ->
- case peg_meta:file(InputFile) of
+ case neotoma_parse:file(InputFile) of
{fail, Index} ->
throw({grammar_error, {fail, Index}});
{Parsed, Remainder, Index} ->
@@ -99,6 +99,6 @@ generate_transform_stub(XfFile,ModName) ->
file:write_file(XfFile, Data).
%% @doc Bootstraps the neotoma metagrammar. Intended only for internal development!
-%% @equiv file("src/peg_meta.peg", [{transform_module, peg_meta_gen}])
+%% @equiv file("src/neotoma_parse.peg")
bootstrap() ->
- file("src/peg_meta.peg", [{transform_module, peg_meta_gen}]).
+ file("src/neotoma_parse.peg").
@@ -1,4 +1,4 @@
--module(peg_meta).
+-module(neotoma_parse).
-export([parse/1,file/1]).
-compile(nowarn_unused_vars).
-compile({nowarn_unused_function,[p/4, p/5, p_eof/0, p_optional/1, p_not/1, p_assert/1, p_seq/1, p_and/1, p_choose/1, p_zero_or_more/1, p_one_or_more/1, p_label/2, p_string/1, p_anything/0, p_charclass/1]}).
@@ -21,7 +21,7 @@ parse(Input) ->
{code, Block} -> Block;
_ -> []
end,
- [{rules, Rules ++ "\n" ++ Code}, {root, RootRule}, {transform, ets:lookup(peg_meta,gen_transform)}]
+ [{rules, Rules ++ "\n" ++ Code}, {root, RootRule}, {transform, ets:lookup(?MODULE,gen_transform)}]
end).
'declaration_sequence'(Input, Index) ->
@@ -6,7 +6,7 @@ rules <- space? declaration_sequence space? code_block? space?
{code, Block} -> Block;
_ -> []
end,
- [{rules, Rules ++ "\n" ++ Code}, {root, RootRule}, {transform, ets:lookup(peg_meta,gen_transform)}]
+ [{rules, Rules ++ "\n" ++ Code}, {root, RootRule}, {transform, ets:lookup(?MODULE,gen_transform)}]
`;
declaration_sequence <- head:declaration tail:(space declaration)*
@@ -1,4 +1,4 @@
--module(peg).
+-module(neotoma_peg).
-author("Sean Cribbs <seancribbs@gmail.com>").
% Thanks to Jeffrey A. Meunier for the original parser.erl library from which I
View
@@ -1,135 +0,0 @@
--module(peg_meta_gen).
--export([transform/3]).
--author("Sean Cribbs <seancribbs@gmail.com>").
-
-transform(rules, Node, _Index) ->
- RootRule = verify_rules(),
- Rules = string:join(lists:nth(2, Node), "\n\n"),
- Code = case lists:nth(4, Node) of
- {code, Block} -> Block;
- _ -> []
- end,
- [{rules, Rules ++ "\n" ++ Code}, {root, RootRule}, {transform, ets:lookup(peg_meta,gen_transform)}];
-transform(declaration_sequence, Node, _Index) ->
- FirstRule = proplists:get_value(head, Node),
- OtherRules = [lists:last(I) || I <- proplists:get_value(tail, Node, [])],
- [FirstRule|OtherRules];
-transform(declaration, [{nonterminal,Symbol}|Node], Index) ->
- add_lhs(Symbol, Index),
- Transform = case lists:nth(6,Node) of
- {code, CodeBlock} -> CodeBlock;
- _ ->
- ets:insert_new(peg_meta,{gen_transform, true}),
- "transform('"++Symbol++"', Node, Idx)"
- end,
- "'"++Symbol++"'"++"(Input, Index) ->\n " ++
- "p(Input, Index, '"++Symbol++"', fun(I,D) -> ("++
- lists:nth(4, Node) ++
- ")(I,D) end, fun(Node, Idx) -> "++Transform++" end).";
-transform(sequence, Node, _Index) ->
- Tail = [lists:nth(2, S) || S <- proplists:get_value(tail, Node)],
- Statements = [proplists:get_value(head, Node)|Tail],
- "p_seq(["++ string:join(Statements, ", ") ++ "])";
-transform(choice, Node, _Index) ->
- Tail = [lists:last(S) || S <- proplists:get_value(tail, Node)],
- Statements = [proplists:get_value(head, Node)|Tail],
- "p_choose([" ++ string:join(Statements, ", ") ++ "])";
-transform(label, Node, _Index) ->
- String = lists:flatten(Node),
- lists:sublist(String, length(String)-1);
-transform(labeled_sequence_primary, Node, _Index) ->
- case hd(Node) of
- [] -> lists:nth(2, Node);
- Label -> "p_label('" ++ Label ++ "', "++lists:nth(2, Node)++")"
- end;
-transform(single_quoted_string, Node, Index) ->
- transform(double_quoted_string, Node, Index);
-transform(double_quoted_string, Node, _Index) ->
- "p_string(\""++escape_quotes(lists:flatten(proplists:get_value(string, Node)))++"\")";
-transform(character_class, Node, _Index) ->
- "p_charclass(\"[" ++ escape_quotes(lists:flatten(proplists:get_value(characters, Node))) ++ "]\")";
-transform(parenthesized_expression, Node, _Index) ->
- lists:nth(3, Node);
-transform(atomic, {nonterminal, Symbol}, Index) ->
- add_nt(Symbol, Index),
- "fun '" ++ Symbol ++ "'/2";
-transform(primary, [Atomic, one_or_more], _Index) ->
- "p_one_or_more("++Atomic++")";
-transform(primary, [Atomic, zero_or_more], _Index) ->
- "p_zero_or_more("++Atomic++")";
-transform(primary, [Atomic, optional], _Index) ->
- "p_optional("++Atomic++")";
-transform(primary, [assert, Atomic], _Index)->
- "p_assert("++Atomic++")";
-transform(primary, [not_, Atomic], _Index) ->
- "p_not("++Atomic++")";
-transform(nonterminal, Node, _Index) ->
- {nonterminal, lists:flatten(Node)};
-transform(anything_symbol, _Node, _Index) ->
- "p_anything()";
-transform(suffix, Node, _Index) ->
- case Node of
- "*" -> zero_or_more;
- "+" -> one_or_more;
- "?" -> optional
- end;
-transform(prefix, Node, _Index) ->
- case Node of
- "&" -> assert;
- "!" -> not_
- end;
-transform(code_block, Node, _Index) ->
- case Node of
- "~" -> {code, "Node"};
- _ -> {code, lists:flatten(proplists:get_value('code', Node))}
- end;
-transform(Rule, Node, _Index) when is_atom(Rule) ->
- Node.
-
-escape_quotes(String) ->
- {ok, RE} = re:compile("\""),
- re:replace(String, RE, "\\\\\"", [global, {return, list}]).
-
-add_lhs(Symbol, Index) ->
- case ets:lookup(peg_meta, lhs) of
- [] ->
- ets:insert(peg_meta, {lhs, [{Symbol,Index}]});
- [{lhs, L}] when is_list(L) ->
- ets:insert(peg_meta, {lhs, [{Symbol,Index}|L]})
- end.
-
-add_nt(Symbol, Index) ->
- case ets:lookup(peg_meta, nts) of
- [] ->
- ets:insert(peg_meta, {nts, [{Symbol,Index}]});
- [{nts, L}] when is_list(L) ->
- case proplists:is_defined(Symbol, L) of
- true ->
- ok;
- _ ->
- ets:insert(peg_meta, {nts, [{Symbol,Index}|L]})
- end
- end.
-
-verify_rules() ->
- [{lhs, LHS}] = ets:lookup(peg_meta, lhs),
- [{nts, NTs}] = ets:lookup(peg_meta, nts),
- [Root|NonRoots] = lists:reverse(LHS),
- lists:foreach(fun({Sym,Idx}) ->
- case proplists:is_defined(Sym, NTs) of
- true ->
- ok;
- _ ->
- io:format("neotoma warning: rule '~s' is unused. ~p~n", [Sym,Idx])
- end
- end, NonRoots),
- lists:foreach(fun({S,I}) ->
- case proplists:is_defined(S, LHS) of
- true ->
- ok;
- _ ->
- io:format("neotoma error: nonterminal '~s' has no reduction. (found at ~p) No parser will be generated!~n", [S,I]),
- exit({neotoma, {no_reduction, list_to_atom(S)}})
- end
- end, NTs),
- Root.
@@ -2,81 +2,81 @@
-author("Sean Cribbs <seancribbs@gmail.com>").
-include_lib("eunit/include/eunit.hrl").
-% Test the parser-combinators in the 'peg' module
+% Test the parser-combinators in the 'neotoma_peg' module
-define(STARTINDEX, {{line,1},{column,1}}).
eof_test_() ->
[
- ?_assertEqual({fail,{expected,eof,?STARTINDEX}}, (peg:p_eof())("abc",?STARTINDEX)),
- ?_assertEqual({eof, [], ?STARTINDEX}, (peg:p_eof())("",?STARTINDEX))
+ ?_assertEqual({fail,{expected,eof,?STARTINDEX}}, (neotoma_peg:p_eof())("abc",?STARTINDEX)),
+ ?_assertEqual({eof, [], ?STARTINDEX}, (neotoma_peg:p_eof())("",?STARTINDEX))
].
optional_test_() ->
[
- ?_assertEqual({[], "xyz",?STARTINDEX}, (peg:p_optional(peg:p_string("abc")))("xyz",?STARTINDEX)),
- ?_assertEqual({"abc", "xyz",{{line,1},{column,4}}}, (peg:p_optional(peg:p_string("abc")))("abcxyz",?STARTINDEX))
+ ?_assertEqual({[], "xyz",?STARTINDEX}, (neotoma_peg:p_optional(neotoma_peg:p_string("abc")))("xyz",?STARTINDEX)),
+ ?_assertEqual({"abc", "xyz",{{line,1},{column,4}}}, (neotoma_peg:p_optional(neotoma_peg:p_string("abc")))("abcxyz",?STARTINDEX))
].
not_test_() ->
[
- ?_assertEqual({[], "xyzabc",?STARTINDEX}, (peg:p_not(peg:p_string("abc")))("xyzabc",?STARTINDEX)),
- ?_assertEqual({fail,{expected, {no_match, "abc"}, ?STARTINDEX}}, (peg:p_not(peg:p_string("abc")))("abcxyz",?STARTINDEX))
+ ?_assertEqual({[], "xyzabc",?STARTINDEX}, (neotoma_peg:p_not(neotoma_peg:p_string("abc")))("xyzabc",?STARTINDEX)),
+ ?_assertEqual({fail,{expected, {no_match, "abc"}, ?STARTINDEX}}, (neotoma_peg:p_not(neotoma_peg:p_string("abc")))("abcxyz",?STARTINDEX))
].
assert_test_() ->
[
- ?_assertEqual({fail,{expected, {string, "abc"}, ?STARTINDEX}}, (peg:p_assert(peg:p_string("abc")))("xyzabc",?STARTINDEX)),
- ?_assertEqual({[], "abcxyz",?STARTINDEX}, (peg:p_assert(peg:p_string("abc")))("abcxyz",?STARTINDEX))
+ ?_assertEqual({fail,{expected, {string, "abc"}, ?STARTINDEX}}, (neotoma_peg:p_assert(neotoma_peg:p_string("abc")))("xyzabc",?STARTINDEX)),
+ ?_assertEqual({[], "abcxyz",?STARTINDEX}, (neotoma_peg:p_assert(neotoma_peg:p_string("abc")))("abcxyz",?STARTINDEX))
].
seq_test_() ->
[
- ?_assertEqual({["abc","def"], "xyz",{{line,1},{column,7}}}, (peg:p_seq([peg:p_string("abc"), peg:p_string("def")]))("abcdefxyz",?STARTINDEX)),
- ?_assertEqual({fail,{expected, {string, "def"}, {{line,1},{column,4}}}}, (peg:p_seq([peg:p_string("abc"), peg:p_string("def")]))("abcxyz",?STARTINDEX))
+ ?_assertEqual({["abc","def"], "xyz",{{line,1},{column,7}}}, (neotoma_peg:p_seq([neotoma_peg:p_string("abc"), neotoma_peg:p_string("def")]))("abcdefxyz",?STARTINDEX)),
+ ?_assertEqual({fail,{expected, {string, "def"}, {{line,1},{column,4}}}}, (neotoma_peg:p_seq([neotoma_peg:p_string("abc"), neotoma_peg:p_string("def")]))("abcxyz",?STARTINDEX))
].
choose_test_() ->
[
- ?_assertEqual({"abc", "xyz", {{line,1},{column,4}}}, (peg:p_choose([peg:p_string("abc"), peg:p_string("def")]))("abcxyz",?STARTINDEX)),
- ?_assertEqual({"def", "xyz", {{line,1},{column,4}}}, (peg:p_choose([peg:p_string("abc"), peg:p_string("def")]))("defxyz",?STARTINDEX)),
- ?_assertEqual({"xyz", "xyz", {{line,1},{column,4}}}, (peg:p_choose([peg:p_string("abc"), peg:p_string("def"), peg:p_string("xyz")]))("xyzxyz",?STARTINDEX)),
- ?_assertEqual({fail,{expected,{string,"abc"},?STARTINDEX}}, (peg:p_choose([peg:p_string("abc"),peg:p_string("def")]))("xyz", ?STARTINDEX))
+ ?_assertEqual({"abc", "xyz", {{line,1},{column,4}}}, (neotoma_peg:p_choose([neotoma_peg:p_string("abc"), neotoma_peg:p_string("def")]))("abcxyz",?STARTINDEX)),
+ ?_assertEqual({"def", "xyz", {{line,1},{column,4}}}, (neotoma_peg:p_choose([neotoma_peg:p_string("abc"), neotoma_peg:p_string("def")]))("defxyz",?STARTINDEX)),
+ ?_assertEqual({"xyz", "xyz", {{line,1},{column,4}}}, (neotoma_peg:p_choose([neotoma_peg:p_string("abc"), neotoma_peg:p_string("def"), neotoma_peg:p_string("xyz")]))("xyzxyz",?STARTINDEX)),
+ ?_assertEqual({fail,{expected,{string,"abc"},?STARTINDEX}}, (neotoma_peg:p_choose([neotoma_peg:p_string("abc"),neotoma_peg:p_string("def")]))("xyz", ?STARTINDEX))
].
zero_or_more_test_() ->
[
- ?_assertEqual({[], [], ?STARTINDEX}, (peg:p_zero_or_more(peg:p_string("abc")))("",?STARTINDEX)),
- ?_assertEqual({[], "def",?STARTINDEX}, (peg:p_zero_or_more(peg:p_string("abc")))("def",?STARTINDEX)),
- ?_assertEqual({["abc"], "def",{{line,1},{column,4}}}, (peg:p_zero_or_more(peg:p_string("abc")))("abcdef",?STARTINDEX)),
- ?_assertEqual({["abc", "abc"], "def",{{line,1},{column,7}}}, (peg:p_zero_or_more(peg:p_string("abc")))("abcabcdef",?STARTINDEX))
+ ?_assertEqual({[], [], ?STARTINDEX}, (neotoma_peg:p_zero_or_more(neotoma_peg:p_string("abc")))("",?STARTINDEX)),
+ ?_assertEqual({[], "def",?STARTINDEX}, (neotoma_peg:p_zero_or_more(neotoma_peg:p_string("abc")))("def",?STARTINDEX)),
+ ?_assertEqual({["abc"], "def",{{line,1},{column,4}}}, (neotoma_peg:p_zero_or_more(neotoma_peg:p_string("abc")))("abcdef",?STARTINDEX)),
+ ?_assertEqual({["abc", "abc"], "def",{{line,1},{column,7}}}, (neotoma_peg:p_zero_or_more(neotoma_peg:p_string("abc")))("abcabcdef",?STARTINDEX))
].
one_or_more_test_() ->
[
- ?_assertEqual({fail,{expected, {at_least_one, {string, "abc"}}, ?STARTINDEX}}, (peg:p_one_or_more(peg:p_string("abc")))("def",?STARTINDEX)),
- ?_assertEqual({["abc"], "def",{{line,1},{column,4}}}, (peg:p_one_or_more(peg:p_string("abc")))("abcdef",?STARTINDEX)),
- ?_assertEqual({["abc","abc"], "def",{{line,1},{column,7}}}, (peg:p_one_or_more(peg:p_string("abc")))("abcabcdef",?STARTINDEX))
+ ?_assertEqual({fail,{expected, {at_least_one, {string, "abc"}}, ?STARTINDEX}}, (neotoma_peg:p_one_or_more(neotoma_peg:p_string("abc")))("def",?STARTINDEX)),
+ ?_assertEqual({["abc"], "def",{{line,1},{column,4}}}, (neotoma_peg:p_one_or_more(neotoma_peg:p_string("abc")))("abcdef",?STARTINDEX)),
+ ?_assertEqual({["abc","abc"], "def",{{line,1},{column,7}}}, (neotoma_peg:p_one_or_more(neotoma_peg:p_string("abc")))("abcabcdef",?STARTINDEX))
].
label_test_() ->
[
- ?_assertEqual({fail,{expected, {string, "!"}, ?STARTINDEX}}, (peg:p_label(bang, peg:p_string("!")))("?",?STARTINDEX)),
- ?_assertEqual({{bang, "!"}, "",{{line,1},{column,2}}}, (peg:p_label(bang, peg:p_string("!")))("!",?STARTINDEX))
+ ?_assertEqual({fail,{expected, {string, "!"}, ?STARTINDEX}}, (neotoma_peg:p_label(bang, neotoma_peg:p_string("!")))("?",?STARTINDEX)),
+ ?_assertEqual({{bang, "!"}, "",{{line,1},{column,2}}}, (neotoma_peg:p_label(bang, neotoma_peg:p_string("!")))("!",?STARTINDEX))
].
string_test_() ->
[
- ?_assertEqual({"abc", "def",{{line,1},{column,4}}}, (peg:p_string("abc"))("abcdef",?STARTINDEX)),
- ?_assertEqual({fail,{expected, {string, "abc"}, ?STARTINDEX}}, (peg:p_string("abc"))("defabc",?STARTINDEX))
+ ?_assertEqual({"abc", "def",{{line,1},{column,4}}}, (neotoma_peg:p_string("abc"))("abcdef",?STARTINDEX)),
+ ?_assertEqual({fail,{expected, {string, "abc"}, ?STARTINDEX}}, (neotoma_peg:p_string("abc"))("defabc",?STARTINDEX))
].
anything_test_() ->
[
- ?_assertEqual({$a,"bcde",{{line,1},{column,2}}}, (peg:p_anything())("abcde",?STARTINDEX)),
- ?_assertEqual({fail,{expected, any_character, ?STARTINDEX}}, (peg:p_anything())("",?STARTINDEX))
+ ?_assertEqual({$a,"bcde",{{line,1},{column,2}}}, (neotoma_peg:p_anything())("abcde",?STARTINDEX)),
+ ?_assertEqual({fail,{expected, any_character, ?STARTINDEX}}, (neotoma_peg:p_anything())("",?STARTINDEX))
].
charclass_test_() ->
[
- ?_assertEqual({$+,"----",{{line,1},{column,2}}}, (peg:p_charclass("[+]"))("+----",?STARTINDEX)),
- ?_assertEqual({fail,{expected, {character_class, "[+]"}, ?STARTINDEX}}, (peg:p_charclass("[+]"))("----",?STARTINDEX))
+ ?_assertEqual({$+,"----",{{line,1},{column,2}}}, (neotoma_peg:p_charclass("[+]"))("+----",?STARTINDEX)),
+ ?_assertEqual({fail,{expected, {character_class, "[+]"}, ?STARTINDEX}}, (neotoma_peg:p_charclass("[+]"))("----",?STARTINDEX))
].
@@ -3,20 +3,20 @@
-include_lib("eunit/include/eunit.hrl").
setup_memo_test() ->
- peg:setup_memo(),
- ?assertNot(undefined == ets:info(peg)),
- peg:release_memo().
+ neotoma_peg:setup_memo(),
+ ?assertNot(undefined == ets:info(neotoma_peg)),
+ neotoma_peg:release_memo().
release_memo_test() ->
- peg:setup_memo(),
- peg:release_memo(),
- ?assertEqual(undefined, ets:info(peg)).
+ neotoma_peg:setup_memo(),
+ neotoma_peg:release_memo(),
+ ?assertEqual(undefined, ets:info(neotoma_peg)).
step_memo_test() ->
- peg:setup_memo(),
- Result = peg:p("abcdefghi", {{line,1},{column,1}}, anything, peg:p_anything()),
+ neotoma_peg:setup_memo(),
+ Result = neotoma_peg:p("abcdefghi", {{line,1},{column,1}}, anything, neotoma_peg:p_anything()),
?assertEqual({$a, "bcdefghi", {{line,1},{column,2}}}, Result),
- Result2 = peg:p("abcdefghi", {{line,1},{column,1}}, anything, fun(_) ->
+ Result2 = neotoma_peg:p("abcdefghi", {{line,1},{column,1}}, anything, fun(_) ->
throw(bork) end),
?assertEqual(Result, Result2),
- peg:release_memo().
+ neotoma_peg:release_memo().

0 comments on commit cbf275a

Please sign in to comment.