diff --git a/Makefile b/Makefile index c4bfe5f..8c769a9 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,8 @@ priv: ebin_tests: mkdir ebin_tests -priv/peg_includes.erl: priv src/peg.erl - cat src/peg.erl | grep -v "^%" | grep -v "^-" > priv/peg_includes.erl +priv/peg_includes.erl: priv src/neotoma_peg.erl + cat src/neotoma_peg.erl | grep -v "^%" | grep -v "^-" > priv/peg_includes.erl src_src: ebin src/neotoma.app priv/peg_includes.erl cd src;erl -pz ../ebin -make @@ -33,5 +33,5 @@ clean: rm -rf ebin_tests bootstrap: src_src - ${ERL} -pz ebin -b start_sasl -noshell -s init stop -eval 'peg_gen:bootstrap().' - cd src;erl -pz ../ebin -make \ No newline at end of file + ${ERL} -pz ebin -b start_sasl -noshell -s init stop -eval 'neotoma:bootstrap().' + cd src;erl -pz ../ebin -make diff --git a/src/Emakefile b/src/Emakefile index 62d5014..16d34a1 100644 --- a/src/Emakefile +++ b/src/Emakefile @@ -1,2 +1,2 @@ -{[peg, peg_gen, peg_meta, peg_meta_gen], +{[neotoma_peg, neotoma, neotoma_parse], [{outdir, "../ebin"}, {i, "../../"}, debug_info]}. diff --git a/src/neotoma.app b/src/neotoma.app index 14d460f..69260e0 100644 --- a/src/neotoma.app +++ b/src/neotoma.app @@ -1,8 +1,8 @@ {application, neotoma, [ {description, "PEG/Packrat toolkit and parser-generator."}, - {vsn, "1.1"}, - {modules, [peg, peg_transform, peg_itransform, peg_gen, peg_meta, peg_meta_gen]}, + {vsn, "1.3"}, + {modules, [neotoma, neotoma_parse, neotoma_peg]}, {applications, [kernel, stdlib]} ] }. diff --git a/src/peg_gen.erl b/src/neotoma.erl similarity index 96% rename from src/peg_gen.erl rename to src/neotoma.erl index 0a0e996..6943904 100644 --- a/src/peg_gen.erl +++ b/src/neotoma.erl @@ -1,4 +1,4 @@ --module(peg_gen). +-module(neotoma). -author("Sean Cribbs "). -export([file/1, file/2, bootstrap/0]). @@ -69,7 +69,7 @@ generate_module_attrs(ModName, Root) -> " release_memo(), Result.\n"]. parse_grammar(InputFile) -> - case peg_meta:file(InputFile) of + case neotoma_parse:file(InputFile) of {fail, Index} -> throw({grammar_error, {fail, Index}}); {Parsed, Remainder, Index} -> @@ -99,6 +99,6 @@ generate_transform_stub(XfFile,ModName) -> file:write_file(XfFile, Data). %% @doc Bootstraps the neotoma metagrammar. Intended only for internal development! -%% @equiv file("src/peg_meta.peg", [{transform_module, peg_meta_gen}]) +%% @equiv file("src/neotoma_parse.peg") bootstrap() -> - file("src/peg_meta.peg", [{transform_module, peg_meta_gen}]). + file("src/neotoma_parse.peg"). diff --git a/src/peg_meta.erl b/src/neotoma_parse.erl similarity index 99% rename from src/peg_meta.erl rename to src/neotoma_parse.erl index 8ba7d44..d40b837 100644 --- a/src/peg_meta.erl +++ b/src/neotoma_parse.erl @@ -1,4 +1,4 @@ --module(peg_meta). +-module(neotoma_parse). -export([parse/1,file/1]). -compile(nowarn_unused_vars). -compile({nowarn_unused_function,[p/4, p/5, p_eof/0, p_optional/1, p_not/1, p_assert/1, p_seq/1, p_and/1, p_choose/1, p_zero_or_more/1, p_one_or_more/1, p_label/2, p_string/1, p_anything/0, p_charclass/1]}). @@ -21,7 +21,7 @@ parse(Input) -> {code, Block} -> Block; _ -> [] end, - [{rules, Rules ++ "\n" ++ Code}, {root, RootRule}, {transform, ets:lookup(peg_meta,gen_transform)}] + [{rules, Rules ++ "\n" ++ Code}, {root, RootRule}, {transform, ets:lookup(?MODULE,gen_transform)}] end). 'declaration_sequence'(Input, Index) -> diff --git a/src/peg_meta.peg b/src/neotoma_parse.peg similarity index 99% rename from src/peg_meta.peg rename to src/neotoma_parse.peg index c8a75bb..24f3cdc 100644 --- a/src/peg_meta.peg +++ b/src/neotoma_parse.peg @@ -6,7 +6,7 @@ rules <- space? declaration_sequence space? code_block? space? {code, Block} -> Block; _ -> [] end, - [{rules, Rules ++ "\n" ++ Code}, {root, RootRule}, {transform, ets:lookup(peg_meta,gen_transform)}] + [{rules, Rules ++ "\n" ++ Code}, {root, RootRule}, {transform, ets:lookup(?MODULE,gen_transform)}] `; declaration_sequence <- head:declaration tail:(space declaration)* diff --git a/src/peg.erl b/src/neotoma_peg.erl similarity index 99% rename from src/peg.erl rename to src/neotoma_peg.erl index 6e3be02..638d60d 100644 --- a/src/peg.erl +++ b/src/neotoma_peg.erl @@ -1,4 +1,4 @@ --module(peg). +-module(neotoma_peg). -author("Sean Cribbs "). % Thanks to Jeffrey A. Meunier for the original parser.erl library from which I diff --git a/src/peg_meta_gen.erl b/src/peg_meta_gen.erl deleted file mode 100644 index 3bff7e3..0000000 --- a/src/peg_meta_gen.erl +++ /dev/null @@ -1,135 +0,0 @@ --module(peg_meta_gen). --export([transform/3]). --author("Sean Cribbs "). - -transform(rules, Node, _Index) -> - RootRule = verify_rules(), - Rules = string:join(lists:nth(2, Node), "\n\n"), - Code = case lists:nth(4, Node) of - {code, Block} -> Block; - _ -> [] - end, - [{rules, Rules ++ "\n" ++ Code}, {root, RootRule}, {transform, ets:lookup(peg_meta,gen_transform)}]; -transform(declaration_sequence, Node, _Index) -> - FirstRule = proplists:get_value(head, Node), - OtherRules = [lists:last(I) || I <- proplists:get_value(tail, Node, [])], - [FirstRule|OtherRules]; -transform(declaration, [{nonterminal,Symbol}|Node], Index) -> - add_lhs(Symbol, Index), - Transform = case lists:nth(6,Node) of - {code, CodeBlock} -> CodeBlock; - _ -> - ets:insert_new(peg_meta,{gen_transform, true}), - "transform('"++Symbol++"', Node, Idx)" - end, - "'"++Symbol++"'"++"(Input, Index) ->\n " ++ - "p(Input, Index, '"++Symbol++"', fun(I,D) -> ("++ - lists:nth(4, Node) ++ - ")(I,D) end, fun(Node, Idx) -> "++Transform++" end)."; -transform(sequence, Node, _Index) -> - Tail = [lists:nth(2, S) || S <- proplists:get_value(tail, Node)], - Statements = [proplists:get_value(head, Node)|Tail], - "p_seq(["++ string:join(Statements, ", ") ++ "])"; -transform(choice, Node, _Index) -> - Tail = [lists:last(S) || S <- proplists:get_value(tail, Node)], - Statements = [proplists:get_value(head, Node)|Tail], - "p_choose([" ++ string:join(Statements, ", ") ++ "])"; -transform(label, Node, _Index) -> - String = lists:flatten(Node), - lists:sublist(String, length(String)-1); -transform(labeled_sequence_primary, Node, _Index) -> - case hd(Node) of - [] -> lists:nth(2, Node); - Label -> "p_label('" ++ Label ++ "', "++lists:nth(2, Node)++")" - end; -transform(single_quoted_string, Node, Index) -> - transform(double_quoted_string, Node, Index); -transform(double_quoted_string, Node, _Index) -> - "p_string(\""++escape_quotes(lists:flatten(proplists:get_value(string, Node)))++"\")"; -transform(character_class, Node, _Index) -> - "p_charclass(\"[" ++ escape_quotes(lists:flatten(proplists:get_value(characters, Node))) ++ "]\")"; -transform(parenthesized_expression, Node, _Index) -> - lists:nth(3, Node); -transform(atomic, {nonterminal, Symbol}, Index) -> - add_nt(Symbol, Index), - "fun '" ++ Symbol ++ "'/2"; -transform(primary, [Atomic, one_or_more], _Index) -> - "p_one_or_more("++Atomic++")"; -transform(primary, [Atomic, zero_or_more], _Index) -> - "p_zero_or_more("++Atomic++")"; -transform(primary, [Atomic, optional], _Index) -> - "p_optional("++Atomic++")"; -transform(primary, [assert, Atomic], _Index)-> - "p_assert("++Atomic++")"; -transform(primary, [not_, Atomic], _Index) -> - "p_not("++Atomic++")"; -transform(nonterminal, Node, _Index) -> - {nonterminal, lists:flatten(Node)}; -transform(anything_symbol, _Node, _Index) -> - "p_anything()"; -transform(suffix, Node, _Index) -> - case Node of - "*" -> zero_or_more; - "+" -> one_or_more; - "?" -> optional - end; -transform(prefix, Node, _Index) -> - case Node of - "&" -> assert; - "!" -> not_ - end; -transform(code_block, Node, _Index) -> - case Node of - "~" -> {code, "Node"}; - _ -> {code, lists:flatten(proplists:get_value('code', Node))} - end; -transform(Rule, Node, _Index) when is_atom(Rule) -> - Node. - -escape_quotes(String) -> - {ok, RE} = re:compile("\""), - re:replace(String, RE, "\\\\\"", [global, {return, list}]). - -add_lhs(Symbol, Index) -> - case ets:lookup(peg_meta, lhs) of - [] -> - ets:insert(peg_meta, {lhs, [{Symbol,Index}]}); - [{lhs, L}] when is_list(L) -> - ets:insert(peg_meta, {lhs, [{Symbol,Index}|L]}) - end. - -add_nt(Symbol, Index) -> - case ets:lookup(peg_meta, nts) of - [] -> - ets:insert(peg_meta, {nts, [{Symbol,Index}]}); - [{nts, L}] when is_list(L) -> - case proplists:is_defined(Symbol, L) of - true -> - ok; - _ -> - ets:insert(peg_meta, {nts, [{Symbol,Index}|L]}) - end - end. - -verify_rules() -> - [{lhs, LHS}] = ets:lookup(peg_meta, lhs), - [{nts, NTs}] = ets:lookup(peg_meta, nts), - [Root|NonRoots] = lists:reverse(LHS), - lists:foreach(fun({Sym,Idx}) -> - case proplists:is_defined(Sym, NTs) of - true -> - ok; - _ -> - io:format("neotoma warning: rule '~s' is unused. ~p~n", [Sym,Idx]) - end - end, NonRoots), - lists:foreach(fun({S,I}) -> - case proplists:is_defined(S, LHS) of - true -> - ok; - _ -> - io:format("neotoma error: nonterminal '~s' has no reduction. (found at ~p) No parser will be generated!~n", [S,I]), - exit({neotoma, {no_reduction, list_to_atom(S)}}) - end - end, NTs), - Root. diff --git a/tests/test_combinators.erl b/tests/test_combinators.erl index aed40ae..c83e828 100644 --- a/tests/test_combinators.erl +++ b/tests/test_combinators.erl @@ -2,81 +2,81 @@ -author("Sean Cribbs "). -include_lib("eunit/include/eunit.hrl"). -% Test the parser-combinators in the 'peg' module +% Test the parser-combinators in the 'neotoma_peg' module -define(STARTINDEX, {{line,1},{column,1}}). eof_test_() -> [ - ?_assertEqual({fail,{expected,eof,?STARTINDEX}}, (peg:p_eof())("abc",?STARTINDEX)), - ?_assertEqual({eof, [], ?STARTINDEX}, (peg:p_eof())("",?STARTINDEX)) + ?_assertEqual({fail,{expected,eof,?STARTINDEX}}, (neotoma_peg:p_eof())("abc",?STARTINDEX)), + ?_assertEqual({eof, [], ?STARTINDEX}, (neotoma_peg:p_eof())("",?STARTINDEX)) ]. optional_test_() -> [ - ?_assertEqual({[], "xyz",?STARTINDEX}, (peg:p_optional(peg:p_string("abc")))("xyz",?STARTINDEX)), - ?_assertEqual({"abc", "xyz",{{line,1},{column,4}}}, (peg:p_optional(peg:p_string("abc")))("abcxyz",?STARTINDEX)) + ?_assertEqual({[], "xyz",?STARTINDEX}, (neotoma_peg:p_optional(neotoma_peg:p_string("abc")))("xyz",?STARTINDEX)), + ?_assertEqual({"abc", "xyz",{{line,1},{column,4}}}, (neotoma_peg:p_optional(neotoma_peg:p_string("abc")))("abcxyz",?STARTINDEX)) ]. not_test_() -> [ - ?_assertEqual({[], "xyzabc",?STARTINDEX}, (peg:p_not(peg:p_string("abc")))("xyzabc",?STARTINDEX)), - ?_assertEqual({fail,{expected, {no_match, "abc"}, ?STARTINDEX}}, (peg:p_not(peg:p_string("abc")))("abcxyz",?STARTINDEX)) + ?_assertEqual({[], "xyzabc",?STARTINDEX}, (neotoma_peg:p_not(neotoma_peg:p_string("abc")))("xyzabc",?STARTINDEX)), + ?_assertEqual({fail,{expected, {no_match, "abc"}, ?STARTINDEX}}, (neotoma_peg:p_not(neotoma_peg:p_string("abc")))("abcxyz",?STARTINDEX)) ]. assert_test_() -> [ - ?_assertEqual({fail,{expected, {string, "abc"}, ?STARTINDEX}}, (peg:p_assert(peg:p_string("abc")))("xyzabc",?STARTINDEX)), - ?_assertEqual({[], "abcxyz",?STARTINDEX}, (peg:p_assert(peg:p_string("abc")))("abcxyz",?STARTINDEX)) + ?_assertEqual({fail,{expected, {string, "abc"}, ?STARTINDEX}}, (neotoma_peg:p_assert(neotoma_peg:p_string("abc")))("xyzabc",?STARTINDEX)), + ?_assertEqual({[], "abcxyz",?STARTINDEX}, (neotoma_peg:p_assert(neotoma_peg:p_string("abc")))("abcxyz",?STARTINDEX)) ]. seq_test_() -> [ - ?_assertEqual({["abc","def"], "xyz",{{line,1},{column,7}}}, (peg:p_seq([peg:p_string("abc"), peg:p_string("def")]))("abcdefxyz",?STARTINDEX)), - ?_assertEqual({fail,{expected, {string, "def"}, {{line,1},{column,4}}}}, (peg:p_seq([peg:p_string("abc"), peg:p_string("def")]))("abcxyz",?STARTINDEX)) + ?_assertEqual({["abc","def"], "xyz",{{line,1},{column,7}}}, (neotoma_peg:p_seq([neotoma_peg:p_string("abc"), neotoma_peg:p_string("def")]))("abcdefxyz",?STARTINDEX)), + ?_assertEqual({fail,{expected, {string, "def"}, {{line,1},{column,4}}}}, (neotoma_peg:p_seq([neotoma_peg:p_string("abc"), neotoma_peg:p_string("def")]))("abcxyz",?STARTINDEX)) ]. choose_test_() -> [ - ?_assertEqual({"abc", "xyz", {{line,1},{column,4}}}, (peg:p_choose([peg:p_string("abc"), peg:p_string("def")]))("abcxyz",?STARTINDEX)), - ?_assertEqual({"def", "xyz", {{line,1},{column,4}}}, (peg:p_choose([peg:p_string("abc"), peg:p_string("def")]))("defxyz",?STARTINDEX)), - ?_assertEqual({"xyz", "xyz", {{line,1},{column,4}}}, (peg:p_choose([peg:p_string("abc"), peg:p_string("def"), peg:p_string("xyz")]))("xyzxyz",?STARTINDEX)), - ?_assertEqual({fail,{expected,{string,"abc"},?STARTINDEX}}, (peg:p_choose([peg:p_string("abc"),peg:p_string("def")]))("xyz", ?STARTINDEX)) + ?_assertEqual({"abc", "xyz", {{line,1},{column,4}}}, (neotoma_peg:p_choose([neotoma_peg:p_string("abc"), neotoma_peg:p_string("def")]))("abcxyz",?STARTINDEX)), + ?_assertEqual({"def", "xyz", {{line,1},{column,4}}}, (neotoma_peg:p_choose([neotoma_peg:p_string("abc"), neotoma_peg:p_string("def")]))("defxyz",?STARTINDEX)), + ?_assertEqual({"xyz", "xyz", {{line,1},{column,4}}}, (neotoma_peg:p_choose([neotoma_peg:p_string("abc"), neotoma_peg:p_string("def"), neotoma_peg:p_string("xyz")]))("xyzxyz",?STARTINDEX)), + ?_assertEqual({fail,{expected,{string,"abc"},?STARTINDEX}}, (neotoma_peg:p_choose([neotoma_peg:p_string("abc"),neotoma_peg:p_string("def")]))("xyz", ?STARTINDEX)) ]. zero_or_more_test_() -> [ - ?_assertEqual({[], [], ?STARTINDEX}, (peg:p_zero_or_more(peg:p_string("abc")))("",?STARTINDEX)), - ?_assertEqual({[], "def",?STARTINDEX}, (peg:p_zero_or_more(peg:p_string("abc")))("def",?STARTINDEX)), - ?_assertEqual({["abc"], "def",{{line,1},{column,4}}}, (peg:p_zero_or_more(peg:p_string("abc")))("abcdef",?STARTINDEX)), - ?_assertEqual({["abc", "abc"], "def",{{line,1},{column,7}}}, (peg:p_zero_or_more(peg:p_string("abc")))("abcabcdef",?STARTINDEX)) + ?_assertEqual({[], [], ?STARTINDEX}, (neotoma_peg:p_zero_or_more(neotoma_peg:p_string("abc")))("",?STARTINDEX)), + ?_assertEqual({[], "def",?STARTINDEX}, (neotoma_peg:p_zero_or_more(neotoma_peg:p_string("abc")))("def",?STARTINDEX)), + ?_assertEqual({["abc"], "def",{{line,1},{column,4}}}, (neotoma_peg:p_zero_or_more(neotoma_peg:p_string("abc")))("abcdef",?STARTINDEX)), + ?_assertEqual({["abc", "abc"], "def",{{line,1},{column,7}}}, (neotoma_peg:p_zero_or_more(neotoma_peg:p_string("abc")))("abcabcdef",?STARTINDEX)) ]. one_or_more_test_() -> [ - ?_assertEqual({fail,{expected, {at_least_one, {string, "abc"}}, ?STARTINDEX}}, (peg:p_one_or_more(peg:p_string("abc")))("def",?STARTINDEX)), - ?_assertEqual({["abc"], "def",{{line,1},{column,4}}}, (peg:p_one_or_more(peg:p_string("abc")))("abcdef",?STARTINDEX)), - ?_assertEqual({["abc","abc"], "def",{{line,1},{column,7}}}, (peg:p_one_or_more(peg:p_string("abc")))("abcabcdef",?STARTINDEX)) + ?_assertEqual({fail,{expected, {at_least_one, {string, "abc"}}, ?STARTINDEX}}, (neotoma_peg:p_one_or_more(neotoma_peg:p_string("abc")))("def",?STARTINDEX)), + ?_assertEqual({["abc"], "def",{{line,1},{column,4}}}, (neotoma_peg:p_one_or_more(neotoma_peg:p_string("abc")))("abcdef",?STARTINDEX)), + ?_assertEqual({["abc","abc"], "def",{{line,1},{column,7}}}, (neotoma_peg:p_one_or_more(neotoma_peg:p_string("abc")))("abcabcdef",?STARTINDEX)) ]. label_test_() -> [ - ?_assertEqual({fail,{expected, {string, "!"}, ?STARTINDEX}}, (peg:p_label(bang, peg:p_string("!")))("?",?STARTINDEX)), - ?_assertEqual({{bang, "!"}, "",{{line,1},{column,2}}}, (peg:p_label(bang, peg:p_string("!")))("!",?STARTINDEX)) + ?_assertEqual({fail,{expected, {string, "!"}, ?STARTINDEX}}, (neotoma_peg:p_label(bang, neotoma_peg:p_string("!")))("?",?STARTINDEX)), + ?_assertEqual({{bang, "!"}, "",{{line,1},{column,2}}}, (neotoma_peg:p_label(bang, neotoma_peg:p_string("!")))("!",?STARTINDEX)) ]. string_test_() -> [ - ?_assertEqual({"abc", "def",{{line,1},{column,4}}}, (peg:p_string("abc"))("abcdef",?STARTINDEX)), - ?_assertEqual({fail,{expected, {string, "abc"}, ?STARTINDEX}}, (peg:p_string("abc"))("defabc",?STARTINDEX)) + ?_assertEqual({"abc", "def",{{line,1},{column,4}}}, (neotoma_peg:p_string("abc"))("abcdef",?STARTINDEX)), + ?_assertEqual({fail,{expected, {string, "abc"}, ?STARTINDEX}}, (neotoma_peg:p_string("abc"))("defabc",?STARTINDEX)) ]. anything_test_() -> [ - ?_assertEqual({$a,"bcde",{{line,1},{column,2}}}, (peg:p_anything())("abcde",?STARTINDEX)), - ?_assertEqual({fail,{expected, any_character, ?STARTINDEX}}, (peg:p_anything())("",?STARTINDEX)) + ?_assertEqual({$a,"bcde",{{line,1},{column,2}}}, (neotoma_peg:p_anything())("abcde",?STARTINDEX)), + ?_assertEqual({fail,{expected, any_character, ?STARTINDEX}}, (neotoma_peg:p_anything())("",?STARTINDEX)) ]. charclass_test_() -> [ - ?_assertEqual({$+,"----",{{line,1},{column,2}}}, (peg:p_charclass("[+]"))("+----",?STARTINDEX)), - ?_assertEqual({fail,{expected, {character_class, "[+]"}, ?STARTINDEX}}, (peg:p_charclass("[+]"))("----",?STARTINDEX)) + ?_assertEqual({$+,"----",{{line,1},{column,2}}}, (neotoma_peg:p_charclass("[+]"))("+----",?STARTINDEX)), + ?_assertEqual({fail,{expected, {character_class, "[+]"}, ?STARTINDEX}}, (neotoma_peg:p_charclass("[+]"))("----",?STARTINDEX)) ]. diff --git a/tests/test_memoization.erl b/tests/test_memoization.erl index 79e19b1..8e5df8e 100644 --- a/tests/test_memoization.erl +++ b/tests/test_memoization.erl @@ -3,20 +3,20 @@ -include_lib("eunit/include/eunit.hrl"). setup_memo_test() -> - peg:setup_memo(), - ?assertNot(undefined == ets:info(peg)), - peg:release_memo(). + neotoma_peg:setup_memo(), + ?assertNot(undefined == ets:info(neotoma_peg)), + neotoma_peg:release_memo(). release_memo_test() -> - peg:setup_memo(), - peg:release_memo(), - ?assertEqual(undefined, ets:info(peg)). + neotoma_peg:setup_memo(), + neotoma_peg:release_memo(), + ?assertEqual(undefined, ets:info(neotoma_peg)). step_memo_test() -> - peg:setup_memo(), - Result = peg:p("abcdefghi", {{line,1},{column,1}}, anything, peg:p_anything()), + neotoma_peg:setup_memo(), + Result = neotoma_peg:p("abcdefghi", {{line,1},{column,1}}, anything, neotoma_peg:p_anything()), ?assertEqual({$a, "bcdefghi", {{line,1},{column,2}}}, Result), - Result2 = peg:p("abcdefghi", {{line,1},{column,1}}, anything, fun(_) -> + Result2 = neotoma_peg:p("abcdefghi", {{line,1},{column,1}}, anything, fun(_) -> throw(bork) end), ?assertEqual(Result, Result2), - peg:release_memo(). + neotoma_peg:release_memo().