priv/peg_includes.erl @@ -1,3 +1,4 @@ ebin/* ebin_tests/* -extra/*.beam \ No newline at end of file +extra/*.beam +*#* \ No newline at end of file .gitignore @@ -10,26 +10,24 @@ tests: clean src_src src_tests ebin: mkdir ebin +priv: + mkdir priv + ebin_tests: mkdir ebin_tests -ebin_tests/examples: - mkdir ebin_tests/examples - -special: src/herml_scan.erl src/herml_parse.erl +priv/peg_includes.erl: priv src/peg.erl + cat src/peg.erl | grep -v "^%" | grep -v "^-" > priv/peg_includes.erl -src_src: ebin src/neotoma.app +src_src: ebin src/neotoma.app priv/peg_includes.erl cd src;erl -pz ../ebin -make -src_tests: ebin_tests test_examples +src_tests: ebin_tests cd tests;erl -pz ../ebin -make src/neotoma.app: ebin cp src/neotoma.app ebin -test_examples: ebin_tests/examples - cd tests/examples;erl -pz ../../ebin -make - clean: rm -rf ebin rm -rf ebin_tests Makefile @@ -1,23 +1,205 @@ -module(csv). -export([parse/1,file/1]). --include_lib("neotoma/include/peg.hrl"). +-compile(nowarn_unused_function). -rule(rows) -> - peg:choose([peg:seq([peg:label('head', fun row/2), peg:label('tail', peg:zero_or_more(peg:seq([fun crlf/2, fun row/2])))]), peg:string("")]); +file(Filename) -> {ok, Bin} = file:read_file(Filename), parse(binary_to_list(Bin)). -rule(row) -> - peg:choose([peg:seq([peg:label('head', fun field/2), peg:label('tail', peg:zero_or_more(peg:seq([fun field_sep/2, fun field/2])))]), peg:string("")]); +parse(Input) -> + setup_memo(csv), + Result = case rows(Input,{{line,1},{column,1}}) of + {AST, [], _Index} -> AST; + Any -> Any + end, + release_memo(), Result. -rule(field) -> - peg:choose([fun quoted_field/2, peg:zero_or_more(peg:seq([peg:not_(peg:choose([fun field_sep/2, fun crlf/2])), peg:anything()]))]); +'rows'(Input, Index) -> + p(Input, Index, 'rows', fun(I,D) -> (p_choose([p_seq([p_label('head', fun 'row'/2), p_label('tail', p_zero_or_more(p_seq([fun 'crlf'/2, fun 'row'/2])))]), p_string("")]))(I,D) end, fun(Node, Idx) -> transform('rows', Node, Idx) end). -rule(quoted_field) -> - peg:seq([peg:string("\""), peg:label('string', peg:zero_or_more(peg:choose([peg:string("\"\""), peg:seq([peg:not_(peg:string("\"")), peg:anything()])]))), peg:string("\"")]); +'row'(Input, Index) -> + p(Input, Index, 'row', fun(I,D) -> (p_choose([p_seq([p_label('head', fun 'field'/2), p_label('tail', p_zero_or_more(p_seq([fun 'field_sep'/2, fun 'field'/2])))]), p_string("")]))(I,D) end, fun(Node, Idx) -> transform('row', Node, Idx) end). -rule(field_sep) -> - peg:string(","); +'field'(Input, Index) -> + p(Input, Index, 'field', fun(I,D) -> (p_choose([fun 'quoted_field'/2, p_zero_or_more(p_seq([p_not(p_choose([fun 'field_sep'/2, fun 'crlf'/2])), p_anything()]))]))(I,D) end, fun(Node, Idx) -> transform('field', Node, Idx) end). -rule(crlf) -> - peg:choose([peg:string("\r\n"), peg:string("\n")]). +'quoted_field'(Input, Index) -> + p(Input, Index, 'quoted_field', fun(I,D) -> (p_seq([p_string("\""), p_label('string', p_zero_or_more(p_choose([p_string("\"\""), p_seq([p_not(p_string("\"")), p_anything()])]))), p_string("\"")]))(I,D) end, fun(Node, Idx) -> transform('quoted_field', Node, Idx) end). -transform(Symbol,Node,Index) -> csv_gen:transform(Symbol, Node). +'field_sep'(Input, Index) -> + p(Input, Index, 'field_sep', fun(I,D) -> (p_string(","))(I,D) end, fun(Node, Idx) -> transform('field_sep', Node, Idx) end). + +'crlf'(Input, Index) -> + p(Input, Index, 'crlf', fun(I,D) -> (p_choose([p_string("\r\n"), p_string("\n")]))(I,D) end, fun(Node, Idx) -> transform('crlf', Node, Idx) end). + +transform(Symbol,Node,Index) -> csv_gen:transform(Symbol, Node, Index). + + + + + +p(Inp, Index, Name, ParseFun) -> + p(Inp, Index, Name, ParseFun, fun(N, _Idx) -> N end). + +p(Inp, StartIndex, Name, ParseFun, TransformFun) -> + % Grab the memo table from ets + Memo = get_memo(StartIndex), + % See if the current reduction is memoized + case dict:find(Name, Memo) of + % If it is, return the result + {ok, Result} -> Result; + % If not, attempt to parse + _ -> + case ParseFun(Inp, StartIndex) of + % If it fails, memoize the failure + {fail,_} = Failure -> + memoize(StartIndex, dict:store(Name, Failure, Memo)), + Failure; + % If it passes, transform and memoize the result. + {Result, InpRem, NewIndex} -> + Transformed = TransformFun(Result, StartIndex), + memoize(StartIndex, dict:store(Name, {Transformed, InpRem, NewIndex}, Memo)), + {Transformed, InpRem, NewIndex} + end + end. + +setup_memo(Name) -> + TID = ets:new(Name, [set]), + put(ets_table, TID). + +release_memo() -> + ets:delete(get(ets_table)), + erase(ets_table). + +memoize(Position, Struct) -> + ets:insert(get(ets_table), {Position, Struct}). + +get_memo(Position) -> + case ets:lookup(get(ets_table), Position) of + [] -> dict:new(); + [{Position, Dict}] -> Dict + end. + +p_eof() -> + fun([], Index) -> {eof, [], Index}; + (_, Index) -> {fail, {expected, eof, Index}} end. + +p_optional(P) -> + fun(Input, Index) -> + case P(Input, Index) of + {fail,_} -> {[], Input, Index}; + {_, _, _} = Success -> Success + end + end. + +p_not(P) -> + fun(Input, Index)-> + case P(Input,Index) of + {fail,_} -> + {[], Input, Index}; + {Result, _, _} -> {fail, {expected, {no_match, Result},Index}} + end + end. + +p_assert(P) -> + fun(Input,Index) -> + case P(Input,Index) of + {fail,_} = Failure-> Failure; + _ -> {[], Input, Index} + end + end. + +p_and(P) -> + p_seq(P). + +p_seq(P) -> + fun(Input, Index) -> + p_all(P, Input, Index, []) + end. + +p_all([], Inp, Index, Accum ) -> {lists:reverse( Accum ), Inp, Index}; +p_all([P|Parsers], Inp, Index, Accum) -> + case P(Inp, Index) of + {fail, _} = Failure -> Failure; + {Result, InpRem, NewIndex} -> p_all(Parsers, InpRem, NewIndex, [Result|Accum]) + end. + +p_choose(Parsers) -> + fun(Input, Index) -> + p_attempt(Parsers, Input, Index, none) + end. + +p_attempt([], _Input, _Index, Failure) -> Failure; +p_attempt([P|Parsers], Input, Index, FirstFailure)-> + case P(Input, Index) of + {fail, _} = Failure -> + case FirstFailure of + none -> p_attempt(Parsers, Input, Index, Failure); + _ -> p_attempt(Parsers, Input, Index, FirstFailure) + end; + Result -> Result + end. + +p_zero_or_more(P) -> + fun(Input, Index) -> + p_scan(P, Input, Index, []) + end. + +p_one_or_more(P) -> + fun(Input, Index)-> + Result = p_scan(P, Input, Index, []), + case Result of + {[_|_], _, _} -> + Result; + _ -> + {fail, {expected, Failure, _}} = P(Input,Index), + {fail, {expected, {at_least_one, Failure}, Index}} + end + end. + +p_label(Tag, P) -> + fun(Input, Index) -> + case P(Input, Index) of + {fail,_} = Failure -> + Failure; + {Result, InpRem, NewIndex} -> + {{Tag, Result}, InpRem, NewIndex} + end + end. + +p_scan(_, [], Index, Accum) -> {lists:reverse( Accum ), [], Index}; +p_scan(P, Inp, Index, Accum) -> + case P(Inp, Index) of + {fail,_} -> {lists:reverse(Accum), Inp, Index}; + {Result, InpRem, NewIndex} -> p_scan(P, InpRem, NewIndex, [Result | Accum]) + end. + +p_string(S) -> + fun(Input, Index) -> + case lists:prefix(S, Input) of + true -> {S, lists:sublist(Input, length(S)+1, length(Input)), p_advance_index(S,Index)}; + _ -> {fail, {expected, {string, S}, Index}} + end + end. + +p_anything() -> + fun([], Index) -> {fail, {expected, any_character, Index}}; + ([H|T], Index) -> {H, T, p_advance_index(H, Index)} + end. + +p_charclass(Class) -> + fun(Inp, Index) -> + {ok, RE} = re:compile("^"++Class), + case re:run(Inp, RE) of + {match, _} -> + {hd(Inp), tl(Inp), p_advance_index(hd(Inp), Index)}; + _ -> {fail,{expected, {character_class, Class}, Index}} + end + end. + +p_advance_index(MatchedInput, Index) when is_list(MatchedInput) -> % strings + lists:foldl(fun p_advance_index/2, Index, MatchedInput); +p_advance_index(MatchedInput, Index) when is_integer(MatchedInput) -> % single characters + {{line, Line}, {column, Col}} = Index, + case MatchedInput of + $\n -> {{line, Line+1}, {column, 1}}; + _ -> {{line, Line}, {column, Col+1}} + end. extra/csv.erl @@ -1,53 +1,235 @@ -module(json). -export([parse/1,file/1]). --include_lib("neotoma/include/peg.hrl"). +-compile(nowarn_unused_function). -rule(json_value) -> - peg:seq([peg:optional(fun space/2), peg:choose([fun object/2, fun array/2, fun string/2, fun number/2, fun true/2, fun false/2, fun null/2]), peg:optional(fun space/2)]); +file(Filename) -> {ok, Bin} = file:read_file(Filename), parse(binary_to_list(Bin)). -rule(object) -> - peg:choose([peg:seq([peg:string("{"), peg:optional(fun space/2), peg:label('head', fun pair/2), peg:label('tail', peg:zero_or_more(peg:seq([peg:optional(fun space/2), peg:string(","), peg:optional(fun space/2), fun pair/2]))), peg:optional(fun space/2), peg:string("}")]), peg:seq([peg:string("{"), peg:optional(fun space/2), peg:string("}")])]); +parse(Input) -> + setup_memo(json), + Result = case json_value(Input,{{line,1},{column,1}}) of + {AST, [], _Index} -> AST; + Any -> Any + end, + release_memo(), Result. -rule(pair) -> - peg:seq([peg:optional(fun space/2), peg:label('key', fun string/2), peg:optional(fun space/2), peg:string(":"), peg:optional(fun space/2), peg:label('value', fun json_value/2), peg:optional(fun space/2)]); +'json_value'(Input, Index) -> + p(Input, Index, 'json_value', fun(I,D) -> (p_seq([p_optional(fun 'space'/2), p_choose([fun 'object'/2, fun 'array'/2, fun 'string'/2, fun 'number'/2, fun 'true'/2, fun 'false'/2, fun 'null'/2]), p_optional(fun 'space'/2)]))(I,D) end, fun(Node, Idx) -> transform('json_value', Node, Idx) end). -rule(array) -> - peg:choose([peg:seq([peg:string("["), peg:optional(fun space/2), peg:label('head', fun json_value/2), peg:label('tail', peg:zero_or_more(peg:seq([peg:optional(fun space/2), peg:string(","), peg:optional(fun space/2), fun json_value/2]))), peg:optional(fun space/2), peg:string("]")]), peg:seq([peg:string("["), peg:optional(fun space/2), peg:string("]")])]); +'object'(Input, Index) -> + p(Input, Index, 'object', fun(I,D) -> (p_choose([p_seq([p_string("{"), p_optional(fun 'space'/2), p_label('head', fun 'pair'/2), p_label('tail', p_zero_or_more(p_seq([p_optional(fun 'space'/2), p_string(","), p_optional(fun 'space'/2), fun 'pair'/2]))), p_optional(fun 'space'/2), p_string("}")]), p_seq([p_string("{"), p_optional(fun 'space'/2), p_string("}")])]))(I,D) end, fun(Node, Idx) -> transform('object', Node, Idx) end). -rule(string) -> - peg:seq([peg:string("\""), peg:label('chars', peg:zero_or_more(peg:seq([peg:not_(peg:string("\"")), peg:choose([peg:string("\\\\"), peg:string("\\\""), peg:anything()])]))), peg:string("\"")]); +'pair'(Input, Index) -> + p(Input, Index, 'pair', fun(I,D) -> (p_seq([p_optional(fun 'space'/2), p_label('key', fun 'string'/2), p_optional(fun 'space'/2), p_string(":"), p_optional(fun 'space'/2), p_label('value', fun 'json_value'/2), p_optional(fun 'space'/2)]))(I,D) end, fun(Node, Idx) -> transform('pair', Node, Idx) end). -rule(number) -> - peg:seq([fun int/2, peg:optional(fun frac/2), peg:optional(fun exp/2)]); +'array'(Input, Index) -> + p(Input, Index, 'array', fun(I,D) -> (p_choose([p_seq([p_string("["), p_optional(fun 'space'/2), p_label('head', fun 'json_value'/2), p_label('tail', p_zero_or_more(p_seq([p_optional(fun 'space'/2), p_string(","), p_optional(fun 'space'/2), fun 'json_value'/2]))), p_optional(fun 'space'/2), p_string("]")]), p_seq([p_string("["), p_optional(fun 'space'/2), p_string("]")])]))(I,D) end, fun(Node, Idx) -> transform('array', Node, Idx) end). -rule(int) -> - peg:choose([peg:seq([peg:optional(peg:string("-")), peg:seq([fun non_zero_digit/2, peg:one_or_more(fun digit/2)])]), fun digit/2]); +'string'(Input, Index) -> + p(Input, Index, 'string', fun(I,D) -> (p_seq([p_string("\""), p_label('chars', p_zero_or_more(p_seq([p_not(p_string("\"")), p_choose([p_string("\\\\"), p_string("\\\""), p_anything()])]))), p_string("\"")]))(I,D) end, fun(Node, Idx) -> transform('string', Node, Idx) end). -rule(frac) -> - peg:seq([peg:string("."), peg:one_or_more(fun digit/2)]); +'number'(Input, Index) -> + p(Input, Index, 'number', fun(I,D) -> (p_seq([fun 'int'/2, p_optional(fun 'frac'/2), p_optional(fun 'exp'/2)]))(I,D) end, fun(Node, Idx) -> transform('number', Node, Idx) end). -rule(exp) -> - peg:seq([fun e/2, peg:one_or_more(fun digit/2)]); +'int'(Input, Index) -> + p(Input, Index, 'int', fun(I,D) -> (p_choose([p_seq([p_optional(p_string("-")), p_seq([fun 'non_zero_digit'/2, p_one_or_more(fun 'digit'/2)])]), fun 'digit'/2]))(I,D) end, fun(Node, Idx) -> transform('int', Node, Idx) end). -rule(e) -> - peg:seq([peg:charclass("[eE]"), peg:optional(peg:choose([peg:string("+"), peg:string("-")]))]); +'frac'(Input, Index) -> + p(Input, Index, 'frac', fun(I,D) -> (p_seq([p_string("."), p_one_or_more(fun 'digit'/2)]))(I,D) end, fun(Node, Idx) -> transform('frac', Node, Idx) end). + +'exp'(Input, Index) -> + p(Input, Index, 'exp', fun(I,D) -> (p_seq([fun 'e'/2, p_one_or_more(fun 'digit'/2)]))(I,D) end, fun(Node, Idx) -> transform('exp', Node, Idx) end). -rule(non_zero_digit) -> - peg:charclass("[1-9]"); +'e'(Input, Index) -> + p(Input, Index, 'e', fun(I,D) -> (p_seq([p_charclass("[eE]"), p_optional(p_choose([p_string("+"), p_string("-")]))]))(I,D) end, fun(Node, Idx) -> transform('e', Node, Idx) end). -rule(digit) -> - peg:charclass("[0-9]"); +'non_zero_digit'(Input, Index) -> + p(Input, Index, 'non_zero_digit', fun(I,D) -> (p_charclass("[1-9]"))(I,D) end, fun(Node, Idx) -> transform('non_zero_digit', Node, Idx) end). -rule(true) -> - peg:string("true"); - -rule(false) -> - peg:string("false"); - -rule(null) -> - peg:string("null"); - -rule(space) -> - peg:zero_or_more(peg:charclass("[ \t\n\s\r]")). - -transform(Symbol,Node) -> json_tree:transform(Symbol, Node). \ No newline at end of file +'digit'(Input, Index) -> + p(Input, Index, 'digit', fun(I,D) -> (p_charclass("[0-9]"))(I,D) end, fun(Node, Idx) -> transform('digit', Node, Idx) end). + +'true'(Input, Index) -> + p(Input, Index, 'true', fun(I,D) -> (p_string("true"))(I,D) end, fun(Node, Idx) -> transform('true', Node, Idx) end). + +'false'(Input, Index) -> + p(Input, Index, 'false', fun(I,D) -> (p_string("false"))(I,D) end, fun(Node, Idx) -> transform('false', Node, Idx) end). + +'null'(Input, Index) -> + p(Input, Index, 'null', fun(I,D) -> (p_string("null"))(I,D) end, fun(Node, Idx) -> transform('null', Node, Idx) end). + +'space'(Input, Index) -> + p(Input, Index, 'space', fun(I,D) -> (p_zero_or_more(p_charclass("[ \t\n\s\r]")))(I,D) end, fun(Node, Idx) -> transform('space', Node, Idx) end). + +transform(Symbol,Node,Index) -> json_tree:transform(Symbol, Node, Index). + + + + + +p(Inp, Index, Name, ParseFun) -> + p(Inp, Index, Name, ParseFun, fun(N, _Idx) -> N end). + +p(Inp, StartIndex, Name, ParseFun, TransformFun) -> + % Grab the memo table from ets + Memo = get_memo(StartIndex), + % See if the current reduction is memoized + case dict:find(Name, Memo) of + % If it is, return the result + {ok, Result} -> Result; + % If not, attempt to parse + _ -> + case ParseFun(Inp, StartIndex) of + % If it fails, memoize the failure + {fail,_} = Failure -> + memoize(StartIndex, dict:store(Name, Failure, Memo)), + Failure; + % If it passes, transform and memoize the result. + {Result, InpRem, NewIndex} -> + Transformed = TransformFun(Result, StartIndex), + memoize(StartIndex, dict:store(Name, {Transformed, InpRem, NewIndex}, Memo)), + {Transformed, InpRem, NewIndex} + end + end. + +setup_memo(Name) -> + TID = ets:new(Name, [set]), + put(ets_table, TID). + +release_memo() -> + ets:delete(get(ets_table)), + erase(ets_table). + +memoize(Position, Struct) -> + ets:insert(get(ets_table), {Position, Struct}). + +get_memo(Position) -> + case ets:lookup(get(ets_table), Position) of + [] -> dict:new(); + [{Position, Dict}] -> Dict + end. + +p_eof() -> + fun([], Index) -> {eof, [], Index}; + (_, Index) -> {fail, {expected, eof, Index}} end. + +p_optional(P) -> + fun(Input, Index) -> + case P(Input, Index) of + {fail,_} -> {[], Input, Index}; + {_, _, _} = Success -> Success + end + end. + +p_not(P) -> + fun(Input, Index)-> + case P(Input,Index) of + {fail,_} -> + {[], Input, Index}; + {Result, _, _} -> {fail, {expected, {no_match, Result},Index}} + end + end. + +p_assert(P) -> + fun(Input,Index) -> + case P(Input,Index) of + {fail,_} = Failure-> Failure; + _ -> {[], Input, Index} + end + end. + +p_and(P) -> + p_seq(P). + +p_seq(P) -> + fun(Input, Index) -> + p_all(P, Input, Index, []) + end. + +p_all([], Inp, Index, Accum ) -> {lists:reverse( Accum ), Inp, Index}; +p_all([P|Parsers], Inp, Index, Accum) -> + case P(Inp, Index) of + {fail, _} = Failure -> Failure; + {Result, InpRem, NewIndex} -> p_all(Parsers, InpRem, NewIndex, [Result|Accum]) + end. + +p_choose(Parsers) -> + fun(Input, Index) -> + p_attempt(Parsers, Input, Index, none) + end. + +p_attempt([], _Input, _Index, Failure) -> Failure; +p_attempt([P|Parsers], Input, Index, FirstFailure)-> + case P(Input, Index) of + {fail, _} = Failure -> + case FirstFailure of + none -> p_attempt(Parsers, Input, Index, Failure); + _ -> p_attempt(Parsers, Input, Index, FirstFailure) + end; + Result -> Result + end. + +p_zero_or_more(P) -> + fun(Input, Index) -> + p_scan(P, Input, Index, []) + end. + +p_one_or_more(P) -> + fun(Input, Index)-> + Result = p_scan(P, Input, Index, []), + case Result of + {[_|_], _, _} -> + Result; + _ -> + {fail, {expected, Failure, _}} = P(Input,Index), + {fail, {expected, {at_least_one, Failure}, Index}} + end + end. + +p_label(Tag, P) -> + fun(Input, Index) -> + case P(Input, Index) of + {fail,_} = Failure -> + Failure; + {Result, InpRem, NewIndex} -> + {{Tag, Result}, InpRem, NewIndex} + end + end. + +p_scan(_, [], Index, Accum) -> {lists:reverse( Accum ), [], Index}; +p_scan(P, Inp, Index, Accum) -> + case P(Inp, Index) of + {fail,_} -> {lists:reverse(Accum), Inp, Index}; + {Result, InpRem, NewIndex} -> p_scan(P, InpRem, NewIndex, [Result | Accum]) + end. + +p_string(S) -> + fun(Input, Index) -> + case lists:prefix(S, Input) of + true -> {S, lists:sublist(Input, length(S)+1, length(Input)), p_advance_index(S,Index)}; + _ -> {fail, {expected, {string, S}, Index}} + end + end. + +p_anything() -> + fun([], Index) -> {fail, {expected, any_character, Index}}; + ([H|T], Index) -> {H, T, p_advance_index(H, Index)} + end. + +p_charclass(Class) -> + fun(Inp, Index) -> + {ok, RE} = re:compile("^"++Class), + case re:run(Inp, RE) of + {match, _} -> + {hd(Inp), tl(Inp), p_advance_index(hd(Inp), Index)}; + _ -> {fail,{expected, {character_class, Class}, Index}} + end + end. + +p_advance_index(MatchedInput, Index) when is_list(MatchedInput) -> % strings + lists:foldl(fun p_advance_index/2, Index, MatchedInput); +p_advance_index(MatchedInput, Index) when is_integer(MatchedInput) -> % single characters + {{line, Line}, {column, Col}} = Index, + case MatchedInput of + $\n -> {{line, Line+1}, {column, 1}}; + _ -> {{line, Line}, {column, Col+1}} + end. extra/json.erl @@ -18,7 +18,7 @@ transform(array, Node, _Idx) -> Rest = [lists:nth(4, I) || I <- proplists:get_value(tail, Node)], [Head|Rest]; transform(number, [Int, [],[]], _Idx) -> - list_to_integer(lists:flatten(Int)); + list_to_integer(lists:flatten([Int])); transform(number, [Int, Frac, []], _Idx) -> list_to_float(lists:flatten([Int, Frac])); transform(number, [Int, [], Exp], Idx) -> extra/json_tree.erl @@ -1,2 +1,2 @@ -{[peg_transform, peg_itransform, peg, peg_gen, peg_meta, peg_meta_gen], +{[peg, peg_gen, peg_meta, peg_meta_gen], [{outdir, "../ebin"}, {i, "../../"}, debug_info]}. src/Emakefile @@ -13,13 +13,7 @@ -export([p/4, p/5]). -export([setup_memo/1, release_memo/0]). --export([eof/0, optional/1, - not_/1, assert/1, seq/1, - and_/1, choose/1, - zero_or_more/1, one_or_more/1, - label/2, - string/1, anything/0, - charclass/1]). +-export([p_eof/0, p_optional/1, p_not/1, p_assert/1, p_seq/1, p_and/1, p_choose/1, p_zero_or_more/1, p_one_or_more/1, p_label/2, p_string/1, p_anything/0, p_charclass/1]). %% @doc Memoizing parsing function wrapper. This form does not transform the result of a successful parse. %% @see p/5. @@ -71,14 +65,14 @@ get_memo(Position) -> end. %% @doc Generates a parse function that matches the end of the buffer. -%% @spec eof() -> parse_fun() -eof() -> +%% @spec p_eof() -> parse_fun() +p_eof() -> fun([], Index) -> {eof, [], Index}; (_, Index) -> {fail, {expected, eof, Index}} end. %% @doc Generates a parse function that treats the passed parse function as optional. -%% @spec optional(parse_fun()) -> parse_fun() -optional(P) -> +%% @spec p_optional(parse_fun()) -> parse_fun() +p_optional(P) -> fun(Input, Index) -> case P(Input, Index) of {fail,_} -> {[], Input, Index}; @@ -87,8 +81,8 @@ optional(P) -> end. %% @doc Generates a parse function that ensures the passed function will not match without consuming any input, that is, negative lookahead. -%% @spec not_(parse_fun()) -> parse_fun() -not_(P) -> +%% @spec p_not(parse_fun()) -> parse_fun() +p_not(P) -> fun(Input, Index)-> case P(Input,Index) of {fail,_} -> @@ -98,8 +92,8 @@ not_(P) -> end. %% @doc Generates a parse function that ensures the passed function will match, without consuming any input, that is, positive lookahead. -%% @spec assert(parse_fun()) -> parse_fun() -assert(P) -> +%% @spec p_assert(parse_fun()) -> parse_fun() +p_assert(P) -> fun(Input,Index) -> case P(Input,Index) of {fail,_} = Failure-> Failure; @@ -107,55 +101,55 @@ assert(P) -> end end. -%% @doc Alias for seq/1. -%% @see seq/1. -and_(P) -> - seq(P). +%% @doc Alias for p_seq/1. +%% @see p_seq/1. +p_and(P) -> + p_seq(P). %% @doc Generates a parse function that will match the passed parse functions in order. -%% @spec seq([parse_fun()]) -> parse_fun() -seq(P) -> +%% @spec p_seq([parse_fun()]) -> parse_fun() +p_seq(P) -> fun(Input, Index) -> - all(P, Input, Index, []) + p_all(P, Input, Index, []) end. -all([], Inp, Index, Accum ) -> {lists:reverse( Accum ), Inp, Index}; -all([P|Parsers], Inp, Index, Accum) -> +p_all([], Inp, Index, Accum ) -> {lists:reverse( Accum ), Inp, Index}; +p_all([P|Parsers], Inp, Index, Accum) -> case P(Inp, Index) of {fail, _} = Failure -> Failure; - {Result, InpRem, NewIndex} -> all(Parsers, InpRem, NewIndex, [Result|Accum]) + {Result, InpRem, NewIndex} -> p_all(Parsers, InpRem, NewIndex, [Result|Accum]) end. %% @doc Generates a parse function that will match at least one of the passed parse functions (ordered choice). %% @spec choose([parse_fun()]) -> parse_fun() -choose(Parsers) -> +p_choose(Parsers) -> fun(Input, Index) -> - attempt(Parsers, Input, Index, none) + p_attempt(Parsers, Input, Index, none) end. -attempt([], _Input, _Index, Failure) -> Failure; -attempt([P|Parsers], Input, Index, FirstFailure)-> +p_attempt([], _Input, _Index, Failure) -> Failure; +p_attempt([P|Parsers], Input, Index, FirstFailure)-> case P(Input, Index) of {fail, _} = Failure -> case FirstFailure of - none -> attempt(Parsers, Input, Index, Failure); - _ -> attempt(Parsers, Input, Index, FirstFailure) + none -> p_attempt(Parsers, Input, Index, Failure); + _ -> p_attempt(Parsers, Input, Index, FirstFailure) end; Result -> Result end. %% @doc Generates a parse function that will match any number of the passed parse function in sequence (optional greedy repetition). -%% @spec zero_or_more(parse_fun()) -> parse_fun() -zero_or_more(P) -> +%% @spec p_zero_or_more(parse_fun()) -> parse_fun() +p_zero_or_more(P) -> fun(Input, Index) -> - scan(P, Input, Index, []) + p_scan(P, Input, Index, []) end. %% @doc Generates a parse function that will match at least one of the passed parse function in sequence (greedy repetition). -%% @spec one_or_more(parse_fun()) -> parse_fun() -one_or_more(P) -> +%% @spec p_one_or_more(parse_fun()) -> parse_fun() +p_one_or_more(P) -> fun(Input, Index)-> - Result = scan(P, Input, Index, []), + Result = p_scan(P, Input, Index, []), case Result of {[_|_], _, _} -> Result; @@ -166,8 +160,8 @@ one_or_more(P) -> end. %% @doc Generates a parse function that will tag the result of the passed parse function with a label when it succeeds. The tagged result will be a 2-tuple of {Tag, Result}. -%% @spec label(Tag::anything(), P::parse_fun()) -> parse_fun() -label(Tag, P) -> +%% @spec p_label(Tag::anything(), P::parse_fun()) -> parse_fun() +p_label(Tag, P) -> fun(Input, Index) -> case P(Input, Index) of {fail,_} = Failure -> @@ -177,28 +171,28 @@ label(Tag, P) -> end end. -scan(_, [], Index, Accum) -> {lists:reverse( Accum ), [], Index}; -scan(P, Inp, Index, Accum) -> +p_scan(_, [], Index, Accum) -> {lists:reverse( Accum ), [], Index}; +p_scan(P, Inp, Index, Accum) -> case P(Inp, Index) of {fail,_} -> {lists:reverse(Accum), Inp, Index}; - {Result, InpRem, NewIndex} -> scan(P, InpRem, NewIndex, [Result | Accum]) + {Result, InpRem, NewIndex} -> p_scan(P, InpRem, NewIndex, [Result | Accum]) end. %% @doc Generates a parse function that will match the passed string on the head of the buffer. -%% @spec string(string()) -> parse_fun() -string(S) -> +%% @spec p_string(string()) -> parse_fun() +p_string(S) -> fun(Input, Index) -> case lists:prefix(S, Input) of - true -> {S, lists:sublist(Input, length(S)+1, length(Input)), advance_index(S,Index)}; + true -> {S, lists:sublist(Input, length(S)+1, length(Input)), p_advance_index(S,Index)}; _ -> {fail, {expected, {string, S}, Index}} end end. %% @doc Generates a parse function that will match any single character. %% @spec anything() -> parse_fun() -anything() -> +p_anything() -> fun([], Index) -> {fail, {expected, any_character, Index}}; - ([H|T], Index) -> {H, T, advance_index(H, Index)} + ([H|T], Index) -> {H, T, p_advance_index(H, Index)} end. %% @doc Generates a parse function that will match any single character from the passed "class". The class should be a PCRE-compatible character class in a string. @@ -207,19 +201,19 @@ anything() -> %% "[0-9]" %% "[^z]" . %% @spec charclass(string()) -> parse_fun() -charclass(Class) -> +p_charclass(Class) -> fun(Inp, Index) -> {ok, RE} = re:compile("^"++Class), case re:run(Inp, RE) of {match, _} -> - {hd(Inp), tl(Inp), advance_index(hd(Inp), Index)}; + {hd(Inp), tl(Inp), p_advance_index(hd(Inp), Index)}; _ -> {fail,{expected, {character_class, Class}, Index}} end end. -advance_index(MatchedInput, Index) when is_list(MatchedInput) -> % strings - lists:foldl(fun advance_index/2, Index, MatchedInput); -advance_index(MatchedInput, Index) when is_integer(MatchedInput) -> % single characters +p_advance_index(MatchedInput, Index) when is_list(MatchedInput) -> % strings + lists:foldl(fun p_advance_index/2, Index, MatchedInput); +p_advance_index(MatchedInput, Index) when is_integer(MatchedInput) -> % single characters {{line, Line}, {column, Col}} = Index, case MatchedInput of $\n -> {{line, Line+1}, {column, 1}}; src/peg.erl @@ -23,10 +23,11 @@ file(InputGrammar, Options) -> ModuleName, TransformModule, filename:absname(OutputFilename)), - ModuleAttrs = generate_module_attrs(ModuleName), Rules = parse_grammar(InputGrammar), + ModuleAttrs = generate_module_attrs(ModuleName), TransformFun = create_transform(TransformModule, OutputDir), - file:write_file(OutputFilename, [ModuleAttrs, "\n", Rules, "\n", TransformFun]). + {ok, PegIncludes} = file:read_file(code:priv_dir(neotoma) ++ "/peg_includes.erl"), + file:write_file(OutputFilename, [ModuleAttrs, "\n", Rules, "\n", TransformFun, "\n", PegIncludes]). validate_params(InputGrammar, _, _, OutputFile) when InputGrammar =:= OutputFile -> throw({badarg, "Input and output file are the same!"}); @@ -45,9 +46,22 @@ validate_params(_,_, TransformModule, OutputFile) -> end. generate_module_attrs(ModName) -> + {RootRule,_} = erase(neotoma_root_rule), ["-module(",atom_to_list(ModName),").\n", "-export([parse/1,file/1]).\n", - "-include_lib(\"neotoma/include/peg.hrl\").\n"]. + % This option could be problematic if your grammar is broken in + % some way, but hides warnings about unused parser combinators. + % In a future version we should just emit the used combinators, + % excluding the rest. + "-compile(nowarn_unused_function).\n\n", + "file(Filename) -> {ok, Bin} = file:read_file(Filename), parse(binary_to_list(Bin)).\n\n", + "parse(Input) ->\n", + " setup_memo('", atom_to_list(ModName),"'),\n", + " Result = case '",RootRule,"'(Input,{{line,1},{column,1}}) of\n", + " {AST, [], _Index} -> AST;\n", + " Any -> Any\n" + " end,\n", + " release_memo(), Result.\n"]. parse_grammar(InputFile) -> case peg_meta:file(InputFile) of src/peg_gen.erl @@ -1,89 +1,271 @@ -module(peg_meta). -export([parse/1,file/1]). --include_lib("neotoma/include/peg.hrl"). +-compile(nowarn_unused_function). -rule(rules) -> - peg:seq([peg:optional(fun space/2), fun declaration_sequence/2, peg:optional(fun space/2)]); +file(Filename) -> {ok, Bin} = file:read_file(Filename), parse(binary_to_list(Bin)). -rule(declaration_sequence) -> - peg:seq([peg:label('head', fun declaration/2), peg:label('tail', peg:zero_or_more(peg:seq([fun space/2, fun declaration/2])))]); +parse(Input) -> + setup_memo(peg_meta), + Result = case rules(Input,{{line,1},{column,1}}) of + {AST, [], _Index} -> AST; + Any -> Any + end, + release_memo(), Result. -rule(declaration) -> - peg:seq([fun nonterminal/2, fun space/2, peg:string("<-"), fun space/2, fun parsing_expression/2, peg:optional(fun space/2), peg:string(";")]); +'rules'(Input, Index) -> + p(Input, Index, 'rules', fun(I,D) -> (p_seq([p_optional(fun 'space'/2), fun 'declaration_sequence'/2, p_optional(fun 'space'/2)]))(I,D) end, fun(Node, Idx) -> transform('rules', Node, Idx) end). -rule(parsing_expression) -> - peg:choose([fun choice/2, fun sequence/2, fun primary/2]); +'declaration_sequence'(Input, Index) -> + p(Input, Index, 'declaration_sequence', fun(I,D) -> (p_seq([p_label('head', fun 'declaration'/2), p_label('tail', p_zero_or_more(p_seq([fun 'space'/2, fun 'declaration'/2])))]))(I,D) end, fun(Node, Idx) -> transform('declaration_sequence', Node, Idx) end). -rule(choice) -> - peg:seq([peg:label('head', fun alternative/2), peg:label('tail', peg:one_or_more(peg:seq([fun space/2, peg:string("/"), fun space/2, fun alternative/2])))]); +'declaration'(Input, Index) -> + p(Input, Index, 'declaration', fun(I,D) -> (p_seq([fun 'nonterminal'/2, fun 'space'/2, p_string("<-"), fun 'space'/2, fun 'parsing_expression'/2, p_optional(fun 'space'/2), p_string(";")]))(I,D) end, fun(Node, Idx) -> transform('declaration', Node, Idx) end). -rule(alternative) -> - peg:choose([fun sequence/2, fun primary/2]); +'parsing_expression'(Input, Index) -> + p(Input, Index, 'parsing_expression', fun(I,D) -> (p_choose([fun 'choice'/2, fun 'sequence'/2, fun 'primary'/2]))(I,D) end, fun(Node, Idx) -> transform('parsing_expression', Node, Idx) end). -rule(primary) -> - peg:choose([peg:seq([fun prefix/2, fun atomic/2]), peg:seq([fun atomic/2, fun suffix/2]), fun atomic/2]); +'choice'(Input, Index) -> + p(Input, Index, 'choice', fun(I,D) -> (p_seq([p_label('head', fun 'alternative'/2), p_label('tail', p_one_or_more(p_seq([fun 'space'/2, p_string("/"), fun 'space'/2, fun 'alternative'/2])))]))(I,D) end, fun(Node, Idx) -> transform('choice', Node, Idx) end). -rule(sequence) -> - peg:seq([peg:label('head', fun labeled_sequence_primary/2), peg:label('tail', peg:one_or_more(peg:seq([fun space/2, fun labeled_sequence_primary/2])))]); +'alternative'(Input, Index) -> + p(Input, Index, 'alternative', fun(I,D) -> (p_choose([fun 'sequence'/2, fun 'primary'/2]))(I,D) end, fun(Node, Idx) -> transform('alternative', Node, Idx) end). -rule(labeled_sequence_primary) -> - peg:seq([peg:optional(fun label/2), fun primary/2]); +'primary'(Input, Index) -> + p(Input, Index, 'primary', fun(I,D) -> (p_choose([p_seq([fun 'prefix'/2, fun 'atomic'/2]), p_seq([fun 'atomic'/2, fun 'suffix'/2]), fun 'atomic'/2]))(I,D) end, fun(Node, Idx) -> transform('primary', Node, Idx) end). -rule(label) -> - peg:seq([fun alpha_char/2, peg:zero_or_more(fun alphanumeric_char/2), peg:string(":")]); +'sequence'(Input, Index) -> + p(Input, Index, 'sequence', fun(I,D) -> (p_seq([p_label('head', fun 'labeled_sequence_primary'/2), p_label('tail', p_one_or_more(p_seq([fun 'space'/2, fun 'labeled_sequence_primary'/2])))]))(I,D) end, fun(Node, Idx) -> transform('sequence', Node, Idx) end). -rule(suffix) -> - peg:choose([fun repetition_suffix/2, fun optional_suffix/2]); +'labeled_sequence_primary'(Input, Index) -> + p(Input, Index, 'labeled_sequence_primary', fun(I,D) -> (p_seq([p_optional(fun 'label'/2), fun 'primary'/2]))(I,D) end, fun(Node, Idx) -> transform('labeled_sequence_primary', Node, Idx) end). -rule(optional_suffix) -> - peg:string("?"); +'label'(Input, Index) -> + p(Input, Index, 'label', fun(I,D) -> (p_seq([fun 'alpha_char'/2, p_zero_or_more(fun 'alphanumeric_char'/2), p_string(":")]))(I,D) end, fun(Node, Idx) -> transform('label', Node, Idx) end). -rule(repetition_suffix) -> - peg:choose([peg:string("+"), peg:string("*")]); +'suffix'(Input, Index) -> + p(Input, Index, 'suffix', fun(I,D) -> (p_choose([fun 'repetition_suffix'/2, fun 'optional_suffix'/2]))(I,D) end, fun(Node, Idx) -> transform('suffix', Node, Idx) end). -rule(prefix) -> - peg:choose([peg:string("&"), peg:string("!")]); +'optional_suffix'(Input, Index) -> + p(Input, Index, 'optional_suffix', fun(I,D) -> (p_string("?"))(I,D) end, fun(Node, Idx) -> transform('optional_suffix', Node, Idx) end). -rule(atomic) -> - peg:choose([fun terminal/2, fun nonterminal/2, fun parenthesized_expression/2]); +'repetition_suffix'(Input, Index) -> + p(Input, Index, 'repetition_suffix', fun(I,D) -> (p_choose([p_string("+"), p_string("*")]))(I,D) end, fun(Node, Idx) -> transform('repetition_suffix', Node, Idx) end). -rule(parenthesized_expression) -> - peg:seq([peg:string("("), peg:optional(fun space/2), fun parsing_expression/2, peg:optional(fun space/2), peg:string(")")]); +'prefix'(Input, Index) -> + p(Input, Index, 'prefix', fun(I,D) -> (p_choose([p_string("&"), p_string("!")]))(I,D) end, fun(Node, Idx) -> transform('prefix', Node, Idx) end). -rule(nonterminal) -> - peg:seq([fun alpha_char/2, peg:zero_or_more(fun alphanumeric_char/2)]); +'atomic'(Input, Index) -> + p(Input, Index, 'atomic', fun(I,D) -> (p_choose([fun 'terminal'/2, fun 'nonterminal'/2, fun 'parenthesized_expression'/2]))(I,D) end, fun(Node, Idx) -> transform('atomic', Node, Idx) end). -rule(terminal) -> - peg:choose([fun quoted_string/2, fun character_class/2, fun anything_symbol/2]); +'parenthesized_expression'(Input, Index) -> + p(Input, Index, 'parenthesized_expression', fun(I,D) -> (p_seq([p_string("("), p_optional(fun 'space'/2), fun 'parsing_expression'/2, p_optional(fun 'space'/2), p_string(")")]))(I,D) end, fun(Node, Idx) -> transform('parenthesized_expression', Node, Idx) end). -rule(quoted_string) -> - peg:choose([fun single_quoted_string/2, fun double_quoted_string/2]); +'nonterminal'(Input, Index) -> + p(Input, Index, 'nonterminal', fun(I,D) -> (p_seq([fun 'alpha_char'/2, p_zero_or_more(fun 'alphanumeric_char'/2)]))(I,D) end, fun(Node, Idx) -> transform('nonterminal', Node, Idx) end). -rule(double_quoted_string) -> - peg:seq([peg:string("\""), peg:label('string', peg:zero_or_more(peg:seq([peg:not_(peg:string("\"")), peg:choose([peg:string("\\\\"), peg:string("\\\""), peg:anything()])]))), peg:string("\"")]); +'terminal'(Input, Index) -> + p(Input, Index, 'terminal', fun(I,D) -> (p_choose([fun 'quoted_string'/2, fun 'character_class'/2, fun 'anything_symbol'/2]))(I,D) end, fun(Node, Idx) -> transform('terminal', Node, Idx) end). -rule(single_quoted_string) -> - peg:seq([peg:string("'"), peg:label('string', peg:zero_or_more(peg:seq([peg:not_(peg:string("'")), peg:choose([peg:string("\\\\"), peg:string("\\'"), peg:anything()])]))), peg:string("'")]); +'quoted_string'(Input, Index) -> + p(Input, Index, 'quoted_string', fun(I,D) -> (p_choose([fun 'single_quoted_string'/2, fun 'double_quoted_string'/2]))(I,D) end, fun(Node, Idx) -> transform('quoted_string', Node, Idx) end). -rule(character_class) -> - peg:seq([peg:string("["), peg:label('characters', peg:one_or_more(peg:seq([peg:not_(peg:string("]")), peg:choose([peg:seq([peg:string("\\\\"), peg:anything()]), peg:seq([peg:not_(peg:string("\\\\")), peg:anything()])])]))), peg:string("]")]); +'double_quoted_string'(Input, Index) -> + p(Input, Index, 'double_quoted_string', fun(I,D) -> (p_seq([p_string("\""), p_label('string', p_zero_or_more(p_seq([p_not(p_string("\"")), p_choose([p_string("\\\\"), p_string("\\\""), p_anything()])]))), p_string("\"")]))(I,D) end, fun(Node, Idx) -> transform('double_quoted_string', Node, Idx) end). -rule(anything_symbol) -> - peg:string("."); +'single_quoted_string'(Input, Index) -> + p(Input, Index, 'single_quoted_string', fun(I,D) -> (p_seq([p_string("'"), p_label('string', p_zero_or_more(p_seq([p_not(p_string("'")), p_choose([p_string("\\\\"), p_string("\\'"), p_anything()])]))), p_string("'")]))(I,D) end, fun(Node, Idx) -> transform('single_quoted_string', Node, Idx) end). -rule(alpha_char) -> - peg:charclass("[a-z_]"); +'character_class'(Input, Index) -> + p(Input, Index, 'character_class', fun(I,D) -> (p_seq([p_string("["), p_label('characters', p_one_or_more(p_seq([p_not(p_string("]")), p_choose([p_seq([p_string("\\\\"), p_anything()]), p_seq([p_not(p_string("\\\\")), p_anything()])])]))), p_string("]")]))(I,D) end, fun(Node, Idx) -> transform('character_class', Node, Idx) end). -rule(alphanumeric_char) -> - peg:choose([fun alpha_char/2, peg:charclass("[0-9]")]); +'anything_symbol'(Input, Index) -> + p(Input, Index, 'anything_symbol', fun(I,D) -> (p_string("."))(I,D) end, fun(Node, Idx) -> transform('anything_symbol', Node, Idx) end). -rule(space) -> - peg:one_or_more(peg:choose([fun white/2, fun comment_to_eol/2])); +'alpha_char'(Input, Index) -> + p(Input, Index, 'alpha_char', fun(I,D) -> (p_charclass("[a-z_]"))(I,D) end, fun(Node, Idx) -> transform('alpha_char', Node, Idx) end). -rule(comment_to_eol) -> - peg:seq([peg:string("%"), peg:zero_or_more(peg:seq([peg:not_(peg:string("\n")), peg:anything()]))]); +'alphanumeric_char'(Input, Index) -> + p(Input, Index, 'alphanumeric_char', fun(I,D) -> (p_choose([fun 'alpha_char'/2, p_charclass("[0-9]")]))(I,D) end, fun(Node, Idx) -> transform('alphanumeric_char', Node, Idx) end). -rule(white) -> - peg:charclass("[ \t\n\r]"). +'space'(Input, Index) -> + p(Input, Index, 'space', fun(I,D) -> (p_one_or_more(p_choose([fun 'white'/2, fun 'comment_to_eol'/2])))(I,D) end, fun(Node, Idx) -> transform('space', Node, Idx) end). -transform(Symbol,Node,Index) -> peg_meta_gen:transform(Symbol, Node, Index). \ No newline at end of file +'comment_to_eol'(Input, Index) -> + p(Input, Index, 'comment_to_eol', fun(I,D) -> (p_seq([p_string("%"), p_zero_or_more(p_seq([p_not(p_string("\n")), p_anything()]))]))(I,D) end, fun(Node, Idx) -> transform('comment_to_eol', Node, Idx) end). + +'white'(Input, Index) -> + p(Input, Index, 'white', fun(I,D) -> (p_charclass("[ \t\n\r]"))(I,D) end, fun(Node, Idx) -> transform('white', Node, Idx) end). + +transform(Symbol,Node,Index) -> peg_meta_gen:transform(Symbol, Node, Index). + + + + + +p(Inp, Index, Name, ParseFun) -> + p(Inp, Index, Name, ParseFun, fun(N, _Idx) -> N end). + +p(Inp, StartIndex, Name, ParseFun, TransformFun) -> + % Grab the memo table from ets + Memo = get_memo(StartIndex), + % See if the current reduction is memoized + case dict:find(Name, Memo) of + % If it is, return the result + {ok, Result} -> Result; + % If not, attempt to parse + _ -> + case ParseFun(Inp, StartIndex) of + % If it fails, memoize the failure + {fail,_} = Failure -> + memoize(StartIndex, dict:store(Name, Failure, Memo)), + Failure; + % If it passes, transform and memoize the result. + {Result, InpRem, NewIndex} -> + Transformed = TransformFun(Result, StartIndex), + memoize(StartIndex, dict:store(Name, {Transformed, InpRem, NewIndex}, Memo)), + {Transformed, InpRem, NewIndex} + end + end. + +setup_memo(Name) -> + TID = ets:new(Name, [set]), + put(ets_table, TID). + +release_memo() -> + ets:delete(get(ets_table)), + erase(ets_table). + +memoize(Position, Struct) -> + ets:insert(get(ets_table), {Position, Struct}). + +get_memo(Position) -> + case ets:lookup(get(ets_table), Position) of + [] -> dict:new(); + [{Position, Dict}] -> Dict + end. + +p_eof() -> + fun([], Index) -> {eof, [], Index}; + (_, Index) -> {fail, {expected, eof, Index}} end. + +p_optional(P) -> + fun(Input, Index) -> + case P(Input, Index) of + {fail,_} -> {[], Input, Index}; + {_, _, _} = Success -> Success + end + end. + +p_not(P) -> + fun(Input, Index)-> + case P(Input,Index) of + {fail,_} -> + {[], Input, Index}; + {Result, _, _} -> {fail, {expected, {no_match, Result},Index}} + end + end. + +p_assert(P) -> + fun(Input,Index) -> + case P(Input,Index) of + {fail,_} = Failure-> Failure; + _ -> {[], Input, Index} + end + end. + +p_and(P) -> + p_seq(P). + +p_seq(P) -> + fun(Input, Index) -> + p_all(P, Input, Index, []) + end. + +p_all([], Inp, Index, Accum ) -> {lists:reverse( Accum ), Inp, Index}; +p_all([P|Parsers], Inp, Index, Accum) -> + case P(Inp, Index) of + {fail, _} = Failure -> Failure; + {Result, InpRem, NewIndex} -> p_all(Parsers, InpRem, NewIndex, [Result|Accum]) + end. + +p_choose(Parsers) -> + fun(Input, Index) -> + p_attempt(Parsers, Input, Index, none) + end. + +p_attempt([], _Input, _Index, Failure) -> Failure; +p_attempt([P|Parsers], Input, Index, FirstFailure)-> + case P(Input, Index) of + {fail, _} = Failure -> + case FirstFailure of + none -> p_attempt(Parsers, Input, Index, Failure); + _ -> p_attempt(Parsers, Input, Index, FirstFailure) + end; + Result -> Result + end. + +p_zero_or_more(P) -> + fun(Input, Index) -> + p_scan(P, Input, Index, []) + end. + +p_one_or_more(P) -> + fun(Input, Index)-> + Result = p_scan(P, Input, Index, []), + case Result of + {[_|_], _, _} -> + Result; + _ -> + {fail, {expected, Failure, _}} = P(Input,Index), + {fail, {expected, {at_least_one, Failure}, Index}} + end + end. + +p_label(Tag, P) -> + fun(Input, Index) -> + case P(Input, Index) of + {fail,_} = Failure -> + Failure; + {Result, InpRem, NewIndex} -> + {{Tag, Result}, InpRem, NewIndex} + end + end. + +p_scan(_, [], Index, Accum) -> {lists:reverse( Accum ), [], Index}; +p_scan(P, Inp, Index, Accum) -> + case P(Inp, Index) of + {fail,_} -> {lists:reverse(Accum), Inp, Index}; + {Result, InpRem, NewIndex} -> p_scan(P, InpRem, NewIndex, [Result | Accum]) + end. + +p_string(S) -> + fun(Input, Index) -> + case lists:prefix(S, Input) of + true -> {S, lists:sublist(Input, length(S)+1, length(Input)), p_advance_index(S,Index)}; + _ -> {fail, {expected, {string, S}, Index}} + end + end. + +p_anything() -> + fun([], Index) -> {fail, {expected, any_character, Index}}; + ([H|T], Index) -> {H, T, p_advance_index(H, Index)} + end. + +p_charclass(Class) -> + fun(Inp, Index) -> + {ok, RE} = re:compile("^"++Class), + case re:run(Inp, RE) of + {match, _} -> + {hd(Inp), tl(Inp), p_advance_index(hd(Inp), Index)}; + _ -> {fail,{expected, {character_class, Class}, Index}} + end + end. + +p_advance_index(MatchedInput, Index) when is_list(MatchedInput) -> % strings + lists:foldl(fun p_advance_index/2, Index, MatchedInput); +p_advance_index(MatchedInput, Index) when is_integer(MatchedInput) -> % single characters + {{line, Line}, {column, Col}} = Index, + case MatchedInput of + $\n -> {{line, Line+1}, {column, 1}}; + _ -> {{line, Line}, {column, Col+1}} + end. src/peg_meta.erl @@ -3,57 +3,60 @@ -author("Sean Cribbs <seancribbs@gmail.com>"). transform(rules, Node, _Index) -> - verify_rules(), - Rules = string:join(lists:nth(2, Node), ";\n\n"), - Rules ++ ".\n"; + put(neotoma_root_rule, verify_rules()), + Rules = string:join(lists:nth(2, Node), "\n\n"), + Rules ++ "\n"; transform(declaration_sequence, Node, _Index) -> FirstRule = proplists:get_value(head, Node), OtherRules = [lists:last(I) || I <- proplists:get_value(tail, Node, [])], [FirstRule|OtherRules]; transform(declaration, [{nonterminal,Symbol}|Node], Index) -> add_lhs(Symbol, Index), - "rule("++escape_symbol(Symbol)++") ->\n " ++ lists:nth(4, Node); + "'"++Symbol++"'"++"(Input, Index) ->\n " ++ + "p(Input, Index, '"++Symbol++"', fun(I,D) -> ("++ + lists:nth(4, Node) ++ + ")(I,D) end, fun(Node, Idx) -> transform('"++Symbol++"', Node, Idx) end)."; transform(sequence, Node, _Index) -> Tail = [lists:nth(2, S) || S <- proplists:get_value(tail, Node)], Statements = [proplists:get_value(head, Node)|Tail], - "peg:seq(["++ string:join(Statements, ", ") ++ "])"; + "p_seq(["++ string:join(Statements, ", ") ++ "])"; transform(choice, Node, _Index) -> Tail = [lists:last(S) || S <- proplists:get_value(tail, Node)], Statements = [proplists:get_value(head, Node)|Tail], - "peg:choose([" ++ string:join(Statements, ", ") ++ "])"; + "p_choose([" ++ string:join(Statements, ", ") ++ "])"; transform(label, Node, _Index) -> String = lists:flatten(Node), lists:sublist(String, length(String)-1); transform(labeled_sequence_primary, Node, _Index) -> case hd(Node) of [] -> lists:nth(2, Node); - Label -> "peg:label('" ++ Label ++ "', "++lists:nth(2, Node)++")" + Label -> "p_label('" ++ Label ++ "', "++lists:nth(2, Node)++")" end; transform(single_quoted_string, Node, Index) -> transform(double_quoted_string, Node, Index); transform(double_quoted_string, Node, _Index) -> - "peg:string(\""++escape_quotes(lists:flatten(proplists:get_value(string, Node)))++"\")"; + "p_string(\""++escape_quotes(lists:flatten(proplists:get_value(string, Node)))++"\")"; transform(character_class, Node, _Index) -> - "peg:charclass(\"[" ++ escape_quotes(lists:flatten(proplists:get_value(characters, Node))) ++ "]\")"; + "p_charclass(\"[" ++ escape_quotes(lists:flatten(proplists:get_value(characters, Node))) ++ "]\")"; transform(parenthesized_expression, Node, _Index) -> lists:nth(3, Node); transform(atomic, {nonterminal, Symbol}, Index) -> add_nt(Symbol, Index), - "fun " ++ escape_symbol(Symbol) ++ "/2"; + "fun '" ++ Symbol ++ "'/2"; transform(primary, [Atomic, one_or_more], _Index) -> - "peg:one_or_more("++Atomic++")"; + "p_one_or_more("++Atomic++")"; transform(primary, [Atomic, zero_or_more], _Index) -> - "peg:zero_or_more("++Atomic++")"; + "p_zero_or_more("++Atomic++")"; transform(primary, [Atomic, optional], _Index) -> - "peg:optional("++Atomic++")"; + "p_optional("++Atomic++")"; transform(primary, [assert, Atomic], _Index)-> - "peg:assert("++Atomic++")"; + "p_assert("++Atomic++")"; transform(primary, [not_, Atomic], _Index) -> - "peg:not_("++Atomic++")"; + "p_not("++Atomic++")"; transform(nonterminal, Node, _Index) -> {nonterminal, lists:flatten(Node)}; transform(anything_symbol, _Node, _Index) -> - "peg:anything()"; + "p_anything()"; transform(suffix, Node, _Index) -> case Node of "*" -> zero_or_more; @@ -96,7 +99,7 @@ add_nt(Symbol, Index) -> verify_rules() -> LHS = erase(lhs), NTs = erase(nts), - NonRoots = tl(lists:reverse(LHS)), + [Root|NonRoots] = lists:reverse(LHS), lists:foreach(fun({Sym,Idx}) -> case proplists:is_defined(Sym, NTs) of true -> @@ -110,38 +113,8 @@ verify_rules() -> true -> ok; _ -> - io:format("neotoma error: symbol '~s' has no reduction. (found at ~p) No parser will be generated!~n", [S,I]), + io:format("neotoma error: nonterminal '~s' has no reduction. (found at ~p) No parser will be generated!~n", [S,I]), exit({neotoma, {no_reduction, list_to_atom(S)}}) end - end, NTs). - -escape_symbol("after") -> "'after'"; -escape_symbol("and") -> "'and'"; -escape_symbol("andalso") -> "'andalso'"; -escape_symbol("band") -> "'band'"; -escape_symbol("begin") -> "'begin'"; -escape_symbol("bnot") -> "'bnot'"; -escape_symbol("bor") -> "'bor'"; -escape_symbol("bsl") -> "'bsl'"; -escape_symbol("bsr") -> "'bsr'"; -escape_symbol("bxor") -> "'bxor'"; -escape_symbol("case") -> "'case'"; -escape_symbol("catch") -> "'catch'"; -escape_symbol("cond") -> "'cond'"; -escape_symbol("div") -> "'div'"; -escape_symbol("end") -> "'end'"; -escape_symbol("fun") -> "'fun'"; -escape_symbol("if") -> "'if'"; -escape_symbol("let") -> "'let'"; -escape_symbol("not") -> "'not'"; -escape_symbol("of") -> "'of'"; -escape_symbol("or") -> "'or'"; -escape_symbol("orelse") -> "'orelse'"; -escape_symbol("query") -> "'query'"; -escape_symbol("receive") -> "'receive'"; -escape_symbol("rem") -> "'rem'"; -escape_symbol("try") -> "'try'"; -escape_symbol("when") -> "'when'"; -escape_symbol("xor") -> "'xor'"; - -escape_symbol(Symbol) -> Symbol. + end, NTs), + Root. src/peg_meta_gen.erl @@ -6,77 +6,77 @@ -define(STARTINDEX, {{line,1},{column,1}}). eof_test_() -> [ - ?_assertEqual({fail,{expected,eof,?STARTINDEX}}, (peg:eof())("abc",?STARTINDEX)), - ?_assertEqual({eof, [], ?STARTINDEX}, (peg:eof())("",?STARTINDEX)) + ?_assertEqual({fail,{expected,eof,?STARTINDEX}}, (peg:p_eof())("abc",?STARTINDEX)), + ?_assertEqual({eof, [], ?STARTINDEX}, (peg:p_eof())("",?STARTINDEX)) ]. optional_test_() -> [ - ?_assertEqual({[], "xyz",?STARTINDEX}, (peg:optional(peg:string("abc")))("xyz",?STARTINDEX)), - ?_assertEqual({"abc", "xyz",{{line,1},{column,4}}}, (peg:optional(peg:string("abc")))("abcxyz",?STARTINDEX)) + ?_assertEqual({[], "xyz",?STARTINDEX}, (peg:p_optional(peg:p_string("abc")))("xyz",?STARTINDEX)), + ?_assertEqual({"abc", "xyz",{{line,1},{column,4}}}, (peg:p_optional(peg:p_string("abc")))("abcxyz",?STARTINDEX)) ]. not_test_() -> [ - ?_assertEqual({[], "xyzabc",?STARTINDEX}, (peg:not_(peg:string("abc")))("xyzabc",?STARTINDEX)), - ?_assertEqual({fail,{expected, {no_match, "abc"}, ?STARTINDEX}}, (peg:not_(peg:string("abc")))("abcxyz",?STARTINDEX)) + ?_assertEqual({[], "xyzabc",?STARTINDEX}, (peg:p_not(peg:p_string("abc")))("xyzabc",?STARTINDEX)), + ?_assertEqual({fail,{expected, {no_match, "abc"}, ?STARTINDEX}}, (peg:p_not(peg:p_string("abc")))("abcxyz",?STARTINDEX)) ]. assert_test_() -> [ - ?_assertEqual({fail,{expected, {string, "abc"}, ?STARTINDEX}}, (peg:assert(peg:string("abc")))("xyzabc",?STARTINDEX)), - ?_assertEqual({[], "abcxyz",?STARTINDEX}, (peg:assert(peg:string("abc")))("abcxyz",?STARTINDEX)) + ?_assertEqual({fail,{expected, {string, "abc"}, ?STARTINDEX}}, (peg:p_assert(peg:p_string("abc")))("xyzabc",?STARTINDEX)), + ?_assertEqual({[], "abcxyz",?STARTINDEX}, (peg:p_assert(peg:p_string("abc")))("abcxyz",?STARTINDEX)) ]. seq_test_() -> [ - ?_assertEqual({["abc","def"], "xyz",{{line,1},{column,7}}}, (peg:seq([peg:string("abc"), peg:string("def")]))("abcdefxyz",?STARTINDEX)), - ?_assertEqual({fail,{expected, {string, "def"}, {{line,1},{column,4}}}}, (peg:seq([peg:string("abc"), peg:string("def")]))("abcxyz",?STARTINDEX)) + ?_assertEqual({["abc","def"], "xyz",{{line,1},{column,7}}}, (peg:p_seq([peg:p_string("abc"), peg:p_string("def")]))("abcdefxyz",?STARTINDEX)), + ?_assertEqual({fail,{expected, {string, "def"}, {{line,1},{column,4}}}}, (peg:p_seq([peg:p_string("abc"), peg:p_string("def")]))("abcxyz",?STARTINDEX)) ]. choose_test_() -> [ - ?_assertEqual({"abc", "xyz", {{line,1},{column,4}}}, (peg:choose([peg:string("abc"), peg:string("def")]))("abcxyz",?STARTINDEX)), - ?_assertEqual({"def", "xyz", {{line,1},{column,4}}}, (peg:choose([peg:string("abc"), peg:string("def")]))("defxyz",?STARTINDEX)), - ?_assertEqual({"xyz", "xyz", {{line,1},{column,4}}}, (peg:choose([peg:string("abc"), peg:string("def"), peg:string("xyz")]))("xyzxyz",?STARTINDEX)), - ?_assertEqual({fail,{expected,{string,"abc"},?STARTINDEX}}, (peg:choose([peg:string("abc"),peg:string("def")]))("xyz", ?STARTINDEX)) + ?_assertEqual({"abc", "xyz", {{line,1},{column,4}}}, (peg:p_choose([peg:p_string("abc"), peg:p_string("def")]))("abcxyz",?STARTINDEX)), + ?_assertEqual({"def", "xyz", {{line,1},{column,4}}}, (peg:p_choose([peg:p_string("abc"), peg:p_string("def")]))("defxyz",?STARTINDEX)), + ?_assertEqual({"xyz", "xyz", {{line,1},{column,4}}}, (peg:p_choose([peg:p_string("abc"), peg:p_string("def"), peg:p_string("xyz")]))("xyzxyz",?STARTINDEX)), + ?_assertEqual({fail,{expected,{string,"abc"},?STARTINDEX}}, (peg:p_choose([peg:p_string("abc"),peg:p_string("def")]))("xyz", ?STARTINDEX)) ]. zero_or_more_test_() -> [ - ?_assertEqual({[], [], ?STARTINDEX}, (peg:zero_or_more(peg:string("abc")))("",?STARTINDEX)), - ?_assertEqual({[], "def",?STARTINDEX}, (peg:zero_or_more(peg:string("abc")))("def",?STARTINDEX)), - ?_assertEqual({["abc"], "def",{{line,1},{column,4}}}, (peg:zero_or_more(peg:string("abc")))("abcdef",?STARTINDEX)), - ?_assertEqual({["abc", "abc"], "def",{{line,1},{column,7}}}, (peg:zero_or_more(peg:string("abc")))("abcabcdef",?STARTINDEX)) + ?_assertEqual({[], [], ?STARTINDEX}, (peg:p_zero_or_more(peg:p_string("abc")))("",?STARTINDEX)), + ?_assertEqual({[], "def",?STARTINDEX}, (peg:p_zero_or_more(peg:p_string("abc")))("def",?STARTINDEX)), + ?_assertEqual({["abc"], "def",{{line,1},{column,4}}}, (peg:p_zero_or_more(peg:p_string("abc")))("abcdef",?STARTINDEX)), + ?_assertEqual({["abc", "abc"], "def",{{line,1},{column,7}}}, (peg:p_zero_or_more(peg:p_string("abc")))("abcabcdef",?STARTINDEX)) ]. one_or_more_test_() -> [ - ?_assertEqual({fail,{expected, {at_least_one, {string, "abc"}}, ?STARTINDEX}}, (peg:one_or_more(peg:string("abc")))("def",?STARTINDEX)), - ?_assertEqual({["abc"], "def",{{line,1},{column,4}}}, (peg:one_or_more(peg:string("abc")))("abcdef",?STARTINDEX)), - ?_assertEqual({["abc","abc"], "def",{{line,1},{column,7}}}, (peg:one_or_more(peg:string("abc")))("abcabcdef",?STARTINDEX)) + ?_assertEqual({fail,{expected, {at_least_one, {string, "abc"}}, ?STARTINDEX}}, (peg:p_one_or_more(peg:p_string("abc")))("def",?STARTINDEX)), + ?_assertEqual({["abc"], "def",{{line,1},{column,4}}}, (peg:p_one_or_more(peg:p_string("abc")))("abcdef",?STARTINDEX)), + ?_assertEqual({["abc","abc"], "def",{{line,1},{column,7}}}, (peg:p_one_or_more(peg:p_string("abc")))("abcabcdef",?STARTINDEX)) ]. label_test_() -> [ - ?_assertEqual({fail,{expected, {string, "!"}, ?STARTINDEX}}, (peg:label(bang, peg:string("!")))("?",?STARTINDEX)), - ?_assertEqual({{bang, "!"}, "",{{line,1},{column,2}}}, (peg:label(bang, peg:string("!")))("!",?STARTINDEX)) + ?_assertEqual({fail,{expected, {string, "!"}, ?STARTINDEX}}, (peg:p_label(bang, peg:p_string("!")))("?",?STARTINDEX)), + ?_assertEqual({{bang, "!"}, "",{{line,1},{column,2}}}, (peg:p_label(bang, peg:p_string("!")))("!",?STARTINDEX)) ]. string_test_() -> [ - ?_assertEqual({"abc", "def",{{line,1},{column,4}}}, (peg:string("abc"))("abcdef",?STARTINDEX)), - ?_assertEqual({fail,{expected, {string, "abc"}, ?STARTINDEX}}, (peg:string("abc"))("defabc",?STARTINDEX)) + ?_assertEqual({"abc", "def",{{line,1},{column,4}}}, (peg:p_string("abc"))("abcdef",?STARTINDEX)), + ?_assertEqual({fail,{expected, {string, "abc"}, ?STARTINDEX}}, (peg:p_string("abc"))("defabc",?STARTINDEX)) ]. anything_test_() -> [ - ?_assertEqual({$a,"bcde",{{line,1},{column,2}}}, (peg:anything())("abcde",?STARTINDEX)), - ?_assertEqual({fail,{expected, any_character, ?STARTINDEX}}, (peg:anything())("",?STARTINDEX)) + ?_assertEqual({$a,"bcde",{{line,1},{column,2}}}, (peg:p_anything())("abcde",?STARTINDEX)), + ?_assertEqual({fail,{expected, any_character, ?STARTINDEX}}, (peg:p_anything())("",?STARTINDEX)) ]. charclass_test_() -> [ - ?_assertEqual({$+,"----",{{line,1},{column,2}}}, (peg:charclass("[+]"))("+----",?STARTINDEX)), - ?_assertEqual({fail,{expected, {character_class, "[+]"}, ?STARTINDEX}}, (peg:charclass("[+]"))("----",?STARTINDEX)) + ?_assertEqual({$+,"----",{{line,1},{column,2}}}, (peg:p_charclass("[+]"))("+----",?STARTINDEX)), + ?_assertEqual({fail,{expected, {character_class, "[+]"}, ?STARTINDEX}}, (peg:p_charclass("[+]"))("----",?STARTINDEX)) ]. tests/test_combinators.erl @@ -15,7 +15,7 @@ release_memo_test() -> step_memo_test() -> peg:setup_memo(?MODULE), - Result = peg:p("abcdefghi", {{line,1},{column,1}}, anything, peg:anything()), + Result = peg:p("abcdefghi", {{line,1},{column,1}}, anything, peg:p_anything()), ?assertEqual({$a, "bcdefghi", {{line,1},{column,2}}}, Result), Result2 = peg:p("abcdefghi", {{line,1},{column,1}}, anything, fun(_) -> throw(bork) end), tests/test_memoization.erl @@ -3,4 +3,4 @@ -include_lib("eunit/include/eunit.hrl"). all_test_() -> - [{module, test_combinators},{module, test_memoization},{module, test_transform}]. + [{module, test_combinators},{module, test_memoization}]. tests/test_suite.erl include/peg.hrl include/peg_i.hrl src/peg_itransform.erl src/peg_transform.erl tests/examples/Emakefile tests/examples/arithmetic.erl tests/examples/arithmetic_xf.erl tests/test_transform.erl 84057b99def0825dc6a600c6528d1789f56c71eb Sean Cribbs seancribbs@gmail.com http://github.com/seancribbs/neotoma/commit/86e09575b6e6abdad173ff78303718309006724b 86e09575b6e6abdad173ff78303718309006724b 2009-10-31T12:16:06-07:00 2009-10-31T12:16:06-07:00 Big spike: standalone parsers, escape NTs, rename combinators, remove parse_transforms. e036429b13f85b32ff86f6fe1dccf8569f2de711 Sean Cribbs seancribbs@gmail.com