Permalink
Browse files

Blocktrans utilities

  • Loading branch information...
1 parent 13362ee commit 86b1199e0ad1bb64deba0477b019cc00c51c551e Evan Miller committed Jun 27, 2011
Showing with 177 additions and 29 deletions.
  1. +18 −29 src/erlydtl_scanner.erl
  2. +13 −0 src/i18n/blocktrans_parser.erl
  3. +146 −0 src/i18n/blocktrans_scanner.erl
View
@@ -128,27 +128,27 @@ scan("<!--{{" ++ T, Scanned, {Row, Column}, in_text) ->
scan(T, [{open_var, {Row, Column}, '<!--{{'} | Scanned], {Row, Column + length("<!--{{")}, {in_code, "}}-->"});
scan("{{" ++ T, Scanned, {Row, Column}, in_text) ->
- scan(T, [{open_var, {Row, Column}, '{{'} | Scanned], {Row, Column + 2}, {in_code, "}}"});
+ scan(T, [{open_var, {Row, Column}, '{{'} | Scanned], {Row, Column + length("{{")}, {in_code, "}}"});
scan("<!--{#" ++ T, Scanned, {Row, Column}, in_text) ->
scan(T, Scanned, {Row, Column + length("<!--{#")}, {in_comment, "#}-->"});
scan("{#" ++ T, Scanned, {Row, Column}, in_text) ->
- scan(T, Scanned, {Row, Column + 2}, {in_comment, "#}"});
+ scan(T, Scanned, {Row, Column + length("{#")}, {in_comment, "#}"});
scan("#}-->" ++ T, Scanned, {Row, Column}, {in_comment, "#}-->"}) ->
scan(T, Scanned, {Row, Column + length("#}-->")}, in_text);
scan("#}" ++ T, Scanned, {Row, Column}, {in_comment, "#}"}) ->
- scan(T, Scanned, {Row, Column + 2}, in_text);
+ scan(T, Scanned, {Row, Column + length("#}")}, in_text);
scan("<!--{%" ++ T, Scanned, {Row, Column}, in_text) ->
scan(T, [{open_tag, {Row, Column}, '<!--{%'} | Scanned],
{Row, Column + length("<!--{%")}, {in_code, "%}-->"});
scan("{%" ++ T, Scanned, {Row, Column}, in_text) ->
scan(T, [{open_tag, {Row, Column}, '{%'} | Scanned],
- {Row, Column + 2}, {in_code, "%}"});
+ {Row, Column + length("{%")}, {in_code, "%}"});
scan([_ | T], Scanned, {Row, Column}, {in_comment, Closer}) ->
scan(T, Scanned, {Row, Column + 1}, {in_comment, Closer});
@@ -289,30 +289,19 @@ scan([H | T], Scanned, {Row, Column}, {in_identifier, Closer}) ->
% internal functions
-append_char(Scanned, Char) ->
- [String | Scanned1] = Scanned,
- [setelement(3, String, [Char | element(3, String)]) | Scanned1].
+append_char([{Type, Pos, Chars}|Scanned], Char) ->
+ [{Type, Pos, [Char | Chars]} | Scanned].
+append_text_char([], {Row, Column}, Char) ->
+ [{string, {Row, Column}, [Char]}];
+append_text_char([{string, StrPos, Chars} |Scanned1], _, Char) ->
+ [{string, StrPos, [Char | Chars]} | Scanned1];
append_text_char(Scanned, {Row, Column}, Char) ->
- case length(Scanned) of
- 0 ->
- [{string, {Row, Column}, [Char]}];
- _ ->
- [Token | Scanned1] = Scanned,
- case element(1, Token) of
- string ->
- [{string, element(2, Token), [Char | element(3, Token)]} | Scanned1];
- _ ->
- [{string, element(2, Token), [Char]} | Scanned]
- end
- end.
-
-char_type(Char) ->
- case Char of
- C when ((C >= $a) and (C =< $z)) or ((C >= $A) and (C =< $Z)) or (C == $_) ->
- letter_underscore;
- C when ((C >= $0) and (C =< $9)) ->
- digit;
- _ ->
- undefined
- end.
+ [{string, {Row, Column}, [Char]} | Scanned].
+
+char_type(C) when ((C >= $a) andalso (C =< $z)) orelse ((C >= $A) andalso (C =< $Z)) orelse (C == $_) ->
+ letter_underscore;
+char_type(C) when ((C >= $0) andalso (C =< $9)) ->
+ digit;
+char_type(_C) ->
+ undefined.
@@ -0,0 +1,13 @@
+-module(blocktrans_parser).
+
+-export([parse/1]).
+
+parse(Tokens) ->
+ parse(Tokens, []).
+
+parse([], Acc) ->
+ lists:reverse(Acc);
+parse([{open_blocktrans, _, Name}, {text, _, Text}, {close_blocktrans, _}|Rest], Acc) ->
+ parse(Rest, [{Name, unicode:characters_to_binary(Text)}|Acc]);
+parse([{text, _, _}|Rest], Acc) ->
+ parse(Rest, Acc).
@@ -0,0 +1,146 @@
+% Module for extracting blocktrans blocks with original source formatting preserved.
+
+-module(blocktrans_scanner).
+
+-export([scan/1]).
+
+scan(Template) ->
+ scan(Template, [], {1, 1}, in_text).
+
+scan([], Scanned, _, in_text) ->
+ {ok, lists:reverse(lists:map(
+ fun
+ ({text, Pos, Text}) ->
+ {text, Pos, lists:reverse(Text)};
+ (Other) ->
+ Other
+ end, Scanned))};
+
+scan([], _Scanned, _, {in_comment, _}) ->
+ {error, "Reached end of file inside a comment."};
+
+scan([], _Scanned, _, _) ->
+ {error, "Reached end of file inside a code block."};
+
+scan("<!--{{" ++ T, Scanned, {Row, Column}, in_text) ->
+ scan(T, append_text("<!--{{", {Row, Column}, Scanned), {Row, Column + length("<!--{{")}, {in_code, "}}-->"});
+
+scan("{{" ++ T, Scanned, {Row, Column}, in_text) ->
+ scan(T, append_text("{{", {Row, Column}, Scanned), {Row, Column + 2}, {in_code, "}}"});
+
+scan("<!--{#" ++ T, Scanned, {Row, Column}, in_text) ->
+ scan(T, append_text("<!--{#", {Row, Column}, Scanned), {Row, Column + length("<!--{#")}, {in_comment, "#}-->"});
+
+scan("{#" ++ T, Scanned, {Row, Column}, in_text) ->
+ scan(T, append_text("{#", {Row, Column}, Scanned), {Row, Column + length("{#")}, {in_comment, "#}"});
+
+scan("#}-->" ++ T, Scanned, {Row, Column}, {in_comment, "#}-->"}) ->
+ scan(T, append_text("#}-->", {Row, Column}, Scanned), {Row, Column + length("#}-->")}, in_text);
+
+scan("#}" ++ T, Scanned, {Row, Column}, {in_comment, "#}"}) ->
+ scan(T, append_text("#}", {Row, Column}, Scanned), {Row, Column + length("#}")}, in_text);
+
+scan("<!--{% blocktrans " ++ T, Scanned, {Row, Column}, in_text) ->
+ scan(T, [{open_blocktrans, {Row, Column}, ""} | Scanned],
+ {Row, Column + length("<!--{% blocktrans ")}, {in_code, "%}-->"});
+
+scan("{% blocktrans " ++ T, Scanned, {Row, Column}, in_text) ->
+ scan(T, [{open_blocktrans, {Row, Column}, ""} | Scanned],
+ {Row, Column + length("{% blocktrans ")}, {in_code, "%}"});
+
+scan("{% endblocktrans %}" ++ T, Scanned, {Row, Column}, in_text) ->
+ scan(T, [{close_blocktrans, {Row, Column}} | Scanned],
+ {Row, Column + length("{% endblocktrans %}")}, in_text);
+
+scan("<!--{% endblocktrans %}-->" ++ T, Scanned, {Row, Column}, in_text) ->
+ scan(T, [{close_blocktrans, {Row, Column}} | Scanned],
+ {Row, Column + length("<!--{% endblocktrans %}-->")}, {in_text});
+
+scan("<!--{%" ++ T, Scanned, {Row, Column}, in_text) ->
+ scan(T, append_text("<!--{%", {Row, Column}, Scanned),
+ {Row, Column + length("<!--{%")}, {in_code, "%}-->"});
+
+scan("{%" ++ T, Scanned, {Row, Column}, in_text) ->
+ scan(T, append_text("{%", {Row, Column}, Scanned),
+ {Row, Column + length("{%")}, {in_code, "%}"});
+
+scan([H | T], Scanned, {Row, Column}, {in_comment, Closer}) ->
+ scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_comment, Closer});
+
+scan("\n" ++ T, Scanned, {Row, Column}, in_text) ->
+ scan(T, append_text("\n", {Row, Column}, Scanned), {Row + 1, 1}, in_text);
+
+scan([H | T], Scanned, {Row, Column}, in_text) ->
+ scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, in_text);
+
+scan("\"" ++ T, Scanned, {Row, Column}, {in_code, Closer}) ->
+ scan(T, append_text("\"", {Row, Column}, Scanned), {Row, Column + 1} , {in_double_quote, Closer});
+
+scan("\'" ++ T, Scanned, {Row, Column}, {in_code, Closer}) ->
+ scan(T, append_text("\'", {Row, Column}, Scanned), {Row, Column + 1}, {in_single_quote, Closer});
+
+scan("\\" ++ T, Scanned, {Row, Column}, {in_double_quote, Closer}) ->
+ scan(T, append_text("\\", {Row, Column}, Scanned), {Row, Column + 1}, {in_double_quote_slash, Closer});
+
+scan([H | T], Scanned, {Row, Column}, {in_double_quote_slash, Closer}) ->
+ scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_double_quote, Closer});
+
+scan("\\" ++ T, Scanned, {Row, Column}, {in_single_quote, Closer}) ->
+ scan(T, append_text("\\", {Row, Column}, Scanned), {Row, Column + 1}, {in_single_quote_slash, Closer});
+
+scan([H | T], Scanned, {Row, Column}, {in_single_quote_slash, Closer}) ->
+ scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_single_quote, Closer});
+
+% end quote
+scan("\"" ++ T, Scanned, {Row, Column}, {in_double_quote, Closer}) ->
+ scan(T, append_text("\"", {Row, Column}, Scanned), {Row, Column + 1}, {in_code, Closer});
+
+scan("\'" ++ T, Scanned, {Row, Column}, {in_single_quote, Closer}) ->
+ scan(T, append_text("\'", {Row, Column}, Scanned), {Row, Column + 1}, {in_code, Closer});
+
+scan([H | T], Scanned, {Row, Column}, {in_double_quote, Closer}) ->
+ scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_double_quote, Closer});
+
+scan([H | T], Scanned, {Row, Column}, {in_single_quote, Closer}) ->
+ scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_single_quote, Closer});
+
+
+scan("}}-->" ++ T, Scanned, {Row, Column}, {_, "}}-->"}) ->
+ scan(T, append_text("}}-->", {Row, Column}, Scanned),
+ {Row, Column + length("}}-->")}, in_text);
+
+scan("}}" ++ T, Scanned, {Row, Column}, {_, "}}"}) ->
+ scan(T, append_text("}}", {Row, Column}, Scanned), {Row, Column + length("}}")}, in_text);
+
+scan("%}-->" ++ T, Scanned, {Row, Column}, {_, "%}-->"}) ->
+ scan(T, append_text("%}-->", {Row, Column}, Scanned),
+ {Row, Column + length("%}-->")}, in_text);
+
+scan("%}" ++ T, Scanned, {Row, Column}, {_, "%}"}) ->
+ scan(T, append_text("%}", {Row, Column}, Scanned),
+ {Row, Column + length("%}")}, in_text);
+
+
+scan([H | T], Scanned, {Row, Column}, {in_code, Closer}) ->
+ scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_code, Closer}).
+
+% internal functions
+
+append_text(Text, Pos, []) ->
+ [{text, Pos, Text}];
+append_text(Text, Pos, [{close_blocktrans, _}|_] = Scanned) ->
+ [{text, Pos, Text}|Scanned];
+append_text([C], _Pos, [{open_blocktrans, BPos, ""}|Rest]) when ((C >= $a) and (C =< $z)) or ((C >= $A) and (C =< $Z)) or (C =:= $_) ->
+ [{open_blocktrans, BPos, [C]}|Rest];
+append_text(" ", _Pos, [{open_blocktrans, BPos, ""}|Rest]) ->
+ [{open_blocktrans, BPos, ""}|Rest];
+append_text([C], _Pos, [{open_blocktrans, BPos, Name}|Rest]) when ((C >= $a) and (C =< $z)) or ((C >= $A) and (C =< $Z)) or (C =:= $_) orelse (C >= $0 andalso C =< $9) ->
+ [{open_blocktrans, BPos, [C|Name]}|Rest];
+append_text(" ", _Pos, [{open_blocktrans, BPos, Name}|Rest]) when is_list(Name) ->
+ [{open_blocktrans, BPos, list_to_atom(lists:reverse(Name))}|Rest];
+append_text("%}", {Row, Column}, [{open_blocktrans, _BPos, _Name}|_] = Scanned) ->
+ [{text, {Row, Column + 2}, ""}|Scanned];
+append_text(_Chars, _Pos, [{open_blocktrans, _BPos, _Name}|_] = Scanned) ->
+ Scanned;
+append_text(Chars, _Pos, [{text, TPos, TChars}|Rest]) ->
+ [{text, TPos, lists:reverse(Chars, TChars)}|Rest].

0 comments on commit 86b1199

Please sign in to comment.