Permalink
Browse files

All compiled templates now use UTF8 as the default binary encoding.

On "entry" all data is converted to UTF8 (proper feedback about conversion errors is still missing though).
Tests are passing again (except eunit_render_types).
  • Loading branch information...
1 parent 8af01c5 commit 879c07e3a65ad79d3488f67986546b47433e9d52 @davide davide committed May 4, 2009
Showing with 21 additions and 19 deletions.
  1. +2 −2 src/sgte.erl
  2. +1 −1 src/sgte_gettext.erl
  3. +7 −5 src/sgte_parse.erl
  4. +1 −1 src/sgte_render.erl
  5. +7 −7 test/src/eunit_render.erl
  6. +3 −3 test/src/eunit_render_bin.erl
View
@@ -58,7 +58,7 @@
%% @end
%%--------------------------------------------------------------------
compile(T) when is_binary(T) ->
- sgte_parse:parse(binary_to_list(T));
+ sgte_parse:parse(unicode:characters_to_list(T));
compile(T) when is_list(T) ->
sgte_parse:parse(T).
@@ -146,7 +146,7 @@ render_bin(Compiled, Data) ->
%% @end
%%--------------------------------------------------------------------
gettext_strings(Template) when is_binary(Template) ->
- sgte_parse:gettext_strings(binary_to_list(Template));
+ sgte_parse:gettext_strings(unicode:characters_to_list(Template));
gettext_strings(FileName) ->
case file:read_file(FileName) of
{ok, Bin} ->
View
@@ -82,7 +82,7 @@ gettext_init(TargetDir, SrcFiles, Domain) ->
write_file(Fd, FName) ->
case file:read_file(FName) of
{ok, Bin} ->
- Keys = sgte_parse:gettext_strings(binary_to_list(Bin)),
+ Keys = sgte_parse:gettext_strings(unicode:characters_to_list(Bin)),
Entries = [gettext_entry(
{FName, {Key, LineNo}, Key})
|| {Key, LineNo} <- Keys],
View
@@ -52,13 +52,15 @@
%% template or an error.
%% @end
%%--------------------------------------------------------------------
-parse(Template) ->
- parse(Template, [], [], 1).
+parse(Template) when is_list(Template) ->
+ parse(Template, [], [], 1);
+parse(_Template) ->
+ throw(parse_undefined).
parse([], Parsed, [], _Line) ->
{ok, lists:reverse(Parsed)};
parse([], Parsed, Acc, _Line) ->
- Raw = list_to_binary(lists:reverse(Acc)),
+ Raw = unicode:characters_to_binary(lists:reverse(Acc)),
{ok, lists:reverse([Raw | Parsed])};
parse("$$" ++ T, Parsed, Acc, Line) ->
parse(T, Parsed, "$" ++ Acc, Line);
@@ -69,7 +71,7 @@ parse("$" ++ T, Parsed, Acc, Line) ->
[] ->
parse(Rest, [Keyword|Parsed], [], Line+LinesParsed);
_ ->
- Raw = list_to_binary(lists:reverse(Acc)),
+ Raw = unicode:characters_to_binary(lists:reverse(Acc)),
parse(Rest, [Keyword,Raw|Parsed], [], Line+LinesParsed)
end;
false ->
@@ -88,7 +90,7 @@ parse([{ift, _, ParsedLines}=H|T], Parsed, Acc, Line) ->
[] ->
parse(T, [H|Parsed], Acc, Line+ParsedLines);
_ ->
- parse(T, [H|[list_to_binary(lists:reverse(Acc))|Parsed]], [], Line+ParsedLines)
+ parse(T, [H|[unicode:characters_to_binary(lists:reverse(Acc))|Parsed]], [], Line+ParsedLines)
end;
parse([H|T], Parsed, Acc, Line) ->
case simple([H|T]) of
View
@@ -470,6 +470,6 @@ printable(Term, _Else) when is_float(Term) ->
printable(Term, _Else) when is_atom(Term) ->
atom_to_list(Term);
printable(Term, _Else) when is_binary(Term) ->
- binary_to_list(Term);
+ unicode:characters_to_list(Term);
printable(Term, Else) ->
Else(Term).
View
@@ -19,17 +19,17 @@ string_test_() ->
"foo, bar, baz",
<<" followed by ">>,
"my test data with unicode characters: àèìòù",
- list_to_binary(" and unicode characters àèìòù")],
+ unicode:characters_to_binary(" and unicode characters àèìòù")],
%% error test
Str2 = "This is a test:\n" ++
"$testFun()$ followed by $testData$ and unicode chars àèìòù",
{ok, Compiled2} = sgte:compile(Str2),
- Res2 = sgte:render_str(Compiled2, []),
- ResultStr2 = "This is a test:\n"++
- "[SGTE Warning: template: attribute - key 'testFun()' not found on line 2]"++
- " followed by "++
- "[SGTE Warning: template: attribute - key testData not found on line 2]"++
- " and unicode chars àèìòù",
+ Res2 = sgte:render_bin(Compiled2, []),
+ ResultStr2 = [<<"This is a test:\n">>,
+ <<"[SGTE Warning: template: attribute - key 'testFun()' not found on line 2]">>,
+ <<" followed by ">>,
+ <<"[SGTE Warning: template: attribute - key testData not found on line 2]">>,
+ unicode:characters_to_binary(" and unicode chars àèìòù")],
[?_assert(Res =:= ResultStr),
?_assert(Res2 =:= ResultStr2)].
@@ -18,8 +18,8 @@ string_test_() ->
ResultStr = [<<"This is a test:\n">>,
<<"foo, bar, baz">>,
<<" followed by ">>,
- list_to_binary("my test data with unicode characters: àèìòù"),
- list_to_binary(" and unicode characters àèìòù")],
+ unicode:characters_to_binary("my test data with unicode characters: àèìòù"),
+ unicode:characters_to_binary(" and unicode characters àèìòù")],
%% error test
Str2 = "This is a test:\n" ++
"$testFun()$ followed by $testData$ and unicode chars àèìòù",
@@ -29,7 +29,7 @@ string_test_() ->
<<"[SGTE Warning: template: attribute - key 'testFun()' not found on line 2]">>,
<<" followed by ">>,
<<"[SGTE Warning: template: attribute - key testData not found on line 2]">>,
- <<" and unicode chars àèìòù">>],
+ unicode:characters_to_binary(" and unicode chars àèìòù")],
[?_assert(Res =:= ResultStr),
?_assert(Res2 =:= ResultStr2)].

0 comments on commit 879c07e

Please sign in to comment.