Skip to content
Browse files

Unicode fixes

 - mostly backported from erlang.org website
 - unicode in templates now work
  • Loading branch information...
1 parent 0405d58 commit aa65b6f26f5b280c4716feb89fcd9ce2208e196c @goj goj committed Jun 17, 2010
View
5 lib/eptic-1.4/src/e_cache_disk.erl
@@ -62,9 +62,10 @@ read_file(File, Expander) ->
%%====================================================================
-spec(cache/2 :: (string(), atom()) -> xmlElement() | no_return()).
cache(File, xmerl_xs) ->
- XML = case xmerl_scan:file(File, []) of
+ XML = case xmerl_scan:file(File, [{encoding, "utf-8"}]) of
{error, enoent} ->
- case xmerl_scan:file(e_conf:template_root() ++ "/" ++ File, []) of
+ File1 = e_conf:template_root() ++ "/" ++ File,
+ case xmerl_scan:file(File1, [{encoding, "utf-8"}]) of
{error, Reason} ->
erlang:error({Reason, File});
{XML2, _} ->
View
2 lib/eptic-1.4/src/e_cache_ets.erl
@@ -81,7 +81,7 @@ valid_cache(File) ->
-spec(cache/2 :: (string(), atom()) -> term()).
cache(File, wpart_xs) ->
- XML = case xmerl_scan:file(File, []) of
+ XML = case xmerl_scan:file(File, [{encoding, "utf-8"}]) of
{error, Reason} ->
erlang:error(Reason);
{XML3, _} ->
View
46 lib/wpart-1.4/src/wpart_utils.erl
@@ -33,30 +33,34 @@
%% @spec url_encode(URL :: string()) -> EncodedURL :: string()
%% @doc Url-encodes a string.
%% All URLs in HTML documents must be URL encoded. <br/>
-%% This function has been taken from <i>Yaws</i> web server.
+%%
+%% This function escapes everything except url-safe characters
+%% [-._0-9a-zA-Z] AND SLASHES, which sould be very DWIM-y
+%% It should be consistent with Python3's urllib.parse.quote
%%
-spec(url_encode/1 :: (string()) -> string()).
-url_encode([H|T]) ->
- if
- H >= $a, $z >= H ->
- [H|url_encode(T)];
- H >= $A, $Z >= H ->
- [H|url_encode(T)];
- H >= $0, $9 >= H ->
- [H|url_encode(T)];
- H == $_; H == $.; H == $-; H == $/; H == $: -> % FIXME: more..
- [H|url_encode(T)];
- true ->
- case integer_to_hex(H) of
- [X, Y] ->
- [37, X, Y | url_encode(T)];
- [X] ->
- [37, $0, X | url_encode(T)]
- end
- end;
+url_encode(Str) when is_list(Str) ->
+ S = xmerl_ucs:to_utf8(Str),
+ url_encode_char(lists:reverse(S), []).
+
+url_encode_char([X | T], Acc) when X >= $0, X =< $9 ->
+ url_encode_char(T, [X | Acc]);
+url_encode_char([X | T], Acc) when X >= $a, X =< $z ->
+ url_encode_char(T, [X | Acc]);
+url_encode_char([X | T], Acc) when X >= $A, X =< $Z ->
+ url_encode_char(T, [X | Acc]);
+url_encode_char([X | T], Acc) when X == $-; X == $_; X == $. ->
+url_encode_char(T, [X | Acc]);
+url_encode_char([X | T], Acc) when X == $/ -> % this is DWIM bit
+ url_encode_char(T, [X | Acc]);
+url_encode_char([X | T], Acc) ->
+ url_encode_char(T, [$%, d2h(X bsr 4), d2h(X band 16#0f)
+ | Acc]);
+url_encode_char([], Acc) ->
+ Acc.
-url_encode([]) ->
- [].
+d2h(N) when N<10 -> N+$0;
+d2h(N) -> N+$a-10.
%%
%% @spec url_decode(URL :: string()) -> DecodedURL :: string()
View
5 lib/wparts-1.4/src/wpart_derived.erl
@@ -49,8 +49,9 @@ handle_call(E) ->
Inputs = build_tags(Type, FormType, Prefix),
Templater = fun(Tag) ->
- Flatten = "<?xml version=\"1.0\" encoding=\"utf-8\"?><div>" ++ Tag ++ "</div>",
- {XML, _} = xmerl_scan:string(Flatten),
+ Flatten = "<div>" ++ Tag ++ "</div>",
+ OH_NOEZ_UGLY_HACK = [{encoding, 'latin1'}],
+ {XML, _} = xmerl_scan:string(Flatten, OH_NOEZ_UGLY_HACK),
"<div>" ++ R = lists:flatten(wpart_xs:template(XML)),
">vid/<" ++ R2 = lists:reverse(R),
lists:reverse(R2)
View
14 lib/wparts-1.4/src/wtype_string.erl
@@ -45,8 +45,7 @@ validate({Types,undefined}) ->
end
end;
-validate({Types,RawString}) when is_list(RawString) ->
- String = utf8_api:ustring(RawString),
+validate({Types,String}) when is_list(String) ->
case wpart_valid:is_private(Types) of
true ->
{ok, String};
@@ -57,7 +56,12 @@ validate({Types,RawString}) when is_list(RawString) ->
{ok, String} ->
case check_regexp(String, Types) of
{ok, String} ->
- check_html(String, Types);
+ case check_html(String, Types) of
+ {ok, S} ->
+ {ok, unicode:characters_to_list(list_to_binary(S))};
+ Err ->
+ Err
+ end;
ErrorRegexp ->
ErrorRegexp
end;
@@ -72,7 +76,7 @@ validate({Types,RawString}) when is_list(RawString) ->
check_min_length(String, Types) ->
case lists:keysearch(min_length, 1, Types) of
{value, {min_length, Min}} ->
- N = length(String),
+ N = utf8_api:ulength(String),
if
N < Min ->
{error, {too_short, String}};
@@ -86,7 +90,7 @@ check_min_length(String, Types) ->
check_max_length(String, Types) ->
case lists:keysearch(max_length, 1, Types) of
{value, {max_length, Max}} ->
- N = length(String),
+ N = utf8_api:ulength(String),
if
N > Max ->
{error, {too_long, String}};
View
14 lib/wparts-1.4/src/wtype_text.erl
@@ -43,8 +43,7 @@ validate({Types, undefined}) ->
end
end;
-validate({Types,RawText}) when is_list(RawText) ->
- Text = utf8_api:ustring(RawText),
+validate({Types,Text}) when is_list(Text) ->
case wpart_valid:is_private(Types) of
true ->
{ok, Text};
@@ -53,7 +52,12 @@ validate({Types,RawText}) when is_list(RawText) ->
{ok, Text} ->
case check_max_length(Text, Types) of
{ok, Text} ->
- check_html(Text, Types);
+ case check_html(Text, Types) of
+ {ok, S} ->
+ {ok, unicode:characters_to_list(list_to_binary(S))};
+ Err ->
+ Err
+ end;
ErrorMax ->
ErrorMax
end;
@@ -71,7 +75,7 @@ validate({Types, Text}) ->
check_min_length(String, Types) ->
case lists:keysearch(min_length, 1, Types) of
{value, {min_length, Min}} ->
- X = length(String),
+ X = utf8_api:ulength(String),
if
X < Min ->
{error, {too_short, String}};
@@ -85,7 +89,7 @@ check_min_length(String, Types) ->
check_max_length(String, Types) ->
case lists:keysearch(max_length, 1, Types) of
{value, {max_length, Max}} ->
- X = length(String),
+ X = utf8_api:ulength(String),
if
X > Max ->
{error, {too_long, String}};

0 comments on commit aa65b6f

Please sign in to comment.
Something went wrong with that request. Please try again.