Merge pull request #12 from tjarvstrand/source-file-encoding-support

Add support for latin1 encoded source files
inaka · Apr 25, 2016 · bce3671 · bce3671
2 parents 58f0a9e + 171242c
commit bce3671
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 7 deletions.
diff --git a/src/ktn_code.erl b/src/ktn_code.erl
@@ -9,7 +9,8 @@
          parse_tree/1,
          parse_tree/2,
          eval/1,
-         consult/1
+         consult/1,
+         to_str/1
         ]).
 
 %% Getters
@@ -187,20 +188,37 @@ content(#{content := Content}) ->
 content(_Node) ->
     [].
 
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%%% Internal
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
 -spec to_str(binary() | list() | atom()) -> string().
 to_str(Arg) when is_binary(Arg) ->
-    unicode:characters_to_list(Arg);
+    Encoding = source_encoding(Arg),
+    unicode:characters_to_list(Arg, Encoding);
 to_str(Arg) when is_atom(Arg) ->
     atom_to_list(Arg);
 to_str(Arg) when is_integer(Arg) ->
     integer_to_list(Arg);
 to_str(Arg) when is_list(Arg) ->
     Arg.
 
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%% Internal
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+-spec source_encoding(binary() | list()) -> latin1 | utf8.
+source_encoding(Source) ->
+    Re = ".*\n?.*(coding *[:=] *(?<encoding>[-a-zA-Z0-9]+))",
+    ReOpts = [firstline, {capture, all_names, list}],
+    case re:run(Source, Re, ReOpts) of
+        {match, [Encoding]} ->
+            case string:to_lower(Encoding) of
+                "latin-1" ->
+                    latin1;
+                _ ->
+                    utf8
+            end;
+        nomatch ->
+            utf8
+    end.
+
 -spec is_dot(tuple()) -> boolean().
 is_dot({dot, _}) -> true;
 is_dot(_) -> false.

diff --git a/test/ktn_code_SUITE.erl b/test/ktn_code_SUITE.erl
@@ -9,7 +9,9 @@
 -export([
          consult/1,
          beam_to_string/1,
-         parse_tree/1
+         parse_tree/1,
+         latin1_parse_tree/1,
+         to_string/1
         ]).
 
 -define(EXCLUDED_FUNS,
@@ -73,3 +75,19 @@ parse_tree(_Config) ->
 
     #{type := root,
       content := [ModuleNode]} = ktn_code:parse_tree("-module(x).").
+
+
+latin1_parse_tree(_Config) ->
+    error = try ktn_code:parse_tree(<<"%% �\n-module(x).">>)
+            catch error:_ -> error
+            end,
+    #{type := root,
+      content := _} = ktn_code:parse_tree(<<"%% -*- coding: latin-1 -*-\n"
+                                            "%% �"
+                                            "-module(x).">>).
+
+-spec to_string(config()) -> any().
+to_string(_Config) ->
+    "1" = ktn_code:to_str(1),
+    "hello" = ktn_code:to_str(<<"hello">>),
+    "atom" = ktn_code:to_str(atom).