Merge pull request #73 from malcolmstill/bad-encoding-replacement

Allow replacement of bad encoding

lib/csv/decoding/decoder.ex

@@ -38,6 +38,8 @@ defmodule CSV.Decoding.Decoder do
       When set to `false` (default), will use no header values.
       When set to anything but `false`, the resulting rows in the matrix will
       be maps instead of lists.
+  * `:replacement`   – The replacement string to use where lines have bad
+      encoding. Defaults to `nil`, which disables replacement.
 
   ## Examples
 

lib/csv/decoding/lexer.ex

@@ -17,13 +17,22 @@ defmodule CSV.Decoding.Lexer do
 
     * `:separator`   – The separator token to use, defaults to `?,`. Must be a
       codepoint.
+
+    * `:replacement`    – The replacement string to use where lines have bad
+      encoding. Defaults to `nil`, which disables replacement.
   """
 
   def lex({ line, index }, options \\ []) when is_list(options) do
     separator = options |> Keyword.get(:separator, @separator)
+    replacement = options |> Keyword.get(:replacement, @replacement)
 
     case String.valid?(line) do
-      false -> { :error, EncodingError, "Invalid encoding", index }
+      false ->
+        if replacement do
+          replace_bad_encoding(line, replacement) |> lex(index, separator)
+        else
+          { :error, EncodingError, "Invalid encoding", index }
+        end
       true -> lex(line, index, separator)
     end
   end
@@ -68,4 +77,11 @@ defmodule CSV.Decoding.Lexer do
   defp add_token(tokens, token) do
     tokens ++ [token]
   end
+
+  defp replace_bad_encoding(line, replacement) do
+    line
+    |> String.codepoints
+    |> Enum.map(fn codepoint -> if String.valid?(codepoint), do: codepoint, else: replacement end)
+    |> Enum.join
+  end
 end

lib/csv/defaults.ex

@@ -11,6 +11,7 @@ defmodule CSV.Defaults do
       @delimiter          << @carriage_return :: utf8 >> <> << @newline :: utf8 >>
       @double_quote       ?"
       @escape_max_lines   1000
+      @replacement        nil
     end
   end
 

test/decoding/baseline_exceptions_test.exs

@@ -22,6 +22,13 @@ defmodule DecodingTests.BaselineExceptionsTest do
     ]
   end
 
+  test "invalid encoding can be replaced" do
+    stream = [<<"a,", 255>>, "c,d"] |> to_stream
+    result = Decoder.decode(stream, replacement: "?") |> Enum.take(2)
+
+    assert result == [ok: ~w(a ?), ok: ~w(c d)]
+  end
+
   test "discards any state in the current message queues when halted" do
     stream = ["a,be", "c,d", "e,f", "g,h", "i,j", "k,l"] |> to_stream
     result = Decoder.decode(stream) |> Enum.take(2)