Skip to content

Commit

Permalink
fix(deprecations): Fix deprecations, string.trim, and format code
Browse files Browse the repository at this point in the history
  • Loading branch information
raksonibs committed Aug 10, 2018
1 parent a26aef1 commit 94cf5b3
Show file tree
Hide file tree
Showing 35 changed files with 907 additions and 696 deletions.
4 changes: 4 additions & 0 deletions .formatter.exs
@@ -0,0 +1,4 @@
# Used by "mix format"
[
inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}"]
]
30 changes: 21 additions & 9 deletions lib/csv.ex
Expand Up @@ -184,6 +184,7 @@ defmodule CSV do
case options |> Keyword.get(:preprocessor) do
:none ->
stream |> Preprocessing.None.process(options)

_ ->
stream |> Preprocessing.Lines.process(options)
end
Expand All @@ -195,26 +196,38 @@ defmodule CSV do
stream |> Stream.map(&yield_or_raise!(&1, escape_max_lines))
end

defp yield_or_raise!({ :error, EscapeSequenceError, escape_sequence, index }, escape_max_lines) do
raise EscapeSequenceError, escape_sequence: escape_sequence, line: index + 1, escape_max_lines: escape_max_lines
defp yield_or_raise!({:error, EscapeSequenceError, escape_sequence, index}, escape_max_lines) do
raise EscapeSequenceError,
escape_sequence: escape_sequence,
line: index + 1,
escape_max_lines: escape_max_lines
end
defp yield_or_raise!({ :error, mod, message, index }, _) do

defp yield_or_raise!({:error, mod, message, index}, _) do
raise mod, message: message, line: index + 1
end
defp yield_or_raise!({ :ok, row }, _), do: row

defp yield_or_raise!({:ok, row}, _), do: row

defp inline_errors!(stream, options) do
escape_max_lines = options |> Keyword.get(:escape_max_lines, @escape_max_lines)

stream |> Stream.map(&yield_or_inline!(&1, escape_max_lines))
end

defp yield_or_inline!({ :error, EscapeSequenceError, escape_sequence, index }, escape_max_lines) do
{ :error, EscapeSequenceError.exception(escape_sequence: escape_sequence, line: index + 1, escape_max_lines: escape_max_lines).message }
defp yield_or_inline!({:error, EscapeSequenceError, escape_sequence, index}, escape_max_lines) do
{:error,
EscapeSequenceError.exception(
escape_sequence: escape_sequence,
line: index + 1,
escape_max_lines: escape_max_lines
).message}
end
defp yield_or_inline!({ :error, errormod, message, index }, _) do
{ :error, errormod.exception(message: message, line: index + 1).message }

defp yield_or_inline!({:error, errormod, message, index}, _) do
{:error, errormod.exception(message: message, line: index + 1).message}
end

defp yield_or_inline!(value, _), do: value

@doc """
Expand Down Expand Up @@ -251,5 +264,4 @@ defmodule CSV do
def encode(stream, options \\ []) do
Encoder.encode(stream, options)
end

end
108 changes: 60 additions & 48 deletions lib/csv/decoding/decoder.ex
@@ -1,5 +1,4 @@
defmodule CSV.Decoding.Decoder do

@moduledoc ~S"""
The Decoder CSV module sends lines of delimited values from a stream to the
parser and converts rows coming from the CSV parser module to a consumable
Expand Down Expand Up @@ -95,7 +94,7 @@ defmodule CSV.Decoding.Decoder do
options = options |> with_defaults

stream
|> Stream.with_index
|> Stream.with_index()
|> with_headers(options)
|> with_row_length(options)
|> decode_rows(options)
Expand All @@ -104,86 +103,99 @@ defmodule CSV.Decoding.Decoder do
defp with_defaults(options) do
options
|> Keyword.merge(
num_workers: options |> Keyword.get(:num_workers, Defaults.num_workers),
headers: options |> Keyword.get(:headers, false)
)
num_workers: options |> Keyword.get(:num_workers, Defaults.num_workers()),
headers: options |> Keyword.get(:headers, false)
)
end

defp decode_rows(stream, options) do
stream
|> ParallelStream.map(&(decode_row(&1, options)), options)
|> ParallelStream.map(&decode_row(&1, options), options)
end

defp decode_row({ nil, 0 }, _) do
{ :ok, [] }
defp decode_row({nil, 0}, _) do
{:ok, []}
end
defp decode_row({ line, index, headers, row_length }, options) do
with { :ok, parsed, _ } <- parse_row({ line, index }, options),
{ :ok, _ } <- validate_row_length({ parsed, index }, row_length),
do: build_row(parsed, headers)

defp decode_row({line, index, headers, row_length}, options) do
with {:ok, parsed, _} <- parse_row({line, index}, options),
{:ok, _} <- validate_row_length({parsed, index}, row_length),
do: build_row(parsed, headers)
end

defp parse_row({ line, index}, options) do
with { :ok, lex, _ } <- Lexer.lex({ line, index }, options),
do: Parser.parse({ lex, index }, options)
defp parse_row({line, index}, options) do
with {:ok, lex, _} <- Lexer.lex({line, index}, options),
do: Parser.parse({lex, index}, options)
end

defp build_row(data, headers) when is_list(headers) do
{ :ok, headers |> Enum.zip(data) |> Enum.into(%{}) }
{:ok, headers |> Enum.zip(data) |> Enum.into(%{})}
end
defp build_row(data, _), do: { :ok, data }

defp build_row(data, _), do: {:ok, data}

defp with_headers(stream, options) do
headers = options |> Keyword.get(:headers, false)
stream |> Stream.transform({ headers, options }, &add_headers/2)
stream |> Stream.transform({headers, options}, &add_headers/2)
end

defp add_headers({ line, 0 }, { headers, options }) when is_list(headers) do
{ [{ line, 0, headers }], { headers, options } }
defp add_headers({line, 0}, {headers, options}) when is_list(headers) do
{[{line, 0, headers}], {headers, options}}
end
defp add_headers({ line, 0 }, { true, options }) do
case parse_row({ line, 0 }, options) do
{ :ok, headers, _ } ->
{ [], { headers, options } }

defp add_headers({line, 0}, {true, options}) do
case parse_row({line, 0}, options) do
{:ok, headers, _} ->
{[], {headers, options}}

_ ->
{ [], { false, options } }
{[], {false, options}}
end
end
defp add_headers({ line, 0 }, { false, options }) do
{ [{ line, 0, false }], { false, options } }

defp add_headers({line, 0}, {false, options}) do
{[{line, 0, false}], {false, options}}
end
defp add_headers({ line, index }, { headers, options }) do
{ [{ line, index, headers }], { headers, options } }

defp add_headers({line, index}, {headers, options}) do
{[{line, index, headers}], {headers, options}}
end

defp with_row_length(stream, options) do
stream |> Stream.transform({ nil, options }, &add_row_length/2)
stream |> Stream.transform({nil, options}, &add_row_length/2)
end

defp add_row_length({ line, 0, false }, { row_length, options }) do
case parse_row({ line, 0 }, options) do
{ :ok, row, _ } ->
row_length = row |> Enum.count
{ [{ line, 0, false, row_length }], { row_length, options } }
defp add_row_length({line, 0, false}, {row_length, options}) do
case parse_row({line, 0}, options) do
{:ok, row, _} ->
row_length = row |> Enum.count()
{[{line, 0, false, row_length}], {row_length, options}}

_ ->
{ [{ line, 0, false, false }], { row_length, options } }
{[{line, 0, false, false}], {row_length, options}}
end
end
defp add_row_length({ line, _, headers }, { nil, options }) when is_list(headers) do
row_length = headers |> Enum.count
{ [{ line, 0, headers, row_length }], { row_length, options } }

defp add_row_length({line, _, headers}, {nil, options}) when is_list(headers) do
row_length = headers |> Enum.count()
{[{line, 0, headers, row_length}], {row_length, options}}
end
defp add_row_length({ line, index, headers }, { row_length, options }) do
{ [{ line, index, headers, row_length }], { row_length, options } }

defp add_row_length({line, index, headers}, {row_length, options}) do
{[{line, index, headers, row_length}], {row_length, options}}
end

defp validate_row_length({ data, _ }, false ), do: { :ok, data }
defp validate_row_length({ data, _ }, nil ), do: { :ok, data }
defp validate_row_length({ data, index }, expected_length) do
case data |> Enum.count do
^expected_length -> { :ok, data }
actual_length -> { :error, RowLengthError, "Row has length #{actual_length} - expected length #{expected_length}", index }
defp validate_row_length({data, _}, false), do: {:ok, data}
defp validate_row_length({data, _}, nil), do: {:ok, data}

defp validate_row_length({data, index}, expected_length) do
case data |> Enum.count() do
^expected_length ->
{:ok, data}

actual_length ->
{:error, RowLengthError,
"Row has length #{actual_length} - expected length #{expected_length}", index}
end
end

end
67 changes: 44 additions & 23 deletions lib/csv/decoding/lexer.ex
Expand Up @@ -22,7 +22,7 @@ defmodule CSV.Decoding.Lexer do
encoding. Defaults to `nil`, which disables replacement.
"""

def lex({ line, index }, options \\ []) when is_list(options) do
def lex({line, index}, options \\ []) when is_list(options) do
separator = options |> Keyword.get(:separator, @separator)
replacement = options |> Keyword.get(:replacement, @replacement)

Expand All @@ -31,57 +31,78 @@ defmodule CSV.Decoding.Lexer do
if replacement do
replace_bad_encoding(line, replacement) |> lex(index, separator)
else
{ :error, EncodingError, "Invalid encoding", index }
{:error, EncodingError, "Invalid encoding", index}
end
true -> lex(line, index, separator)

true ->
lex(line, index, separator)
end
end

defp lex(line, index, separator) do
case lex([], nil, line, separator) do
{ :ok, tokens } -> { :ok, tokens, index }
{:ok, tokens} -> {:ok, tokens, index}
end
end

defp lex(tokens, { :delimiter, value }, << @newline :: utf8 >> <> tail, separator) do
lex(tokens, { :delimiter, value <> << @newline :: utf8 >> }, tail, separator)
defp lex(tokens, {:delimiter, value}, <<@newline::utf8>> <> tail, separator) do
lex(tokens, {:delimiter, value <> <<@newline::utf8>>}, tail, separator)
end
defp lex(tokens, current_token, << @newline :: utf8 >> <> tail, separator) do
lex(tokens |> add_token(current_token), { :delimiter, << @newline :: utf8 >> }, tail, separator)

defp lex(tokens, current_token, <<@newline::utf8>> <> tail, separator) do
lex(tokens |> add_token(current_token), {:delimiter, <<@newline::utf8>>}, tail, separator)
end
defp lex(tokens, current_token, << @carriage_return :: utf8 >> <> tail, separator) do
lex(tokens |> add_token(current_token), { :delimiter, << @carriage_return :: utf8 >> }, tail, separator)

defp lex(tokens, current_token, <<@carriage_return::utf8>> <> tail, separator) do
lex(
tokens |> add_token(current_token),
{:delimiter, <<@carriage_return::utf8>>},
tail,
separator
)
end
defp lex(tokens, current_token, << @double_quote :: utf8 >> <> tail, separator) do
lex(tokens |> add_token(current_token), { :double_quote, << @double_quote :: utf8 >> }, tail, separator)

defp lex(tokens, current_token, <<@double_quote::utf8>> <> tail, separator) do
lex(
tokens |> add_token(current_token),
{:double_quote, <<@double_quote::utf8>>},
tail,
separator
)
end
defp lex(tokens, current_token, << head :: utf8 >> <> tail, separator) when head == separator do
lex(tokens |> add_token(current_token), { :separator, << separator :: utf8 >> }, tail, separator)

defp lex(tokens, current_token, <<head::utf8>> <> tail, separator) when head == separator do
lex(tokens |> add_token(current_token), {:separator, <<separator::utf8>>}, tail, separator)
end
defp lex(tokens, { :content, value }, << head :: utf8 >> <> tail, separator) do
lex(tokens, { :content, value <> << head :: utf8 >> }, tail, separator)

defp lex(tokens, {:content, value}, <<head::utf8>> <> tail, separator) do
lex(tokens, {:content, value <> <<head::utf8>>}, tail, separator)
end
defp lex(tokens, nil, << head :: utf8 >> <> tail, separator) do
lex(tokens, { :content, << head :: utf8 >> }, tail, separator)

defp lex(tokens, nil, <<head::utf8>> <> tail, separator) do
lex(tokens, {:content, <<head::utf8>>}, tail, separator)
end
defp lex(tokens, current_token, << head :: utf8 >> <> tail, separator) do
lex(tokens |> add_token(current_token), { :content, << head :: utf8 >> }, tail, separator)

defp lex(tokens, current_token, <<head::utf8>> <> tail, separator) do
lex(tokens |> add_token(current_token), {:content, <<head::utf8>>}, tail, separator)
end

defp lex(tokens, current_token, "", _) do
{ :ok, tokens |> add_token(current_token) }
{:ok, tokens |> add_token(current_token)}
end

defp add_token(tokens, nil) do
tokens
end

defp add_token(tokens, token) do
tokens ++ [token]
end

defp replace_bad_encoding(line, replacement) do
line
|> String.codepoints
|> String.codepoints()
|> Enum.map(fn codepoint -> if String.valid?(codepoint), do: codepoint, else: replacement end)
|> Enum.join
|> Enum.join()
end
end

0 comments on commit 94cf5b3

Please sign in to comment.