Skip to content

Commit

Permalink
Should only support UTF-8
Browse files Browse the repository at this point in the history
For the same reasons as BurntSushi/xsv#42 we
should only support UTF-8 and other encodings should be converted to
UTF-8 before processing.
  • Loading branch information
silasb committed May 15, 2019
1 parent 6becba8 commit 18c37ca
Show file tree
Hide file tree
Showing 3 changed files with 5 additions and 11 deletions.
12 changes: 5 additions & 7 deletions spec/csv-to-json_spec.cr
Expand Up @@ -94,15 +94,13 @@ describe Csv::To::Json do
out_io.to_s.should contain(%([\n{"field 1":"あ"}\n]\n))
end

it "will parse bad_data.tsv fixture" do
in_io = File.open("spec/fixtures/bad_data.tsv")
in_io.set_encoding(encoding: "latin1")
it "will throw exception for latin1 encoded file" do
in_io = File.open("spec/fixtures/latin1.tsv")
out_io = IO::Memory.new()

Csv::To::Json.run(in_io, out_io)

out_io.seek(0)
out_io.to_s.should contain(%([\n{"field 1":"Associação Paulista de Cirurgiões-Den"}\n]\n))
expect_raises(InvalidByteSequenceError) do
Csv::To::Json.run(in_io, out_io)
end
end

it "will parse \u0041 char" do
Expand Down
2 changes: 0 additions & 2 deletions spec/fixtures/bad_data.tsv

This file was deleted.

2 changes: 0 additions & 2 deletions src/csv-to-json.cr
Expand Up @@ -7,8 +7,6 @@ module Csv::To::Json
def self.run(in_io, out_io, options = {} of Symbol => String)
STDIN.blocking = true

# in_io.set_encoding(encoding: "UTF-8", invalid: :skip)

empty_value_replace_char = options.fetch(:empty_value_replace_char, "")
tail = options.delete :tail
delimiter = options.fetch(:delimiter, ',').as(Char)
Expand Down

0 comments on commit 18c37ca

Please sign in to comment.