Permalink
Browse files

Merge pull request #5 from technophobia/master

json2csv feature, fixed tests
  • Loading branch information...
2 parents 954c21f + 0dce7e4 commit ed22c1fc0bcee75fde0d6e606086775d166aafd6 @darwin committed Sep 16, 2012
View
23 README.md
@@ -6,12 +6,12 @@ Tried to google for solution and surprisingly enough nothing solid existed.
## Solution
- export XLS as a CSV file (I use OpenOffice.org for this)
-- run `csv2json file.csv > file.json`
+- run `csv2json file.csv > file.json` (or `json2csv file.json > file.csv`)
- there is no step 3
### Sample
-note: make sure your XLS table has the first row with column names
+note: make sure your XLS table has the first row with column names, or use the -H option to provide some
This CSV file:
@@ -50,14 +50,31 @@ gets turned into this JSON:
### Usage
- Usage: csv2json [INPUT] [OPTIONS]
+Usage: csv2json [INPUT] [OPTIONS]
Specific options:
-s, --separator SEP Set separator character surrounded by single quotes (default is ',')
-o, --output FILE Write output to a file
+ -p, --pretty Pretty-format JSON output
+ -k, --skip-headers-from-file Ignore the headers (first line) in the file; use with --headers to set your own
+ -H, --headers HEADERS Supply list of headers, where no headers exist in the file, or where you're using -k to ignore them
-h, --help Show this message
-v, --version Show version
+We also provide `json2csv`, which converts in the opposite direction.
+
+ Usage: json2csv [INPUT] [OPTIONS]
+
+ Specific options:
+ -s, --separator SEP Set separator character surrounded by single quotes (default is ',')
+ -o, --output FILE Write output to a file
+ -H, --headers HEADERS Supply sorted list of headers, by which to order the columns in the CSV. These must match the key names in the JSON.
+ -h, --help Show this message
+ -v, --version Show version
+
+
+Note: JSON field order is explicitly unsorted, so if you want predictable field order for CSV output, use the -H option.
+
### Alternative usage
common usage is `csv2json file.csv > file.json`
View
7 bin/csv2json
@@ -35,14 +35,13 @@ module CSV2JSONRunner
options.pretty = true
end
- o.on("-H", "--force-headers HEADERS", "Supply list of headers, and skip first row of input; use to override headers in file") do |headers|
+ o.on("-k", "--skip-headers-from-file", "Ignore the headers (first line) in the file; use with --headers to set your own") do |headers|
if headers then
- options.headers = headers.split(",")
options.skipFirstRow = true
end
end
- o.on("-h", "--headers HEADERS", "Supply list of headers, where no headers exist in the file") do |headers|
+ o.on("-H", "--headers HEADERS", "Supply list of headers, where no headers exist in the file, or where you're using -k to ignore them") do |headers|
if headers then
options.headers = headers.split(",")
end
@@ -62,7 +61,7 @@ module CSV2JSONRunner
begin
opts.parse!(ARGV)
rescue
- die "Unable to parse options: #{$!}"
+ raise "Unable to parse options: #{$!}"
end
# initialize output handle
View
73 bin/json2csv
@@ -0,0 +1,73 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require 'optparse'
+require 'ostruct'
+require File.join(File.expand_path(File.dirname(__FILE__)), '..', 'lib', 'json2csv.rb') # this form is important for local development
+
+module JSON2CSVRunner
+
+ # command-line parsing
+ COMMAND = File.basename($0)
+ USAGE = "Usage: #{COMMAND} [INPUT] [OPTIONS]"
+
+ options = OpenStruct.new
+ options.output = "-"
+ options.separator = ","
+ options.headers = nil
+
+ opts = OptionParser.new do |o|
+ o.banner = USAGE
+ o.separator ""
+ o.separator "Specific options:"
+
+ o.on("-s", "--separator SEP", "Set separator character surrounded by single quotes (default is ',')") do |sep|
+ options.separator = sep
+ end
+
+ o.on("-o", "--output FILE", "Write output to a file") do |fn|
+ options.output = fn
+ end
+
+ o.on("-H", "--headers HEADERS", "Supply sorted list of headers, by which to order the columns in the CSV. These must match the key names in the JSON.") do |headers|
+ if headers then
+ options.headers = headers.split(",")
+ end
+ end
+
+ o.on_tail("-h", "--help", "Show this message") do
+ puts o
+ exit
+ end
+
+ o.on_tail("-v", "--version", "Show version") do
+ puts CSV2JSON::VERSION
+ exit
+ end
+ end
+
+ begin
+ opts.parse!(ARGV)
+ rescue
+ raise "Unable to parse options: #{$!}"
+ end
+
+ # initialize output handle
+ if options.output == "-"
+ OUT = $stdout.clone
+ else
+ OUT = File.open(options.output, "w")
+ end
+
+ if ARGV.size > 0
+ IN = File.read(ARGV[0])
+ else
+ IN = StringIO.new($stdin.read) # cannot be just $stdin.clone because FasterCSV is seeking in file :-(
+ end
+
+ # run the command
+ JSON2CSV.parse(IN, OUT, options.headers, {:col_sep => options.separator})
+
+ # leave in peace
+ OUT.flush
+end
View
19 csv2json.gemspec
@@ -13,10 +13,10 @@ Gem::Specification.new do |s|
s.default_executable = %q{csv2json}
s.description = %q{handy for converting xls files to json}
s.email = %q{antonin@hildebrand.cz}
- s.executables = ["csv2json"]
+ s.executables = ["csv2json", "json2csv"]
s.extra_rdoc_files = [
"LICENSE",
- "README.md"
+ "README.md"
]
s.files = [
".document",
@@ -25,9 +25,11 @@ Gem::Specification.new do |s|
"README.md",
"Rakefile",
"bin/csv2json",
+ "bin/json2csv",
"csv2json.gemspec",
"lib/csv2json-version.rb",
"lib/csv2json.rb",
+ "lib/json2csv.rb",
"test/fixtures/addresses.json",
"test/fixtures/addresses_comma.csv",
"test/fixtures/addresses_pipe.csv",
@@ -40,8 +42,13 @@ Gem::Specification.new do |s|
"test/fixtures/population_comma.csv",
"test/fixtures/population_pipe.csv",
"test/fixtures/population_semicolon.csv",
+ "test/json2csv-fixtures/photos.json",
+ "test/json2csv-fixtures/photos_comma.csv",
+ "test/json2csv-fixtures/photos_pipe.csv",
+ "test/json2csv-fixtures/photos_semicolon.csv",
"test/helper.rb",
- "test/test_csv2json.rb"
+ "test/test_csv2json.rb",
+ "test/test_json2csv.rb"
]
s.homepage = %q{http://github.com/darwin/csv2json}
s.rdoc_options = ["--charset=UTF-8"]
@@ -50,7 +57,8 @@ Gem::Specification.new do |s|
s.summary = %q{.csv to .json converter}
s.test_files = [
"test/helper.rb",
- "test/test_csv2json.rb"
+ "test/test_csv2json.rb",
+ "test/test_json2csv.rb"
]
if s.respond_to? :specification_version then
@@ -60,15 +68,18 @@ Gem::Specification.new do |s|
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
s.add_runtime_dependency(%q<json>, [">= 0"])
s.add_runtime_dependency(%q<fastercsv>, [">= 0"])
+ s.add_runtime_dependency(%q<orderedhash>, [">= 0"])
s.add_development_dependency(%q<shoulda>, [">= 0"])
else
s.add_dependency(%q<json>, [">= 0"])
s.add_dependency(%q<fastercsv>, [">= 0"])
+ s.add_dependency(%q<orderedhash>, [">= 0"])
s.add_dependency(%q<shoulda>, [">= 0"])
end
else
s.add_dependency(%q<json>, [">= 0"])
s.add_dependency(%q<fastercsv>, [">= 0"])
+ s.add_dependency(%q<orderedhash>, [">= 0"])
s.add_dependency(%q<shoulda>, [">= 0"])
end
end
View
5 lib/csv2json.rb
@@ -1,7 +1,8 @@
require 'rubygems'
require 'fastercsv'
require 'json'
-require 'csv2json-version.rb'
+require 'orderedhash'
+require File.join(File.expand_path(File.dirname(__FILE__)), '..', 'lib', 'csv2json-version.rb')
module CSV2JSON
@@ -28,7 +29,7 @@ def parse(input, output, headers=nil, csvOptions={}, gemOptions={})
end
# build JSON snippet and append it to the result
- snippet = Hash.new
+ snippet = OrderedHash.new
headers.each_index { |i| snippet[headers[i]] = self.convert(row[i]) }
result << snippet
end
View
36 lib/json2csv.rb
@@ -0,0 +1,36 @@
+require 'rubygems'
+require 'fastercsv'
+require 'json'
+require File.join(File.expand_path(File.dirname(__FILE__)), '..', 'lib', 'csv2json-version.rb')
+
+module JSON2CSV
+
+ def parse(input, output, headers=nil, csvOptions=nil)
+ json = JSON.parse(input)
+
+ unless headers
+ headers = json[0].keys
+ end
+
+ outputCSV = FasterCSV.generate(csvOptions) do |csv|
+ csv << headers
+
+ json.each do |obj|
+ sortedValues = Array.new(headers.length)
+
+ obj.each do |key, val|
+ headers.each_index { |i|
+ sortedValues[i] = val if headers[i] == key
+ }
+ end
+
+ csv << sortedValues
+ end
+ end
+
+ output << outputCSV
+ end
+
+ module_function :parse
+
+end
View
1 test/helper.rb
@@ -5,6 +5,7 @@
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
$LOAD_PATH.unshift(File.dirname(__FILE__))
require 'csv2json'
+require 'json2csv'
class Test::Unit::TestCase
end
View
122 test/json2csv-fixtures/photos.json
@@ -0,0 +1,122 @@
+[
+ {
+ "thumbwidth": 75,
+ "imgsrc": "paris_01.jpg",
+ "thumbsrc": "paris_01.jpg",
+ "height": 262,
+ "imgpath": "images/",
+ "thumbheight": 56,
+ "thumbpath": "thumbnails/",
+ "width": 350
+ },
+ {
+ "thumbwidth": 75,
+ "imgsrc": "paris_02.jpg",
+ "thumbsrc": "paris_02.jpg",
+ "height": 350,
+ "imgpath": "images/",
+ "thumbheight": 56,
+ "thumbpath": "thumbnails/",
+ "width": 262
+ },
+ {
+ "thumbwidth": 75,
+ "imgsrc": "paris_03.jpg",
+ "thumbsrc": "paris_03.jpg",
+ "height": 262,
+ "imgpath": "images/",
+ "thumbheight": 56,
+ "thumbpath": "thumbnails/",
+ "width": 350
+ },
+ {
+ "thumbwidth": 56,
+ "imgsrc": "paris_04.jpg",
+ "thumbsrc": "paris_04.jpg",
+ "height": 350,
+ "imgpath": "images/",
+ "thumbheight": 75,
+ "thumbpath": "thumbnails/",
+ "width": 262
+ },
+ {
+ "thumbwidth": 56,
+ "imgsrc": "paris_05.jpg",
+ "thumbsrc": "paris_05.jpg",
+ "height": 350,
+ "imgpath": "images/",
+ "thumbheight": 75,
+ "thumbpath": "thumbnails/",
+ "width": 262
+ },
+ {
+ "thumbwidth": 75,
+ "imgsrc": "paris_06.jpg",
+ "thumbsrc": "paris_06.jpg",
+ "height": 262,
+ "imgpath": "images/",
+ "thumbheight": 56,
+ "thumbpath": "thumbnails/",
+ "width": 350
+ },
+ {
+ "thumbwidth": 56,
+ "imgsrc": "paris_07.jpg",
+ "thumbsrc": "paris_07.jpg",
+ "height": 350,
+ "imgpath": "images/",
+ "thumbheight": 75,
+ "thumbpath": "thumbnails/",
+ "width": 262
+ },
+ {
+ "thumbwidth": 56,
+ "imgsrc": "paris_08.jpg",
+ "thumbsrc": "paris_08.jpg",
+ "height": 350,
+ "imgpath": "images/",
+ "thumbheight": 75,
+ "thumbpath": "thumbnails/",
+ "width": 262
+ },
+ {
+ "thumbwidth": 56,
+ "imgsrc": "paris_09.jpg",
+ "thumbsrc": "paris_09.jpg",
+ "height": 350,
+ "imgpath": "images/",
+ "thumbheight": 75,
+ "thumbpath": "thumbnails/",
+ "width": 262
+ },
+ {
+ "thumbwidth": 75,
+ "imgsrc": "paris_10.jpg",
+ "thumbsrc": "paris_10.jpg",
+ "height": 262,
+ "imgpath": "images/",
+ "thumbheight": 56,
+ "thumbpath": "thumbnails/",
+ "width": 350
+ },
+ {
+ "thumbwidth": 75,
+ "imgsrc": "paris_11.jpg",
+ "thumbsrc": "paris_11.jpg",
+ "height": 262,
+ "imgpath": "images/",
+ "thumbheight": 56,
+ "thumbpath": "thumbnails/",
+ "width": 350
+ },
+ {
+ "thumbwidth": 75,
+ "imgsrc": "paris_12.jpg",
+ "thumbsrc": "paris_12.jpg",
+ "height": 262,
+ "imgpath": "images/",
+ "thumbheight": 56,
+ "thumbpath": "thumbnails/",
+ "width": 350
+ }
+]
View
13 test/json2csv-fixtures/photos_comma.csv
@@ -0,0 +1,13 @@
+thumbpath,imgpath,imgsrc,width,height,thumbsrc,thumbwidth,thumbheight
+thumbnails/,images/,paris_01.jpg,350,262,paris_01.jpg,75,56
+thumbnails/,images/,paris_02.jpg,262,350,paris_02.jpg,75,56
+thumbnails/,images/,paris_03.jpg,350,262,paris_03.jpg,75,56
+thumbnails/,images/,paris_04.jpg,262,350,paris_04.jpg,56,75
+thumbnails/,images/,paris_05.jpg,262,350,paris_05.jpg,56,75
+thumbnails/,images/,paris_06.jpg,350,262,paris_06.jpg,75,56
+thumbnails/,images/,paris_07.jpg,262,350,paris_07.jpg,56,75
+thumbnails/,images/,paris_08.jpg,262,350,paris_08.jpg,56,75
+thumbnails/,images/,paris_09.jpg,262,350,paris_09.jpg,56,75
+thumbnails/,images/,paris_10.jpg,350,262,paris_10.jpg,75,56
+thumbnails/,images/,paris_11.jpg,350,262,paris_11.jpg,75,56
+thumbnails/,images/,paris_12.jpg,350,262,paris_12.jpg,75,56
View
13 test/json2csv-fixtures/photos_pipe.csv
@@ -0,0 +1,13 @@
+thumbpath|imgpath|imgsrc|width|height|thumbsrc|thumbwidth|thumbheight
+thumbnails/|images/|paris_01.jpg|350|262|paris_01.jpg|75|56
+thumbnails/|images/|paris_02.jpg|262|350|paris_02.jpg|75|56
+thumbnails/|images/|paris_03.jpg|350|262|paris_03.jpg|75|56
+thumbnails/|images/|paris_04.jpg|262|350|paris_04.jpg|56|75
+thumbnails/|images/|paris_05.jpg|262|350|paris_05.jpg|56|75
+thumbnails/|images/|paris_06.jpg|350|262|paris_06.jpg|75|56
+thumbnails/|images/|paris_07.jpg|262|350|paris_07.jpg|56|75
+thumbnails/|images/|paris_08.jpg|262|350|paris_08.jpg|56|75
+thumbnails/|images/|paris_09.jpg|262|350|paris_09.jpg|56|75
+thumbnails/|images/|paris_10.jpg|350|262|paris_10.jpg|75|56
+thumbnails/|images/|paris_11.jpg|350|262|paris_11.jpg|75|56
+thumbnails/|images/|paris_12.jpg|350|262|paris_12.jpg|75|56
View
13 test/json2csv-fixtures/photos_semicolon.csv
@@ -0,0 +1,13 @@
+thumbpath;imgpath;imgsrc;width;height;thumbsrc;thumbwidth;thumbheight
+thumbnails/;images/;paris_01.jpg;350;262;paris_01.jpg;75;56
+thumbnails/;images/;paris_02.jpg;262;350;paris_02.jpg;75;56
+thumbnails/;images/;paris_03.jpg;350;262;paris_03.jpg;75;56
+thumbnails/;images/;paris_04.jpg;262;350;paris_04.jpg;56;75
+thumbnails/;images/;paris_05.jpg;262;350;paris_05.jpg;56;75
+thumbnails/;images/;paris_06.jpg;350;262;paris_06.jpg;75;56
+thumbnails/;images/;paris_07.jpg;262;350;paris_07.jpg;56;75
+thumbnails/;images/;paris_08.jpg;262;350;paris_08.jpg;56;75
+thumbnails/;images/;paris_09.jpg;262;350;paris_09.jpg;56;75
+thumbnails/;images/;paris_10.jpg;350;262;paris_10.jpg;75;56
+thumbnails/;images/;paris_11.jpg;350;262;paris_11.jpg;75;56
+thumbnails/;images/;paris_12.jpg;350;262;paris_12.jpg;75;56
View
11 test/test_csv2json.rb
@@ -1,20 +1,25 @@
require 'helper'
-require File.join(File.expand_path(File.dirname(__FILE__)), '..', 'lib', 'csv2json.rb') # this form is important for local development
class TestCsv2json < Test::Unit::TestCase
SEPS = {:comma => ',', :pipe => '|', :semicolon => ';'}
+
should "parse some test files" do
fixtures_dir = File.expand_path(File.join(File.dirname(__FILE__), 'fixtures'))
+
Dir.chdir(fixtures_dir) do
Dir.glob('*.csv') do |filename|
filename_parts = File.basename(filename, ".csv").split('_')
json_template = filename_parts[0] + '.json'
+
+ print "Testing csv2json against #{filename}\n"
+
File.open(filename, "r") do |input|
output = StringIO.new()
- CSV2JSON.parse(input, output, nil, :col_sep => SEPS[filename_parts[1].to_sym] )
+ CSV2JSON.parse(input, output, nil, {:col_sep => SEPS[filename_parts[1].to_sym]}, {:pretty => true} )
template = File.read(json_template)
+
output.pos = 0
- assert template == output.read
+ assert "Content doesn't match", template == output.read
end
end
end
View
32 test/test_json2csv.rb
@@ -0,0 +1,32 @@
+require 'helper'
+
+class TestJson2Csv < Test::Unit::TestCase
+ SEPS = {:comma => ',', :pipe => '|', :semicolon => ';'}
+
+ should "parse some test files" do
+ fixtures_dir = File.expand_path(File.join(File.dirname(__FILE__), 'json2csv-fixtures'))
+
+ Dir.chdir(fixtures_dir) do
+ Dir.glob('*.json') do |filename|
+ basename = File.basename(filename, ".json")
+
+ print "Testing json2csv against #{filename}\n"
+
+ SEPS.each do |key, val|
+ template = "#{basename}_#{key.to_sym}.csv"
+ print " ... for comparison with #{template} "
+ headers = ['thumbpath','imgpath','imgsrc','width','height','thumbsrc','thumbwidth','thumbheight']
+
+ input = File.read(filename)
+ output = StringIO.new()
+ JSON2CSV.parse(input, output, headers, {:col_sep => val} )
+
+ output.pos = 0
+ fileToCompareTo = File.read(template)
+ assert_equal output.read.strip, fileToCompareTo.strip
+ print "OK\n"
+ end
+ end
+ end
+ end
+end

0 comments on commit ed22c1f

Please sign in to comment.