Skip to content
Browse files

Initial Import

  • Loading branch information...
0 parents commit e28c0dad736674b0e9be641e2faf253b775cdc92 Wilco van Duinkerken committed Nov 4, 2009
Showing with 430 additions and 0 deletions.
  1. +4 −0 History.txt
  2. +16 −0 Manifest.txt
  3. +1 −0 PostInstall.txt
  4. +67 −0 README.rdoc
  5. +21 −0 Rakefile
  6. +4 −0 lib/csv_importer.rb
  7. +79 −0 lib/csv_importer/csv_importer.rb
  8. +10 −0 script/console
  9. +14 −0 script/destroy
  10. +14 −0 script/generate
  11. +71 −0 script/txt2html
  12. +6 −0 spec/csv/sample.csv
  13. +83 −0 spec/csv_importer_spec.rb
  14. +1 −0 spec/spec.opts
  15. +18 −0 spec/spec_helper.rb
  16. +21 −0 tasks/rspec.rake
4 History.txt
@@ -0,0 +1,4 @@
+=== 0.0.1 2009-11-04
+
+* 1 major enhancement:
+ * Initial release
16 Manifest.txt
@@ -0,0 +1,16 @@
+History.txt
+Manifest.txt
+PostInstall.txt
+Rakefile
+README.rdoc
+lib/csv_importer.rb
+lib/csv_importer/csv_importer.rb
+script/console
+script/destroy
+script/generate
+script/txt2html
+spec/csv/sample.csv
+spec/csv_importer_spec.rb
+spec/spec.opts
+spec/spec_helper.rb
+tasks/rspec.rake
1 PostInstall.txt
@@ -0,0 +1 @@
+For more information on csv_importer, see http://csv_importer.rubyforge.org
67 README.rdoc
@@ -0,0 +1,67 @@
+== DESCRIPTION:
+
+Ever needed to import csv files where every row needs to be converted into a model?
+
+The CSV importer turns every row of a CSV file into an object. Each column is matched and tested against a given class.
+You can provide a dictionary with translations between the CSV column names and the object properties.
+
+== FEATURES/PROBLEMS:
+
+* Probably no windows support right now because of different line endings.
+
+== SYNOPSIS:
+
+ require 'csv_importer'
+
+ class Product < Struct.new(:title, :price, :brand, :image); end
+ file = File.open("filename.csv", "rb")
+ importer = CSV_Importer::Importer.new(file, Product)
+ products = importer.objects
+
+or
+
+ require 'csv_importer'
+
+ class Product < Struct.new(:title, :price, :brand, :image); end
+ csv_string = "title, the price, the brand \n jeans, 10, fashionable"
+ dictionary = {"the brand"=>"brand", "the price"=>"price"}
+ importer = CSV_Importer::Importer.new(csv_string, Product, dictionary)
+ products = importer.objects
+
+For use with ActiveRecord objects you can provide an argument to call save on every product when created
+
+ products = importer.object(true)
+
+
+== REQUIREMENTS:
+
+* This gem uses the standard ruby CSV libary. Ruby 1.9 will have a new library which will probably be compatible.
+
+== INSTALL:
+
+sudo gem install csv_importer
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2009 FIXME full name
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21 Rakefile
@@ -0,0 +1,21 @@
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+require 'fileutils'
+require './lib/csv_importer'
+
+Hoe.plugin :newgem
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.spec 'csv_importer' do
+ self.developer 'Sparkboxx', 'wilco@sparkboxx.com'
+ self.rubyforge_name = self.name # TODO this is default value
+
+end
+
+require 'newgem/tasks'
+Dir['tasks/**/*.rake'].each { |t| load t }
+
+# TODO - want other tests/tasks run by default? Add them to the list
+# remove_task :default
+# task :default => [:spec, :features]
4 lib/csv_importer.rb
@@ -0,0 +1,4 @@
+$:.unshift(File.dirname(__FILE__)) unless
+ $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'csv_importer/csv_importer'
79 lib/csv_importer/csv_importer.rb
@@ -0,0 +1,79 @@
+module CSVImporter
+ VERSION = "0.0.1"
+ require 'csv'
+
+
+ class Importer
+ attr_reader :reader, :delimiter, :csv_columns, :columns, :klass, :dictionary, :objects, :save
+
+ def initialize(str_or_readable, klass=nil, dictionary={})
+ @klass = klass
+ @save = save
+ @dictionary = {}
+ dictionary.each_pair{|k,v| @dictionary[k.downcase] = v.downcase}
+ @delimiter, @csv_columns = determine_delimiter_and_columns(str_or_readable)
+ @columns = match_columns
+ @reader = CSV::Reader.create(str_or_readable, @delimiter)
+ end
+
+ def objects(save=false)
+ return @objects if !@objects.nil?
+ @objects = Array.new if @object.nil?
+ first = true
+ @reader.each do |row|
+ #TODO: Ruby 1.9 has header option would be nicer instead of this hack.
+ if !first
+ object = @klass.new
+ @columns.each_pair do |column, row_number|
+ object.send("#{column}=", row[row_number].strip)
+ end
+ object.save if save
+ @objects << object
+ else
+ first = false
+ end
+ end
+ return @objects
+ end
+
+ private
+ def determine_delimiter_and_columns(str_or_readable)
+ case str_or_readable
+ when IO
+ first_line = str_or_readable.gets
+ str_or_readable.pos = 0 #Reset IO cursor position, else the first line will be skipped
+ when String
+ #TODO: Find efficient way to get the first line by stoping at first newline character
+ #TODO: Test for different newline characters on windows machines
+ first_line = str_or_readable.split("\n")[0]
+ else
+ #TODO: What's the best exception to raise?
+ raise ArgumentException, "No String or IO argument detected"
+ end
+
+ #TODO: Fix repitition in determine_delimiter_and_columns
+ comma = first_line.split(",")
+ semicolon = first_line.split(";")
+ if comma.count > semicolon.count
+ return ",",comma.map!{|c| c.strip.downcase}
+ else
+ return ";",semicolon.map!{|c| c.strip.downcase}
+ end
+ end
+
+ def match_columns
+ columns = {} if @columns.nil?
+ klass = @klass.new
+ @csv_columns.each_with_index do |column, index|
+ if klass.respond_to?("#{column}=")
+ columns[column] = index
+ elsif klass.respond_to?("#{@dictionary[column.downcase]}=")
+ columns[@dictionary[column.downcase]] = index
+ end
+ end
+ return columns
+ end
+
+
+ end
+end
10 script/console
@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs = " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs << " -r #{File.dirname(__FILE__) + '/../lib/csv_importer.rb'}"
+puts "Loading csv_importer gem"
+exec "#{irb} #{libs} --simple-prompt"
14 script/destroy
@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+ require 'rubigen'
+rescue LoadError
+ require 'rubygems'
+ require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)
14 script/generate
@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+ require 'rubigen'
+rescue LoadError
+ require 'rubygems'
+ require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)
71 script/txt2html
@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+
+load File.dirname(__FILE__) + "/../Rakefile"
+require 'rubyforge'
+require 'redcloth'
+require 'syntax/convertors/html'
+require 'erb'
+
+download = "http://rubyforge.org/projects/#{$hoe.rubyforge_name}"
+version = $hoe.version
+
+def rubyforge_project_id
+ RubyForge.new.configure.autoconfig["group_ids"][$hoe.rubyforge_name]
+end
+
+class Fixnum
+ def ordinal
+ # teens
+ return 'th' if (10..19).include?(self % 100)
+ # others
+ case self % 10
+ when 1: return 'st'
+ when 2: return 'nd'
+ when 3: return 'rd'
+ else return 'th'
+ end
+ end
+end
+
+class Time
+ def pretty
+ return "#{mday}#{mday.ordinal} #{strftime('%B')} #{year}"
+ end
+end
+
+def convert_syntax(syntax, source)
+ return Syntax::Convertors::HTML.for_syntax(syntax).convert(source).gsub(%r!^<pre>|</pre>$!,'')
+end
+
+if ARGV.length >= 1
+ src, template = ARGV
+ template ||= File.join(File.dirname(__FILE__), '/../website/template.html.erb')
+else
+ puts("Usage: #{File.split($0).last} source.txt [template.html.erb] > output.html")
+ exit!
+end
+
+template = ERB.new(File.open(template).read)
+
+title = nil
+body = nil
+File.open(src) do |fsrc|
+ title_text = fsrc.readline
+ body_text_template = fsrc.read
+ body_text = ERB.new(body_text_template).result(binding)
+ syntax_items = []
+ body_text.gsub!(%r!<(pre|code)[^>]*?syntax=['"]([^'"]+)[^>]*>(.*?)</\1>!m){
+ ident = syntax_items.length
+ element, syntax, source = $1, $2, $3
+ syntax_items << "<#{element} class='syntax'>#{convert_syntax(syntax, source)}</#{element}>"
+ "syntax-temp-#{ident}"
+ }
+ title = RedCloth.new(title_text).to_html.gsub(%r!<.*?>!,'').strip
+ body = RedCloth.new(body_text).to_html
+ body.gsub!(%r!(?:<pre><code>)?syntax-temp-(\d+)(?:</code></pre>)?!){ syntax_items[$1.to_i] }
+end
+stat = File.stat(src)
+created = stat.ctime
+modified = stat.mtime
+
+$stdout << template.result(binding)
6 spec/csv/sample.csv
@@ -0,0 +1,6 @@
+Description;Brand;SellPrice;SalePrice;Image;Category;ArticleCode
+SINGLE JERSEY L.S R;PME Jeans; 59.95; 59.95;http://www.example.com/images/JTS91573.JPG;T-shirts;JTS91573
+MONTANA L.SL. GRAND;PME legend; 59.95; 59.95;http://www.example.com/images/PTS88510.JPG;T-shirts;PTS88510
+CO DOUBLE JERSEY, 1;Cast Iron; 29.95; 29.95;http://www.example.com/images/CAC91101.JPG;Belts/ Keyholders;CAC91101
+PLAYER L.SLV. VEST;Cast Iron; 99.95; 99.95;http://www.example.com/images/CKW91406.JPG;Knitwear;CKW91406
+SLUBOR L.SL POLO;Cast Iron; 79.95; 79.95;http://www.example.com/images/CPS91301.JPG;Polo l.sl;CPS91301
83 spec/csv_importer_spec.rb
@@ -0,0 +1,83 @@
+require File.dirname(__FILE__) + '/spec_helper.rb'
+
+describe "Initialze CSV" do
+ include CSVImporter
+
+ it "should open a reader on the given CSV file" do
+ importer = new_importer
+ importer.reader.class == CSV::Reader
+ end
+
+ it "should open a reader given a string" do
+ importer = Importer.new("foo,bar,foo,bar", Product)
+ importer.reader.class == CSV::Reader
+ end
+
+ it "should detect the ; file delimiter" do
+ importer = new_importer
+ importer.delimiter.should == ";"
+ end
+
+ it "should detect the , file delimiter" do
+ importer = Importer.new("foo,bar,foo,bar,foo", Product)
+ importer.delimiter.should == ","
+ end
+
+ it "should pick the delimiter with the maximum split count" do
+ importer = Importer.new("column,col;umn,column,col;umn,column,column", Product)
+ importer.delimiter.should == ","
+ end
+
+ it "should set the columns of the CSV file" do
+ importer = Importer.new("a,b,c,d,e\nfoo,bar,foo,bar,foo", Product)
+ importer.csv_columns.should == ["a","b","c","d","e"]
+ end
+
+ it "should strip the column names" do
+ importer = Importer.new("a, b, c , d ", Product)
+ importer.csv_columns.should == ["a","b","c","d"]
+ end
+
+ it "should memorize the class to handle the rows" do
+ importer = Importer.new("a,b", Product)
+ importer.klass.should == Product
+ end
+
+ it "should match the CSV columns with the Object columns" do
+ importer = Importer.new("title, price, brand, foo, barred", Product)
+ importer.columns.should == {"title"=>0, "price"=>1, "brand"=>2}
+ end
+
+ it "should match the CSV columns with the Object columns through the dictionary" do
+ dictionary = {"the title"=>"title", "the price"=>"price", "the brand" => "brand"}
+ importer = Importer.new("the title, the price, the brand", Product, dictionary)
+ importer.columns.should == {"title"=>0, "price"=>1, "brand"=>2}
+ end
+
+ it "should ignore case differences in the dictionary" do
+ dictionary = {"THE title"=>"title", "the PRice"=>"price", "The Brand" => "brand"}
+ importer = Importer.new("the title, the price, the brand", Product, dictionary)
+ importer.columns.should == {"title"=>0, "price"=>1, "brand"=>2}
+ end
+
+ it "should be able to cope with partial dictionaries" do
+ dictionary = {"Description"=>"title", "Saleprice"=>"Price"}
+ importer = Importer.new(File.open(File.dirname(__FILE__)+ "/csv/sample.csv", "rb"), Product, dictionary)
+ importer.columns.should == {"title"=>0, "price"=>3, "brand"=>1, "image"=>4}
+ end
+
+ it "should create the collection of objects" do
+ dictionary = {"Description"=>"title", "Saleprice"=>"Price"}
+ importer = Importer.new(File.open(File.dirname(__FILE__)+ "/csv/sample.csv", "rb"), Product, dictionary)
+ importer.objects.count.should == 5 #See spec/csv/sample.csv
+ end
+
+ it "shoud not turn the title line into an object" do
+ importer = Importer.new("title,brand\nThis is a nice title!, belonging to a brand", Product)
+ importer.objects[0].title.should == "This is a nice title!"
+ importer.objects[0].brand.should == "belonging to a brand"
+ importer.objects.count.should == 1
+ end
+
+
+end
1 spec/spec.opts
@@ -0,0 +1 @@
+--colour
18 spec/spec_helper.rb
@@ -0,0 +1,18 @@
+begin
+ require 'spec'
+rescue LoadError
+ require 'rubygems' unless ENV['NO_RUBYGEMS']
+ gem 'rspec'
+ require 'spec'
+end
+
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+require 'csv_importer'
+include CSVImporter
+
+# Define a Product Struct for test purposes
+class Product < Struct.new(:title, :price, :brand, :image); end
+
+def new_importer
+ Importer.new(File.open(File.dirname(__FILE__)+ "/csv/sample.csv", "rb"), Product)
+end
21 tasks/rspec.rake
@@ -0,0 +1,21 @@
+begin
+ require 'spec'
+rescue LoadError
+ require 'rubygems' unless ENV['NO_RUBYGEMS']
+ require 'spec'
+end
+begin
+ require 'spec/rake/spectask'
+rescue LoadError
+ puts <<-EOS
+To use rspec for testing you must install rspec gem:
+ gem install rspec
+EOS
+ exit(0)
+end
+
+desc "Run the specs under spec/models"
+Spec::Rake::SpecTask.new do |t|
+ t.spec_opts = ['--options', "spec/spec.opts"]
+ t.spec_files = FileList['spec/**/*_spec.rb']
+end

0 comments on commit e28c0da

Please sign in to comment.
Something went wrong with that request. Please try again.