@@ -2,38 +2,146 @@ Simple API for importing csv files. -== Usage +== Defining importers -define custom importers by including simple_importer - class CrazyImporter - include SimpleImporter +The basic importer has a name, file, and a foreach block. - file 'data.csv' - callbacks true + importer :items do + file 'items.csv' + + foreach do |row| + Item.create(:name => row[:name]) + end + end + +simple_importer uses fastercsv, which replaces the csv standard lib in ruby1.9. Importers have configuration options that match those of fastercsv. The foreach block is called for each row in the csv file. The argument passed into the foreach block is a fastercsv row created with the specified configuration options. + + importer :items do + file 'items.csv' + + # all fastercsv options are supported + headers false + converters :numeric + # etc.. + + foreach do |row| + Item.create(:name => row[:name]) + end + end + +Importers have a default configuration that matches the following importer. + + importer :items do + file 'items.csv' + + # default configuration + col_sep "," + row_sep :auto + quote_char '"' + field_size_limit nil + converters :all + unconverted_fields nil headers true + return_headers false header_converters :symbol - converters :all + skip_blanks false + force_quotes false + + foreach do |row| + Item.create(:name => row[:name]) + end + end + +== Before callback - before_import :delete_all +Define before_filter-style callbacks in the form of the before block. This block is run once before the importer run. + + importer :items do + file 'items.csv' + + before do + Item.delete_all + end foreach do |row| + Item.create(:name => row[:name]) end + end + +== File processing + +Passing an array of file paths to the file configuration field will process all the rows in each file. - def delete_all + importer :items do + file Dir.glob("data/*.csv") + + foreach do |row| + Item.create(:name => row[:name]) + end + end + +Use the foreach_file block to add per-file behavior. + + importer :items do + file Dir.glob("data/*.csv") + + foreach_file do |file| + List.create(:name => File.basename(file, ".csv")) + end + + foreach do |row| + Item.create(:name => row[:name]) + end + end + +Nest the foreach block to give your row processing block access to foreach_file block variables. + + importer :items do + file Dir.glob("data/*.csv") + + foreach_file do |file| + list = List.create(:name => File.basename(file, ".csv")) + + foreach do |row| + Item.create(:name => row[:name], :list => list) + end end end -run your importers - CrazyImporter.run +== Loading and running importers + +simple_importer provides a method that will find and load importer definitions in importers, lib/importers, and app/importers directories. -== TODO + # importer definition is in lib/importers/item.rb + SimpleImporter.find_importers + +SimpleImporter has a collection of importers and each importer can be run once it has been loaded. + + # the following will fire off the actual importers + SimpleImporter.importers.each do |importer| + importer.run + end -find files automatically -rake tasks generated automatically +== Rake tasks and autoloaded importers + +simple_importer provides rake tasks to run importers individually or all at once. + + # add this to your rakefile + # lib/tasks/simple_importer.rake is a good place in your rails app + require 'simple_importer/tasks' + +rake -T simple_importer will show a rake task for each importer as well as an import task which runs all of the importers. Descriptions defined in the importer will be used for the rake task description. + + importer :items do + file 'items.csv' + desc 'Import all of the cool items' + + ... + end == Requirements -* fast_csv (if you are not using ruby 1.9) +* fastercsv (if you are not using ruby 1.9) == Note on Patches/Pull Requests README.rdoc @@ -10,7 +10,6 @@ begin gem.email = "gotascii@gmail.com" gem.homepage = "http://github.com/vigetlabs/simple_importer" gem.authors = ["Justin Marney"] - gem.add_development_dependency "thoughtbot-shoulda", ">= 0" # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings end rescue LoadError Rakefile @@ -1,8 +1,10 @@ -require 'csv' +require "csv" -# require 'lib/simple_importer' -# SimpleImporter.find_importers -# SimpleImporter[:crazy].run +if CSV.const_defined? :Reader + require 'fastercsv' + Object.send(:remove_const, :CSV) + CSV = FasterCSV +end module SimpleImporter def self.included(base) @@ -15,7 +17,7 @@ module SimpleImporter METH end - [:before, :foreach].each do |meth| + [:before, :foreach, :foreach_file].each do |meth| base.class_eval <<-METH def #{meth}(&block) config[:#{meth}] = block if block_given? @@ -23,6 +25,10 @@ module SimpleImporter end METH end + + base.class_eval do + attr_accessor :name + end end def self.importer_paths @@ -43,19 +49,19 @@ module SimpleImporter end def self.config_meths - csv_config_meths + [:file, :callbacks] + csv_config_meths + [:file, :callbacks, :desc] end def self.importers - @importers ||= {} + @importers ||= [] end def self.importer(name, &block) - importers[name] = Importer.new(&block) + importers << Importer.new(name, &block) end def self.[](name) - importers[name] + importers.select{|i| i.name == name}.first end def config @@ -63,7 +69,8 @@ module SimpleImporter :callbacks => true, :headers => true, :header_converters => :symbol, - :converters => :all + :converters => :all, + :desc => 'a simple_import importer' } end @@ -73,7 +80,10 @@ module SimpleImporter def run run_callbacks - CSV.foreach(file, csv_config, &foreach) + [file].flatten.each do |f| + foreach_file.call(f) if foreach_file + CSV.foreach(f, csv_config, &foreach) if foreach + end end def run_callbacks @@ -83,7 +93,8 @@ module SimpleImporter class Importer include SimpleImporter - def initialize(&block) + def initialize(name, &block) + self.name = name instance_eval(&block) end end lib/simple_importer.rb @@ -1,7 +1,24 @@ namespace :simple_importer do - desc "find SimpleImporter" - task :find do + def importers require 'simple_importer' - puts SimpleImporter + SimpleImporter.find_importers + SimpleImporter.importers + end + + importers.each do |i| + desc i.desc + task i.name do + Rake::Task[:environment].invoke if Rake::Task.task_defined?(:environment) + puts "Importing #{i.name}" + i.run + puts "Finished importing #{i.name}" + end + end + + desc "run all importers" + task :import do + importers.each do |i| + Rake::Task["simple_importer:#{i.name}"].invoke + end end end \ No newline at end of file lib/simple_importer/tasks/simple_importer.rake @@ -10,7 +10,6 @@ begin gem.email = "gotascii@gmail.com" gem.homepage = "http://github.com/gotascii/simple_importer" gem.authors = ["gotascii"] - gem.add_development_dependency "thoughtbot-shoulda", ">= 0" # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings end rescue LoadError simple_importer.gemspec @@ -11,19 +11,18 @@ class SimpleImporterTest < Test::Unit::TestCase end should "have an importers hash field" do - SimpleImporter.importers.should == {} + SimpleImporter.importers.should == [] end should "create a new importer" do - block = lambda{ 'omg' } - SimpleImporter::Importer.expects(:new).with(&block) - SimpleImporter.importer(:bob, &block) + SimpleImporter::Importer.expects(:new).with(:bob) + SimpleImporter.importer(:bob) end should "add new importer to importers" do SimpleImporter::Importer.stubs(:new).returns('hi') SimpleImporter.importer(:bob) - SimpleImporter.importers[:bob].should == 'hi' + SimpleImporter.importers.should == ['hi'] end should "look in importer, app/importers, and lib/importers for importers" do @@ -36,12 +35,13 @@ class SimpleImporterTest < Test::Unit::TestCase should "return csv_config_meths plus file and callbacks for config_meths" do SimpleImporter.stubs(:csv_config_meths).returns([:yo]) - SimpleImporter.config_meths.should == [:yo, :file, :callbacks] + SimpleImporter.config_meths.should == [:yo, :file, :callbacks, :desc] end - should "forward [] to importers" do - SimpleImporter.importers.expects(:[]).with(:omg).returns('hey') - SimpleImporter[:omg].should == 'hey' + should "return first importer whos name matches arg for []" do + importer = stub(:name => :omg) + SimpleImporter.stubs(:importers).returns([importer]) + SimpleImporter[:omg].should == importer end end @@ -131,6 +131,14 @@ class SimpleImporterTest < Test::Unit::TestCase CSV.expects(:foreach).with('file', 'csv_config') @importer.run end + + should "call CSV.foreach with each file if file is an array" do + @importer.stubs(:file).returns(['file1', 'file2']) + @importer.stubs(:csv_config).returns('csv_config') + CSV.expects(:foreach).with('file1', 'csv_config') + CSV.expects(:foreach).with('file2', 'csv_config') + @importer.run + end end context "The Importer class" do @@ -140,7 +148,7 @@ class SimpleImporterTest < Test::Unit::TestCase should "instance eval block on initialize" do SimpleImporter::Importer.any_instance.expects(:bobo) - SimpleImporter::Importer.new do + SimpleImporter::Importer.new(:hi) do bobo end end test/test_simple_importer.rb bac421afec835186bfc2231ed8901d3340395632 Justin Marney gotascii@gmail.com http://github.com/vigetlabs/simple_importer/commit/000aaf2b3e6259c68c6bf4b069dccf8dc898a933 000aaf2b3e6259c68c6bf4b069dccf8dc898a933 2009-10-22T14:03:55-07:00 2009-10-22T14:03:55-07:00 documentation and some polish, sausage 411f9170eabb54927f968d769f6d8a236fd24e89 Justin Marney gotascii@gmail.com