documentation and some polish, sausage

README.rdoc

@@ -2,38 +2,146 @@
 
 Simple API for importing csv files.
 
-== Usage
+== Defining importers
 
-define custom importers by including simple_importer
-  class CrazyImporter
-    include SimpleImporter
+The basic importer has a name, file, and a foreach block.
 
-    file 'data.csv'
-    callbacks true
+  importer :items do
+    file 'items.csv'
+
+    foreach do |row|
+      Item.create(:name => row[:name])
+    end
+  end
+
+simple_importer uses fastercsv, which replaces the csv standard lib in ruby1.9. Importers have configuration options that match those of fastercsv. The foreach block is called for each row in the csv file. The argument passed into the foreach block is a fastercsv row created with the specified configuration options.
+
+  importer :items do
+    file 'items.csv'
+
+    # all fastercsv options are supported
+    headers false
+    converters :numeric
+    # etc..
+
+    foreach do |row|
+      Item.create(:name => row[:name])
+    end
+  end
+
+Importers have a default configuration that matches the following importer.
+
+  importer :items do
+    file 'items.csv'
+
+    # default configuration
+    col_sep ","
+    row_sep :auto
+    quote_char '"'
+    field_size_limit nil
+    converters :all
+    unconverted_fields nil
     headers true
+    return_headers false
     header_converters :symbol
-    converters :all
+    skip_blanks false
+    force_quotes false
+
+    foreach do |row|
+      Item.create(:name => row[:name])
+    end
+  end
+
+== Before callback
 
-    before_import :delete_all
+Define before_filter-style callbacks in the form of the before block. This block is run once before the importer run.
+
+  importer :items do
+    file 'items.csv'
+
+    before do
+      Item.delete_all
+    end
 
     foreach do |row|
+      Item.create(:name => row[:name])
     end
+  end
+
+== File processing
+
+Passing an array of file paths to the file configuration field will process all the rows in each file.
 
-    def delete_all
+  importer :items do
+    file Dir.glob("data/*.csv")
+
+    foreach do |row|
+      Item.create(:name => row[:name])
+    end
+  end
+
+Use the foreach_file block to add per-file behavior.
+
+  importer :items do
+    file Dir.glob("data/*.csv")
+
+    foreach_file do |file|
+      List.create(:name => File.basename(file, ".csv"))
+    end
+
+    foreach do |row|
+      Item.create(:name => row[:name])
+    end
+  end
+
+Nest the foreach block to give your row processing block access to foreach_file block variables.
+
+  importer :items do
+    file Dir.glob("data/*.csv")
+
+    foreach_file do |file|
+      list = List.create(:name => File.basename(file, ".csv"))
+
+      foreach do |row|
+        Item.create(:name => row[:name], :list => list)
+      end
     end
   end
 
-run your importers
-  CrazyImporter.run
+== Loading and running importers
+
+simple_importer provides a method that will find and load importer definitions in importers, lib/importers, and app/importers directories.
 
-== TODO
+  # importer definition is in lib/importers/item.rb
+  SimpleImporter.find_importers
+
+SimpleImporter has a collection of importers and each importer can be run once it has been loaded.
+
+  # the following will fire off the actual importers
+  SimpleImporter.importers.each do |importer|
+    importer.run
+  end
 
-find files automatically
-rake tasks generated automatically
+== Rake tasks and autoloaded importers
+
+simple_importer provides rake tasks to run importers individually or all at once.
+
+  # add this to your rakefile
+  # lib/tasks/simple_importer.rake is a good place in your rails app
+  require 'simple_importer/tasks'
+
+rake -T simple_importer will show a rake task for each importer as well as an import task which runs all of the importers. Descriptions defined in the importer will be used for the rake task description.
+
+  importer :items do
+    file 'items.csv'
+    desc 'Import all of the cool items'
+
+    ...
+  end
 
 == Requirements
 
-* fast_csv (if you are not using ruby 1.9)
+* fastercsv (if you are not using ruby 1.9)
 
 == Note on Patches/Pull Requests 
 

Rakefile

@@ -10,7 +10,6 @@ begin
     gem.email = "gotascii@gmail.com"
     gem.homepage = "http://github.com/vigetlabs/simple_importer"
     gem.authors = ["Justin Marney"]
-    gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
 rescue LoadError

lib/simple_importer.rb

@@ -1,8 +1,10 @@
-require 'csv'
+require "csv"
 
-# require 'lib/simple_importer'
-# SimpleImporter.find_importers
-# SimpleImporter[:crazy].run
+if CSV.const_defined? :Reader
+  require 'fastercsv'
+  Object.send(:remove_const, :CSV)
+  CSV = FasterCSV
+end
 
 module SimpleImporter
   def self.included(base)
@@ -15,14 +17,18 @@ def #{meth}(val = nil)
       METH
     end
 
-    [:before, :foreach].each do |meth|
+    [:before, :foreach, :foreach_file].each do |meth|
       base.class_eval <<-METH
         def #{meth}(&block)
           config[:#{meth}] = block if block_given?
           config[:#{meth}]
         end
       METH
     end
+
+    base.class_eval do
+      attr_accessor :name
+    end
   end
 
   def self.importer_paths
@@ -43,27 +49,28 @@ def self.csv_config_meths
   end
 
   def self.config_meths
-    csv_config_meths + [:file, :callbacks]
+    csv_config_meths + [:file, :callbacks, :desc]
   end
 
   def self.importers
-    @importers ||= {}
+    @importers ||= []
   end
 
   def self.importer(name, &block)
-    importers[name] = Importer.new(&block)
+    importers << Importer.new(name, &block)
   end
 
   def self.[](name)
-    importers[name]
+    importers.select{|i| i.name == name}.first
   end
 
   def config
     @config ||= {
       :callbacks => true,
       :headers => true,
       :header_converters => :symbol,
-      :converters => :all
+      :converters => :all,
+      :desc => 'a simple_import importer'
     }
   end
 
@@ -73,7 +80,10 @@ def csv_config
 
   def run
     run_callbacks
-    CSV.foreach(file, csv_config, &foreach)
+    [file].flatten.each do |f|
+      foreach_file.call(f) if foreach_file
+      CSV.foreach(f, csv_config, &foreach) if foreach
+    end
   end
 
   def run_callbacks
@@ -83,7 +93,8 @@ def run_callbacks
   class Importer
     include SimpleImporter
 
-    def initialize(&block)
+    def initialize(name, &block)
+      self.name = name
       instance_eval(&block)
     end
   end

lib/simple_importer/tasks/simple_importer.rake

@@ -1,7 +1,24 @@
 namespace :simple_importer do
-  desc "find SimpleImporter"
-  task :find do
+  def importers
     require 'simple_importer'
-    puts SimpleImporter
+    SimpleImporter.find_importers
+    SimpleImporter.importers
+  end
+
+  importers.each do |i|
+    desc i.desc
+    task i.name do
+      Rake::Task[:environment].invoke if Rake::Task.task_defined?(:environment)
+      puts "Importing #{i.name}"
+      i.run
+      puts "Finished importing #{i.name}"
+    end
+  end
+
+  desc "run all importers"
+  task :import do
+    importers.each do |i|
+      Rake::Task["simple_importer:#{i.name}"].invoke
+    end
   end
 end

simple_importer.gemspec

@@ -10,7 +10,6 @@ begin
     gem.email = "gotascii@gmail.com"
     gem.homepage = "http://github.com/gotascii/simple_importer"
     gem.authors = ["gotascii"]
-    gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
 rescue LoadError

test/test_simple_importer.rb

@@ -11,19 +11,18 @@ class SimpleImporterTest < Test::Unit::TestCase
     end
 
     should "have an importers hash field" do
-      SimpleImporter.importers.should == {}
+      SimpleImporter.importers.should == []
     end
 
     should "create a new importer" do
-      block = lambda{ 'omg' }
-      SimpleImporter::Importer.expects(:new).with(&block)
-      SimpleImporter.importer(:bob, &block)
+      SimpleImporter::Importer.expects(:new).with(:bob)
+      SimpleImporter.importer(:bob)
     end
 
     should "add new importer to importers" do
       SimpleImporter::Importer.stubs(:new).returns('hi')
       SimpleImporter.importer(:bob)
-      SimpleImporter.importers[:bob].should == 'hi'
+      SimpleImporter.importers.should == ['hi']
     end
 
     should "look in importer, app/importers, and lib/importers for importers" do
@@ -36,12 +35,13 @@ class SimpleImporterTest < Test::Unit::TestCase
 
     should "return csv_config_meths plus file and callbacks for config_meths" do
       SimpleImporter.stubs(:csv_config_meths).returns([:yo])
-      SimpleImporter.config_meths.should == [:yo, :file, :callbacks]
+      SimpleImporter.config_meths.should == [:yo, :file, :callbacks, :desc]
     end
 
-    should "forward [] to importers" do
-      SimpleImporter.importers.expects(:[]).with(:omg).returns('hey')
-      SimpleImporter[:omg].should == 'hey'
+    should "return first importer whos name matches arg for []" do
+      importer = stub(:name => :omg)
+      SimpleImporter.stubs(:importers).returns([importer])
+      SimpleImporter[:omg].should == importer
     end
   end
 
@@ -131,6 +131,14 @@ class SimpleImporterTest < Test::Unit::TestCase
       CSV.expects(:foreach).with('file', 'csv_config')
       @importer.run
     end
+
+    should "call CSV.foreach with each file if file is an array" do
+      @importer.stubs(:file).returns(['file1', 'file2'])
+      @importer.stubs(:csv_config).returns('csv_config')
+      CSV.expects(:foreach).with('file1', 'csv_config')
+      CSV.expects(:foreach).with('file2', 'csv_config')
+      @importer.run
+    end
   end
 
   context "The Importer class" do
@@ -140,7 +148,7 @@ class SimpleImporterTest < Test::Unit::TestCase
 
     should "instance eval block on initialize" do
       SimpleImporter::Importer.any_instance.expects(:bobo)
-      SimpleImporter::Importer.new do
+      SimpleImporter::Importer.new(:hi) do
         bobo
       end
     end