first paperclip specs pass. more options for generate to remove columns

lib/datashift/method_detail.rb

@@ -12,6 +12,7 @@
 #
 require 'to_b'
 require 'logging'
+require 'populator'
 require 'set'
 
 module DataShift
@@ -20,6 +21,9 @@ class MethodDetail
 
     include DataShift::Logging
 
+    include DataShift::Populator
+    extend DataShift::Populator
+
     def self.supported_types_enum
       @type_enum ||= Set[:assignment, :belongs_to, :has_one, :has_many]
       @type_enum
@@ -169,12 +173,6 @@ def pp
       "#{@name} => #{operator}"
     end
 
-
-    def self.insistent_method_list
-      @insistent_method_list ||= [:to_s, :to_i, :to_f, :to_b]
-      @insistent_method_list
-    end
-
     private
 
     # Attempt to find the associated object via id, name, title ....

lib/datashift/querying.rb

@@ -0,0 +1,86 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2010
+# License::   MIT
+#
+#  Details::  Base class for loaders, providing a process hook which populates a model,
+#             based on a method map and supplied value from a file - i.e a single column/row's string value.
+#             Note that although a single column, the string can be formatted to contain multiple values.
+#
+#             Tightly coupled with MethodMapper classes (in lib/engine) which contains full details of
+#             a file's column and it's correlated AR associations.
+#
+module DataShift
+
+  require 'datashift/method_mapper'
+
+  module  Querying
+
+    def search_for_record(klazz, field, search_term, options = {})
+
+      begin
+
+        if(options[:case_sensitive]) 
+          return klazz.send("find_by_#{field}", search_term)
+        elsif(options[:use_like])
+          return klazz.where("#{field} like ?", "#{search_term}%").first
+        else
+          return klazz.where("lower(#{field}) = ?", search_term.downcase).first
+        end
+
+      rescue => e
+        puts e.inspect
+        logger.error("Exception attempting to find a record for [#{search_term}] on #{klazz}.#{field}")
+        logger.error e.backtrace
+        logger.error e.inspect
+      end
+
+      nil
+    end
+
+    # Find a record for model klazz, looking up on field containing search_terms
+    # Responds to global Options :
+    #   :case_sensitive : Default is a case insensitive lookup.
+    #   :use_like : Attempts a lookup using ike and x% rather than equality 
+    #
+    # Returns nil if no record found
+    def get_record_by(klazz, field, search_term, split_on = ' ', split_on_prefix = nil)
+
+      begin
+
+        record = search_for_record(klazz, field, search_term)
+
+        # try individual portions of search_term, front -> back i.e "A_B_C_D" => A, B, C etc
+        search_term.split(split_on).each do |str|
+          z = (split_on_prefix) ? "#{split_on_prefix}#{str}": str
+          record = search_for_record(klazz, field, z)
+          break if record
+        end unless(record)
+
+        # this time try incrementally scanning i.e "A_B_C_D" => A, A_B, A_B_C etc
+        search_term.split(split_on).inject("") do |str, term|
+          z = (split_on_prefix) ? "#{split_on_prefix}#{str}#{split_on}#{term}": "#{str}#{split_on}#{term}"
+          record = search_for_record(klazz, field, z)
+          break if record
+          term
+        end unless(record)
+
+        return record
+      rescue => e
+        logger.error("Exception attempting to find a record for [#{search_term}] on #{klazz}.#{field}")
+        logger.error e.backtrace
+        logger.error e.inspect
+        return nil
+      end
+    end
+
+    def get_record_by!(klazz, field, search_terms, split_on = ' ', split_on_prefix = nil)
+      x = get_record_by(klazz, field, search_terms, split_on, split_on_prefix)
+
+      raise RecordNotFound, "No #{klazz} record found for [#{search_terms}] on #{field}" unless(x)
+
+      x
+    end
+  end
+
+end

lib/generators/csv_generator.rb

@@ -12,11 +12,8 @@ module DataShift
 
   class CsvGenerator < GeneratorBase
 
-    attr_accessor :excel, :filename
-
     def initialize(filename)
-      @excel = nil
-      @filename = filename
+      super(filename)
     end
 
     # Create CSV file representing supplied Model

lib/generators/excel_generator.rb

@@ -8,20 +8,19 @@
 # TOD : Can we switch between .xls and XSSF (POI implementation of Excel 2007 OOXML (.xlsx) file format.)
 #
 #
+require 'generator_base'
+require 'excel'
+
 module DataShift
-
-  require 'generator_base'
 
-  require 'excel'
-
   class ExcelGenerator < GeneratorBase
 
     include DataShift::Logging
 
-    attr_accessor :excel, :filename
+    attr_accessor :excel
 
     def initialize(filename)
-      @filename = filename
+      super(filename)
     end
 
     # Create an Excel file template (header row) representing supplied Model
@@ -31,8 +30,14 @@ def initialize(filename)
     def generate(klass, options = {})
 
       prepare_excel(klass, options)
-
-      @excel.set_headers(MethodDictionary.assignments[klass])
+
+      prep_remove_list(options)  
+
+      @headers = MethodDictionary.assignments[klass]
+
+      @headers.delete_if{|h| @remove_list.include?( h.to_sym ) }
+
+      @excel.set_headers( @headers )
 
       logger.info("ExcelGenerator saving generated template #{@filename}")
 
@@ -57,6 +62,9 @@ def generate(klass, options = {})
     #       
     # * <tt>:remove</tt> - Association NAME(s) to remove .. :title, :id, :name
     # .
+    # * <tt>:remove_rails</tt> - Remove Rails DB columns :
+    #           :id, :created_at, :created_on, :updated_at, :updated_on
+    #   
     def generate_with_associations(klass, options = {})
 
       prepare_excel(klass, options)
@@ -65,9 +73,9 @@ def generate_with_associations(klass, options = {})
 
       work_list = MethodDetail::supported_types_enum.to_a - [ *options[:exclude] ]
 
-      remove_list = [ *options[:remove] ].compact.collect{|x| x.to_s.downcase.to_sym }
-         
-      headers = []
+      prep_remove_list(options)    
+
+      @headers = []
 
       details_mgr = MethodDictionary.method_details_mgrs[klass]
 
@@ -94,6 +102,15 @@ def generate_with_associations(klass, options = {})
 
     private
 
+    # Take options and create a list of symbols to remove from headers
+    # 
+    def prep_remove_list( options )
+      @remove_list = [ *options[:remove] ].compact.collect{|x| x.to_s.downcase.to_sym }
+
+      @remove_list += GeneratorBase::rails_columns if(options[:remove_rails])
+    end
+
+
     def prepare_excel(klass, options = {})
       @filename = options[:filename] if  options[:filename]
 

lib/generators/generator_base.rb

@@ -9,6 +9,18 @@ module DataShift
 
   class GeneratorBase
 
+    attr_accessor :filename, :headers, :remove_list
+
+    def initialize(filename)
+      @filename = filename
+      @headers = []
+      @remove_list =[]
+    end
+
+
+    def self.rails_columns
+      @rails_standard_columns ||= [:id, :created_at, :created_on, :updated_at, :updated_on]
+    end
   end
 
 end

lib/loaders/loader_base.rb

@@ -13,11 +13,13 @@
 module DataShift
 
   require 'datashift/method_mapper'
+  require 'datashift/querying'
 
   class LoaderBase
 
     include DataShift::Logging
     include DataShift::Populator
+    include DataShift::Querying
 
     attr_reader :headers
 
@@ -71,7 +73,9 @@ def initialize(object_class, object = nil, options = {})
       @prefixes       = {}
       @postfixes      = {}
 
+      # TODO - move to own LoadStats or LoadReport class
       @loaded_objects = []
+      @failed_objects = []
 
       reset(object)
     end
@@ -185,72 +189,6 @@ def find_and_process(column_name, data)
     end
 
 
-    def search_for_record(klazz, field, search_term)
-
-      begin
-
-        if(@config[:case_sensitive]) 
-          return klazz.send("find_by_#{field}", search_term)
-        elsif(@config[:use_like])
-          return klazz.where("#{field} like ?", "#{search_term}%").first
-        else
-          return klazz.where("lower(#{field}) = ?", search_term.downcase).first
-        end
-
-      rescue => e
-        puts e.inspect
-        logger.error("Exception attempting to find a record for [#{search_term}] on #{klazz}.#{field}")
-        logger.error e.backtrace
-        logger.error e.inspect
-      end
-
-      nil
-    end
-
-    # Find a record for model klazz, looking up on field containing search_terms
-    # Responds to global Options :
-    #   :case_sensitive : Default is a case insensitive lookup.
-    #   :use_like : Attempts a lookup using ike and x% rather than equality 
-    #
-    # Returns nil if no record found
-    def get_record_by(klazz, field, search_term, split_on = ' ', split_on_prefix = nil)
-
-      begin
-
-        record = search_for_record(klazz, field, search_term)
-
-        # try individual portions of search_term, front -> back i.e "A_B_C_D" => A, B, C etc
-        search_term.split(split_on).each do |str|
-          z = (split_on_prefix) ? "#{split_on_prefix}#{str}": str
-          record = search_for_record(klazz, field, z)
-          break if record
-        end unless(record)
-
-        # this time try incrementally scanning i.e "A_B_C_D" => A, A_B, A_B_C etc
-        search_term.split(split_on).inject("") do |str, term|
-          z = (split_on_prefix) ? "#{split_on_prefix}#{str}#{split_on}#{term}": "#{str}#{split_on}#{term}"
-          record = search_for_record(klazz, field, z)
-          break if record
-          term
-        end unless(record)
-
-        return record
-      rescue => e
-        logger.error("Exception attempting to find a record for [#{search_term}] on #{klazz}.#{field}")
-        logger.error e.backtrace
-        logger.error e.inspect
-        return nil
-      end
-    end
-
-    def get_record_by!(klazz, field, search_terms, split_on = ' ', split_on_prefix = nil)
-      x = get_record_by(klazz, field, search_terms, split_on, split_on_prefix)
-
-      raise RecordNotFound, "No #{klazz} record found for [#{search_terms}] on #{field}" unless(x)
-
-      x
-    end
-
     # Default values and over rides can be provided in YAML config file.
     # 
     # Any Config under key 'LoaderBase' is merged over existing options - taking precedence.