start sorting out the mess that is configuration of operators

README.md

@@ -249,9 +249,14 @@ is to run our rail's install generator :
 ```ruby
 rails g datashift:install
 ```
+
+You can create a model specific file at anytime via
+```bash
+    thor datashift:config:generate:import
+```
 
-To create such a file manually, create an initialisation file within `config/initializers`, 
-and see `lib/datashift/configuration.rb` for details of options, for example
+To create a Rails tyle config block manually, create an initialisation file within `config/initializers`, 
+and see [lib/datashift/configuration.rb](lib/datashift/configuration.rb`) for details of all possible options, for example
 
 ```ruby
 DataShift::Configuration.call do |c|

datashift.gemspec

@@ -21,13 +21,13 @@ Gem::Specification.new do |s|
   s.files = Dir['{lib}/**/*', 'LICENSE.md', 'README.md', 'datashift.thor']
   s.require_paths = ['lib']
 
-  s.add_runtime_dependency 'rails', '>= 4.2', '< 5.1'
+  s.add_runtime_dependency 'rails', '>= 4.2', '< 5.2'
   s.add_runtime_dependency 'thor', '<= 0.20', '>= 0.18'
 
   s.add_runtime_dependency 'paperclip', '~> 5.2', '>= 5.2.0'
   s.add_runtime_dependency 'spreadsheet', '~> 1.1'
 
-  s.add_runtime_dependency 'rubyzip', '~> 0.9', '>= 0.9.9'
+  s.add_runtime_dependency 'rubyzip', '>= 0.9'
 
   s.add_runtime_dependency 'thread_safe', '~> 0.3', '>= 0.3'
   s.add_runtime_dependency 'erubis', '~> 2.7', '>= 2.7.0'

datashift.thor

@@ -62,7 +62,7 @@ module Datashift
 
     desc "build", 'Build gem and install in one step'
 
-    method_option :version, :aliases => '-v',  :desc => "New version", required: false
+    method_option :bump, :aliases => '-b', type: :string, desc:  "Bump the version", required: false
 
     method_option :push, :aliases => '-p', :desc => "Push resulting gem to rubygems.org"
 
@@ -71,14 +71,15 @@ module Datashift
 
     def build
 
-      version = options[:version] || DataShift::VERSION
+      raise "Please bump to a new version to install at rubygems" if options[:install] && options[:bump].blank?
+      version = options[:bump]
 
       # Bump the VERSION file in library
       File.open( File.join('lib/datashift/version.rb'), 'w') do |f|
         f << "module DataShift\n"
         f << "    VERSION = '#{version}'.freeze unless defined?(VERSION)\n"
         f << "end\n"
-      end if(options[:version] != DataShift::VERSION)
+      end if(options[:bump].present?)
 
       build_cmd =  "gem build datashift.gemspec"
 

lib/datashift.rb

@@ -128,7 +128,7 @@ def self.load_commands
 
     Dir["#{base}/*.thor"].each do |f|
       next unless File.file?(f)
-      load(f)
+      Thor::Util.load_thorfile(f)
     end
   end
 

lib/datashift/binder.rb

@@ -101,14 +101,16 @@ def map_inbound_headers(klass, columns)
 
       # If klass not in Dictionary yet, add to dictionary all possible operators on klass
       # which can be used to map headers and populate an object of type klass
-      model_method_mgr = ModelMethods::Manager.catalog_class(klass)
+      model_methods_collection = ModelMethods::Manager.catalog_class(klass)
+
+      bound = bindings.map(&:source)
 
       [*columns].each_with_index do |col_data, col_index|
         raw_col_data = col_data.to_s.strip
 
         if raw_col_data.nil? || raw_col_data.empty?
           logger.warn("Column list contains empty or null header at index #{col_index}")
-          bindings << NoMethodBinding.new(raw_col_data, col_index)
+          bindings << NoMethodBinding.new(raw_col_data, idx: col_index)
           next
         end
 
@@ -119,25 +121,31 @@ def map_inbound_headers(klass, columns)
         #
         raw_col_name, where_field, where_value, *data = raw_col_data.split(column_delim).map(&:strip)
 
+        puts "WTF #{raw_col_name}", bound.include?(raw_col_name)
+        next if bound.include?(raw_col_name)
+
+        pp bindings.map(&:source).inspect
+
         # Find the domain model method details
-        model_method = model_method_mgr.search(raw_col_name)
+        model_method = model_methods_collection.search(raw_col_name)
+
 
         # No such column, but if config set to include it, for example for delegated methods, add as op type :assignment
         if( model_method.nil? && (include_all? || forced?(raw_col_name)) )
           logger.debug("Operator #{raw_col_name} not found but forced inclusion set - adding as :assignment")
-          model_method = model_method_mgr.insert(raw_col_name, :assignment)
+          model_method = model_methods_collection.insert(raw_col_name, :assignment)
         end
 
         unless model_method
           Binder.substitutions(raw_col_name).each do |n|
-            model_method = model_method_mgr.search(n)
+            model_method = model_methods_collection.search(n)
             break if model_method
           end
         end
 
         if(model_method)
 
-          binding = MethodBinding.new(raw_col_name, col_index, model_method)
+          binding = MethodBinding.new(raw_col_name, model_method, idx: col_index)
 
           # we slurped up all possible data in split, turn it back into original string
           binding.add_column_data(data.join(column_delim))
@@ -168,7 +176,9 @@ def map_inbound_headers(klass, columns)
 
     def add_bindings_from_nodes( nodes )
       logger.debug("Adding  [#{nodes.size}] custom bindings")
-      nodes.each { |n| bindings << n.method_binding }
+      nodes.each { |n| bindings << n.method_binding unless n.is_a?(NoMethodBinding) }
+      puts "@ADD"
+      pp bindings.inspect
     end
 
     # Essentially we map any string collection of field names, not just headers from files
@@ -177,7 +187,7 @@ def add_bindings_from_nodes( nodes )
     def add_missing(col_data, col_index, reason)
       logger.warn(reason)
 
-      missing = NoMethodBinding.new(col_data, col_index, reason: reason)
+      missing = NoMethodBinding.new(col_data, reason: reason, idx: col_index)
 
       missing_bindings << missing
       bindings << missing
@@ -210,6 +220,11 @@ def operator_names
       bindings.collect( &:operator )
     end
 
+    # Find a binding, matches raw client supplied names e.g header and has a valid index
+    def find_for_source( name )
+      bindings.find{|b| b.source == name && b.index}
+    end
+
   end
 
 end

lib/datashift/inbound_data/method_binding.rb

@@ -36,7 +36,7 @@ class MethodBinding
     #
     # col_types can typically be derived from klass.columns - set of ActiveRecord::ConnectionAdapters::Column
 
-    def initialize(name, idx, model_method)
+    def initialize(name, model_method, idx: nil)
       @inbound_column = InboundData::Column.new(name, idx)
 
       @model_method = model_method
@@ -130,10 +130,10 @@ class NoMethodBinding < MethodBinding
 
     attr_accessor :reason
 
-    def initialize(client_name = '', client_idx = -1, options = {})
-      super(client_name, client_idx, nil)
+    def initialize(client_name = '', reason: nil, idx: 1 )
+      super(client_name, nil, idx: idx)
 
-      @reason = options[:reason] || ''
+      @reason = reason || ''
     end
 
     def invalid?

lib/datashift/loaders/excel_loader.rb

@@ -66,13 +66,18 @@ def perform_load( options = {} )
 
             @binder.bindings.each do |method_binding|
 
+              # TODO - how does this get inserted - bind headers ?? ignore if no index
+              # #<DataShift::MethodBinding:0x0000000003e26280 @inbound_column=#<DataShift::InboundData::Column:0x0000000003e26258 @header=#<DataShift::Header:0x0000000003e26190 @source="audio", @presentation="audio">, @index=nil,
+              next if method_binding.index.nil?
+
               unless method_binding.valid?
                 logger.warn("No binding was found for column (#{current_row_idx})")
                 next
               end
 
               # If binding to a column, get the value from the cell (bindings can be to internal methods)
-              value = method_binding.index ? row[method_binding.index] : nil
+              #
+              value = row[method_binding.index]
 
               context = doc_context.create_node_context(method_binding, current_row_idx, value)
 

lib/datashift/loaders/loader_base.rb

@@ -89,8 +89,6 @@ def bind_headers( headers )
 
       logger.info("Binding #{headers.size} inbound headers to #{load_object_class.name}")
 
-      @binder ||= DataShift::Binder.new
-
       begin
         binder.map_inbound_headers(load_object_class, headers)
       rescue => e
@@ -145,10 +143,6 @@ def configure_from(yaml_file, klass = nil, locale_key = 'data_flow_schema')
 
       logger.info("Read Datashift config: #{data.inspect}")
 
-      @config.merge!(data['LoaderBase']) if data['LoaderBase']
-
-      @config.merge!(data[self.class.name]) if data[self.class.name]
-
       @binder ||= DataShift::Binder.new
 
       data_flow_schema = DataShift::DataFlowSchema.new

lib/datashift/loaders/loader_factory.rb

@@ -16,6 +16,13 @@ module Loader
 
     class Factory
 
+      # organizer
+
+        #include Interactor::Organizer
+
+        #organize DetermineLoaderKlass, BuildKlassCatalog, LoadConfig, ApplyConfig
+
+
       # Based on file_name find appropriate Loader
 
       # Currently supports :

lib/datashift/mapping/data_flow_schema.rb

@@ -85,7 +85,7 @@ def prepare_from_klass( klass, doc_context = nil )
       klass_to_model_methods( klass ).each_with_index do |mm, i|
         @nodes.headers.add(mm.operator)   # for a class, the header names, default to the operators (methods)
 
-        binding = MethodBinding.new(mm.operator, i, mm)
+        binding = MethodBinding.new(mm.operator, mm, idx: i)
 
         # TODO - do we really need to pass in the doc context when parent nodes already has it ?
         @nodes << DataShift::NodeContext.new(@nodes.doc_context, binding, i, nil)
@@ -121,6 +121,8 @@ def klass_to_model_methods(klass)
     #
     # See Config generation or lib/datashift/templates/import_export_config.erb for full syntax
     #
+    # Returns  DataShift::NodeCollection
+    #
     def prepare_from_file(file_name, locale_key = 'data_flow_schema')
       @raw_data = ERB.new(File.read(file_name)).result
 
@@ -144,6 +146,23 @@ def prepare_from_yaml(yaml, locale_key = 'data_flow_schema')
 
       locale_section = yaml[locale_key]
 
+      if(locale_section.has_key?('Global'))
+        global_nodes = locale_section.delete('Global')
+
+        [*global_nodes].each do |c|
+
+          # TODO   what is c ?   a list or hash ?
+          # if DataShift::Configuration.call.respond_to #{c}=
+          # Set the global value e.g
+          # DataShift::Configuration.call.force_inclusion_of_columns = [:audio]
+        end
+      end
+
+      unless locale_section.keys.present?
+        logger.warn('No class related configuration found in YAML syntax- Nothing to process')
+        return  DataShift::NodeCollection.new
+      end
+
       class_name = locale_section.keys.first
 
       klass = MapperUtils.class_from_string_or_raise(class_name)
@@ -201,19 +220,26 @@ def prepare_from_yaml(yaml, locale_key = 'data_flow_schema')
 
           if(section['operator'])
             # Find the domain model method details
+            # byebug
             model_method = model_method_mgr.search(section['operator'])
 
             unless model_method
               operator_type = section['operator_type'] || :method
 
-              # expect one of ModelMethod.supported_types_enum
+              # TODO validate type ? guess we expect one of ModelMethod.supported_types_enum
               model_method = model_method_mgr.insert(section['operator'], operator_type)
+              # TODO - This index could be hard coded by the user in YAML or we try to derive it from the headers
+              # byebug
+              method_binding = MethodBinding.new(source, model_method, idx: section['index'])
             end
-
-            method_binding = InternalMethodBinding.new(model_method)
           end
 
-          method_binding ||= MethodBinding.new(source, i, model_method)
+          # Now ensure we bind source/header(and index) to the method tht performs assignment of inbound datum to the model
+          #
+          # TOFIX - This is a bug waiting to happen right ? i is not coming from the headers
+          # so chances are user hasn't made config indexed as per headers
+          # index could be hard coded by the user in YAML or we try to derive it from the headers via the binder ???
+          method_binding ||= MethodBinding.new(source,  model_method, idx: i)
 
           node_context = DataShift::NodeContext.new(@nodes.doc_context, method_binding, i, nil)
 

lib/datashift/model_methods/model_method.rb

@@ -4,7 +4,7 @@
 # License::   MIT
 #
 # Details::   This class holds info on a single Method callable on a domain Model
-#             By holding information on the Type inbound data can be manipulated
+#             By holding information on the Type, inbound data can be manipulated
 #             into the right format for the style of operator; simple assignment,
 #             appending to an association collection or a method call
 #

lib/datashift/model_methods/operator.rb

@@ -18,7 +18,7 @@ class Operator
     # N.B these are in priority order ie. often prefer to process assignments first, then associations
     #
     def self.supported_types_enum
-      @type_enum ||= [:assignment, :enum, :belongs_to, :has_one, :has_many, :method]
+      @type_enum ||= [:assignment, :enum, :belongs_to, :has_one, :has_many, :method, :paperclip]
       @type_enum
     end
 

lib/datashift/node_context.rb

@@ -61,7 +61,7 @@ def process
   class EmptyContext < NodeContext
 
     def initialize
-      super(NilClass, DataShift::NoMethodBinding.new, -1, [])
+      super(NilClass, DataShift::NoMethodBinding.new, [], idx: -1)
     end
   end
 

lib/datashift/populators/populator.rb

@@ -99,7 +99,7 @@ def prepare_data(method_binding, data)
           @value = data
 
         elsif(!DataShift::Guards.jruby? &&
-          (data.is_a?(Spreadsheet::Formula) || data.class.ancestors.include?(Spreadsheet::Formula)) )
+            (data.is_a?(Spreadsheet::Formula) || data.class.ancestors.include?(Spreadsheet::Formula)) )
 
           @value = data.value # TOFIX jruby/apache poi equivalent ?
 
@@ -156,11 +156,17 @@ def assign(method_binding, record)
 
       elsif model_method.operator_for(:has_one)
 
-        if value.is_a?(model_method.klass)
+        begin
           record.send(operator + '=', value)
-        else
-          logger.error("Cannot assign value [#{value.inspect}]")
-          logger.error("Value was Type (#{value.class}) - Required Type for has_one #{operator} is [#{klass}]")
+        rescue => x
+          logger.error("Cannot assign value [#{value.inspect}] for has_one [#{operator}]")
+          logger.error(x.inspect)
+
+          if value.is_a?(model_method.klass)
+            logger.error("Value was Correct Type (#{value.class}) - [#{model_method.klass}]")
+          else
+            logger.error("Value was Type (#{value.class}) - Required Type is [#{model_method.klass}]")
+          end
         end
 
       elsif model_method.operator_for(:assignment)
@@ -179,11 +185,14 @@ def assign(method_binding, record)
 
       elsif model_method.operator_for(:method)
 
+
         begin
           params_num = record.method(operator.to_sym).arity
 
-          # think this should be == 0 but seen situations where -1 returned even though method accepts ZERO params
-          if(params_num < 1)
+          # There are situations where -1 returned, this is normally related to variable number of arguments,
+          # but maybe buggy - have seen -1 for what seems perfceatlly normal method e.g   def attach_audio_file_helper(file_name)
+          #
+          if(params_num == 0)
             logger.debug("Calling Custom Method (no value) [#{operator}]")
             record.send(operator)
           elsif(value)

lib/datashift/version.rb

@@ -1,3 +1,3 @@
 module DataShift
-    VERSION = '0.40.2'.freeze unless defined?(VERSION)
+    VERSION = '0.40.4'.freeze unless defined?(VERSION)
 end