Merge branch 'takeover'

app/controllers/cachers.rb

@@ -22,6 +22,11 @@ def missing
   end
 
   def generate
+    @model = params[:by] ? Kernel.const_get(params[:by].camel_case + "Cache") : BranchCache
+    if @from_date and @to_date
+      (@from_date..@to_date).each{|date| @model.update(:date => date)}
+    else
+      @model.update(:date => (@date || Date.today))
     if Branch.count > 0
       if @from_date and @to_date
         (@from_date..@to_date).each{|date| BranchCache.update(date)}
@@ -52,10 +57,21 @@ def freshen
   def consolidate
     get_cachers
     group_by = @level.to_s.singularize
-    group_by_model = Kernel.const_get(group_by.camelcase) 
+    group_by_model = Kernel.const_get(group_by.camelcase) rescue Kernel.const_get(params[:by].camel_case)
     unless group_by == "loan"
       @cachers = @cachers.group_by{|c| c.send("#{group_by}_id".to_sym)}.to_hash.map do |group_by_id, cachers| 
-        cachers.reduce(:consolidate)
+        # when we are aggregating "by" something else we need to consolidate cachers that span across dates and 
+        # add cachers for the same date
+        if params[:by]
+          $debug = true if group_by_id == 2
+          cachers_for_date = cachers.group_by{|c| c.date}.to_hash.map{|d, cs| [d,cs.reduce(:+)]}.to_hash
+          r = cachers_for_date.values.reduce(:consolidate)
+          r.model_id = group_by_id
+          r.branch_id = nil
+          r
+        else
+          cachers.reduce(:consolidate)
+        end
       end
     end
     display @cachers, :template => 'cachers/index', :layout => (params[:layout] or Nothing).to_sym
@@ -103,37 +119,60 @@ def parse_dates
   def get_cachers
     q = {}
     q[:branch_id] = params[:branch_id] unless params[:branch_id].blank? 
-    if (not params[:branch_id].blank?)
-      q[:center_id] = params[:center_id] unless (params[:center_id].blank? or params[:center_id].to_i == 0)
-    else
-      q[:model_name] = "Branch"
+    unless params[:branch_id].blank?
+      if params[:center_id]
+        q[:center_id] = params[:center_id] unless (params[:center_id].blank? or params[:center_id].to_i == 0)
+      else
+        q[:center_id.not] = 0
+      end
+    end
+    if true
+      if params[:by]
+        q[:model_name] = params[:by].camel_case
+        q[:center_id] ||= 0 unless q[:center_id.not]
+        q[:model_id] = params[:model_id] if params[:model_id]
+      else
+        q[:model_name] = ["Branch","Center"] 
+      end
     end
     q[:date] = @date if @date
     q[:date] = @from_date..@to_date if (@from_date and @to_date)
     q[:stale] = true if params[:stale]
     @cachers = Cacher.all(q)
     q.delete(:model_name)
-    @missing_centers = CenterCache.missing(q)
+    if params[:by]
+      @missing_centers = {} # TODO
+    else
+      @missing_centers = {} #CenterCache.missing(q)
+    end
     get_context
   end
 
   def get_context
     @center = params[:center_id].blank? ? nil : Center.get(params[:center_id])
     @branch = params[:branch_id].blank? ? nil : Branch.get(params[:branch_id])
+    @area = params[:area_id].blank? ? nil : Area.get(params[:area_id])
+    @region = params[:region_id].blank? ? nil : Region.get(params[:region_id])
     @center_names = @cachers.blank? ? {} : Center.all(:id => @cachers.aggregate(:center_id)).aggregate(:id, :name).to_hash
     @branch_names = @cachers.blank? ? {} : Branch.all(:id => @cachers.aggregate(:branch_id)).aggregate(:id, :name).to_hash
     q = (@from_date and @to_date) ? {:date => @from_date..@to_date} : {:date => @date}
-    @stale_centers = CenterCache.all(q.merge(:stale => true))
+    @stale_centers = Cacher.all(q.merge(:stale => true))
     @stale_branches = BranchCache.all(q.merge(:stale => true))
     @last_cache_update = @cachers.aggregate(:updated_at.min)
     @resource = params[:action] == "index" ? :cachers : (params[:action].to_s + "_" + "cachers").to_sym
     @keys = [:branch_id, :center_id] + (ReportFormat.get(params[:report_format]) || ReportFormat.first).keys
     @total_keys = @keys[2..-1]
     if @resource == :split_cachers
-      @level = params[:center_id].blank? ? :branches : :centers
+      @level = (params[:center_id].blank? ? :branches : :centers)
       @keys = [:date] + @keys
     else 
-      @level = (not params[:center_id].blank?) ? :loans : ((not params[:branch_id].blank?) ? :centers : :branches)
+      @level = @center ? :loans : (@branch ? :centers : (@area ? :branches : (@region ? :areas : :branches)))
+      if params[:by]
+        @level = :model unless @level == :loans
+        @keys = [:model_name] + @keys
+        @model = Kernel.const_get(params[:by].camel_case)
+      else
+      end
     end
   end
 

app/models/cache_observer.rb

@@ -4,13 +4,16 @@ class CacheObserver
   observe Payment, Loan
 
   def self.get_date(obj)
-    date = obj.respond_to?(:applied_on) ? obj.applied_on : (obj.respond_to?(:received_on) ? obj.received_on : nil)
+    date = obj.respond_to?(:applied_on) ? obj.applied_on : (obj.respond_to?(:received_on) ? obj.received_on : enil)
   end
 
   def self.make_stale(obj)
     center_id = obj.c_center_id
     date = CacheObserver.get_date(obj)
+    loan = obj.is_a?(Loan) ? obj : obj.loan
+    info = loan.info(date)
     CenterCache.stalify(:center_id => obj.c_center_id, :date => date)  if date
+    FundingLineCache.stalify(:center_id => obj.c_center_id, :date => date, :model_id => info.funding_line_id)
   end
 
   after :create do

app/models/cacher.rb

@@ -8,6 +8,11 @@ class Cacher
   property :model_id,                        Integer, :nullable => false, :index => true, :unique => [:model_name, :date], :key => true
   property :branch_id,                       Integer, :index => true, :key => true
   property :center_id,                       Integer, :index => true, :key => true
+<<<<<<< HEAD
+
+  property :funding_line_id,                 Integer, :index => true
+=======
+>>>>>>> 53f1094295262f8d6c7e371a03dedd10ddff71af
   property :scheduled_outstanding_total,     Float, :nullable => false
   property :scheduled_outstanding_principal, Float, :nullable => false
   property :actual_outstanding_total,        Float, :nullable => false
@@ -150,6 +155,7 @@ def + (other)
     me = self.attributes; other = other.attributes;
     attrs = me + other; attrs[:date] = date; attrs[:model_name] = "Sum";
     attrs[:model_id] = nil; attrs[:branch_id] = nil; attrs[:center_id] = nil;
+    attrs[:stale] = me[:stale] || other[:stale]
     Cacher.new(attrs)
   end
 
@@ -163,7 +169,7 @@ def self.process_queue
 class BranchCache < Cacher
 
   def self.recreate(date = Date.today, branch_ids = nil)
-    self.update(date, branch_ids, true)
+    self.update(:date => date, :branch_ids => branch_ids, :force => true)
   end
 
   def self.update(date = Date.today, branch_ids = nil, force = false)
@@ -271,7 +277,8 @@ class CenterCache < Cacher
 
   # these have to be hooks, for obvious reasons, but for now we make do with some hardcoded magic!
 
-s  EXTRA_FIELDS = [:delayed_disbursals]
+  EXTRA_FIELDS = [:delayed_disbursals]
+
   def self.update(hash = {})
     # creates a cache per center for branches and centers per the hash passed as argument
     t = Time.now
@@ -348,8 +355,16 @@ def self.stalify(options = {})
     @center = Center.get(cid)
     d = options[:date].class != Date ? (Date.parse(options[:date]) rescue nil) : options[:date]
     raise ArgumentError.new("Cannot parse date") unless d
-
-    repository.adapter.execute("UPDATE cachers SET stale=1 WHERE center_id=#{cid} OR (center_id = 0 AND branch_id = #{@center.branch_id}) AND date >= '#{d.strftime('%Y-%m-%d')}' AND stale=0")
+    sql = %Q{
+        UPDATE cachers 
+        SET stale=1 
+        WHERE (center_id=#{cid} 
+             OR (center_id = 0 
+               AND branch_id = #{@center.branch_id})) 
+           AND date >= '#{d.strftime('%Y-%m-%d')}' 
+           AND stale=0 
+           AND type IN ('CenterCache','BranchCache')}
+    repository.adapter.execute(sql)
     # puts "STALIFIED CENTERS in #{(Time.now - t).round(2)} secs"
 
   end
@@ -370,3 +385,189 @@ def self.missing(selection)
 
 
 end
+
+
+class Cache < Cacher
+
+  # this class is for caching according to properties other than branch and center id i.e. funding line, staff member, loan product and
+  # any other arbitrary collection of loan including portfolios
+  # all references to funding line in comments is purely for illustration and not meant literally
+
+
+  def self.update(hash = {})
+    debugger
+    # is our cache based off fields in the loan history table?
+    base_model_name = self.to_s.gsub("Cache","")
+    loan_history_field = "#{base_model_name.snake_case}_id".to_sym
+    # creates a cache per funding line for branches and funding lines per the hash passed as argument
+    date = hash.delete(:date) || Date.today
+    force = hash.delete(:force) || false
+    hash = hash.select{|k,v| [:branch_id, :center_id, loan_history_field].include?(k)}.to_hash
+
+    unless force
+      # the problem of doing 2 factor stalification - we might have within the same center some loans that belong to a stale funding line
+      # and others that belong to other fuding lines.
+      # therefore, the incremental update must necessarily be per loan_ids.
+      # we have to find the loan ids in the stale funding line
+      fl_caches = self.all(:center_id.not => 0, :date => date)
+      stale_caches = fl_caches.stale.aggregate(:model_id, :center_id)
+      missing_caches = LoanHistory.all(:date.gte => date).aggregate(loan_history_field, :center_id) - fl_caches.aggregate(:model_id, :center_id)
+      caches_to_do = stale_caches + missing_caches
+      unless caches_to_do.blank?
+        ids = caches_to_do.map{|x| "(#{x.join(',')})"}
+        sql = "select loan_id from loan_history where (#{loan_history_field}, center_id) in (#{ids.join(',')}) group by loan_id"
+        loan_ids = repository.adapter.query(sql)
+        hash[:loan_id] = loan_ids
+      end
+    end
+
+    fl_data = self.create(hash.merge(:date => date, :group_by => [:branch_id, :center_id, loan_history_field])).deepen.values.sum
+    return false if fl_data == nil
+    now = DateTime.now
+    fl_data.delete(:no_group)
+    return true if fl_data.empty?
+    fls = fl_data.map do |center_id,funding_line_hash|
+      funding_line_hash.map do |fl_id, fl|
+        fl_data[center_id][fl_id].merge({:type => self.to_s,:model_name => base_model_name, :model_id => fl_id, :date => date, :updated_at => now})
+      end
+    end.flatten
+    if fls.nil?
+      return false 
+    end
+    _fls = fls.map{|fl| fl.delete(loan_history_field); fl}
+    sql = get_bulk_insert_sql("cachers", _fls)
+    # destroy the relevant funding_line caches in the database
+    debugger
+    ids = fl_data.map{|center_id, models|
+      models.map{|fl_id, data|
+        [center_id, fl_id]
+      }
+    }
+    ids = ids.flatten(1).map{|x| "(#{x.join(',')})"}.join(",")
+    raise unless repository.adapter.execute("delete from cachers where model_name = '#{base_model_name}' and (center_id, model_id) in (#{ids})")
+    repository.adapter.execute(sql)
+
+    # now do the branch aggregates for each funding line cache
+    Kernel.const_get(base_model_name).all.each do |fl|
+      debugger
+      relevant_branch_ids = (hash[:center_ids] ? Center.all(:id => hash[:center_ids]) : Center.all).aggregate(:branch_id)
+      branch_data_hash = self.all(:model_name => base_model_name, :branch_id => relevant_branch_ids, :date => date, :center_id.gt => 0, :model_id => fl.id).group_by{|x| x.branch_id}.to_hash
+
+      # we now have {:branch => [{...center data...}, {...center data...}]}, ...
+      # we have to convert this to {:branch => { sum of centers data }, ...}
+
+      branch_data = branch_data_hash.map do |bid,ccs|
+        sum_centers = ccs.map do |c|
+          center_sum_attrs = c.attributes.select{|k,v| v.is_a? Numeric}.to_hash
+        end
+        [bid, sum_centers.reduce({}){|s,h| s+h}]
+      end.to_hash
+
+      # TODO then add the loans that do not belong to any center
+      # this does not exist right now so there is no code here.
+      # when you add clients directly to the branch, do also update the code here
+
+      branch_data.map do |bid, c|
+        debugger
+        bc = self.first_or_new({:model_name => base_model_name, :branch_id => bid, :date => date, :model_id => fl.id, :center_id => 0})
+        attrs = c.merge(:branch_id => bid, :center_id => 0, :model_id => fl.id, :stale => false, :updated_at => DateTime.now, :model_name => base_model_name)
+        if bc.new?
+          bc.attributes = attrs.merge(:id => nil, :updated_at => DateTime.now)
+          bc.save
+        else
+          bc.update(attrs.merge(:id => bc.id))
+        end
+      end
+    end
+  end
+
+
+  def self.create(hash = {})
+    # creates a cacher from loan_history table for any arbitrary condition. Also does grouping
+    debugger
+    base_model_name = self.to_s.gsub("Cache","")
+    loan_history_field = "#{base_model_name.snake_case}_id".to_sym
+    date = hash.delete(:date) || Date.today
+    group_by = hash.delete(:group_by) || [:branch_id, :center_id, loan_history_field]
+    cols = hash.delete(:cols) || COLS
+    flow_cols = FLOW_COLS
+    balances = LoanHistory.latest_sum(hash,date, group_by, cols)
+    pmts = LoanHistory.composite_key_sum(LoanHistory.all(hash.merge(:date => date)).aggregate(:composite_key), group_by, flow_cols)
+    # if there are no loan history rows that match today, then pmts is just a single hash, else it is a hash of hashes
+
+``    # set up some default hashes to use in case we get dodgy results back.
+    ng_pmts = flow_cols.map{|c| [c,0]}.to_hash # ng = no good. we return this if we get dodgy data
+    ng_bals = cols.map{|c| [c,0]}.to_hash # ng = no good. we return this if we get dodgy data
+    # workaround for the situation where no rows get returned for centers without loans.
+    # this makes it very difficult to find missing center caches so we must have a row for all centers, even if it is full of zeros
+    # find all relevant centers
+
+    # if we are doing only a subset of centers / funding lines, we do it using loan_ids.
+    # so if we have some loan_ids, then we just use these
+    if hash[:loan_id]
+      universe = LoanHistory.all(hash).aggregate(:branch_id, :center_id, loan_history_field)
+    else
+      # else we find all the centers that we're interested in
+       _u = Center.all(hash[:center_id] ? {:id => hash[:center_id]} : {}).aggregate(:branch_id, :id) # array of [[:branch_id, :center_id]...] for all branches and centers
+      # now add the funding line id to each of these universes
+      universe = Kernel.const_get(base_model_name).all.map{|f| 
+        __u = Marshal.load(Marshal.dump(_u)) # effing ruby pass by reference....my ass
+        __u.map{|u| u.push(f.id)}
+      }.flatten(1)
+    end
+    universe.map do |k| 
+      _p = pmts[k] || ng_pmts
+      _b = balances[k] || ng_bals
+      extra = balances[k] ? {} : {:center_id => k[1], :branch_id => k[0], :model_id => k[2]} # for ng rows, we need to insert center_id and branch_id
+      [k, _p.merge(_b).merge(extra)]
+    end.to_hash
+
+  end
+
+
+  # executes an SQL statement to mark all center caches and branch caches for this center as stale. Only does this for cachers on or after options[:date]
+  # params [Hash] a hash of options thus {:center_id => Integer, :date => Date or String, :model_id => Integer}
+  def self.stalify(options = {})
+    base_model_name = self.to_s.gsub("Cache","")
+    loan_history_field = "#{base_model_name.snake_case}_id".to_sym
+
+    t = Time.now
+    raise NotAcceptable unless [:center_id, :date].map{|o| options[o]}.compact.size == 2
+    cid = options[:center_id]
+    @center = Center.get(cid)
+    d = options[:date].class != Date ? (Date.parse(options[:date]) rescue nil) : options[:date]
+    raise ArgumentError.new("Cannot parse date") unless d
+
+    sql = %Q{
+      UPDATE cachers SET stale=1 
+      WHERE model_name = '#{base_model_name}'
+        AND model_id=#{options[:model_id]} 
+        AND (center_id = #{cid} 
+           OR (center_id = 0 AND branch_id = #{@center.branch_id}) 
+           OR (branch_id = 0 and center_id = 0 and model_id = #{options[:model_id]})
+          ) 
+        AND date >= '#{d.strftime('%Y-%m-%d')}' AND stale=0}
+    repository.adapter.execute(sql)
+  end
+
+  # finds the missing caches given some caches 
+  def self.missing(selection)
+    base_model_name = self.to_s.gsub("Cache","")
+    loan_history_field = "#{base_model_name.snake_case}_id".to_sym
+
+    bs = self.all(selection).aggregate(:date, :model_id).group_by{|x| x[0]}.to_hash.map{|k,v| [k, v.map{|x| x[1]}]}.to_hash
+    # bs is a hash of {:date => [:center_id,...]}
+    date = selection.delete(:date)
+    selection[:id] = selection.delete(loan_history_field) if selection[loan_history_field]
+    hs = Kernel.const_get(base_model_name).all(selection.merge(:creation_date.lte => date)).aggregate(:id)
+    date.map{|d| [d,hs - (bs[d] || [])]}.to_hash
+  end
+end
+
+
+class FundingLineCache < Cache
+end
+
+class LoanProductCache < Cache
+end
+

app/models/data_access_observer.rb

@@ -1,6 +1,6 @@
 class DataAccessObserver
   include DataMapper::Observer
-  observe *(DataMapper::Model.descendants.to_a - [AuditTrail, Cacher, BranchCache, CenterCache] + [Branch, Center, ClientGroup, Client, Loan, Payment, Fee]).uniq # strange bug where observer drops some of the descnedants.
+  observe *(DataMapper::Model.descendants.to_a - [AuditTrail, Cacher, BranchCache, CenterCache, FundingLineCache] + [Branch, Center, ClientGroup, Client, Loan, Payment, Fee]).uniq # strange bug where observer drops some of the descnedants.
 
 
   def self.insert_session(id)

app/models/loan.rb

@@ -86,6 +86,7 @@ def set_bullet_installments
   property :created_by_user_id,                Integer, :nullable => true, :index => true
   property :cheque_number,                     String,  :length => 20, :nullable => true, :index => true
   property :cycle_number,                      Integer, :default => 1, :nullable => false, :index => true
+  property :loan_pool_id,                      Integer, :nullable => true, :index => true
 
   #these amount and disbursal dates are required for TakeOver loan types. 
   property :original_amount,                    Integer
@@ -144,6 +145,8 @@ def set_bullet_installments
   belongs_to :verified_by,               :child_key => [:verified_by_user_id],                :model => 'User'
   belongs_to :repayment_style
 
+  belongs_to :loan_pool
+
   belongs_to :organization, :parent_key => [:org_guid], :child_key => [:parent_org_guid], :nullable => true  
   property   :parent_org_guid, String, :nullable => true