Permalink
Browse files

Merge branch 'takeover'

  • Loading branch information...
2 parents 159b9ca + 3865b54 commit 051f44fd3b4a015faa5beda15530f037b3c3b409 Siddharth Sharma committed Feb 6, 2012
View
@@ -22,6 +22,11 @@ def missing
end
def generate
+ @model = params[:by] ? Kernel.const_get(params[:by].camel_case + "Cache") : BranchCache
+ if @from_date and @to_date
+ (@from_date..@to_date).each{|date| @model.update(:date => date)}
+ else
+ @model.update(:date => (@date || Date.today))
if Branch.count > 0
if @from_date and @to_date
(@from_date..@to_date).each{|date| BranchCache.update(date)}
@@ -52,10 +57,21 @@ def freshen
def consolidate
get_cachers
group_by = @level.to_s.singularize
- group_by_model = Kernel.const_get(group_by.camelcase)
+ group_by_model = Kernel.const_get(group_by.camelcase) rescue Kernel.const_get(params[:by].camel_case)
unless group_by == "loan"
@cachers = @cachers.group_by{|c| c.send("#{group_by}_id".to_sym)}.to_hash.map do |group_by_id, cachers|
- cachers.reduce(:consolidate)
+ # when we are aggregating "by" something else we need to consolidate cachers that span across dates and
+ # add cachers for the same date
+ if params[:by]
+ $debug = true if group_by_id == 2
+ cachers_for_date = cachers.group_by{|c| c.date}.to_hash.map{|d, cs| [d,cs.reduce(:+)]}.to_hash
+ r = cachers_for_date.values.reduce(:consolidate)
+ r.model_id = group_by_id
+ r.branch_id = nil
+ r
+ else
+ cachers.reduce(:consolidate)
+ end
end
end
display @cachers, :template => 'cachers/index', :layout => (params[:layout] or Nothing).to_sym
@@ -103,37 +119,60 @@ def parse_dates
def get_cachers
q = {}
q[:branch_id] = params[:branch_id] unless params[:branch_id].blank?
- if (not params[:branch_id].blank?)
- q[:center_id] = params[:center_id] unless (params[:center_id].blank? or params[:center_id].to_i == 0)
- else
- q[:model_name] = "Branch"
+ unless params[:branch_id].blank?
+ if params[:center_id]
+ q[:center_id] = params[:center_id] unless (params[:center_id].blank? or params[:center_id].to_i == 0)
+ else
+ q[:center_id.not] = 0
+ end
+ end
+ if true
+ if params[:by]
+ q[:model_name] = params[:by].camel_case
+ q[:center_id] ||= 0 unless q[:center_id.not]
+ q[:model_id] = params[:model_id] if params[:model_id]
+ else
+ q[:model_name] = ["Branch","Center"]
+ end
end
q[:date] = @date if @date
q[:date] = @from_date..@to_date if (@from_date and @to_date)
q[:stale] = true if params[:stale]
@cachers = Cacher.all(q)
q.delete(:model_name)
- @missing_centers = CenterCache.missing(q)
+ if params[:by]
+ @missing_centers = {} # TODO
+ else
+ @missing_centers = {} #CenterCache.missing(q)
+ end
get_context
end
def get_context
@center = params[:center_id].blank? ? nil : Center.get(params[:center_id])
@branch = params[:branch_id].blank? ? nil : Branch.get(params[:branch_id])
+ @area = params[:area_id].blank? ? nil : Area.get(params[:area_id])
+ @region = params[:region_id].blank? ? nil : Region.get(params[:region_id])
@center_names = @cachers.blank? ? {} : Center.all(:id => @cachers.aggregate(:center_id)).aggregate(:id, :name).to_hash
@branch_names = @cachers.blank? ? {} : Branch.all(:id => @cachers.aggregate(:branch_id)).aggregate(:id, :name).to_hash
q = (@from_date and @to_date) ? {:date => @from_date..@to_date} : {:date => @date}
- @stale_centers = CenterCache.all(q.merge(:stale => true))
+ @stale_centers = Cacher.all(q.merge(:stale => true))
@stale_branches = BranchCache.all(q.merge(:stale => true))
@last_cache_update = @cachers.aggregate(:updated_at.min)
@resource = params[:action] == "index" ? :cachers : (params[:action].to_s + "_" + "cachers").to_sym
@keys = [:branch_id, :center_id] + (ReportFormat.get(params[:report_format]) || ReportFormat.first).keys
@total_keys = @keys[2..-1]
if @resource == :split_cachers
- @level = params[:center_id].blank? ? :branches : :centers
+ @level = (params[:center_id].blank? ? :branches : :centers)
@keys = [:date] + @keys
else
- @level = (not params[:center_id].blank?) ? :loans : ((not params[:branch_id].blank?) ? :centers : :branches)
+ @level = @center ? :loans : (@branch ? :centers : (@area ? :branches : (@region ? :areas : :branches)))
+ if params[:by]
+ @level = :model unless @level == :loans
+ @keys = [:model_name] + @keys
+ @model = Kernel.const_get(params[:by].camel_case)
+ else
+ end
end
end
@@ -4,13 +4,16 @@ class CacheObserver
observe Payment, Loan
def self.get_date(obj)
- date = obj.respond_to?(:applied_on) ? obj.applied_on : (obj.respond_to?(:received_on) ? obj.received_on : nil)
+ date = obj.respond_to?(:applied_on) ? obj.applied_on : (obj.respond_to?(:received_on) ? obj.received_on : enil)
end
def self.make_stale(obj)
center_id = obj.c_center_id
date = CacheObserver.get_date(obj)
+ loan = obj.is_a?(Loan) ? obj : obj.loan
+ info = loan.info(date)
CenterCache.stalify(:center_id => obj.c_center_id, :date => date) if date
+ FundingLineCache.stalify(:center_id => obj.c_center_id, :date => date, :model_id => info.funding_line_id)
end
after :create do
View
@@ -8,6 +8,11 @@ class Cacher
property :model_id, Integer, :nullable => false, :index => true, :unique => [:model_name, :date], :key => true
property :branch_id, Integer, :index => true, :key => true
property :center_id, Integer, :index => true, :key => true
+<<<<<<< HEAD
+
+ property :funding_line_id, Integer, :index => true
+=======
+>>>>>>> 53f1094295262f8d6c7e371a03dedd10ddff71af
property :scheduled_outstanding_total, Float, :nullable => false
property :scheduled_outstanding_principal, Float, :nullable => false
property :actual_outstanding_total, Float, :nullable => false
@@ -150,6 +155,7 @@ def + (other)
me = self.attributes; other = other.attributes;
attrs = me + other; attrs[:date] = date; attrs[:model_name] = "Sum";
attrs[:model_id] = nil; attrs[:branch_id] = nil; attrs[:center_id] = nil;
+ attrs[:stale] = me[:stale] || other[:stale]
Cacher.new(attrs)
end
@@ -163,7 +169,7 @@ def self.process_queue
class BranchCache < Cacher
def self.recreate(date = Date.today, branch_ids = nil)
- self.update(date, branch_ids, true)
+ self.update(:date => date, :branch_ids => branch_ids, :force => true)
end
def self.update(date = Date.today, branch_ids = nil, force = false)
@@ -271,7 +277,8 @@ class CenterCache < Cacher
# these have to be hooks, for obvious reasons, but for now we make do with some hardcoded magic!
-s EXTRA_FIELDS = [:delayed_disbursals]
+ EXTRA_FIELDS = [:delayed_disbursals]
+
def self.update(hash = {})
# creates a cache per center for branches and centers per the hash passed as argument
t = Time.now
@@ -348,8 +355,16 @@ def self.stalify(options = {})
@center = Center.get(cid)
d = options[:date].class != Date ? (Date.parse(options[:date]) rescue nil) : options[:date]
raise ArgumentError.new("Cannot parse date") unless d
-
- repository.adapter.execute("UPDATE cachers SET stale=1 WHERE center_id=#{cid} OR (center_id = 0 AND branch_id = #{@center.branch_id}) AND date >= '#{d.strftime('%Y-%m-%d')}' AND stale=0")
+ sql = %Q{
+ UPDATE cachers
+ SET stale=1
+ WHERE (center_id=#{cid}
+ OR (center_id = 0
+ AND branch_id = #{@center.branch_id}))
+ AND date >= '#{d.strftime('%Y-%m-%d')}'
+ AND stale=0
+ AND type IN ('CenterCache','BranchCache')}
+ repository.adapter.execute(sql)
# puts "STALIFIED CENTERS in #{(Time.now - t).round(2)} secs"
end
@@ -370,3 +385,189 @@ def self.missing(selection)
end
+
+
+class Cache < Cacher
+
+ # this class is for caching according to properties other than branch and center id i.e. funding line, staff member, loan product and
+ # any other arbitrary collection of loan including portfolios
+ # all references to funding line in comments is purely for illustration and not meant literally
+
+
+ def self.update(hash = {})
+ debugger
+ # is our cache based off fields in the loan history table?
+ base_model_name = self.to_s.gsub("Cache","")
+ loan_history_field = "#{base_model_name.snake_case}_id".to_sym
+ # creates a cache per funding line for branches and funding lines per the hash passed as argument
+ date = hash.delete(:date) || Date.today
+ force = hash.delete(:force) || false
+ hash = hash.select{|k,v| [:branch_id, :center_id, loan_history_field].include?(k)}.to_hash
+
+ unless force
+ # the problem of doing 2 factor stalification - we might have within the same center some loans that belong to a stale funding line
+ # and others that belong to other fuding lines.
+ # therefore, the incremental update must necessarily be per loan_ids.
+ # we have to find the loan ids in the stale funding line
+ fl_caches = self.all(:center_id.not => 0, :date => date)
+ stale_caches = fl_caches.stale.aggregate(:model_id, :center_id)
+ missing_caches = LoanHistory.all(:date.gte => date).aggregate(loan_history_field, :center_id) - fl_caches.aggregate(:model_id, :center_id)
+ caches_to_do = stale_caches + missing_caches
+ unless caches_to_do.blank?
+ ids = caches_to_do.map{|x| "(#{x.join(',')})"}
+ sql = "select loan_id from loan_history where (#{loan_history_field}, center_id) in (#{ids.join(',')}) group by loan_id"
+ loan_ids = repository.adapter.query(sql)
+ hash[:loan_id] = loan_ids
+ end
+ end
+
+ fl_data = self.create(hash.merge(:date => date, :group_by => [:branch_id, :center_id, loan_history_field])).deepen.values.sum
+ return false if fl_data == nil
+ now = DateTime.now
+ fl_data.delete(:no_group)
+ return true if fl_data.empty?
+ fls = fl_data.map do |center_id,funding_line_hash|
+ funding_line_hash.map do |fl_id, fl|
+ fl_data[center_id][fl_id].merge({:type => self.to_s,:model_name => base_model_name, :model_id => fl_id, :date => date, :updated_at => now})
+ end
+ end.flatten
+ if fls.nil?
+ return false
+ end
+ _fls = fls.map{|fl| fl.delete(loan_history_field); fl}
+ sql = get_bulk_insert_sql("cachers", _fls)
+ # destroy the relevant funding_line caches in the database
+ debugger
+ ids = fl_data.map{|center_id, models|
+ models.map{|fl_id, data|
+ [center_id, fl_id]
+ }
+ }
+ ids = ids.flatten(1).map{|x| "(#{x.join(',')})"}.join(",")
+ raise unless repository.adapter.execute("delete from cachers where model_name = '#{base_model_name}' and (center_id, model_id) in (#{ids})")
+ repository.adapter.execute(sql)
+
+ # now do the branch aggregates for each funding line cache
+ Kernel.const_get(base_model_name).all.each do |fl|
+ debugger
+ relevant_branch_ids = (hash[:center_ids] ? Center.all(:id => hash[:center_ids]) : Center.all).aggregate(:branch_id)
+ branch_data_hash = self.all(:model_name => base_model_name, :branch_id => relevant_branch_ids, :date => date, :center_id.gt => 0, :model_id => fl.id).group_by{|x| x.branch_id}.to_hash
+
+ # we now have {:branch => [{...center data...}, {...center data...}]}, ...
+ # we have to convert this to {:branch => { sum of centers data }, ...}
+
+ branch_data = branch_data_hash.map do |bid,ccs|
+ sum_centers = ccs.map do |c|
+ center_sum_attrs = c.attributes.select{|k,v| v.is_a? Numeric}.to_hash
+ end
+ [bid, sum_centers.reduce({}){|s,h| s+h}]
+ end.to_hash
+
+ # TODO then add the loans that do not belong to any center
+ # this does not exist right now so there is no code here.
+ # when you add clients directly to the branch, do also update the code here
+
+ branch_data.map do |bid, c|
+ debugger
+ bc = self.first_or_new({:model_name => base_model_name, :branch_id => bid, :date => date, :model_id => fl.id, :center_id => 0})
+ attrs = c.merge(:branch_id => bid, :center_id => 0, :model_id => fl.id, :stale => false, :updated_at => DateTime.now, :model_name => base_model_name)
+ if bc.new?
+ bc.attributes = attrs.merge(:id => nil, :updated_at => DateTime.now)
+ bc.save
+ else
+ bc.update(attrs.merge(:id => bc.id))
+ end
+ end
+ end
+ end
+
+
+ def self.create(hash = {})
+ # creates a cacher from loan_history table for any arbitrary condition. Also does grouping
+ debugger
+ base_model_name = self.to_s.gsub("Cache","")
+ loan_history_field = "#{base_model_name.snake_case}_id".to_sym
+ date = hash.delete(:date) || Date.today
+ group_by = hash.delete(:group_by) || [:branch_id, :center_id, loan_history_field]
+ cols = hash.delete(:cols) || COLS
+ flow_cols = FLOW_COLS
+ balances = LoanHistory.latest_sum(hash,date, group_by, cols)
+ pmts = LoanHistory.composite_key_sum(LoanHistory.all(hash.merge(:date => date)).aggregate(:composite_key), group_by, flow_cols)
+ # if there are no loan history rows that match today, then pmts is just a single hash, else it is a hash of hashes
+
+`` # set up some default hashes to use in case we get dodgy results back.
+ ng_pmts = flow_cols.map{|c| [c,0]}.to_hash # ng = no good. we return this if we get dodgy data
+ ng_bals = cols.map{|c| [c,0]}.to_hash # ng = no good. we return this if we get dodgy data
+ # workaround for the situation where no rows get returned for centers without loans.
+ # this makes it very difficult to find missing center caches so we must have a row for all centers, even if it is full of zeros
+ # find all relevant centers
+
+ # if we are doing only a subset of centers / funding lines, we do it using loan_ids.
+ # so if we have some loan_ids, then we just use these
+ if hash[:loan_id]
+ universe = LoanHistory.all(hash).aggregate(:branch_id, :center_id, loan_history_field)
+ else
+ # else we find all the centers that we're interested in
+ _u = Center.all(hash[:center_id] ? {:id => hash[:center_id]} : {}).aggregate(:branch_id, :id) # array of [[:branch_id, :center_id]...] for all branches and centers
+ # now add the funding line id to each of these universes
+ universe = Kernel.const_get(base_model_name).all.map{|f|
+ __u = Marshal.load(Marshal.dump(_u)) # effing ruby pass by reference....my ass
+ __u.map{|u| u.push(f.id)}
+ }.flatten(1)
+ end
+ universe.map do |k|
+ _p = pmts[k] || ng_pmts
+ _b = balances[k] || ng_bals
+ extra = balances[k] ? {} : {:center_id => k[1], :branch_id => k[0], :model_id => k[2]} # for ng rows, we need to insert center_id and branch_id
+ [k, _p.merge(_b).merge(extra)]
+ end.to_hash
+
+ end
+
+
+ # executes an SQL statement to mark all center caches and branch caches for this center as stale. Only does this for cachers on or after options[:date]
+ # params [Hash] a hash of options thus {:center_id => Integer, :date => Date or String, :model_id => Integer}
+ def self.stalify(options = {})
+ base_model_name = self.to_s.gsub("Cache","")
+ loan_history_field = "#{base_model_name.snake_case}_id".to_sym
+
+ t = Time.now
+ raise NotAcceptable unless [:center_id, :date].map{|o| options[o]}.compact.size == 2
+ cid = options[:center_id]
+ @center = Center.get(cid)
+ d = options[:date].class != Date ? (Date.parse(options[:date]) rescue nil) : options[:date]
+ raise ArgumentError.new("Cannot parse date") unless d
+
+ sql = %Q{
+ UPDATE cachers SET stale=1
+ WHERE model_name = '#{base_model_name}'
+ AND model_id=#{options[:model_id]}
+ AND (center_id = #{cid}
+ OR (center_id = 0 AND branch_id = #{@center.branch_id})
+ OR (branch_id = 0 and center_id = 0 and model_id = #{options[:model_id]})
+ )
+ AND date >= '#{d.strftime('%Y-%m-%d')}' AND stale=0}
+ repository.adapter.execute(sql)
+ end
+
+ # finds the missing caches given some caches
+ def self.missing(selection)
+ base_model_name = self.to_s.gsub("Cache","")
+ loan_history_field = "#{base_model_name.snake_case}_id".to_sym
+
+ bs = self.all(selection).aggregate(:date, :model_id).group_by{|x| x[0]}.to_hash.map{|k,v| [k, v.map{|x| x[1]}]}.to_hash
+ # bs is a hash of {:date => [:center_id,...]}
+ date = selection.delete(:date)
+ selection[:id] = selection.delete(loan_history_field) if selection[loan_history_field]
+ hs = Kernel.const_get(base_model_name).all(selection.merge(:creation_date.lte => date)).aggregate(:id)
+ date.map{|d| [d,hs - (bs[d] || [])]}.to_hash
+ end
+end
+
+
+class FundingLineCache < Cache
+end
+
+class LoanProductCache < Cache
+end
+
@@ -1,6 +1,6 @@
class DataAccessObserver
include DataMapper::Observer
- observe *(DataMapper::Model.descendants.to_a - [AuditTrail, Cacher, BranchCache, CenterCache] + [Branch, Center, ClientGroup, Client, Loan, Payment, Fee]).uniq # strange bug where observer drops some of the descnedants.
+ observe *(DataMapper::Model.descendants.to_a - [AuditTrail, Cacher, BranchCache, CenterCache, FundingLineCache] + [Branch, Center, ClientGroup, Client, Loan, Payment, Fee]).uniq # strange bug where observer drops some of the descnedants.
def self.insert_session(id)
View
@@ -86,6 +86,7 @@ def set_bullet_installments
property :created_by_user_id, Integer, :nullable => true, :index => true
property :cheque_number, String, :length => 20, :nullable => true, :index => true
property :cycle_number, Integer, :default => 1, :nullable => false, :index => true
+ property :loan_pool_id, Integer, :nullable => true, :index => true
#these amount and disbursal dates are required for TakeOver loan types.
property :original_amount, Integer
@@ -144,6 +145,8 @@ def set_bullet_installments
belongs_to :verified_by, :child_key => [:verified_by_user_id], :model => 'User'
belongs_to :repayment_style
+ belongs_to :loan_pool
+
belongs_to :organization, :parent_key => [:org_guid], :child_key => [:parent_org_guid], :nullable => true
property :parent_org_guid, String, :nullable => true
Oops, something went wrong.

0 comments on commit 051f44f

Please sign in to comment.