Initial commit of extracted Reunion gem

Gemfile

@@ -0,0 +1,5 @@
+source 'http://rubygems.org'
+ruby "2.0.0"
+
+gem "ofx", "~> 0.3.2"
+gem "byebug", :group => :development

Gemfile.lock

@@ -0,0 +1,20 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    byebug (2.6.0)
+      columnize (~> 0.3)
+      debugger-linecache (~> 1.2)
+    columnize (0.3.6)
+    debugger-linecache (1.2.0)
+    mini_portile (0.5.1)
+    nokogiri (1.6.0)
+      mini_portile (~> 0.5.0)
+    ofx (0.3.2)
+      nokogiri
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  byebug
+  ofx (~> 0.3.2)

Rakefile

@@ -0,0 +1,9 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs.push "lib"
+  t.libs.push "test"
+  t.pattern = "test/**/*.rb"
+end
+
+task :default => ['test']

lib/reunion.rb

@@ -0,0 +1,44 @@
+require 'yaml'
+require 'bigdecimal'
+require 'csv'
+require 'date'
+require 'ofx'
+require 'stringio'
+require 'benchmark'
+require 'delegate'
+require 'set'
+require 'fileutils'
+
+class Array
+  def stable_sort_by (&block)
+      n = 0
+      sort_by {|x| n+= 1; [block.call(x), n]}
+  end
+end
+class Numeric
+    def to_usd
+        delimiter = ','
+        separator = '.'
+        parts = ("%.2f" % self).split(separator)
+        parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{delimiter}")
+        "$" + parts.join(separator)
+    end
+end
+
+module Reunion
+end
+
+require 'reunion/transaction'
+require 'reunion/account'
+require 'reunion/input_file'
+require 'reunion/parsers'
+require 'reunion/standard_convention'
+require 'reunion/account_merge'
+require 'reunion/account_reconcile'
+require 'reunion/vendors'
+require 'reunion/expectations'
+
+
+require 'reunion/output'
+require 'reunion/transfers'
+require 'reunion/rules'

lib/reunion/account.rb

@@ -0,0 +1,36 @@
+module Reunion
+  class Account
+
+    def initialize(name, currency, tags)
+      @name = name
+      @currency = currency
+      @tags = tags
+      @input_files = []
+    end
+
+    attr_accessor :name, :currency, :tags
+
+    attr_accessor :input_files, :transactions, :statements, :final_discrepancy
+
+
+    #Deletes any transactions in 'secondary_files' that have a similar transaction in primary_files (same date and amount)
+    #returns an array of deleted transactions
+    def delete_overlaps(primary_files, secondary_files)
+      rejected = []
+      secondary_files.each do |sf|
+        sf.transactions.reject! do |txn|
+          #For speed, check if there is a date overlap first
+          overlaps_date = primary_files.any?{|f| f.first_txn_date <= txn[:date] && f.last_txn_date >= txn[:date]}
+          other_exists = overlaps_date && primary_files.any?{|f| f.transactions.any?{|t| t[:date] == txn[:date] && t[:amount] == txn[:amount]}}
+          rejected << txn if other_exists
+          other_exists
+        end
+        ##Todo - update first_txn_date and last_txn_date now that they have changed?
+      end
+
+      rejected
+    end
+
+
+  end
+end 

lib/reunion/account_merge.rb

@@ -0,0 +1,88 @@
+require 'pp'
+class Reunion::Account
+  def merge_duplicate_transactions(all_transactions)
+
+
+    match = lambda { |t|  t.date_str + "|" + ("%.2f" %  t.amount) + "|" + t.description.strip.squeeze(" ").downcase  }
+
+
+
+    #We have to give each transaction a unique index so we can do set math without accidentially removing similar txns
+    #And so we can resort correctly at the end
+    all_transactions.each_with_index { |v, i| v[:temp_index] = i}
+
+    all_transactions = all_transactions.stable_sort_by { |t| match.call(t)}
+
+    #Group into matching transactions
+    matches = all_transactions.chunk(&match).map{|t| t[1]}
+    matches = matches.map do |group|
+
+      log_it = false #group.any? {|r| r[:description] =~ /DELTA AIR LINE/i}
+      if log_it
+        p 
+        pp group 
+        p
+      end 
+
+      next group unless group.count > 1
+
+
+      uniq_ids = group.uniq{|t| t.id}.reject { |t| t.id.nil? }
+      remainder = group - uniq_ids
+
+      subgroups = []
+      #Start with with IDs. Merge matching ID transactions, followed by 1 non-id  transaction from every unrepresented source 
+      subgroups += uniq_ids.map do |with_id|
+        # Collect transactions with matching IDs
+        take = [with_id] + remainder.select {|r| r.id == with_id.id}
+        remainder -= take
+        take = (take + remainder).uniq{|k| k.source}
+        remainder -= take
+        take 
+      end 
+      #Group remaining transactions into subgroups (each subgroup only has 1 txn per source)
+      until remainder.empty? do
+        take = remainder.uniq{|t| t.source}
+        subgroups << take
+        remainder -= take;
+      end
+
+
+
+      p "Merging #{group.count} transactions into #{subgroups.count}" if log_it
+
+      result = []
+      result = subgroups.map do |subgroup| 
+        subgroup = subgroup.sort_by{|t| t[:priority]}
+
+        if log_it
+          pp "--- this sub group ---"
+          pp subgroup
+
+          pp "--- Becomes ---"
+        end 
+
+        has_primary_txn = subgroup.any?{|t| t[:discard_if_unmerged].nil? }
+
+        #Merge them into a single result transaction
+        result_row = subgroup.inject(Reunion::Transaction.new){|acc, current| acc.merge_transaction(current)}
+        result_row.delete(:discard_if_unmerged) if has_primary_txn
+
+        pp result_row if log_it
+        #result_row[:source_rows] = subgroup.dup
+        #result_row[:description] += match.call(result_row)
+        result_row
+      end 
+
+
+
+      result
+    end
+    matches = matches.flatten.compact
+
+    #Restore order
+    matches.sort_by! {|x| [x.date_str, x[:temp_index]]}
+
+    matches
+  end 
+end

lib/reunion/account_reconcile.rb

@@ -0,0 +1,146 @@
+
+class Reunion::Account
+
+  def sort_to_reduce_discrepancies(startbal, combined)
+
+    sorted = []
+    #Group by day
+    by_day = combined.compact.group_by{|r| r[:date].strftime("%Y-%m-%d")}.values
+
+    balance = startbal || 0
+    by_day.each do |day|
+      day_start = balance
+      daily_delta = 0
+
+      index = 2
+      wrapped = day.map do |row|
+        #Calculate 'before' transaction balances
+        w = {row: row}
+
+        w[:amount] = row[:amount] || 0
+
+        if row.key?(:balance_after)
+          w[:daily_delta] = daily_delta = row[:balance_after] - day_start
+        elsif w[:amount] != 0
+          w[:daily_delta] = daily_delta += w[:amount]
+        elsif row[:bal] && !row[:balance].nil?
+          w[:daily_delta] = row[:balance] - day_start
+        else
+          w[:daily_delta] = daily_delta
+        end
+        w[:index] = index += 2
+        w
+      end
+
+      txns = [{amount: 0, daily_delta:0, placeholder:true, index: 0}] + wrapped.select{ |b| b[:amount] != 0 }
+      bals = wrapped.select{ |b| b[:amount] == 0 }
+      bals.each do |b|
+        closest = txns.map{ |t| { delta: (b[:daily_delta] - t[:daily_delta]).abs, txn: t}}.sort_by { |p| p[:delta]}
+        b[:index] = closest.first[:txn][:index] + 1
+      end
+
+      results = (txns + bals).stable_sort_by{|e| e[:index]}
+
+      balance += results.reverse.detect {|e| e[:daily_delta]}[:daily_delta]
+
+      sorted << results.reject{|e|e[:placeholder]}.map{|e| e[:row]}
+
+    end 
+
+    sorted.flatten
+  end
+
+  def reconcile
+    transactions = self.transactions
+    statements = self.statements
+
+    #input: transaction amounts, transaction after_balance values, transaction 
+    # Establish knowns. 
+
+
+    #Order statements first...
+    combined = statements.map {|t| t[:bal] = true; t} + transactions.map {|t| t[:tran] = true; t}
+
+    combined = combined.compact.stable_sort_by { |t| t[:date].iso8601 }
+
+    #Shuffle statements forward to minimize discrepancies
+    combined = sort_to_reduce_discrepancies(0,combined)
+
+    report = []
+    # Output columns:
+    # Date, Amount, Balance, Discrepancy, Description, Source
+
+    last_balance_row = nil
+    last_statement = nil
+    balance = 0
+
+
+    combined.each_with_index do | row, index |
+      result_row = {}
+      result_row[:id] = row[:id] if row[:id]
+      result_row[:date] = row[:date] if row[:date]
+      result_row[:amount] = row[:amount] if row[:amount]
+      result_row[:description] = row[:description] if row[:description]
+      result_row[:source] = File.basename(row[:source].to_s) if row[:source]
+
+      row_amount = row[:amount] ? row[:amount] : 0
+      #What should the balance be after this row?
+      balance_after = row[:bal] ? row[:balance] : (row.key?(:balance_after) ? row[:balance_after] : (balance + row_amount))
+      #p row if balance_after == 0
+      result_row[:balance] = balance_after
+
+      discrepancy_amount = balance_after - (row_amount + balance)
+
+      report << nil if row[:bal]
+
+      if discrepancy_amount.abs > 0.001
+
+        #Get the path of the file that provided the balance in the given row
+        get_balance_source = lambda do |for_row|
+          next for_row[:source] if for_row[:bal] || for_row[:source_rows].nil? || for_row[:balance_after].nil?
+          next for_row[:source_rows].detect { |r| r[:balance_after] == for_row[:balance_after]}[:source]
+        end
+
+        source = ""
+
+        if last_balance_row.nil?
+          description = "Using starting balance of " + ("%.2f" % (balance + discrepancy_amount))
+          source = "From #{File.basename(get_balance_source.call(row).to_s)}"
+        else
+          description = "Discrepancy between #{last_balance_row[:date].strftime("%Y-%m-%d")} and #{result_row[:date].strftime("%Y-%m-%d")} of " + "%.2f" % discrepancy_amount 
+
+          last_balance_source = get_balance_source.call(last_balance_row)
+          balance_source = get_balance_source.call(row)
+
+          #p last_balance_row if last_balance_source.nil? 
+          #p row if balance_source.nil? 
+
+          if (last_balance_source == balance_source)
+            source = "Discrepancy within file: #{File.basename(balance_source.to_s)}"
+          else
+            source = "Discrepancy between file #{File.basename(last_balance_source.to_s)} and #{File.basename(balance_source.to_s)}"
+          end 
+
+        end
+
+        #We have a discrepancy
+        report << {amount:discrepancy_amount, balance: balance, discrepancy: discrepancy_amount, description: description, source: source}
+      end
+
+      balance = balance_after
+
+      report << result_row
+
+      report << nil if row[:bal]
+
+      last_balance_row = row if row[:bal] || row.key?(:balance_after)
+      last_statement = row if row[:bal]
+
+    end
+
+    @final_discrepancy = report.compact.map { |r| r[:discrepancy]}.compact.inject(0, :+)
+    @reconciliation_report = report
+    report
+  end 
+  attr_accessor :reconciliation_report 
+end