Switch to CombinePopoloMemberships

Gemfile

@@ -11,3 +11,5 @@ gem 'pry'
 gem 'rake'
 gem 'capybara'
 gem 'poltergeist'
+gem "combine_popolo_memberships", git: "https://github.com/everypolitician/combine_popolo_memberships.git"
+

Gemfile.lock

@@ -1,3 +1,9 @@
+GIT
+  remote: https://github.com/everypolitician/combine_popolo_memberships.git
+  revision: 0a18a0defb11cf08f59f7f6cdc8de898227cbe89
+  specs:
+    combine_popolo_memberships (0.2.0)
+
 GIT
   remote: https://github.com/openaustralia/scraperwiki-ruby.git
   revision: fc50176812505e463077d5c673d504a6a234aa78
@@ -57,10 +63,14 @@ PLATFORMS
 DEPENDENCIES
   capybara
   colorize
+  combine_popolo_memberships!
   json
   nokogiri
   open-uri-cached
   poltergeist
   pry
   rake
   scraperwiki!
+
+BUNDLED WITH
+   1.11.2

scraper.rb

@@ -5,6 +5,7 @@
 require 'capybara'
 require 'capybara/poltergeist'
 require 'nokogiri'
+require 'combine_popolo_memberships'
 
 require 'colorize'
 require 'pry'
@@ -20,29 +21,6 @@ def tidy
   end
 end
 
-def overlap(mem, term)
-  mS = mem[:start_date].to_s.empty?  ? '0000-00-00' : mem[:start_date]
-  mE = mem[:end_date].to_s.empty?    ? '9999-99-99' : mem[:end_date]
-  tS = term[:start_date].to_s.empty? ? '0000-00-00' : term[:start_date]
-  tE = term[:end_date].to_s.empty?   ? '9999-99-99' : term[:end_date]
-
-  return unless mS < tE && mE > tS
-  (s, e) = [mS, mE, tS, tE].sort[1,2]
-  return { 
-    _data: [mem, term],
-    start_date: s == '0000-00-00' ? nil : s,
-    end_date:   e == '9999-99-99' ? nil : e,
-  }
-end
-
-def combine(h)
-  into_name, into_data, from_name, from_data = h.flatten
-  from_data.product(into_data).map { |a,b| overlap(a,b) }.compact.map { |h|
-    data = h.delete :_data
-    h.merge({ from_name => data.first[:id], into_name => data.last[:id] })
-  }.sort_by { |h| h[:start_date] }
-end
-
 pages = [
   '/personal/deputati/saeima12_depweb_public.nsf/deputies?OpenView&lang=EN&count=1000',
   # '/personal/deputati/saeima12_depweb_public.nsf/deputiesByMandate?OpenView&restricttocategory=1&lang=EN&count=1000',
@@ -100,7 +78,9 @@ def combine(h)
     binding.pry if terms.count.zero? || groups.count.zero?
 
     puts person[:name]
-    combine(note: terms, party: groups).each do |mem|
+
+    CombinePopoloMemberships.combine(note: terms, party: groups).each do |mem|
+      %i(role style).each { |i| mem.delete(i) }
       data = person.merge(front).merge(mem)
       data[:party] = data[:party].sub(' parliamentary group','')
       ScraperWiki.save_sqlite([:id, :term, :party, :start_date], data)