improved generation of bill_events for third reading bills - turn imp…

…rest debates into third readings

app/controllers/application_controller.rb

@@ -106,13 +106,7 @@ def third_reading_and_negatived_votes_by_parliament
     @parliament = Parliament.find(params[:id])
     if @parliament
       respond_to do |format|
-        votes = Vote.third_reading_and_negatived_votes(@parliament.id).sort_by{|x| x.debate.date}
-        parent_bills = votes.collect(&:bill)
-        # parent_bill_names = %Q|"","#{bills.collect(&:bill_name).join('","')}"|
-        # parent_bill_urls = %Q|"","#{bill_urls.join('","')}"|
-        # child_bill_names = %Q|"","#{votes.collect(&:bill_name).join('","')}"|
-        # vote_vectors = Vote.vote_vectors(@parliament.id)
-        # format.csv { render :text => parent_bill_names + "\n" + parent_bill_urls + "\n" + child_bill_names + "\n" + vote_vectors.collect(&:to_s).join("\n") }
+        votes = Vote.third_reading_and_negatived_votes(@parliament.id).sort_by{|x| x.debate.date.to_s + x.id.to_s}
 
         bills = votes.collect(&:vote_bill)
         bill_urls = bills.collect{|x| show_bill_url(x.id_hash)}
@@ -122,14 +116,8 @@ def third_reading_and_negatived_votes_by_parliament
 
         vote_vectors = Vote.vote_vectors(@parliament.id, to_string=false)
 
-        parent_bill_names = ["Formerly part of bill"] + votes.collect do |v|
-          name = if v.bill
-            v.bill.bill_name
-          elsif v.vote_bill && v.vote_bill.formerly_part_of
-            v.vote_bill.formerly_part_of.bill_name
-          else
-            ''
-          end
+        parent_bill_names = ["Formerly part of bill"] + bills.collect do |bill|
+          (bill && bill.formerly_part_of) ? bill.formerly_part_of.bill_name : ''
         end
 
         child_bill_names.each_with_index do |x,i|

app/helpers/bills_helper.rb

@@ -286,6 +286,7 @@ def bill_event_result_summary bill_event
       result
     else
       result = bill_event.reading_result_from_contributions self
+
       if result.blank?
         bill_event.result_from_contributions self
       else

app/models/bill.rb

@@ -18,6 +18,7 @@ class Bill < ActiveRecord::Base
   validates_presence_of :member_in_charge_id
 
   validates_presence_of :parliament_url
+  validates_uniqueness_of :parliament_id, :allow_blank => true
 
   before_validation :populate_former_name,
       :populate_formerly_part_of,
@@ -36,6 +37,22 @@ class Bill < ActiveRecord::Base
 
   class << self
 
+    def bill_names text
+      bill_text = text.to_s
+      bill_text.gsub!(/(\d)\), Te/, '\1), the Te')
+      bill_text.gsub!(/Bill( \(No \d+\))? and the/,'Bill\1, and the')
+      bill_text.gsub!(/Bill( \([^\)]+\))? and the/,'Bill\1, and the')
+      bill_text.gsub!(/\sbe now read a (\w+) time and the /, ', and the ')
+
+      bills = bill_text.split(/,( and)? the/)
+      bills = bills.select { |b| b.match(/[a-z ]*(.*)/)[1].length > 0 }
+      bills.collect { |b| b.match(/[a-z ]*(.*)/)[1].chomp(', ').strip }
+    end
+
+    def parliament_id parliament_url
+      parliament_url.split('/').last.split('.').first
+    end
+
     def passed_third_reading_no_vote
       readings = BillEvent.find(:all, :conditions => 'name like "%Third%"'); nil
       readings = Parliament.find(48).in_parliament(readings); nil
@@ -51,11 +68,7 @@ def all_bill_names
     end
 
     def bills_from_text_and_date text, date
-      bill_text = text.gsub(/(\d)\), Te/, '\1), the Te')
-      bill_text = bill_text.gsub(/Bill( \([^\)]+\))? and the/,'Bill\1, and the')
-      bills = bill_text.split(/,( and)? the/).collect do |name|
-        name = name.match(/[a-z ]*(.*)/)[1]
-        name = name.chomp(', ').strip unless name.empty?
+      bills = bill_names(text).collect do |name|
         name.empty? ? nil : Bill.from_name_and_date(name, date)
       end.compact
     end
@@ -384,6 +397,10 @@ def full_name
     bill_name
   end
 
+  def is_appropriation_bill?
+    bill_name[/^Appropriation/]
+  end
+
   def negatived?
     first_reading_negatived or second_reading_negatived
   end
@@ -729,7 +746,7 @@ def reset_earliest_date
     end
 
     def populate_parliament_id
-      self.parliament_id = parliament_url.split('/').last.split('.').first if parliament_id.blank?
+      self.parliament_id = Bill.parliament_id(parliament_url) if parliament_id.blank?
     end
 
   protected

app/models/bill_event.rb

@@ -78,12 +78,16 @@ def create_from_bill_debate bill, stage, debate
     end
 
     def create_from_bill_stage bill, stage, date
+      if stage == 'Imprest Supply Debate' && bill.is_appropriation_bill?
+        stage = 'Third Reading' if bill.third_reading.to_s == date.to_s
+      end
       returning(BillEvent.new) do |e|
         e.bill_id     = bill.id
         e.name        = stage
         e.date        = date
       end
     end
+
   end
 
   def date_method
@@ -93,7 +97,15 @@ def date_method
 
   def debates
     debates_in_groups_by_name, votes_by_name = bill.debates_by_name_names_votes_by_name
-    debates = debates_in_groups_by_name.blank? ? [] : debates_in_groups_by_name.select {|list| list.first.normalized_name == self.name}.flatten
+    debates = if debates_in_groups_by_name.blank?
+                []
+              else
+                debates_in_groups_by_name.select {|list| list.first.normalized_name == self.name}.flatten
+              end
+
+    if debates.empty? && bill.is_appropriation_bill?
+      debates = debates_in_groups_by_name.select {|list| list.first.normalized_name == 'Imprest Supply Debate'}.flatten
+    end
 
     debates.sort! do |a,b|
       comparison = b.date <=> a.date
@@ -110,6 +122,10 @@ def debates
   def votes
     votes_by_name = bill.votes_in_groups_by_name
     votes = votes_by_name.blank? ? nil : votes_by_name[self.name]
+    if votes.nil? && bill.is_appropriation_bill?
+      votes = votes_by_name.blank? ? nil : votes_by_name['Imprest Supply Debate']
+    end
+
     votes = votes.compact.uniq if votes
 
     selected_votes = votes.select do |vote|
@@ -227,6 +243,7 @@ def result_from_vote helper
       debate = self.debates.first
       contributions = debate.contributions
       last = contributions.last
+      # raise name + ' ' + debate.inspect if last.nil?
       if last.is_procedural?
         if motion_agreed_to? last.text
           result += '<br /><br />' + strip_tags(last.text).chomp('.') + ':<br />'

app/models/contribution.rb

@@ -232,6 +232,14 @@ def is_motion?
     starts_with?('I move that')
   end
 
+  def is_motion_to_now_read?
+    text.include?('I move') && text.include?('be now read')
+  end
+
+  def bill_names
+    Bill.bill_names text.match(/That the (.*)be now read/)[1]
+  end
+
   def is_interjection?
     false
   end

app/models/debate.rb

@@ -413,6 +413,35 @@ def expire_cached_pages date
       puts 'found: '+debates.size.to_s
       debates.each { |d| d.expire_cached_pages }
     end
+
+    def create_debate_topics debates
+      debates = debates.select { |debate| debate.debate_topics.blank? }
+      contributions = debates.collect(&:motion_to_now_read_contributions).flatten
+
+      debate_to_bill_names = {}
+      contributions.each { |c| debate_to_bill_names[c.debate] = c.bill_names }
+
+      debate_to_bill_names.each_pair { |d,b| puts d.id.to_s + " -> " + b.join(' | ') }
+      puts "\n unknown bills:"
+      debate_to_bill_names.each_pair do |d, bill_names|
+        bill_names.select{|n| Bill.from_name_and_date(n, d.date).nil? }.each { |name| puts "#{d.date} #{name}" }
+      end ;nil
+
+      debate_to_bill_names.each do |debate, bill_names|
+        bills = bill_names.collect { |name| Bill.from_name_and_date(name, debate.date) }.compact
+
+        bills.each do |bill|
+          topic = DebateTopic.find_or_create_by_debate_id_and_topic_id_and_topic_type(debate.id, bill.id, 'Bill')
+          BillEvent.refresh_events_from_bill bill
+          puts topic.inspect
+        end
+        nil
+      end ; nil
+    end
+  end
+
+  def motion_to_now_read_contributions
+    contributions.select(&:is_motion_to_now_read?)
   end
 
   def formerly_about_bill

app/models/vote.rb

@@ -195,10 +195,10 @@ def vote_bill
   end
 
   def bill_name
-    if vote_question[/That the (.+) be now read a third time/] && !$1.include?('Bill, ') && !$1.include?('Bill and')
+    if vote_question[/That the (.+) be now read a (.+) time/] && !$1.include?('Bill, ') && !$1.include?('Bill and')
       $1
     else
-      bill ? bill.bill_name : ('no bill ' + debate.date.to_s + ' ' + bill_names)
+      bill ? bill.bill_name : ('no bill for: ' + debate.date.to_s + ' ' + bill_names)
     end
   end
 

app/views/debates/_debate.haml

@@ -1,7 +1,7 @@
-- if debate.contributions.size > 100
-  = render :partial => 'contribution', :collection => debate.contributions[0..100], :locals => { :debate => debate }
+- if debate.contributions.size > 115
+  = render :partial => 'contribution', :collection => debate.contributions[0..115], :locals => { :debate => debate }
   %div
     %p
-      = "[... plus a further #{debate.contributions.size - 100} contributions not shown here]"
+      = "[... plus a further #{debate.contributions.size - 115} contributions not shown here]"
 - else
   = render :partial => 'contribution', :collection => debate.contributions, :locals => { :debate => debate }

lib/tasks/clean_imprest_debates.rake

@@ -0,0 +1,18 @@
+
+namespace :kiwimp do
+
+  desc 'convert imprest supply debates to third reading debates'
+  task :clean_imprest => [:environment] do
+    debates = Debate.find_by_sql("select * from debates where name like 'Appropriation%' and type != 'SubDebate'") ; nil
+    subdebates = debates.map {|x| x.sub_debates}.flatten ; nil
+    empty_debates = subdebates.select{|x| x.contributions.empty?} ; nil
+    parent_debates = empty_debates.map(&:parent) ; nil
+    imprest_debates = parent_debates.map{|x| x.sub_debates }.flatten.select {|x| x.name == 'Imprest Supply Debate'} ; nil
+    events = imprest_debates.map{|x| x.bill_events}.flatten.select {|x| x.name == 'Imprest Supply Debate'} ; nil
+    events.each {|x| x.name = 'Third Reading' ; x.save } ; nil
+    imprest_debates.each {|x| x.name = 'Third Reading' ; x.save } ; nil
+
+    empty_debates.each {|x| x.destroy} ; nil
+  end
+
+end

lib/tasks/correct_third_readings.rake

@@ -3,56 +3,6 @@ namespace :kiwimp do
   desc "corrects topic of third readings"
   task :correct_third_readings => :environment do
     debates = Debate.find(:all, :conditions => 'name like "%Third Readings%"')
-
-    contributions = debates.inject([]) do |all, debate|
-      if debate.debate_topics.blank?
-        all += debate.contributions.select do |contribution|
-          contribution.text.include?('I move') and contribution.text.include?('be now read')
-        end
-      end
-      all
-    end
-
-    debate_to_bills = contributions.inject({}) do |debate_bills, contribution|
-      bill_text = contribution.text.match(/That the (.*)be now read/)[1].gsub(/Bill( \(No \d+\))? and the/,'Bill\1, and the')
-      bills = bill_text.split(/,( and)? the/)
-      bills = bills.select { |b| b.match(/[a-z ]*(.*)/)[1].length > 0 }
-      bills = bills.collect { |b| b.match(/[a-z ]*(.*)/)[1].chomp(', ') }
-      debate_bills[contribution.debate] = bills
-      debate_bills
-    end
-
-    debate_to_bills.each_pair { |k,v| puts k.id.to_s + " " + v.join(' | ') }
-    puts ''
-    puts 'unknown bills:'
-    debate_to_bills.each_pair do |debate, bills|
-      bills.each do |name|
-        puts debate.date.to_s + ' ' + name if Bill.from_name_and_date(name, debate.date).nil?
-      end
-    end
-
-    debate_to_bill_ids = debate_to_bills.keys.inject({}) do |debate_bill_ids, debate|
-      bills = debate_to_bills[debate].collect do |name|
-        Bill.from_name_and_date(name, debate.date)
-      end.compact
-      debate_bill_ids[debate]= bills.collect {|b| b.id}
-      debate_bill_ids
-    end
-
-    debate_to_bill_ids.each do |debate, bill_ids|
-      bill_ids.each do |bill_id|
-        topic = DebateTopic.find_by_debate_id_and_topic_id_and_topic_type(debate.id, bill_id, 'Bill')
-        unless topic
-          topic = DebateTopic.new
-          topic.debate_id = debate.id
-          topic.topic_type = 'Bill'
-          topic.topic_id = bill_id
-          topic.save
-
-          BillEvent.refresh_events_from_bill Bill.find(bill_id)
-          puts topic
-        end
-      end
-    end
+    Debate.create_debate_topics debates
   end
 end

lib/tasks/update_bills.rake

@@ -27,7 +27,8 @@ namespace :kiwimp do
 
   def update_the_bill bill, updated_bill
     if bill.parliament_url != updated_bill.parliament_url
-      updated_bill.parliament_id = updated_bill.parliament_url.split('/').last.split('.').first
+      updated_bill.populate_parliament_id
+      updated_bill.save
     end
     puts '  update bill ' + bill.bill_name
     bill.update_attributes! updated_bill.attributes
@@ -101,7 +102,7 @@ namespace :kiwimp do
 
     elsif old_bills.size == 0
       puts 'trying to save'
-      bill.parliament_id = bill.parliament_url.split('/').last.split('.').first
+      bill.populate_parliament_id
 
       if Bill.find_all_by_url(bill.url).size > 0
         if bill.earliest_date
@@ -120,7 +121,10 @@ namespace :kiwimp do
 
   def update_bill update_existing, url, name, bill_no
     bill_no = nil if bill_no.blank?
-    bill = Bill.find_by_parliament_url(url)
+    parliament_id = Bill.parliament_id(url)
+
+    bill = Bill.find_by_parliament_id(parliament_id)
+    bill = Bill.find_by_parliament_url(url) unless bill
 
     if bill
       try_bill_update url, bill if update_existing