@@ -110,14 +110,6 @@ class People < Array # End of methods that return Period objects # Facade for readers and writers - def People.read_members_csv(people_filename, members_filename) - PeopleCSVReader.read_members(people_filename, members_filename) - end - - def read_ministers_csv(filename) - PeopleCSVReader.read_ministers(filename, self) - end - def write_xml(people_filename, members_filename, senators_filename, ministers_filename) PeopleXMLWriter.write(self, people_filename, members_filename, senators_filename, ministers_filename) end lib/people.rb @@ -15,7 +15,8 @@ class PeopleCSVReader data end - def PeopleCSVReader.read_members(people_filename, members_filename) + def PeopleCSVReader.read_members(people_filename = "#{File.dirname(__FILE__)}/../data/people.csv", + members_filename = "#{File.dirname(__FILE__)}/../data/members.csv") data = read_raw_csv(people_filename) data.shift data.shift @@ -81,7 +82,7 @@ class PeopleCSVReader end # Attaches ministerial information to people - def PeopleCSVReader.read_ministers(filename, people) + def PeopleCSVReader.read_ministers(people, filename) data = CSV.readlines(filename) # Remove the first two rows data.shift @@ -101,6 +102,12 @@ class PeopleCSVReader end end + def PeopleCSVReader.read_all_ministers(people, ministers_filename = "#{File.dirname(__FILE__)}/../data/ministers.csv", + shadow_ministers_filename = "#{File.dirname(__FILE__)}/../data/shadow-ministers.csv") + read_ministers(people, ministers_filename) + read_ministers(people, shadow_ministers_filename) + end + private def PeopleCSVReader.parse_party(party) lib/people_csv_reader.rb @@ -7,6 +7,6 @@ require 'people' conf = Configuration.new -people = People.read_members_csv("data/people.csv", "data/members.csv") +people = PeopleCSVReader.read_members puts "Downloading person images..." people.download_images("#{conf.file_image_path}/mps", "#{conf.file_image_path}/mpsL") member-images.rb @@ -13,7 +13,7 @@ agent = MechanizeProxy.new agent.cache_subdirectory = "parse-member-links" puts "Reading member data..." -people = People.read_members_csv("data/people.csv", "data/members.csv") +people = PeopleCSVReader.read_members puts "Personal home page & Contact Details (Gov website)..." parse-member-links.rb @@ -20,9 +20,8 @@ system("mkdir -p #{conf.members_xml_path}") end puts "Reading members data..." -people = People.read_members_csv("data/people.csv", "data/members.csv") -people.read_ministers_csv("data/ministers.csv") -people.read_ministers_csv("data/shadow-ministers.csv") +people = PeopleCSVReader.read_members +PeopleCSVReader.read_all_ministers(people) puts "Running consistency checks..." # First check that each constituency is showing a continuous period of members with there never being more than one member at any time. # Collect all the division names parse-members.rb @@ -72,7 +72,7 @@ system("mkdir -p #{conf.xml_path}/scrapedxml/debates #{conf.xml_path}/scrapedxml system("cp #{File.dirname(__FILE__)}/data/empty-template.xml #{conf.xml_path}/scrapedxml/regmem/regmem2000-01-01.xml") # First load people back in so that we can look up member id's -people = People.read_members_csv("#{File.dirname(__FILE__)}/data/people.csv", "#{File.dirname(__FILE__)}/data/members.csv") +people = PeopleCSVReader.read_members parser = HansardParser.new(people) parse-speeches.rb @@ -21,7 +21,7 @@ data.shift data.shift puts "Reading members data..." -people = People.read_members_csv("data/people.csv", "data/members.csv") +people = PeopleCSVReader.read_members all_members = people.all_periods_in_house(House.representatives) # First check that all the constituencies are valid postcodes.rb @@ -29,9 +29,8 @@ system("mkdir -p #{conf.members_xml_path}") puts "Reading CSV data..." data_path = "#{File.dirname(__FILE__)}/../data" -people = People.read_members_csv("#{data_path}/people.csv", "#{data_path}/members.csv") -people.read_ministers_csv("#{data_path}/ministers.csv") -people.read_ministers_csv("#{data_path}/shadow-ministers.csv") +people = PeopleCSVReader.read_members +PeopleCSVReader.read_all_ministers(people) puts "Writing XML..." people.write_xml("#{conf.members_xml_path}/people.xml", "#{conf.members_xml_path}/all-members.xml", "#{conf.members_xml_path}/peers-ucl.xml", "#{conf.members_xml_path}/ministers.xml") tests-regression/test_parse_members.rb @@ -31,7 +31,7 @@ skip_dates = [] conf = Configuration.new # First load people back in so that we can look up member id's -people = People.read_members_csv("#{File.dirname(__FILE__)}/../data/people.csv", "#{File.dirname(__FILE__)}/../data/members.csv") +people = PeopleCSVReader.read_members parser = HansardParser.new(people) tests-regression/test_parse_speeches.rb @@ -5,8 +5,8 @@ require 'people' class TestPeople < Test::Unit::TestCase def test_speaker - people = People.read_members_csv("#{File.dirname(__FILE__)}/../data/people.csv", "#{File.dirname(__FILE__)}/../data/members.csv") - people.read_ministers_csv("#{File.dirname(__FILE__)}/../data/ministers.csv") + people = PeopleCSVReader.read_members + PeopleCSVReader.read_all_ministers(people) member = people.house_speaker(Date.new(2007, 10, 1)) assert_equal("David Peter Maxwell Hawker", member.person.name.full_name) assert(member.house_speaker?) tests/test_people.rb @@ -13,9 +13,8 @@ class TestPeopleCSVReader < Test::Unit::TestCase ref.add_minister_position(:count => 790, :from_date => Date.new(2007,12,6), :to_date => DateWithFuture.future, :position => "Shadow Parliamentary Secretary for Local Government") - people = People.read_members_csv("#{File.dirname(__FILE__)}/../data/people.csv", "#{File.dirname(__FILE__)}/../data/members.csv") - people.read_ministers_csv("#{File.dirname(__FILE__)}/../data/ministers.csv") - people.read_ministers_csv("#{File.dirname(__FILE__)}/../data/shadow-ministers.csv") + people = PeopleCSVReader.read_members + PeopleCSVReader.read_all_ministers(people) sophie_mirabella = people.find_person_by_name(Name.new(:first => "Sophie", :last => "Mirabella")) assert_equal(ref, sophie_mirabella) tests/test_people_csv_reader.rb c58e1997047caa50d22c64396e2e0c963aba2f9d Matthew Landauer matthew@openaustralia.org http://github.com/mlandauer/openaustralia-parser/commit/bdc6a5685b3a1f74857ca8b42e0bf4031170f83b bdc6a5685b3a1f74857ca8b42e0bf4031170f83b 2008-07-15T20:51:02-07:00 2008-07-15T20:51:02-07:00 Got rid of duplication of paths to all the csv files a274a8526d2299182e6d8d194b03d71616a52e93 Matthew Landauer matthew@openaustralia.org