@@ -35,7 +35,3 @@ qanda_all_senators_url: "http://www.abc.net.au/tv/qanda/find-a-senate-member-by- # ABC 2007 election results election_web_root: "http://www.abc.net.au/elections/federal/2007" - -# Selectively allows us to turn off and on the writing of the XML -write_xml_representatives: true -write_xml_senators: true configuration.yml @@ -33,26 +33,11 @@ class PeopleXMLWriter x.ministers do people.each do |person| person.minister_positions.each do |p| - was_representative = !person.house_periods.empty? - was_senator = !person.senate_periods.empty? - # TODO: Add "dept" and "source" - if (conf.write_xml_representatives && was_representative) || - (conf.write_xml_senators && was_senator) - if conf.write_xml_representatives && !conf.write_xml_senators - matchid = person.house_periods.first.id - elsif conf.write_xml_senators && !conf.write_xml_representatives - matchid = person.senate_periods.first.id - elsif conf.write_xml_representatives && conf.write_xml_senators - matchid = person.periods.first.id - else - throw "Hmmm... what's the point of that?" - end - x.ministerofficegroup do - x.moffice(:id => p.id, :name => person.name.full_name, - :matchid => matchid, :position => p.position, - :fromdate => p.from_date, :todate => p.to_date, :dept => "", :source => "") - end + x.ministerofficegroup do + x.moffice(:id => p.id, :name => person.name.full_name, + :matchid => person.periods.first.id, :position => p.position, + :fromdate => p.from_date, :todate => p.to_date, :dept => "", :source => "") end end end @@ -67,14 +52,12 @@ class PeopleXMLWriter x = Builder::XmlMarkup.new(:target => xml, :indent => 1) x.instruct! x.members do - if conf.write_xml_representatives - people.each do |person| - person.house_periods.each do |period| - x.member(:id => period.id, - :house => "representatives", :title => period.person.name.title, :firstname => period.person.name.first, - :lastname => period.person.name.last, :division => period.division, :party => period.party, - :fromdate => period.from_date, :todate => period.to_date, :fromwhy => period.from_why, :towhy => period.to_why) - end + people.each do |person| + person.house_periods.each do |period| + x.member(:id => period.id, + :house => "representatives", :title => period.person.name.title, :firstname => period.person.name.first, + :lastname => period.person.name.last, :division => period.division, :party => period.party, + :fromdate => period.from_date, :todate => period.to_date, :fromwhy => period.from_why, :towhy => period.to_why) end end end @@ -88,14 +71,12 @@ class PeopleXMLWriter x = Builder::XmlMarkup.new(:target => xml, :indent => 1) x.instruct! x.members do - if conf.write_xml_senators - people.each do |person| - person.senate_periods.each do |period| - x.member(:id => period.id, - :house => "senate", :title => period.person.name.title, :firstname => period.person.name.first, - :lastname => period.person.name.last, :division => period.state, :party => period.party, - :fromdate => period.from_date, :todate => period.to_date, :fromwhy => period.from_why, :towhy => period.to_why) - end + people.each do |person| + person.senate_periods.each do |period| + x.member(:id => period.id, + :house => "senate", :title => period.person.name.title, :firstname => period.person.name.first, + :lastname => period.person.name.last, :division => period.state, :party => period.party, + :fromdate => period.from_date, :todate => period.to_date, :fromwhy => period.from_why, :towhy => period.to_why) end end end @@ -111,22 +92,18 @@ class PeopleXMLWriter x.people do people.each do |person| x.person(:id => person.id, :latestname => person.name.informal_name) do - if conf.write_xml_representatives - person.house_periods.each do |period| - if period.current? - x.office(:id => period.id, :current => "yes") - else - x.office(:id => period.id) - end + person.house_periods.each do |period| + if period.current? + x.office(:id => period.id, :current => "yes") + else + x.office(:id => period.id) end end - if conf.write_xml_senators - person.senate_periods.each do |period| - if period.current? - x.office(:id => period.id, :current => "yes") - else - x.office(:id => period.id) - end + person.senate_periods.each do |period| + if period.current? + x.office(:id => period.id, :current => "yes") + else + x.office(:id => period.id) end end person.minister_positions.each do |p| lib/people_xml_writer.rb @@ -66,20 +66,16 @@ xml = File.open("#{conf.members_xml_path}/websites.xml", 'w') x = Builder::XmlMarkup.new(:target => xml, :indent => 1) x.instruct! x.peopleinfo do - if conf.write_xml_representatives - agent.get(conf.alternative_current_house_members_url).links.each do |link| - if link.to_s =~ /Member for/ - name = Name.last_title_first(link.text.split(',')[0..1].join(',')) - extract_links(name, people, agent, link, x) - end + agent.get(conf.alternative_current_house_members_url).links.each do |link| + if link.to_s =~ /Member for/ + name = Name.last_title_first(link.text.split(',')[0..1].join(',')) + extract_links(name, people, agent, link, x) end end - if conf.write_xml_senators - agent.get(conf.alternative_current_senate_members_url).links.each do |link| - if link.to_s =~ /Senator/ - name = Name.last_title_first(link.to_s.split('-')[0..-2].join('-')) - extract_links(name, people, agent, link, x) - end + agent.get(conf.alternative_current_senate_members_url).links.each do |link| + if link.to_s =~ /Senator/ + name = Name.last_title_first(link.to_s.split('-')[0..-2].join('-')) + extract_links(name, people, agent, link, x) end end end @@ -120,55 +116,50 @@ puts "Q&A Links..." data = {} -if conf.write_xml_representatives +# First get mapping between constituency name and web page +page = agent.get(conf.qanda_electorate_url) +map = {} - # First get mapping between constituency name and web page - page = agent.get(conf.qanda_electorate_url) - map = {} - - page.links[35..184].each do |link| - map[link.text.downcase] = (page.uri + link.uri).to_s - end - # Hack to deal with "Flynn" constituency incorrectly spelled as "Flyn" - map["flynn"] = "http://www.abc.net.au/tv/qanda/mp-profiles/flyn.htm" - - bad_divisions = [] - # Check that the links point to valid pages - map.each_pair do |division, url| - begin - agent.get(url) - rescue WWW::Mechanize::ResponseCodeError - bad_divisions << division - puts "ERROR: Invalid url #{url} for division #{division}" - end - end - # Clear out bad divisions - bad_divisions.each { |division| map.delete(division) } - - people.find_current_members(House.representatives).each do |member| - short_division = member.division.downcase[0..3] - link = map[member.division.downcase] - data[member.person.id] = link - puts "ERROR: Couldn't lookup division #{member.division}" if link.nil? +page.links[35..184].each do |link| + map[link.text.downcase] = (page.uri + link.uri).to_s +end +# Hack to deal with "Flynn" constituency incorrectly spelled as "Flyn" +map["flynn"] = "http://www.abc.net.au/tv/qanda/mp-profiles/flyn.htm" + +bad_divisions = [] +# Check that the links point to valid pages +map.each_pair do |division, url| + begin + agent.get(url) + rescue WWW::Mechanize::ResponseCodeError + bad_divisions << division + puts "ERROR: Invalid url #{url} for division #{division}" end end +# Clear out bad divisions +bad_divisions.each { |division| map.delete(division) } + +people.find_current_members(House.representatives).each do |member| + short_division = member.division.downcase[0..3] + link = map[member.division.downcase] + data[member.person.id] = link + puts "ERROR: Couldn't lookup division #{member.division}" if link.nil? +end -if conf.write_xml_senators - page = agent.get(conf.qanda_all_senators_url) - page.links.each do |link| - if link.uri.to_s =~ /^\/tv\/qanda\/senators\// - # HACK to handle Unicode in Kerry O'Brien's name on Q&A site - if link.to_s == "Kerry O\222Brien" - name_text = "Kerry O'Brien" - else - name_text = link.to_s - end - member = people.find_member_by_name_current_on_date(Name.title_first_last(name_text), Date.today, House.senate) - if member.nil? - puts "WARNING: Can't find Senator #{link}" - else - data[member.person.id] = page.uri + link.uri - end +page = agent.get(conf.qanda_all_senators_url) +page.links.each do |link| + if link.uri.to_s =~ /^\/tv\/qanda\/senators\// + # HACK to handle Unicode in Kerry O'Brien's name on Q&A site + if link.to_s == "Kerry O\222Brien" + name_text = "Kerry O'Brien" + else + name_text = link.to_s + end + member = people.find_member_by_name_current_on_date(Name.title_first_last(name_text), Date.today, House.senate) + if member.nil? + puts "WARNING: Can't find Senator #{link}" + else + data[member.person.id] = page.uri + link.uri end end end parse-member-links.rb @@ -78,20 +78,16 @@ progress = ProgressBar.new("parse-speeches", ((to_date - from_date + 1) * 2).to_ # Kind of helpful to start at the end date and go backwards when using the "--proof" option. So, always going to do this now. date = to_date while date >= from_date - if conf.write_xml_representatives - if options[:proof] - parser.parse_date_house_only_in_proof(date, "#{conf.xml_path}/scrapedxml/representatives_debates/#{date}.xml", House.representatives) - else - parser.parse_date_house(date, "#{conf.xml_path}/scrapedxml/representatives_debates/#{date}.xml", House.representatives) - end + if options[:proof] + parser.parse_date_house_only_in_proof(date, "#{conf.xml_path}/scrapedxml/representatives_debates/#{date}.xml", House.representatives) + else + parser.parse_date_house(date, "#{conf.xml_path}/scrapedxml/representatives_debates/#{date}.xml", House.representatives) end progress.inc - if conf.write_xml_senators - if options[:proof] - parser.parse_date_house_only_in_proof(date, "#{conf.xml_path}/scrapedxml/senate_debates/#{date}.xml", House.senate) - else - parser.parse_date_house(date, "#{conf.xml_path}/scrapedxml/senate_debates/#{date}.xml", House.senate) - end + if options[:proof] + parser.parse_date_house_only_in_proof(date, "#{conf.xml_path}/scrapedxml/senate_debates/#{date}.xml", House.senate) + else + parser.parse_date_house(date, "#{conf.xml_path}/scrapedxml/senate_debates/#{date}.xml", House.senate) end progress.inc date = date - 1 @@ -102,8 +98,8 @@ progress.finish # And load up the database if options[:load_database] command_options = " --from=#{from_date} --to=#{to_date}" - command_options << " --debates" if conf.write_xml_representatives - command_options << " --lordsdebates" if conf.write_xml_senators + command_options << " --debates" + command_options << " --lordsdebates" command_options << " --force" if options[:force] # Starts with 'perl' to be friendly with Windows parse-speeches.rb @@ -30,16 +30,12 @@ agent = MechanizeProxy.new agent.user_agent_alias = 'Mac Safari' agent.cache_subdirectory = "wikipedia" -if conf.write_xml_representatives - puts "Wikipedia links for Representatives..." - links = extract_all_representative_wikipedia_links(people, agent) - write_links(links, "#{conf.members_xml_path}/wikipedia-commons.xml") - # For Representatives just for curiousity sake find out which has a link back to OpenAustralia - links.each {|link| check_wikipedia_page(link[1], agent) } -end -if conf.write_xml_senators - puts "Wikipedia links for Senators..." - write_links(extract_all_senator_wikipedia_links(people, agent), "#{conf.members_xml_path}/wikipedia-lords.xml") -end +puts "Wikipedia links for Representatives..." +links = extract_all_representative_wikipedia_links(people, agent) +write_links(links, "#{conf.members_xml_path}/wikipedia-commons.xml") +# For Representatives just for curiousity sake find out which has a link back to OpenAustralia +links.each {|link| check_wikipedia_page(link[1], agent) } +puts "Wikipedia links for Senators..." +write_links(extract_all_senator_wikipedia_links(people, agent), "#{conf.members_xml_path}/wikipedia-lords.xml") system(conf.web_root + "/twfy/scripts/mpinfoin.pl links") wikipedia.rb 69841c4733500945aa38168f1614ae0a62233c33 Matthew Landauer matthew@openaustralia.org http://github.com/mlandauer/openaustralia-parser/commit/4b6ee4e4e2a66440b62d870ca998023011cbc195 4b6ee4e4e2a66440b62d870ca998023011cbc195 2009-11-01T04:45:21-08:00 2009-11-01T04:45:21-08:00 Always write out both the representatives and the senators xml 431df9d1f536c8f541a396538e850232330e5b0b Matthew Landauer matthew@openaustralia.org