@@ -10,15 +10,6 @@ conf = Configuration.new FileUtils.mkdir_p conf.members_xml_path -# Copy across files that are needed for the script xml2db to run but are not yet populated with data -["bbc-links.xml", "constituencies.xml", "diocese-bishops.xml", "edm-links.xml", "expenses200102.xml", - "expenses200203.xml", "expenses200304.xml", "expenses200405.xml", "expenses200506.xml", "expenses200506former.xml", - "expenses200607.xml", "guardian-links.xml", "journa-list.xml", "lordbiogs.xml", "ni-members.xml", - "royals.xml", "sp-members.xml", "wikipedia-commons.xml", "wikipedia-lords.xml", "wikipedia-mla.xml", - "wikipedia-msp.xml"].each do |file| - FileUtils.cp File.join("data", "empty-template.xml"), File.join(conf.members_xml_path, file) -end - puts "Reading members data..." people = PeopleCSVReader.read_members PeopleCSVReader.read_all_ministers(people) @@ -66,8 +57,8 @@ people.each do |person| end puts "Writing XML..." -people.write_xml("#{conf.members_xml_path}/people.xml", "#{conf.members_xml_path}/all-members.xml", "#{conf.members_xml_path}/peers-ucl.xml", - "#{conf.members_xml_path}/ministers.xml", "#{conf.members_xml_path}/constituencies.xml") +people.write_xml("#{conf.members_xml_path}/people.xml", "#{conf.members_xml_path}/representatives.xml", "#{conf.members_xml_path}/senators.xml", + "#{conf.members_xml_path}/ministers.xml", "#{conf.members_xml_path}/divisions.xml") # And load up the database # Starts with 'perl' to be friendly with Windows parse-members.rb @@ -65,12 +65,8 @@ end conf = Configuration.new -FileUtils.mkdir_p "#{conf.xml_path}/scrapedxml/debates" -FileUtils.mkdir_p "#{conf.xml_path}/scrapedxml/lordspages" -FileUtils.mkdir_p "#{conf.xml_path}/scrapedxml/regmem" - -# Copy across file that is needed for the script xml2db to run but is not yet populated with data -system("cp #{File.dirname(__FILE__)}/data/empty-template.xml #{conf.xml_path}/scrapedxml/regmem/regmem2000-01-01.xml") +FileUtils.mkdir_p "#{conf.xml_path}/scrapedxml/representatives_debates" +FileUtils.mkdir_p "#{conf.xml_path}/scrapedxml/senate_debates" # First load people back in so that we can look up member id's people = PeopleCSVReader.read_members @@ -84,17 +80,17 @@ date = to_date while date >= from_date if conf.write_xml_representatives if options[:proof] - parser.parse_date_house_only_in_proof(date, "#{conf.xml_path}/scrapedxml/debates/debates#{date}.xml", House.representatives) + parser.parse_date_house_only_in_proof(date, "#{conf.xml_path}/scrapedxml/representatives_debates/#{date}.xml", House.representatives) else - parser.parse_date_house(date, "#{conf.xml_path}/scrapedxml/debates/debates#{date}.xml", House.representatives) + parser.parse_date_house(date, "#{conf.xml_path}/scrapedxml/representatives_debates/#{date}.xml", House.representatives) end end progress.inc if conf.write_xml_senators if options[:proof] - parser.parse_date_house_only_in_proof(date, "#{conf.xml_path}/scrapedxml/lordspages/daylord#{date}.xml", House.senate) + parser.parse_date_house_only_in_proof(date, "#{conf.xml_path}/scrapedxml/senate_debates/#{date}.xml", House.senate) else - parser.parse_date_house(date, "#{conf.xml_path}/scrapedxml/lordspages/daylord#{date}.xml", House.senate) + parser.parse_date_house(date, "#{conf.xml_path}/scrapedxml/senate_debates/#{date}.xml", House.senate) end end progress.inc parse-speeches.rb data/empty-template.xml c8418ecc7f04655d381e19ffaee3ac4f5950e7e3 Matthew Landauer matthew@openaustralia.org http://github.com/mlandauer/openaustralia-parser/commit/2fd9727bb0d73e99f5c5d21ebaa1a8351c0b16d5 2fd9727bb0d73e99f5c5d21ebaa1a8351c0b16d5 2009-05-26T18:52:01-07:00 2009-05-25T21:08:02-07:00 Don't generate empty XML files. Rename created XML into more sensible naming convention for Australia. 95b9c4bc8ab223623a478e1ca9d117381297cd67 Matthew Landauer matthew@openaustralia.org