features/fixtures/hansard_reps_2009_06_04.html features/fixtures/hansard_reps_2009_06_04.xml features/hansard_parser.feature features/step_definitions/hansard_parser_steps.rb @@ -1,6 +1,7 @@ require 'activesupport' require 'hpricot' require 'fake_web' +require 'ostruct' module OA class Namespace @@ -11,6 +12,30 @@ module OA def before!; end + + def self.build(ivar,&block) + define_method("make",&block) + + define_method("make!") do |*args| + instance_variable_set("@#{ivar}", make(*args)) + self + end + + define_method('made') do + instance_variable_get("@#{ivar}") || raise("Please create the #{ivar} with #make!(...) first") + end + end + + def make!(*args) + @made = make(*args) + self + end + + def made + @made || raise("Please create the #{self.class.to_s.underscore} with #make!(...) first") + end + alias :o :made + def method_missing(method_id,*args,&block) @app.send(method_id,*args,&block) end @@ -93,14 +118,19 @@ module OA prepare_for_string_get!(url,text) end + def prepare_for_type_get!(url,fixture_name,content_type) + text = fixture.open(fixture_name).read + prepare_for_string_get!(url,text,content_type) + end + def prepare_for_file_get!(url,fixture_name) ::FakeWeb.register_uri(:get, url, :file => fixture.path(fixture_name)) end - def prepare_for_string_get!(url,text) + def prepare_for_string_get!(url,text,content_type='text/html') response = <<-EOR HTTP/1.1 200 OK -Content-Type: text/html +Content-Type: #{content_type} Content-Length: #{text.size} #{text} @@ -129,14 +159,44 @@ Content-Length: #{text.size} web.prepare_for_string_get!(person_url(name), page_content("Hooray #{name}")) end + + # hansard + def prepare_hansard_page! + web.prepare_for_html_get!(hansard_search_url, hansard_fixture('html')) + end + + def prepare_hansard_xml!(id) + web.prepare_for_type_get!(hansard_xml_url(id), hansard_fixture('xml'), 'text/xml') + end + + def parlinfo_url + "http://parlinfo.aph.gov.au/parlInfo/search/display/display.w3p" + end def person_url(name) - "http://parlinfo.aph.gov.au/parlInfo/search/display/display.w3p;query=Dataset:allmps%20" + name.gsub(' ', '%20') + "#{parlinfo_url};query=Dataset:allmps%20" + name.gsub(' ', '%20') end def person_image_url(name) name = name.underscore.gsub(/\s+/,'_') "http://parlinfo.aph.gov.au/parlInfo/download/handbook/allmps/0J4/upload_ref_binary/#{name}.jpg" end + + def hansard_letter + house.is_reps?(hansard.o.house) ? "r" : "s" + end + + def hansard_search_url + "#{parlinfo_url};query=Id:chamber/hansard#{hansard_letter}/#{hansard.o.date}/0000" + end + + def hansard_xml_url(id) + "http://parlinfo.aph.gov.au:80/parlInfo/download/chamber/hansard#{hansard_letter}/#{hansard.o.date}/toc_unixml/#{id}.xml" + end + + def hansard_fixture(ext) + base_name = "hansard_#{hansard.o.house}_#{hansard.o.date.underscore}" + "#{base_name}.#{ext}" + end def person_html_file(name) @@ -148,6 +208,54 @@ Content-Length: #{text.size} end end + require 'house' + class House < Namespace + def interpret(house) + if is_reps?(house) + 'reps' + elsif house[/senate/i] + 'senate' + else + raise "can't interpret house '#{house}'" + end + end + + def make(named) + case interpret(named) + when 'reps' + ::House.representatives + when 'senate' + ::House.senate + end + end + + def is_reps?(house) + !!house[/rep/i] + end + end + + class Hansard < Namespace + def make(house_str,date) + OpenStruct.new(:house => house.interpret(house_str), :date => date) + end + end + + require 'hansard_parser' + class HansardParser < Namespace + def make(people) + ::HansardParser.new(people) + end + + def download! + @body = o.unpatched_hansard_xml_source_data_on_date(hansard.o.date, house.make(hansard.o.house)) + self + end + + def body? + @body.should_not be_blank + end + end + require 'person' class Person < Namespace def make(name) @@ -299,5 +407,8 @@ Content-Length: #{text.size} namespace(:people_downloader,:PeopleDownloader) namespace(:person,:Person) namespace(:people,:People) + namespace(:house,:House) + namespace(:hansard,:Hansard) + namespace(:hansard_parser,:HansardParser) end end features/support/oa_app.rb @@ -65,6 +65,7 @@ end conf = Configuration.new +puts conf.xml_path FileUtils.mkdir_p "#{conf.xml_path}/scrapedxml/representatives_debates" FileUtils.mkdir_p "#{conf.xml_path}/scrapedxml/senate_debates" @@ -85,6 +86,7 @@ while date >= from_date parser.parse_date_house(date, "#{conf.xml_path}/scrapedxml/representatives_debates/#{date}.xml", House.representatives) end end + progress.inc if conf.write_xml_senators if options[:proof] parse-speeches.rb 8d9ac023845952bb78630b9ce9fea10dab2da52f lachie lachiec@gmail.com http://github.com/lachie/openaustralia-parser/commit/986fa119363f5ce62d17452e4176c83438becf7e 986fa119363f5ce62d17452e4176c83438becf7e 2009-06-14T22:21:26-07:00 2009-06-14T22:21:26-07:00 feature: hansard parser, scenario: downloading the hansard 39b75c594d46ce08c0be21fe6a98f8826aa48da8 lachie lachiec@gmail.com