Merge pull request #1 from kaydensigh/master

Save council_website.
openaustralia · Jun 17, 2015 · f28b18c · f28b18c
2 parents 3e5aee3 + d843646
commit f28b18c
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 1 deletion.
diff --git a/.gitignore b/.gitignore
@@ -1,2 +1,5 @@
 # Ignore output of scraper
 data.sqlite
+
+Gemfile
+Gemfile.lock
diff --git a/scraper.rb b/scraper.rb
@@ -14,6 +14,8 @@ def scrape_council(url)
   agent = Mechanize.new
   page = agent.get(url)
   council = page.at("h1").inner_text
+  website_h = page.search("h2").find{|h| h.inner_text == "Website"}
+  website = website_h.next_element.inner_text
   h = page.search("h2").find{|h| h.inner_text == "Councillors"}
   block = h.next_element.inner_html.split("<br>")
   block[1..-1].each do |line|
@@ -35,7 +37,8 @@ def scrape_council(url)
       "council" => council,
       "ward" => ward,
       "councillor" => name,
-      "position" => position
+      "position" => position,
+      "council_website" => website
     }
     p record
     ScraperWiki.save_sqlite(["council", "councillor"], record)