Merge pull request #1 from everypolitician-scrapers/add-scraped-page-…

…archiver

Add scraped page archiver

Gemfile

@@ -2,16 +2,18 @@
 # specified here will be installed and made available to your morph.io scraper.
 # Find out more: https://morph.io/documentation/ruby
 
-source "https://rubygems.org"
+source 'https://rubygems.org'
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}.git" }
 
-ruby "2.0.0"
+ruby '2.0.0'
 
-gem "scraperwiki", git: "https://github.com/openaustralia/scraperwiki-ruby.git", branch: "morph_defaults"
-gem "execjs"
-gem "pry"
-gem "colorize"
-gem "nokogiri"
-gem "open-uri-cached"
-gem "fuzzy_match"
+gem 'scraperwiki', github: 'openaustralia/scraperwiki-ruby',
+                   branch: 'morph_defaults'
+gem 'execjs'
+gem 'pry'
+gem 'colorize'
+gem 'nokogiri'
+gem 'open-uri-cached'
+gem 'fuzzy_match'
 gem 'yajl-ruby', require: 'yajl'
-
+gem 'scraped_page_archive'

Gemfile.lock

@@ -10,10 +10,16 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
+    addressable (2.5.0)
+      public_suffix (~> 2.0, >= 2.0.2)
     coderay (1.1.0)
     colorize (0.7.7)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
     execjs (2.5.2)
     fuzzy_match (2.1.0)
+    git (1.3.0)
+    hashdiff (0.3.0)
     httpclient (2.6.0.1)
     method_source (0.8.2)
     mini_portile (0.6.2)
@@ -24,10 +30,23 @@ GEM
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
+    public_suffix (2.0.4)
+    safe_yaml (1.0.4)
+    scraped_page_archive (0.5.0)
+      git (~> 1.3.0)
+      vcr-archive (~> 0.3.0)
     slop (3.6.0)
     sqlite3 (1.3.10)
     sqlite_magic (0.0.3)
       sqlite3
+    vcr (3.0.3)
+    vcr-archive (0.3.0)
+      vcr (~> 3.0.2)
+      webmock (~> 2.0.3)
+    webmock (2.0.3)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
     yajl-ruby (1.2.1)
 
 PLATFORMS
@@ -40,5 +59,12 @@ DEPENDENCIES
   nokogiri
   open-uri-cached
   pry
+  scraped_page_archive
   scraperwiki!
   yajl-ruby
+
+RUBY VERSION
+   ruby 2.0.0p648
+
+BUNDLED WITH
+   1.13.5

scraper.rb

@@ -5,8 +5,9 @@
 require 'nokogiri'
 require 'colorize'
 require 'pry'
-require 'open-uri/cached'
-OpenURI::Cache.cache_path = '.cache'
+# require 'open-uri/cached'
+# OpenURI::Cache.cache_path = '.cache'
+require 'scraped_page_archive/open-uri'
 
 @ConstituencyRegion = {}