Track posts to Slack in new table. Fixes #15.

lib/runners.rb → lib/jacaranda.rb

@@ -13,11 +13,31 @@
 # Wrapper for all runners
 module Jacaranda
   def self.run(args)
+    upgrade_schema!
     parse(args)
     announce
     runners.each(&:run)
   end
 
+  def self.upgrade_schema!
+    return if been_migrated?
+    data = ScraperWiki.select('* from data')
+    upgraded = data.each { |r| r['runner'] = 'RightToKnow::Runner' unless r['runner'] }
+    primary_keys = %w[date_posted runner]
+    table_name = 'posts'
+    ScraperWiki.save_sqlite(primary_keys, upgraded, table_name)
+  end
+
+  def self.been_migrated?
+    query = %(SELECT sql FROM sqlite_master where name = 'data' AND type = 'table')
+    return true if ScraperWiki.sqliteexecute(query).empty?
+
+    query = %(SELECT sql FROM sqlite_master where name = 'posts' AND type = 'table')
+    return true if ScraperWiki.sqliteexecute(query).any?
+
+    false
+  end
+
   # rubocop:disable Metrics/MethodLength
   def self.parse(args = [])
     @whitelist = []

lib/runners/base_runner.rb

@@ -34,7 +34,7 @@ def validate_environment_variables!
       end
 
       def posts
-        posts = ScraperWiki.select("* from data where runner = '#{self}'")
+        posts = ScraperWiki.select("* from posts where runner = '#{self}'")
         normalise_dates(posts)
       rescue
         []
@@ -110,7 +110,7 @@ def record_successful_post(message)
           text: message,
           runner: to_s
         }
-        ScraperWiki.save_sqlite(%i[date_posted runner], record)
+        ScraperWiki.save_sqlite(%i[date_posted runner], record, 'posts')
       end
 
       def print(message)

scraper.rb

@@ -2,6 +2,6 @@
 
 require 'dotenv'
 Dotenv.load
-require_relative 'lib/runners'
+require_relative 'lib/jacaranda'
 
 Jacaranda.run(ARGV) if $PROGRAM_NAME == __FILE__

spec/jacaranda_spec.rb

@@ -11,6 +11,60 @@ def all_request_bodies
 end
 
 describe 'Jacaranda' do
+  describe '.upgrade_schema!' do
+    let(:insert_data_statements) do
+      [
+        %(INSERT INTO data (date_posted,text,runner) VALUES ('2016-08-29','hello',null)),
+        %(INSERT INTO data (date_posted,text,runner) VALUES ('2016-09-12','world',null)),
+        %(INSERT INTO data (date_posted,text,runner) VALUES ('2017-09-04','foo','RightToKnow::Runner'))
+      ]
+    end
+
+    before(:each) do
+      create_table_statements.each { |statement| ScraperWiki.sqliteexecute(statement) }
+      insert_data_statements.each { |statement| ScraperWiki.sqliteexecute(statement) }
+    end
+
+    context 'when the schema has not been upgraded' do
+      let(:create_table_statements) do
+        [
+          %[CREATE TABLE data (date_posted,text,runner,UNIQUE (date_posted))]
+        ]
+      end
+
+      it 'applies the upgrade' do
+        expect(Jacaranda.been_migrated?).to be false
+        Jacaranda.upgrade_schema!
+        expect(Jacaranda.been_migrated?).to be true
+
+        query = %(SELECT sql FROM sqlite_master WHERE name = 'posts' AND type = 'table')
+        expect(ScraperWiki.sqliteexecute(query).any?).to be true
+
+        query = %(SELECT count(*) AS count FROM posts)
+        expect(ScraperWiki.sqliteexecute(query).first['count']).to eq(insert_data_statements.size)
+
+        query = %(SELECT * FROM posts WHERE runner IS NULL)
+        expect(ScraperWiki.sqliteexecute(query).empty?).to be true
+      end
+    end
+
+    context 'when the schema has been upgraded' do
+      let(:create_table_statements) do
+        [
+          %[CREATE TABLE data (date_posted,text,runner,UNIQUE (date_posted))],
+          %[CREATE TABLE posts (date_posted,text,runner,UNIQUE (date_posted,runner))]
+        ]
+      end
+
+      it 'skips the upgrade' do
+        Jacaranda.upgrade_schema!
+
+        query = %[SELECT count(*) AS count FROM posts]
+        expect(ScraperWiki.sqliteexecute(query).first['count']).to eq(0)
+      end
+    end
+  end
+
   describe '.parse' do
     context 'when filtering' do
       it 'sorts runners alphabetically' do

spec/runners/base_runner_spec.rb

@@ -77,7 +77,7 @@
         end
       end
 
-      after(:each) { ScraperWiki.sqliteexecute('DELETE FROM data') }
+      after(:each) { ScraperWiki.sqliteexecute('DELETE FROM posts') }
     end
 
     describe '.run' do
@@ -109,7 +109,7 @@
 
       after(:each) do
         restore_env
-        ScraperWiki.sqliteexecute('DELETE FROM data')
+        ScraperWiki.sqliteexecute('DELETE FROM posts')
       end
     end