Merge fd1562c into 1e7f99b

lib/ayashige.rb

@@ -16,7 +16,6 @@
 require "ayashige/jobs/cron_job"
 
 require "rollbar"
-require "dotenv/load"
 
 Rollbar.configure do |config|
   config.access_token = ENV["ROLLBAR_ACCESS_TOKEN"]

lib/ayashige/jobs/cron_job.rb

@@ -4,10 +4,6 @@ module Ayashige
   module Jobs
     class CronJob
       def perform
-        store_newly_registered_domains
-      end
-
-      def store_newly_registered_domains
         Sources::WebAnalyzer.new.store_newly_registered_domains
       end
     end

lib/ayashige/sources/web_analyzer.rb

@@ -18,22 +18,18 @@ def initialize
         @store = Store.new
       end
 
-      def tlds
-        TLDS
-      end
-
       def already_stored?(date)
         @store.exists? date
       end
 
       def store_newly_registered_domains
         date = latest_indexed_date
         if already_stored?(date)
-          puts "#{date} domains are already stored."
+          puts "domains which updated on #{date} are already stored."
           return
         end
 
-        Parallel.map(tlds) do |tld|
+        Parallel.each(TLDS) do |tld|
           index = 1
           while index < LIMIT
             page = get_page(date, tld, index)

spec/sources/web_analyzer_spec.rb

@@ -5,6 +5,30 @@
 RSpec.describe Ayashige::Sources::WebAnalyzer, :vcr do
   subject { Ayashige::Sources::WebAnalyzer.new }
 
+  let(:redis) { MockRedis.new }
+
+  before do
+    allow(Ayashige::Redis).to receive(:client).and_return(redis)
+  end
+
+  after do
+    redis.flushdb
+  end
+
+  describe "#already_stored?" do
+    context "when already stored" do
+      before { redis.set "2018-01-01", "set" }
+      it "should return true" do
+        expect(subject.already_stored?("2018-01-01")).to eq(true)
+      end
+    end
+    context "when not already stored" do
+      it "should return false" do
+        expect(subject.already_stored?("2018-01-01")).to eq(false)
+      end
+    end
+  end
+
   describe "#latest_indexed_date" do
     it "should return the latest indexed date as a String" do
       expect(subject.latest_indexed_date).to be_a(String)
@@ -36,4 +60,28 @@
       end
     end
   end
+
+  describe "#store_newly_registered_domains" do
+    let(:updated_on) { "2018-01-01" }
+
+    before do
+      stub_const("Ayashige::Sources::WebAnalyzer::LIMIT", 2)
+      stub_const("Ayashige::Sources::WebAnalyzer::TLDS", ["world"])
+
+      allow(subject).to receive(:latest_indexed_date).and_return("2018-11-23")
+      allow(subject).to receive(:get_page).and_return(nil)
+      allow(subject).to receive(:get_domains_from_doc).and_return([
+        { updated: updated_on, domain: "paypal.pay.pay.world" }
+      ])
+
+      allow(Parallel).to receive(:each).with(["world"]).and_yield("world")
+    end
+
+    it "should store parsed domains into Redis" do
+      output = capture(:stdout) { subject.store_newly_registered_domains }
+      expect(output.include?("paypal.pay.pay.world")).to eq(true)
+
+      expect(redis.exists(updated_on)).to eq(true)
+    end
+  end
 end