Merge 983481f into a713854

Gemfile

@@ -11,6 +11,7 @@ gem "public_suffix"
 gem "puma"
 gem "redis"
 gem "rollbar"
+gem "rubyzip"
 gem "simpleidn"
 gem "sinatra-contrib"
 gem "sinatra"

Gemfile.lock

@@ -78,6 +78,7 @@ GEM
     ruby-ll (2.1.2)
       ansi
       ast
+    rubyzip (1.2.2)
     safe_yaml (1.0.4)
     simplecov (0.16.1)
       docile (~> 1.1)
@@ -135,6 +136,7 @@ DEPENDENCIES
   redis
   rollbar
   rspec
+  rubyzip
   simpleidn
   sinatra
   sinatra-contrib

Rakefile

@@ -4,14 +4,3 @@ require "rspec/core/rake_task"
 
 RSpec::Core::RakeTask.new(:spec)
 task default: :spec
-
-$LOAD_PATH.unshift("#{__dir__}/lib")
-require "ayashige"
-
-desc "Fetching domains via WebAnalyzer"
-task :fetch_domains_via_webanalyzer do
-  puts "Fetching domains via WebAnalyzer..."
-  job = Ayashige::Jobs::CronJob.new
-  job.perform
-  puts "done."
-end

bin/cron_job.rb

@@ -2,13 +2,15 @@
 
 require "ayashige"
 
-begin
-  job = Ayashige::Jobs::CronJob.new
-  job.perform
+def with_error_handling
+  yield
 rescue StandardError => e
   if Ayashige::Rollbar.available?
     Rollbar.error e
   else
     puts e
   end
 end
+
+with_error_handling { Ayashige::Sources::WebAnalyzer.new.store_newly_registered_domains }
+with_error_handling { Ayashige::Sources::WhoisDS.new.store_newly_registered_domains }

lib/ayashige.rb

@@ -9,14 +9,14 @@
 
 require "ayashige/store"
 
+require "ayashige/sources/source"
 require "ayashige/sources/web_analyzer"
+require "ayashige/sources/whoisds"
 
 require "ayashige/rollbar"
 
 require "ayashige/application"
 
-require "ayashige/jobs/cron_job"
-
 require "rollbar"
 
 if Ayashige::Rollbar.available?

lib/ayashige/sources/source.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "http"
+require "oga"
+
+module Ayashige
+  module Sources
+    class Source
+      def initialize
+        @store = Store.new
+      end
+
+      def store_newly_registered_domains
+        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+      end
+
+      private
+
+      def html2doc(html)
+        Oga.parse_html html
+      rescue StandardError => _
+        nil
+      end
+
+      def xml2doc(xml)
+        Oga.parse_xml xml
+      rescue StandardError => _
+        nil
+      end
+    end
+  end
+end

lib/ayashige/sources/web_analyzer.rb

@@ -1,33 +1,19 @@
 # frozen_string_literal: true
 
-require "http"
 require "json"
-require "oga"
 require "uri"
 require "simpleidn"
 require "parallel"
 
 module Ayashige
   module Sources
-    class WebAnalyzer
+    class WebAnalyzer < Source
       BASE_URL = "https://wa-com.com"
       TLDS = %w(com net org info us bid biz cat club download life live ltd men pro review shop stream tech today trade win world xyz).freeze
       LIMIT = 5_000
 
-      def initialize
-        @store = Store.new
-      end
-
-      def already_stored?(date)
-        @store.exists? date
-      end
-
       def store_newly_registered_domains
         date = latest_indexed_date
-        if already_stored?(date)
-          puts "domains which updated on #{date} are already stored."
-          return
-        end
 
         Parallel.each(TLDS) do |tld|
           index = 1
@@ -42,7 +28,7 @@ def store_newly_registered_domains
               next unless domain.suspicious?
 
               @store.store updated, domain.to_s, domain.score
-              puts "#{domain} is stored."
+              puts "WebAnalyzer: #{domain} is stored."
             end
             index += 1
           end
@@ -86,12 +72,6 @@ def latest_indexed_date
           out << time.text
         end.first
       end
-
-      def html2doc(html)
-        Oga.parse_html html
-      rescue StandardError => _
-        nil
-      end
     end
   end
 end

lib/ayashige/sources/whoisds.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "zip"
+
+module Ayashige
+  module Sources
+    class WhoisDS < Source
+      BASE_URL = "https://whoisds.com/newly-registered-domains"
+
+      def store_newly_registered_domains
+        date = latest_indexed_date
+        res = HTTP.get(latest_zip_file_link)
+        lines = unzip(res.body.to_s)
+
+        lines.each do |line|
+          domain = Domain.new(line)
+          next unless domain.suspicious?
+
+          @store.store date, domain.to_s, domain.score
+          puts "WhoisDS: #{domain} is stored."
+        end
+      end
+
+      def unzip(data)
+        fin = StringIO.new(data)
+
+        lines = []
+        Zip::InputStream.open(fin) do |zip_file|
+          while (entry = zip_file.get_next_entry)
+            content = entry.get_input_stream.read
+            lines << content.lines.map(&:chomp)
+          end
+        end
+        lines.flatten
+      end
+
+      def doc
+        @doc ||= [].tap do |out|
+          res = HTTP.get(BASE_URL)
+          out << xml2doc(res.body.to_s)
+        end.first
+      end
+
+      def latest_zip_file_link
+        @latest_zip_file_link ||= [].tap do |out|
+          break unless doc
+
+          table = doc.at_css("table")
+          a = table.at_css("tr > td > a")
+          out << a.get("href") if a
+        end.first
+      end
+
+      def latest_indexed_date
+        @latest_indexed_date ||= [].tap do |out|
+          break unless doc
+
+          table = doc.at_css("table")
+          td = table.at_css("tr > td")
+          out << td.text.split.first if td
+        end.first
+      end
+    end
+  end
+end