Merge 707cef2 into 5b90854

README.md

@@ -8,7 +8,7 @@ Ayashige provides a list of suspicious newly registered domains as a JSON feed.
 
 ## How it works
 
-- It collects newly registered domains via [WebAnalyzer](https://wa-com.com/), [WhoisDS](https://whoisds.com/), [DomainWatch](https://domainwat.ch/) and a Certififate Transparency log server (Google Rocketeer).
+- It collects newly registered domains via [WebAnalyzer](https://wa-com.com/), [WhoisDS](https://whoisds.com/), [DomainWatch](https://domainwat.ch/) and Certififate Transparency log servers.
 - It computes a suspicious score of a given domain.
   - The scoring rule comes from [x0rz/phishing_catcher](https://github.com/x0rz/phishing_catcher).
 - It stores suspicious domains into a Redis instance.

lib/ayashige/sources/ct.rb

@@ -1,40 +1,50 @@
 # frozen_string_literal: true
 
 require "certificate-transparency-client"
+require "set"
 
 module Ayashige
   module Sources
     class CT < Source
       BASE_URL = "https://ct.googleapis.com/rocketeer"
-      LIMIT = 10_000
+      LIMIT = 5_000
 
       def initialize
         super
 
-        @ct = CertificateTransparency::Client.new(BASE_URL)
+        @ct_log_servers = [
+          "https://ct.googleapis.com/icarus",
+          "https://ct.googleapis.com/pilot",
+          "https://ct.googleapis.com/rocketeer",
+          "https://ct.googleapis.com/logs/argon2019",
+          "https://yeti2019.ct.digicert.com/log"
+        ]
       end
 
       def name
-        "CT(Google Rocketeer)"
+        "CT log"
       end
 
       def store_newly_registered_domains
         records.each { |record| store record }
       end
 
-      def sth
-        @sth ||= @ct.get_sth
-      end
-
       def get_domain_name(subject)
         cn = subject.to_a.find { |a| a.first == "CN" }
-        cn[1]
+        domain = cn[1]
+        domain.gsub /\*\./, ""
       end
 
       def x509_entries
-        @x509_entries ||= @ct.get_entries(sth.tree_size - LIMIT, sth.tree_size).select do |entry|
-          entry.leaf_input.timestamped_entry.x509_entry
-        end
+        @x509_entries ||= [].tap do |entries|
+          @ct_log_servers.each do |ct_log_server|
+            ct = CertificateTransparency::Client.new(ct_log_server)
+            sth = ct.get_sth
+            entries << ct.get_entries(sth.tree_size - LIMIT, sth.tree_size).select do |entry|
+              entry.leaf_input.timestamped_entry.x509_entry
+            end
+          end
+        end.flatten
       end
 
       def records

spec/sources/ct_spec.rb

@@ -34,7 +34,7 @@
 
   describe "#name" do
     it "should return a name of the class" do
-      expect(subject.name).to eq("CT(Google Rocketeer)")
+      expect(subject.name).to eq("CT log")
     end
   end