Merge pull request #46 from ninoseki/fix-ct-issue

fix: fix CT related issues

lib/ayashige/config/ct.yml

@@ -0,0 +1,31 @@
+ct_log_servers:
+  - https://ct.cloudflare.com/logs/nimbus2019
+  - https://ct.cloudflare.com/logs/nimbus2020
+  - https://ct.cloudflare.com/logs/nimbus2021
+  - https://ct.cloudflare.com/logs/nimbus2022
+  - https://ct.cloudflare.com/logs/nimbus2023
+  - https://ct.googleapis.com/aviator
+  - https://ct.googleapis.com/icarus
+  - https://ct.googleapis.com/logs/argon2019
+  - https://ct.googleapis.com/logs/argon2020
+  - https://ct.googleapis.com/logs/argon2021
+  - https://ct.googleapis.com/logs/argon2022
+  - https://ct.googleapis.com/logs/xenon2019
+  - https://ct.googleapis.com/logs/xenon2020
+  - https://ct.googleapis.com/logs/xenon2021
+  - https://ct.googleapis.com/logs/xenon2022
+  - https://ct.googleapis.com/pilot
+  - https://ct.googleapis.com/rocketeer
+  - https://ct.googleapis.com/skydiver
+  - https://ct.googleapis.com/submariner
+  - https://ct1.digicert-ct.com/log
+  - https://ct2.digicert-ct.com/log
+  - https://golem.ct.digicert.com/log
+  - https://nessie2019.ct.digicert.com/log
+  - https://nessie2020.ct.digicert.com/log
+  - https://nessie2021.ct.digicert.com/log
+  - https://nessie2022.ct.digicert.com/log
+  - https://yeti2019.ct.digicert.com/log
+  - https://yeti2020.ct.digicert.com/log
+  - https://yeti2021.ct.digicert.com/log
+  - https://yeti2022.ct.digicert.com/log

lib/ayashige/sources/ct.rb

@@ -29,38 +29,28 @@ def x509_entries
         @cache.set(url, sth.tree_size)
 
         entries
+      rescue StandardError => _
+        []
       end
     end
 
     class CT < Source
-      CTL_LIST = "https://www.gstatic.com/ct/log_list/all_logs_list.json"
-      BAD_CTL_SERVERS = [
-        "alpha.ctlogs.org/", "clicky.ct.letsencrypt.org/", "ct.akamai.com/", "ct.filippo.io/behindthesofa/",
-        "ct.gdca.com.cn/", "ct.izenpe.com/", "ct.izenpe.eus/", "ct.sheca.com/", "ct.startssl.com/", "ct.wosign.com/",
-        "ctserver.cnnic.cn/", "ctlog.api.venafi.com/", "ctlog.gdca.com.cn/", "ctlog.sheca.com/", "ctlog.wosign.com/",
-        "ctlog2.wosign.com/", "flimsy.ct.nordu.net:8080/", "log.certly.io/", "nessie2021.ct.digicert.com/log/",
-        "plausible.ct.nordu.net/", "www.certificatetransparency.cn/ct/", "ct.googleapis.com/testtube/",
-        "ct.googleapis.com/daedalus/"
-      ].freeze
-
-      def initialize
-        super
-        @cache = FileCache.new("ct")
+      CTL_LIST = "https://ct.grahamedgecombe.com/logs.json"
+
+      def initialize(cache_dir = "/tmp")
+        super()
+        @cache = FileCache.new("ct", cache_dir)
+      end
+
+      def config
+        @config ||= YAML.safe_load File.read(File.expand_path("./../config/ct.yml", __dir__))
       end
 
       def ctl_servers
         @ctl_servers ||= [].tap do |servers|
-          res = HTTP.get(CTL_LIST)
-          json = JSON.parse(res.body.to_s)
-          logs = json.dig("logs")
-          break unless logs
-
-          logs.each do |log|
-            url = log.dig("url")
-            next unless url || BAD_CTL_SERVERS.include?(url)
-
-            # remove "/" from end of url
-            servers << CTLServer.new("https://#{url[0..-2]}", @cache)
+          urls = config.dig("ct_log_servers") || []
+          urls.each do |url|
+            servers << CTLServer.new(url, @cache)
           end
         end
       end
@@ -75,26 +65,25 @@ def store_newly_registered_domains
 
       def get_domain_name(subject)
         cn = subject.to_a.find { |a| a.first == "CN" }
+        return nil unless cn
+
         domain = cn[1]
         domain.gsub /\*\./, ""
       end
 
       def all_x509_entries
-        ctl_servers.map do |ctl_server|
-          entries << ctl_server.x509_entries
-        rescue StandardError => _
-          []
-        end.flatten
+        ctl_servers.map(&:x509_entries).flatten
       end
 
       def records
         all_x509_entries.map do |entry|
+          domain_name = get_domain_name(entry.leaf_input.timestamped_entry.x509_entry.subject)
+          next unless domain_name
+
           Record.new(
-            domain_name: get_domain_name(entry.leaf_input.timestamped_entry.x509_entry.subject),
+            domain_name: domain_name,
             updated: entry.leaf_input.timestamped_entry.timestamp.to_s
           )
-        rescue NoMethodError => _
-          nil
         end.compact
       end
     end