Merge 4d2030c into 69a8ea8

ninoseki · Mar 30, 2019 · 700cc82 · 700cc82
2 parents 69a8ea8 + 4d2030c
commit 700cc82
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 6 deletions.
diff --git a/lib/miteru/crawler.rb b/lib/miteru/crawler.rb
@@ -33,7 +33,7 @@ def execute
         website = Website.new(url)
         downloader.download_kits(website.kits) if website.has_kits? && auto_download?
         notify(website) if verbose || website.has_kits?
-      rescue OpenSSL::SSL::SSLError, HTTP::Error, LL::ParserError, Addressable::URI::InvalidURIError => _
+      rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _
         next
       end
     end

diff --git a/lib/miteru/website.rb b/lib/miteru/website.rb
@@ -10,12 +10,11 @@ def initialize(url)
     end
 
     def title
-      doc.at_css("title")&.text
+      doc&.at_css("title")&.text
     end
 
     def kits
-      @kits ||= doc.css("a").map do |a|
-        link = a.get("href")
+      @kits ||= links.map do |link|
         kit = Kit.new(base_url: url, link: link.to_s)
         kit.valid? ? kit : nil
       end.compact
@@ -35,7 +34,7 @@ def kits?
 
     def has_kits?
       ok? && index? && kits?
-    rescue OpenSSL::SSL::SSLError, HTTP::Error, LL::ParserError, Addressable::URI::InvalidURIError => _
+    rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _
       false
     end
 
@@ -55,7 +54,19 @@ def get
     end
 
     def doc
-      @doc ||= Oga.parse_html(response.body.to_s)
+      @doc ||= [].tap do |out|
+        out << Oga.parse_html(response.body.to_s)
+               rescue LL::ParserError => _
+                 out << nil
+      end.first
+    end
+
+    def links
+      if doc
+        doc.css("a").map { |a| a.get("href") }.compact
+      else
+        []
+      end
     end
   end
 end