Merge pull request #3 from ninoseki/improve-detection

refactor: improve detection method

lib/miteru.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "miteru/error"
+require "miteru/website"
 require "miteru/crawler"
 require "miteru/cli"
 require "miteru/version"

lib/miteru/crawler.rb

@@ -16,23 +16,17 @@ def suspicous_urls
       res["results"].map { |result| result.dig("task", "url") }
     end
 
-    def has_kit?(url)
-      begin
-        res = get(url)
-      rescue HTTPResponseError => _
-        false
-      end
-
-      rules = ["Index of", ".zip"]
-      rules.all? { |rule| res.include? rule }
-    end
-
     def execute
       pool = Thread.pool(threads)
       results = []
 
       suspicous_urls.each do |url|
-        pool.process { results << url if has_kit?(url) }
+        pool.process do
+          doc = Website.new(url)
+          results << url if doc.has_kit?
+        rescue HTTPResponseError => _
+          next
+        end
       end
       pool.shutdown
 

lib/miteru/website.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "http"
+require "oga"
+
+module Miteru
+  class Website
+    attr_reader :url
+    def initialize(url)
+      @url = url
+    end
+
+    def title
+      doc.at_css("title")&.text
+    end
+
+    def zip_files
+      @zip_files ||= doc.css("a").map do |a|
+        href = a.get("href")
+        href&.end_with?(".zip") ? href : nil
+      end.compact
+    end
+
+    def index?
+      title == "Index of /"
+    end
+
+    def zip_files?
+      !zip_files.empty?
+    end
+
+    def has_kit?
+      index? && zip_files?
+    end
+
+    private
+
+    def get
+      res = HTTP.get(url)
+      raise HTTPResponseError if res.code != 200
+
+      res.body.to_s
+    end
+
+    def doc
+      @doc ||= Oga.parse_html(get)
+    end
+  end
+end

miteru.gemspec

@@ -32,6 +32,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "webmock", "~> 3.4"
 
   spec.add_dependency "http", "~> 3.3"
+  spec.add_dependency "oga", "~> 2.15"
   spec.add_dependency "thor", "~> 0.19"
   spec.add_dependency "thread", "~> 0.2.2"
 end

spec/crawler_spec.rb

@@ -10,25 +10,15 @@
       expect(results.length).to eq(100)
     end
   end
-  describe "#has_kit?" do
-    context "when giving a url which contains a phishint kit" do
-      before do
-        path = File.expand_path("./fixtures/index.html", __dir__)
-        data = File.read(path)
-
-        allow_any_instance_of(Miteru::Crawler).to receive(:get).and_return(data)
-      end
-      it "should return true" do
-        expect(subject.has_kit?("http://localhost")).to eq(true)
-      end
-    end
-  end
-  context "when giving a url which doesn't contain a phishint kit" do
+  describe "#execute" do
     before do
-      allow_any_instance_of(Miteru::Crawler).to receive(:get).and_return("None")
+      allow_any_instance_of(Miteru::Crawler).to receive(:suspicous_urls).and_return(%w(http://localhost))
+      allow_any_instance_of(Miteru::Website).to receive(:has_kit?).and_return(true)
     end
-    it "should return false" do
-      expect(subject.has_kit?("http://localhost")).to eq(false)
+    it "should return an Array" do
+      results = subject.execute
+      expect(results).to be_an(Array)
+      expect(results.length).to eq(1)
     end
   end
 end

spec/website_spec.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+RSpec.describe Miteru::Website, :vcr do
+  subject { Miteru::Website }
+  describe "#has_kit?" do
+    context "when giving a url which contains a phishint kit" do
+      before do
+        path = File.expand_path("./fixtures/index.html", __dir__)
+        data = File.read(path)
+
+        allow_any_instance_of(Miteru::Website).to receive(:get).and_return(data)
+      end
+      it "should return true" do
+        expect(subject.new("http://localhost").has_kit?).to eq(true)
+      end
+    end
+  end
+  context "when giving a url which doesn't contain a phishint kit" do
+    before do
+      allow_any_instance_of(Miteru::Website).to receive(:get).and_return("None")
+    end
+    it "should return false" do
+      expect(subject.new("http://localhost").has_kit?).to eq(false)
+    end
+  end
+end