Merge pull request #73 from ninoseki/refactoring

refactor: add Kit class

lib/miteru.rb

@@ -4,6 +4,7 @@
 
 require "miteru/error"
 require "miteru/http_client"
+require "miteru/kit"
 require "miteru/website"
 require "miteru/downloader"
 require "miteru/feeds"

lib/miteru/cli.rb

@@ -4,7 +4,7 @@
 
 module Miteru
   class CLI < Thor
-    method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of compressed file(s)"
+    method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of phishing kits"
     method_option :directory_traveling, type: :boolean, default: false, desc: "Enable or disable directory traveling"
     method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
     method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"

lib/miteru/crawler.rb

@@ -31,12 +31,8 @@ def execute
 
       Parallel.each(feeds.suspicious_urls, in_threads: threads) do |url|
         website = Website.new(url)
-        if website.has_kit?
-          downloader.download_compressed_files(website.url, website.compressed_files) if auto_download?
-          notify(website.url, website.compressed_files)
-        else
-          notify(website.url, website.compressed_files) if verbose
-        end
+        downloader.download_kits(website.kits) if website.has_kits? && auto_download?
+        notify(website) if verbose || website.has_kits?
       rescue OpenSSL::SSL::SSLError, HTTP::Error, LL::ParserError, Addressable::URI::InvalidURIError => _
         next
       end
@@ -54,8 +50,8 @@ def self.execute(auto_download: false, directory_traveling: false, download_to:
       ).execute
     end
 
-    def notify(url, message)
-      @notifier.notify(url, message)
+    def notify(website)
+      @notifier.notify(url: website.url, kits: website.kits, message: website.message)
     end
 
     def auto_download?

lib/miteru/downloader.rb

@@ -13,33 +13,30 @@ def initialize(base_dir = "/tmp")
       raise ArgumentError, "#{base_dir} is not existing." unless Dir.exist?(base_dir)
     end
 
-    def download_compressed_files(url, compressed_files)
-      compressed_files.each do |path|
-        target_url = "#{url}/#{path}"
-        filename = download_filename(target_url)
+    def download_kits(kits)
+      kits.each do |kit|
+        filename = download_filename(kit)
         destination = filepath_to_download(filename)
         begin
-          download_filepath = HTTPClient.download(target_url, destination)
-          if duplicated?(download_filepath)
-            puts "Do not download #{target_url} because there is a file that has a same hash value in the directory (SHA256: #{sha256(download_filepath)})."
-            FileUtils.rm download_filepath
+          downloaded_filepath = HTTPClient.download(kit.url, destination)
+          if duplicated?(downloaded_filepath)
+            puts "Do not download #{kit.url} because there is a file that has a same hash value in the directory (SHA256: #{sha256(downloaded_filepath)})."
+            FileUtils.rm downloaded_filepath
           else
-            puts "Download #{target_url} as #{download_filepath}"
+            puts "Download #{kit.url} as #{downloaded_filepath}"
           end
         rescue Down::Error => e
-          puts "Failed to download: #{target_url} (#{e})"
+          puts "Failed to download: #{kit.url} (#{e})"
         end
       end
     end
 
     private
 
-    def download_filename(url)
-      filename = url.split("/").last
-      extname = File.extname(filename)
-      domain = URI(url).hostname
+    def download_filename(kit)
+      domain = URI(kit.base_url).hostname
 
-      "#{domain}_#{filename}_#{SecureRandom.alphanumeric(10)}#{extname}"
+      "#{domain}_#{kit.basename}_#{SecureRandom.alphanumeric(10)}#{kit.extname}"
     end
 
     def filepath_to_download(filename)

lib/miteru/kit.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Miteru
+  class Kit
+    VALID_EXTENSIONS = [".zip", ".rar", ".7z", ".tar", ".gz"].freeze
+
+    attr_reader :base_url, :link
+
+    def initialize(base_url:, link:)
+      @base_url = base_url
+      @link = link.start_with?("/") ? link[1..-1] : link
+    end
+
+    def valid?
+      VALID_EXTENSIONS.include? extname
+    end
+
+    def extname
+      File.extname(link)
+    end
+
+    def basename
+      File.basename(link)
+    end
+
+    def url
+      "#{base_url}/#{basename}"
+    end
+  end
+end

lib/miteru/notifier.rb

@@ -9,11 +9,10 @@ def initialize(post_to_slack = false)
       @post_to_slack = post_to_slack
     end
 
-    def notify(url, compressed_files)
-      message = compressed_files.empty? ? "it doesn't contain a phishing kit." : "it might contain phishing kit(s): (#{compressed_files.join(', ')})."
+    def notify(url:, kits:, message:)
       attachement = Attachement.new(url)
 
-      if post_to_slack? && !compressed_files.empty?
+      if post_to_slack? && !kits.empty?
         slack = Slack::Incoming::Webhooks.new(slack_webhook_url, channel: slack_channel)
         slack.post(
           url,
@@ -24,7 +23,7 @@ def notify(url, compressed_files)
         )
       end
 
-      message = message.colorize(:light_red) unless compressed_files.empty?
+      message = message.colorize(:light_red) unless kits.empty?
       puts "#{url}: #{message}"
     end
 

lib/miteru/website.rb

@@ -13,13 +13,12 @@ def title
       doc.at_css("title")&.text
     end
 
-    def compressed_files
-      @compressed_files ||= doc.css("a").map do |a|
-        href = a.get("href")
-        [".zip", ".rar", ".7z", ".tar", ".gz"].any? { |ext| href&.end_with? ext } ? href : nil
-      end.compact.map do |href|
-        href.start_with?("/") ? href[1..-1] : href
-      end
+    def kits
+      @kits ||= doc.css("a").map do |a|
+        link = a.get("href")
+        kit = Kit.new(base_url: url, link: link.to_s)
+        kit.valid? ? kit : nil
+      end.compact
     end
 
     def ok?
@@ -30,16 +29,21 @@ def index?
       title == "Index of /"
     end
 
-    def compressed_files?
-      !compressed_files.empty?
+    def kits?
+      !kits.empty?
     end
 
-    def has_kit?
-      ok? && index? && compressed_files?
-    rescue StandardError => _
+    def has_kits?
+      ok? && index? && kits?
+    rescue OpenSSL::SSL::SSLError, HTTP::Error, LL::ParserError, Addressable::URI::InvalidURIError => _
       false
     end
 
+    def message
+      kit_names = kits.map(&:basename).join(", ")
+      kits? ? "it might contain phishing kit(s): (#{kit_names})." : "it doesn't contain a phishing kit."
+    end
+
     private
 
     def response

spec/crawler_spec.rb

@@ -2,7 +2,7 @@
 
 RSpec.describe Miteru::Crawler do
   include_context "http_server"
-  include_context "download_compressed_files"
+  include_context "download_kits"
 
   before { allow(ENV).to receive(:[]).with("SLACK_WEBHOOK_URL").and_return(nil) }
 

spec/downloader_spec.rb

@@ -2,21 +2,23 @@
 
 RSpec.describe Miteru::Downloader do
   include_context "http_server"
-  include_context "download_compressed_files"
+  include_context "download_kits"
 
-  describe "#download_compressed_files" do
+  describe "#download_kits" do
     subject { Miteru::Downloader.new(base_dir) }
 
     before { WebMock.disable! }
     after { WebMock.enable! }
 
     context "when it runs once" do
       it "should download a file" do
-        url = "http://#{host}:#{port}/has_kit"
-        compressed_files = ["test.zip", "test.tar"]
+        kits = [
+          Miteru::Kit.new(base_url: "http://#{host}:#{port}/has_kit", link: "test.zip"),
+          Miteru::Kit.new(base_url: "http://#{host}:#{port}/has_kit", link: "test.tar")
+        ]
         expect(Dir.glob("#{base_dir}/*.zip").empty?).to be(true)
 
-        out = capture(:stdout) { subject.download_compressed_files(url, compressed_files) }
+        out = capture(:stdout) { subject.download_kits(kits) }
         lines = out.split("\n")
         expect(lines.length).to eq(2)
         expect(lines.first.end_with?(".zip")).to be(true)
@@ -34,13 +36,14 @@
 
     context "when it runs multiple times" do
       it "should remove duplicated files" do
-        url = "http://#{host}:#{port}/has_kit"
-        compressed_files = ["test.zip"]
+        kits = [
+          Miteru::Kit.new(base_url: "http://#{host}:#{port}/has_kit", link: "test.zip")
+        ]
         expect(Dir.glob("#{base_dir}/*.zip").empty?).to be(true)
 
-        capture(:stdout) { subject.download_compressed_files(url, compressed_files) }
-        capture(:stdout) { subject.download_compressed_files(url, compressed_files) }
-        out = capture(:stdout) { subject.download_compressed_files(url, compressed_files) }
+        capture(:stdout) { subject.download_kits(kits) }
+        capture(:stdout) { subject.download_kits(kits) }
+        out = capture(:stdout) { subject.download_kits(kits) }
         expect(out.start_with?("Do not download")).to be(true)
 
         download_files = Dir.glob("#{base_dir}/*.zip")

spec/http_client_spec.rb

@@ -2,7 +2,7 @@
 
 RSpec.describe Miteru::HTTPClient do
   include_context "http_server"
-  include_context "download_compressed_files"
+  include_context "download_kits"
   subject { Miteru::HTTPClient }
 
   describe ".download" do

spec/kit_spec.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+RSpec.describe Miteru::Kit do
+
+  subject { Miteru::Kit.new(base_url: "http://test.com", link: "/test.zip") }
+
+  describe "#basename" do
+    it "should return a base name" do
+      expect(subject.basename).to eq("test.zip")
+    end
+  end
+
+  describe "#extname" do
+    it "should return a base extname" do
+      expect(subject.extname).to eq(".zip")
+    end
+  end
+
+  describe "#url" do
+    it "should return a URL" do
+      expect(subject.url).to eq("http://test.com/test.zip")
+    end
+  end
+
+  describe "#valid?" do
+    it "should return true" do
+      expect(subject.valid?).to eq(true)
+    end
+  end
+end

spec/notifier_spec.rb

@@ -7,7 +7,7 @@
     context "when not given SLACK setting via ENV" do
       it "should output a message to STDOUT" do
         out = capture(:stdout) {
-          subject.notify("http://test.com", ["test.zip"])
+          subject.notify(url: "http://test.com", kits: [], message: "test")
         }
         expect(out).to include("test.com")
       end

spec/spec_helper.rb

@@ -27,12 +27,12 @@
   config.shared_context_metadata_behavior = :apply_to_host_groups
 end
 
-require_relative "./support/shared_contexts/download_compressed_files_context"
+require_relative "./support/shared_contexts/download_kits_context"
 require_relative "./support/shared_contexts/http_server_context"
 require_relative "./support/helpers/helpers"
 
 RSpec.configure do |config|
-  config.include_context "download_compressed_files"
+  config.include_context "download_kits"
   config.include_context "http_server"
   config.include Spec::Support::Helpers
 end

spec/support/shared_contexts/download_compressed_files_context.rb → spec/support/shared_contexts/download_kits_context.rb

@@ -2,7 +2,7 @@
 
 require "fileutils"
 
-RSpec.shared_context "download_compressed_files" do
+RSpec.shared_context "download_kits" do
   before do
     @path = File.expand_path("../../../tmp", __dir__)
     FileUtils.mkdir_p(@path)

spec/website_spec.rb

@@ -11,24 +11,24 @@
     end
   end
 
-  describe "#compressed_files" do
+  describe "#kits" do
     it "should return an Array" do
-      compressed_files = subject.new("http://#{host}:#{port}/has_kit").compressed_files
-      expect(compressed_files).to be_an(Array)
-      expect(compressed_files.length).to eq(2)
+      kits = subject.new("http://#{host}:#{port}/has_kit").kits
+      expect(kits).to be_an(Array)
+      expect(kits.length).to eq(2)
     end
   end
 
-  describe "#has_kit?" do
+  describe "#has_kits?" do
     context "when giving a url which contains a phishing kit" do
       it "should return true" do
-        expect(subject.new("http://#{host}:#{port}/has_kit").has_kit?).to eq(true)
+        expect(subject.new("http://#{host}:#{port}/has_kit").has_kits?).to eq(true)
       end
     end
 
     context "when giving a url which doesn't contain a phishing kit" do
       it "should return false" do
-        expect(subject.new("http://#{host}:#{port}/no_kit").has_kit?).to eq(false)
+        expect(subject.new("http://#{host}:#{port}/no_kit").has_kits?).to eq(false)
       end
     end
   end