Merge pull request #33 from ninoseki/add-more-extension-support

feat: add more extension support

lib/miteru/cli.rb

@@ -8,7 +8,7 @@
 
 module Miteru
   class CLI < Thor
-    method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of *.zip file(s)"
+    method_option :auto_download, type: :boolean, default: false, desc: "Enable or disable auto-download of compressed file(s)"
     method_option :directory_traveling, type: :boolean, default: false, desc: "Enable or disable directory traveling"
     method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
     method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"
@@ -26,16 +26,16 @@ def execute
       websites.each do |website|
         next unless website.has_kit?
 
-        message = "#{website.url}: it might contain phishing kit(s) (#{website.zip_files.join(', ')})."
+        message = "#{website.url}: it might contain phishing kit(s) (#{website.compressed_files.join(', ')})."
         puts message.colorize(:light_red)
-        post_to_slack(message) if options[:post_to_slack] && valid_slack_setting?
-        download_zip_files(website.url, website.zip_files, options[:download_to]) if options[:auto_download]
+        post_to_slack(website.message) if options[:post_to_slack] && valid_slack_setting?
+        download_compressed_files(website.url, website.compressed_files, options[:download_to]) if options[:auto_download]
       end
     end
 
     no_commands do
-      def download_zip_files(url, zip_files, base_dir)
-        zip_files.each do |path|
+      def download_compressed_files(url, compressed_files, base_dir)
+        compressed_files.each do |path|
           target_url = "#{url}/#{path}"
           begin
             download_file_path = HTTPClient.download(target_url, base_dir)

lib/miteru/crawler.rb

@@ -29,17 +29,23 @@ def urlscan_feed
       url = "#{URLSCAN_ENDPOINT}/search/?q=certstream-suspicious&size=#{size}"
       res = JSON.parse(get(url))
       res["results"].map { |result| result.dig("task", "url") }
+    rescue HTTPResponseError => _
+      []
     end
 
     def openphish_feed
       res = get("#{OPENPHISH_ENDPOINT}/feed.txt")
       res.lines.map(&:chomp)
+    rescue HTTPResponseError => _
+      []
     end
 
     def phishtank_feed
       res = get("#{PHISHTANK_ENDPOINT}/data/online-valid.csv")
       table = CSV.parse(res, headers: true)
       table.map { |row| row["url"] }
+    rescue HTTPResponseError => _
+      []
     end
 
     def breakdown(url)
@@ -62,11 +68,15 @@ def breakdown(url)
     end
 
     def suspicious_urls
-      urls = (urlscan_feed + openphish_feed + phishtank_feed)
-      urls.map { |url| breakdown(url) }.flatten.uniq.sort
+      @suspicious_urls ||= [].tap do |arr|
+        urls = (urlscan_feed + openphish_feed + phishtank_feed)
+        urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
+      end
     end
 
     def execute
+      puts "Loaded #{suspicious_urls.length} URLs to crawl." if verbose
+
       pool = Thread.pool(threads)
       websites = []
 

lib/miteru/website.rb

@@ -14,10 +14,10 @@ def title
       doc.at_css("title")&.text
     end
 
-    def zip_files
-      @zip_files ||= doc.css("a").map do |a|
+    def compressed_files
+      @compressed_files ||= doc.css("a").map do |a|
         href = a.get("href")
-        href&.end_with?(".zip") ? href : nil
+        [".zip", ".rar", ".7z", ".tar", ".gz"].any? { |ext| href&.end_with? ext } ? href : nil
       end.compact.map do |href|
         href.start_with?("/") ? href[1..-1] : href
       end
@@ -31,12 +31,12 @@ def index?
       title == "Index of /"
     end
 
-    def zip_files?
-      !zip_files.empty?
+    def compressed_files?
+      !compressed_files.empty?
     end
 
     def has_kit?
-      @has_kit ||= ok? && index? && zip_files?
+      @has_kit ||= ok? && index? && compressed_files?
     end
 
     def build
@@ -46,7 +46,7 @@ def build
     def unbuild
       @doc = nil
       @response = nil
-      @zip_files = nil
+      @compressed_files = nil
     end
 
     private

spec/cli_spec.rb

@@ -2,7 +2,7 @@
 
 RSpec.describe Miteru::CLI do
   include_context "http_server"
-  include_context "download_zip_files"
+  include_context "download_compressed_files"
   subject { Miteru::CLI.new }
   before(:each) { ENV.delete "SLACK_WEBHOOK_URL" }
 
@@ -15,15 +15,15 @@
     end
   end
 
-  describe "#download_zip_files" do
+  describe "#download_compressed_files" do
     before { WebMock.disable! }
     after { WebMock.enable! }
     context "when it runs once" do
       it "should download a file" do
         url = "http://#{host}:#{port}/has_kit"
-        zip_files = ["test.zip"]
+        compressed_files = ["test.zip"]
         expect(Dir.glob("#{base_dir}/*.zip").empty?).to be(true)
-        capture(:stdout) { subject.download_zip_files(url, zip_files, base_dir) }
+        capture(:stdout) { subject.download_compressed_files(url, compressed_files, base_dir) }
         download_files = Dir.glob("#{base_dir}/*.zip")
         expect(download_files.empty?).to be(false)
         expect(download_files.length).to eq(1)
@@ -32,11 +32,11 @@
     context "when it runs multiple times" do
       it "should remove duplicated files" do
         url = "http://#{host}:#{port}/has_kit"
-        zip_files = ["test.zip"]
+        compressed_files = ["test.zip"]
         expect(Dir.glob("#{base_dir}/*.zip").empty?).to be(true)
-        capture(:stdout) { subject.download_zip_files(url, zip_files, base_dir) }
-        capture(:stdout) { subject.download_zip_files(url, zip_files, base_dir) }
-        capture(:stdout) { subject.download_zip_files(url, zip_files, base_dir) }
+        capture(:stdout) { subject.download_compressed_files(url, compressed_files, base_dir) }
+        capture(:stdout) { subject.download_compressed_files(url, compressed_files, base_dir) }
+        capture(:stdout) { subject.download_compressed_files(url, compressed_files, base_dir) }
         download_files = Dir.glob("#{base_dir}/*.zip")
         expect(download_files.empty?).to be(false)
         expect(download_files.length).to eq(1)

spec/fixtures/index.html

@@ -7,6 +7,7 @@ <h1>Index of /</h1>
   <ul>
     <li><a href=".well-known/"> .well-known/</a></li>
     <li><a href="test.zip"> test.zip</a></li>
+    <li><a href="test.7z"> test.7z</a></li>
     <li><a href="12/"> 12/</a></li>
     <li><a href="cgi-bin/"> cgi-bin/</a></li>
   </ul>

spec/http_client.rb

@@ -2,7 +2,7 @@
 
 RSpec.describe Miteru::HTTPClient do
   include_context "http_server"
-  include_context "download_zip_files"
+  include_context "download_compressed_files"
   subject { Miteru::HTTPClient }
 
   describe ".download" do

spec/spec_helper.rb

@@ -22,12 +22,12 @@
   config.shared_context_metadata_behavior = :apply_to_host_groups
 end
 
-require_relative "./support/shared_contexts/download_zip_files_context"
+require_relative "./support/shared_contexts/download_compressed_files_context"
 require_relative "./support/shared_contexts/http_server_context"
 require_relative "./support/helpers/helpers"
 
 RSpec.configure do |config|
-  config.include_context "download_zip_files"
+  config.include_context "download_compressed_files"
   config.include_context "http_server"
   config.include Spec::Support::Helpers
 end

spec/support/shared_contexts/download_zip_files_context.rb → spec/support/shared_contexts/download_compressed_files_context.rb

@@ -2,7 +2,7 @@
 
 require "fileutils"
 
-RSpec.shared_context "download_zip_files" do
+RSpec.shared_context "download_compressed_files" do
   before do
     @path = File.expand_path("../../../tmp", __dir__)
     FileUtils.mkdir_p(@path)

spec/website_spec.rb

@@ -9,11 +9,11 @@
       expect(subject.new("http://#{host}:#{port}/has_kit").title).to be_a(String)
     end
   end
-  describe "#zip_files" do
+  describe "#compressed_files" do
     it "should return an Array" do
-      zip_files = subject.new("http://#{host}:#{port}/has_kit").zip_files
-      expect(zip_files).to be_an(Array)
-      expect(zip_files.length).to eq(1)
+      compressed_files = subject.new("http://#{host}:#{port}/has_kit").compressed_files
+      expect(compressed_files).to be_an(Array)
+      expect(compressed_files.length).to eq(2)
     end
   end
   describe "#has_kit?" do