Merge pull request #4 from ninoseki/add-verbose-option

feat: add verbose option

lib/miteru/cli.rb

@@ -1,18 +1,21 @@
 # frozen_string_literal: true
 
+require "colorize"
 require "http"
 require "thor"
 
 module Miteru
   class CLI < Thor
+    method_option :verbose, type: :boolean, default: true
     method_option :post_to_slack, type: :boolean, default: false
     desc "execute", "Execute the crawler"
     def execute
-      results = Crawler.execute
-      results.each do |result|
-        message = "#{result} might contain a phishing kit."
-        puts message
-        post_to_slack(message) if options[:post_to_slack] && valid_slack_setting?
+      websites = Crawler.execute(options[:verbose])
+      websites.each do |website|
+        if website.has_kit?
+          puts "#{website.url}: it might contain a phishing kit (#{website.zip_files.join(',')}).".colorize(:light_red)
+          post_to_slack(message) if options[:post_to_slack] && valid_slack_setting?
+        end
       end
     end
 

lib/miteru/crawler.rb

@@ -16,25 +16,24 @@ def suspicous_urls
       res["results"].map { |result| result.dig("task", "url") }
     end
 
-    def execute
+    def execute(verbose = false)
       pool = Thread.pool(threads)
-      results = []
+      websites = []
 
       suspicous_urls.each do |url|
         pool.process do
-          doc = Website.new(url)
-          results << url if doc.has_kit?
-        rescue HTTPResponseError => _
-          next
+          website = Website.new(url)
+          puts "#{website.url}: it doesn't contain a phishing kit." if verbose && !website.has_kit?
+          websites << website
         end
       end
       pool.shutdown
 
-      results
+      websites
     end
 
-    def self.execute
-      new.execute
+    def self.execute(verbose = false)
+      new.execute(verbose)
     end
 
     private

lib/miteru/website.rb

@@ -8,6 +8,7 @@ class Website
     attr_reader :url
     def initialize(url)
       @url = url
+      build
     end
 
     def title
@@ -21,6 +22,10 @@ def zip_files
       end.compact
     end
 
+    def ok?
+      response.code == 200
+    end
+
     def index?
       title == "Index of /"
     end
@@ -30,20 +35,27 @@ def zip_files?
     end
 
     def has_kit?
-      index? && zip_files?
+      ok? && index? && zip_files?
+    end
+
+    def build
+      doc
     end
 
     private
 
-    def get
-      res = HTTP.get(url)
-      raise HTTPResponseError if res.code != 200
+    def response
+      @response ||= get
+    end
 
-      res.body.to_s
+    def get
+      ctx = OpenSSL::SSL::SSLContext.new
+      ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      HTTP.get(url, ssl_context: ctx)
     end
 
     def doc
-      @doc ||= Oga.parse_html(get)
+      @doc ||= Oga.parse_html(response.body.to_s)
     end
   end
 end

miteru.gemspec

@@ -26,11 +26,13 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.16"
   spec.add_development_dependency "coveralls", "~> 0.8"
+  spec.add_development_dependency "glint", "~> 0.1"
   spec.add_development_dependency "rake", "~> 12.3"
   spec.add_development_dependency "rspec", "~> 3.8"
   spec.add_development_dependency "vcr", "~> 4.0"
   spec.add_development_dependency "webmock", "~> 3.4"
 
+  spec.add_dependency "colorize", "~> 0.8"
   spec.add_dependency "http", "~> 3.3"
   spec.add_dependency "oga", "~> 2.15"
   spec.add_dependency "thor", "~> 0.19"

spec/crawler_spec.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 RSpec.describe Miteru::Crawler, :vcr do
+  include_context "http_server"
   subject { Miteru::Crawler.new }
   describe "#suspicous_urls" do
     it "should return an Array" do
@@ -12,8 +13,7 @@
   end
   describe "#execute" do
     before do
-      allow_any_instance_of(Miteru::Crawler).to receive(:suspicous_urls).and_return(%w(http://localhost))
-      allow_any_instance_of(Miteru::Website).to receive(:has_kit?).and_return(true)
+      allow_any_instance_of(Miteru::Crawler).to receive(:suspicous_urls).and_return(["http://#{host}:#{port}/has_kit"])
     end
     it "should return an Array" do
       results = subject.execute

spec/spec_helper.rb

@@ -5,12 +5,11 @@
 require 'coveralls'
 Coveralls.wear!
 
+require "glint"
 require "miteru"
-
 require "vcr"
 require "webrick"
 
-
 RSpec.configure do |config|
   # Enable flags like --only-failures and --next-failure
   config.example_status_persistence_file_path = ".rspec_status"
@@ -35,6 +34,55 @@ def capture(stream)
   end
 end
 
+def server
+  server = Glint::Server.new do |port|
+    http = WEBrick::HTTPServer.new(
+      BindAddress: "0.0.0.0",
+      Port: port,
+      Logger: WEBrick::Log.new(File.open(File::NULL, "w")),
+      AccessLog: []
+    )
+    http.mount_proc("/has_kit") do |_, res|
+      path = File.expand_path("./fixtures/index.html", __dir__)
+      body = File.read(path)
+
+      res.status = 200
+      res.content_length = body.size
+      res.content_type = 'text/plain'
+      res.body = body
+    end
+
+    http.mount_proc("/no_kit") do |_, res|
+      body = "None"
+
+      res.status = 200
+      res.content_length = body.size
+      res.content_type = 'text/plain'
+      res.body = body
+    end
+
+    trap(:INT) { http.shutdown }
+    trap(:TERM) { http.shutdown }
+    http.start
+  end
+  Glint::Server.info[:http_server] = {
+    host: "0.0.0.0",
+    port: server.port
+  }
+  server
+end
+
+RSpec.shared_context "http_server" do
+  before(:all) {
+    @server = server
+    @server.start
+  }
+  after(:all) { @server.stop }
+
+  let(:host) { "0.0.0.0" }
+  let(:port) { @server.port }
+end
+
 VCR.configure do |config|
   config.cassette_library_dir = "spec/fixtures/vcr_cassettes"
   config.configure_rspec_metadata!

spec/website_spec.rb

@@ -1,26 +1,18 @@
 # frozen_string_literal: true
 
-RSpec.describe Miteru::Website, :vcr do
+RSpec.describe Miteru::Website do
+  include_context "http_server"
   subject { Miteru::Website }
   describe "#has_kit?" do
     context "when giving a url which contains a phishint kit" do
-      before do
-        path = File.expand_path("./fixtures/index.html", __dir__)
-        data = File.read(path)
-
-        allow_any_instance_of(Miteru::Website).to receive(:get).and_return(data)
-      end
       it "should return true" do
-        expect(subject.new("http://localhost").has_kit?).to eq(true)
+        expect(subject.new("http://#{host}:#{port}/has_kit").has_kit?).to eq(true)
       end
     end
   end
   context "when giving a url which doesn't contain a phishint kit" do
-    before do
-      allow_any_instance_of(Miteru::Website).to receive(:get).and_return("None")
-    end
     it "should return false" do
-      expect(subject.new("http://localhost").has_kit?).to eq(false)
+      expect(subject.new("http://#{host}:#{port}/no_kit").has_kit?).to eq(false)
     end
   end
 end