diff --git a/lib/miteru/cli.rb b/lib/miteru/cli.rb index f5dc035..3e4cf95 100644 --- a/lib/miteru/cli.rb +++ b/lib/miteru/cli.rb @@ -1,18 +1,21 @@ # frozen_string_literal: true +require "colorize" require "http" require "thor" module Miteru class CLI < Thor + method_option :verbose, type: :boolean, default: true method_option :post_to_slack, type: :boolean, default: false desc "execute", "Execute the crawler" def execute - results = Crawler.execute - results.each do |result| - message = "#{result} might contain a phishing kit." - puts message - post_to_slack(message) if options[:post_to_slack] && valid_slack_setting? + websites = Crawler.execute(options[:verbose]) + websites.each do |website| + if website.has_kit? + puts "#{website.url}: it might contain a phishing kit (#{website.zip_files.join(',')}).".colorize(:light_red) + post_to_slack(message) if options[:post_to_slack] && valid_slack_setting? + end end end diff --git a/lib/miteru/crawler.rb b/lib/miteru/crawler.rb index 124f0be..0ad08f2 100644 --- a/lib/miteru/crawler.rb +++ b/lib/miteru/crawler.rb @@ -16,25 +16,24 @@ def suspicous_urls res["results"].map { |result| result.dig("task", "url") } end - def execute + def execute(verbose = false) pool = Thread.pool(threads) - results = [] + websites = [] suspicous_urls.each do |url| pool.process do - doc = Website.new(url) - results << url if doc.has_kit? - rescue HTTPResponseError => _ - next + website = Website.new(url) + puts "#{website.url}: it doesn't contain a phishing kit." if verbose && !website.has_kit? + websites << website end end pool.shutdown - results + websites end - def self.execute - new.execute + def self.execute(verbose = false) + new.execute(verbose) end private diff --git a/lib/miteru/website.rb b/lib/miteru/website.rb index e46acc7..f49071f 100644 --- a/lib/miteru/website.rb +++ b/lib/miteru/website.rb @@ -8,6 +8,7 @@ class Website attr_reader :url def initialize(url) @url = url + build end def title @@ -21,6 +22,10 @@ def zip_files end.compact end + def ok? + response.code == 200 + end + def index? title == "Index of /" end @@ -30,20 +35,27 @@ def zip_files? end def has_kit? - index? && zip_files? + ok? && index? && zip_files? + end + + def build + doc end private - def get - res = HTTP.get(url) - raise HTTPResponseError if res.code != 200 + def response + @response ||= get + end - res.body.to_s + def get + ctx = OpenSSL::SSL::SSLContext.new + ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE + HTTP.get(url, ssl_context: ctx) end def doc - @doc ||= Oga.parse_html(get) + @doc ||= Oga.parse_html(response.body.to_s) end end end diff --git a/miteru.gemspec b/miteru.gemspec index 3b356e9..0c57f3b 100644 --- a/miteru.gemspec +++ b/miteru.gemspec @@ -26,11 +26,13 @@ Gem::Specification.new do |spec| spec.add_development_dependency "bundler", "~> 1.16" spec.add_development_dependency "coveralls", "~> 0.8" + spec.add_development_dependency "glint", "~> 0.1" spec.add_development_dependency "rake", "~> 12.3" spec.add_development_dependency "rspec", "~> 3.8" spec.add_development_dependency "vcr", "~> 4.0" spec.add_development_dependency "webmock", "~> 3.4" + spec.add_dependency "colorize", "~> 0.8" spec.add_dependency "http", "~> 3.3" spec.add_dependency "oga", "~> 2.15" spec.add_dependency "thor", "~> 0.19" diff --git a/spec/crawler_spec.rb b/spec/crawler_spec.rb index d17f9f4..a932784 100644 --- a/spec/crawler_spec.rb +++ b/spec/crawler_spec.rb @@ -2,6 +2,7 @@ # frozen_string_literal: true RSpec.describe Miteru::Crawler, :vcr do + include_context "http_server" subject { Miteru::Crawler.new } describe "#suspicous_urls" do it "should return an Array" do @@ -12,8 +13,7 @@ end describe "#execute" do before do - allow_any_instance_of(Miteru::Crawler).to receive(:suspicous_urls).and_return(%w(http://localhost)) - allow_any_instance_of(Miteru::Website).to receive(:has_kit?).and_return(true) + allow_any_instance_of(Miteru::Crawler).to receive(:suspicous_urls).and_return(["http://#{host}:#{port}/has_kit"]) end it "should return an Array" do results = subject.execute diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index f6ecd41..f1ebda1 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -5,12 +5,11 @@ require 'coveralls' Coveralls.wear! +require "glint" require "miteru" - require "vcr" require "webrick" - RSpec.configure do |config| # Enable flags like --only-failures and --next-failure config.example_status_persistence_file_path = ".rspec_status" @@ -35,6 +34,55 @@ def capture(stream) end end +def server + server = Glint::Server.new do |port| + http = WEBrick::HTTPServer.new( + BindAddress: "0.0.0.0", + Port: port, + Logger: WEBrick::Log.new(File.open(File::NULL, "w")), + AccessLog: [] + ) + http.mount_proc("/has_kit") do |_, res| + path = File.expand_path("./fixtures/index.html", __dir__) + body = File.read(path) + + res.status = 200 + res.content_length = body.size + res.content_type = 'text/plain' + res.body = body + end + + http.mount_proc("/no_kit") do |_, res| + body = "None" + + res.status = 200 + res.content_length = body.size + res.content_type = 'text/plain' + res.body = body + end + + trap(:INT) { http.shutdown } + trap(:TERM) { http.shutdown } + http.start + end + Glint::Server.info[:http_server] = { + host: "0.0.0.0", + port: server.port + } + server +end + +RSpec.shared_context "http_server" do + before(:all) { + @server = server + @server.start + } + after(:all) { @server.stop } + + let(:host) { "0.0.0.0" } + let(:port) { @server.port } +end + VCR.configure do |config| config.cassette_library_dir = "spec/fixtures/vcr_cassettes" config.configure_rspec_metadata! diff --git a/spec/website_spec.rb b/spec/website_spec.rb index a75654b..f0a75da 100644 --- a/spec/website_spec.rb +++ b/spec/website_spec.rb @@ -1,26 +1,18 @@ # frozen_string_literal: true -RSpec.describe Miteru::Website, :vcr do +RSpec.describe Miteru::Website do + include_context "http_server" subject { Miteru::Website } describe "#has_kit?" do context "when giving a url which contains a phishint kit" do - before do - path = File.expand_path("./fixtures/index.html", __dir__) - data = File.read(path) - - allow_any_instance_of(Miteru::Website).to receive(:get).and_return(data) - end it "should return true" do - expect(subject.new("http://localhost").has_kit?).to eq(true) + expect(subject.new("http://#{host}:#{port}/has_kit").has_kit?).to eq(true) end end end context "when giving a url which doesn't contain a phishint kit" do - before do - allow_any_instance_of(Miteru::Website).to receive(:get).and_return("None") - end it "should return false" do - expect(subject.new("http://localhost").has_kit?).to eq(false) + expect(subject.new("http://#{host}:#{port}/no_kit").has_kit?).to eq(false) end end end