chore: change default number of threads

Change default number of threads to Parallel.processor_count(Number of processors seen by the OS)

lib/miteru/cli.rb

@@ -9,7 +9,7 @@ class CLI < Thor
     method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download file(s)"
     method_option :post_to_slack, type: :boolean, default: false, desc: "Post a message to Slack if it detects a phishing kit"
     method_option :size, type: :numeric, default: 100, desc: "Number of urlscan.io's results. (Max: 10,000)"
-    method_option :threads, type: :numeric, default: 10, desc: "Number of threads to use"
+    method_option :threads, type: :numeric, desc: "Number of threads to use"
     method_option :verbose, type: :boolean, default: true
     desc "execute", "Execute the crawler"
     def execute

lib/miteru/crawler.rb

@@ -6,15 +6,14 @@
 
 module Miteru
   class Crawler
-    attr_reader :auto_download
     attr_reader :directory_traveling
     attr_reader :downloader
     attr_reader :feeds
     attr_reader :size
     attr_reader :threads
     attr_reader :verbose
 
-    def initialize(auto_download: false, directory_traveling: false, download_to: "/tmp", post_to_slack: false, size: 100, threads: 10, verbose: false)
+    def initialize(auto_download: false, directory_traveling: false, download_to: "/tmp", post_to_slack: false, size: 100, threads: Parallel.processor_count, verbose: false)
       @auto_download = auto_download
       @directory_traveling = directory_traveling
       @downloader = Downloader.new(download_to)
@@ -27,18 +26,18 @@ def initialize(auto_download: false, directory_traveling: false, download_to: "/
     end
 
     def execute
-      puts "Loaded #{feeds.suspicious_urls.length} URLs to crawl." if verbose
+      puts "Loaded #{feeds.suspicious_urls.length} URLs to crawl. (crawling in #{threads} threads)" if verbose
 
       Parallel.each(feeds.suspicious_urls, in_threads: threads) do |url|
         website = Website.new(url)
         downloader.download_kits(website.kits) if website.has_kits? && auto_download?
         notify(website) if verbose || website.has_kits?
-      rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _
+      rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _e
         next
       end
     end
 
-    def self.execute(auto_download: false, directory_traveling: false, download_to: "/tmp", post_to_slack: false, size: 100, threads: 10, verbose: false)
+    def self.execute(auto_download: false, directory_traveling: false, download_to: "/tmp", post_to_slack: false, size: 100, threads: Parallel.processor_count, verbose: false)
       new(
         auto_download: auto_download,
         directory_traveling: directory_traveling,

spec/crawler_spec.rb

@@ -4,8 +4,6 @@
   include_context "http_server"
   include_context "download_kits"
 
-  subject { described_class }
-
   before { allow(ENV).to receive(:[]).with("SLACK_WEBHOOK_URL").and_return(nil) }
 
   describe ".execute" do
@@ -14,8 +12,14 @@
       allow(Parallel).to receive(:processor_count).and_return(0)
     end
 
-    it "does not raise any error" do
-      capture(:stdout) { expect { subject.execute }.not_to raise_error }
+    it do
+      capture(:stdout) { expect { described_class.execute }.not_to raise_error }
+    end
+  end
+
+  describe "#threads" do
+    it do
+      expect(described_class.new.threads).to eq(Parallel.processor_count)
     end
   end
 end