From 567e02d70876817a8469610846d8a5167ad4964b Mon Sep 17 00:00:00 2001 From: Manabu Niseki Date: Sat, 1 Dec 2018 10:36:27 +0900 Subject: [PATCH] chore: split jobs into each task --- README.md | 7 ++--- bin/cron_job.rb | 16 ----------- bin/web_analyzer_job.rb | 7 +++++ bin/whoisds_job.rb | 7 +++++ lib/ayashige.rb | 4 +++ lib/ayashige/jobs/job.rb | 21 +++++++++++++++ lib/ayashige/jobs/web_analyzer.rb | 15 +++++++++++ lib/ayashige/jobs/whoisds.rb | 15 +++++++++++ spec/jobs/web_analyzer_spec.rb | 6 +++++ spec/jobs/whoisds_spec.rb | 6 +++++ spec/spec_helper.rb | 6 +++++ spec/support/shared_contexts/job_context.rb | 30 +++++++++++++++++++++ 12 files changed, 121 insertions(+), 19 deletions(-) delete mode 100644 bin/cron_job.rb create mode 100755 bin/web_analyzer_job.rb create mode 100755 bin/whoisds_job.rb create mode 100644 lib/ayashige/jobs/job.rb create mode 100644 lib/ayashige/jobs/web_analyzer.rb create mode 100644 lib/ayashige/jobs/whoisds.rb create mode 100644 spec/jobs/web_analyzer_spec.rb create mode 100644 spec/jobs/whoisds_spec.rb create mode 100644 spec/support/shared_contexts/job_context.rb diff --git a/README.md b/README.md index 6c6a5be..dc6b3a6 100644 --- a/README.md +++ b/README.md @@ -31,13 +31,14 @@ REDIS_PORT = YOUR_REDIS_PORT REDIS_PASSWORD = YOUR_REDIS_PASSWORD ``` -### Run a Cron job +### Run Cron jobs ```sh -bundle exec ruby bin/cron_job.rb +bundle exec ruby bin/web_analyzer_job.rb +bundle exec ruby bin/whoisds_job.rb ``` -- The job crawls WebAnalyzer websites and parsed the latest registered domains. +- The jobs collects the latest registered domains from WebAnalyzer & WhoisDS. - It checks a suspicious score of a given each domain and stores a suspicious one into a Redis instance. ### Run a Web app diff --git a/bin/cron_job.rb b/bin/cron_job.rb deleted file mode 100644 index 3d91907..0000000 --- a/bin/cron_job.rb +++ /dev/null @@ -1,16 +0,0 @@ -$LOAD_PATH.unshift("#{__dir__}/../lib") - -require "ayashige" - -def with_error_handling - yield -rescue StandardError => e - if Ayashige::Rollbar.available? - Rollbar.error e - else - puts e - end -end - -with_error_handling { Ayashige::Sources::WebAnalyzer.new.store_newly_registered_domains } -with_error_handling { Ayashige::Sources::WhoisDS.new.store_newly_registered_domains } diff --git a/bin/web_analyzer_job.rb b/bin/web_analyzer_job.rb new file mode 100755 index 0000000..e906237 --- /dev/null +++ b/bin/web_analyzer_job.rb @@ -0,0 +1,7 @@ +#!/usr/bin/env ruby + +$LOAD_PATH.unshift("#{__dir__}/../lib") + +require "ayashige" + +Ayashige::Jobs::WebAnalyzer.new.perform diff --git a/bin/whoisds_job.rb b/bin/whoisds_job.rb new file mode 100755 index 0000000..acc0523 --- /dev/null +++ b/bin/whoisds_job.rb @@ -0,0 +1,7 @@ +#!/usr/bin/env ruby + +$LOAD_PATH.unshift("#{__dir__}/../lib") + +require "ayashige" + +Ayashige::Jobs::WhoisDS.new.perform diff --git a/lib/ayashige.rb b/lib/ayashige.rb index 01ce239..724b0f8 100644 --- a/lib/ayashige.rb +++ b/lib/ayashige.rb @@ -13,6 +13,10 @@ require "ayashige/sources/web_analyzer" require "ayashige/sources/whoisds" +require "ayashige/jobs/job" +require "ayashige/jobs/web_analyzer" +require "ayashige/jobs/whoisds" + require "ayashige/rollbar" require "ayashige/application" diff --git a/lib/ayashige/jobs/job.rb b/lib/ayashige/jobs/job.rb new file mode 100644 index 0000000..211c906 --- /dev/null +++ b/lib/ayashige/jobs/job.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Ayashige + module Jobs + class Job + def perform + raise NotImplementedError, "You must implement #{self.class}##{__method__}" + end + + def with_error_handling + yield + rescue StandardError => e + if Ayashige::Rollbar.available? + Rollbar.error e + else + puts e + end + end + end + end +end diff --git a/lib/ayashige/jobs/web_analyzer.rb b/lib/ayashige/jobs/web_analyzer.rb new file mode 100644 index 0000000..b2d65f9 --- /dev/null +++ b/lib/ayashige/jobs/web_analyzer.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Ayashige + module Jobs + class WebAnalyzer < Job + def initialize + @source = Ayashige::Sources::WebAnalyzer.new + end + + def perform + with_error_handling { @source.store_newly_registered_domains } + end + end + end +end diff --git a/lib/ayashige/jobs/whoisds.rb b/lib/ayashige/jobs/whoisds.rb new file mode 100644 index 0000000..422b051 --- /dev/null +++ b/lib/ayashige/jobs/whoisds.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Ayashige + module Jobs + class WhoisDS < Job + def initialize + @source = Ayashige::Sources::WhoisDS.new + end + + def perform + with_error_handling { @source.store_newly_registered_domains } + end + end + end +end diff --git a/spec/jobs/web_analyzer_spec.rb b/spec/jobs/web_analyzer_spec.rb new file mode 100644 index 0000000..19aaaf4 --- /dev/null +++ b/spec/jobs/web_analyzer_spec.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +RSpec.describe Ayashige::Jobs::WebAnalyzer, :vcr do + include_context "job context" + subject { Ayashige::Jobs::WebAnalyzer.new } +end diff --git a/spec/jobs/whoisds_spec.rb b/spec/jobs/whoisds_spec.rb new file mode 100644 index 0000000..8342733 --- /dev/null +++ b/spec/jobs/whoisds_spec.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +RSpec.describe Ayashige::Jobs::WhoisDS, :vcr do + include_context "job context" + subject { Ayashige::Jobs::WhoisDS.new } +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 70b2846..16e75e7 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -27,6 +27,12 @@ config.include Spec::Support::IOHelper end +require_relative "./support/shared_contexts/job_context" + +RSpec.configure do |rspec| + rspec.include_context "job context", include_shared: true +end + VCR.configure do |config| config.cassette_library_dir = "spec/fixtures/vcr_cassettes" config.configure_rspec_metadata! diff --git a/spec/support/shared_contexts/job_context.rb b/spec/support/shared_contexts/job_context.rb new file mode 100644 index 0000000..7751be4 --- /dev/null +++ b/spec/support/shared_contexts/job_context.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +RSpec.shared_context "job context", shared_context: :metadata do + describe "#perform" do + context "when no error is raised" do + before do + mock = double + allow(mock).to receive(:store_newly_registered_domains).and_return([]) + subject.instance_variable_set(:@source, mock) + end + + it "should return the latest indexed date as a String" do + output = capture(:stdout){ subject.perform } + expect(output.empty?).to eq(true) + end + end + context "when an error is raised" do + before do + mock = double + allow(mock).to receive(:store_newly_registered_domains).and_raise(ArgumentError, "argument error") + subject.instance_variable_set(:@source, mock) + end + + it "should return the latest indexed date as a String" do + output = capture(:stdout) { subject.perform } + expect(output).to include("argument error") + end + end + end +end