Merge pull request #55 from ninoseki/add-st-feed

Add SecurityTrails feed

Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 ruby "~> 2.6"
@@ -14,6 +16,7 @@ gem "puma"
 gem "redis"
 gem "rollbar"
 gem "rubyzip"
+gem "securitytrails"
 gem "simpleidn"
 gem "sinatra"
 

Gemfile.lock

@@ -74,6 +74,7 @@ GEM
       ast
     rubyzip (1.2.3)
     safe_yaml (1.0.5)
+    securitytrails (0.2.0)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
@@ -122,6 +123,7 @@ DEPENDENCIES
   rollbar
   rspec
   rubyzip
+  securitytrails
   simpleidn
   sinatra
   vcr

bin/securitytrails_job.rb

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift("#{__dir__}/../lib")
+
+require "ayashige"
+
+Ayashige::Jobs::SecurityTrails.new.perform

lib/ayashige.rb

@@ -13,12 +13,14 @@
 require "ayashige/sources/source"
 require "ayashige/sources/ct"
 require "ayashige/sources/domain_watch"
+require "ayashige/sources/securitytrails"
 require "ayashige/sources/web_analyzer"
 require "ayashige/sources/whoisds"
 
 require "ayashige/jobs/job"
 require "ayashige/jobs/ct"
 require "ayashige/jobs/domain_watch"
+require "ayashige/jobs/securitytrails"
 require "ayashige/jobs/web_analyzer"
 require "ayashige/jobs/whoisds"
 

lib/ayashige/jobs/securitytrails.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Ayashige
+  module Jobs
+    class SecurityTrails < Job
+      def initialize
+        @source = Ayashige::Sources::SecurityTrails.new
+      end
+    end
+  end
+end

lib/ayashige/sources/securitytrails.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "date"
+require "securitytrails"
+
+module Ayashige
+  module Sources
+    class SecurityTrails < Source
+      def initialize
+        super
+
+        @api = ::SecurityTrails::API.new
+      end
+
+      def store_newly_registered_domains
+        records.each { |record| store record }
+      end
+
+      def records
+        domains = @api.feeds.domains("new")
+        date = DateTime.now.to_s
+        domains.map do |domain|
+          Record.new( domain_name: domain, updated: date)
+        end
+      end
+    end
+  end
+end

spec/jobs/securitytrails_spec.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+RSpec.describe Ayashige::Jobs::SecurityTrails, :vcr do
+  include_context "job context"
+  subject { Ayashige::Jobs::SecurityTrails.new }
+end

spec/sources/securitytrails_spec.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "mock_redis"
+
+RSpec.describe Ayashige::Sources::SecurityTrails do
+  subject { described_class.new }
+
+  let(:redis) { MockRedis.new }
+
+  before do
+    allow(Ayashige::Redis).to receive(:client).and_return(redis)
+  end
+
+  after do
+    redis.flushdb
+  end
+
+  describe "#get_records" do
+    before do
+      mock = double("Feeds API")
+      allow(mock).to receive(:domains).and_return(["example.com"])
+      allow(SecurityTrails::Clients::Feeds).to receive(:new).and_return(mock)
+    end
+
+    it do
+      records = subject.records
+      records.each do |record|
+        expect(record.domain).to be_a(Ayashige::Domain)
+        expect(record.updated_on).to be_a(String)
+      end
+    end
+  end
+
+  describe "#store_newly_registered_domains" do
+    before do
+      allow(subject).to receive(:records).and_return(records)
+    end
+
+    include_examples "#store_newly_registered_domains example"
+  end
+end

spec/support/shared_contexts/job_context.rb

@@ -9,19 +9,20 @@
         subject.instance_variable_set(:@source, mock)
       end
 
-      it "should return the latest indexed date as a String" do
+      it "returns the latest indexed date as a String" do
         output = capture(:stdout){ subject.perform }
         expect(output.empty?).to eq(true)
       end
     end
+
     context "when an error is raised" do
       before do
         mock = double
         allow(mock).to receive(:store_newly_registered_domains).and_raise(ArgumentError, "argument error")
         subject.instance_variable_set(:@source, mock)
       end
 
-      it "should return the latest indexed date as a String" do
+      it "returns the latest indexed date as a String" do
         output = capture(:stdout) { subject.perform }
         expect(output).to include("argument error")
       end

spec/support/shared_examples/source_example.rb

@@ -3,8 +3,10 @@
 RSpec.shared_examples "#store_newly_registered_domains example" do
   let(:updated_on) { "2018-01-01" }
   let(:domain_name) { "paypal.pay.pay.world" }
+  let(:record) { Ayashige::Record.new(updated: updated_on, domain_name: "paypal.pay.pay.world") }
+  let(:records) { [record] }
 
-  it "should store parsed domains into Redis" do
+  it "stores parsed domains into Redis" do
     output = capture(:stdout) { subject.store_newly_registered_domains }
     expect(output.include?(domain_name)).to eq(true)
     expect(redis.exists(domain_name)).to eq(true)