v0.1.0

.gitignore

@@ -11,7 +11,7 @@
 /tmp/
 
 # Used by dotenv library to load environment variables.
-# .env
+.env
 
 ## Specific to RubyMotion:
 .dat*
@@ -42,9 +42,11 @@ build-iPhoneSimulator/
 
 # for a library or gem, you might want to ignore these files since the code is
 # intended to run in multiple environments; otherwise, check them in:
-# Gemfile.lock
-# .ruby-version
-# .ruby-gemset
+Gemfile.lock
+.ruby-version
+.ruby-gemset
 
 # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
 .rvmrc
+
+.rspec_status

.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.6.1
+before_install: gem install bundler -v 2.0.1

Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in urlhaus_monitor.gemspec
+gemspec

README.md

@@ -0,0 +1,53 @@
+# URLhaus monitor
+
+[![Build Status](https://travis-ci.org/ninoseki/urlhaus_monitor.svg?branch=master)](https://travis-ci.org/ninoseki/urlhaus_monitor)
+[![Coverage Status](https://coveralls.io/repos/github/ninoseki/urlhaus_monitor/badge.svg?branch=master)](https://coveralls.io/github/ninoseki/urlhaus_monitor?branch=master)
+
+A monitoring tool for [URLhaus](https://urlhaus.abuse.ch/) feeds.
+
+## Features
+
+- URLhaus feeds lookup by AS number, country code and TLD.
+- Slack notification.
+
+## Installation
+
+```bash
+$ gem install urlhaus_monitor
+```
+
+## Usage
+
+```bash
+$ urlhaus_monitor
+Commands:
+  urlhaus_monitor help [COMMAND]               # Describe available commands or one specific command
+  urlhaus_monitor lookup_by_asn [asn]          # lookup by a given asn
+  urlhaus_monitor lookup_by_country [COUNTRY]  # lookup by a given country code
+  urlhaus_monitor lookup_by_tld [tld]          # lookup by a given tld
+```
+
+## Configuration
+
+### Slack
+
+Please set the following environmental variables for enabling Slack notification.
+
+- `SLACK_WEBHOOK_URL`: A Slack webhook URL.
+- `SLACK_CHANNEL_NAME`: A Slask channel name which will be notified.
+
+## Screenshots
+
+```bash
+$ urlhaus_monitor lookup_by_tld jp
+```
+
+![img](./screenshots/example.png)
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+```
+
+```

Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "urlhaus_monitor"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

exe/urlhaus_monitor

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift("#{__dir__}/../lib")
+
+require "urlhaus_monitor"
+
+URLhausMonitor::CLI.start

lib/urlhaus_monitor.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "urlhaus_monitor/version"
+
+require "urlhaus_monitor/entry"
+require "urlhaus_monitor/checker"
+require "urlhaus_monitor/notifier"
+require "urlhaus_monitor/monitor"
+require "urlhaus_monitor/cli"
+
+module URLhausMonitor
+  class Error < StandardError; end
+end

lib/urlhaus_monitor/checker.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "http"
+
+module URLhausMonitor
+  class Checker
+    BASE_URL = "https://urlhaus.abuse.ch"
+    LIMIT = 50
+
+    def lookup_by_country(country)
+      lookup "#{BASE_URL}/feeds/country/#{country}"
+    end
+
+    def lookup_by_tld(tld)
+      lookup "#{BASE_URL}/feeds/tld/#{tld}"
+    end
+
+    def lookup_by_asn(asn)
+      lookup "#{BASE_URL}/feeds/asn/#{asn}"
+    end
+
+    private
+
+    def lookup(url)
+      res = HTTP.get(url)
+      return nil unless res.code == 200
+
+      convert res.body.to_s.lines[0..LIMIT]
+    end
+
+    def convert(lines)
+      [].tap do |entries|
+        lines.each do |line|
+          next if line.start_with? "#"
+
+          entries << Entry.new(line)
+        end
+      end
+    end
+  end
+end

lib/urlhaus_monitor/cli.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "thor"
+
+module URLhausMonitor
+  class CLI < Thor
+    desc "lookup_by_country [COUNTRY]", "lookup by a given country code"
+    def lookup_by_country(country)
+      Monitor.new.lookup_by_country country
+    end
+
+    desc "lookup_by_tld [tld]", "lookup by a given tld"
+    def lookup_by_tld(tld)
+      Monitor.new.lookup_by_tld tld
+    end
+
+    desc "lookup_by_asn [asn]", "lookup by a given asn"
+    def lookup_by_asn(asn)
+      Monitor.new.lookup_by_asn asn
+    end
+  end
+end

lib/urlhaus_monitor/entry.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module URLhausMonitor
+  class Entry
+    attr_reader :date_added
+    attr_reader :url
+    attr_reader :url_status
+    attr_reader :threat
+    attr_reader :host
+    attr_reader :ip_address
+    attr_reader :asnumber
+    attr_reader :country
+
+    def initialize(line)
+      parts = CSV.parse(line.chomp).flatten
+      raise ArgumentError, "#{line} is not valid." unless parts.length == 8
+
+      @date_added = parts.shift
+      @url = parts.shift
+      @url_status = parts.shift
+      @threat = parts.shift
+      @host = parts.shift
+      @ip_address = parts.shift
+      @asnumber = parts.shift
+      @country = parts.shift
+    end
+
+    def defanged_url
+      @defanged_url ||= url.gsub(/\./, "[.]")
+    end
+
+    def defanged_host
+      @defanged_host ||=  host.gsub(/\./, "[.]")
+    end
+
+    def vt_link
+      "https://www.virustotal.com/#/domain/#{host}"
+    end
+
+    def urlhaus_link
+      "https://urlhaus.abuse.ch/host/#{host}/"
+    end
+
+    def title
+      "#{defanged_url} (#{defanged_host} / #{ip_address}) (#{date_added})"
+    end
+
+    def to_attachements
+      [
+        {
+          fallback: "urlhaus link",
+          title: defanged_host,
+          title_link: urlhaus_link,
+          footer: "urlhaus.abuse.ch",
+          footer_icon: "http://www.google.com/s2/favicons?domain=urlhaus.abuse.ch"
+        },
+        {
+          fallback: "vt link",
+          title: defanged_host,
+          title_link: vt_link,
+          footer: "virustotal.com",
+          footer_icon: "http://www.google.com/s2/favicons?domain=virustotal.com"
+        }
+      ]
+    end
+  end
+end

lib/urlhaus_monitor/monitor.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "lightly"
+
+module URLhausMonitor
+  class Monitor
+    attr_reader :checker
+    attr_reader :cache
+
+    def initialize
+      @checker = Checker.new
+      @cache = Lightly.new(dir: "/tmp/urlhause_monitor", life: "180d")
+    end
+
+    def lookup_by_country(country)
+      entries = checker.lookup_by_country(country)
+      process entries
+    end
+
+    def lookup_by_tld(tld)
+      entries = checker.lookup_by_tld(tld)
+      process entries
+    end
+
+    def lookup_by_asn(asn)
+      entries = checker.lookup_by_asn(asn)
+      process entries
+    end
+
+    def process(entries)
+      return nil unless entries
+
+      entries.each do |entry|
+        next if cache.cached? entry.url
+
+        Notifier.notify entry.title, entry.to_attachements
+        cache.save entry.url, true
+      end
+    end
+  end
+end

lib/urlhaus_monitor/notifier.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "slack/incoming/webhooks"
+
+module URLhausMonitor
+  class Notifier
+    def notifiy(title, attachments = [])
+      attachments << { title: "N/A" } if attachments.empty?
+
+      if slack_webhook_url?
+        slack = Slack::Incoming::Webhooks.new(slack_webhook_url, channel: slack_channel)
+        slack.post title, attachments: attachments
+      else
+        puts title
+        attachments.each do |attachment|
+          puts attachment.dig(:title_link)
+        end
+      end
+    end
+
+    def slack_webhook_url
+      ENV.fetch "SLACK_WEBHOOK_URL"
+    end
+
+    def slack_channel
+      ENV.fetch "SLACK_CHANNEL", "#general"
+    end
+
+    def slack_webhook_url?
+      ENV.key? "SLACK_WEBHOOK_URL"
+    end
+
+    def self.notify(title, attachments)
+      new.notifiy(title, attachments)
+    end
+  end
+end

lib/urlhaus_monitor/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module URLhausMonitor
+  VERSION = "0.1.0"
+end