Add support for ignoring IPs by ASN in riemann-http

Monitoring a website behind a Content Delivery Network (CDN) may lead to
flapping metrics when the short-lived IP addresses where the service is
accessible change.

Allow to provide a list of Autonomous System Numbers (ASN) that we can
ignore for well-known CDN service providers.  Use the MaxMind ASN
database provided by the user for IP lookups.

This is not a hard dependency as no ASN filtering is done by default, so
only add this dependency for testing and assume the end-user will handle
the soft requirement on his own if they want to do filter-out some ASN.

.gitignore

@@ -6,3 +6,4 @@ pkg/
 .*.swp
 *.log
 lib/riemann/tools/*_parser.tab.rb
+spec/fixtures/test-asn/test-asn

Gemfile

@@ -6,6 +6,7 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'github_changelog_generator'
+gem 'maxmind-geoip2'
 gem 'racc'
 gem 'rake'
 gem 'rspec'

lib/riemann/tools/http_check.rb

@@ -32,6 +32,8 @@ class HttpCheck
       opt :resolvers, 'Run this number of resolver threads', short: :none, type: :integer, default: 5
       opt :workers, 'Run this number of worker threads', short: :none, type: :integer, default: 20
       opt :user_agent, 'User-Agent header for HTTP requests', short: :none, default: "#{File.basename($PROGRAM_NAME)}/#{Riemann::Tools::VERSION} (+https://github.com/riemann/riemann-tools)"
+      opt :ignored_asn, 'Ignore addresses belonging to these ASN', short: :none, type: :integers, default: []
+      opt :geoip_asn_database, 'Path to the GeoIP ASN database', short: :none, default: '/usr/share/GeoIP/GeoLite2-ASN.mmdb'
 
       def initialize
         super
@@ -60,6 +62,14 @@ def initialize
                 end
               end
 
+              if opts[:ignored_asn].any?
+                addresses.reject! do |address|
+                  address_belongs_to_ignored_asn?(address)
+                end
+              end
+
+              next if addresses.empty?
+
               @work_queue.push([uri, addresses])
             end
           end
@@ -77,6 +87,21 @@ def initialize
         end
       end
 
+      def address_belongs_to_ignored_asn?(address)
+        begin
+          require 'maxmind/geoip2'
+        rescue LoadError
+          raise StandardError, 'MaxMind::GeoIP2 is not available. Please install the maxmind-geoip2 gem for filtering by ASN.'
+        end
+
+        @reader ||= MaxMind::GeoIP2::Reader.new(database: opts[:geoip_asn_database])
+        asn = @reader.asn(address.to_s)
+
+        opts[:ignored_asn].include?(asn&.autonomous_system_number)
+      rescue MaxMind::GeoIP2::AddressNotFoundError
+        false
+      end
+
       # Under normal operation, we have a single instance of this class for the
       # lifetime of the process.  But when testing, we create a new instance
       # for each test, each with its resolvers and worker threads.  The test

spec/fixtures/test-asn/GeoLite2-ASN-Blocks-IPv4.csv

@@ -0,0 +1,4 @@
+network,autonomous_system_number,autonomous_system_organization
+1.1.1.0/24,64512,FOO
+2.2.2.0/24,64513,BAR
+3.3.3.0/24,64514,BAZ

spec/fixtures/test-asn/GeoLite2-ASN-Blocks-IPv6.csv

@@ -0,0 +1,4 @@
+network,autonomous_system_number,autonomous_system_organization
+2001:1::/20,64512,FOO
+2001:2::/20,64513,BAR
+2001:3::/20,64514,BAZ

spec/fixtures/test-asn/README.md

@@ -0,0 +1,19 @@
+# test-asn
+
+This is a copy of the asn-writer example from [MaxMind's `mmdbwriter` repository](https://github.com/maxmind/mmdbwriter), with some tooling to build the `test-asn.mmdb` file from the `GeoLite2-ASN-Blocks-IPv4.csv` and `GeoLite2-ASN-Blocks-IPv6.csv` files.
+
+## Usage
+
+Adjsut the `.cvs` files, then (re)generate `test-asn.mmdb` with:
+
+```sh
+go get
+go build
+./test-asn
+```
+
+## Note
+
+The `mmdbwriter` code does not allow to use private neworks nor networks reserved for documentation.
+The test ASN database therefore contains (obviously incorrect) information about *real* networks.
+It goes without saying, but I will still say it: do not use this database for anything else than testing the riemann-tools.

spec/fixtures/test-asn/go.mod

@@ -0,0 +1,11 @@
+module test-asn
+
+go 1.21
+
+require github.com/maxmind/mmdbwriter v1.0.0
+
+require (
+	github.com/oschwald/maxminddb-golang v1.12.0 // indirect
+	go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d // indirect
+	golang.org/x/sys v0.10.0 // indirect
+)

spec/fixtures/test-asn/go.sum

@@ -0,0 +1,8 @@
+github.com/maxmind/mmdbwriter v1.0.0 h1:bieL4P6yaYaHvbtLSwnKtEvScUKKD6jcKaLiTM3WSMw=
+github.com/maxmind/mmdbwriter v1.0.0/go.mod h1:noBMCUtyN5PUQ4H8ikkOvGSHhzhLok51fON2hcrpKj8=
+github.com/oschwald/maxminddb-golang v1.12.0 h1:9FnTOD0YOhP7DGxGsq4glzpGy5+w7pq50AS6wALUMYs=
+github.com/oschwald/maxminddb-golang v1.12.0/go.mod h1:q0Nob5lTCqyQ8WT6FYgS1L7PXKVVbgiymefNwIjPzgY=
+go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d h1:ggxwEf5eu0l8v+87VhX1czFh8zJul3hK16Gmruxn7hw=
+go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d/go.mod h1:tgPU4N2u9RByaTN3NC2p9xOzyFpte4jYwsIIRF7XlSc=
+golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

spec/fixtures/test-asn/main.go

@@ -0,0 +1,88 @@
+// asn-writer is an example of how to create an ASN MaxMind DB file from the
+// GeoLite2 ASN CSVs. You must have the CSVs in the current working directory.
+package main
+
+import (
+	"encoding/csv"
+	"io"
+	"log"
+	"net"
+	"os"
+	"strconv"
+
+	"github.com/maxmind/mmdbwriter"
+	"github.com/maxmind/mmdbwriter/mmdbtype"
+)
+
+func main() {
+	writer, err := mmdbwriter.New(
+		mmdbwriter.Options{
+			DatabaseType: "GeoLite2-ASN",
+			RecordSize:   24,
+		},
+	)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, file := range []string{"GeoLite2-ASN-Blocks-IPv4.csv", "GeoLite2-ASN-Blocks-IPv6.csv"} {
+		fh, err := os.Open(file)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		r := csv.NewReader(fh)
+
+		// first line
+		r.Read()
+
+		for {
+			row, err := r.Read()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			if len(row) != 3 {
+				log.Fatalf("unexpected CSV rows: %v", row)
+			}
+
+			_, network, err := net.ParseCIDR(row[0])
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			asn, err := strconv.Atoi(row[1])
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			record := mmdbtype.Map{}
+
+			if asn != 0 {
+				record["autonomous_system_number"] = mmdbtype.Uint32(asn)
+			}
+
+			if row[2] != "" {
+				record["autonomous_system_organization"] = mmdbtype.String(row[2])
+			}
+
+			err = writer.Insert(network, record)
+			if err != nil {
+				log.Fatal(err)
+			}
+		}
+	}
+
+	fh, err := os.Create("test-asn.mmdb")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	_, err = writer.WriteTo(fh)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

spec/riemann/tools/http_check_spec.rb

@@ -300,4 +300,28 @@ def protected!
       it { is_expected.to have_received(:report).with(hash_including({ service: 'get https://invalid.example.com/ consistency', state: 'critical', description: 'Could not get any response from invalid.example.com' })) }
     end
   end
+
+  describe '#address_belongs_to_ignored_asn?' do
+    subject { described_class.new.address_belongs_to_ignored_asn?(address) }
+
+    before { ARGV.replace(['--geoip-asn-database', 'spec/fixtures/test-asn/test-asn.mmdb', '--ignored-asn', '64512', '64514']) }
+
+    context 'when the address does not belong to an ignored ASN' do
+      let(:address) { IPAddr.new('1.1.1.2') }
+
+      it { is_expected.to be_truthy }
+    end
+
+    context 'when the address belongs to an ignored ASN' do
+      let(:address) { IPAddr.new('2.2.2.1') }
+
+      it { is_expected.to be_falsey }
+    end
+
+    context 'when the address is unknown' do
+      let(:address) { IPAddr.new('4.4.4.1') }
+
+      it { is_expected.to be_falsey }
+    end
+  end
 end