Merge pull request #210 from readthedocs/davidfischer/block-ips

IP Geolocation and Proxy detection improvements

.gitignore

@@ -83,5 +83,4 @@ db.sqlite3
 _build
 
 # GeoIP data files
-geoip/*.mmdb
-geoip/*.lock
+geoip/*

Makefile

@@ -1,8 +1,14 @@
-.PHONY: help test clean dockerbuild dockerserve dockershell dockerprod geoip
+.PHONY: help test clean dockerbuild dockerserve dockershell dockerprod geoip ipproxy
+
 
-GEOIP_UPDATE = geoipupdate
 GEOIP_DIR = geoip
-GEOIP_CONF_FILE = $(GEOIP_DIR)/GeoIP.conf
+GEOIP_CITY_FILE = dbip-city-lite.mmdb.gz
+GEOIP_COUNTRY_FILE = dbip-country-lite.mmdb.gz
+GEOIP_CITY_DB_URL = https://download.db-ip.com/free/dbip-city-lite-2020-07.mmdb.gz
+GEOIP_COUNTRY_DB_URL = https://download.db-ip.com/free/dbip-country-lite-2020-07.mmdb.gz
+
+TOR_EXIT_LIST_FILE = torbulkexitlist.txt
+TOR_EXIT_LIST_URL = https://check.torproject.org/torbulkexitlist
 
 DOCKER_CONFIG=docker-compose-local.yml
 DOCKER_IMAGE_NAME=ethicaladserver
@@ -17,7 +23,8 @@ help:
 	@echo "  dockershell    Connect to a shell on the Django docker container"
 	@echo "  dockerstart    Start all services in the background"
 	@echo "  dockerstop     Stop all services started by dockerstart"
-	@echo "  geoip          Download the GeoIP database from MaxMind"
+	@echo "  geoip          Download the GeoIP databases"
+	@echo "  ipproxy        Download proxy databases"
 
 test:
 	tox
@@ -54,7 +61,12 @@ dockerstop:
 dockershell:
 	docker-compose -f $(DOCKER_CONFIG) run --rm django /bin/ash
 
-# Get the GeoIP database from MaxMind
-# This command will probably fail unless you have "geoipupdate" installed
+# Get the GeoIP databases from DB-IP
 geoip:
-	$(GEOIP_UPDATE) -f $(GEOIP_CONF_FILE) -d $(GEOIP_DIR) --verbose
+	curl -o $(GEOIP_DIR)/$(GEOIP_CITY_FILE) "$(GEOIP_CITY_DB_URL)"
+	curl -o $(GEOIP_DIR)/$(GEOIP_COUNTRY_FILE) "$(GEOIP_COUNTRY_DB_URL)"
+	gunzip $(GEOIP_DIR)/$(GEOIP_CITY_FILE)
+	gunzip $(GEOIP_DIR)/$(GEOIP_COUNTRY_FILE)
+
+ipproxy:
+	curl -o $(GEOIP_DIR)/$(TOR_EXIT_LIST_FILE) "$(TOR_EXIT_LIST_URL)"

README.rst

@@ -31,7 +31,8 @@ Features
   - mobile, tablet, or desktop targeting
   - extensible custom targeting options
 
-The Ethical Ad Server uses GeoLite2 data created by MaxMind.
+The Ethical Ad Server uses GeoLite2 data created by MaxMind
+or IP Geolocation by DB-IP.
 
 
 Documentation

adserver/tests/test_api.py

@@ -995,6 +995,16 @@ def setUp(self):
             "click-proxy", kwargs={"advertisement_id": self.ad.pk, "nonce": self.nonce}
         )
 
+    def tearDown(self):
+        # Reset the UA blocklist
+        adserver_utils.BLOCKLISTED_UA_REGEXES = []
+
+        # Reset the referrer blocklist
+        adserver_utils.BLOCKLISTED_REFERRERS_REGEXES = []
+
+        # Reset the IP blocklist
+        adserver_utils.BLOCKLISTED_IPS = []
+
     def test_view_tracking_valid(self):
         resp = self.client.get(self.url)
 
@@ -1060,9 +1070,6 @@ def test_view_tracking_blocked_ua(self):
         self.assertEqual(resp.status_code, 200)
         self.assertEqual(resp["X-Adserver-Reason"], "Blocked UA impression")
 
-        # Reset the UA blocklist
-        adserver_utils.BLOCKLISTED_UA_REGEXES = []
-
     @override_settings(ADSERVER_BLOCKLISTED_REFERRERS=["http://invalid.referrer"])
     def test_view_tracking_blocked_referrer(self):
         # Override the settings for the blocklist
@@ -1076,8 +1083,13 @@ def test_view_tracking_blocked_referrer(self):
         self.assertEqual(resp.status_code, 200)
         self.assertEqual(resp["X-Adserver-Reason"], "Blocked referrer impression")
 
-        # Reset the referrer blocklist
-        adserver_utils.BLOCKLISTED_REFERRERS_REGEXES = []
+    def test_view_tracking_blocked_ip(self):
+        adserver_utils.BLOCKLISTED_IPS = set([self.ip_address])
+
+        resp = self.client.get(self.url)
+
+        self.assertEqual(resp.status_code, 200)
+        self.assertEqual(resp["X-Adserver-Reason"], "Blocked IP impression")
 
     def test_view_tracking_invalid_ad(self):
         url = reverse(

adserver/tests/test_utils.py

@@ -20,6 +20,7 @@
 from ..utils import get_client_id
 from ..utils import get_client_user_agent
 from ..utils import get_geolocation
+from ..utils import is_blocklisted_ip
 from ..utils import is_blocklisted_referrer
 from ..utils import is_blocklisted_user_agent
 from ..utils import is_click_ratelimited
@@ -87,6 +88,13 @@ def test_blocklisted_referrer(self):
         regexes = [re.compile("this isn't found"), re.compile("neither is this")]
         self.assertFalse(is_blocklisted_referrer(referrer, regexes))
 
+    def test_blocklisted_ip(self):
+        ip = "1.1.1.1"
+        self.assertFalse(is_blocklisted_ip(ip))
+
+        self.assertTrue(is_blocklisted_ip(ip, ["1.1.1.1", "2.2.2.2"]))
+        self.assertFalse(is_blocklisted_ip(ip, ["2.2.2.2"]))
+
     def test_ratelimited(self):
         factory = RequestFactory()
         request = factory.get("/")

adserver/utils.py

@@ -2,6 +2,7 @@
 import hashlib
 import ipaddress
 import logging
+import os
 import re
 from collections import namedtuple
 from datetime import datetime
@@ -21,13 +22,6 @@
 
 log = logging.getLogger(__name__)  # noqa
 
-# Compile these regular expressions at startup time for performance purposes
-BLOCKLISTED_UA_REGEXES = [
-    re.compile(s) for s in settings.ADSERVER_BLOCKLISTED_USER_AGENTS
-]
-BLOCKLISTED_REFERRERS_REGEXES = [
-    re.compile(s) for s in settings.ADSERVER_BLOCKLISTED_REFERRERS
-]
 
 try:
     geoip = GeoIP2()
@@ -198,6 +192,21 @@ def is_blocklisted_referrer(referrer, blocklist_regexes=None):
     return False
 
 
+def is_blocklisted_ip(ip, blocked_ips=None):
+    """
+    Returns ``True`` if the IP is blocklisted and ``False`` otherwise.
+
+    IPs can be blocked because they are anonymous proxies or other reasons.
+    """
+    if blocked_ips is None:
+        blocked_ips = BLOCKLISTED_IPS
+
+    if ip and ip in blocked_ips:
+        return True
+
+    return False
+
+
 def get_geolocation(ip_address):
     try:
         ipaddress.ip_address(force_text(ip_address))
@@ -216,6 +225,21 @@ def get_geolocation(ip_address):
     return None
 
 
+def build_blocked_ip_set():
+    """Build a set of blocked IPs for preventing bogus ad impressions."""
+    blocked_ips = set()
+
+    filepath = os.path.join(settings.GEOIP_PATH, "torbulkexitlist.txt")
+    if os.path.exists(filepath):
+        with open(filepath) as fd:
+            for line in fd.readlines():
+                line = line.strip()
+                if line:
+                    blocked_ips.add(line)
+
+    return blocked_ips
+
+
 def generate_client_id(ip_address, user_agent):
     """
     Create an advertising ID.
@@ -240,3 +264,13 @@ def generate_client_id(ip_address, user_agent):
         hash_id.update(force_bytes(get_random_string()))
 
     return hash_id.hexdigest()
+
+
+# Compile these regular expressions at startup time for performance purposes
+BLOCKLISTED_UA_REGEXES = [
+    re.compile(s) for s in settings.ADSERVER_BLOCKLISTED_USER_AGENTS
+]
+BLOCKLISTED_REFERRERS_REGEXES = [
+    re.compile(s) for s in settings.ADSERVER_BLOCKLISTED_REFERRERS
+]
+BLOCKLISTED_IPS = build_blocked_ip_set()

adserver/views.py

@@ -55,6 +55,7 @@
 from .utils import get_client_ip
 from .utils import get_client_user_agent
 from .utils import get_geolocation
+from .utils import is_blocklisted_ip
 from .utils import is_blocklisted_referrer
 from .utils import is_blocklisted_user_agent
 from .utils import is_click_ratelimited
@@ -375,6 +376,13 @@ def ignore_tracking_reason(self, request, advertisement, nonce, publisher):
                 user_agent,
             )
             reason = "Blocked referrer impression"
+        elif is_blocklisted_ip(ip_address):
+            log.log(
+                self.log_security_level,
+                "Blocked IP impression, Publisher: [%s]",
+                publisher,
+            )
+            reason = "Blocked IP impression"
         elif not publisher:
             log.log(self.log_level, "Ad impression for unknown publisher")
             reason = "Unknown publisher"