Add global server IP blacklist against scrapers. (#1770)

* Added global IP blacklist (against scrapers) to webserver.

* Added --disable-blacklist argument.

* Changed dottedQuadToNum type.

* Added logging.

* Updated format.

pogom/app.py

@@ -12,19 +12,39 @@
 from pogom.utils import get_args
 from datetime import timedelta
 from collections import OrderedDict
+from bisect import bisect_left
 
 from . import config
 from .models import (Pokemon, Gym, Pokestop, ScannedLocation,
                      MainWorker, WorkerStatus)
-from .utils import now
+from .utils import now, dottedQuadToNum, get_blacklist
 log = logging.getLogger(__name__)
 compress = Compress()
 
 
 class Pogom(Flask):
+
     def __init__(self, import_name, **kwargs):
         super(Pogom, self).__init__(import_name, **kwargs)
         compress.init_app(self)
+
+        args = get_args()
+
+        # Global blist
+        if not args.disable_blacklist:
+            log.info('Retrieving blacklist...')
+            self.blacklist = get_blacklist()
+            # Sort & index for binary search
+            self.blacklist.sort(key=lambda r: r[0])
+            self.blacklist_keys = [
+                dottedQuadToNum(r[0]) for r in self.blacklist
+            ]
+        else:
+            log.info('Blacklist disabled for this session.')
+            self.blacklist = []
+            self.blacklist_keys = []
+
+        # Routes
         self.json_encoder = CustomJSONEncoder
         self.route("/", methods=['GET'])(self.fullmap)
         self.route("/raw_data", methods=['GET'])(self.raw_data)
@@ -39,6 +59,24 @@ def __init__(self, import_name, **kwargs):
         self.route("/status", methods=['POST'])(self.post_status)
         self.route("/gym_data", methods=['GET'])(self.get_gymdata)
 
+    def validate_request(self):
+        if self._ip_is_blacklisted(request.remote_addr):
+            log.debug('Denied access to %s.', request.remote_addr)
+            abort(403)
+
+    def _ip_is_blacklisted(self, ip):
+        if not self.blacklist:
+            return False
+
+        # Get the nearest IP range
+        pos = max(bisect_left(self.blacklist_keys, ip) - 1, 0)
+        ip_range = self.blacklist[pos]
+
+        start = dottedQuadToNum(ip_range[0])
+        end = dottedQuadToNum(ip_range[1])
+
+        return start <= dottedQuadToNum(ip) <= end
+
     def set_search_control(self, control):
         self.search_control = control
 

pogom/utils.py

@@ -11,6 +11,9 @@
 import pprint
 import time
 import random
+import socket
+import struct
+import requests
 from uuid import uuid4
 from s2sphere import CellId, LatLng
 
@@ -353,6 +356,9 @@ def get_args():
                         help=('Pause searching while web UI is inactive ' +
                               'for this timeout(in seconds).'),
                         type=int, default=0)
+    parser.add_argument('--disable-blacklist',
+                        help=('Disable the global anti-scraper IP blacklist.'),
+                        action='store_true', default=False)
     verbosity = parser.add_mutually_exclusive_group()
     verbosity.add_argument('-v', '--verbose',
                            help=('Show debug messages from PokemonGo-Map ' +
@@ -878,6 +884,22 @@ def complete_tutorial(api, account, tutorial_state):
     return True
 
 
+
+def dottedQuadToNum(ip):
+    return struct.unpack("!L", socket.inet_aton(ip))[0]
+
+
+def get_blacklist():
+    try:
+        url = 'https://blist.devkat.org/blacklist.json'
+        blacklist = requests.get(url).json()
+        log.debug('Entries in blacklist: %s.', len(blacklist))
+        return blacklist
+    except (requests.exceptions.RequestException, IndexError, KeyError):
+        log.error('Unable to retrieve blacklist, setting to empty.')
+        return []
+
+
 # Generate random device info.
 # Original by Noctem.
 IPHONES = {'iPhone5,1': 'N41AP',

runserver.py

@@ -222,6 +222,8 @@ def main():
     config['CHINA'] = args.china
 
     app = Pogom(__name__)
+    app.before_request(app.validate_request)
+
     db = init_database(app)
     if args.clear_db:
         log.info('Clearing database')