#130 : maxmind for ip import instead of whois.

app/__init__.py

@@ -67,6 +67,8 @@ def run(debug=False, port=5000, host='127.0.0.1'):
     from app.celeryapp import make_celery
     celery = make_celery(app)
 
+    from app.geo_ip_helper import get_geo_for_ip
+
     from app.routes import index
     from app.routes import authentication
     from app.routes import c2ips

app/geo_ip_helper.py

@@ -0,0 +1,45 @@
+import geoip2.database
+import os
+import pyzipcode
+
+from ipaddr import IPAddress
+from app import app
+from app.models import cfg_settings
+
+app.config["GEOIP_ASN_DATABASE_FILE"] = cfg_settings.Cfg_settings.get_private_setting("GEOIP_ASN_DATABASE_FILE")
+if not app.config["GEOIP_ASN_DATABASE_FILE"]:
+    app.config["GEOIP_ASN_DATABASE_FILE"] = os.getenv('GEOIP_ASN_DATABASE_FILE',
+                                                      '/usr/local/ThreatKB/MaxMind/GeoLite2-ASN.mmdb')
+
+app.config["GEOIP_CITY_DATABASE_FILE"] = cfg_settings.Cfg_settings.get_private_setting("GEOIP_CITY_DATABASE_FILE")
+if not app.config["GEOIP_CITY_DATABASE_FILE"]:
+    app.config["GEOIP_CITY_DATABASE_FILE"] = os.getenv('GEOIP_CITY_DATABASE_FILE',
+                                                       '/usr/local/ThreatKB/MaxMind/GeoLite2-City.mmdb')
+
+GEOIP_ASN_DATABASE_FILE = app.config["GEOIP_ASN_DATABASE_FILE"]
+GEOIP_CITY_DATABASE_FILE = app.config["GEOIP_CITY_DATABASE_FILE"]
+ASN_READER = geoip2.database.Reader(GEOIP_ASN_DATABASE_FILE)
+CITY_READER = geoip2.database.Reader(GEOIP_CITY_DATABASE_FILE)
+ZIPCODE_DB = pyzipcode.ZipCodeDatabase()
+
+
+def get_geo_for_ip(ip_address):
+    try:
+        ip_address = IPAddress(ip_address)
+        if not ip_address.is_private:
+            asn_info = ASN_READER.asn(ip_address)
+            city_info = CITY_READER.city(ip_address)
+            zip_code = city_info.postal.code
+            return dict(
+                ip=ip_address,
+                asn=asn_info.autonomous_system_organization,
+                country_code=city_info.country.iso_code,
+                city=city_info.city.names["en"],
+                state=ZIPCODE_DB[zip_code].state,
+                zip_code=zip_code,
+                country=city_info.country.names["en"],
+                continent=city_info.continent.names["en"]
+            )
+        return None
+    except Exception as e:
+        return None

app/models/c2ip.py

@@ -4,6 +4,7 @@
 from sqlalchemy.event import listens_for
 
 from app import db, current_user
+from app.geo_ip_helper import get_geo_for_ip
 from app.models.whitelist import Whitelist
 from app.routes import tags_mapping
 from app.models.comments import Comments
@@ -76,21 +77,14 @@ def to_dict(self):
 
     @classmethod
     def get_c2ip_from_ip(cls, ip):
-        whois = ipwhois.IPWhois(ip).lookup_whois()
+        geo_ip = get_geo_for_ip(str(ip))
 
         c2ip = C2ip()
         c2ip.ip = ip
-        c2ip.asn = whois.get("asn_description", None)
-
-        net = {}
-        for range in whois.get("nets", []):
-            if range["cidr"] == whois["asn_cidr"]:
-                net = range
-                break
-
-        c2ip.country = net.get("country", None)
-        c2ip.city = net.get("city", None)
-        c2ip.state = net.get("state", None)
+        c2ip.asn = geo_ip["asn"]
+        c2ip.country = geo_ip["country_code"]
+        c2ip.city = geo_ip["city"]
+        c2ip.state = geo_ip["state"]
         return c2ip
 
 

migrations/versions/665baa4d3f57_geoip_database_config_settings.py

@@ -0,0 +1,44 @@
+"""GEOIP database config settings
+
+Revision ID: 665baa4d3f57
+Revises: 34d5b6b940a7
+Create Date: 2017-11-12 16:20:53.697870
+
+"""
+import datetime
+from alembic import op
+
+# revision identifiers, used by Alembic.
+from app.models import cfg_settings
+
+revision = '665baa4d3f57'
+down_revision = '34d5b6b940a7'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    date_created = datetime.datetime.now().isoformat()
+    date_modified = datetime.datetime.now().isoformat()
+
+    op.bulk_insert(
+        cfg_settings.Cfg_settings.__table__,
+        [
+            {"key": "GEOIP_ASN_DATABASE_FILE",
+             "value": '/usr/local/ThreatKB/MaxMind/GeoLite2-ASN.mmdb',
+             "public": True,
+             "date_created": date_created,
+             "date_modified": date_modified},
+            {"key": "GEOIP_CITY_DATABASE_FILE",
+             "value": '/usr/local/ThreatKB/MaxMind/GeoLite2-City.mmdb',
+             "public": True,
+             "date_created": date_created,
+             "date_modified": date_modified}
+        ]
+    )
+
+
+def downgrade():
+    keys = ["GEOIP_ASN_DATABASE_FILE", "GEOIP_CITY_DATABASE_FILE"]
+    for key in keys:
+        op.execute("""DELETE from cfg_settings where `key`='%s';""" % key)

requirements.txt

@@ -1,16 +1,20 @@
 alembic==0.9.4
 amqp==2.2.1
+app==0.0.1
 Babel==2.4.0
 bcrypt==3.1.3
 billiard==3.5.0.3
 blinker==1.4
 celery==4.1.0
+certifi==2017.11.5
 cffi==1.10.0
+chardet==3.0.4
 click==6.7
 decorator==4.0.11
 deepdiff==3.3.0
 dnspython==1.15.0
 Flask==0.9
+Flask-Autodoc==0.1.2
 Flask-Babel==0.8
 Flask-Bcrypt==0.7.1
 Flask-Login==0.4.0
@@ -21,31 +25,38 @@ Flask-Script==2.0.5
 Flask-SQLAlchemy==0.16
 Flask-WhooshAlchemy==0.54a0
 Flask-WTF==0.8.4
-Flask-Autodoc==0.1.2
 flup==1.0.2
+geoip2==2.6.0
+idna==2.6
 ipaddr==2.1.11
+ipaddress==1.0.18
 ipwhois==1.0.0
 itsdangerous==0.24
 Jinja2==2.9.6
 jsonpickle==0.9.5
 kombu==4.1.0
 Mako==1.0.7
 MarkupSafe==1.0
+maxminddb==1.3.0
 migrate==0.3.8
 more-itertools==3.2.0
 MySQL-python==1.2.5
 ply==3.10
 pycparser==2.18
+pysqlite==2.8.3
 python-dateutil==2.6.1
 python-editor==1.0.3
 python-openid==2.2.5
 pytz==2017.2
+pyzipcode==1.0
 redis==2.10.6
+requests==2.18.4
 six==1.10.0
 speaklater==1.3
 SQLAlchemy==0.7.9
 sqlalchemy-migrate==0.7.2
 Tempita==0.5.2
+urllib3==1.22
 vine==1.1.4
 Werkzeug==0.12.2
 Whoosh==2.7.4

setup.py

@@ -29,17 +29,21 @@
     install_requires=[
         'alembic==0.9.4',
         'amqp==2.2.1',
+        'app==0.0.1',
         'Babel==2.4.0',
         'bcrypt==3.1.3',
         'billiard==3.5.0.3',
         'blinker==1.4',
         'celery==4.1.0',
+        'certifi==2017.11.5',
         'cffi==1.10.0',
+        'chardet==3.0.4',
         'click==6.7',
         'decorator==4.0.11',
         'deepdiff==3.3.0',
         'dnspython==1.15.0',
         'Flask==0.9',
+        'Flask-Autodoc==0.1.2',
         'Flask-Babel==0.8',
         'Flask-Bcrypt==0.7.1',
         'Flask-Login==0.4.0',
@@ -51,35 +55,42 @@
         'Flask-WhooshAlchemy==0.54a0',
         'Flask-WTF==0.8.4',
         'flup==1.0.2',
+        'geoip2==2.6.0',
+        'idna==2.6',
         'ipaddr==2.1.11',
+        'ipaddress==1.0.18',
         'ipwhois==1.0.0',
         'itsdangerous==0.24',
         'Jinja2==2.9.6',
         'jsonpickle==0.9.5',
         'kombu==4.1.0',
         'Mako==1.0.7',
         'MarkupSafe==1.0',
+        'maxminddb==1.3.0',
         'migrate==0.3.8',
         'more-itertools==3.2.0',
         'MySQL-python==1.2.5',
         'ply==3.10',
         'pycparser==2.18',
+        'pysqlite==2.8.3',
         'python-dateutil==2.6.1',
         'python-editor==1.0.3',
         'python-openid==2.2.5',
         'pytz==2017.2',
+        'pyzipcode==1.0',
         'redis==2.10.6',
+        'requests==2.18.4',
         'six==1.10.0',
         'speaklater==1.3',
         'SQLAlchemy==0.7.9',
         'sqlalchemy-migrate==0.7.2',
         'Tempita==0.5.2',
+        'urllib3==1.22',
         'vine==1.1.4',
         'Werkzeug==0.12.2',
         'Whoosh==2.7.4',
         'WTForms==2.1',
-        'yara-python==3.6.3',
-        'Flask-Autodoc==0.1.2',
+        'yara-python==3.6.3'
     ],
 
     extra_require={
@@ -93,7 +104,6 @@
         'config': ['config.py']
     },
 
-
     entry_points={
         'console_scripts': [
             'hunt = hunting.macro_hunter.cli:main',