Skip to content
Permalink
Browse files

Use GeoIP2 database

  • Loading branch information...
ayeowch committed Aug 7, 2018
1 parent 7cd08e7 commit a9be4af2e28016701b7d1c5f7bbcc22d5eb48da5
Showing with 58 additions and 57 deletions.
  1. +2 −0 .gitignore
  2. +12 −9 crawl.py
  3. +3 −13 geoip/update.sh
  4. +1 −1 requirements.txt
  5. +40 −34 resolve.py
@@ -8,4 +8,6 @@
/tests/
clean.sh
geoip/*.dat
geoip/*.mmdb
geoip/*.txt
rsync.sh
@@ -31,11 +31,11 @@
from gevent import monkey
monkey.patch_all()

import geoip2.database
import gevent
import json
import logging
import os
import pygeoip
import redis
import redis.connection
import requests
@@ -46,6 +46,7 @@
from binascii import hexlify, unhexlify
from collections import Counter
from ConfigParser import ConfigParser
from geoip2.errors import AddressNotFoundError
from ipaddress import ip_address, ip_network

from protocol import (
@@ -63,8 +64,7 @@
CONF = {}

# MaxMind databases
ASN4 = pygeoip.GeoIP("geoip/GeoIPASNum.dat", pygeoip.MEMORY_CACHE)
ASN6 = pygeoip.GeoIP("geoip/GeoIPASNumv6.dat", pygeoip.MEMORY_CACHE)
ASN = geoip2.database.Reader("geoip/GeoLite2-ASN.mmdb")


def enumerate_node(redis_pipe, addr_msgs, now):
@@ -363,11 +363,16 @@ def is_excluded(address):
if ":" in address:
address_family = socket.AF_INET6
key = 'exclude_ipv6_networks'
asn_record = ASN6.org_by_addr(address)
else:
address_family = socket.AF_INET
key = 'exclude_ipv4_networks'
asn_record = ASN4.org_by_addr(address)

try:
asn_record = ASN.asn(address)
except AddressNotFoundError:
asn = None
else:
asn = 'AS{}'.format(asn_record.autonomous_system_number)

try:
addr = int(hexlify(socket.inet_pton(address_family, address)), 16)
@@ -378,10 +383,8 @@ def is_excluded(address):
if any([(addr & net[1] == net[0]) for net in CONF[key]]):
return True

if asn_record:
asn = asn_record.split(" ", 1)[0]
if asn in CONF['exclude_asns']:
return True
if asn and asn in CONF['exclude_asns']:
return True

return False

@@ -4,16 +4,6 @@ USER_AGENT="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (K

cd $DIR

wget --user-agent="$USER_AGENT" http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
wget --user-agent="$USER_AGENT" http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz
wget --user-agent="$USER_AGENT" http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz
wget --user-agent="$USER_AGENT" http://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz
wget --user-agent="$USER_AGENT" http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz
wget --user-agent="$USER_AGENT" http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNumv6.dat.gz

gzip -f -d GeoIP.dat.gz
gzip -f -d GeoIPv6.dat.gz
gzip -f -d GeoLiteCity.dat.gz
gzip -f -d GeoLiteCityv6.dat.gz
gzip -f -d GeoIPASNum.dat.gz
gzip -f -d GeoIPASNumv6.dat.gz
wget --user-agent="$USER_AGENT" http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz && tar --strip-components=1 -zxf GeoLite2-City.tar.gz && rm GeoLite2-City.tar.gz
wget --user-agent="$USER_AGENT" http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz && tar --strip-components=1 -zxf GeoLite2-Country.tar.gz && rm GeoLite2-Country.tar.gz
wget --user-agent="$USER_AGENT" http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz && tar --strip-components=1 -zxf GeoLite2-ASN.tar.gz && rm GeoLite2-ASN.tar.gz
@@ -1,7 +1,7 @@
dpkt==1.9.1
geoip2==2.9.0
gevent==1.3.4
ipaddress==1.0.22
pygeoip==0.3.2
PySocks==1.6.8
redis==2.10.6
requests==2.19.1
@@ -31,11 +31,11 @@
from gevent import monkey
monkey.patch_all()

import geoip2.database
import gevent
import gevent.pool
import logging
import os
import pygeoip
import redis
import redis.connection
import socket
@@ -45,6 +45,7 @@
from collections import defaultdict
from ConfigParser import ConfigParser
from decimal import Decimal
from geoip2.errors import AddressNotFoundError

from utils import new_redis_conn

@@ -54,10 +55,9 @@
CONF = {}

# MaxMind databases
GEOIP4 = pygeoip.GeoIP("geoip/GeoLiteCity.dat", pygeoip.MEMORY_CACHE)
GEOIP6 = pygeoip.GeoIP("geoip/GeoLiteCityv6.dat", pygeoip.MEMORY_CACHE)
ASN4 = pygeoip.GeoIP("geoip/GeoIPASNum.dat", pygeoip.MEMORY_CACHE)
ASN6 = pygeoip.GeoIP("geoip/GeoIPASNumv6.dat", pygeoip.MEMORY_CACHE)
GEOIP_CITY = geoip2.database.Reader("geoip/GeoLite2-City.mmdb")
GEOIP_COUNTRY = geoip2.database.Reader("geoip/GeoLite2-Country.mmdb")
ASN = geoip2.database.Reader("geoip/GeoLite2-ASN.mmdb")


class Resolve(object):
@@ -170,43 +170,49 @@ def raw_geoip(address):
"""
Resolves GeoIP data for the specified address using MaxMind databases.
"""
city = None
country = None
latitude = 0.0
longitude = 0.0
city = None
lat = 0.0
lng = 0.0
timezone = None
asn = None
org = None

geoip_record = None
prec = Decimal('.000001')

if not address.endswith(".onion"):
try:
gcountry = GEOIP_COUNTRY.country(address)
except AddressNotFoundError:
pass
else:
country = gcountry.country.iso_code

try:
gcity = GEOIP_CITY.city(address)
except AddressNotFoundError:
pass
else:
city = gcity.city.name
if gcity.location.latitude is not None and \
gcity.location.longitude is not None:
lat = float(Decimal(gcity.location.latitude).quantize(prec))
lng = float(Decimal(gcity.location.longitude).quantize(prec))
timezone = gcity.location.time_zone

if address.endswith(".onion"):
geoip_record = None
elif ":" in address:
geoip_record = GEOIP6.record_by_addr(address)
else:
geoip_record = GEOIP4.record_by_addr(address)
if geoip_record:
city = geoip_record['city']
country = geoip_record['country_code']
latitude = float(Decimal(geoip_record['latitude']).quantize(prec))
longitude = float(Decimal(geoip_record['longitude']).quantize(prec))
timezone = geoip_record['time_zone']

asn_record = None
if address.endswith(".onion"):
asn_record = "TOR Tor network"
elif ":" in address:
asn_record = ASN6.org_by_addr(address)
asn = "TOR"
org = "Tor network"
else:
asn_record = ASN4.org_by_addr(address)
if asn_record:
data = asn_record.split(" ", 1)
asn = data[0]
if len(data) > 1:
org = data[1]

return (city, country, latitude, longitude, timezone, asn, org)
try:
asn_record = ASN.asn(address)
except AddressNotFoundError:
pass
else:
asn = 'AS{}'.format(asn_record.autonomous_system_number)
org = asn_record.autonomous_system_organization

return (city, country, lat, lng, timezone, asn, org)


def init_conf(argv):

0 comments on commit a9be4af

Please sign in to comment.
You can’t perform that action at this time.