Implement GeoIP lookup support

* From configuration options it is possible to choice what level of privacy the
  prober is willing to accept.

ooni/config.py

@@ -29,6 +29,11 @@ def get_root_path():
 for k, v in configuration['basic'].items():
     basic[k] = v
 
+# Process the privacy configuration options
+privacy = Storage()
+for k, v in configuration['privacy'].items():
+    privacy[k] = v
+
 # Process the advanced configuration options
 advanced = Storage()
 for k, v in configuration['advanced'].items():

ooni/reporter.py

@@ -186,33 +186,35 @@ def writeHeader(self):
         self._writeln("###########################################")
 
         client_geodata = {}
-        log.msg("Running geo IP lookup via check.torproject.org")
 
-        client_ip = yield geodata.myIP()
-        if config.includeip:
+        if config.privacy.includeip or \
+                config.privacy.includeasn or \
+                config.privacy.includecountry or \
+                config.privacy.includecity:
+            log.msg("Running geo IP lookup via check.torproject.org")
+            client_ip = yield geodata.myIP()
+            client_location = geodata.IPToLocation(client_ip)
+        else:
+            client_ip = "127.0.0.1"
+
+        if config.privacy.includeip:
             client_geodata['ip'] = client_ip
         else:
-            client_geodata['ip'] = None
+            client_geodata['ip'] = "127.0.0.1"
 
         client_geodata['asn'] = None
         client_geodata['city'] = None
         client_geodata['countrycode'] = None
 
-        try:
-            import txtorcon
-            client_location = txtorcon.util.NetLocation(client_geodata['ip'])
-            if config.includeasn:
-                client_geodata['asn'] = client_location.asn
+        if config.privacy.includeasn:
+            client_geodata['asn'] = client_location['asn']
 
-            if config.includecity:
-                client_geodata['city'] = client_location.city
+        if config.privacy.includecity:
+            client_geodata['city'] = client_location['city']
 
-            if config.includecountry:
-                client_geodata['countrycode'] = client_location.countrycode
+        if config.privacy.includecountry:
+            client_geodata['countrycode'] = client_location['countrycode']
 
-        except ImportError:
-            log.err("txtorcon is not installed. Geolocation lookup is not"\
-                    "supported")
 
         test_details = {'start_time': repr(date.now()),
                         'probe_asn': client_geodata['asn'],

ooni/utils/geodata.py

@@ -1,7 +1,9 @@
 import re
 import pygeoip
+import os
 
 from ooni import config
+from ooni.utils import log
 
 from twisted.web.client import Agent
 from twisted.internet import reactor, defer, protocol
@@ -38,3 +40,28 @@ def myIP():
 
     defer.returnValue(myip)
 
+class GeoIPDataFilesNotFound(Exception):
+    pass
+
+def IPToLocation(ipaddr):
+    city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat')
+    country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat')
+    asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat')
+
+    location = {'city': None, 'countrycode': None, 'asn': None}
+    try:
+        city_dat = pygeoip.GeoIP(city_file)
+        location['city'] = city_dat.record_by_addr(ipaddr)['city']
+
+        country_dat = pygeoip.GeoIP(country_file)
+        location['countrycode'] = country_dat.country_code_by_addr(ipaddr)
+
+        asn_dat = pygeoip.GeoIP(asn_file)
+        location['asn'] = asn_dat.org_by_addr(ipaddr)
+    except IOError:
+        log.err("Could not find GeoIP data files. Go into data/ "
+                "and run make geoip")
+        raise GeoIPDataFilesNotFound
+
+    return location
+

ooni/utils/log.py

@@ -11,13 +11,13 @@
 from twisted.python.logfile import DailyLogFile
 
 from ooni.utils import otime
-from ooni import oconfig
+from ooni import config
 
 def start(logfile=None):
     daily_logfile = None
 
     if not logfile:
-        logfile = oconfig.basic.logfile
+        logfile = config.basic.logfile
 
     log_folder = os.path.dirname(logfile)
     log_filename = os.path.basename(logfile)
@@ -29,7 +29,7 @@ def start(logfile=None):
     logging.basicConfig()
     python_logging = txlog.PythonLoggingObserver()
 
-    if oconfig.advanced.debug:
+    if config.advanced.debug:
         python_logging.logger.setLevel(logging.DEBUG)
     else:
         python_logging.logger.setLevel(logging.INFO)

ooniprobe.conf

@@ -5,15 +5,21 @@
 basic:
     # Where OONIProbe should be writing it's log file
     logfile: /tmp/ooniprobe.log
+
+privacy:
     # Should we include the IP address of the probe in the report?
-    includeip: true
+    includeip: false
     # Should we include the ASN of the probe in the report?
-    includeasn: true
+    includeasn: false
     # Should we include the ASN of the probe in the report?
-    includecountry: true
+    includecountry: false
     # Should we include the ASN of the probe in the report?
-    includecity: true
+    includecity: false
+
 advanced:
-    geoip_data_dir: /home/x/code/networking/ooni-probe/data
+    # XXX change this to point to the directory where you have stored the GeoIP
+    # database file. This should be the directory in which OONI is installed
+    # /path/to/ooni-probe/data/
+    geoip_data_dir: /home/x/code/networking/ooni-probe/data/
     debug: true