Refactoring of geoip related functionality to have it less coupled

ooni/director.py

@@ -127,7 +127,6 @@ def start(self):
             log.msg("Connecting to Tor Control Port...")
             yield self.getTorState()
 
-        config.probe_ip = geoip.ProbeIP()
         yield config.probe_ip.lookup()
 
     @property
@@ -222,7 +221,7 @@ def startNetTest(self, net_test_loader, reporters):
         """
         if config.privacy.includepcap:
             if not config.reports.pcap:
-                config.reports.pcap = config.generatePcapFilename(net_test_loader.testDetails)
+                config.reports.pcap = config.generate_pcap_filename(net_test_loader.testDetails)
             self.startSniffing()
 
         report = Report(reporters, self.reportEntryManager)

ooni/geoip.py

@@ -5,11 +5,9 @@
 from twisted.web import client, http_headers
 client._HTTP11ClientFactory.noisy = False
 
-from ooni.utils.net import userAgents, BodyReceiver
 from twisted.internet import reactor, defer, protocol
 
 from ooni.utils import log, net, checkForRoot
-from ooni.settings import config
 from ooni import errors
 
 try:
@@ -26,6 +24,8 @@ class GeoIPDataFilesNotFound(Exception):
     pass
 
 def IPToLocation(ipaddr):
+    from ooni.settings import config
+
     city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat')
     country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat')
     asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat')
@@ -67,6 +67,8 @@ def __init__(self):
         self.agent = self._agent(reactor)
 
     def _response(self, response):
+        from ooni.utils.net import BodyReceiver
+
         content_length = response.headers.getRawHeaders('content-length')
 
         finished = defer.Deferred()
@@ -88,6 +90,8 @@ def failed(self, failure):
         return failure
 
     def lookup(self):
+        from ooni.utils.net import userAgents
+
         headers = {}
         headers['User-Agent'] = [random.choice(userAgents)]
 
@@ -117,16 +121,37 @@ class ProbeIP(object):
     address = None
 
     def __init__(self):
-        self.tor_state = config.tor_state
-        self.geoIPServices = {'ubuntu': UbuntuGeoIP,
+        self.geoIPServices = {
+            'ubuntu': UbuntuGeoIP,
             'torproject': TorProjectGeoIP
         }
+        self.geodata = {
+            'asn': 'AS0',
+            'city': None,
+            'countrycode': 'ZZ',
+            'ip': '127.0.0.1'
+        }
+
+    def resolveGeodata(self):
+        from ooni.settings import config
+
+        self.geodata = IPToLocation(self.address)
+        self.geodata['ip'] = self.address
+        if not config.privacy.includeasn:
+            self.geodata['asn'] = 'AS0'
+        if not config.privacy.includecity:
+            self.geodata['city'] = None
+        if not config.privacy.includecountry:
+            self.geodata['countrycode'] = 'ZZ'
+        if not config.privacy.includeip:
+            self.geodata['ip'] = '127.0.0.1'
 
     @defer.inlineCallbacks
     def lookup(self):
         try:
             yield self.askTor()
             log.msg("Found your IP via Tor %s" % self.address)
+            self.resolveGeodata()
             defer.returnValue(self.address)
         except errors.TorStateNotFound:
             log.debug("Tor is not running. Skipping IP lookup via Tor.")
@@ -136,6 +161,7 @@ def lookup(self):
         try:
             yield self.askTraceroute()
             log.msg("Found your IP via Traceroute %s" % self.address)
+            self.resolveGeodata()
             defer.returnValue(self.address)
         except errors.InsufficientPrivileges:
             log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges")
@@ -145,6 +171,7 @@ def lookup(self):
         try:
             yield self.askGeoIPService()
             log.msg("Found your IP via a GeoIP service: %s" % self.address)
+            self.resolveGeodata()
             defer.returnValue(self.address)
         except Exception, e:
             log.msg("Unable to lookup the probe IP via GeoIPService")
@@ -183,8 +210,10 @@ def askTor(self):
         XXX this lookup method is currently broken when there are cached descriptors or consensus documents
         see: https://trac.torproject.org/projects/tor/ticket/8214
         """
-        if self.tor_state:
-            d = self.tor_state.protocol.get_info("address")
+        from ooni.settings import config
+
+        if config.tor_state:
+            d = config.tor_state.protocol.get_info("address")
             @d.addCallback
             def cb(result):
                 self.strategy = 'tor_get_info_address'

ooni/nettest.py

@@ -238,45 +238,15 @@ def inputFiles(self):
     def testDetails(self):
         from ooni import __version__ as software_version
 
-        client_geodata = {
-                'city': None, 
-                'countrycode': 'ZZ',
-                'asn': 'AS0',
-                'ip': '127.0.0.1'
-        }
-
-        if config.probe_ip and config.probe_ip.address and (config.privacy.includeip or \
-                config.privacy.includeasn or \
-                config.privacy.includecountry or \
-                config.privacy.includecity):
-            log.msg("We will include some geo data in the report")
-            client_geodata = geoip.IPToLocation(config.probe_ip.address)
-
-        if config.privacy.includeip:
-            client_geodata['ip'] = config.probe_ip.address
-        else:
-            client_geodata['ip'] = "127.0.0.1"
-
-        # Here we unset all the client geodata if the option to not include then
-        # has been specified
-        if not config.privacy.includeasn:
-            client_geodata['asn'] = 'AS0'
-
-        if not config.privacy.includecity:
-            client_geodata['city'] = None
-
-        if not config.privacy.includecountry:
-            client_geodata['countrycode'] = None
-
         input_file_hashes = []
         for input_file in self.inputFiles:
             input_file_hashes.append(input_file['hash'])
 
         test_details = {'start_time': time.time(),
-            'probe_asn': client_geodata['asn'],
-            'probe_cc': client_geodata['countrycode'],
-            'probe_ip': client_geodata['ip'],
-            'probe_city': client_geodata['city'],
+            'probe_asn': config.probe_ip.geodata['asn'],
+            'probe_cc': config.probe_ip.geodata['countrycode'],
+            'probe_ip': config.probe_ip.geodata['ip'],
+            'probe_city': config.probe_ip.geodata['city'],
             'test_name': self.testName,
             'test_version': self.testVersion,
             'software_name': 'ooniprobe',

ooni/settings.py

@@ -8,7 +8,7 @@
 
 from twisted.internet import reactor, threads, defer
 
-from ooni import otime
+from ooni import otime, geoip
 from ooni.utils import Storage
 
 class OConfig(object):
@@ -19,7 +19,7 @@ def __init__(self):
         self.scapyFactory = None
         self.tor_state = None
         # This is used to store the probes IP address obtained via Tor
-        self.probe_ip = None
+        self.probe_ip = geoip.ProbeIP()
         # This is used to keep track of the state of the sniffer
         self.sniffer_running = None
         self.logging = True
@@ -91,7 +91,7 @@ def read_config_file(self):
                     pass
         self.set_paths()
 
-    def generatePcapFilename(self, testDetails):
+    def generate_pcap_filename(self, testDetails):
         test_name, start_time = testDetails['test_name'], testDetails['start_time']
         start_time = otime.epochToTimestamp(start_time)
         return "report-%s-%s.%s" % (test_name, start_time, "pcap")

ooni/utils/log.py

@@ -9,7 +9,6 @@
 from twisted.python.logfile import DailyLogFile
 
 from ooni import otime
-from ooni.settings import config
 
 ## Get rid of the annoying "No route found for
 ## IPv6 destination warnings":
@@ -26,6 +25,8 @@ def emit(self, eventDict):
 
 class OONILogger(object):
     def start(self, logfile=None, application_name="ooniprobe"):
+        from ooni.settings import config
+
         daily_logfile = None
 
         if not logfile:
@@ -58,14 +59,17 @@ def stop():
     oonilogger.stop()
 
 def msg(msg, *arg, **kw):
+    from ooni.settings import config
     if config.logging:
         print "%s" % msg
 
 def debug(msg, *arg, **kw):
+    from ooni.settings import config
     if config.advanced.debug and config.logging:
         print "[D] %s" % msg
 
 def err(msg, *arg, **kw):
+    from ooni.settings import config
     if config.logging:
         print "[!] %s" % msg