Merge pull request #347 from kudrom/feature/deckgen

Review of deckgen branch

ooni/deckgen/cli.py

@@ -5,7 +5,7 @@
 
 import yaml
 
-from twisted.internet import defer, reactor
+from twisted.internet import defer
 from twisted.python import usage
 
 from ooni.geoip import ProbeIP
@@ -36,7 +36,7 @@ def opt_version(self):
         sys.exit(0)
 
 
-class Deck():
+class Deck(object):
     _base_entry = {
         "options": {
             "collector": None,
@@ -105,7 +105,8 @@ def generate_deck(options):
     deck.add_test('manipulation/http_invalid_request_line')
     deck.add_test('manipulation/http_header_field_manipulation')
     # deck.add_test('manipulation/traceroute')
-    deck.pprint()
+    if config.advanced.debug:
+        deck.pprint()
     deck_filename = os.path.join(options['output'],
                                  "%s-%s-user.deck" % (__version__,
                                                       options['country-code']))
@@ -163,11 +164,3 @@ def run():
             raise
 
     generate_deck(options)
-
-if __name__ == "__main__":
-    d = run()
-
-    @d.addBoth
-    def cb(_):
-        reactor.stop()
-    reactor.start()

ooni/deckgen/processors/citizenlab_test_lists.py

@@ -2,6 +2,17 @@
 import csv
 from ooni.settings import config
 
+
+def load_input(file_input, file_output):
+    fw = open(file_output, "w+")
+    with open(file_input) as f:
+        csvreader = csv.reader(f)
+        csvreader.next()
+        for row in csvreader:
+            fw.write("%s\n" % row[0])
+    fw.close()
+
+
 def generate_country_input(country_code, dst):
     """
     Write to dst/citizenlab-urls-{country_code}.txt
@@ -14,7 +25,6 @@ def generate_country_input(country_code, dst):
 
     country_code = country_code.lower()
     filename = os.path.join(dst, "citizenlab-urls-%s.txt" % country_code)
-    fw = open(filename, "w+")
 
     input_list = os.path.join(config.resources_directory,
                               "citizenlab-test-lists",
@@ -24,30 +34,19 @@ def generate_country_input(country_code, dst):
     if not os.path.exists(input_list):
         raise Exception("Could not find list for country %s" % country_code)
 
-    with open(input_list) as f:
-        csvreader = csv.reader(f)
-        csvreader.next()
-        for row in csvreader:
-            fw.write("%s\n" % row[0])
+    load_input(input_list, filename)
 
-    fw.close()
     return filename
 
 
 def generate_global_input(dst):
-
     filename = os.path.join(dst, "citizenlab-urls-global.txt")
-    fw = open(filename, "w+")
 
     input_list = os.path.join(config.resources_directory,
                               "citizenlab-test-lists",
                               "test-lists-master",
                               "csv", "global.csv")
-    with open(input_list) as f:
-        csvreader = csv.reader(f)
-        csvreader.next()
-        for row in csvreader:
-            fw.write("%s\n" % row[0])
 
-    fw.close()
+    load_input(input_list, filename)
+
     return filename

ooni/deckgen/processors/namebench_dns_servers.py

@@ -14,7 +14,6 @@ def __init__(self):
         self.__dict__ = self._borg
         if not self.country:
             try:
-                print config.advanced.geoip_data_dir
                 country_file = os.path.join(config.advanced.geoip_data_dir,
                                             'GeoIP.dat')
                 self.country = GeoIP.open(country_file,

ooni/resources/cli.py

@@ -4,7 +4,6 @@
 from twisted.python import usage
 
 from ooni.utils import log
-from ooni.settings import config
 
 from ooni.resources import __version__
 from ooni.resources import update
@@ -26,7 +25,6 @@ def opt_version(self):
 
 @defer.inlineCallbacks
 def run():
-    config.read_config_file()
     options = Options()
     try:
         options.parseOptions()

ooni/resources/update.py

@@ -13,6 +13,15 @@ def download_resource(resources):
         print "Downloading %s" % filename
 
         filename = os.path.join(config.resources_directory, filename)
+        if not os.path.exists(filename):
+            directory = os.path.dirname(filename)
+            if not os.path.isdir(directory):
+                os.makedirs(directory)
+            f = open(filename, 'w')
+            f.close()
+        elif not os.path.isfile(filename):
+            print "[!] %s must be a file." % filename
+            defer.returnValue(False)
         yield downloadPage(resource['url'], filename)
 
         if resource['action'] is not None:

ooni/utils/__init__.py

@@ -135,6 +135,7 @@ def generate_filename(testDetails, prefix=None, extension=None, filename=None):
 
     return final_filename
 
+
 def sanitize_options(options):
     """
     Strips all possible user identifying information from the ooniprobe test
@@ -147,8 +148,8 @@ def sanitize_options(options):
         sanitized_options.append(option)
     return sanitized_options
 
-def unzip(filename, dst):
 
+def unzip(filename, dst):
     assert filename.endswith('.zip')
     dst_path = os.path.join(
         dst,
@@ -159,6 +160,7 @@ def unzip(filename, dst):
         zip_file.extractall(dst_path)
     return dst_path
 
+
 def gunzip(filename, dst):
     assert filename.endswith(".gz")
     dst_path = os.path.join(