Implement test deck system for automating the specification of comman…

…d line arguments for tests

* Change before_i_commit script to use test deck system

before_i_commit.sh

@@ -14,15 +14,7 @@ rm before_i_commit.log
 
 find . -type f -name "*.py[co]" -delete
 
-./bin/ooniprobe -l before_i_commit.log -o http_url_lists.yamloo nettests/core/http_url_list.py -f test_inputs/url_lists_file.txt
-
-./bin/ooniprobe -l before_i_commit.log -o dns_tamper_test.yamloo nettests/core/dnstamper.py -T test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/dns_tamper_file.txt
-
-./bin/ooniprobe -l before_i_commit.log -o captive_portal_test.yamloo nettests/core/captiveportal.py
-
-./bin/ooniprobe -l before_i_commit.log -o http_host.yamloo nettests/core/http_host.py -b http://ooni.nu/test -f test_inputs/http_host_file.txt
-
-./bin/ooniprobe -l before_i_commit.log -o http_keyword_filtering.yamloo nettests/core/http_keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt
+./bin/ooniprobe -i before_i_commit.testdeck
 
 echo "Below you should not see anything"
 echo "---------------------------------"
@@ -32,5 +24,6 @@ echo "If you do, it means something is wrong."
 echo "Read through the log file and fix it."
 echo "If you are having some problems fixing some things that have to do with"
 echo "the core of OONI, let's first discuss it on IRC, or open a ticket"
-less *yamloo
+read
+cat *yamloo | less
 rm -f *yamloo

before_i_commit.testdeck

@@ -0,0 +1,40 @@
+- options:
+    collector: null
+    help: 0
+    logfile: before_i_commit.log
+    pcapfile: null
+    reportfile: captive_portal_test.yamloo
+    subargs: []
+    test: nettests/core/captiveportal.py
+- options:
+    collector: null
+    help: 0
+    logfile: before_i_commit.log
+    pcapfile: null
+    reportfile: dns_tamper_test.yamloo
+    subargs: [-T, test_inputs/dns_tamper_test_resolvers.txt, -f, test_inputs/dns_tamper_file.txt]
+    test: nettests/core/dnstamper.py
+- options:
+    collector: null
+    help: 0
+    logfile: before_i_commit.log
+    pcapfile: null
+    reportfile: http_host.yamloo
+    subargs: [-b, 'http://ooni.nu/test', -f, test_inputs/http_host_file.txt]
+    test: nettests/core/http_host.py
+- options:
+    collector: null
+    help: 0
+    logfile: before_i_commit.log
+    pcapfile: null
+    reportfile: http_keyword_filtering.yamloo
+    subargs: [-b, 'http://ooni.nu/test/', -f, test_inputs/keyword_filtering_file.txt]
+    test: nettests/core/http_keyword_filtering.py
+- options:
+    collector: null
+    help: 0
+    logfile: before_i_commit.log
+    pcapfile: null
+    reportfile: http_url_lists.yamloo
+    subargs: [-f, test_inputs/url_lists_file.txt]
+    test: nettests/core/http_url_list.py

ooni/oonicli.py

@@ -13,6 +13,7 @@
 import os
 import random
 import time
+import yaml
 
 from twisted.internet import defer, reactor
 from twisted.application import app
@@ -27,20 +28,20 @@
 from ooni.utils import checkForRoot, NotRootError
 from ooni.utils import log
 
-class Options(usage.Options, app.ReactorSelectionMixin):
+class Options(usage.Options):
     synopsis = """%s [options] [path to test].py
     """ % (os.path.basename(sys.argv[0]),)
 
     longdesc = ("ooniprobe loads and executes a suite or a set of suites of"
                 " network tests. These are loaded from modules, packages and"
                 " files listed on the command line")
 
-    optFlags = [["help", "h"],
-                ['debug-stacktraces', 'B',
-                    'Report deferred creation and callback stack traces'],]
+    optFlags = [["help", "h"]]
 
     optParameters = [["reportfile", "o", None, "report file name"],
-                     ["collector", "c", None, 
+                     ["testdeck", "i", None,
+                         "Specify as input a test deck: a yaml file containig the tests to run an their arguments"],
+                     ["collector", "c", None,
                          "Address of the collector of test results. (example: http://127.0.0.1:8888)"],
                      ["logfile", "l", None, "log file name"],
                      ["pcapfile", "p", None, "pcap file name"]]
@@ -68,6 +69,8 @@ def opt_spew(self):
         sys.settrace(spewer)
 
     def parseArgs(self, *args):
+        if self['testdeck']:
+            return
         try:
             self['test'] = args[0]
             self['subargs'] = args[1:]
@@ -79,22 +82,9 @@ def testsEnded(*arg, **kw):
     You can place here all the post shutdown tasks.
     """
     log.debug("testsEnded: Finished running all tests")
+    reactor.stop()
 
-def run():
-    """
-    Call me to begin testing from a file.
-    """
-    cmd_line_options = Options()
-    if len(sys.argv) == 1:
-        cmd_line_options.getUsage()
-    try:
-        cmd_line_options.parseOptions()
-    except usage.UsageError, ue:
-        raise SystemExit, "%s: %s" % (sys.argv[0], ue)
-
-    if cmd_line_options['debug-stacktraces']:
-        defer.setDebugging(True)
-
+def runTest(cmd_line_options):
     config.cmd_line_options = cmd_line_options
     config.generateReportFilenames()
 
@@ -120,10 +110,37 @@ def run():
         print "Starting sniffer"
         net.capturePackets(config.reports.pcap)
 
-    log.start(cmd_line_options['logfile'])
+    return runner.runTestCases(test_cases, options, cmd_line_options)
 
-    tests_d = runner.runTestCases(test_cases, options, cmd_line_options)
-    tests_d.addBoth(testsEnded)
+def run():
+    """
+    Call me to begin testing from a file.
+    """
+    cmd_line_options = Options()
+    if len(sys.argv) == 1:
+        cmd_line_options.getUsage()
+    try:
+        cmd_line_options.parseOptions()
+    except usage.UsageError, ue:
+        raise SystemExit, "%s: %s" % (sys.argv[0], ue)
 
-    reactor.run()
+    deck_dl = []
 
+    log.start(cmd_line_options['logfile'])
+    if cmd_line_options['testdeck']:
+        test_deck = yaml.safe_load(open(cmd_line_options['testdeck']))
+        for test in test_deck:
+            del cmd_line_options
+            cmd_line_options = test['options']
+            d1 = runTest(cmd_line_options)
+            deck_dl.append(d1)
+    else:
+        log.msg("No test deck detected")
+        del cmd_line_options['testdeck']
+        d1 = runTest(cmd_line_options)
+        deck_dl.append(d1)
+
+    d2 = defer.DeferredList(deck_dl)
+    d2.addCallback(testsEnded)
+
+    reactor.run()

ooni/reporter.py

@@ -197,10 +197,8 @@ def allDone(self):
         log.debug("allDone: Finished running all tests")
         try:
             log.debug("Stopping the reactor")
-            reactor.stop()
         except:
             log.debug("Unable to stop the reactor")
-            pass
         return None
 
 class YAMLReporter(OReporter):

ooni/runner.py

@@ -262,7 +262,6 @@ def runTestCases(test_cases, options, cmd_line_options):
         yield oreporter.createReport(options)
     except reporter.OONIBReportCreationFailed:
         log.err("Error in creating new report")
-        reactor.stop()
         raise
     except Exception, e:
         log.exception(e)
@@ -280,6 +279,5 @@ def runTestCases(test_cases, options, cmd_line_options):
 
     except Exception:
         log.exception("Problem in running test")
-        reactor.stop()
     oreporter.allDone()