Add support for generating summary results for more tests (#738)

* Add support for generating summary results for more tests

It's now possible to see from the measurement list if the following
tests presents anomalies:
* http_header_field_manipulation
* http_invalid_request_line
* facebook_messenger
* whatsapp
* vanilla_tor

* Add .DS_Store to gitignore

* Improve debug level output

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 *.pyc
 
 ENV/*

ooni/backend_client.py

@@ -120,7 +120,7 @@ def errback(err, attempts):
                 # We we will recursively keep trying to perform a request until
                 # we have reached the retry count.
                 if attempts < retries:
-                    log.err("Lookup failed. Retrying.")
+                    log.err("Lookup {} failed. Retrying.".format(uri))
                     attempts += 1
                     perform_request(attempts)
                 else:

ooni/measurements.py

@@ -15,8 +15,80 @@ class MeasurementTypes():
     supported_tests = [
         "web_connectivity",
         "http_requests",
-        "tcp_connect"
+        "tcp_connect",
+        "http_invalid_request_line",
+        "http_header_field_manipulation",
+        "facebook_messenger",
+        "whatsapp",
+        "vanilla_tor"
     ]
+
+    @staticmethod
+    def vanilla_tor(entry):
+        result = {}
+        result['anomaly'] = False
+        if entry['test_keys'].get('success', None) == False:
+            result['anomaly'] = True
+        return result
+
+    @staticmethod
+    def whatsapp(entry):
+        result = {}
+        result['anomaly'] = False
+        for key in ['registration_server_status', 'whatsapp_endpoints_status', 'whatsapp_web_status']:
+            if entry['test_keys'][key] != 'ok':
+                result['anomaly'] = True
+        return result
+
+    @staticmethod
+    def facebook_messenger(entry):
+        result = {}
+        result['anomaly'] = False
+        true_calc_keys = [
+                'facebook_b_api_dns_consistent',
+                'facebook_b_api_reachable',
+                'facebook_b_graph_dns_consistent',
+                'facebook_b_graph_reachable',
+                'facebook_edge_dns_consistent',
+                'facebook_edge_reachable',
+                'facebook_external_cdn_dns_consistent',
+                'facebook_external_cdn_reachable',
+                'facebook_scontent_cdn_dns_consistent',
+                'facebook_scontent_cdn_reachable',
+                'facebook_star_dns_consistent',
+                'facebook_star_reachable',
+                'facebook_stun_dns_consistent',
+                # facebook_stun_reachable',
+                ]
+        false_calc_keys = [
+                'facebook_tcp_blocking',
+                'facebook_dns_blocking'
+                ]
+        for key in false_calc_keys:
+            if entry['test_keys'][key] == True:
+                result['anomaly'] = True
+        for key in true_calc_keys:
+            if entry['test_keys'][key] == False:
+                result['anomaly'] = True
+        return result
+
+    @staticmethod
+    def http_invalid_request_line(entry):
+        result = {}
+        result['anomaly'] = False
+        if entry['test_keys']['tampering'] == True:
+            result['anomaly'] = True
+        return result
+
+    @staticmethod
+    def http_header_field_manipulation(entry):
+        result = {}
+        result['anomaly'] = False
+        for t in entry['test_keys'].get('tampering', {}).values():
+            if t == True:
+                result['anomaly'] = True
+        return result
+
     @staticmethod
     def web_connectivity(entry):
         result = {}