Added test of simple commands and table integrity, bug fixes

openwpm · May 4, 2016 · 4cc559f · 4cc559f
1 parent 39e31ed
commit 4cc559f
Show file tree

Hide file tree

Showing 6 changed files with 52 additions and 24 deletions.
diff --git a/automation/Commands/browser_commands.py b/automation/Commands/browser_commands.py
@@ -147,7 +147,7 @@ def browse_website(url, num_links, sleep, visit_id, webdriver, proxy_queue,
     calls get_website before visiting <num_links> present on the page
     """
     # First get the site
-    get_website(url, visit_id, sleep, webdriver, proxy_queue, browser_params, extension_socket)
+    get_website(url, sleep, visit_id, webdriver, proxy_queue, browser_params, extension_socket)
 
     # Connect to logger
     logger = loggingclient(*manager_params['logger_address'])

diff --git a/automation/Commands/command_executor.py b/automation/Commands/command_executor.py
@@ -20,17 +20,22 @@ def execute_command(command, webdriver, proxy_queue, browser_settings, browser_p
                                         manager_params=manager_params, extension_socket=extension_socket)
 
     if command[0] == 'DUMP_FLASH_COOKIES':
-        browser_commands.dump_flash_cookies(command[1], command[2], webdriver,
-                                              browser_params, manager_params)
+        browser_commands.dump_flash_cookies(start_time=command[1], visit_id=command[2],
+                                            webdriver=webdriver, browser_params=browser_params,
+                                            manager_params=manager_params)
 
     if command[0] == 'DUMP_PROFILE_COOKIES':
-        browser_commands.dump_profile_cookies(command[1], command[2], webdriver,
-                                              browser_params, manager_params)
+        browser_commands.dump_profile_cookies(start_time=command[1], visit_id=command[2],
+                                              webdriver=webdriver, browser_params=browser_params,
+                                              manager_params=manager_params)
 
     if command[0] == 'DUMP_PROF':
-        profile_commands.dump_profile(browser_params['profile_path'], manager_params,
-                                      browser_params, command[1], command[2], webdriver,
-                                      browser_settings, compress=command[3],
+        profile_commands.dump_profile(browser_profile_folder=browser_params['profile_path'],
+                                      manager_params=manager_params,
+                                      browser_params=browser_params,
+                                      tar_location=command[1], close_webdriver=command[2],
+                                      webdriver=webdriver, browser_settings=browser_settings,
+                                      compress=command[3],
                                       save_flash=browser_params['disable_flash'] is False)
 
     if command[0] == 'EXTRACT_LINKS':

diff --git a/automation/Extension/firefox/@openwpm-0.0.1.xpi b/automation/Extension/firefox/@openwpm-0.0.1.xpi
diff --git a/demo.py b/demo.py
@@ -2,9 +2,9 @@
 
 # The list of sites that we wish to crawl
 NUM_BROWSERS = 3
-sites = ['example.com',
-         'princeton.edu',
-         'citp.princeton.edu/']
+sites = ['http://www.example.com',
+         'http://www.princeton.edu',
+         'http://citp.princeton.edu/']
 
 # Loads the manager preference and 3 copies of the default browser dictionaries
 manager_params, browser_params = TaskManager.load_default_params(NUM_BROWSERS)

diff --git a/test/test_storage_vectors.py b/test/test_storage_vectors.py
@@ -5,14 +5,22 @@
 from ..automation import CommandSequence
 from ..automation import TaskManager
 
-expected_lso_content = [
+expected_lso_content_a = [
                1, # visit id
                u'localtest.me',
                u'FlashCookie.sol',
                u'localtest.me/FlashCookie.sol',
                u'test_key',
                u'REPLACEME']
 
+expected_lso_content_b = [
+               2, # visit id
+               u'localtest.me',
+               u'FlashCookie.sol',
+               u'localtest.me/FlashCookie.sol',
+               u'test_key',
+               u'REPLACEME']
+
 expected_js_cookie = (
              1, # visit id
              u'%s' % utilities.BASE_TEST_URL_DOMAIN,
@@ -40,34 +48,49 @@ def get_config(self, data_dir):
         return manager_params, browser_params
 
     def test_flash_cookies(self, tmpdir):
-        """ Check that some Flash LSOs are saved """
+        """ Check that some Flash LSOs are saved and
+        are properly keyed in db."""
         # Run the test crawl
         manager_params, browser_params = self.get_config(str(tmpdir))
         browser_params[0]['disable_flash'] = False
         manager = TaskManager.TaskManager(manager_params, browser_params)
 
-        # Get a site we know sets Flash cookies
-        lso_value = utilities.rand_str(8)
-        expected_lso_content[5] = lso_value  # we'll expect this to be present
+        # Get a site we know sets Flash cookies and visit it twice
+        lso_value_a = utilities.rand_str(8)
+        expected_lso_content_a[5] = lso_value_a  # we'll expect this to be present
         qry_str = '?lso_test_key=%s&lso_test_value=%s' % ("test_key",
-                                                          lso_value)
-        test_url = utilities.BASE_TEST_URL + '/lso/setlso.html' + qry_str
-        cs = CommandSequence.CommandSequence(test_url)
+                                                          lso_value_a)
+        test_url_a = utilities.BASE_TEST_URL + '/lso/setlso.html' + qry_str
+        cs = CommandSequence.CommandSequence(test_url_a)
         cs.get(sleep=3, timeout=120)
         cs.dump_flash_cookies()
         manager.execute_command_sequence(cs)
+
+        lso_value_b = utilities.rand_str(8)
+        expected_lso_content_b[5] = lso_value_b  # we'll expect this to be present
+        qry_str = '?lso_test_key=%s&lso_test_value=%s' % ("test_key",
+                                                          lso_value_b)
+        test_url_b = utilities.BASE_TEST_URL + '/lso/setlso.html' + qry_str
+        cs = CommandSequence.CommandSequence(test_url_b)
+        cs.get(sleep=3, timeout=120)
+        cs.dump_flash_cookies()
+        manager.execute_command_sequence(cs)
+
         manager.close(post_process=False)
 
         #  Check that some flash cookies are recorded
         qry_res = utilities.query_db(manager_params['db'],
                                      "SELECT * FROM flash_cookies")
         lso_count = len(qry_res)
-        assert lso_count == 1
-        lso_content = list(qry_res[0][2:])  # Remove first two items
+        assert lso_count == 2
+        lso_content_a = list(qry_res[0][2:])  # Remove first two items
+        lso_content_b = list(qry_res[1][2:])  # Remove first two items
         # remove randomly generated LSO directory name
         # e.g. TY2FOJUG/localtest.me/Flash.sol -> localtest.me/Flash.sol
-        lso_content[3] = lso_content[3].split("/", 1)[-1]  # remove LSO dirname
-        assert lso_content == expected_lso_content
+        lso_content_a[3] = lso_content_a[3].split("/", 1)[-1]  # remove LSO dirname
+        lso_content_b[3] = lso_content_b[3].split("/", 1)[-1]  # remove LSO dirname
+        assert lso_content_a == expected_lso_content_a
+        assert lso_content_b == expected_lso_content_b
 
     def test_profile_cookies(self, tmpdir):
         """ Check that some profile cookies are saved """

diff --git a/test/utilities.py b/test/utilities.py
@@ -6,7 +6,7 @@
 import os
 import sqlite3
 from random import choice
-LOCAL_WEBSERVER_PORT = 9000
+LOCAL_WEBSERVER_PORT = 8000
 PSL_CACHE_LOC = '/tmp/public_suffix_list.dat'
 BASE_TEST_URL_DOMAIN = "localtest.me"
 BASE_TEST_URL_NOPATH = "http://%s:%s" % (BASE_TEST_URL_DOMAIN,