Downloads data from server using requests over Tor

The test requirements now have requests[socks] as dependency.
Using the same we are now directly downloading the files/messages
from the .onion address for functional tests.

The old external command file also got removed this committ.

We are creating the gpg object for both container based local
testing and external testing (in functional tests).
Fixes: #3691 #3687

securedrop/requirements/test-requirements.in

@@ -7,6 +7,7 @@ py
 pytest
 pytest-cov
 pytest-mock
+requests[socks]
 selenium
 tbselenium
 pyvirtualdisplay

securedrop/requirements/test-requirements.txt

@@ -7,13 +7,16 @@
 attrs==17.4.0             # via pytest
 beautifulsoup4==4.6.0
 blinker==1.4
+certifi==2018.4.16        # via requests
+chardet==3.0.4            # via requests
 click==6.7                # via flask, pip-tools
 coverage==4.4.2           # via pytest-cov
 easyprocess==0.2.3        # via pyvirtualdisplay
 first==2.0.1              # via pip-tools
 flask-testing==0.7.1
 flask==0.12.2             # via flask-testing
 funcsigs==1.0.2           # via mock, pytest
+idna==2.7                 # via requests
 itsdangerous==0.24        # via flask
 jinja2==2.10              # via flask
 markupsafe==1.0           # via jinja2
@@ -22,11 +25,14 @@ pbr==3.1.1                # via mock
 pip-tools==1.11.0
 pluggy==0.6.0             # via pytest
 py==1.5.2
+pysocks==1.6.8            # via requests
 pytest-cov==2.5.1
 pytest-mock==1.7.1
 pytest==3.3.2
 pyvirtualdisplay==0.2.1
+requests[socks]==2.19.1
 selenium==3.13.0
 six==1.11.0               # via mock, pip-tools, pytest
 tbselenium==0.3.3
+urllib3==1.23             # via requests
 werkzeug==0.12.2          # via flask

securedrop/tests/functional/functional_test.py

@@ -202,15 +202,6 @@ def init_gpg(self):
             gpg.import_keys(open(keyfile).read())
         return gpg
 
-    def system(self, cmd):
-        """
-        Invoke a shell command. Primary replacement for os.system calls.
-        """
-        ret = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
-                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                               close_fds=True)
-        out, err = ret.communicate()
-        return out
 
     def setup(self, session_expiration=30):
 
@@ -222,6 +213,9 @@ def setup(self, session_expiration=30):
         instance_information_path = join(FUNCTIONAL_TEST_DIR,
                                          'instance_information.json')
 
+        env.create_directories()
+        self.gpg = env.init_gpg()
+
         if os.path.exists(instance_information_path):
             with open(instance_information_path) as fobj:
                 data = json.load(fobj)
@@ -244,8 +238,6 @@ def setup(self, session_expiration=30):
             self.mock_get_entropy_estimate = self.patcher2.start()
             self.mock_get_entropy_estimate.return_value = 8192
 
-            env.create_directories()
-            self.gpg = env.init_gpg()
             db.create_all()
 
             # Add our test user

securedrop/tests/functional/journalist_navigation_steps.py

@@ -6,6 +6,7 @@
 import time
 import os
 import random
+import requests
 
 from os.path import abspath, realpath, dirname, join
 
@@ -51,19 +52,19 @@ def return_downloaded_content(self, url, cookies):
         :param cookies: the cookies to access
         :return: Content of the URL
         """
-        temp_cookie_file = tempfile.NamedTemporaryFile('w', delete=False)
-        data = {'url': url, 'cookies': cookies}
-        json.dump(data, temp_cookie_file)
-        temp_cookie_file.close()
-
-        cmd_path = abspath(join(dirname(realpath(__file__)),
-                                'download_content.py'))
-        # Now call the external program to handle the download
-        cmd = 'python {0} {1}'.format(cmd_path, temp_cookie_file.name)
-        if '.onion' in url:
-            cmd = 'nc -x 127.0.0.1:9150 ' + cmd
-        raw_content = self.system(cmd).strip()
-        return raw_content
+        proxies = None
+        if ".onion" in url:
+            proxies = {
+            'http': 'socks5h://127.0.0.1:9150',
+            'https': 'socks5h://127.0.0.1:9150'
+        }
+        r = requests.get(url, cookies=cookies, proxies=proxies,  stream=True)
+        if r.status_code != 200:
+            raise Exception("Failed to download the data.")
+        data = b""
+        for chunk in r.iter_content(1024):
+            data += chunk
+        return data
 
     def _input_text_in_login_form(self, username, password, token):
         self.driver.get(self.journalist_location + "/login")
@@ -640,14 +641,13 @@ def _journalist_downloads_message(self):
 
         # Downloading files with Selenium is tricky because it cannot automate
         # the browser's file download dialog. We can directly request the file
-        # using urllib2, but we need to pass the cookies for the logged in user
+        # using requests, but we need to pass the cookies for the logged in user
         # for Flask to allow this.
         def cookie_string_from_selenium_cookies(cookies):
-            cookie_strs = []
+            result = {}
             for cookie in cookies:
-                cookie_str = "=".join([cookie['name'], cookie['value']]) + ';'
-                cookie_strs.append(cookie_str)
-            return ' '.join(cookie_strs)
+                result[cookie['name']] = cookie['value']
+            return result
 
         cks = cookie_string_from_selenium_cookies(self.driver.get_cookies())
         raw_content = self.return_downloaded_content(file_url, cks)