From 17c1c12fe8bbdff22d887b40922f8437cbd44313 Mon Sep 17 00:00:00 2001 From: Kushal Das Date: Fri, 3 Aug 2018 18:25:28 +0530 Subject: [PATCH] Downloads data from server using requests over Tor The test requirements now have requests[socks] as dependency. Using the same we are now directly downloading the files/messages from the .onion address for functional tests. The old external command file also got removed this committ. We are creating the gpg object for both container based local testing and external testing (in functional tests). Fixes: #3691 #3687 --- securedrop/requirements/test-requirements.in | 1 + securedrop/requirements/test-requirements.txt | 6 ++++ .../tests/functional/download_content.py | 22 ------------ .../tests/functional/functional_test.py | 14 ++------ .../functional/journalist_navigation_steps.py | 36 +++++++++---------- 5 files changed, 28 insertions(+), 51 deletions(-) delete mode 100644 securedrop/tests/functional/download_content.py diff --git a/securedrop/requirements/test-requirements.in b/securedrop/requirements/test-requirements.in index cf0d7a553a..62aa2ec03d 100644 --- a/securedrop/requirements/test-requirements.in +++ b/securedrop/requirements/test-requirements.in @@ -7,6 +7,7 @@ py pytest pytest-cov pytest-mock +requests[socks] selenium tbselenium pyvirtualdisplay diff --git a/securedrop/requirements/test-requirements.txt b/securedrop/requirements/test-requirements.txt index 0864679f79..5e2bd02cec 100644 --- a/securedrop/requirements/test-requirements.txt +++ b/securedrop/requirements/test-requirements.txt @@ -7,6 +7,8 @@ attrs==17.4.0 # via pytest beautifulsoup4==4.6.0 blinker==1.4 +certifi==2018.4.16 # via requests +chardet==3.0.4 # via requests click==6.7 # via flask, pip-tools coverage==4.4.2 # via pytest-cov easyprocess==0.2.3 # via pyvirtualdisplay @@ -14,6 +16,7 @@ first==2.0.1 # via pip-tools flask-testing==0.7.1 flask==0.12.2 # via flask-testing funcsigs==1.0.2 # via mock, pytest +idna==2.7 # via requests itsdangerous==0.24 # via flask jinja2==2.10 # via flask markupsafe==1.0 # via jinja2 @@ -22,11 +25,14 @@ pbr==3.1.1 # via mock pip-tools==1.11.0 pluggy==0.6.0 # via pytest py==1.5.2 +pysocks==1.6.8 # via requests pytest-cov==2.5.1 pytest-mock==1.7.1 pytest==3.3.2 pyvirtualdisplay==0.2.1 +requests[socks]==2.19.1 selenium==3.13.0 six==1.11.0 # via mock, pip-tools, pytest tbselenium==0.3.3 +urllib3==1.23 # via requests werkzeug==0.12.2 # via flask diff --git a/securedrop/tests/functional/download_content.py b/securedrop/tests/functional/download_content.py deleted file mode 100644 index 51343dff02..0000000000 --- a/securedrop/tests/functional/download_content.py +++ /dev/null @@ -1,22 +0,0 @@ -import sys -import json -import urllib2 - - -def main(): - fpath = sys.argv[1] - with open(fpath) as fobj: - data = json.load(fobj) - - url = data['url'] - - submission_req = urllib2.Request(url) - submission_req.add_header( - 'Cookie', - data['cookies']) - raw_content = urllib2.urlopen(submission_req).read() - print(raw_content) - - -if __name__ == '__main__': - main() diff --git a/securedrop/tests/functional/functional_test.py b/securedrop/tests/functional/functional_test.py index 45928c31ef..c9d9a54d31 100644 --- a/securedrop/tests/functional/functional_test.py +++ b/securedrop/tests/functional/functional_test.py @@ -202,15 +202,6 @@ def init_gpg(self): gpg.import_keys(open(keyfile).read()) return gpg - def system(self, cmd): - """ - Invoke a shell command. Primary replacement for os.system calls. - """ - ret = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - close_fds=True) - out, err = ret.communicate() - return out def setup(self, session_expiration=30): @@ -222,6 +213,9 @@ def setup(self, session_expiration=30): instance_information_path = join(FUNCTIONAL_TEST_DIR, 'instance_information.json') + env.create_directories() + self.gpg = env.init_gpg() + if os.path.exists(instance_information_path): with open(instance_information_path) as fobj: data = json.load(fobj) @@ -244,8 +238,6 @@ def setup(self, session_expiration=30): self.mock_get_entropy_estimate = self.patcher2.start() self.mock_get_entropy_estimate.return_value = 8192 - env.create_directories() - self.gpg = env.init_gpg() db.create_all() # Add our test user diff --git a/securedrop/tests/functional/journalist_navigation_steps.py b/securedrop/tests/functional/journalist_navigation_steps.py index 8cc81dfb6f..667f41dd8d 100644 --- a/securedrop/tests/functional/journalist_navigation_steps.py +++ b/securedrop/tests/functional/journalist_navigation_steps.py @@ -6,6 +6,7 @@ import time import os import random +import requests from os.path import abspath, realpath, dirname, join @@ -51,19 +52,19 @@ def return_downloaded_content(self, url, cookies): :param cookies: the cookies to access :return: Content of the URL """ - temp_cookie_file = tempfile.NamedTemporaryFile('w', delete=False) - data = {'url': url, 'cookies': cookies} - json.dump(data, temp_cookie_file) - temp_cookie_file.close() - - cmd_path = abspath(join(dirname(realpath(__file__)), - 'download_content.py')) - # Now call the external program to handle the download - cmd = 'python {0} {1}'.format(cmd_path, temp_cookie_file.name) - if '.onion' in url: - cmd = 'nc -x 127.0.0.1:9150 ' + cmd - raw_content = self.system(cmd).strip() - return raw_content + proxies = None + if ".onion" in url: + proxies = { + 'http': 'socks5h://127.0.0.1:9150', + 'https': 'socks5h://127.0.0.1:9150' + } + r = requests.get(url, cookies=cookies, proxies=proxies, stream=True) + if r.status_code != 200: + raise Exception("Failed to download the data.") + data = b"" + for chunk in r.iter_content(1024): + data += chunk + return data def _input_text_in_login_form(self, username, password, token): self.driver.get(self.journalist_location + "/login") @@ -640,14 +641,13 @@ def _journalist_downloads_message(self): # Downloading files with Selenium is tricky because it cannot automate # the browser's file download dialog. We can directly request the file - # using urllib2, but we need to pass the cookies for the logged in user + # using requests, but we need to pass the cookies for the logged in user # for Flask to allow this. def cookie_string_from_selenium_cookies(cookies): - cookie_strs = [] + result = {} for cookie in cookies: - cookie_str = "=".join([cookie['name'], cookie['value']]) + ';' - cookie_strs.append(cookie_str) - return ' '.join(cookie_strs) + result[cookie['name']] = cookie['value'] + return result cks = cookie_string_from_selenium_cookies(self.driver.get_cookies()) raw_content = self.return_downloaded_content(file_url, cks)