Skip to content

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also compare across forks.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also compare across forks.
...
  • 3 commits
  • 11 files changed
  • 0 commit comments
  • 2 contributors
Commits on Feb 24, 2014
Garrett Robinson Remove Flask-Testing dependency
In the unit tests, creates the self.app and self.client variables that were
being automatically created by Flask-Testing's TestCase.

Removes unnecessary (and inconsistently used) special status code asserts,
replaces them with standard lookups on the response.
4b52c1f
Commits on Mar 03, 2014
@garrettr garrettr Merge pull request #319 from freedomofpress/remove-flask-testing
Remove Flask-Testing dependency
c59609d
@gnusosa Added MAT to source.py as a metadata purge option.
- MAT consist of a python library that makes
use of other metadata tools dedicated to specific
file-formats.

Always nice tidy and neat.

Added checkbox for metadata purge.

- Also added more validation on the
file selection for the cleanup.

Better validation on what file to write to.

Added MAT to requirements.txt

- Note: MAT is not available in
  PyPi, therefore, we clone it
  from the Tor repository.

Added MAT to source-requirements.txt

Added exiftool and poppler dependency to Debian/Ubuntu script.

Added tests for binary file upload.

Added test images for MAT
8d7c5e4
View
2 .travis.yml
@@ -5,7 +5,7 @@ install:
- pip install --upgrade distribute
- pip install -r securedrop/source-requirements.txt
- pip install -r securedrop/document-requirements.txt
- - pip install --allow-external twill --allow-unverified twill -r securedrop/test-requirements.txt
+ - pip install -r securedrop/test-requirements.txt
before_script:
- sudo apt-get install gnupg2 secure-delete python-dev haveged
- cp securedrop/config/base.py.example securedrop/config/base.py
View
2 install_files/source-requirements.txt
@@ -7,3 +7,5 @@ apache2-mpm-worker
libapache2-mod-wsgi
python-pip
python-dev
+libimage-exiftool-perl
+python-poppler
View
5 securedrop/source-requirements.txt
@@ -12,3 +12,8 @@ pycrypto==2.6.1
gnupg-securedrop==1.2.5-9-g6f9d63a-dirty
scrypt==0.6.1
wsgiref==0.1.2
+hachoir-core==1.3.3
+hachoir-parser==1.3.4
+mutagen==1.22
+pdfrw==0.1
+-e git+https://git.torproject.org/user/jvoisin/mat.git#egg=MAT
View
3 securedrop/source.py
@@ -172,12 +172,13 @@ def async_genkey(sid, codename):
def submit():
msg = request.form['msg']
fh = request.files['fh']
+ not_clean = True if 'notclean' in request.form else False
if msg:
store.save_message_submission(g.sid, msg)
flash("Thanks! We received your message.", "notification")
if fh:
- store.save_file_submission(g.sid, fh.filename, fh.stream)
+ store.save_file_submission(g.sid, fh.filename, fh.stream, fh.content_type, not_clean)
flash("Thanks! We received your document '%s'."
% fh.filename or '[unnamed]', "notification")
View
4 securedrop/source_templates/lookup.html
@@ -29,7 +29,9 @@
<input name="csrf_token" type="hidden" value="{{ csrf_token() }}"/>
<p style="padding-bottom: 0"><b>Upload a file:</b></p>
<div id="browse-select">
- <input type="file" name="fh" autocomplete="off"/>
+ <input type="file" name="fh" autocomplete="off"/><br />
+ <input type="checkbox" id="cleanup" name="notclean" value="True" />
+ <label for="cleanup">Remove all the metadata of the file.</label>
</div>
<p><b>Or just enter a message:</b></p>
View
10 securedrop/static/css/securedrop.css
@@ -145,6 +145,16 @@ form input#filename{
padding:0 10px;
}
+form input#cleanup{
+ color:#666;
+ border:none;
+ font-family: Helvetica, Arial, Verdana, sans-serif;
+ font-weight:400;
+ font-size:12px;
+ height:30px;
+ margin:15px 5px 0 5px;
+ padding:0 10px;
+}
/* add back (removed by reset) indents and bullets for plain lists in text */
ul {
View
32 securedrop/store.py
@@ -8,6 +8,10 @@
import tempfile
import subprocess
from cStringIO import StringIO
+from shutil import copyfileobj
+
+from MAT import mat
+from MAT import strippers
import logging
log = logging.getLogger(__name__)
@@ -74,15 +78,34 @@ def get_bulk_archive(filenames):
zip.write(filename, arcname=os.path.basename(filename))
return zip_file
-
-def save_file_submission(sid, filename, stream):
+def save_file_submission(sid, filename, stream, content_type, not_clean):
sanitized_filename = secure_filename(filename)
+ text_plain = content_type == 'text/plain'
+
+ f = None
+ t = None
+ clean_file = False
+
+ if not_clean and not text_plain:
+ t = tempfile.NamedTemporaryFile()
+ copyfileobj(stream, t)
+ t.flush()
+ file_meta = metadata_handler(t.name)
+
+ if not file_meta.is_clean():
+ file_meta.remove_all()
+ f = open(t.name)
+ clean_file = True
s = StringIO()
with zipfile.ZipFile(s, 'w') as zf:
- zf.writestr(sanitized_filename, stream.read())
+ zf.writestr(sanitized_filename, f.read() if clean_file else stream.read())
s.reset()
+ if clean_file:
+ f.close()
+ t.close()
+
file_loc = path(sid, "%s_doc.zip.gpg" % uuid.uuid4())
crypto_util.encrypt(config.JOURNALIST_KEY, s, file_loc)
@@ -103,3 +126,6 @@ def secure_unlink(fn, recursive=False):
def delete_source_directory(source_id):
secure_unlink(path(source_id), recursive=True)
+
+def metadata_handler(f):
+ return mat.create_class_file(f, False, add2archive=True)
View
1 securedrop/test-requirements.txt
@@ -1,3 +1,2 @@
selenium==2.39.0
-Flask-Testing==0.4
mock==1.0.1
View
BIN securedrop/tests/test_images/clean.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
BIN securedrop/tests/test_images/dirty.jpg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
View
112 securedrop/tests/unit_tests.py
@@ -13,7 +13,6 @@
import gnupg
from flask import session, g, escape
-from flask_testing import TestCase
from flask_wtf import CsrfProtect
from bs4 import BeautifulSoup
@@ -26,20 +25,19 @@
import journalist
import test_setup
-
def _block_on_reply_keypair_gen(codename):
sid = crypto_util.hash_codename(codename)
while not crypto_util.getkey(sid):
sleep(0.1)
-def _logout(app):
+def _logout(test_client):
# See http://flask.pocoo.org/docs/testing/#accessing-and-modifying-sessions
# This is necessary because SecureDrop doesn't have a logout button, so a
# user is logged in until they close the browser, which clears the session.
# For testing, this function simulates closing the browser at places
# where a source is likely to do so (for instance, between submitting a
# document and checking for a journalist reply).
- with app.session_transaction() as sess:
+ with test_client.session_transaction() as sess:
sess.clear()
def shared_setup():
@@ -55,23 +53,20 @@ def shared_teardown():
test_setup.clean_root()
-class TestSource(TestCase):
+class TestSource(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
+ def setUp(self):
shared_setup()
+ self.app = source.app
+ self.client = self.app.test_client()
- @classmethod
- def tearDownClass(cls):
+ def tearDown(self):
shared_teardown()
- def create_app(self):
- return source.app
-
def test_index(self):
"""Test that the landing page loads and looks how we expect"""
response = self.client.get('/')
- self.assert200(response)
+ self.assertEqual(response.status_code, 200)
self.assertIn("Submit documents for the first time", response.data)
self.assertIn("Already submitted something?", response.data)
@@ -87,17 +82,17 @@ def _find_codename(self, html):
def test_generate(self):
with self.client as c:
rv = c.get('/generate')
- self.assert200(rv)
+ self.assertEqual(rv.status_code, 200)
session_codename = session['codename']
self.assertIn("Submitting for the first time", rv.data)
self.assertIn(
"To protect your identity, we're assigning you a unique code name.", rv.data)
codename = self._find_codename(rv.data)
# default codename length is 8 words
- self.assertEquals(len(codename.split()), 8)
+ self.assertEqual(len(codename.split()), 8)
# codename is also stored in the session - make sure it matches the
# codename displayed to the source
- self.assertEquals(codename, escape(session_codename))
+ self.assertEqual(codename, escape(session_codename))
def test_regenerate_valid_lengths(self):
"""Make sure we can regenerate all valid length codenames"""
@@ -105,7 +100,7 @@ def test_regenerate_valid_lengths(self):
response = self.client.post('/generate', data={
'number-words': str(codename_len),
})
- self.assert200(response)
+ self.assertEqual(response.status_code, 200)
codename = self._find_codename(response.data)
self.assertEquals(len(codename.split()), codename_len)
@@ -115,7 +110,7 @@ def test_regenerate_invalid_lengths(self):
response = self.client.post('/generate', data={
'number-words': str(codename_len),
})
- self.assert403(response)
+ self.assertEqual(response.status_code, 403)
def test_create(self):
with self.client as c:
@@ -147,23 +142,22 @@ def test_lookup(self):
def test_login_and_logout(self):
rv = self.client.get('/login')
- self.assert200(rv)
+ self.assertEqual(rv.status_code, 200)
self.assertIn("Already submitted something?", rv.data)
codename = self._new_codename()
with self.client as c:
rv = c.post('/login', data=dict(codename=codename),
follow_redirects=True)
- self.assert200(rv)
+ self.assertEqual(rv.status_code, 200)
self.assertIn("Submit a document, message, or both", rv.data)
self.assertTrue(session['logged_in'])
_logout(c)
- self.assertEquals(len(session), 0)
with self.client as c:
rv = self.client.post('/login', data=dict(codename='invalid'),
follow_redirects=True)
- self.assert200(rv)
+ self.assertEqual(rv.status_code, 200)
self.assertIn('Sorry, that is not a recognized codename.', rv.data)
self.assertNotIn('logged_in', session)
@@ -173,7 +167,7 @@ def test_submit_message(self):
msg="This is a test.",
fh=(StringIO(''), ''),
), follow_redirects=True)
- self.assert200(rv)
+ self.assertEqual(rv.status_code, 200)
self.assertIn("Thanks! We received your message.", rv.data)
def test_submit_file(self):
@@ -182,7 +176,7 @@ def test_submit_file(self):
msg="",
fh=(StringIO('This is a test'), 'test.txt'),
), follow_redirects=True)
- self.assert200(rv)
+ self.assertEqual(rv.status_code, 200)
self.assertIn(escape("Thanks! We received your document 'test.txt'."),
rv.data)
@@ -192,11 +186,68 @@ def test_submit_both(self):
msg="This is a test",
fh=(StringIO('This is a test'), 'test.txt'),
), follow_redirects=True)
- self.assert200(rv)
+ self.assertEqual(rv.status_code, 200)
self.assertIn("Thanks! We received your message.", rv.data)
self.assertIn(escape("Thanks! We received your document 'test.txt'."),
rv.data)
+ def test_submit_dirty_file(self):
+ self.gpg = gnupg.GPG(homedir=config.GPG_KEY_DIR)
+ self._new_codename()
+ img = open(os.getcwd()+'/tests/test_images/dirty.jpg')
+ img_metadata = store.metadata_handler(img.name)
+ self.assertFalse(img_metadata.is_clean(), "The file is dirty.")
+ del(img_metadata)
+ codename = self._new_codename()
+ rv = self.client.post('/submit', data=dict(
+ msg="This is a test",
+ fh=(img, 'dirty.jpg'),
+ notclean='True',
+ ), follow_redirects=True)
+ self.assert200(rv)
+ self.assertIn("Thanks! We received your message.", rv.data)
+ self.assertIn(escape("Thanks! We received your document 'dirty.jpg'."),
+ rv.data)
+
+ store_dirs = [os.path.join(config.STORE_DIR,d) for d in os.listdir(config.STORE_DIR) if os.path.isdir(os.path.join(config.STORE_DIR,d))]
+ latest_subdir = max(store_dirs, key=os.path.getmtime)
+ zip_gpg_files = [os.path.join(latest_subdir,f) for f in os.listdir(latest_subdir) if os.path.isfile(os.path.join(latest_subdir,f))]
+ zip_gpg = max(zip_gpg_files, key=os.path.getmtime)
+
+ zip_gpg_file = open(zip_gpg)
+ decrypted_data = self.gpg.decrypt_file(zip_gpg_file)
+ self.assertTrue(decrypted_data.ok, 'Checking the integrity of the data after decryption.')
+
+ s = StringIO(decrypted_data.data)
+ zip_file = zipfile.ZipFile(s, 'r')
+ clean_file = open(os.path.join(latest_subdir,'dirty.jpg'), 'w+b')
+ clean_file.write(zip_file.read('dirty.jpg'))
+ clean_file.seek(0)
+ zip_file.close()
+
+ # check for the actual file been clean
+ clean_file_metadata = store.metadata_handler(clean_file.name)
+ self.assertTrue(clean_file_metadata.is_clean(), "the file is now clean.")
+ del(clean_file_metadata)
+ zip_gpg_file.close()
+ clean_file.close()
+ img.close()
+
+ def test_submit_clean_file(self):
+ self._new_codename()
+ img = open(os.getcwd()+'/tests/test_images/clean.jpg')
+ codename = self._new_codename()
+ rv = self.client.post('/submit', data=dict(
+ msg="This is a test",
+ fh=(img, 'clean.jpg'),
+ notclean='True',
+ ), follow_redirects=True)
+ self.assert200(rv)
+ self.assertIn("Thanks! We received your message.", rv.data)
+ self.assertIn(escape("Thanks! We received your document 'clean.jpg'."),
+ rv.data)
+ img.close()
+
@patch('zipfile.ZipFile.writestr')
def test_submit_sanitizes_filename(self, zipfile_write):
"""Test that upload file name is sanitized"""
@@ -212,24 +263,23 @@ def test_submit_sanitizes_filename(self, zipfile_write):
def test_tor2web_warning(self):
rv = self.client.get('/', headers=[('X-tor2web', 'encrypted')])
- self.assert200(rv)
+ self.assertEqual(rv.status_code, 200)
self.assertIn("You appear to be using Tor2Web.", rv.data)
-class TestJournalist(TestCase):
+class TestJournalist(unittest.TestCase):
def setUp(self):
shared_setup()
+ self.app = journalist.app
+ self.client = self.app.test_client()
def tearDown(self):
shared_teardown()
- def create_app(self):
- return journalist.app
-
def test_index(self):
rv = self.client.get('/')
- self.assert200(rv)
+ self.assertEqual(rv.status_code, 200)
self.assertIn("Latest submissions", rv.data)
self.assertIn("No documents have been submitted!", rv.data)

No commit comments for this range

Something went wrong with that request. Please try again.