diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py
index efaf429e2..12b051cb6 100644
--- a/qiita_db/meta_util.py
+++ b/qiita_db/meta_util.py
@@ -13,7 +13,6 @@
..autosummary::
:toctree: generated/
- get_accessible_filepath_ids
get_lat_longs
"""
# -----------------------------------------------------------------------------
@@ -59,78 +58,117 @@ def _get_data_fpids(constructor, object_id):
return {fpid for fpid, _, _ in obj.get_filepaths()}
-def get_accessible_filepath_ids(user):
- """Gets all filepaths that this user should have access to
-
- This gets all raw, preprocessed, and processed filepaths, for studies
- that the user has access to, as well as all the mapping files and biom
- tables associated with the analyses that the user has access to.
+def validate_filepath_access_by_user(user, filepath_id):
+ """Validates if the user has access to the filepath_id
Parameters
----------
user : User object
The user we are interested in
-
+ filepath_id : int
+ The filepath id
Returns
-------
- set
- A set of filepath ids
+ bool
+ If the user has access or not to the filepath_id
Notes
-----
- Admins have access to all files, so all filepath ids are returned for
- admins
+ Admins have access to all files so True is always returned
"""
- with qdb.sql_connection.TRN:
+ TRN = qdb.sql_connection.TRN
+ with TRN:
if user.level == "admin":
+ return True
# admins have access all files
- qdb.sql_connection.TRN.add(
- "SELECT filepath_id FROM qiita.filepath")
- return set(qdb.sql_connection.TRN.execute_fetchflatten())
-
- # First, the studies
- # There are private and shared studies
- studies = user.user_studies | user.shared_studies
-
- filepath_ids = set()
- for study in studies:
- # Add the sample template files
- if study.sample_template:
- filepath_ids.update(
- {fid for fid, _ in study.sample_template.get_filepaths()})
-
- # Add the prep template filepaths
- for pt in study.prep_templates():
- filepath_ids.update({fid for fid, _ in pt.get_filepaths()})
-
- # Add the artifact filepaths
- for artifact in study.artifacts():
- filepath_ids.update({fid for fid, _, _ in artifact.filepaths})
-
- # Next, the public artifacts
- for artifact in qdb.artifact.Artifact.iter_public():
- # Add the filepaths of the artifact
- filepath_ids.update({fid for fid, _, _ in artifact.filepaths})
-
- # Then add the filepaths of the prep templates
- for pt in artifact.prep_templates:
- filepath_ids.update({fid for fid, _ in pt.get_filepaths()})
-
- # Then add the filepaths of the sample template
- filepath_ids.update(
- {fid
- for fid, _ in artifact.study.sample_template.get_filepaths()})
-
- # Next, analyses
- # Same as before, there are public, private, and shared
- analyses = qdb.analysis.Analysis.get_by_status('public') | \
- user.private_analyses | user.shared_analyses
-
- for analysis in analyses:
- filepath_ids.update(analysis.all_associated_filepath_ids)
-
- return filepath_ids
+
+ access = False
+ sql = """SELECT
+ (SELECT count(*) FROM qiita.artifact_filepath
+ WHERE filepath_id = {0}) AS artifact,
+ (SELECT count(*) FROM qiita.sample_template_filepath
+ WHERE filepath_id = {0}) AS sample_info,
+ (SELECT count(*) FROM qiita.prep_template_filepath
+ WHERE filepath_id = {0}) AS prep_info,
+ (SELECT count(*) FROM qiita.job_results_filepath
+ WHERE filepath_id = {0}) AS job_results,
+ (SELECT count(*) FROM qiita.analysis_filepath
+ WHERE filepath_id = {0}) AS analysis""".format(filepath_id)
+ TRN.add(sql)
+
+ arid, sid, pid, jid, anid = TRN.execute_fetchflatten()
+
+ # artifacts
+ if arid:
+ # check the public artifacts
+ public_artifacts = qdb.artifact.Artifact.iter_public()
+ for artifact in public_artifacts:
+ if filepath_id in [fid for fid, _, _ in artifact.filepaths]:
+ access = True
+ break
+ # if not found check the user artifacts from their studies
+ if not access:
+ user_studies = user.user_studies | user.shared_studies
+ for s in user_studies:
+ if s.sample_template:
+ for a in s.artifacts():
+ if filepath_id in [fid[0] for fid in a.filepaths]:
+ access = True
+ break
+ # just avoiding extra loops if found
+ if access:
+ break
+ # sample info files
+ elif sid:
+ # check private and shared studies with the user
+ user_studies = user.user_studies | user.shared_studies
+ for s in user_studies:
+ st = s.sample_template
+ if st is not None:
+ # sample info files
+ if filepath_id in [fid for fid, _ in st.get_filepaths()]:
+ access = True
+ break
+ # if that didn't work let's check the public sample info files
+ if not access:
+ public_studies = qdb.study.Study.get_by_status('public')
+ for s in public_studies:
+ st = s.sample_template
+ if st is not None:
+ if filepath_id in [fid[0] for fid in
+ st.get_filepaths()]:
+ access = True
+ break
+ # prep info files
+ elif pid:
+ # check private and shared studies with the user
+ user_studies = user.user_studies | user.shared_studies
+ for s in user_studies:
+ for pt in s.prep_templates():
+ # sample info files
+ if filepath_id in [fid for fid, _ in pt.get_filepaths()]:
+ access = True
+ break
+ # if that didn't work let's check the public prep info files
+ if not access:
+ public_studies = qdb.study.Study.get_by_status('public')
+ for s in public_studies:
+ for pt in s.prep_templates():
+ if filepath_id in [fid[0]
+ for fid in pt.get_filepaths()]:
+ access = True
+ break
+ # next analyses
+ elif anid or jid:
+ analyses = qdb.analysis.Analysis.get_by_status('public') | \
+ user.private_analyses | user.shared_analyses
+ for analysis in analyses:
+ if filepath_id in analysis.all_associated_filepath_ids:
+ access = True
+ break
+
+ return access
def update_redis_stats():
diff --git a/qiita_db/test/test_meta_util.py b/qiita_db/test/test_meta_util.py
index f9f513467..891ed3b66 100644
--- a/qiita_db/test/test_meta_util.py
+++ b/qiita_db/test/test_meta_util.py
@@ -39,40 +39,38 @@ def _unshare_studies(self):
def _unshare_analyses(self):
self.conn_handler.execute("DELETE FROM qiita.analysis_users")
- def test_get_accessible_filepath_ids(self):
+ def test_validate_filepath_access_by_user(self):
self._set_artifact_private()
# shared has access to all study files and analysis files
-
- obs = qdb.meta_util.get_accessible_filepath_ids(
- qdb.user.User('shared@foo.bar'))
- self.assertItemsEqual(obs, {
- 1, 2, 3, 4, 5, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21})
+ user = qdb.user.User('shared@foo.bar')
+ for i in [1, 2, 3, 4, 5, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]:
+ self.assertTrue(qdb.meta_util.validate_filepath_access_by_user(
+ user, i))
# Now shared should not have access to the study files
self._unshare_studies()
- obs = qdb.meta_util.get_accessible_filepath_ids(
- qdb.user.User('shared@foo.bar'))
- self.assertItemsEqual(obs, {16, 14, 15, 13})
+ for i in [1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21]:
+ self.assertFalse(qdb.meta_util.validate_filepath_access_by_user(
+ user, i))
+
+ for i in [13, 14, 15, 16]:
+ self.assertTrue(qdb.meta_util.validate_filepath_access_by_user(
+ user, i))
# Now shared should not have access to any files
self._unshare_analyses()
- obs = qdb.meta_util.get_accessible_filepath_ids(
- qdb.user.User('shared@foo.bar'))
- self.assertEqual(obs, set())
+ for i in [1, 2, 3, 4, 5, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]:
+ self.assertFalse(qdb.meta_util.validate_filepath_access_by_user(
+ user, i))
# Now shared has access to public study files
self._set_artifact_public()
- obs = qdb.meta_util.get_accessible_filepath_ids(
- qdb.user.User('shared@foo.bar'))
- self.assertEqual(obs, {1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21})
+ for i in [1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21]:
+ self.assertTrue(qdb.meta_util.validate_filepath_access_by_user(
+ user, i))
# Test that it doesn't break: if the SampleTemplate hasn't been added
- exp = {1, 2, 3, 4, 5, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}
- obs = qdb.meta_util.get_accessible_filepath_ids(
- qdb.user.User('test@foo.bar'))
- self.assertEqual(obs, exp)
-
info = {
"timeseries_type_id": 1,
"metadata_complete": True,
@@ -86,26 +84,28 @@ def test_get_accessible_filepath_ids(self):
"principal_investigator_id": 1,
"lab_person_id": 1
}
- qdb.study.Study.create(
+ study = qdb.study.Study.create(
qdb.user.User('test@foo.bar'), "Test study", [1], info)
- obs = qdb.meta_util.get_accessible_filepath_ids(
- qdb.user.User('test@foo.bar'))
- self.assertEqual(obs, exp)
+ for i in [1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21]:
+ self.assertTrue(qdb.meta_util.validate_filepath_access_by_user(
+ user, i))
# test in case there is a prep template that failed
self.conn_handler.execute(
"INSERT INTO qiita.prep_template (data_type_id) VALUES (2)")
- obs = qdb.meta_util.get_accessible_filepath_ids(
- qdb.user.User('test@foo.bar'))
- self.assertEqual(obs, exp)
+ for i in [1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21]:
+ self.assertTrue(qdb.meta_util.validate_filepath_access_by_user(
+ user, i))
# admin should have access to everything
- count = self.conn_handler.execute_fetchone("SELECT count(*) FROM "
- "qiita.filepath")[0]
- exp = set(range(1, count + 1))
- obs = qdb.meta_util.get_accessible_filepath_ids(
- qdb.user.User('admin@foo.bar'))
- self.assertEqual(obs, exp)
+ admin = qdb.user.User('admin@foo.bar')
+ fids = self.conn_handler.execute_fetchall(
+ "SELECT filepath_id FROM qiita.filepath")
+ for i in fids:
+ self.assertTrue(qdb.meta_util.validate_filepath_access_by_user(
+ admin, i[0]))
+
+ qdb.study.Study.delete(study.id)
def test_get_lat_longs(self):
exp = [
@@ -181,18 +181,21 @@ def test_get_lat_longs_EMP_portal(self):
self.assertItemsEqual(obs, exp)
+ qdb.metadata_template.sample_template.SampleTemplate.delete(study.id)
+ qdb.study.Study.delete(study.id)
+
def test_update_redis_stats(self):
qdb.meta_util.update_redis_stats()
portal = qiita_config.portal
vals = [
- ('number_studies', {'sanbox': '2', 'public': '0',
- 'private': '1'}, r_client.hgetall),
- ('number_of_samples', {'sanbox': '1', 'public': '0',
- 'private': '27'}, r_client.hgetall),
+ ('number_studies', {'sanbox': '0', 'public': '1',
+ 'private': '0'}, r_client.hgetall),
+ ('number_of_samples', {'sanbox': '0', 'public': '27',
+ 'private': '0'}, r_client.hgetall),
('num_users', '4', r_client.get),
('lat_longs', EXP_LAT_LONG, r_client.get),
- ('num_studies_ebi', '3', r_client.get),
+ ('num_studies_ebi', '1', r_client.get),
('num_samples_ebi', '27', r_client.get),
('number_samples_ebi_prep', '54', r_client.get)
# not testing img/time for simplicity
@@ -205,19 +208,19 @@ def test_update_redis_stats(self):
EXP_LAT_LONG = (
- '[[0.291867635913, 68.5945325743], [68.0991287718, 34.8360987059],'
- ' [10.6655599093, 70.784770579], [40.8623799474, 6.66444220187],'
+ '[[60.1102854322, 74.7123248382], [23.1218032799, 42.838497795],'
+ ' [3.21190859967, 26.8138925876], [74.0894932572, 65.3283470202],'
+ ' [53.5050692395, 31.6056761814], [12.6245524972, 96.0693176066],'
+ ' [43.9614715197, 82.8516734159], [10.6655599093, 70.784770579],'
+ ' [78.3634273709, 74.423907894], [82.8302905615, 86.3615778099],'
+ ' [44.9725384282, 66.1920014699], [4.59216095574, 63.5115213108],'
+ ' [57.571893782, 32.5563076447], [40.8623799474, 6.66444220187],'
+ ' [95.2060749748, 27.3592668624], [38.2627021402, 3.48274264219],'
' [13.089194595, 92.5274472082], [84.0030227585, 66.8954849864],'
- ' [12.7065957714, 84.9722975792], [78.3634273709, 74.423907894],'
- ' [82.8302905615, 86.3615778099], [53.5050692395, 31.6056761814],'
- ' [43.9614715197, 82.8516734159], [29.1499460692, 82.1270418227],'
- ' [23.1218032799, 42.838497795], [12.6245524972, 96.0693176066],'
- ' [38.2627021402, 3.48274264219], [74.0894932572, 65.3283470202],'
- ' [35.2374368957, 68.5041623253], [4.59216095574, 63.5115213108],'
- ' [95.2060749748, 27.3592668624], [68.51099627, 2.35063674718],'
- ' [85.4121476399, 15.6526750776], [60.1102854322, 74.7123248382],'
- ' [3.21190859967, 26.8138925876], [57.571893782, 32.5563076447],'
- ' [44.9725384282, 66.1920014699], [42.42, 41.41]]')
+ ' [68.51099627, 2.35063674718], [29.1499460692, 82.1270418227],'
+ ' [35.2374368957, 68.5041623253], [12.7065957714, 84.9722975792],'
+ ' [0.291867635913, 68.5945325743], [85.4121476399, 15.6526750776],'
+ ' [68.0991287718, 34.8360987059]]')
if __name__ == '__main__':
main()
diff --git a/qiita_db/util.py b/qiita_db/util.py
index 938451450..9c9a09611 100644
--- a/qiita_db/util.py
+++ b/qiita_db/util.py
@@ -894,12 +894,12 @@ def filepath_id_to_rel_path(filepath_id):
LEFT JOIN qiita.artifact_filepath USING (filepath_id)
WHERE filepath_id = %s"""
qdb.sql_connection.TRN.add(sql, [filepath_id])
+ # It should be only one row
mp, fp, sd, a_id = qdb.sql_connection.TRN.execute_fetchindex()[0]
if sd:
result = join(mp, str(a_id), fp)
else:
result = join(mp, fp)
- # It should be only one row
return result
diff --git a/qiita_pet/handlers/download.py b/qiita_pet/handlers/download.py
index 1a113e928..b6d3e3fab 100644
--- a/qiita_pet/handlers/download.py
+++ b/qiita_pet/handlers/download.py
@@ -1,11 +1,13 @@
-from tornado.web import authenticated
+from tornado.web import authenticated, HTTPError
from os.path import basename
+from datetime import datetime
from .base_handlers import BaseHandler
-from qiita_pet.exceptions import QiitaPetAuthorizationError
-from qiita_db.util import filepath_id_to_rel_path
-from qiita_db.meta_util import get_accessible_filepath_ids
+from qiita_pet.handlers.api_proxy import study_get_req
+from qiita_db.study import Study
+from qiita_db.util import filepath_id_to_rel_path, get_db_files_base_dir
+from qiita_db.meta_util import validate_filepath_access_by_user
from qiita_core.util import execute_as_transaction
@@ -13,15 +15,14 @@ class DownloadHandler(BaseHandler):
@authenticated
@execute_as_transaction
def get(self, filepath_id):
- filepath_id = int(filepath_id)
- # Check access to file
- accessible_filepaths = get_accessible_filepath_ids(self.current_user)
+ fid = int(filepath_id)
- if filepath_id not in accessible_filepaths:
- raise QiitaPetAuthorizationError(
- self.current_user, 'filepath id %s' % str(filepath_id))
+ if not validate_filepath_access_by_user(self.current_user, fid):
+ raise HTTPError(
+ 404, "%s doesn't have access to "
+ "filepath_id: %s" % (self.current_user.email, str(fid)))
- relpath = filepath_id_to_rel_path(filepath_id)
+ relpath = filepath_id_to_rel_path(fid)
fname = basename(relpath)
# If we don't have nginx, write a file that indicates this
@@ -39,3 +40,66 @@ def get(self, filepath_id):
'attachment; filename=%s' % fname)
self.finish()
+
+
+class DownloadStudyBIOMSHandler(BaseHandler):
+ @authenticated
+ @execute_as_transaction
+ def get(self, study_id):
+ study_id = int(study_id)
+ # Check access to study
+ study_info = study_get_req(study_id, self.current_user.id)
+
+ if study_info['status'] != 'success':
+ raise HTTPError(405, "%s: %s, %s" % (study_info['message'],
+ self.current_user.email,
+ str(study_id)))
+
+ study = Study(study_id)
+ user = self.current_user
+ basedir = get_db_files_base_dir()
+ basedir_len = len(basedir) + 1
+ # loop over artifacts and retrieve those that we have access to
+ to_download = []
+ vfabu = validate_filepath_access_by_user
+ for a in study.artifacts():
+ if a.artifact_type == 'BIOM':
+ to_add = True
+ for i, (fid, path, data_type) in enumerate(a.filepaths):
+ # validate access only of the first artifact filepath,
+ # the rest have the same permissions
+ if (i == 0 and not vfabu(user, fid)):
+ to_add = False
+ break
+ if path.startswith(basedir):
+ path = path[basedir_len:]
+ to_download.append((path, data_type))
+
+ if to_add:
+ for pt in a.prep_templates:
+ qmf = pt.qiime_map_fp
+ if qmf is not None:
+ if qmf.startswith(basedir):
+ qmf = qmf[basedir_len:]
+ to_download.append((qmf, 'QIIME map file'))
+
+ # If we don't have nginx, write a file that indicates this
+ all_files = '\n'.join(['%s: %s' % (n, fp) for fp, n in to_download])
+ self.write("This installation of Qiita was not equipped with nginx, "
+ "so it is incapable of serving files. The files you "
+ "attempted to download are located at:\n%s" % all_files)
+
+ zip_fn = 'study_%d_%s.zip' % (
+ study_id, datetime.now().strftime('%m%d%y-%H%M%S'))
+
+ self.set_header('Content-Description', 'File Transfer')
+ self.set_header('Content-Type', 'application/octet-stream')
+ self.set_header('Content-Transfer-Encoding', 'binary')
+ self.set_header('Expires', '0')
+ self.set_header('Cache-Control', 'no-cache')
+ self.set_header('X-Accel-Files', 'zip')
+ for fp, n in to_download:
+ self.set_header('X-Accel-Redirect', '/protected/' + fp)
+ self.set_header('Content-Disposition',
+ 'attachment; filename=%s' % zip_fn)
+ self.finish()
diff --git a/qiita_pet/templates/study_base.html b/qiita_pet/templates/study_base.html
index 3e16e9fa9..34b604bf4 100644
--- a/qiita_pet/templates/study_base.html
+++ b/qiita_pet/templates/study_base.html
@@ -239,6 +239,7 @@
Upload Files
{% end %}
+ All QIIME maps and BIOMs
diff --git a/qiita_pet/test/test_download.py b/qiita_pet/test/test_download.py
new file mode 100644
index 000000000..aa3b94526
--- /dev/null
+++ b/qiita_pet/test/test_download.py
@@ -0,0 +1,72 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) 2014--, The Qiita Development Team.
+#
+# Distributed under the terms of the BSD 3-clause License.
+#
+# The full license is in the file LICENSE, distributed with this software.
+# -----------------------------------------------------------------------------
+
+from unittest import main
+from mock import Mock
+
+from qiita_pet.test.tornado_test_base import TestHandlerBase
+from qiita_pet.handlers.base_handlers import BaseHandler
+from qiita_db.user import User
+
+
+class TestDownloadHandler(TestHandlerBase):
+
+ def setUp(self):
+ super(TestDownloadHandler, self).setUp()
+
+ def tearDown(self):
+ super(TestDownloadHandler, self).tearDown()
+
+ def test_download(self):
+ # check success
+ response = self.get('/download/1')
+ self.assertEqual(response.code, 200)
+ self.assertEqual(response.body, (
+ "This installation of Qiita was not equipped with nginx, so it "
+ "is incapable of serving files. The file you attempted to "
+ "download is located at raw_data/1_s_G1_L001_sequences.fastq.gz"))
+
+ # failure
+ response = self.get('/download/1000')
+ self.assertEqual(response.code, 404)
+
+
+class TestDownloadStudyBIOMSHandler(TestHandlerBase):
+
+ def setUp(self):
+ super(TestDownloadStudyBIOMSHandler, self).setUp()
+
+ def tearDown(self):
+ super(TestDownloadStudyBIOMSHandler, self).tearDown()
+
+ def test_download_study(self):
+ response = self.get('/download_study_bioms/1')
+ self.assertEqual(response.code, 200)
+ self.assertEqual(response.body, (
+ "This installation of Qiita was not equipped with nginx, so it "
+ "is incapable of serving files. The files you attempted to "
+ "download are located at:\nbiom: processed_data/1_study_1001_"
+ "closed_reference_otu_table.biom\nQIIME map file: templates/"
+ "1_prep_1_qiime_19700101-000000.txt\nbiom: processed_data/"
+ "1_study_1001_closed_reference_otu_table.biom\nQIIME map file: "
+ "templates/1_prep_1_qiime_19700101-000000.txt\nbiom: "
+ "processed_data/1_study_1001_closed_reference_otu_table_Silva.biom"
+ "\nQIIME map file: templates/1_prep_1_qiime_19700101-000000.txt"))
+
+ response = self.get('/download_study_bioms/200')
+ self.assertEqual(response.code, 405)
+
+ # changing user so we can test the failures
+ BaseHandler.get_current_user = Mock(
+ return_value=User("demo@microbio.me"))
+ response = self.get('/download_study_bioms/1')
+ self.assertEqual(response.code, 405)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py
index 42952d478..15185601d 100644
--- a/qiita_pet/webserver.py
+++ b/qiita_pet/webserver.py
@@ -38,7 +38,8 @@
from qiita_pet.handlers.logger_handlers import LogEntryViewerHandler
from qiita_pet.handlers.upload import UploadFileHandler, StudyUploadFileHandler
from qiita_pet.handlers.stats import StatsHandler
-from qiita_pet.handlers.download import DownloadHandler
+from qiita_pet.handlers.download import (
+ DownloadHandler, DownloadStudyBIOMSHandler)
from qiita_pet.handlers.prep_template import PrepTemplateHandler
from qiita_pet.handlers.ontology import OntologyHandler
from qiita_db.handlers.processing_job import (
@@ -144,6 +145,7 @@ def __init__(self):
(r"/check_study/", CreateStudyAJAX),
(r"/stats/", StatsHandler),
(r"/download/(.*)", DownloadHandler),
+ (r"/download_study_bioms/(.*)", DownloadStudyBIOMSHandler),
(r"/vamps/(.*)", VAMPSHandler),
# Plugin handlers - the order matters here so do not change
# qiita_db/jobs/(.*) should go after any of the