Skip to content

Commit

Permalink
Merge ade8cbe into 4e380e0
Browse files Browse the repository at this point in the history
  • Loading branch information
@antgonza
antgonza committed Jan 30, 2017
2 parents 4e380e0 + ade8cbe commit 62140d9
Show file tree
Hide file tree
Showing 7 changed files with 302 additions and 122 deletions.
156 changes: 97 additions & 59 deletions qiita_db/meta_util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
..autosummary::
:toctree: generated/
get_accessible_filepath_ids
get_lat_longs
"""
# -----------------------------------------------------------------------------
Expand Down Expand Up @@ -59,78 +58,117 @@ def _get_data_fpids(constructor, object_id):
return {fpid for fpid, _, _ in obj.get_filepaths()}


def get_accessible_filepath_ids(user):
"""Gets all filepaths that this user should have access to
This gets all raw, preprocessed, and processed filepaths, for studies
that the user has access to, as well as all the mapping files and biom
tables associated with the analyses that the user has access to.
def validate_filepath_access_by_user(user, filepath_id):
"""Validates if the user has access to the filepath_id
Parameters
----------
user : User object
The user we are interested in
filepath_id : int
The filepath id
Returns
-------
set
A set of filepath ids
bool
If the user has access or not to the filepath_id
Notes
-----
Admins have access to all files, so all filepath ids are returned for
admins
Admins have access to all files so True is always returned
"""
with qdb.sql_connection.TRN:
TRN = qdb.sql_connection.TRN
with TRN:
if user.level == "admin":
return True
# admins have access all files
qdb.sql_connection.TRN.add(
"SELECT filepath_id FROM qiita.filepath")
return set(qdb.sql_connection.TRN.execute_fetchflatten())

# First, the studies
# There are private and shared studies
studies = user.user_studies | user.shared_studies

filepath_ids = set()
for study in studies:
# Add the sample template files
if study.sample_template:
filepath_ids.update(
{fid for fid, _ in study.sample_template.get_filepaths()})

# Add the prep template filepaths
for pt in study.prep_templates():
filepath_ids.update({fid for fid, _ in pt.get_filepaths()})

# Add the artifact filepaths
for artifact in study.artifacts():
filepath_ids.update({fid for fid, _, _ in artifact.filepaths})

# Next, the public artifacts
for artifact in qdb.artifact.Artifact.iter_public():
# Add the filepaths of the artifact
filepath_ids.update({fid for fid, _, _ in artifact.filepaths})

# Then add the filepaths of the prep templates
for pt in artifact.prep_templates:
filepath_ids.update({fid for fid, _ in pt.get_filepaths()})

# Then add the filepaths of the sample template
filepath_ids.update(
{fid
for fid, _ in artifact.study.sample_template.get_filepaths()})

# Next, analyses
# Same as before, there are public, private, and shared
analyses = qdb.analysis.Analysis.get_by_status('public') | \
user.private_analyses | user.shared_analyses

for analysis in analyses:
filepath_ids.update(analysis.all_associated_filepath_ids)

return filepath_ids

access = False
sql = """SELECT
(SELECT count(*) FROM qiita.artifact_filepath
WHERE filepath_id = {0}) AS artifact,
(SELECT count(*) FROM qiita.sample_template_filepath
WHERE filepath_id = {0}) AS sample_info,
(SELECT count(*) FROM qiita.prep_template_filepath
WHERE filepath_id = {0}) AS prep_info,
(SELECT count(*) FROM qiita.job_results_filepath
WHERE filepath_id = {0}) AS job_results,
(SELECT count(*) FROM qiita.analysis_filepath
WHERE filepath_id = {0}) AS analysis""".format(filepath_id)
TRN.add(sql)

arid, sid, pid, jid, anid = TRN.execute_fetchflatten()

# artifacts
if arid:
# check the public artifacts
public_artifacts = qdb.artifact.Artifact.iter_public()
for artifact in public_artifacts:
if filepath_id in [fid for fid, _, _ in artifact.filepaths]:
access = True
break
# if not found check the user artifacts from their studies
if not access:
user_studies = user.user_studies | user.shared_studies
for s in user_studies:
if s.sample_template:
for a in s.artifacts():
if filepath_id in [fid[0] for fid in a.filepaths]:
access = True
break
# just avoiding extra loops if found
if access:
break
# sample info files
elif sid:
# check private and shared studies with the user
user_studies = user.user_studies | user.shared_studies
for s in user_studies:
st = s.sample_template
if st is not None:
# sample info files
if filepath_id in [fid for fid, _ in st.get_filepaths()]:
access = True
break
# if that didn't work let's check the public sample info files
if not access:
public_studies = qdb.study.Study.get_by_status('public')
for s in public_studies:
st = s.sample_template
if st is not None:
if filepath_id in [fid[0] for fid in
st.get_filepaths()]:
access = True
break
# prep info files
elif pid:
# check private and shared studies with the user
user_studies = user.user_studies | user.shared_studies
for s in user_studies:
for pt in s.prep_templates():
# sample info files
if filepath_id in [fid for fid, _ in pt.get_filepaths()]:
access = True
break
# if that didn't work let's check the public prep info files
if not access:
public_studies = qdb.study.Study.get_by_status('public')
for s in public_studies:
for pt in s.prep_templates():
if filepath_id in [fid[0]
for fid in pt.get_filepaths()]:
access = True
break
# next analyses
elif anid or jid:
analyses = qdb.analysis.Analysis.get_by_status('public') | \
user.private_analyses | user.shared_analyses
for analysis in analyses:
if filepath_id in analysis.all_associated_filepath_ids:
access = True
break

return access


def update_redis_stats():
Expand Down
103 changes: 53 additions & 50 deletions qiita_db/test/test_meta_util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,40 +39,38 @@ def _unshare_studies(self):
def _unshare_analyses(self):
self.conn_handler.execute("DELETE FROM qiita.analysis_users")

def test_get_accessible_filepath_ids(self):
def test_validate_filepath_access_by_user(self):
self._set_artifact_private()

# shared has access to all study files and analysis files

obs = qdb.meta_util.get_accessible_filepath_ids(
qdb.user.User('shared@foo.bar'))
self.assertItemsEqual(obs, {
1, 2, 3, 4, 5, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21})
user = qdb.user.User('shared@foo.bar')
for i in [1, 2, 3, 4, 5, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]:
self.assertTrue(qdb.meta_util.validate_filepath_access_by_user(
user, i))

# Now shared should not have access to the study files
self._unshare_studies()
obs = qdb.meta_util.get_accessible_filepath_ids(
qdb.user.User('shared@foo.bar'))
self.assertItemsEqual(obs, {16, 14, 15, 13})
for i in [1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21]:
self.assertFalse(qdb.meta_util.validate_filepath_access_by_user(
user, i))

for i in [13, 14, 15, 16]:
self.assertTrue(qdb.meta_util.validate_filepath_access_by_user(
user, i))

# Now shared should not have access to any files
self._unshare_analyses()
obs = qdb.meta_util.get_accessible_filepath_ids(
qdb.user.User('shared@foo.bar'))
self.assertEqual(obs, set())
for i in [1, 2, 3, 4, 5, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]:
self.assertFalse(qdb.meta_util.validate_filepath_access_by_user(
user, i))

# Now shared has access to public study files
self._set_artifact_public()
obs = qdb.meta_util.get_accessible_filepath_ids(
qdb.user.User('shared@foo.bar'))
self.assertEqual(obs, {1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21})
for i in [1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21]:
self.assertTrue(qdb.meta_util.validate_filepath_access_by_user(
user, i))

# Test that it doesn't break: if the SampleTemplate hasn't been added
exp = {1, 2, 3, 4, 5, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}
obs = qdb.meta_util.get_accessible_filepath_ids(
qdb.user.User('test@foo.bar'))
self.assertEqual(obs, exp)

info = {
"timeseries_type_id": 1,
"metadata_complete": True,
Expand All @@ -86,26 +84,28 @@ def test_get_accessible_filepath_ids(self):
"principal_investigator_id": 1,
"lab_person_id": 1
}
qdb.study.Study.create(
study = qdb.study.Study.create(
qdb.user.User('test@foo.bar'), "Test study", [1], info)
obs = qdb.meta_util.get_accessible_filepath_ids(
qdb.user.User('test@foo.bar'))
self.assertEqual(obs, exp)
for i in [1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21]:
self.assertTrue(qdb.meta_util.validate_filepath_access_by_user(
user, i))

# test in case there is a prep template that failed
self.conn_handler.execute(
"INSERT INTO qiita.prep_template (data_type_id) VALUES (2)")
obs = qdb.meta_util.get_accessible_filepath_ids(
qdb.user.User('test@foo.bar'))
self.assertEqual(obs, exp)
for i in [1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21]:
self.assertTrue(qdb.meta_util.validate_filepath_access_by_user(
user, i))

# admin should have access to everything
count = self.conn_handler.execute_fetchone("SELECT count(*) FROM "
"qiita.filepath")[0]
exp = set(range(1, count + 1))
obs = qdb.meta_util.get_accessible_filepath_ids(
qdb.user.User('admin@foo.bar'))
self.assertEqual(obs, exp)
admin = qdb.user.User('admin@foo.bar')
fids = self.conn_handler.execute_fetchall(
"SELECT filepath_id FROM qiita.filepath")
for i in fids:
self.assertTrue(qdb.meta_util.validate_filepath_access_by_user(
admin, i[0]))

qdb.study.Study.delete(study.id)

def test_get_lat_longs(self):
exp = [
Expand Down Expand Up @@ -181,18 +181,21 @@ def test_get_lat_longs_EMP_portal(self):

self.assertItemsEqual(obs, exp)

qdb.metadata_template.sample_template.SampleTemplate.delete(study.id)
qdb.study.Study.delete(study.id)

def test_update_redis_stats(self):
qdb.meta_util.update_redis_stats()

portal = qiita_config.portal
vals = [
('number_studies', {'sanbox': '2', 'public': '0',
'private': '1'}, r_client.hgetall),
('number_of_samples', {'sanbox': '1', 'public': '0',
'private': '27'}, r_client.hgetall),
('number_studies', {'sanbox': '0', 'public': '1',
'private': '0'}, r_client.hgetall),
('number_of_samples', {'sanbox': '0', 'public': '27',
'private': '0'}, r_client.hgetall),
('num_users', '4', r_client.get),
('lat_longs', EXP_LAT_LONG, r_client.get),
('num_studies_ebi', '3', r_client.get),
('num_studies_ebi', '1', r_client.get),
('num_samples_ebi', '27', r_client.get),
('number_samples_ebi_prep', '54', r_client.get)
# not testing img/time for simplicity
Expand All @@ -205,19 +208,19 @@ def test_update_redis_stats(self):


EXP_LAT_LONG = (
'[[0.291867635913, 68.5945325743], [68.0991287718, 34.8360987059],'
' [10.6655599093, 70.784770579], [40.8623799474, 6.66444220187],'
'[[60.1102854322, 74.7123248382], [23.1218032799, 42.838497795],'
' [3.21190859967, 26.8138925876], [74.0894932572, 65.3283470202],'
' [53.5050692395, 31.6056761814], [12.6245524972, 96.0693176066],'
' [43.9614715197, 82.8516734159], [10.6655599093, 70.784770579],'
' [78.3634273709, 74.423907894], [82.8302905615, 86.3615778099],'
' [44.9725384282, 66.1920014699], [4.59216095574, 63.5115213108],'
' [57.571893782, 32.5563076447], [40.8623799474, 6.66444220187],'
' [95.2060749748, 27.3592668624], [38.2627021402, 3.48274264219],'
' [13.089194595, 92.5274472082], [84.0030227585, 66.8954849864],'
' [12.7065957714, 84.9722975792], [78.3634273709, 74.423907894],'
' [82.8302905615, 86.3615778099], [53.5050692395, 31.6056761814],'
' [43.9614715197, 82.8516734159], [29.1499460692, 82.1270418227],'
' [23.1218032799, 42.838497795], [12.6245524972, 96.0693176066],'
' [38.2627021402, 3.48274264219], [74.0894932572, 65.3283470202],'
' [35.2374368957, 68.5041623253], [4.59216095574, 63.5115213108],'
' [95.2060749748, 27.3592668624], [68.51099627, 2.35063674718],'
' [85.4121476399, 15.6526750776], [60.1102854322, 74.7123248382],'
' [3.21190859967, 26.8138925876], [57.571893782, 32.5563076447],'
' [44.9725384282, 66.1920014699], [42.42, 41.41]]')
' [68.51099627, 2.35063674718], [29.1499460692, 82.1270418227],'
' [35.2374368957, 68.5041623253], [12.7065957714, 84.9722975792],'
' [0.291867635913, 68.5945325743], [85.4121476399, 15.6526750776],'
' [68.0991287718, 34.8360987059]]')

if __name__ == '__main__':
main()
2 changes: 1 addition & 1 deletion qiita_db/util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -894,12 +894,12 @@ def filepath_id_to_rel_path(filepath_id):
LEFT JOIN qiita.artifact_filepath USING (filepath_id)
WHERE filepath_id = %s"""
qdb.sql_connection.TRN.add(sql, [filepath_id])
# It should be only one row
mp, fp, sd, a_id = qdb.sql_connection.TRN.execute_fetchindex()[0]
if sd:
result = join(mp, str(a_id), fp)
else:
result = join(mp, fp)
# It should be only one row
return result


Expand Down

0 comments on commit 62140d9

Please sign in to comment.