diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index bf81ddf41..e5eb581eb 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -1463,8 +1463,17 @@ def prep_templates(self): FROM qiita.preparation_artifact WHERE artifact_id = %s""" qdb.sql_connection.TRN.add(sql, [self.id]) - return [qdb.metadata_template.prep_template.PrepTemplate(pt_id) - for pt_id in qdb.sql_connection.TRN.execute_fetchflatten()] + templates = [qdb.metadata_template.prep_template.PrepTemplate(pt_id) # noqa + for pt_id in qdb.sql_connection.TRN.execute_fetchflatten()] # noqa + + if len(templates) > 1: + # We never expect an artifact to be associated with multiple + # preparations + ids = [p.id for p in templates] + msg = f"Artifact({self.id}) associated with preps: {sorted(ids)}" + raise ValueError(msg) + + return templates @property def study(self): diff --git a/qiita_pet/handlers/rest/__init__.py b/qiita_pet/handlers/rest/__init__.py index 73ad9382a..913758457 100644 --- a/qiita_pet/handlers/rest/__init__.py +++ b/qiita_pet/handlers/rest/__init__.py @@ -7,6 +7,7 @@ # ----------------------------------------------------------------------------- from .study import StudyHandler, StudyCreatorHandler, StudyStatusHandler +from .study_association import StudyAssociationHandler from .study_samples import (StudySamplesHandler, StudySamplesInfoHandler, StudySamplesCategoriesHandler, StudySamplesDetailHandler, @@ -25,6 +26,7 @@ ENDPOINTS = ( (r"/api/v1/study$", StudyCreatorHandler), (r"/api/v1/study/([0-9]+)$", StudyHandler), + (r"/api/v1/study/([0-9]+)/associations$", StudyAssociationHandler), (r"/api/v1/study/([0-9]+)/samples/categories=([a-zA-Z\-0-9\.:,_]*)", StudySamplesCategoriesHandler), (r"/api/v1/study/([0-9]+)/samples", StudySamplesHandler), diff --git a/qiita_pet/handlers/rest/study_association.py b/qiita_pet/handlers/rest/study_association.py new file mode 100644 index 000000000..855d33d42 --- /dev/null +++ b/qiita_pet/handlers/rest/study_association.py @@ -0,0 +1,220 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) 2014--, The Qiita Development Team. +# +# Distributed under the terms of the BSD 3-clause License. +# +# The full license is in the file LICENSE, distributed with this software. +# ----------------------------------------------------------------------------- + + +from qiita_db.handlers.oauth2 import authenticate_oauth +from .rest_handler import RESTHandler + + +# terms used more than once +_STUDY = 'study' +_PREP = 'prep' +_FILEPATH = 'filepath' +_STATUS = 'status' +_ARTIFACT = 'artifact' +_SAMPLE = 'sample' +_METADATA = 'metadata' +_TEMPLATE = 'template' +_ID = 'id' +_PROCESSING = 'processing' +_TYPE = 'type' + +# payload keys +STUDY_ID = f'{_STUDY}_{_ID}' +STUDY_SAMPLE_METADATA_FILEPATH = f'{_STUDY}_{_SAMPLE}_{_METADATA}_{_FILEPATH}' +PREP_TEMPLATES = f'{_PREP}_{_TEMPLATE}s' +PREP_ID = f'{_PREP}_{_ID}' +PREP_STATUS = f'{_PREP}_{_STATUS}' +PREP_SAMPLE_METADATA_FILEPATH = f'{_PREP}_{_SAMPLE}_{_METADATA}_{_FILEPATH}' +PREP_DATA_TYPE = f'{_PREP}_data_{_TYPE}' +PREP_HUMAN_FILTERING = f'{_PREP}_human_filtering' +PREP_ARTIFACTS = f'{_PREP}_{_ARTIFACT}s' +ARTIFACT_ID = f'{_ARTIFACT}_{_ID}' +ARTIFACT_STATUS = f'{_ARTIFACT}_{_STATUS}' +ARTIFACT_PARENT_IDS = f'{_ARTIFACT}_parent_{_ID}s' +ARTIFACT_BASAL_ID = f'{_ARTIFACT}_basal_{_ID}' +ARTIFACT_PROCESSING_ID = f'{_ARTIFACT}_{_PROCESSING}_{_ID}' +ARTIFACT_PROCESSING_NAME = f'{_ARTIFACT}_{_PROCESSING}_name' +ARTIFACT_PROCESSING_ARGUMENTS = f'{_ARTIFACT}_{_PROCESSING}_arguments' +ARTIFACT_FILEPATHS = f'{_ARTIFACT}_{_FILEPATH}s' +ARTIFACT_FILEPATH = f'{_ARTIFACT}_{_FILEPATH}' +ARTIFACT_FILEPATH_TYPE = f'{_ARTIFACT}_{_FILEPATH}_{_TYPE}' +ARTIFACT_FILEPATH_ID = f'{_ARTIFACT}_{_FILEPATH}_{_ID}' + + +def _most_recent_template_path(template): + """Obtain the most recent available template filepath""" + filepaths = template.get_filepaths() + + # the test dataset shows that a prep can exist without a prep template + if len(filepaths) == 0: + return None + + # [0] -> the highest file by ID + # [1] -> the filepath + return filepaths[0][1] + + +def _set_study(payload, study): + """Set study level information""" + filepath = _most_recent_template_path(study.sample_template) + + payload[STUDY_ID] = study.id + payload[STUDY_SAMPLE_METADATA_FILEPATH] = filepath + + +def _set_prep_templates(payload, study): + """Set prep template level information""" + template_data = [] + for pt in study.prep_templates(): + _set_prep_template(template_data, pt) + payload[PREP_TEMPLATES] = template_data + + +def _get_human_filtering(prep_template): + """Obtain the human filtering if applied""" + # .current_human_filtering does not describe what the human filter is + # so we will examine the first artifact off the prep + if prep_template.artifact is not None: + return prep_template.artifact.human_reads_filter_method + + +def _set_prep_template(template_payload, prep_template): + """Set an individual prep template information""" + filepath = _most_recent_template_path(prep_template) + + current_template = {} + current_template[PREP_ID] = prep_template.id + current_template[PREP_STATUS] = prep_template.status + current_template[PREP_SAMPLE_METADATA_FILEPATH] = filepath + current_template[PREP_DATA_TYPE] = prep_template.data_type() + current_template[PREP_HUMAN_FILTERING] = _get_human_filtering(prep_template) # noqa + + _set_artifacts(current_template, prep_template) + + template_payload.append(current_template) + + +def _get_artifacts(prep_template): + """Get artifact information associated with a prep""" + if prep_template.artifact is None: + return [] + + pending_artifact_objects = [prep_template.artifact, ] + all_artifact_objects = set(pending_artifact_objects[:]) + + while pending_artifact_objects: + artifact = pending_artifact_objects.pop() + pending_artifact_objects.extend(artifact.children) + all_artifact_objects.update(set(artifact.children)) + + return sorted(all_artifact_objects, key=lambda artifact: artifact.id) + + +def _set_artifacts(template_payload, prep_template): + """Set artifact information specific to a prep""" + prep_artifacts = [] + + if prep_template.artifact is None: + basal_id = None + else: + basal_id = prep_template.artifact.id + + for artifact in _get_artifacts(prep_template): + _set_artifact(prep_artifacts, artifact, basal_id) + template_payload[PREP_ARTIFACTS] = prep_artifacts + + +def _set_artifact(prep_artifacts, artifact, basal_id): + """Set artifact specific information""" + artifact_payload = {} + artifact_payload[ARTIFACT_ID] = artifact.id + + # Prep uses .status, artifact uses .visibility + # favoring .status as visibility implies a UI + artifact_payload[ARTIFACT_STATUS] = artifact.visibility + + parents = [parent.id for parent in artifact.parents] + artifact_payload[ARTIFACT_PARENT_IDS] = parents if parents else None + artifact_payload[ARTIFACT_BASAL_ID] = basal_id + + _set_artifact_processing(artifact_payload, artifact) + _set_artifact_filepaths(artifact_payload, artifact) + + prep_artifacts.append(artifact_payload) + + +def _set_artifact_processing(artifact_payload, artifact): + """Set processing parameter information associated with an artifact""" + processing_parameters = artifact.processing_parameters + if processing_parameters is None: + artifact_processing_id = None + artifact_processing_name = None + artifact_processing_arguments = None + else: + command = processing_parameters.command + artifact_processing_id = command.id + artifact_processing_name = command.name + artifact_processing_arguments = processing_parameters.values + + artifact_payload[ARTIFACT_PROCESSING_ID] = artifact_processing_id + artifact_payload[ARTIFACT_PROCESSING_NAME] = artifact_processing_name + artifact_payload[ARTIFACT_PROCESSING_ARGUMENTS] = artifact_processing_arguments # noqa + + +def _set_artifact_filepaths(artifact_payload, artifact): + """Set filepath information associated with an artifact""" + artifact_filepaths = [] + for filepath_data in artifact.filepaths: + local_payload = {} + local_payload[ARTIFACT_FILEPATH] = filepath_data['fp'] + local_payload[ARTIFACT_FILEPATH_ID] = filepath_data['fp_id'] + local_payload[ARTIFACT_FILEPATH_TYPE] = filepath_data['fp_type'] + artifact_filepaths.append(local_payload) + + # the test study includes an artifact which does not have filepaths + if len(artifact_filepaths) == 0: + artifact_filepaths = None + + artifact_payload[ARTIFACT_FILEPATHS] = artifact_filepaths + + +class StudyAssociationHandler(RESTHandler): + @authenticate_oauth + def get(self, study_id): + study = self.safe_get_study(study_id) + if study is None: + return + + # schema: + # STUDY_ID: , + # STUDY_SAMPLE_METADATA_FILEPATH: , + # PREP_TEMPLATES: None | list[dict] + # PREP_ID: , + # PREP_STATUS: , + # PREP_SAMPLE_METADATA_FILEPATH: , + # PREP_DATA_TYPE: , + # PREP_HUMAN_FILTERING: None | , + # PREP_ARTIFACTS: None | list[dict] + # ARTIFACT_ID: , + # ARTIFACT_STATUS: , + # ARTIFACT_PARENT_IDS: None | list[int], + # ARTIFACT_BASAL_ID: None | , + # ARTIFACT_PROCESSING_ID: None | , + # ARTIFACT_PROCESSING_NAME: None | , + # ARTIFACT_FILEPATH: , + # ARTIFACT_FILEPATH_TYPE': + # + payload = {} + _set_study(payload, study) + _set_prep_templates(payload, study) + self.write(payload) + self.finish() diff --git a/qiita_pet/support_files/doc/source/dev/rest.rst b/qiita_pet/support_files/doc/source/dev/rest.rst index 8707831d7..fee62f6da 100755 --- a/qiita_pet/support_files/doc/source/dev/rest.rst +++ b/qiita_pet/support_files/doc/source/dev/rest.rst @@ -106,6 +106,8 @@ This is the currently internal but planned to be external (general users) API. +--------+-----------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ |GET | ``/api/v1/study//status`` | The status of a study (whether or not the study: is public, has sample information, sample information has warnings and a list of existing preparations. | +--------+-----------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ +|GET | ``/api/v1/study//associations`` | Comprehensive information about a study, associated prep and artifact information, and file locations | ++--------+-----------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ |GET | ``/api/v1/person`` | Get list of persons. | +--------+-----------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+ |GET | ``/api/v1/person?name=foo&affiliation=bar`` | See if a person exists. | diff --git a/qiita_pet/test/rest/test_study_associations.py b/qiita_pet/test/rest/test_study_associations.py new file mode 100644 index 000000000..43df423b3 --- /dev/null +++ b/qiita_pet/test/rest/test_study_associations.py @@ -0,0 +1,197 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) 2014--, The Qiita Development Team. +# +# Distributed under the terms of the BSD 3-clause License. +# +# The full license is in the file LICENSE, distributed with this software. +# ----------------------------------------------------------------------------- + +from unittest import main + +from tornado.escape import json_decode + +from qiita_pet.test.rest.test_base import RESTHandlerTestCase + + +class StudyAssociationTests(RESTHandlerTestCase): + def test_get_valid(self): + IGNORE = 'IGNORE' + exp = {'study_id': 1, + 'study_sample_metadata_filepath': IGNORE, + 'prep_templates': [ + {'prep_id': 1, + 'prep_status': 'private', + 'prep_sample_metadata_filepath': IGNORE, + 'prep_data_type': '18S', + 'prep_human_filtering': 'The greatest human filtering method', # noqa + 'prep_artifacts': [ + {'artifact_id': 1, + 'artifact_status': 'private', + 'artifact_parent_ids': None, + 'artifact_basal_id': 1, + 'artifact_processing_id': None, + 'artifact_processing_name': None, + 'artifact_processing_arguments': None, + 'artifact_filepaths': [ + {'artifact_filepath_id': 1, + 'artifact_filepath': IGNORE, + 'artifact_filepath_type': 'raw_forward_seqs'}, + {'artifact_filepath_id': 2, + 'artifact_filepath': IGNORE, + 'artifact_filepath_type': 'raw_barcodes'}]}, + {'artifact_id': 2, + 'artifact_status': 'private', + 'artifact_parent_ids': [1], + 'artifact_basal_id': 1, + 'artifact_processing_id': 1, + 'artifact_processing_name': 'Split libraries FASTQ', + 'artifact_processing_arguments': { + 'input_data': '1', + 'max_bad_run_length': '3', + 'min_per_read_length_fraction': '0.75', + 'sequence_max_n': '0', + 'rev_comp_barcode': 'False', + 'rev_comp_mapping_barcodes': 'False', + 'rev_comp': 'False', + 'phred_quality_threshold': '3', + 'barcode_type': 'golay_12', + 'max_barcode_errors': '1.5', + 'phred_offset': 'auto'}, + 'artifact_filepaths': [ + {'artifact_filepath_id': 3, + 'artifact_filepath': IGNORE, + 'artifact_filepath_type': 'preprocessed_fasta'}, + {'artifact_filepath': IGNORE, + 'artifact_filepath_id': 4, + 'artifact_filepath_type': 'preprocessed_fastq'}, + {'artifact_filepath': IGNORE, + 'artifact_filepath_id': 5, + 'artifact_filepath_type': 'preprocessed_demux'}]}, + {'artifact_id': 3, + 'artifact_status': 'private', + 'artifact_parent_ids': [1], + 'artifact_basal_id': 1, + 'artifact_processing_id': 1, + 'artifact_processing_name': 'Split libraries FASTQ', + 'artifact_processing_arguments': { + 'input_data': '1', + 'max_bad_run_length': '3', + 'min_per_read_length_fraction': '0.75', + 'sequence_max_n': '0', + 'rev_comp_barcode': 'False', + 'rev_comp_mapping_barcodes': 'True', + 'rev_comp': 'False', + 'phred_quality_threshold': '3', + 'barcode_type': 'golay_12', + 'max_barcode_errors': '1.5', + 'phred_offset': 'auto'}, + 'artifact_filepaths': None}, + {'artifact_id': 4, + 'artifact_status': 'private', + 'artifact_parent_ids': [2], + 'artifact_basal_id': 1, + 'artifact_processing_id': 3, + 'artifact_processing_name': 'Pick closed-reference OTUs', + 'artifact_processing_arguments': { + 'input_data': '2', + 'reference': '1', + 'sortmerna_e_value': '1', + 'sortmerna_max_pos': '10000', + 'similarity': '0.97', + 'sortmerna_coverage': '0.97', + 'threads': '1'}, + 'artifact_filepaths': [{ + 'artifact_filepath_id': 9, + 'artifact_filepath': IGNORE, + 'artifact_filepath_type': 'biom'}]}, + {'artifact_id': 5, + 'artifact_status': 'private', + 'artifact_parent_ids': [2], + 'artifact_basal_id': 1, + 'artifact_processing_id': 3, + 'artifact_processing_name': 'Pick closed-reference OTUs', + 'artifact_processing_arguments': { + 'input_data': '2', + 'reference': '1', + 'sortmerna_e_value': '1', + 'sortmerna_max_pos': '10000', + 'similarity': '0.97', + 'sortmerna_coverage': '0.97', + 'threads': '1'}, + 'artifact_filepaths': [{ + 'artifact_filepath_id': 9, + 'artifact_filepath': IGNORE, + 'artifact_filepath_type': 'biom'}]}, + {'artifact_id': 6, + 'artifact_status': 'private', + 'artifact_parent_ids': [2], + 'artifact_basal_id': 1, + 'artifact_processing_id': 3, + 'artifact_processing_name': 'Pick closed-reference OTUs', + 'artifact_processing_arguments': { + 'input_data': '2', + 'reference': '2', + 'sortmerna_e_value': '1', + 'sortmerna_max_pos': '10000', + 'similarity': '0.97', + 'sortmerna_coverage': '0.97', + 'threads': '1'}, + 'artifact_filepaths': [{ + 'artifact_filepath_id': 12, + 'artifact_filepath': IGNORE, + 'artifact_filepath_type': 'biom'}]}]}, + {'prep_id': 2, + 'prep_status': 'private', + 'prep_sample_metadata_filepath': IGNORE, + 'prep_data_type': '18S', + 'prep_human_filtering': None, + 'prep_artifacts': [{ + 'artifact_id': 7, + 'artifact_parent_ids': None, + 'artifact_basal_id': 7, + 'artifact_status': 'private', + 'artifact_processing_id': None, + 'artifact_processing_name': None, + 'artifact_processing_arguments': None, + 'artifact_filepaths': [{ + 'artifact_filepath_id': 22, + 'artifact_filepath': IGNORE, + 'artifact_filepath_type': 'biom'}]}]}]} + + response = self.get('/api/v1/study/1/associations', + headers=self.headers) + self.assertEqual(response.code, 200) + obs = json_decode(response.body) + + def _process_dict(d): + return [(d, k) for k in d] + + def _process_list(list_): + if list_ is None: + return [] + + return [dk for d in list_ + for dk in _process_dict(d)] + + stack = _process_dict(obs) + while stack: + (d, k) = stack.pop() + if k.endswith('filepath'): + d[k] = IGNORE + elif k.endswith('filepaths'): + stack.extend(_process_list(d[k])) + elif k.endswith('templates'): + stack.extend(_process_list(d[k])) + elif k.endswith('artifacts'): + stack.extend(_process_list(d[k])) + + self.assertEqual(obs, exp) + + def test_get_invalid(self): + response = self.get('/api/v1/study/0/associations', + headers=self.headers) + self.assertEqual(response.code, 404) + + +if __name__ == '__main__': + main()