From 155fcaf05525452dcfd2a160b097256a88395a81 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Thu, 21 May 2015 22:11:09 -0700 Subject: [PATCH 01/10] Fixing test_base.py --- qiita_db/test/test_base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/qiita_db/test/test_base.py b/qiita_db/test/test_base.py index 1f439d6c1..7716f3981 100644 --- a/qiita_db/test/test_base.py +++ b/qiita_db/test/test_base.py @@ -13,7 +13,7 @@ from qiita_db.base import QiitaObject, QiitaStatusObject from qiita_db.exceptions import QiitaDBUnknownIDError from qiita_db.data import RawData -from qiita_db.study import Study +from qiita_db.study import Study, StudyPerson from qiita_db.analysis import Analysis @@ -63,8 +63,9 @@ def test_equal(self): def test_not_equal(self): """Not equals works with object of the same type""" - new = RawData(2) - self.assertNotEqual(self.tester, new) + sp1 = StudyPerson(1) + sp2 = StudyPerson(2) + self.assertNotEqual(sp1, sp2) def test_not_equal_type(self): """Not equals works with object of different type""" From 4e5b9ce8826578710b062a91cf51aa0f6f730975 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Thu, 21 May 2015 22:14:22 -0700 Subject: [PATCH 02/10] Fixing qiita_db/test/test_setup.py --- qiita_db/test/test_setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/qiita_db/test/test_setup.py b/qiita_db/test/test_setup.py index 247291cc9..83273f951 100644 --- a/qiita_db/test/test_setup.py +++ b/qiita_db/test/test_setup.py @@ -39,19 +39,19 @@ def test_processed_data_status(self): self.assertEqual(get_count("qiita.processed_data_status"), 4) def test_filepath(self): - self.assertEqual(get_count("qiita.filepath"), 20) + self.assertEqual(get_count("qiita.filepath"), 16) def test_filepath_type(self): self.assertEqual(get_count("qiita.filepath_type"), 19) def test_raw_data(self): - self.assertEqual(get_count("qiita.raw_data"), 4) + self.assertEqual(get_count("qiita.raw_data"), 1) def test_raw_filepath(self): - self.assertEqual(get_count("qiita.raw_filepath"), 4) + self.assertEqual(get_count("qiita.raw_filepath"), 2) - def test_study_raw_data(self): - self.assertEqual(get_count("qiita.study_raw_data"), 4) + def test_study_prep_template(self): + self.assertEqual(get_count("qiita.study_prep_template"), 1) def test_required_sample_info(self): self.assertEqual(get_count("qiita.study_sample"), 27) From 9f5a29ad548de50d5e14edc88a0baa8ae900177d Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Thu, 21 May 2015 22:46:19 -0700 Subject: [PATCH 03/10] Fixing study.py and tests. Removing add_raw_data as it is no longer needed --- qiita_db/study.py | 51 +++++++++++++++++-------------------- qiita_db/test/test_study.py | 20 ++++++--------- 2 files changed, 31 insertions(+), 40 deletions(-) diff --git a/qiita_db/study.py b/qiita_db/study.py index d32019721..2ee98acd9 100644 --- a/qiita_db/study.py +++ b/qiita_db/study.py @@ -642,10 +642,11 @@ def data_types(self): list of str """ conn_handler = SQLConnectionHandler() - sql = ("SELECT DISTINCT DT.data_type FROM qiita.study_raw_data SRD " - "JOIN qiita.prep_template PT ON SRD.raw_data_id = " - "PT.raw_data_id JOIN qiita.data_type DT ON PT.data_type_id = " - "DT.data_type_id WHERE SRD.study_id = %s") + sql = """SELECT DISTINCT data_type + FROM qiita.study_prep_template + JOIN qiita.prep_template USING (prep_template_id) + JOIN qiita.data_type USING (data_type_id) + WHERE study_id = %s""" return [x[0] for x in conn_handler.execute_fetchall(sql, (self._id,))] @property @@ -759,36 +760,30 @@ def raw_data(self, data_type=None): return [x[0] for x in conn_handler.execute_fetchall(sql, (self._id,))] - def add_raw_data(self, raw_data): - """ Adds raw_data to the current study + def prep_templates(self, data_type=None): + """Return list of prep template ids Parameters ---------- - raw_data : list of RawData - The RawData objects to be added to the study + data_type : str, optional + If given, retrieve only prep templates for given datatype. + Default None. - Raises - ------ - QiitaDBError - If the raw_data is already linked to the current study + Returns + ------- + list of PrepTemplate ids """ - conn_handler = SQLConnectionHandler() - self._lock_non_sandbox(conn_handler) - queue = "%d_add_raw_data" % self.id - sql = ("SELECT EXISTS(SELECT * FROM qiita.study_raw_data WHERE " - "study_id=%s AND raw_data_id=%s)") - conn_handler.create_queue(queue) - sql_args = [(self.id, rd.id) for rd in raw_data] - conn_handler.add_to_queue(queue, sql, sql_args, many=True) - linked = conn_handler.execute_queue(queue) - - if any(linked): - raise QiitaDBError("Some of the passed raw datas have been already" - " linked to the study %s" % self.id) + spec_data = "" + if data_type: + spec_data = " AND data_type_id = %s" % convert_to_id(data_type, + "data_type") - conn_handler.executemany( - "INSERT INTO qiita.study_raw_data (study_id, raw_data_id) " - "VALUES (%s, %s)", sql_args) + conn_handler = SQLConnectionHandler() + sql = """SELECT prep_template_id + FROM qiita.study_prep_template + JOIN qiita.prep_template USING (prep_template_id) + WHERE study_id = %s{0}""".format(spec_data) + return [x[0] for x in conn_handler.execute_fetchall(sql, (self._id,))] def preprocessed_data(self, data_type=None): """ Returns list of data ids for preprocessed data info diff --git a/qiita_db/test/test_study.py b/qiita_db/test/test_study.py index bbe98229a..0b4b74d92 100644 --- a/qiita_db/test/test_study.py +++ b/qiita_db/test/test_study.py @@ -2,6 +2,7 @@ from datetime import datetime from future.utils import viewitems +import pandas as pd from qiita_core.exceptions import IncompetentQiitaDeveloperError from qiita_core.util import qiita_test_checker @@ -11,6 +12,7 @@ from qiita_db.user import User from qiita_db.data import RawData from qiita_db.util import convert_to_id +from qiita_db.metadata_template import PrepTemplate from qiita_db.exceptions import ( QiitaDBColumnError, QiitaDBStatusError, QiitaDBError, QiitaDBUnknownIDError) @@ -586,26 +588,20 @@ def test_retrieve_data_types_none(self): self.assertEqual(new.data_types, []) def test_retrieve_raw_data(self): - self.assertEqual(self.study.raw_data(), [1, 2, 3, 4]) + self.assertEqual(self.study.raw_data(), [1]) def test_retrieve_raw_data_none(self): new = Study.create(User('test@foo.bar'), 'NOT Identification of the ' 'Microbiomes for Cannabis Soils', [1], self.info) self.assertEqual(new.raw_data(), []) - def test_add_raw_data(self): - self._change_processed_data_status('awaiting_approval') + def test_retrieve_prep_templates(self): + self.assertEqual(self.study.prep_templates(), [1]) + + def test_retrieve_prep_templates_none(self): new = Study.create(User('test@foo.bar'), 'NOT Identification of the ' 'Microbiomes for Cannabis Soils', [1], self.info) - new.add_raw_data([RawData(1), RawData(2)]) - obs = self.conn_handler.execute_fetchall( - "SELECT * FROM qiita.study_raw_data WHERE study_id=%s", - (new.id,)) - self.assertEqual(obs, [[new.id, 1], [new.id, 2]]) - - def test_add_raw_data_private(self): - with self.assertRaises(QiitaDBStatusError): - self.study.add_raw_data([RawData(2)]) + self.assertEqual(new.prep_templates(), []) def test_retrieve_preprocessed_data(self): self.assertEqual(self.study.preprocessed_data(), [1, 2]) From a76deecc06d80824cd4acbb8e94a1be5e13ffe8e Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Thu, 21 May 2015 23:27:49 -0700 Subject: [PATCH 04/10] Fixing util tests --- qiita_db/test/test_util.py | 45 +++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/qiita_db/test/test_util.py b/qiita_db/test/test_util.py index 6861b9f74..10e2b81f4 100644 --- a/qiita_db/test/test_util.py +++ b/qiita_db/test/test_util.py @@ -12,12 +12,15 @@ from os.path import join, exists, basename from shutil import rmtree +import pandas as pd + from qiita_core.util import qiita_test_checker from qiita_core.exceptions import IncompetentQiitaDeveloperError from qiita_db.exceptions import QiitaDBColumnError, QiitaDBError from qiita_db.data import RawData from qiita_db.study import Study from qiita_db.reference import Reference +from qiita_db.metadata_template import PrepTemplate from qiita_db.util import (exists_table, exists_dynamic_table, scrub_data, compute_checksum, check_table_cols, check_required_columns, convert_to_id, @@ -367,6 +370,18 @@ def _common_purge_filpeaths_test(self): join(raw_data_mp, '2_sequences_barcodes.fastq.gz'), join(raw_data_mp, '2_sequences.fastq.gz')] + for fp in removed_fps: + with open(fp, 'w') as f: + f.write('\n') + + sql = """INSERT INTO qiita.filepath + (filepath, filepath_type_id, checksum, + checksum_algorithm_id, data_directory_id) + VALUES ('2_sequences_barcodes.fastq.gz', 3, '852952723', 1, 5), + ('2_sequences.fastq.gz', 1, '852952723', 1, 5) + RETURNING filepath_id""" + fp_ids = self.conn_handler.execute_fetchall(sql) + fps = set(fps).difference(removed_fps) # Check that the files exist @@ -387,9 +402,9 @@ def _common_purge_filpeaths_test(self): # Check that the 2 rows that have been removed are the correct ones sql = """SELECT EXISTS( SELECT * FROM qiita.filepath WHERE filepath_id = %s)""" - obs = self.conn_handler.execute_fetchone(sql, (3,))[0] + obs = self.conn_handler.execute_fetchone(sql, (fp_ids[0][0],))[0] self.assertFalse(obs) - obs = self.conn_handler.execute_fetchone(sql, (4,))[0] + obs = self.conn_handler.execute_fetchone(sql, (fp_ids[1][0],))[0] self.assertFalse(obs) # Check that the files have been successfully removed @@ -421,9 +436,19 @@ def test_move_filepaths_to_upload_folder(self): # files fd, seqs_fp = mkstemp(suffix='_seqs.fastq') close(fd) - study_id = 1 - - rd = RawData.create(2, [Study(study_id)], [(seqs_fp, 1)]) + st = Study(1) + metadata_dict = { + 'SKB8.640193': {'center_name': 'ANL', + 'primer': 'GTGCCAGCMGCCGCGGTAA', + 'barcode': 'GTCCGCAAGTTA', + 'run_prefix': "s_G1_L001_sequences", + 'platform': 'ILLUMINA', + 'library_construction_protocol': 'AAAA', + 'experiment_design_description': 'BBBB'}} + metadata = pd.DataFrame.from_dict(metadata_dict, orient='index') + pt = PrepTemplate.create(metadata, Study(1), "16S") + + rd = RawData.create(2, [pt], [(seqs_fp, 1)]) filepaths = rd.get_filepaths() # deleting reference so we can directly call # move_filepaths_to_upload_folder @@ -432,10 +457,10 @@ def test_move_filepaths_to_upload_folder(self): "DELETE FROM qiita.raw_filepath WHERE filepath_id=%s", (fid,)) # moving filepaths - move_filepaths_to_upload_folder(study_id, filepaths) + move_filepaths_to_upload_folder(st.id, filepaths) # check that they do not exist in the old path but do in the new one - path_for_removal = join(get_mountpoint("uploads")[0][1], str(study_id)) + path_for_removal = join(get_mountpoint("uploads")[0][1], str(st.id)) for _, fp, _ in filepaths: self.assertFalse(exists(fp)) new_fp = join(path_for_removal, basename(fp).split('_', 1)[1]) @@ -615,21 +640,21 @@ def test_filepath_id_to_rel_path(self): exp = 'raw_data/1_s_G1_L001_sequences.fastq.gz' self.assertEqual(obs, exp) - obs = filepath_id_to_rel_path(5) + obs = filepath_id_to_rel_path(3) exp = 'preprocessed_data/1_seqs.fna' self.assertEqual(obs, exp) def test_filepath_ids_to_rel_paths(self): obs = filepath_ids_to_rel_paths([1, 3]) exp = {1: 'raw_data/1_s_G1_L001_sequences.fastq.gz', - 3: 'raw_data/2_sequences.fastq.gz'} + 3: 'preprocessed_data/1_seqs.fna'} self.assertEqual(obs, exp) def test_check_access_to_analysis_result(self): obs = check_access_to_analysis_result('test@foo.bar', '1_job_result.txt') - exp = [12] + exp = [10] self.assertEqual(obs, exp) From 1c756d9c4c1e681f0e3cb4ae051c9e52ea454cba Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Thu, 21 May 2015 23:33:12 -0700 Subject: [PATCH 05/10] Fixing meta-util tests --- qiita_db/test/test_meta_util.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/qiita_db/test/test_meta_util.py b/qiita_db/test/test_meta_util.py index 8a2519e62..451052e70 100644 --- a/qiita_db/test/test_meta_util.py +++ b/qiita_db/test/test_meta_util.py @@ -36,13 +36,12 @@ def test_get_accessible_filepath_ids(self): # shared has access to all study files and analysis files obs = get_accessible_filepath_ids(User('shared@foo.bar')) - self.assertEqual(obs, set([1, 2, 5, 6, 7, 11, 12, 13, 14, 15, 16, 17, - 18, 19, 20])) + self.assertEqual(obs, {1, 2, 3, 4, 5, 9, 10, 11, 12, 13, 14, 15, 16}) # Now shared should not have access to the study files self._unshare_studies() obs = get_accessible_filepath_ids(User('shared@foo.bar')) - self.assertEqual(obs, set([12, 13, 14, 15])) + self.assertEqual(obs, {10, 11, 12, 13}) # Now shared should not have access to any files self._unshare_analyses() @@ -52,10 +51,10 @@ def test_get_accessible_filepath_ids(self): # Now shared has access to public study files self._set_processed_data_public() obs = get_accessible_filepath_ids(User('shared@foo.bar')) - self.assertEqual(obs, set([1, 2, 5, 6, 7, 11, 16, 19, 20])) + self.assertEqual(obs, {1, 2, 3, 4, 5, 9, 14, 15, 16}) # Test that it doesn't break: if the SampleTemplate hasn't been added - exp = set([1, 2, 5, 6, 7, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]) + exp = {1, 2, 3, 4, 5, 9, 10, 11, 12, 13, 14, 15, 16} obs = get_accessible_filepath_ids(User('test@foo.bar')) self.assertEqual(obs, exp) From 9c61f5b4c364df200d9a500e8b80b49a7698feea Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Thu, 21 May 2015 23:37:24 -0700 Subject: [PATCH 06/10] Fixing test_analysis.py --- qiita_db/test/test_analysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiita_db/test/test_analysis.py b/qiita_db/test/test_analysis.py index fbe344f06..b71964668 100644 --- a/qiita_db/test/test_analysis.py +++ b/qiita_db/test/test_analysis.py @@ -276,7 +276,7 @@ def test_retrieve_biom_tables(self): self.assertEqual(self.analysis.biom_tables, exp) def test_all_associated_filepaths(self): - exp = {12, 13, 14, 15} + exp = {10, 11, 12, 13} self.assertEqual(self.analysis.all_associated_filepath_ids, exp) def test_retrieve_biom_tables_none(self): @@ -427,7 +427,7 @@ def test_build_mapping_file(self): obs = self.conn_handler.execute_fetchall( sql, ("%d_analysis_mapping.txt" % self.analysis.id,)) - exp = [[15, '1_analysis_mapping.txt', 9, '852952723', 1, 1], + exp = [[13, '1_analysis_mapping.txt', 9, '852952723', 1, 1], [new_id, '1_analysis_mapping.txt', 9, '1606265094', 1, 1]] self.assertEqual(obs, exp) From 41070c22353ac320ca5fe81ad1299ad8ef5bd4e1 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Thu, 21 May 2015 23:39:15 -0700 Subject: [PATCH 07/10] Fix test_job.py --- qiita_db/test/test_job.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qiita_db/test/test_job.py b/qiita_db/test/test_job.py index e3f1a95af..1c54043c2 100644 --- a/qiita_db/test/test_job.py +++ b/qiita_db/test/test_job.py @@ -123,7 +123,7 @@ def test_delete_files(self): Job(1) obs = self.conn_handler.execute_fetchall( - "SELECT * FROM qiita.filepath WHERE filepath_id = 12") + "SELECT * FROM qiita.filepath WHERE filepath_id = 10") self.assertEqual(obs, []) obs = self.conn_handler.execute_fetchall( @@ -149,7 +149,7 @@ def test_delete_folders(self): Job(2) obs = self.conn_handler.execute_fetchall( - "SELECT * FROM qiita.filepath WHERE filepath_id = 13") + "SELECT * FROM qiita.filepath WHERE filepath_id = 11") self.assertEqual(obs, []) obs = self.conn_handler.execute_fetchall( @@ -300,7 +300,7 @@ def test_add_results(self): obs = self.conn_handler.execute_fetchall( "SELECT * FROM qiita.job_results_filepath WHERE job_id = 1") - self.assertEqual(obs, [[1, 12], [1, fp_count + 1]]) + self.assertEqual(obs, [[1, 10], [1, fp_count + 1]]) def test_add_results_dir(self): fp_count = get_count('qiita.filepath') @@ -313,7 +313,7 @@ def test_add_results_dir(self): # make sure files attached to job properly obs = self.conn_handler.execute_fetchall( "SELECT * FROM qiita.job_results_filepath WHERE job_id = 1") - self.assertEqual(obs, [[1, 12], [1, fp_count + 1]]) + self.assertEqual(obs, [[1, 10], [1, fp_count + 1]]) def test_add_results_completed(self): self.job.status = "completed" From 1dbd1b7aa9827f4e34b868817a5689a9e49b9a4f Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Thu, 21 May 2015 23:50:30 -0700 Subject: [PATCH 08/10] Fix commands and CLI --- qiita_db/commands.py | 17 +++++------- qiita_db/test/test_commands.py | 47 ++++++++++++---------------------- scripts/qiita | 18 ++++++------- 3 files changed, 32 insertions(+), 50 deletions(-) diff --git a/qiita_db/commands.py b/qiita_db/commands.py index d1c940fb5..8d859837c 100644 --- a/qiita_db/commands.py +++ b/qiita_db/commands.py @@ -153,7 +153,7 @@ def load_sample_template_from_cmd(sample_temp_path, study_id): return SampleTemplate.create(sample_temp, Study(study_id)) -def load_prep_template_from_cmd(prep_temp_path, raw_data_id, study_id, +def load_prep_template_from_cmd(prep_temp_path, study_id, data_type): r"""Adds a prep template to the database @@ -161,19 +161,16 @@ def load_prep_template_from_cmd(prep_temp_path, raw_data_id, study_id, ---------- prep_temp_path : str Path to the prep template file - raw_data_id : int - The raw data id to which the prep template belongs study_id : int The study id to which the prep template belongs data_type : str The data type of the prep template """ prep_temp = load_template_to_dataframe(prep_temp_path) - return PrepTemplate.create(prep_temp, RawData(raw_data_id), - Study(study_id), data_type) + return PrepTemplate.create(prep_temp, Study(study_id), data_type) -def load_raw_data_cmd(filepaths, filepath_types, filetype, study_ids): +def load_raw_data_cmd(filepaths, filepath_types, filetype, prep_template_ids): """Add new raw data by populating the relevant tables Parameters @@ -184,8 +181,8 @@ def load_raw_data_cmd(filepaths, filepath_types, filetype, study_ids): Describes the contents of the files. filetype : str The type of file being loaded - study_ids : iterable of int - The IDs of the studies with which to associate this raw data + prep_template_ids : iterable of int + The IDs of the prep templates with which to associate this raw data Returns ------- @@ -202,9 +199,9 @@ def load_raw_data_cmd(filepaths, filepath_types, filetype, study_ids): filepath_types_dict = get_filepath_types() filepath_types = [filepath_types_dict[x] for x in filepath_types] - studies = [Study(x) for x in study_ids] + prep_templates = [PrepTemplate(x) for x in prep_template_ids] - return RawData.create(filetype_id, studies, + return RawData.create(filetype_id, prep_templates, filepaths=list(zip(filepaths, filepath_types))) diff --git a/qiita_db/test/test_commands.py b/qiita_db/test/test_commands.py index ebf3fa1ac..74d998547 100644 --- a/qiita_db/test/test_commands.py +++ b/qiita_db/test/test_commands.py @@ -15,6 +15,8 @@ from future import standard_library from functools import partial +import pandas as pd + from qiita_db.commands import (load_study_from_cmd, load_raw_data_cmd, load_sample_template_from_cmd, load_prep_template_from_cmd, @@ -28,6 +30,7 @@ from qiita_db.data import RawData, PreprocessedData from qiita_db.util import (get_count, check_count, get_db_files_base_dir, get_mountpoint) +from qiita_db.metadata_template import PrepTemplate from qiita_core.util import qiita_test_checker from qiita_ware.processing_pipeline import generate_demux_file @@ -154,36 +157,12 @@ def test_load_sample_template_from_cmd(self): @qiita_test_checker() class TestLoadPrepTemplateFromCmd(TestCase): def setUp(self): - # Create a sample template file - fd, seqs_fp = mkstemp(suffix='_seqs.fastq') - close(fd) - fd, barcodes_fp = mkstemp(suffix='_barcodes.fastq') - close(fd) - - with open(seqs_fp, "w") as f: - f.write("\n") - with open(barcodes_fp, "w") as f: - f.write("\n") - self.pt_contents = PREP_TEMPLATE - self.raw_data = RawData.create( - 2, [Study(1)], filepaths=[(seqs_fp, 1), (barcodes_fp, 2)]) - - join_f = partial(join, join(get_db_files_base_dir(), 'raw_data')) - self.files_to_remove = [ - join_f("%s_%s" % (self.raw_data.id, basename(seqs_fp))), - join_f("%s_%s" % (self.raw_data.id, basename(barcodes_fp)))] - - def tearDown(self): - for fp in self.files_to_remove: - if exists(fp): - remove(fp) - def test_load_prep_template_from_cmd(self): """Correctly adds a prep template to the DB""" fh = StringIO(self.pt_contents) - st = load_prep_template_from_cmd(fh, self.raw_data.id, 1, '18S') + st = load_prep_template_from_cmd(fh, 1, '18S') self.assertEqual(st.id, 2) @@ -222,14 +201,24 @@ def test_load_data_from_cmd(self): 'raw_barcodes'] filetype = 'FASTQ' - study_ids = [1] + metadata_dict = { + 'SKB8.640193': {'center_name': 'ANL', + 'primer': 'GTGCCAGCMGCCGCGGTAA', + 'barcode': 'GTCCGCAAGTTA', + 'run_prefix': "s_G1_L001_sequences", + 'platform': 'ILLUMINA', + 'library_construction_protocol': 'AAAA', + 'experiment_design_description': 'BBBB'}} + metadata = pd.DataFrame.from_dict(metadata_dict, orient='index') + pt1 = PrepTemplate.create(metadata, Study(1), "16S") + prep_templates = [pt1.id] initial_raw_count = get_count('qiita.raw_data') initial_fp_count = get_count('qiita.filepath') initial_raw_fp_count = get_count('qiita.raw_filepath') new = load_raw_data_cmd(filepaths, filepath_types, filetype, - study_ids) + prep_templates) raw_data_id = new.id self.files_to_remove.append( join(self.db_test_raw_dir, @@ -246,14 +235,12 @@ def test_load_data_from_cmd(self): initial_fp_count + 3)) self.assertTrue(check_count('qiita.raw_filepath', initial_raw_fp_count + 3)) - self.assertTrue(check_count('qiita.study_raw_data', - initial_raw_count + 1)) # Ensure that the ValueError is raised when a filepath_type is not # provided for each and every filepath with self.assertRaises(ValueError): load_raw_data_cmd(filepaths, filepath_types[:-1], filetype, - study_ids) + prep_templates) @qiita_test_checker() diff --git a/scripts/qiita b/scripts/qiita index 2fe3ad944..b73c32438 100755 --- a/scripts/qiita +++ b/scripts/qiita @@ -109,13 +109,13 @@ def load_study(owner, title, info): @click.option('--filetype', required=True, type=click.Choice(get_filetypes().keys()), help='The type of data') -@click.option('--study', multiple=True, help='Associate the data with this ' - 'study. This option can be used multiple times if the data ' - 'should be associated with multiple studies', type=int, - required=True) -def load_raw(fp, fp_type, filetype, study): +@click.option('--prep_template', multiple=True, help='Associate the data with ' + 'this prep template. This option can be used multiple times if ' + 'the data should be associated with multiple prep templates', + type=int, required=True) +def load_raw(fp, fp_type, filetype, prep_template): """Loads a raw data to the database""" - raw_data = load_raw_data_cmd(fp, fp_type, filetype, study) + raw_data = load_raw_data_cmd(fp, fp_type, filetype, prep_template) click.echo("Raw data successfully added to the database with id %s" % raw_data.id) @@ -202,16 +202,14 @@ def load_sample_template(fp, study): @db.command() @click.argument('fp', required=True, type=click.Path(resolve_path=True, readable=True, exists=True)) -@click.option('--raw_data', required=True, type=int, - help='Associate the prep template with this raw data') @click.option('--study', required=True, type=int, help='Associate the prep template with this study') @click.option('--data_type', required=True, type=click.Choice(get_data_types()), help="The data type of data") -def load_prep_template(fp, raw_data, study, data_type): +def load_prep_template(fp, study, data_type): """Loads a sample template to the database""" - prep_template = load_prep_template_from_cmd(fp, raw_data, study, data_type) + prep_template = load_prep_template_from_cmd(fp, study, data_type) click.echo("Prep template successfully added to the database with id %s" % prep_template.id) From 2b613e01329496853af89720a1d548166a073856 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Thu, 21 May 2015 23:52:28 -0700 Subject: [PATCH 09/10] Fixing flake8 --- qiita_db/data.py | 5 ++--- qiita_db/metadata_template/test/test_prep_template.py | 5 ++--- qiita_db/test/test_commands.py | 2 +- qiita_db/test/test_study.py | 3 --- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/qiita_db/data.py b/qiita_db/data.py index f44e62769..cf052bdf3 100644 --- a/qiita_db/data.py +++ b/qiita_db/data.py @@ -87,9 +87,8 @@ from .sql_connection import SQLConnectionHandler from .exceptions import QiitaDBError, QiitaDBUnknownIDError, QiitaDBStatusError from .util import (exists_dynamic_table, insert_filepaths, convert_to_id, - convert_from_id, purge_filepaths, get_filepath_id, - get_mountpoint, move_filepaths_to_upload_folder, - infer_status) + convert_from_id, get_filepath_id, get_mountpoint, + move_filepaths_to_upload_folder, infer_status) class BaseData(QiitaObject): diff --git a/qiita_db/metadata_template/test/test_prep_template.py b/qiita_db/metadata_template/test/test_prep_template.py index e53c866b4..e0cee75db 100644 --- a/qiita_db/metadata_template/test/test_prep_template.py +++ b/qiita_db/metadata_template/test/test_prep_template.py @@ -10,7 +10,7 @@ from unittest import TestCase, main from tempfile import mkstemp from os import close, remove -from os.path import join, basename +from os.path import join from collections import Iterable import numpy.testing as npt @@ -29,8 +29,7 @@ from qiita_db.sql_connection import SQLConnectionHandler from qiita_db.study import Study from qiita_db.data import RawData, ProcessedData -from qiita_db.util import (exists_table, get_db_files_base_dir, get_mountpoint, - get_count) +from qiita_db.util import exists_table, get_mountpoint, get_count from qiita_db.metadata_template.prep_template import PrepTemplate, PrepSample from qiita_db.metadata_template.sample_template import SampleTemplate, Sample from qiita_db.metadata_template.constants import ( diff --git a/qiita_db/test/test_commands.py b/qiita_db/test/test_commands.py index 74d998547..20df805ee 100644 --- a/qiita_db/test/test_commands.py +++ b/qiita_db/test/test_commands.py @@ -27,7 +27,7 @@ from qiita_db.environment_manager import patch from qiita_db.study import Study, StudyPerson from qiita_db.user import User -from qiita_db.data import RawData, PreprocessedData +from qiita_db.data import PreprocessedData from qiita_db.util import (get_count, check_count, get_db_files_base_dir, get_mountpoint) from qiita_db.metadata_template import PrepTemplate diff --git a/qiita_db/test/test_study.py b/qiita_db/test/test_study.py index 0b4b74d92..efde608d2 100644 --- a/qiita_db/test/test_study.py +++ b/qiita_db/test/test_study.py @@ -2,7 +2,6 @@ from datetime import datetime from future.utils import viewitems -import pandas as pd from qiita_core.exceptions import IncompetentQiitaDeveloperError from qiita_core.util import qiita_test_checker @@ -10,9 +9,7 @@ from qiita_db.study import Study, StudyPerson from qiita_db.investigation import Investigation from qiita_db.user import User -from qiita_db.data import RawData from qiita_db.util import convert_to_id -from qiita_db.metadata_template import PrepTemplate from qiita_db.exceptions import ( QiitaDBColumnError, QiitaDBStatusError, QiitaDBError, QiitaDBUnknownIDError) From 7c9e1d2383bd3a2fb5fae8539ac649320fb9e2f8 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Wed, 27 May 2015 18:16:41 -0500 Subject: [PATCH 10/10] Addressing comment --- qiita_db/commands.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qiita_db/commands.py b/qiita_db/commands.py index 8d859837c..9b9859729 100644 --- a/qiita_db/commands.py +++ b/qiita_db/commands.py @@ -153,8 +153,7 @@ def load_sample_template_from_cmd(sample_temp_path, study_id): return SampleTemplate.create(sample_temp, Study(study_id)) -def load_prep_template_from_cmd(prep_temp_path, study_id, - data_type): +def load_prep_template_from_cmd(prep_temp_path, study_id, data_type): r"""Adds a prep template to the database Parameters