From da56fc52d3a058f102f5f05af758f052d0e665f9 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Tue, 7 Apr 2015 13:50:21 -0700 Subject: [PATCH 1/4] Enforcing order in database cols --- qiita_db/metadata_template/base_metadata_template.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiita_db/metadata_template/base_metadata_template.py b/qiita_db/metadata_template/base_metadata_template.py index 24079cddd..f25c90f74 100644 --- a/qiita_db/metadata_template/base_metadata_template.py +++ b/qiita_db/metadata_template/base_metadata_template.py @@ -542,7 +542,7 @@ def _add_common_creation_steps_to_queue(cls, md_template, obj_id, headers = list(md_template.keys()) # Get the required columns from the DB - db_cols = get_table_cols(cls._table, conn_handler) + db_cols = sorted(get_table_cols(cls._table, conn_handler)) # Remove the sample_id and _id_column columns db_cols.remove('sample_id') db_cols.remove(cls._id_column) @@ -561,7 +561,7 @@ def _add_common_creation_steps_to_queue(cls, md_template, obj_id, values, many=True) # Insert rows on *_columns table - headers = list(set(headers).difference(db_cols)) + headers = sorted(list(set(headers).difference(db_cols))) datatypes = get_datatypes(md_template.ix[:, headers]) # psycopg2 requires a list of tuples, in which each tuple is a set # of values to use in the string formatting of the query. We have all From 52505bc4cd9571e4a5a27cfd7bf6f14728a59cae Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Tue, 7 Apr 2015 13:50:41 -0700 Subject: [PATCH 2/4] Fixing prep template tests --- .../test/test_prep_template.py | 106 +++++++++--------- 1 file changed, 56 insertions(+), 50 deletions(-) diff --git a/qiita_db/metadata_template/test/test_prep_template.py b/qiita_db/metadata_template/test/test_prep_template.py index 5a43347fe..dae6a144e 100644 --- a/qiita_db/metadata_template/test/test_prep_template.py +++ b/qiita_db/metadata_template/test/test_prep_template.py @@ -582,39 +582,40 @@ def test_add_common_creation_steps_to_queue(self): 'VALUES (%s, %s, %s)') sql_create_table = ( - 'CREATE TABLE qiita.prep_2 ' - '(sample_id varchar NOT NULL, str_column varchar, ' - 'run_prefix varchar, barcodesequence varchar, platform varchar, ' - 'linkerprimersequence varchar, ' - 'experiment_design_description varchar, ' - 'library_construction_protocol varchar)') + 'CREATE TABLE qiita.prep_2 (sample_id varchar NOT NULL, ' + 'barcodesequence varchar, experiment_design_description varchar, ' + 'library_construction_protocol varchar, ' + 'linkerprimersequence varchar, platform varchar, ' + 'run_prefix varchar, str_column varchar)') sql_insert_dynamic = ( 'INSERT INTO qiita.prep_2 ' - '(sample_id, str_column, run_prefix, barcodesequence, platform, ' - 'linkerprimersequence, experiment_design_description, ' - 'library_construction_protocol) ' + '(sample_id, barcodesequence, experiment_design_description, ' + 'library_construction_protocol, linkerprimersequence, platform, ' + 'run_prefix, str_column) ' 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s)') sql_insert_dynamic_params_1 = ( - '2.SKB8.640193', 'Value for sample 1', 's_G1_L001_sequences', - 'GTCCGCAAGTTA', 'ILLUMINA', 'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA') + '2.SKB8.640193', 'GTCCGCAAGTTA', 'BBBB', 'AAAA', + 'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences', + 'Value for sample 1') sql_insert_dynamic_params_2 = ( - '2.SKD8.640184', 'Value for sample 2', 's_G1_L001_sequences', - 'CGTAGAGCTCTC', 'ILLUMINA', 'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA') + '2.SKD8.640184', 'CGTAGAGCTCTC', 'BBBB', 'AAAA', + 'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences', + 'Value for sample 2') exp = [ (sql_insert_common, sql_insert_common_params_1), (sql_insert_common, sql_insert_common_params_2), - (sql_insert_prep_columns, (2, 'str_column', 'varchar')), - (sql_insert_prep_columns, (2, 'run_prefix', 'varchar')), (sql_insert_prep_columns, (2, 'barcodesequence', 'varchar')), - (sql_insert_prep_columns, (2, 'platform', 'varchar')), - (sql_insert_prep_columns, (2, 'linkerprimersequence', 'varchar')), (sql_insert_prep_columns, (2, 'experiment_design_description', 'varchar')), (sql_insert_prep_columns, (2, 'library_construction_protocol', 'varchar')), + (sql_insert_prep_columns, (2, 'linkerprimersequence', 'varchar')), + (sql_insert_prep_columns, (2, 'platform', 'varchar')), + (sql_insert_prep_columns, (2, 'run_prefix', 'varchar')), + (sql_insert_prep_columns, (2, 'str_column', 'varchar')), (sql_create_table, None), (sql_insert_dynamic, sql_insert_dynamic_params_1), (sql_insert_dynamic, sql_insert_dynamic_params_2)] @@ -794,17 +795,20 @@ def test_create(self): # The new table hosts the correct values obs = self.conn_handler.execute_fetchall( "SELECT * FROM qiita.prep_2") - # sample_id, study_id, str_column, ebi_submission_accession, - # run_prefix, barcodesequence, linkerprimersequence - exp = [['1.SKB7.640196', 'Value for sample 3', 'ILLUMINA', - 's_G1_L002_sequences', 'CCTCTGAGAGCT', None, - 'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'], - ['1.SKB8.640193', 'Value for sample 1', 'ILLUMINA', - 's_G1_L001_sequences', 'GTCCGCAAGTTA', None, - 'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'], - ['1.SKD8.640184', 'Value for sample 2', 'ILLUMINA', - 's_G1_L001_sequences', 'CGTAGAGCTCTC', None, - 'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA']] + + # barcodesequence, ebi_submission_accession, + # experiment_design_description, library_construction_protocol, + # linkerprimersequence, platform, run_prefix, str_column + exp = [['1.SKB7.640196', 'CCTCTGAGAGCT', None, 'BBBB', 'AAAA', + 'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L002_sequences', + 'Value for sample 3'], + ['1.SKB8.640193', 'GTCCGCAAGTTA', None, 'BBBB', 'AAAA', + 'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences', + 'Value for sample 1'], + ['1.SKD8.640184', 'CGTAGAGCTCTC', None, 'BBBB', 'AAAA', + 'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences', + 'Value for sample 2']] + self.assertEqual(sorted(obs), sorted(exp)) # prep and qiime files have been created @@ -858,17 +862,18 @@ def test_create_already_prefixed_samples(self): # The new table hosts the correct values obs = self.conn_handler.execute_fetchall( "SELECT * FROM qiita.prep_2") - # sample_id, study_id, str_column, ebi_submission_accession, - # run_prefix, barcodesequence, linkerprimersequence - exp = [['1.SKB7.640196', 'Value for sample 3', 'ILLUMINA', - 's_G1_L002_sequences', 'CCTCTGAGAGCT', None, - 'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'], - ['1.SKB8.640193', 'Value for sample 1', 'ILLUMINA', - 's_G1_L001_sequences', 'GTCCGCAAGTTA', None, - 'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'], - ['1.SKD8.640184', 'Value for sample 2', 'ILLUMINA', - 's_G1_L001_sequences', 'CGTAGAGCTCTC', None, - 'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA']] + # barcodesequence, ebi_submission_accession, + # experiment_design_description, library_construction_protocol, + # linkerprimersequence, platform, run_prefix, str_column + exp = [['1.SKB7.640196', 'CCTCTGAGAGCT', None, 'BBBB', 'AAAA', + 'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L002_sequences', + 'Value for sample 3'], + ['1.SKB8.640193', 'GTCCGCAAGTTA', None, 'BBBB', 'AAAA', + 'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences', + 'Value for sample 1'], + ['1.SKD8.640184', 'CGTAGAGCTCTC', None, 'BBBB', 'AAAA', + 'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences', + 'Value for sample 2']] self.assertEqual(sorted(obs), sorted(exp)) # prep and qiime files have been created @@ -959,17 +964,18 @@ def test_create_data_type_id(self): # The new table hosts the correct values obs = self.conn_handler.execute_fetchall( "SELECT * FROM qiita.prep_2") - # sample_id, str_column, ebi_submission_accession, - # run_prefix, barcodesequence, linkerprimersequence - exp = [['1.SKB7.640196', 'Value for sample 3', 'ILLUMINA', - 's_G1_L002_sequences', 'CCTCTGAGAGCT', None, - 'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'], - ['1.SKB8.640193', 'Value for sample 1', 'ILLUMINA', - 's_G1_L001_sequences', 'GTCCGCAAGTTA', None, - 'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'], - ['1.SKD8.640184', 'Value for sample 2', 'ILLUMINA', - 's_G1_L001_sequences', 'CGTAGAGCTCTC', None, - 'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA']] + # barcodesequence, ebi_submission_accession, + # experiment_design_description, library_construction_protocol, + # linkerprimersequence, platform, run_prefix, str_column + exp = [['1.SKB7.640196', 'CCTCTGAGAGCT', None, 'BBBB', 'AAAA', + 'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L002_sequences', + 'Value for sample 3'], + ['1.SKB8.640193', 'GTCCGCAAGTTA', None, 'BBBB', 'AAAA', + 'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences', + 'Value for sample 1'], + ['1.SKD8.640184', 'CGTAGAGCTCTC', None, 'BBBB', 'AAAA', + 'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences', + 'Value for sample 2']] self.assertEqual(sorted(obs), sorted(exp)) def test_create_error(self): From e0b269827d9e2774b39b4bd362068e3d6bb423b4 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Tue, 7 Apr 2015 14:11:14 -0700 Subject: [PATCH 3/4] Fixing sample template tests --- .../test/test_sample_template.py | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/qiita_db/metadata_template/test/test_sample_template.py b/qiita_db/metadata_template/test/test_sample_template.py index 0639cdb64..d44f2c902 100644 --- a/qiita_db/metadata_template/test/test_sample_template.py +++ b/qiita_db/metadata_template/test/test_sample_template.py @@ -802,23 +802,20 @@ def test_add_common_creation_steps_to_queue(self): sql_insert_required = ( 'INSERT INTO qiita.required_sample_info ' - '(study_id, sample_id, physical_location, has_physical_specimen, ' - 'has_extracted_data, sample_type, required_sample_info_status_id, ' - 'collection_timestamp, host_subject_id, description, latitude, ' - 'longitude) ' + '(study_id, sample_id, collection_timestamp, description, ' + 'has_extracted_data, has_physical_specimen, host_subject_id, ' + 'latitude, longitude, physical_location, ' + 'required_sample_info_status_id, sample_type) ' 'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)') sql_insert_required_params_1 = ( - 2, '2.Sample1', 'location1', True, True, 'type1', 1, - datetime(2014, 5, 29, 12, 24, 51), 'NotIdentified', - 'Test Sample 1', 42.42, 41.41) + 2, '2.Sample1', datetime(2014, 5, 29, 12, 24, 51), 'Test Sample 1', + True, True, 'NotIdentified', 42.42, 41.41, 'location1', 1, 'type1') sql_insert_required_params_2 = ( - 2, '2.Sample2', 'location1', True, True, 'type1', 1, - datetime(2014, 5, 29, 12, 24, 51), 'NotIdentified', - 'Test Sample 2', 4.2, 1.1) + 2, '2.Sample2', datetime(2014, 5, 29, 12, 24, 51), 'Test Sample 2', + True, True, 'NotIdentified', 4.2, 1.1, 'location1', 1, 'type1') sql_insert_required_params_3 = ( - 2, '2.Sample3', 'location1', True, True, 'type1', 1, - datetime(2014, 5, 29, 12, 24, 51), 'NotIdentified', - 'Test Sample 3', 4.8, 4.41) + 2, '2.Sample3', datetime(2014, 5, 29, 12, 24, 51), 'Test Sample 3', + True, True, 'NotIdentified', 4.8, 4.41, 'location1', 1, 'type1') sql_insert_sample_cols = ( 'INSERT INTO qiita.study_sample_columns ' From 6270a6d9607982c70a8b10eb48c7ea45be49ecb8 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Wed, 8 Apr 2015 10:07:40 -0700 Subject: [PATCH 4/4] Addressing @ElDeveloper's comment --- qiita_db/metadata_template/base_metadata_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_db/metadata_template/base_metadata_template.py b/qiita_db/metadata_template/base_metadata_template.py index f25c90f74..3f8d1e10f 100644 --- a/qiita_db/metadata_template/base_metadata_template.py +++ b/qiita_db/metadata_template/base_metadata_template.py @@ -561,7 +561,7 @@ def _add_common_creation_steps_to_queue(cls, md_template, obj_id, values, many=True) # Insert rows on *_columns table - headers = sorted(list(set(headers).difference(db_cols))) + headers = sorted(set(headers).difference(db_cols)) datatypes = get_datatypes(md_template.ix[:, headers]) # psycopg2 requires a list of tuples, in which each tuple is a set # of values to use in the string formatting of the query. We have all