Fixing merge conflicts

Author: josenavas

qiita_db/analysis.py

@@ -425,14 +425,12 @@ def jobs(self):
         Returns
         -------
         list of ints
-            Job ids for jobs in analysis
+            Job ids for jobs in analysis. Empty list if no jobs attached.
         """
         conn_handler = SQLConnectionHandler()
         sql = ("SELECT job_id FROM qiita.analysis_job WHERE "
                "analysis_id = %s".format(self._table))
         job_ids = conn_handler.execute_fetchall(sql, (self._id, ))
-        if job_ids == []:
-            return None
         return [job_id[0] for job_id in job_ids]
 
     @property

qiita_db/metadata_template/base_metadata_template.py

@@ -637,7 +637,11 @@ def _add_common_creation_steps_to_queue(cls, md_template, obj_id,
                            for col, dtype in zip(headers, datatypes)]
         conn_handler.add_to_queue(
             queue_name,
-            "CREATE TABLE qiita.{0} (sample_id varchar NOT NULL, {1})".format(
+            "CREATE TABLE qiita.{0} ("
+            "sample_id varchar NOT NULL, {1}, "
+            "CONSTRAINT fk_{0} FOREIGN KEY (sample_id) "
+            "REFERENCES qiita.study_sample (sample_id) "
+            "ON UPDATE CASCADE)".format(
                 table_name, ', '.join(column_datatype)))
 
         # Insert values on custom table
@@ -703,9 +707,9 @@ def _add_common_extend_steps_to_queue(self, md_template, conn_handler,
 
             if existing_samples:
                 warnings.warn(
-                    "No values have been modified for samples '%s'. However, "
-                    "the following columns have been added to them: '%s'"
-                    % (", ".join(existing_samples), ", ".join(new_cols)),
+                    "No values have been modified for existing samples (%s). "
+                    "However, the following columns have been added to them: "
+                    "'%s'" % (len(existing_samples), ", ".join(new_cols)),
                     QiitaDBWarning)
                 # The values for the new columns are the only ones that get
                 # added to the database. None of the existing values will be
@@ -727,8 +731,8 @@ def _add_common_extend_steps_to_queue(self, md_template, conn_handler,
                 conn_handler.add_to_queue(queue_name, sql, values, many=True)
         elif existing_samples:
             warnings.warn(
-                "The following samples already exist in the template and "
-                "will be ignored: %s" % ", ".join(existing_samples),
+                "%d samples already exist in the template and "
+                "their values won't be modified" % len(existing_samples),
                 QiitaDBWarning)
 
         if new_samples:

qiita_db/metadata_template/prep_template.py

@@ -232,7 +232,7 @@ def delete(cls, id_):
         conn_handler.execute(
             "DROP TABLE qiita.{0}".format(table_name))
 
-        # Remove the rows from common_prep_info
+        # Remove the rows from prep_template_samples
         conn_handler.execute(
             "DELETE FROM qiita.{0} where {1} = %s".format(cls._table,
                                                           cls._id_column),

qiita_db/metadata_template/sample_template.py

@@ -84,8 +84,6 @@ def metadata_headers():
         return [x[0] for x in
                 conn_handler.execute_fetchall(
                 "SELECT DISTINCT column_name FROM qiita.study_sample_columns "
-                "UNION SELECT column_name FROM information_schema.columns "
-                "WHERE table_name = 'required_sample_info' "
                 "ORDER BY column_name")]
 
     @classmethod

qiita_db/metadata_template/test/test_prep_template.py

@@ -627,7 +627,9 @@ def test_add_common_creation_steps_to_queue(self):
             'emp_status varchar, experiment_design_description varchar, '
             'library_construction_protocol varchar, '
             'linkerprimersequence varchar, platform varchar, '
-            'run_prefix varchar, str_column varchar)')
+            'run_prefix varchar, str_column varchar, '
+            'CONSTRAINT fk_prep_2 FOREIGN KEY (sample_id) REFERENCES '
+            'qiita.study_sample (sample_id) ON UPDATE CASCADE)')
 
         sql_insert_dynamic = (
             'INSERT INTO qiita.prep_2 '

qiita_db/metadata_template/test/test_sample_template.py

@@ -574,6 +574,19 @@ class TestSampleTemplateReadOnly(BaseTestSampleTemplate):
     def setUp(self):
         self._set_up()
 
+    def test_metadata_headers(self):
+        obs = SampleTemplate.metadata_headers()
+        exp = {'physical_specimen_location', 'physical_specimen_remaining',
+               'dna_extracted', 'sample_type', 'collection_timestamp',
+               'host_subject_id', 'description', 'season_environment',
+               'assigned_from_geo', 'texture', 'taxon_id', 'depth',
+               'host_taxid', 'common_name', 'water_content_soil', 'elevation',
+               'temp', 'tot_nitro', 'samp_salinity', 'altitude', 'env_biome',
+               'country', 'ph', 'anonymized_name', 'tot_org_carb',
+               'description_duplicate', 'env_feature', 'latitude', 'longitude',
+               'sample_id'}
+        self.assertEqual(set(obs), exp)
+
     def test_study_id(self):
         """Ensure that the correct study ID is returned"""
         self.assertEqual(self.tester.study_id, 1)
@@ -795,12 +808,14 @@ def test_add_common_creation_steps_to_queue(self):
 
         sql_crate_table = (
             'CREATE TABLE qiita.sample_2 (sample_id varchar NOT NULL, '
-            'collection_timestamp timestamp, ''description varchar, '
+            'collection_timestamp timestamp, description varchar, '
             'dna_extracted bool, host_subject_id varchar, int_column integer, '
             'latitude float8, longitude float8, '
             'physical_specimen_location varchar, '
             'physical_specimen_remaining bool, sample_type varchar, '
-            'str_column varchar)')
+            'str_column varchar, '
+            'CONSTRAINT fk_sample_2 FOREIGN KEY (sample_id) REFERENCES '
+            'qiita.study_sample (sample_id) ON UPDATE CASCADE)')
 
         sql_insert_dynamic = (
             'INSERT INTO qiita.sample_2 '
@@ -1111,7 +1126,7 @@ def test_create_int_prefix(self):
         # The returned object has the correct id
         self.assertEqual(st.id, new_id)
 
-        # The relevant rows to required_sample_info have been added.
+        # The relevant rows to study_sample have been added.
         obs = self.conn_handler.execute_fetchall(
             "SELECT * FROM qiita.study_sample WHERE study_id=%s", (new_id,))
         exp = [["%s.12.Sample1" % new_id, new_id],

qiita_db/metadata_template/util.py

@@ -113,7 +113,7 @@ def prefix_sample_names_with_id(md_template, study_id):
         # Create a new column on the metadata template that includes the
         # metadata template indexes prefixed with the study id
         md_template['sample_name_with_id'] = (study_ids + '.' +
-                                              md_template.index)
+                                              md_template.index.values)
         md_template.index = md_template.sample_name_with_id
         del md_template['sample_name_with_id']
         # The original metadata template had the index column unnamed - remove
@@ -221,13 +221,13 @@ def load_template_to_dataframe(fn, strip_whitespace=True):
                            parse_dates=True, index_col=False, comment='\t',
                            mangle_dupe_cols=False, converters={
                                'sample_name': lambda x: str(x).strip(),
-                               # required_sample_info
+                               # required sample template information
                                'physical_location': str,
                                'sample_type': str,
                                # collection_timestamp is not added here
                                'host_subject_id': str,
                                'description': str,
-                               # common_prep_info
+                               # common prep template information
                                'center_name': str,
                                'center_projct_name': str})
 

qiita_db/support_files/patches/24.sql

@@ -1,30 +1,57 @@
--- May 6, 2015
--- We attach the prep template directly to the study. The raw data is no longer
--- attached to the study directly, the prep template point to them. This will
--- make the RawData to be effectively just a container for the raw files,
--- which is how it was acting previously.
-
-CREATE TABLE qiita.study_prep_template ( 
-    study_id             bigint  NOT NULL,
-    prep_template_id     bigint  NOT NULL,
-    CONSTRAINT idx_study_raw_data PRIMARY KEY ( study_id, prep_template_id )
- ) ;
-
-CREATE INDEX idx_study_prep_template_0 ON qiita.study_prep_template ( study_id ) ;
-
-CREATE INDEX idx_study_prep_template_1 ON qiita.study_prep_template ( prep_template_id ) ;
-
-COMMENT ON TABLE qiita.study_prep_template IS 'links study to its prep templates';
-
-ALTER TABLE qiita.study_prep_template ADD CONSTRAINT fk_study_prep_template_study FOREIGN KEY ( study_id ) REFERENCES qiita.study( study_id )    ;
-
-ALTER TABLE qiita.study_prep_template ADD CONSTRAINT fk_study_prep_template_pt FOREIGN KEY ( prep_template_id ) REFERENCES qiita.prep_template( prep_template_id )    ;
-
--- Connect the existing prep templates in the system with their studies
-WITH pt_st AS (SELECT prep_template_id, study_id
-               FROM qiita.prep_template
-                    JOIN qiita.study_raw_data USING (raw_data_id))
-
-
-DROP TABLE qiita.study_raw_data;
-
+-- May 7, 2015
+-- This patch adds the ON UPDATE CASCADE constraint to all the FK
+-- that are referencing the sample ids
+
+DO $do$
+DECLARE
+    dyn_t varchar;
+    fk_vals RECORD;
+BEGIN
+
+-- The dynamic tables do not have a FK set on their sample ID
+-- We need to find the dynamic tables existing in the system and we add the
+-- FK constraint to them.
+FOR dyn_t IN
+    SELECT DISTINCT table_name
+    FROM information_schema.columns
+    WHERE (SUBSTR(table_name, 1, 7) = 'sample_'
+           OR SUBSTR(table_name, 1, 5) = 'prep_')
+        AND table_schema = 'qiita'
+        AND table_name NOT IN ('prep_template',
+                               'prep_template_preprocessed_data',
+                               'prep_template_filepath',
+                               'prep_columns',
+                               'sample_template_filepath',
+                               'prep_template_sample')
+LOOP
+    EXECUTE 'ALTER TABLE qiita.' || dyn_t || '
+                ADD CONSTRAINT fk_' || dyn_t || '
+                FOREIGN KEY (sample_id)'
+                'REFERENCES qiita.study_sample (sample_id);';
+END LOOP;
+
+-- Search for all the tables that are pointing to the sample_id
+-- and add the FK constraint with ON UPDATE CASCADE
+FOR fk_vals IN 
+    SELECT r.table_name, r.column_name, fk.constraint_name
+        FROM information_schema.constraint_column_usage u
+        INNER JOIN information_schema.referential_constraints fk
+            ON u.constraint_catalog = fk.unique_constraint_catalog
+            AND u.constraint_schema = fk.unique_constraint_schema
+            AND u.constraint_name = fk.unique_constraint_name
+        INNER JOIN information_schema.key_column_usage r
+            ON r.constraint_catalog = fk.constraint_catalog
+            AND r.constraint_schema = fk.constraint_schema
+            AND r.constraint_name = fk.constraint_name
+        WHERE u.column_name = 'sample_id'
+            AND u.table_schema = 'qiita'
+            AND u.table_name = 'study_sample'
+LOOP
+    EXECUTE 'ALTER TABLE qiita.' || fk_vals.table_name || '
+                DROP CONSTRAINT ' || fk_vals.constraint_name || ',
+                ADD CONSTRAINT ' || fk_vals.constraint_name ||'
+                    FOREIGN KEY (' || fk_vals.column_name ||')
+                    REFERENCES qiita.study_sample( sample_id )
+                    ON UPDATE CASCADE;';
+END LOOP;
+END $do$;

qiita_db/support_files/populate_test_db.sql

@@ -107,26 +107,26 @@ INSERT INTO qiita.study_sample (study_id, sample_id) VALUES
 -- Add the study sample columns for study 1
 INSERT INTO qiita.study_sample_columns (study_id, column_name, column_type) VALUES
 	(1, 'sample_id', 'varchar'),
-	(1, 'SEASON_ENVIRONMENT', 'varchar'),
-	(1, 'ASSIGNED_FROM_GEO', 'varchar'),
-	(1, 'TEXTURE', 'varchar'),
-	(1, 'TAXON_ID', 'varchar'),
-	(1, 'DEPTH', 'float8'),
-	(1, 'HOST_TAXID', 'varchar'),
-	(1, 'COMMON_NAME', 'varchar'),
-	(1, 'WATER_CONTENT_SOIL', 'float8'),
-	(1, 'ELEVATION', 'float8'),
-	(1, 'TEMP', 'float8'),
-	(1, 'TOT_NITRO', 'float8'),
-	(1, 'SAMP_SALINITY', 'float8'),
-	(1, 'ALTITUDE', 'float8'),
-	(1, 'ENV_BIOME', 'varchar'),
-	(1, 'COUNTRY', 'varchar'),
-	(1, 'PH', 'float8'),
-	(1, 'ANONYMIZED_NAME', 'varchar'),
-	(1, 'TOT_ORG_CARB', 'float8'),
-	(1, 'Description_duplicate', 'varchar'),
-	(1, 'ENV_FEATURE', 'varchar'),
+	(1, 'season_environment', 'varchar'),
+	(1, 'assigned_from_geo', 'varchar'),
+	(1, 'texture', 'varchar'),
+	(1, 'taxon_id', 'varchar'),
+	(1, 'depth', 'float8'),
+	(1, 'host_taxid', 'varchar'),
+	(1, 'common_name', 'varchar'),
+	(1, 'water_content_soil', 'float8'),
+	(1, 'elevation', 'float8'),
+	(1, 'temp', 'float8'),
+	(1, 'tot_nitro', 'float8'),
+	(1, 'samp_salinity', 'float8'),
+	(1, 'altitude', 'float8'),
+	(1, 'env_biome', 'varchar'),
+	(1, 'country', 'varchar'),
+	(1, 'ph', 'float8'),
+	(1, 'anonymized_name', 'varchar'),
+	(1, 'tot_org_carb', 'float8'),
+	(1, 'description_duplicate', 'varchar'),
+	(1, 'env_feature', 'varchar'),
 	(1, 'physical_specimen_location', 'varchar'),
 	(1, 'physical_specimen_remaining', 'bool'),
 	(1, 'dna_extracted', 'bool'),
@@ -140,26 +140,26 @@ INSERT INTO qiita.study_sample_columns (study_id, column_name, column_type) VALU
 -- Crate the sample_1 dynamic table
 CREATE TABLE qiita.sample_1 (
 	sample_id						varchar,
-	SEASON_ENVIRONMENT				varchar,
-	ASSIGNED_FROM_GEO				varchar,
-	TEXTURE							varchar,
-	TAXON_ID						varchar,
-	DEPTH							float8,
-	HOST_TAXID						varchar,
-	COMMON_NAME						varchar,
-	WATER_CONTENT_SOIL				float8,
-	ELEVATION						float8,
-	TEMP							float8,
-	TOT_NITRO						float8,
-	SAMP_SALINITY					float8,
-	ALTITUDE						float8,
-	ENV_BIOME						varchar,
-	COUNTRY							varchar,
-	PH								float8,
-	ANONYMIZED_NAME					varchar,
-	TOT_ORG_CARB					float8,
-	Description_duplicate			varchar,
-	ENV_FEATURE						varchar,
+	season_environment				varchar,
+	assigned_from_geo				varchar,
+	texture							varchar,
+	taxon_id						varchar,
+	depth							float8,
+	host_taxid						varchar,
+	common_name						varchar,
+	water_content_soil				float8,
+	elevation						float8,
+	temp							float8,
+	tot_nitro						float8,
+	samp_salinity					float8,
+	altitude						float8,
+	env_biome						varchar,
+	country							varchar,
+	ph								float8,
+	anonymized_name					varchar,
+	tot_org_carb					float8,
+	description_duplicate			varchar,
+	env_feature						varchar,
 	physical_specimen_location		varchar,
 	physical_specimen_remaining		bool,
 	dna_extracted					bool,
@@ -173,7 +173,7 @@ CREATE TABLE qiita.sample_1 (
 );
 
 -- Populates the sample_1 dynamic table
-INSERT INTO qiita.sample_1 (sample_id, SEASON_ENVIRONMENT, ASSIGNED_FROM_GEO, TEXTURE, TAXON_ID, DEPTH, HOST_TAXID, COMMON_NAME, WATER_CONTENT_SOIL, ELEVATION, TEMP, TOT_NITRO, SAMP_SALINITY, ALTITUDE, ENV_BIOME, COUNTRY, PH, ANONYMIZED_NAME, TOT_ORG_CARB, Description_duplicate, ENV_FEATURE, physical_specimen_location, physical_specimen_remaining, dna_extracted, sample_type, collection_timestamp, host_subject_id, description, latitude, longitude) VALUES
+INSERT INTO qiita.sample_1 (sample_id, season_environment, assigned_from_geo, texture, taxon_id, depth, host_taxid, common_name, water_content_soil, elevation, temp, tot_nitro, samp_salinity, altitude, env_biome, country, ph, anonymized_name, tot_org_carb, description_duplicate, env_feature, physical_specimen_location, physical_specimen_remaining, dna_extracted, sample_type, collection_timestamp, host_subject_id, description, latitude, longitude) VALUES
 	('1.SKM7.640188', 'winter', 'n', '63.1 sand, 17.7 silt, 19.2 clay', '1118232', 0.15, '3483', 'root metagenome', 0.101, 114, 15, 1.3, 7.44, 0, 'ENVO:Temperate grasslands, savannas, and shrubland biome', 'GAZ:United States of America', 6.82, 'SKM7', 3.31, 'Bucu Roots', 'ENVO:plant-associated habitat', 'ANL', TRUE, TRUE, 'ENVO:soil', '2011-11-11 13:00', '1001:B6', 'Cannabis Soil Microbiome', 60.1102854322, 74.7123248382),
 	('1.SKD9.640182', 'winter', 'n', '66 sand, 16.3 silt, 17.7 clay', '1118232', 0.15, '3483', 'root metagenome', 0.178, 114, 15, 1.51, 7.1, 0, 'ENVO:Temperate grasslands, savannas, and shrubland biome', 'GAZ:United States of America', 6.82, 'SKD9', 4.32, 'Diesel Root', 'ENVO:plant-associated habitat', 'ANL', TRUE, TRUE, 'ENVO:soil', '2011-11-11 13:00', '1001:D3', 'Cannabis Soil Microbiome', 23.1218032799, 42.838497795),
 	('1.SKM8.640201', 'winter', 'n', '63.1 sand, 17.7 silt, 19.2 clay', '1118232', 0.15, '3483', 'root metagenome', 0.101, 114, 15, 1.3, 7.44, 0, 'ENVO:Temperate grasslands, savannas, and shrubland biome', 'GAZ:United States of America', 6.82, 'SKM8', 3.31, 'Bucu Roots', 'ENVO:plant-associated habitat', 'ANL', TRUE, TRUE, 'ENVO:soil', '2011-11-11 13:00', '1001:D8', 'Cannabis Soil Microbiome', 3.21190859967, 26.8138925876),