Merge pull request #1199 from josenavas/issue-1194

Fixes 1194

Author: wasade

qiita_db/metadata_template/constants.py

@@ -15,7 +15,9 @@
 SAMPLE_TEMPLATE_COLUMNS = {
     # The following columns are required by EBI for submission
     'EBI': Restriction(columns={'collection_timestamp': 'timestamp',
-                                'physical_specimen_location': 'varchar'},
+                                'physical_specimen_location': 'varchar',
+                                'taxon_id': 'integer',
+                                'scientific_name': 'varchar'},
                        error_msg="EBI submission disabled"),
     # The following columns are required for the official main QIITA site
     'qiita_main': Restriction(columns={'sample_type': 'varchar',

qiita_db/metadata_template/test/test_sample_template.py

@@ -209,7 +209,9 @@ def test_retrieve_dropped_samples(self):
                             'Description': 'Test Sample 1',
                             'str_column': 'Value for sample 1',
                             'latitude': 42.42,
-                            'longitude': 41.41},
+                            'longitude': 41.41,
+                            'taxon_id': 9606,
+                            'scientific_name': 'homo sapiens'},
             'SKD8.640184': {'physical_specimen_location': 'location1',
                             'physical_specimen_remaining': True,
                             'dna_extracted': True,
@@ -221,7 +223,9 @@ def test_retrieve_dropped_samples(self):
                             'Description': 'Test Sample 2',
                             'str_column': 'Value for sample 2',
                             'latitude': 4.2,
-                            'longitude': 1.1},
+                            'longitude': 1.1,
+                            'taxon_id': 9606,
+                            'scientific_name': 'homo sapiens'},
             'SKB7.640196': {'physical_specimen_location': 'location1',
                             'physical_specimen_remaining': True,
                             'dna_extracted': True,
@@ -233,7 +237,9 @@ def test_retrieve_dropped_samples(self):
                             'Description': 'Test Sample 3',
                             'str_column': 'Value for sample 3',
                             'latitude': 4.8,
-                            'longitude': 4.41},
+                            'longitude': 4.41,
+                            'taxon_id': 9606,
+                            'scientific_name': 'homo sapiens'},
             }
         metadata = pd.DataFrame.from_dict(metadata_dict, orient='index')
 

qiita_db/test/test_analysis.py

@@ -725,19 +725,19 @@ def test_update_preprocessed_data_from_cmd_ppd(self):
     "sample_name\trequired_sample_info_status\tcollection_timestamp\t"
     "sample_type\tphysical_specimen_remaining\tphysical_specimen_location\t"
     "dna_extracted\thost_subject_id\tTreatment\tDOB\tlatitude\tlongitude"
-    "\tDescription\n"
+    "\ttaxon_id\tscientific_name\tDescription\n"
     "PC.354\treceived\t2014-06-18 16:44\ttype_1\tTrue\tLocation_1\tTrue\t"
     "HS_ID_PC.354\tControl\t20061218\t1.88401499993\t56.0003871552\t"
-    "Control_mouse_I.D._354\n"
+    "9606\thomo sapiens\tControl_mouse_I.D._354\n"
     "PC.593\treceived\t2014-06-18 16:44\ttype_1\tTrue\tLocation_1\tTrue\t"
     "HS_ID_PC.593\tControl\t20071210\t35.4079458313\t83.2595338611\t"
-    "Control_mouse_I.D._593\n"
+    "9606\thomo sapiens\tControl_mouse_I.D._593\n"
     "PC.607\treceived\t2014-06-18 16:44\ttype_1\tTrue\tLocation_1\tTrue\t"
     "HS_ID_PC.607\tFast\t20071112\t18.3175615444\t91.3713989729\t"
-    "Fasting_mouse_I.D._607\n"
+    "9606\thomo sapiens\tFasting_mouse_I.D._607\n"
     "PC.636\treceived\t2014-06-18 16:44\ttype_1\tTrue\tLocation_1\tTrue\t"
     "HS_ID_PC.636\tFast\t20080116\t31.0856060708\t4.16781143893\t"
-    "Fasting_mouse_I.D._636")
+    "9606\thomo sapiens\tFasting_mouse_I.D._636")
 
 PREP_TEMPLATE = (
     'sample_name\tbarcode\tcenter_name\tcenter_project_name\t'

qiita_db/test/test_commands.py

@@ -14,7 +14,7 @@
 from skbio.util import safe_md5
 
 from qiita_core.qiita_settings import qiita_config
-
+from qiita_ware.exceptions import EBISumbissionError
 from qiita_db.logger import LogEntry
 from qiita_db.ontology import Ontology
 from qiita_db.util import convert_to_id
@@ -280,18 +280,20 @@ def generate_study_xml(self):
 
         return study_set
 
-    def add_sample(self, sample_name, taxon_id=None, description=None,
-                   **kwargs):
+    def add_sample(self, sample_name, taxon_id, scientific_name,
+                   description, **kwargs):
         """Adds sample information to the current submission
 
         Parameters
         ----------
         sample_name : str
             Unique identifier for the sample
-        taxon_id : str, optional
-            Defaults to ``None``. If not provided, the `empty_value` will be
-            used for the taxon ID
-        description : str, optional
+        taxon_id : str
+            NCBI's taxon ID for the sample
+        scientific_name : str
+            NCBI's scientific name for the `taxon_id`
+        description : str
+
             Defaults to ``None``. If not provided, the `empty_value` will be
             used for the description
 
@@ -308,15 +310,14 @@ def add_sample(self, sample_name, taxon_id=None, description=None,
 
         self.samples[sample_name] = {}
 
-        self.samples[sample_name]['taxon_id'] = self.empty_value if \
-            taxon_id is None else taxon_id
-        self.samples[sample_name]['taxon_id'] = \
-            escape(clean_whitespace(self.samples[sample_name]['taxon_id']))
+        self.samples[sample_name]['taxon_id'] = escape(
+            clean_whitespace(taxon_id))
 
-        self.samples[sample_name]['description'] = self.empty_value if \
-            description is None else description
-        self.samples[sample_name]['description'] = \
-            escape(clean_whitespace(self.samples[sample_name]['description']))
+        self.samples[sample_name]['scientific_name'] = escape(
+            clean_whitespace(scientific_name))
+
+        self.samples[sample_name]['description'] = escape(
+            clean_whitespace(description))
 
         self.samples[sample_name]['attributes'] = self._stringify_kwargs(
             kwargs)
@@ -349,6 +350,10 @@ def generate_sample_xml(self):
             taxon_id = ET.SubElement(sample_name_element, 'TAXON_ID')
             taxon_id.text = escape(clean_whitespace(sample_info['taxon_id']))
 
+            taxon_id = ET.SubElement(sample_name_element, 'SCIENTIFIC_NAME')
+            taxon_id.text = escape(
+                clean_whitespace(sample_info['scientific_name']))
+
             description = ET.SubElement(sample, 'DESCRIPTION')
             description.text = escape(clean_whitespace(
                 sample_info['description']))
@@ -791,21 +796,36 @@ def add_samples_from_templates(self, sample_template, prep_template,
             Path to the directory containing per-sample FASTQ files where
             the sequence labels should be:
             ``SampleID_SequenceNumber And Additional Notes if Applicable``
+
+        Raises
+        ------
+        EBISumbissionError
+            If a sample doesn't have the required EBI submission information
         """
         if not exists(per_sample_fastq_dir):
             raise IOError('The directory with the FASTQ file does not exist.')
 
         for sample in iter_file_via_list_of_dicts(sample_template):
             sample_name = sample.pop('sample_name')
             taxon_id = sample.pop('taxon_id', None)
+            scientific_name = sample.pop('scientific_name', None)
             description = sample.pop('description', None)
 
-            self.add_sample(sample_name, taxon_id=taxon_id,
-                            description=description,
-                            **sample)
+            if taxon_id is None or scientific_name is None or \
+                    description is None:
+                raise EBISumbissionError(
+                    "Sample '%s' is missing required EBI submission "
+                    "information. taxon_id: %s; scientific_name: %s; "
+                    "description: %s" % (sample_name, taxon_id,
+                                         scientific_name, description))
 
+            self.add_sample(sample_name, taxon_id, scientific_name,
+                            description, **sample)
+
+        prep_template_samples = []
         for prep in iter_file_via_list_of_dicts(prep_template):
             sample_name = prep.pop('sample_name')
+            prep_template_samples.append(sample_name)
             platform = prep.pop('platform')
             experiment_design_description = prep.pop(
                 'experiment_design_description')
@@ -818,6 +838,10 @@ def add_samples_from_templates(self, sample_template, prep_template,
                                  library_construction_protocol,
                                  **prep)
 
+        to_remove = set(self.samples).difference(prep_template_samples)
+        for sample in to_remove:
+            del self.samples[sample]
+
     @classmethod
     def from_templates_and_per_sample_fastqs(cls, preprocessed_data_id,
                                              study_title,

qiita_ware/ebi.py

@@ -40,3 +40,8 @@ class StudyDoesNotExistsError(QiitaWareError):
 class ComputeError(QiitaWareError):
     """A compute error happened"""
     pass
+
+
+class EBISumbissionError(QiitaWareError):
+    """Error used when EBI cannot be submitted"""
+    pass