Merge 4d6bb67 into 4a4d33c

qiita-spots · Apr 19, 2016 · b0d57ed · b0d57ed
2 parents 4a4d33c + 4d6bb67
commit b0d57ed
Show file tree

Hide file tree

Showing 11 changed files with 127 additions and 57 deletions.
diff --git a/qiita_db/handlers/tests/test_processing_job.py b/qiita_db/handlers/tests/test_processing_job.py
@@ -52,7 +52,8 @@ def test_get(self):
                   "rev_comp_barcode": False,
                   "rev_comp_mapping_barcodes": False, "rev_comp": False,
                   "phred_quality_threshold": 3, "barcode_type": "golay_12",
-                  "max_barcode_errors": 1.5, "input_data": 1}
+                  "max_barcode_errors": 1.5, "input_data": 1,
+                  'phred_offset': ''}
         exp = {'success': True, 'error': '', 'command': cmd,
                'parameters': params, 'status': 'success'}
         self.assertEqual(loads(obs.body), exp)

diff --git a/qiita_db/software.py b/qiita_db/software.py
@@ -736,6 +736,13 @@ def create(cls, param_set_name, command, **kwargs):
             - If the parameter set already exists
         """
         with qdb.sql_connection.TRN:
+            # setting to default values all parameters not in the user_params
+            cmd_params = command.optional_parameters
+            missing_in_user = {k: cmd_params[k][1]
+                               for k in (set(cmd_params) - set(kwargs))}
+            if missing_in_user:
+                kwargs.update(missing_in_user)
+
             # If the columns in kwargs and command do not match, cls.exists
             # will raise the error for us
             if cls.exists(command, **kwargs):
@@ -869,6 +876,13 @@ def load(cls, command, json_str=None, values_dict=None):
             error_msg = ("The provided values dictionary doesn't encode a "
                          "parameter set for command %s" % command.id)
 
+        # setting to default values all parameters not in the user_params
+        cmd_params = command.optional_parameters
+        missing_in_user = {k: cmd_params[k][1]
+                           for k in (set(cmd_params) - set(parameters))}
+        if missing_in_user:
+            parameters.update(missing_in_user)
+
         with qdb.sql_connection.TRN:
             cmd_reqd_params = command.required_parameters
             cmd_opt_params = command.optional_parameters

diff --git a/qiita_db/support_files/patches/37.sql b/qiita_db/support_files/patches/37.sql
@@ -0,0 +1,36 @@
+-- Apr 18, 2016
+-- Adding phred_offset to split libraries
+
+DO $do$
+DECLARE
+    cmd_id            bigint;
+BEGIN
+    -- selecting command_id of interest
+    SELECT command_id FROM qiita.software_command WHERE name = 'Split libraries FASTQ' INTO cmd_id;
+
+    -- adding new parameter
+    INSERT INTO qiita.command_parameter (command_id, parameter_name, parameter_type, required, default_value)
+        VALUES (cmd_id, 'phred_offset', 'string', False, '');
+
+    -- updating all current artifacts that were generated with this command
+    UPDATE qiita.artifact
+        SET command_parameters = (
+            substring(command_parameters::text FROM 0 FOR char_length(command_parameters::text)) || ',"phred_offset":""}'
+        )::json
+        WHERE command_id=cmd_id;
+
+    -- updating the default_parameter_set
+    UPDATE qiita.default_parameter_set
+        SET parameter_set = (
+            substring(parameter_set::text FROM 0 FOR char_length(parameter_set::text)) || ',"phred_offset":""}'
+        )::json
+        WHERE command_id=cmd_id;
+
+    -- inserting new possible default_parameter_sets
+    INSERT INTO qiita.default_parameter_set (command_id, parameter_set_name, parameter_set)
+        VALUES (cmd_id, 'per sample FASTQ defaults, phred_offset 33',
+                '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"not-barcoded","max_barcode_errors":1.5,"phred_offset":"33"}'::json);
+    INSERT INTO qiita.default_parameter_set (command_id, parameter_set_name, parameter_set)
+        VALUES (cmd_id, 'per sample FASTQ defaults, phred_offset 64',
+                '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"not-barcoded","max_barcode_errors":1.5,"phred_offset":"64"}'::json);
+END $do$;
diff --git a/qiita_db/support_files/populate_test_db.sql b/qiita_db/support_files/populate_test_db.sql
@@ -321,9 +321,9 @@ INSERT INTO qiita.study_prep_template (study_id, prep_template_id) VALUES (1, 1)
 INSERT INTO qiita.artifact (name, generated_timestamp, command_id, command_parameters,
                             visibility_id, artifact_type_id, data_type_id)
     VALUES ('Raw data 1', 'Mon Oct 1 09:30:27 2012', NULL, NULL, 3, 3, 2),
-           ('Demultiplexed 1', 'Mon Oct 1 10:30:27 2012', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1}'::json,
+           ('Demultiplexed 1', 'Mon Oct 1 10:30:27 2012', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":""}'::json,
             3, 6, 2),
-           ('Demultiplexed 2', 'Mon Oct 1 11:30:27 2012', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1}'::json,
+           ('Demultiplexed 2', 'Mon Oct 1 11:30:27 2012', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":""}'::json,
             3, 6, 2),
            ('BIOM', 'Tue Oct 2 17:30:00 2012', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}'::json,
             3, 7, 2),
@@ -339,8 +339,8 @@ INSERT INTO qiita.parent_artifact (parent_id, artifact_id)
 
 -- Insert the jobs that processed the previous artifacts
 INSERT INTO qiita.processing_job (processing_job_id, email, command_id, command_parameters, processing_job_status_id)
-    VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1}'::json, 3),
-           ('4c7115e8-4c8e-424c-bf25-96c292ca1931', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1}'::json, 3),
+    VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":""}'::json, 3),
+           ('4c7115e8-4c8e-424c-bf25-96c292ca1931', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":""}'::json, 3),
            ('3c9991ab-6c14-4368-a48c-841e8837a79c', 'test@foo.bar', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}'::json, 3);
 
 -- Relate the above jobs with the artifacts
@@ -512,9 +512,9 @@ INSERT INTO qiita.logging (time, severity_id, msg, information)
 INSERT INTO qiita.processing_job
         (processing_job_id, email, command_id, command_parameters,
          processing_job_status_id, logging_id, heartbeat, step)
-    VALUES ('063e553b-327c-4818-ab4a-adfe58e49860', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1}'::json, 1, NULL, NULL, NULL),
+    VALUES ('063e553b-327c-4818-ab4a-adfe58e49860', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":""}'::json, 1, NULL, NULL, NULL),
            ('bcc7ebcd-39c1-43e4-af2d-822e3589f14d', 'test@foo.bar', 2, '{"min_seq_len":100,"max_seq_len":1000,"trim_seq_length":false,"min_qual_score":25,"max_ambig":6,"max_homopolymer":6,"max_primer_mismatch":0,"barcode_type":"golay_12","max_barcode_errors":1.5,"disable_bc_correction":false,"qual_score_window":0,"disable_primers":false,"reverse_primers":"disable","reverse_primer_mismatches":0,"truncate_ambi_bases":false,"input_data":1}'::json, 2, NULL, 'Sun Nov 22 21:00:00 2015', 'demultiplexing'),
-           ('b72369f9-a886-4193-8d3d-f7b504168e75', 'shared@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1}'::json, 3, NULL, 'Sun Nov 22 21:15:00 2015', NULL),
+           ('b72369f9-a886-4193-8d3d-f7b504168e75', 'shared@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":""}'::json, 3, NULL, 'Sun Nov 22 21:15:00 2015', NULL),
            ('d19f76ee-274e-4c1b-b3a2-a12d73507c55', 'shared@foo.bar', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}'::json, 4, 1, 'Sun Nov 22 21:30:00 2015', 'generating demux file'),
            ('ac653cb5-76a6-4a45-929e-eb9b2dee6b63', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1}'::json, 5, NULL, NULL, NULL);
 

diff --git a/qiita_db/test/test_artifact.py b/qiita_db/test/test_artifact.py
@@ -76,7 +76,8 @@ def test_processing_parameters(self):
                          'rev_comp_barcode': False,
                          'rev_comp_mapping_barcodes': False,
                          'min_per_read_length_fraction': 0.75,
-                         'barcode_type': 'golay_12'})
+                         'barcode_type': 'golay_12',
+                         'phred_offset': ''})
         self.assertEqual(obs, exp)
         obs = qdb.artifact.Artifact(3).processing_parameters
         exp = qdb.software.Parameters.load(
@@ -87,7 +88,8 @@ def test_processing_parameters(self):
                          'rev_comp_barcode': False,
                          'rev_comp_mapping_barcodes': True,
                          'min_per_read_length_fraction': 0.75,
-                         'barcode_type': 'golay_12'})
+                         'barcode_type': 'golay_12',
+                         'phred_offset': ''})
         self.assertEqual(obs, exp)
 
     def test_visibility(self):
@@ -702,8 +704,8 @@ def test_delete_error_in_construction_job(self):
             '"barcode_type": "golay_12", "max_bad_run_length": 3, '
             '"rev_comp": false, "phred_quality_threshold": 3, '
             '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
-            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}'
-            % test.id)
+            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0, '
+            '"phred_offset": ""}' % test.id)
         qdb.processing_job.ProcessingJob.create(
             qdb.user.User('test@foo.bar'),
             qdb.software.Parameters.load(qdb.software.Command(1),
@@ -720,8 +722,8 @@ def test_delete_error_running_job(self):
             '"barcode_type": "golay_12", "max_bad_run_length": 3, '
             '"rev_comp": false, "phred_quality_threshold": 3, '
             '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
-            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}'
-            % test.id)
+            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0, '
+            '"phred_offset": ""}' % test.id)
         job = qdb.processing_job.ProcessingJob.create(
             qdb.user.User('test@foo.bar'),
             qdb.software.Parameters.load(qdb.software.Command(1),
@@ -779,8 +781,8 @@ def test_delete_with_jobs(self):
             '"barcode_type": "golay_12", "max_bad_run_length": 3, '
             '"rev_comp": false, "phred_quality_threshold": 3, '
             '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
-            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}'
-            % test.id)
+            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0, '
+            '"phred_offset": ""}' % test.id)
         job = qdb.processing_job.ProcessingJob.create(
             qdb.user.User('test@foo.bar'),
             qdb.software.Parameters.load(qdb.software.Command(1),

diff --git a/qiita_db/test/test_commands.py b/qiita_db/test/test_commands.py
@@ -226,7 +226,7 @@ def test_load_parameters_from_cmd(self):
             "test", self.fp, 1)
         obs = new.values
         exp = {"barcode_type": "hamming_8", "max_bad_run_length": "3",
-               "max_barcode_errors": "1.5",
+               "phred_offset": "", "max_barcode_errors": "1.5",
                "min_per_read_length_fraction": "0.75",
                "phred_quality_threshold": "3", "sequence_max_n": "0",
                "rev_comp": "False", "rev_comp_barcode": "False",

diff --git a/qiita_db/test/test_processing_job.py b/qiita_db/test/test_processing_job.py
@@ -112,7 +112,7 @@ def test_parameters(self):
             '"sequence_max_n":0,"rev_comp_barcode":false,'
             '"rev_comp_mapping_barcodes":false,"rev_comp":false,'
             '"phred_quality_threshold":3,"barcode_type":"golay_12",'
-            '"max_barcode_errors":1.5,"input_data":1}')
+            '"max_barcode_errors":1.5,"input_data":1,"phred_offset":""}')
         exp_params = qdb.software.Parameters.load(qdb.software.Command(1),
                                                   json_str=json_str)
         self.assertEqual(self.tester1.parameters, exp_params)
@@ -134,7 +134,7 @@ def test_parameters(self):
             '"sequence_max_n":0,"rev_comp_barcode":false,'
             '"rev_comp_mapping_barcodes":true,"rev_comp":false,'
             '"phred_quality_threshold":3,"barcode_type":"golay_12",'
-            '"max_barcode_errors":1.5,"input_data":1}')
+            '"max_barcode_errors":1.5,"input_data":1,"phred_offset":""}')
         exp_params = qdb.software.Parameters.load(qdb.software.Command(1),
                                                   json_str=json_str)
         self.assertEqual(self.tester3.parameters, exp_params)
@@ -232,7 +232,8 @@ def test_create(self):
             '"barcode_type": "golay_12", "max_bad_run_length": 3, '
             '"rev_comp": false, "phred_quality_threshold": 3, '
             '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
-            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
+            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0, '
+            '"phred_offset": ""}')
         exp_params = qdb.software.Parameters.load(exp_command,
                                                   json_str=json_str)
         exp_user = qdb.user.User('test@foo.bar')
@@ -387,7 +388,8 @@ def test_update_children(self):
             '"barcode_type": "golay_12", "max_bad_run_length": 3, '
             '"rev_comp": false, "phred_quality_threshold": 3, '
             '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
-            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
+            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0, '
+            '"phred_offset": ""}')
         exp_params = qdb.software.Parameters.load(exp_command,
                                                   json_str=json_str)
         exp_user = qdb.user.User('test@foo.bar')
@@ -519,7 +521,8 @@ def test_from_scratch(self):
             '"barcode_type": "golay_12", "max_bad_run_length": 3, '
             '"rev_comp": false, "phred_quality_threshold": 3, '
             '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
-            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
+            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0, '
+            '"phred_offset": ""}')
         exp_params = qdb.software.Parameters.load(exp_command,
                                                   json_str=json_str)
         exp_user = qdb.user.User('test@foo.bar')
@@ -543,7 +546,8 @@ def test_add(self):
             '"barcode_type": "golay_12", "max_bad_run_length": 3, '
             '"rev_comp": false, "phred_quality_threshold": 3, '
             '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
-            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
+            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0, '
+            '"phred_offset": ""}')
         exp_params = qdb.software.Parameters.load(exp_command,
                                                   json_str=json_str)
         exp_user = qdb.user.User('test@foo.bar')
@@ -602,7 +606,8 @@ def test_add(self):
                       'rev_comp': False,
                       'rev_comp_barcode': False,
                       'rev_comp_mapping_barcodes': False,
-                      'sequence_max_n': 0}
+                      'sequence_max_n': 0,
+                      'phred_offset': ''}
         self.assertEqual(obs_job.parameters.values, exp_params)
 
     def test_add_error(self):
@@ -617,7 +622,8 @@ def test_remove(self):
             '"barcode_type": "golay_12", "max_bad_run_length": 3, '
             '"rev_comp": false, "phred_quality_threshold": 3, '
             '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
-            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}')
+            '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0,'
+            '"phred_offset": ""}')
         exp_params = qdb.software.Parameters.load(exp_command,
                                                   json_str=json_str)
         exp_user = qdb.user.User('test@foo.bar')