Merge 6920adb into 02dbb4d

qiita-spots · Sep 28, 2016 · 81b4417 · 81b4417
2 parents 02dbb4d + 6920adb
commit 81b4417
Show file tree

Hide file tree

Showing 8 changed files with 105 additions and 36 deletions.
diff --git a/qiita_db/handlers/tests/test_processing_job.py b/qiita_db/handlers/tests/test_processing_job.py
@@ -47,7 +47,7 @@ def test_get(self):
                   "rev_comp_mapping_barcodes": False, "rev_comp": False,
                   "phred_quality_threshold": 3, "barcode_type": "golay_12",
                   "max_barcode_errors": 1.5, "input_data": 1,
-                  'phred_offset': ''}
+                  'phred_offset': 'auto'}
         exp = {'command': cmd, 'parameters': params, 'status': 'success'}
         self.assertEqual(loads(obs.body), exp)
 

diff --git a/qiita_db/support_files/patches/40.sql b/qiita_db/support_files/patches/40.sql
@@ -5,3 +5,72 @@
 ALTER TABLE qiita.software ADD active bool DEFAULT 'False' NOT NULL;
 
 ALTER TABLE qiita.software_command ADD active bool DEFAULT 'True' NOT NULL;
+
+-- Add function to set a key in a JSON value
+-- Adapted from http://stackoverflow.com/a/23500670/3746629
+CREATE OR REPLACE FUNCTION qiita.json_object_set_key(
+  "json"          json,
+  "key_to_set"    TEXT,
+  "value_to_set"  anyelement
+)
+  RETURNS json
+  LANGUAGE sql
+  IMMUTABLE
+  STRICT
+AS $function$
+SELECT concat('{', string_agg(to_json("key") || ':' || "value", ','), '}')::json
+  FROM (SELECT *
+          FROM json_each("json")
+         WHERE "key" <> "key_to_set"
+         UNION ALL
+        SELECT "key_to_set", to_json("value_to_set")) AS "fields"
+$function$;
+
+-- Change the phred_offset from string to choice
+DO $do$
+DECLARE
+    cmd_id      bigint;
+    dflt_p      RECORD;
+    a_vals      RECORD;
+    j_vals      RECORD;
+BEGIN
+    -- select command id of interest
+    SELECT command_id FROM qiita.software_command WHERE name = 'Split libraries FASTQ' INTO cmd_id;
+
+    -- Update the phred_offset parameter type
+    UPDATE qiita.command_parameter SET parameter_type = 'choice:["auto", "33", "64"]', default_value = 'auto'
+        WHERE parameter_name = 'phred_offset' AND command_id = cmd_id;
+
+    -- Update all the default parameter sets to use "auto" instead of ""
+    FOR dflt_p IN
+        SELECT *
+        FROM qiita.default_parameter_set
+        WHERE command_id = cmd_id AND parameter_set->>'phred_offset' = ''
+    LOOP
+        UPDATE qiita.default_parameter_set
+            SET parameter_set = qiita.json_object_set_key(dflt_p.parameter_set, 'phred_offset', 'auto'::varchar)
+            WHERE default_parameter_set_id = dflt_p.default_parameter_set_id;
+    END LOOP;
+
+    -- Update all current artifacts that have been generated with this command
+    FOR a_vals IN
+        SELECT *
+        FROM qiita.artifact
+        WHERE command_id = cmd_id AND command_parameters->>'phred_offset' = ''
+    LOOP
+        UPDATE qiita.artifact
+            SET command_parameters = qiita.json_object_set_key(a_vals.command_parameters, 'phred_offset', 'auto'::varchar)
+            WHERE artifact_id = a_vals.artifact_id;
+    END LOOP;
+
+    -- Update all the jobs that have been using this parameter set
+    FOR j_vals IN
+        SELECT *
+        FROM qiita.processing_job
+        WHERE command_id = cmd_id AND command_parameters->>'phred_offset' = ''
+    LOOP
+        UPDATE qiita.processing_job
+            SET command_parameters = qiita.json_object_set_key(j_vals.command_parameters, 'phred_offset', 'auto'::varchar)
+            WHERE processing_job_id = j_vals.processing_job_id;
+    END LOOP;
+END $do$;
diff --git a/qiita_db/support_files/populate_test_db.sql b/qiita_db/support_files/populate_test_db.sql
@@ -352,9 +352,9 @@ INSERT INTO qiita.study_prep_template (study_id, prep_template_id) VALUES (1, 2)
 INSERT INTO qiita.artifact (name, generated_timestamp, command_id, command_parameters,
                             visibility_id, artifact_type_id, data_type_id)
     VALUES ('Raw data 1', 'Mon Oct 1 09:30:27 2012', NULL, NULL, 3, 3, 2),
-           ('Demultiplexed 1', 'Mon Oct 1 10:30:27 2012', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":""}'::json,
+           ('Demultiplexed 1', 'Mon Oct 1 10:30:27 2012', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}'::json,
             3, 6, 2),
-           ('Demultiplexed 2', 'Mon Oct 1 11:30:27 2012', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":""}'::json,
+           ('Demultiplexed 2', 'Mon Oct 1 11:30:27 2012', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}'::json,
             3, 6, 2),
            ('BIOM', 'Tue Oct 2 17:30:00 2012', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}'::json,
             3, 7, 2),
@@ -372,8 +372,8 @@ INSERT INTO qiita.parent_artifact (parent_id, artifact_id)
 
 -- Insert the jobs that processed the previous artifacts
 INSERT INTO qiita.processing_job (processing_job_id, email, command_id, command_parameters, processing_job_status_id)
-    VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":""}'::json, 3),
-           ('4c7115e8-4c8e-424c-bf25-96c292ca1931', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":""}'::json, 3),
+    VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}'::json, 3),
+           ('4c7115e8-4c8e-424c-bf25-96c292ca1931', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}'::json, 3),
            ('3c9991ab-6c14-4368-a48c-841e8837a79c', 'test@foo.bar', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}'::json, 3);
 
 -- Relate the above jobs with the artifacts
@@ -550,9 +550,9 @@ INSERT INTO qiita.logging (time, severity_id, msg, information)
 INSERT INTO qiita.processing_job
         (processing_job_id, email, command_id, command_parameters,
          processing_job_status_id, logging_id, heartbeat, step)
-    VALUES ('063e553b-327c-4818-ab4a-adfe58e49860', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":""}'::json, 1, NULL, NULL, NULL),
+    VALUES ('063e553b-327c-4818-ab4a-adfe58e49860', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}'::json, 1, NULL, NULL, NULL),
            ('bcc7ebcd-39c1-43e4-af2d-822e3589f14d', 'test@foo.bar', 2, '{"min_seq_len":100,"max_seq_len":1000,"trim_seq_length":false,"min_qual_score":25,"max_ambig":6,"max_homopolymer":6,"max_primer_mismatch":0,"barcode_type":"golay_12","max_barcode_errors":1.5,"disable_bc_correction":false,"qual_score_window":0,"disable_primers":false,"reverse_primers":"disable","reverse_primer_mismatches":0,"truncate_ambi_bases":false,"input_data":1}'::json, 2, NULL, 'Sun Nov 22 21:00:00 2015', 'demultiplexing'),
-           ('b72369f9-a886-4193-8d3d-f7b504168e75', 'shared@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":""}'::json, 3, NULL, 'Sun Nov 22 21:15:00 2015', NULL),
+           ('b72369f9-a886-4193-8d3d-f7b504168e75', 'shared@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}'::json, 3, NULL, 'Sun Nov 22 21:15:00 2015', NULL),
            ('d19f76ee-274e-4c1b-b3a2-a12d73507c55', 'shared@foo.bar', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}'::json, 4, 1, 'Sun Nov 22 21:30:00 2015', 'generating demux file'),
            ('ac653cb5-76a6-4a45-929e-eb9b2dee6b63', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1}'::json, 5, NULL, NULL, NULL);
 

diff --git a/qiita_db/test/test_artifact.py b/qiita_db/test/test_artifact.py
@@ -95,7 +95,7 @@ def test_processing_parameters(self):
                          'rev_comp_mapping_barcodes': False,
                          'min_per_read_length_fraction': 0.75,
                          'barcode_type': 'golay_12',
-                         'phred_offset': ''})
+                         'phred_offset': 'auto'})
         self.assertEqual(obs, exp)
         obs = qdb.artifact.Artifact(3).processing_parameters
         exp = qdb.software.Parameters.load(
@@ -107,7 +107,7 @@ def test_processing_parameters(self):
                          'rev_comp_mapping_barcodes': True,
                          'min_per_read_length_fraction': 0.75,
                          'barcode_type': 'golay_12',
-                         'phred_offset': ''})
+                         'phred_offset': 'auto'})
         self.assertEqual(obs, exp)
 
     def test_visibility(self):

diff --git a/qiita_db/test/test_commands.py b/qiita_db/test/test_commands.py
@@ -226,7 +226,7 @@ def test_load_parameters_from_cmd(self):
             "test", self.fp, 1)
         obs = new.values
         exp = {"barcode_type": "hamming_8", "max_bad_run_length": "3",
-               "phred_offset": "", "max_barcode_errors": "1.5",
+               "phred_offset": "auto", "max_barcode_errors": "1.5",
                "min_per_read_length_fraction": "0.75",
                "phred_quality_threshold": "3", "sequence_max_n": "0",
                "rev_comp": "False", "rev_comp_barcode": "False",

diff --git a/qiita_db/test/test_processing_job.py b/qiita_db/test/test_processing_job.py
@@ -128,7 +128,7 @@ def test_parameters(self):
             '"sequence_max_n":0,"rev_comp_barcode":false,'
             '"rev_comp_mapping_barcodes":false,"rev_comp":false,'
             '"phred_quality_threshold":3,"barcode_type":"golay_12",'
-            '"max_barcode_errors":1.5,"input_data":1,"phred_offset":""}')
+            '"max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}')
         exp_params = qdb.software.Parameters.load(qdb.software.Command(1),
                                                   json_str=json_str)
         self.assertEqual(self.tester1.parameters, exp_params)
@@ -150,7 +150,7 @@ def test_parameters(self):
             '"sequence_max_n":0,"rev_comp_barcode":false,'
             '"rev_comp_mapping_barcodes":true,"rev_comp":false,'
             '"phred_quality_threshold":3,"barcode_type":"golay_12",'
-            '"max_barcode_errors":1.5,"input_data":1,"phred_offset":""}')
+            '"max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}')
         exp_params = qdb.software.Parameters.load(qdb.software.Command(1),
                                                   json_str=json_str)
         self.assertEqual(self.tester3.parameters, exp_params)
@@ -230,7 +230,7 @@ def test_create(self):
             '"rev_comp": false, "phred_quality_threshold": 3, '
             '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
             '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0, '
-            '"phred_offset": ""}')
+            '"phred_offset": "auto"}')
         exp_params = qdb.software.Parameters.load(exp_command,
                                                   json_str=json_str)
         exp_user = qdb.user.User('test@foo.bar')
@@ -405,7 +405,7 @@ def test_update_children(self):
             '"rev_comp": false, "phred_quality_threshold": 3, '
             '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
             '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0, '
-            '"phred_offset": ""}')
+            '"phred_offset": "auto"}')
         exp_params = qdb.software.Parameters.load(exp_command,
                                                   json_str=json_str)
         exp_user = qdb.user.User('test@foo.bar')
@@ -564,7 +564,7 @@ def test_from_scratch(self):
             '"rev_comp": false, "phred_quality_threshold": 3, '
             '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
             '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0, '
-            '"phred_offset": ""}')
+            '"phred_offset": "auto"}')
         exp_params = qdb.software.Parameters.load(exp_command,
                                                   json_str=json_str)
         exp_user = qdb.user.User('test@foo.bar')
@@ -589,7 +589,7 @@ def test_add(self):
             '"rev_comp": false, "phred_quality_threshold": 3, '
             '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
             '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0, '
-            '"phred_offset": ""}')
+            '"phred_offset": "auto"}')
         exp_params = qdb.software.Parameters.load(exp_command,
                                                   json_str=json_str)
         exp_user = qdb.user.User('test@foo.bar')
@@ -649,7 +649,7 @@ def test_add(self):
                       'rev_comp_barcode': False,
                       'rev_comp_mapping_barcodes': False,
                       'sequence_max_n': 0,
-                      'phred_offset': ''}
+                      'phred_offset': 'auto'}
         self.assertEqual(obs_job.parameters.values, exp_params)
 
     def test_add_error(self):
@@ -665,7 +665,7 @@ def test_remove(self):
             '"rev_comp": false, "phred_quality_threshold": 3, '
             '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, '
             '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0,'
-            '"phred_offset": ""}')
+            '"phred_offset": "auto"}')
         exp_params = qdb.software.Parameters.load(exp_command,
                                                   json_str=json_str)
         exp_user = qdb.user.User('test@foo.bar')