Merge pull request #184 from squirrelo/remove-hardcode-commands

Remove hardcode commands

qiita_db/job.py

@@ -1,11 +1,21 @@
-"""
-Objects for dealing with Qiita jobs
+r"""
+Data objects (:mod: `qiita_db.data`)
+====================================
+
+..currentmodule:: qiita_db.data
 
-This module provides the implementation of the Job class.
+This module provides functionality for creating, running, and storing results
+of jobs in an analysis. It also provides the ability to query what commmands
+are available for jobs, as well as the options for these commands.
 
 Classes
 -------
-- `Job` -- A Qiita Job class
+
+..autosummary::
+    :toctree: generated/
+
+    Job
+    Command
 """
 # -----------------------------------------------------------------------------
 # Copyright (c) 2014--, The Qiita Development Team.
@@ -17,12 +27,12 @@
 from __future__ import division
 from json import dumps, loads
 from os.path import join
-from time import strftime
-from datetime import date
 from functools import partial
+from collections import defaultdict
 
-from qiita_core.exceptions import IncompetentQiitaDeveloperError
+from future.builtins import zip
 
+from qiita_core.exceptions import IncompetentQiitaDeveloperError
 from .base import QiitaStatusObject
 from .util import insert_filepaths, convert_to_id, get_db_files_base_dir
 from .sql_connection import SQLConnectionHandler
@@ -99,7 +109,7 @@ def exists(cls, datatype, command, options):
             sql, (datatype_id, command_id, opts_json))[0]
 
     @classmethod
-    def create(cls, datatype, command, options, analysis):
+    def create(cls, datatype, command, analysis):
         """Creates a new job on the database
 
         Parameters
@@ -108,8 +118,6 @@ def create(cls, datatype, command, options, analysis):
             The datatype in which this job applies
         command : str
             The identifier of the command executed in this job
-        options: dict
-            The options for the command in format {option: value}
         analysis : Analysis object
             The analysis which this job belongs to
 
@@ -130,14 +138,12 @@ def create(cls, datatype, command, options, analysis):
         sql = "SELECT command_id FROM qiita.command WHERE name = %s"
         command_id = conn_handler.execute_fetchone(sql, (command, ))[0]
 
-        # JSON the options dictionary
-        opts_json = dumps(options, sort_keys=True, separators=(',', ':'))
         # Create the job and return it
         sql = ("INSERT INTO qiita.{0} (data_type_id, job_status_id, "
-               "command_id, options) VALUES "
-               "(%s, %s, %s, %s) RETURNING job_id").format(cls._table)
+               "command_id) VALUES "
+               "(%s, %s, %s) RETURNING job_id").format(cls._table)
         job_id = conn_handler.execute_fetchone(sql, (datatype_id, 1,
-                                               command_id, opts_json))[0]
+                                               command_id))[0]
 
         # add job to analysis
         sql = ("INSERT INTO qiita.analysis_job (analysis_id, job_id) VALUES "
@@ -197,6 +203,26 @@ def options(self):
             opts[k] = join_f("%s_%s_%s" % (self._id, db_comm[0], k.strip("-")))
         return opts
 
+    @options.setter
+    def options(self, opts):
+        """ Sets the options for the job
+
+        Parameters
+        ----------
+        opts: dict
+            The options for the command in format {option: value}
+        """
+        conn_handler = SQLConnectionHandler()
+        # make sure job is editable
+        self._lock_job(conn_handler)
+
+        # JSON the options dictionary
+        opts_json = dumps(opts, sort_keys=True, separators=(',', ':'))
+        # Add the options to the job
+        sql = ("UPDATE qiita.{0} SET options = %s WHERE "
+               "job_id = %s").format(self._table)
+        conn_handler.execute(sql, (opts_json, self._id))
+
     @property
     def results(self):
         """List of job result filepaths
@@ -317,6 +343,48 @@ def create_list(cls):
         return [cls(c["name"], c["command"], c["input"], c["required"],
                 c["optional"], c["output"]) for c in commands]
 
+    @classmethod
+    def get_commands_by_datatype(cls, datatypes=None):
+        """Returns the commands available for all or a subset of the datatypes
+
+        Parameters
+        ----------
+        datatypes : list of str, optional
+            List of the datatypes to get commands for. Default is all datatypes
+
+        Returns
+        -------
+        dict of lists of Command objects
+            Returns commands in the format {datatype: [com name1, com name2]}
+
+        Notes
+        -----
+        If no datatypes are passed, the function will default to returning all
+        datatypes available.
+        """
+        conn_handler = SQLConnectionHandler()
+        # get the ids of the datatypes to get commands for
+        if datatypes is not None:
+            datatype_info = [(convert_to_id(dt, "data_type", conn_handler), dt)
+                             for dt in datatypes]
+        else:
+            datatype_info = conn_handler.execute_fetchall(
+                "SELECT data_type_id, data_type from qiita.data_type")
+
+        commands = defaultdict(list)
+        # get commands for each datatype
+        sql = ("SELECT C.* FROM qiita.command C JOIN qiita.command_data_type "
+               "CD on C.command_id = CD.command_id WHERE CD.data_type_id = %s")
+        for dt_id, dt in datatype_info:
+            comms = conn_handler.execute_fetchall(sql, (dt_id, ))
+            for comm in comms:
+                commands[dt].append(cls(comm["name"], comm["command"],
+                                        comm["input"],
+                                        comm["required"],
+                                        comm["optional"],
+                                        comm["output"]))
+        return commands
+
     def __eq__(self, other):
         if type(self) != type(other):
             return False

qiita_db/test/test_job.py

@@ -74,14 +74,12 @@ def test_get_commands(self):
     def test_create(self):
         """Makes sure creation works as expected"""
         # make first job
-        new = Job.create("18S", "Alpha Rarefaction",
-                         self.options, Analysis(1))
+        new = Job.create("18S", "Alpha Rarefaction", Analysis(1))
         self.assertEqual(new.id, 4)
         # make sure job inserted correctly
         obs = self.conn_handler.execute_fetchall("SELECT * FROM qiita.job "
                                                  "WHERE job_id = 4")
-        exp = [[4, 2, 1, 3, '{"option1":false,"option2":25,"option3":"NEW"}',
-                None]]
+        exp = [[4, 2, 1, 3, None, None]]
         self.assertEqual(obs, exp)
         # make sure job added to analysis correctly
         obs = self.conn_handler.execute_fetchall("SELECT * FROM "
@@ -91,14 +89,12 @@ def test_create(self):
         self.assertEqual(obs, exp)
 
         # make second job with diff datatype and command to test column insert
-        new = Job.create("16S", "Beta Diversity",
-                         self.options, Analysis(1))
+        new = Job.create("16S", "Beta Diversity", Analysis(1))
         self.assertEqual(new.id, 5)
         # make sure job inserted correctly
         obs = self.conn_handler.execute_fetchall("SELECT * FROM qiita.job "
                                                  "WHERE job_id = 5")
-        exp = [[5, 1, 1, 2, '{"option1":false,"option2":25,"option3":"NEW"}',
-                None]]
+        exp = [[5, 1, 1, 2, None, None]]
         self.assertEqual(obs, exp)
         # make sure job added to analysis correctly
         obs = self.conn_handler.execute_fetchall("SELECT * FROM "
@@ -130,11 +126,19 @@ def test_retrieve_options(self):
                                  '1_summarize_taxa_through_plots.py'
                                  '_output_dir')})
 
+    def test_set_options(self):
+        new = Job.create("18S", "Alpha Rarefaction", Analysis(1))
+        new.options = self.options
+        self.options['--output_dir'] = join(get_db_files_base_dir(),
+                                            'job/4_alpha_rarefaction.'
+                                            'py_output_dir')
+        self.assertEqual(new.options, self.options)
+
     def test_retrieve_results(self):
         self.assertEqual(self.job.results, [join("job", "1_job_result.txt")])
 
     def test_retrieve_results_empty(self):
-        new = Job.create("18S", "Beta Diversity", self.options, Analysis(1))
+        new = Job.create("18S", "Beta Diversity", Analysis(1))
         self.assertEqual(new.results, [])
 
     def test_retrieve_results_dir(self):
@@ -192,5 +196,40 @@ def test_add_results_completed(self):
             self.job.add_results([("/fake/dir/", "directory")])
 
 
+@qiita_test_checker()
+class CommandTest(TestCase):
+    def setUp(self):
+        com1 = Command('Summarize Taxa', 'summarize_taxa_through_plots.py',
+                       '{"--otu_table_fp":null}', '{}',
+                       '{"--mapping_category":null, "--mapping_fp":null,'
+                       '"--sort":null}', '{"--output_dir":null}')
+        com2 = Command('Beta Diversity', 'beta_diversity_through_plots.py',
+                       '{"--otu_table_fp":null,"--mapping_fp":null}', '{}',
+                       '{"--tree_fp":null,"--color_by_all_fields":null,'
+                       '"--seqs_per_sample":null}', '{"--output_dir":null}')
+        com3 = Command('Alpha Rarefaction', 'alpha_rarefaction.py',
+                       '{"--otu_table_fp":null,"--mapping_fp":null}', '{}',
+                       '{"--tree_fp":null,"--num_steps":null,'
+                       '"--min_rare_depth"'
+                       ':null,"--max_rare_depth":null,'
+                       '"--retain_intermediate_files":false}',
+                       '{"--output_dir":null}')
+        self.all_comms = {
+            "16S": [com1, com2, com3],
+            "18S": [com1, com2, com3],
+            "ITS": [com2, com3],
+            "Proteomic": [com2, com3],
+            "Metabolomic": [com2, com3],
+            "Metagenomic": [com2, com3],
+        }
+
+    def test_get_commands_by_datatype(self):
+        obs = Command.get_commands_by_datatype()
+        self.assertEqual(obs, self.all_comms)
+        obs = Command.get_commands_by_datatype(["16S", "Metabolomic"])
+        exp = {k: self.all_comms[k] for k in ('16S', 'Metabolomic')}
+        self.assertEqual(obs, exp)
+
+
 if __name__ == "__main__":
     main()

qiita_pet/handlers/analysis_handlers.py

@@ -24,7 +24,7 @@
 from qiita_db.study import Study
 from qiita_db.data import ProcessedData
 from qiita_db.metadata_template import SampleTemplate
-from qiita_db.job import Job
+from qiita_db.job import Job, Command
 from qiita_db.util import get_db_files_base_dir
 
 
@@ -94,10 +94,7 @@ def post(self):
         data_types = sorted(list(data_types))
 
         # FIXME: Pull out from the database, see #111
-        commands = {'16S': ['Beta Diversity', 'Summarize Taxa'],
-                    '18S': ['Beta Diversity', 'Summarize Taxa'],
-                    'Metabolomic': ['Beta Diversity'],
-                    'Metagenomic': ['Beta Diversity']}
+        commands = Command.get_commands_by_datatype()
 
         self.render('select_commands.html', user=self.get_current_user(),
                     commands=commands, data_types=data_types, aid=analysis_id)

qiita_pet/templates/select_commands.html

@@ -31,10 +31,10 @@ <h1>Select Commands</h1>
               {% for command in commands[data_type] %}
                 <tr>
                   <td style="width:20px;">
-                    <input id="{{data_type}}#{{command}}" type="checkbox" name="commands" value="{{data_type}}#{{command}}">
+                    <input id="{{data_type}}#{{command.name}}" type="checkbox" name="commands" value="{{data_type}}#{{command.name}}">
                   </td>
                   <td>
-                    <label style="font-weight:normal;" for="{{data_type}}#{{command}}">{{command}}</label>
+                    <label style="font-weight:normal;" for="{{data_type}}#{{command.name}}">{{command.name}}</label>
                   </td>
                 </tr>
               {% end %}