Merge pull request #4 from Clinical-Genomics/extract

Separate 'extract' and 'import' functionalities in CLI

mutacc_auto/cli/extract_command.py

@@ -0,0 +1,73 @@
+import click
+import logging
+from shutil import rmtree
+from pathlib import Path
+import yaml
+import sys
+import os
+
+from mutacc_auto.utils.tmp_dir import TemporaryDirectory
+from mutacc_auto.recipes.input_recipe import get_inputs
+from mutacc_auto.recipes.extract_recipe import run_mutacc_extract
+
+LOG = logging.getLogger(__name__)
+
+@click.command('extract')
+@click.option('-c','--case-id', type=str)
+@click.option('-d','--days-ago', type=int)
+@click.option('-e','--environment', type=str)
+@click.option('-C','--config-file', type=click.Path(exists=True))
+@click.option('-L', '--log-directory', type=click.Path(exists=True))
+@click.option('-E', '--email', type=str)
+@click.option('-p','--padding', type=int)
+@click.option('-D','--dry', is_flag=True)
+@click.option('-V','--verbose', is_flag=True)
+@click.option('-k', '--conda', is_flag=True)
+@click.pass_context
+def extract_command(ctx,
+                    case_id,
+                    days_ago,
+                    environment,
+                    config_file,
+                    log_directory,
+                    email,
+                    padding,
+                    dry,
+                    verbose,
+                    conda):
+
+    #Open and read config fore mutacc
+    with open(Path(config_file)) as yaml_handle:
+
+        mutacc_conf = yaml.load(yaml_handle)
+
+    #Find directory where cases ready for import are stored
+    case_dir = Path(mutacc_conf['case_dir'])
+
+    #Create a temporary dir to store created vcf, yaml, and script files
+    with TemporaryDirectory() as tmp_dir:
+
+        LOG.info("All files are placed in {}".format(tmp_dir))
+
+        #Prepare input for case with case_id or days since updated
+        inputs = get_inputs(tmp_dir, case_id=case_id, days_ago=days_ago, padding = padding)
+
+        for case_input in inputs:
+            #Extract reads for every case
+            sbatch_script = run_mutacc_extract(
+                    tmp_dir,
+                    config_file,
+                    case_input['input_file'],
+                    case_input['padding'],
+                    environment,
+                    log_directory,
+                    email=email,
+                    conda=conda,
+                    dry=dry
+                )
+
+            if verbose:
+
+                with open(sbatch_script) as sbatch_handle:
+
+                    LOG.info("\n{}".format(sbatch_handle.read()))

mutacc_auto/cli/import_command.py

@@ -6,36 +6,19 @@
 import sys
 import os
 
-from mutacc_auto.utils.tmp_dir import TemporaryDirectory
-from mutacc_auto.recipes.input_recipe import get_inputs
-from mutacc_auto.recipes.extract_recipe import run_mutacc_extract
 from mutacc_auto.recipes.import_recipe import import_extracted_case
 
 LOG = logging.getLogger(__name__)
 
 @click.command('import')
-@click.option('-c','--case-id', type=str)
-@click.option('-d','--days-ago', type=int)
-@click.option('-e','--environment', type=str)
 @click.option('-C','--config-file', type=click.Path(exists=True))
-@click.option('-L', '--log-directory', type=click.Path(exists=True))
-@click.option('-E', '--email', type=str)
-@click.option('-p','--padding', type=int)
 @click.option('-D','--dry', is_flag=True)
 @click.option('-V','--verbose', is_flag=True)
-@click.option('-k', '--conda', is_flag=True)
 @click.pass_context
 def import_command(ctx,
-                   case_id,
-                   days_ago,
-                   environment,
                    config_file,
-                   log_directory,
-                   email,
-                   padding,
                    dry,
-                   verbose,
-                   conda):
+                   verbose):
 
     #Open and read config fore mutacc
     with open(Path(config_file)) as yaml_handle:
@@ -45,46 +28,15 @@ def import_command(ctx,
     #Find directory where cases ready for import are stored
     case_dir = Path(mutacc_conf['case_dir'])
 
-    #Create a temporary dir to store created vcf, yaml, and script files
-    with TemporaryDirectory() as tmp_dir:
-
-        LOG.info("All files are placed in {}".format(tmp_dir))
-
-        #Prepare input for case with case_id or days since updated
-        inputs = get_inputs(tmp_dir, case_id=case_id, days_ago=days_ago, padding = padding)
-
-        for case_input in inputs:
-            #Extract reads for every case
-            sbatch_script = run_mutacc_extract(
-                    tmp_dir,
-                    config_file,
-                    case_input['input_file'],
-                    case_input['padding'],
-                    environment,
-                    log_directory,
-                    email=email,
-                    conda=conda,
-                    wait=True,
-                    dry=dry
-                )
-
-            if verbose:
-
-                with open(sbatch_script) as sbatch_handle:
-
-                    LOG.info("\n{}".format(sbatch_handle.read()))
-
-
-
     #For each case found in the case_dir stated in the mutacc config file
     #import to database
     for _, _, case_files in os.walk(case_dir):
         for filename in case_files:
             case_path = case_dir.joinpath(filename)
-            LOG.info("importing {}".format(filename))
 
             ### IMPORT CASE AND DELETE FILE AFTERWARDS
             if str(case_path).endswith('.mutacc'):
+                LOG.info("importing {}".format(filename))
                 if not dry:
                     import_extracted_case(str(case_path), config_file)
                     os.remove(case_path)

mutacc_auto/cli/root.py

@@ -3,6 +3,7 @@
 import coloredlogs
 
 from mutacc_auto import __version__
+from .extract_command import extract_command
 from .import_command import import_command
 
 LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
@@ -17,4 +18,5 @@ def cli(context, loglevel):
     coloredlogs.install(level = loglevel)
     LOG.info("Running mutacc_auto")
 
+cli.add_command(extract_command)
 cli.add_command(import_command)

mutacc_auto/commands/sbatch_command.py

@@ -4,12 +4,8 @@
 
 class SbatchCommand(BaseCommand):
 
-    def __init__(self, sbatch_script_path, wait=False):
+    def __init__(self, sbatch_script_path):
 
         super(SbatchCommand,self).__init__(BASE_COMMAND)
 
-        if wait:
-
-            self.add_option('wait')
-
         self.add_argument(sbatch_script_path)

mutacc_auto/recipes/extract_recipe.py

@@ -61,7 +61,7 @@ def get_mutacc_extract_command(mutacc_conf, input_file, padding):
 
     return str(mutacc_extract_command)
 
-def sbatch_run(sbatch_script_path, wait=False, dry=False):
+def sbatch_run(sbatch_script_path, dry=False):
     """
         Function to execute the sbatch script
 
@@ -70,7 +70,7 @@ def sbatch_run(sbatch_script_path, wait=False, dry=False):
             wait (bool): if True, run sbatch with --wait flag
             dry (bool): if True, does not send job to slurm
     """
-    sbatch_command = SbatchCommand(sbatch_script_path, wait=wait)
+    sbatch_command = SbatchCommand(sbatch_script_path)
 
     if not dry:
         sbatch_command.call()
@@ -106,6 +106,6 @@ def run_mutacc_extract(tmp_dir,
 
 
 
-    sbatch_run(sbatch_script_path, wait, dry=dry)
+    sbatch_run(sbatch_script_path, dry=dry)
 
     return sbatch_script_path

tests/cli/test_extract_command.py

@@ -0,0 +1,58 @@
+import pytest
+from mock import patch
+import yaml
+from pathlib import Path
+
+from click.testing import CliRunner
+
+from mutacc_auto.cli.root import cli
+
+INPUTS_LIST = [{'input_file': 'path_to_file', 'padding':600}]*2
+SBATCH_TEMPLATE = "tests/fixtures/sbatch_template.txt"
+
+@patch('mutacc_auto.cli.extract_command.get_inputs')
+@patch('mutacc_auto.cli.extract_command.run_mutacc_extract')
+def test_extract_command(
+        mock_extract,
+        mock_inputs,
+        tmpdir
+    ):
+
+    tmp_dir = Path(tmpdir.mkdir('test_export_command'))
+    mock_inputs.return_value = INPUTS_LIST
+    mock_extract.return_value = SBATCH_TEMPLATE
+
+    with open(tmp_dir.joinpath('tmp_conf.yaml'), 'w') as conf_handle:
+
+        conf_dict = {'case_dir': str(tmp_dir)}
+        yaml.dump(conf_dict, conf_handle)
+        conf_path = conf_handle.name
+
+    with open(tmp_dir.joinpath('tmp_test.mutacc'), 'w') as handle:
+        handle.write('\n')
+
+    runner = CliRunner()
+    result = runner.invoke(cli, [
+            'extract',
+            '--case-id', 'test_id',
+            '--environment', 'env',
+            '--config-file', conf_path,
+            '--log-directory', str(tmp_dir),
+            '--email', 'email@email.com',
+            '--dry',
+            '--verbose'
+        ]
+    )
+
+    assert result.exit_code == 0
+
+    result = runner.invoke(cli, [
+            'extract',
+            '--days-ago', '300',
+            '--environment', 'env',
+            '--config-file', conf_path,
+            '--log-directory', tmp_dir
+        ]
+    )
+
+    assert result.exit_code == 0

tests/cli/test_import_command.py

@@ -10,37 +10,31 @@
 INPUTS_LIST = [{'input_file': 'path_to_file', 'padding':600}]*2
 SBATCH_TEMPLATE = "tests/fixtures/sbatch_template.txt"
 
-@patch('mutacc_auto.cli.import_command.get_inputs')
-@patch('mutacc_auto.cli.import_command.run_mutacc_extract')
+TMP_DIR_NAME = 'test_import_command'
+CONF_FILE_NAME = 'tmp_conf.yaml'
+IMPORT_FILE_NAME = 'tmp_test.mutacc'
+
 @patch('mutacc_auto.cli.import_command.import_extracted_case')
 def test_import_command(
         mock_import,
-        mock_extract,
-        mock_inputs,
         tmpdir
     ):
 
-    tmp_dir = Path(tmpdir.mkdir('test_import_command'))
-    mock_inputs.return_value = INPUTS_LIST
-    mock_extract.return_value = SBATCH_TEMPLATE
+    tmp_dir = Path(tmpdir.mkdir(TMP_DIR_NAME))
 
-    with open(tmp_dir.joinpath('tmp_conf.yaml'), 'w') as conf_handle:
+    with open(tmp_dir.joinpath(CONF_FILE_NAME), 'w') as conf_handle:
 
         conf_dict = {'case_dir': str(tmp_dir)}
         yaml.dump(conf_dict, conf_handle)
         conf_path = conf_handle.name
 
-    with open(tmp_dir.joinpath('tmp_test.mutacc'), 'w') as handle:
+    with open(tmp_dir.joinpath(IMPORT_FILE_NAME), 'w') as handle:
         handle.write('\n')
 
     runner = CliRunner()
     result = runner.invoke(cli, [
             'import',
-            '--case-id', 'test_id',
-            '--environment', 'env',
             '--config-file', conf_path,
-            '--log-directory', str(tmp_dir),
-            '--email', 'email@email.com',
             '--dry',
             '--verbose'
         ]
@@ -50,11 +44,13 @@ def test_import_command(
 
     result = runner.invoke(cli, [
             'import',
-            '--days-ago', '300',
-            '--environment', 'env',
             '--config-file', conf_path,
-            '--log-directory', tmp_dir
         ]
     )
 
     assert result.exit_code == 0
+
+    mock_import.assert_called_with(
+                str(tmp_dir.joinpath(IMPORT_FILE_NAME)),
+                conf_path
+            )

tests/commands/test_command.py

@@ -35,9 +35,9 @@ def test_Command():
 
 def test_SbatchCommand():
 
-    command = SbatchCommand('script', wait = True)
+    command = SbatchCommand('script')
 
-    assert str(command) == "sbatch --wait script"
+    assert str(command) == "sbatch script"
 
 def test_ScoutExportCases():