From 1880b4b442a385c89970f3e6d0782d88aca05b14 Mon Sep 17 00:00:00 2001 From: tiagofilipe12 Date: Fri, 8 Jun 2018 15:31:41 +0100 Subject: [PATCH 01/10] added patlas recipe --- flowcraft/flowcraft.py | 17 ++++++++++++++--- flowcraft/generator/recipe.py | 7 +------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/flowcraft/flowcraft.py b/flowcraft/flowcraft.py index cd3b84c9..d155c29e 100755 --- a/flowcraft/flowcraft.py +++ b/flowcraft/flowcraft.py @@ -15,7 +15,7 @@ from __init__ import __version__, __build__ from generator.engine import NextflowGenerator, process_map from generator.inspect import NextflowInspector - from generator.recipe import brew_recipe + from generator.recipe import brew_recipe, Innuendo from generator.pipeline_parser import parse_pipeline, SanityError from generator.process_details import proc_collector, colored_print import generator.error_handling as eh @@ -23,7 +23,7 @@ from flowcraft import __version__, __build__ from flowcraft.generator.engine import NextflowGenerator, process_map from flowcraft.generator.inspect import NextflowInspector - from flowcraft.generator.recipe import brew_recipe + from flowcraft.generator.recipe import brew_recipe, Innuendo from flowcraft.generator.pipeline_parser import parse_pipeline, \ SanityError from flowcraft.generator.process_details import proc_collector, \ @@ -32,6 +32,12 @@ logger = logging.getLogger("main") +available_recipes = { + "innuendo": Innuendo, + "plasmid_detection": "integrity_coverage fastqc_trimmomatic (spades pilon " + "(mash_dist | abricate) | mash_screen | mapping_patlas)", +} + def get_args(args=None): @@ -199,7 +205,12 @@ def build(args): # If a recipe is specified, build pipeline based on the # appropriate recipe if args.recipe: - pipeline_string, list_processes = brew_recipe(args) + if args.recipe == "innuendo": + pipeline_string, list_processes = brew_recipe(args, + available_recipes) + else: + pipeline_string = available_recipes[args.recipe] + list_processes = None else: pipeline_string = args.tasks list_processes = None diff --git a/flowcraft/generator/recipe.py b/flowcraft/generator/recipe.py index 5014b130..18b14ac9 100644 --- a/flowcraft/generator/recipe.py +++ b/flowcraft/generator/recipe.py @@ -517,12 +517,7 @@ def __init__(self, *args, **kwargs): } -available_recipes = { - "innuendo": Innuendo -} - - -def brew_recipe(args): +def brew_recipe(args, available_recipes): """Brews a given list of processes according to the recipe Parameters From fea9f32e27f1f61b0854d2ca1f7467dd4214e6a9 Mon Sep 17 00:00:00 2001 From: tiagofilipe12 Date: Fri, 8 Jun 2018 16:06:48 +0100 Subject: [PATCH 02/10] added argument to check recipes and other recipes for plasmids --- flowcraft/flowcraft.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/flowcraft/flowcraft.py b/flowcraft/flowcraft.py index d155c29e..13135fe4 100755 --- a/flowcraft/flowcraft.py +++ b/flowcraft/flowcraft.py @@ -32,10 +32,15 @@ logger = logging.getLogger("main") +# A dictionary of quick recipes available_recipes = { "innuendo": Innuendo, - "plasmid_detection": "integrity_coverage fastqc_trimmomatic (spades pilon " + "plasmids": "integrity_coverage fastqc_trimmomatic (spades pilon " "(mash_dist | abricate) | mash_screen | mapping_patlas)", + "plasmids_mapping": "integrity_coverage fastqc_trimmomatic mapping_patlas", + "plasmids_assembly": "integrity_coverage fastqc_trimmomatic (spades pilon " + "(mash_dist | abricate))", + "plasmids_mash": "integrity_coverage fastqc_trimmomatic mash_screen", } @@ -83,6 +88,12 @@ def get_args(args=None): "-l", "--short-list", action="store_const", dest="short_list", const=True, help="Print a short list of the currently " "available processes") + build_parser.add_argument("-cr", "--check-recipe", dest="check_recipe", + action="store_const", const=True, + help="Check tasks that the recipe contain and " + "their flow. This option might be useful " + "if a user wants to change some components " + "of a given recipe, by using the -t option.") # GENERAL OPTIONS parser.add_argument( @@ -131,7 +142,8 @@ def validate_build_arguments(args): "-l, -L", "red_bold")) sys.exit(1) - if (args.tasks or args.recipe) and not args.output_nf: + if (args.tasks or args.recipe) and not args.check_recipe \ + and not args.output_nf: logger.error(colored_print( "Please provide the path and name of the pipeline file using the" " -o option.", "red_bold")) @@ -211,6 +223,11 @@ def build(args): else: pipeline_string = available_recipes[args.recipe] list_processes = None + if args.check_recipe: + logger.info(colored_print("Pipeline string for recipe: {}" + .format(args.recipe), "purple_bold")) + logger.info(pipeline_string) + sys.exit(0) else: pipeline_string = args.tasks list_processes = None From 866b2fc0ee4f7cc8dc4c3b07b2f5145825b2eff6 Mon Sep 17 00:00:00 2001 From: tiagofilipe12 Date: Mon, 11 Jun 2018 14:54:29 +0100 Subject: [PATCH 03/10] removed prokka from innuendo recipe --- flowcraft/generator/recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowcraft/generator/recipe.py b/flowcraft/generator/recipe.py index 18b14ac9..182edf6c 100644 --- a/flowcraft/generator/recipe.py +++ b/flowcraft/generator/recipe.py @@ -512,7 +512,7 @@ def __init__(self, *args, **kwargs): "mlst": [False, "pilon", "abricate|prokka|chewbbaca|sistr"], "sistr": [True, "mlst", None], "abricate": [True, "mlst", None], - "prokka": [True, "mlst", None], + #"prokka": [True, "mlst", None], "chewbbaca": [True, "mlst", None] } From a5dce88bf7851771045ad1651ac51faeb07cab90 Mon Sep 17 00:00:00 2001 From: tiagofilipe12 Date: Mon, 11 Jun 2018 17:30:46 +0100 Subject: [PATCH 04/10] added checks for list of processes for recipes --- flowcraft/flowcraft.py | 16 ++++++++++------ flowcraft/generator/process_details.py | 24 ++++++++++-------------- flowcraft/generator/recipe.py | 7 +++---- flowcraft/tests/test_assemblerflow.py | 18 ------------------ flowcraft/tests/test_process_details.py | 8 ++++++-- 5 files changed, 29 insertions(+), 44 deletions(-) diff --git a/flowcraft/flowcraft.py b/flowcraft/flowcraft.py index 13135fe4..86106c47 100755 --- a/flowcraft/flowcraft.py +++ b/flowcraft/flowcraft.py @@ -38,8 +38,8 @@ "plasmids": "integrity_coverage fastqc_trimmomatic (spades pilon " "(mash_dist | abricate) | mash_screen | mapping_patlas)", "plasmids_mapping": "integrity_coverage fastqc_trimmomatic mapping_patlas", - "plasmids_assembly": "integrity_coverage fastqc_trimmomatic (spades pilon " - "(mash_dist | abricate))", + "plasmids_assembly": "integrity_coverage fastqc_trimmomatic (spades pilon" + " mash_dist)", "plasmids_mash": "integrity_coverage fastqc_trimmomatic mash_screen", } @@ -218,11 +218,16 @@ def build(args): # appropriate recipe if args.recipe: if args.recipe == "innuendo": - pipeline_string, list_processes = brew_recipe(args, + pipeline_string = brew_recipe(args, available_recipes) else: pipeline_string = available_recipes[args.recipe] - list_processes = None + if args.tasks: + logger.warning(colored_print( + "-t parameter will be ignored for recipe: {}\n" + .format(args.recipe), "yellow_bold") + ) + if args.check_recipe: logger.info(colored_print("Pipeline string for recipe: {}" .format(args.recipe), "purple_bold")) @@ -230,10 +235,9 @@ def build(args): sys.exit(0) else: pipeline_string = args.tasks - list_processes = None # used for lists print - proc_collector(process_map, args, list_processes) + proc_collector(process_map, args, pipeline_string) logger.info(colored_print("Resulting pipeline string:\n")) logger.info(colored_print(pipeline_string + "\n")) diff --git a/flowcraft/generator/process_details.py b/flowcraft/generator/process_details.py index f851997f..85fc5aa8 100644 --- a/flowcraft/generator/process_details.py +++ b/flowcraft/generator/process_details.py @@ -9,7 +9,8 @@ "white_bold": "1;38m", "white_underline": "4;38m", "blue_bold": "1;36m", - "purple_bold": "1;34m" + "purple_bold": "1;34m", + "yellow_bold": "1;93m" } @@ -74,7 +75,7 @@ def procs_dict_parser(procs_dict): )) -def proc_collector(process_map, args, processes_list=None): +def proc_collector(process_map, args, pipeline_string): """ Function that collects all processes available and stores a dictionary of the required arguments of each process class to be passed to @@ -88,9 +89,8 @@ def proc_collector(process_map, args, processes_list=None): args: argparse.Namespace The arguments passed through argparser that will be access to check the type of list to be printed - processes_list: list - List with all the available processes of a recipe. In case no recipe - is passed, the list should come empty. + pipeline_string: str + the pipeline string """ @@ -119,17 +119,13 @@ def proc_collector(process_map, args, processes_list=None): # loops between all process_map Processes for name, cls in process_map.items(): - if processes_list: - # Skip process if process_list is provided and name not in - # processes list - if name not in processes_list: - continue - # instantiates each Process class cls_inst = cls(template=name) - d = {arg_key: vars(cls_inst)[arg_key] for arg_key in vars(cls_inst) - if arg_key in arguments_list} - procs_dict[name] = d + + if name in pipeline_string: + d = {arg_key: vars(cls_inst)[arg_key] for arg_key in + vars(cls_inst) if arg_key in arguments_list} + procs_dict[name] = d procs_dict_parser(procs_dict) diff --git a/flowcraft/generator/recipe.py b/flowcraft/generator/recipe.py index 182edf6c..06067aa2 100644 --- a/flowcraft/generator/recipe.py +++ b/flowcraft/generator/recipe.py @@ -495,7 +495,8 @@ def __init__(self, *args, **kwargs): # The description of the processes # [forkable, input_process, output_process] self.process_descriptions = { - "reads_download": [True, None,"integrity_coverage|seq_typing|patho_typing"], + "reads_download": [True, None, + "integrity_coverage|seq_typing|patho_typing"], "patho_typing": [True, None, None], "seq_typing": [True, None, None], "integrity_coverage": [True, None, "fastqc_trimmomatic"], @@ -550,8 +551,6 @@ def brew_recipe(args, available_recipes): else: input_processes = args.tasks - # Get the list of processes for that recipe - list_processes = automatic_pipeline.get_process_info() # Validate the provided pipeline processes validated = automatic_pipeline.validate_pipeline(input_processes) if not validated: @@ -559,4 +558,4 @@ def brew_recipe(args, available_recipes): # Get the final pipeline string pipeline_string = automatic_pipeline.run_auto_pipeline(input_processes) - return pipeline_string, list_processes + return pipeline_string diff --git a/flowcraft/tests/test_assemblerflow.py b/flowcraft/tests/test_assemblerflow.py index e9310cff..401f6c99 100644 --- a/flowcraft/tests/test_assemblerflow.py +++ b/flowcraft/tests/test_assemblerflow.py @@ -14,24 +14,6 @@ def tmp(): shutil.rmtree("temp") -def test_list_short(): - - sys.argv.append(1) - args = af.get_args(["build", "-l"]) - - with pytest.raises(SystemExit): - af.build(args) - - -def test_list_long(): - - sys.argv.append(1) - args = af.get_args(["build", "-L"]) - - with pytest.raises(SystemExit): - af.build(args) - - def test_check(): sys.argv.append(1) diff --git a/flowcraft/tests/test_process_details.py b/flowcraft/tests/test_process_details.py index a05a081e..39cf5129 100644 --- a/flowcraft/tests/test_process_details.py +++ b/flowcraft/tests/test_process_details.py @@ -19,14 +19,18 @@ def test_long_list(): arguments = af.get_args(["build", "-L"]) + pipeline_string = "fastqc trimmomatic" + with pytest.raises(SystemExit): - pd.proc_collector(process_map, arguments) + pd.proc_collector(process_map, arguments, pipeline_string) def test_short_list(): arguments = af.get_args(["build", "-l"]) + pipeline_string = "fastqc trimmomatic" + with pytest.raises(SystemExit): - pd.proc_collector(process_map, arguments) + pd.proc_collector(process_map, arguments, pipeline_string) \ No newline at end of file From ff58deade8a6aaaa13cd8da4aee233fec3abf71b Mon Sep 17 00:00:00 2001 From: tiagofilipe12 Date: Mon, 11 Jun 2018 17:36:47 +0100 Subject: [PATCH 05/10] added check for list arguments to validate_build_arguments --- docs/index.rst | 1 + docs/user/available_recipes.rst | 27 +++++++++++++++++++++++++++ flowcraft/flowcraft.py | 3 +++ 3 files changed, 31 insertions(+) create mode 100644 docs/user/available_recipes.rst diff --git a/docs/index.rst b/docs/index.rst index 9cdcd1ed..cc095560 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -27,6 +27,7 @@ A NextFlow pipeline assembler for genomics. user/pipeline_building user/pipeline_configuration user/available_components + user/available_recipes user/pipeline_inspect .. _Developer Guide: diff --git a/docs/user/available_recipes.rst b/docs/user/available_recipes.rst new file mode 100644 index 00000000..e4317ffa --- /dev/null +++ b/docs/user/available_recipes.rst @@ -0,0 +1,27 @@ +Recipes +======= + +Recipes are pre-built and static pipelines that give the user a quick-start or +that are required for some specific purpose. + +List of recipes +--------------- + +- ``plasmids``: lala + +- ``plasmids_mapping``: lele + +- ``plasmids_assembly``: lolo + +- ``plasmids_mash``: lili + +- I/O +- para que serve o output + +-link pipeline cfg para mudar directivas e params + + +SAMPLE: +These are the currently available FlowCraft components with a short +description of their tasks. For a more detailed information, follow the +links of each components. diff --git a/flowcraft/flowcraft.py b/flowcraft/flowcraft.py index 86106c47..5e26cf5c 100755 --- a/flowcraft/flowcraft.py +++ b/flowcraft/flowcraft.py @@ -135,6 +135,9 @@ def get_args(args=None): def validate_build_arguments(args): + if args.detailed_list or args.short_list: + return + if not args.tasks and not args.recipe and not args.check_only \ and not args.detailed_list and not args.short_list: logger.error(colored_print( From 7187bc7afd99295eebd00e14010c3b686f094c69 Mon Sep 17 00:00:00 2001 From: tiagofilipe12 Date: Mon, 11 Jun 2018 17:41:07 +0100 Subject: [PATCH 06/10] linted code --- docs/user/available_recipes.rst | 27 --------------------------- flowcraft/flowcraft.py | 3 +-- 2 files changed, 1 insertion(+), 29 deletions(-) delete mode 100644 docs/user/available_recipes.rst diff --git a/docs/user/available_recipes.rst b/docs/user/available_recipes.rst deleted file mode 100644 index e4317ffa..00000000 --- a/docs/user/available_recipes.rst +++ /dev/null @@ -1,27 +0,0 @@ -Recipes -======= - -Recipes are pre-built and static pipelines that give the user a quick-start or -that are required for some specific purpose. - -List of recipes ---------------- - -- ``plasmids``: lala - -- ``plasmids_mapping``: lele - -- ``plasmids_assembly``: lolo - -- ``plasmids_mash``: lili - -- I/O -- para que serve o output - --link pipeline cfg para mudar directivas e params - - -SAMPLE: -These are the currently available FlowCraft components with a short -description of their tasks. For a more detailed information, follow the -links of each components. diff --git a/flowcraft/flowcraft.py b/flowcraft/flowcraft.py index 5e26cf5c..a692b3e0 100755 --- a/flowcraft/flowcraft.py +++ b/flowcraft/flowcraft.py @@ -221,8 +221,7 @@ def build(args): # appropriate recipe if args.recipe: if args.recipe == "innuendo": - pipeline_string = brew_recipe(args, - available_recipes) + pipeline_string = brew_recipe(args, available_recipes) else: pipeline_string = available_recipes[args.recipe] if args.tasks: From 0caf9650df4c20c9b3e2907615dec8b10ee0f71a Mon Sep 17 00:00:00 2001 From: tiagofilipe12 Date: Mon, 11 Jun 2018 19:56:04 +0100 Subject: [PATCH 07/10] commented no longer used method in Recipe class --- docs/index.rst | 1 - flowcraft/generator/recipe.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index cc095560..9cdcd1ed 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -27,7 +27,6 @@ A NextFlow pipeline assembler for genomics. user/pipeline_building user/pipeline_configuration user/available_components - user/available_recipes user/pipeline_inspect .. _Developer Guide: diff --git a/flowcraft/generator/recipe.py b/flowcraft/generator/recipe.py index 06067aa2..63eb3813 100644 --- a/flowcraft/generator/recipe.py +++ b/flowcraft/generator/recipe.py @@ -478,8 +478,8 @@ def run_auto_pipeline(self, tasks): return self.pipeline_string - def get_process_info(self): - return list(self.process_descriptions.keys()) + # def get_process_info(self): + # return list(self.process_descriptions.keys()) class Innuendo(Recipe): From 4d430db28f77439817878f52f0c5b16aeefe20e6 Mon Sep 17 00:00:00 2001 From: tiagofilipe12 Date: Mon, 11 Jun 2018 20:22:18 +0100 Subject: [PATCH 08/10] changed location of available_recipes dict --- flowcraft/flowcraft.py | 15 ++------------- flowcraft/generator/recipe.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/flowcraft/flowcraft.py b/flowcraft/flowcraft.py index a692b3e0..fdfbf308 100755 --- a/flowcraft/flowcraft.py +++ b/flowcraft/flowcraft.py @@ -15,7 +15,7 @@ from __init__ import __version__, __build__ from generator.engine import NextflowGenerator, process_map from generator.inspect import NextflowInspector - from generator.recipe import brew_recipe, Innuendo + from generator.recipe import brew_recipe, available_recipes from generator.pipeline_parser import parse_pipeline, SanityError from generator.process_details import proc_collector, colored_print import generator.error_handling as eh @@ -23,7 +23,7 @@ from flowcraft import __version__, __build__ from flowcraft.generator.engine import NextflowGenerator, process_map from flowcraft.generator.inspect import NextflowInspector - from flowcraft.generator.recipe import brew_recipe, Innuendo + from flowcraft.generator.recipe import brew_recipe, available_recipes from flowcraft.generator.pipeline_parser import parse_pipeline, \ SanityError from flowcraft.generator.process_details import proc_collector, \ @@ -32,17 +32,6 @@ logger = logging.getLogger("main") -# A dictionary of quick recipes -available_recipes = { - "innuendo": Innuendo, - "plasmids": "integrity_coverage fastqc_trimmomatic (spades pilon " - "(mash_dist | abricate) | mash_screen | mapping_patlas)", - "plasmids_mapping": "integrity_coverage fastqc_trimmomatic mapping_patlas", - "plasmids_assembly": "integrity_coverage fastqc_trimmomatic (spades pilon" - " mash_dist)", - "plasmids_mash": "integrity_coverage fastqc_trimmomatic mash_screen", -} - def get_args(args=None): diff --git a/flowcraft/generator/recipe.py b/flowcraft/generator/recipe.py index 63eb3813..b3dfaf9b 100644 --- a/flowcraft/generator/recipe.py +++ b/flowcraft/generator/recipe.py @@ -559,3 +559,15 @@ def brew_recipe(args, available_recipes): pipeline_string = automatic_pipeline.run_auto_pipeline(input_processes) return pipeline_string + + +# A dictionary of quick recipes +available_recipes = { + "innuendo": Innuendo, + "plasmids": "integrity_coverage fastqc_trimmomatic (spades pilon " + "(mash_dist | abricate) | mash_screen | mapping_patlas)", + "plasmids_mapping": "integrity_coverage fastqc_trimmomatic mapping_patlas", + "plasmids_assembly": "integrity_coverage fastqc_trimmomatic (spades pilon" + " mash_dist)", + "plasmids_mash": "integrity_coverage fastqc_trimmomatic mash_screen", +} From 001eae1ab1a7c99518124c0dcde9931986dc17df Mon Sep 17 00:00:00 2001 From: tiagofilipe12 Date: Tue, 19 Jun 2018 16:06:02 +0100 Subject: [PATCH 09/10] fixed pipeline_string parsing through -l and -L options --- flowcraft/generator/process_details.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/flowcraft/generator/process_details.py b/flowcraft/generator/process_details.py index 5b8cf9b0..240359b6 100644 --- a/flowcraft/generator/process_details.py +++ b/flowcraft/generator/process_details.py @@ -123,10 +123,14 @@ def proc_collector(process_map, args, pipeline_string): # instantiates each Process class cls_inst = cls(template=name) - if name in pipeline_string: - d = {arg_key: vars(cls_inst)[arg_key] for arg_key in - vars(cls_inst) if arg_key in arguments_list} - procs_dict[name] = d + # checks if recipe is provided + if pipeline_string: + if name not in pipeline_string: + continue + + d = {arg_key: vars(cls_inst)[arg_key] for arg_key in + vars(cls_inst) if arg_key in arguments_list} + procs_dict[name] = d procs_dict_parser(procs_dict) From 2de23e0c14cff270c85fbaf7df76cd654749459e Mon Sep 17 00:00:00 2001 From: tiagofilipe12 Date: Tue, 19 Jun 2018 16:14:35 +0100 Subject: [PATCH 10/10] updated changelog --- changelog.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index 6d9a87e4..998023a3 100644 --- a/changelog.md +++ b/changelog.md @@ -4,14 +4,28 @@ ### New components -- `maxbin2`: An automatic tool for binning metagenomic sequences -- `bowtie2`: Align short paired-end sequencing reads to long reference sequences +- `maxbin2`: An automatic tool for binning metagenomic sequences. +- `bowtie2`: Align short paired-end sequencing reads to long reference +sequences. + +### New recipes + +- `plasmids`: A recipe to perform mapping, mash screen on reads +and also mash dist for assembly based approaches (all to detect +plasmdis). This also includes annotation with abricate for the assembly. +- `plasmids_mapping`: A recipe to perfmorm mapping for plasmids. +- `plasmids_mash`: A recipe to perform mash screen for plasmids. +- `plasmids_assembly`: A recipe to perform mash dist for plasmid +assemblies. ### Minor/Other changes - Added "smart" check when the user provides a typo in pipeline string for a given process, outputting some "educated" guesses to the the terminal. +- Added "-cr" option to show current recipe `pipeline_string`. +- Changed the way recipes were being parsed by `proc_collector` for the +usage of `-l` and `-L` options. ### Bug fixes