Dag files (#209)

* move DAG JSON files to the resources directory

* added manifest information to the `nextflow.config` file to allow for remote execution (#204) - Partial solve to #194 issue
- Deprecation of the `manifest.config´ file

changelog.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## Upcoming in `dev` 
+
+### Bug fixes
+
+- Fix bug in `downsample_fastq` where the resulting output files was not being saved in the `results` directory
+- Fix bug in `downsample_fastq` where the output files were being saved as broken symlinks when there was no 
+down-sampling occurring 
+
+### Minor/Other changes
+
+- `treeDag.json` and `forktree.json` files are no longer hidden and are now stored in the 
+`resources` directory
+
 ## 1.4.2
 
 ### New components

flowcraft/flowcraft.py

@@ -358,6 +358,11 @@ def build(args):
 
     logger.info(colored_print("Building your awesome pipeline..."))
 
+    # copy template to cwd, to allow for immediate execution
+    # this is executed before the build() method to avoid issues with the resources/ folder
+    if not args.pipeline_only:
+        copy_project(parsed_output_nf)
+
     if args.export_params:
         nfg.export_params()
         sys.exit(0)
@@ -371,10 +376,6 @@ def build(args):
         # building the actual pipeline nf file
         nfg.build()
 
-    # copy template to cwd, to allow for immediate execution
-    if not args.pipeline_only:
-        copy_project(parsed_output_nf)
-
     logger.info(colored_print("DONE!", "green_bold"))
 
 

flowcraft/generator/components/mapping.py

@@ -245,4 +245,4 @@ def __init__(self, **kwargs):
         self.status_channels = [
             "base_recalibrator",
             "apply_bqsr"
-        ]
+        ]

flowcraft/generator/components/variant_calling.py

@@ -49,4 +49,4 @@ def __init__(self, **kwargs):
         self.status_channels = [
             "haplotypecaller",
             "merge_vcfs"
-        ]
+        ]

flowcraft/generator/engine.py

@@ -1300,21 +1300,23 @@ def _set_configurations(self):
         })
         self.user_config = self._render_config("user.config", {})
 
-    def dag_to_file(self, dict_viz, output_file=".treeDag.json"):
-        """Writes dag to output file
+    def dag_info_to_file(self, dict_viz, output_folder, output_file):
+        """Writes dag or fork information to output file
 
         Parameters
         ----------
         dict_viz: dict
             Tree like dictionary that is used to export tree data of processes
-            to html file and here for the dotfile .treeDag.json
+            to html file and here for the treeDag.json, stored in the resources directory
+        output_folder: str
+            Path folder to save the output file
+        output_file: str
+            Output file name
 
         """
 
-        outfile_dag = open(os.path.join(dirname(self.nf_file), output_file)
-                           , "w")
-        outfile_dag.write(json.dumps(dict_viz))
-        outfile_dag.close()
+        with open(os.path.join(output_folder, output_file), "w") as outfile:
+            outfile.write(json.dumps(dict_viz))
 
     def render_pipeline(self):
         """Write pipeline attributes to json
@@ -1379,13 +1381,16 @@ def render_pipeline(self):
 
                 last_of_us[p.lane] = lst[-1]["children"]
 
+        # check if resources dir exists - necessary for dag files
+        resources_dir = os.path.join(dirname(self.nf_file), "resources")
+        if not os.path.exists(resources_dir):
+            os.mkdir(resources_dir)
+
         # write to file dict_viz
-        self.dag_to_file(dict_viz)
+        self.dag_info_to_file(dict_viz, resources_dir, "treeDag.json")
 
-        # Write tree forking information for dotfile
-        with open(os.path.join(dirname(self.nf_file),
-                               ".forkTree.json"), "w") as fh:
-            fh.write(json.dumps(self._fork_tree))
+        # Write tree forking information
+        self.dag_info_to_file(self._fork_tree, resources_dir, "forkTree.json")
 
         # send with jinja to html resource
         return self._render_config("pipeline_graph.html", {"data": dict_viz})

flowcraft/generator/error_handling.py

@@ -34,6 +34,7 @@ def __init__(self, value):
     # def __str__(self):
     #     return repr(self.value)
 
+
 class LogError(Exception):
     def __init__(self, value):
         self.value = "Log ERROR: {}".format(value)

flowcraft/generator/inspect.py

@@ -1465,8 +1465,8 @@ def _prepare_static_info(self):
         return pipeline_files
 
     def _dag_file_to_dict(self):
-        """Function that opens the dotfile named .treeDag.json in the current
-        working directory
+        """Function that opens the accessory file treeDag.json in the
+        resources directory and loads it's contents to a dictionary
 
         Returns
         -------
@@ -1475,11 +1475,11 @@ def _dag_file_to_dict(self):
 
         """
         try:
-            dag_file = open(os.path.join(self.workdir, ".treeDag.json"))
+            dag_file = open(os.path.join(self.workdir, "resources", "treeDag.json"))
             dag_json = json.load(dag_file)
         except (FileNotFoundError, json.decoder.JSONDecodeError):
             logger.warning(colored_print(
-                "WARNING: dotfile named .treeDag.json not found or corrupted",
+                "WARNING: JSON file named treeDag.json not found or corrupted",
                 "red_bold"))
             dag_json = {}
 

flowcraft/generator/templates/Helper.groovy

@@ -65,8 +65,8 @@ class CollectInitialMetadata {
 
     public static void print_metadata(nextflow.script.WorkflowMetadata workflow){
 
-        def treeDag = new File("${workflow.projectDir}/.treeDag.json").text
-        def forkTree = new File("${workflow.projectDir}/.forkTree.json").text
+        def treeDag = new File("${workflow.projectDir}/resources/treeDag.json").text
+        def forkTree = new File("${workflow.projectDir}/resources/forkTree.json").text
 
         def metadataJson = "{'nfMetadata':{'scriptId':'${workflow.scriptId}',\
 'scriptName':'${workflow.scriptName}',\

flowcraft/generator/templates/downsample_fastq.nf

@@ -17,7 +17,7 @@ process downsample_fastq_{{ pid }} {
     {% include "post.txt" ignore missing %}
 
     tag { "${sample_id}" }
-    publishDir "results/downsample_fastq_{{ pid }}/", pattern: "_ss.*"
+    publishDir "results/downsample_fastq_{{ pid }}/", pattern: "*_ss.*"
 
     input:
     set sample_id, file(fastq_pair) from {{ input_channel }}
@@ -37,4 +37,4 @@ process downsample_fastq_{{ pid }} {
 
 }
 
-{{ forks }}
+{{ forks }}

flowcraft/templates/downsample_fastq.py

@@ -36,14 +36,15 @@
 
 """
 
-__version__ = "1.0.0"
-__build__ = "30072018"
+__version__ = "1.0.1"
+__build__ = "21062019"
 __template__ = "sample_fastq-nf"
 
 import os
 import re
 import json
 import subprocess
+import shutil
 
 from os.path import basename
 
@@ -156,8 +157,8 @@ def main(sample_id, fastq_pair, genome_size, depth, clear, seed):
                     os.remove(rp)
 
     else:
-        os.symlink(p1, "{}._ss.fq.gz".format(bn1))
-        os.symlink(p2, "{}._ss.fq.gz".format(bn2))
+        shutil.copy(p1, "{}._ss.fq.gz".format(bn1))
+        shutil.copy(p2, "{}._ss.fq.gz".format(bn2))
 
     # Record the original estimated coverage
     with open(".report.json", "w") as fh:

flowcraft/tests/test_assemblerflow.py

@@ -51,12 +51,13 @@ def test_build_file_2(tmp):
                         "{}".format(p), "--pipeline-only"])
     af.build(args)
 
-    assert sorted(os.listdir(tmp)) == [".forkTree.json", ".treeDag.json",
-                                       "containers.config",
+    assert sorted(os.listdir(tmp)) == ["containers.config",
                                        "lib", "nextflow.config", "params.config",
-                                       "resources.config", "teste.html",
+                                       "resources", "resources.config", "teste.html",
                                        "teste.nf", "user.config"]
 
+    assert sorted(os.listdir(os.path.join(tmp, "resources"))) == ["forkTree.json", "treeDag.json"]
+
 
 def test_build_recipe(tmp):