Merge pull request #322 from oist/feature/debug

Feature/debug

optinist/Snakefile

@@ -1,11 +1,35 @@
-import os
 from optinist.api.dir_path import DIRPATH
 from optinist.api.utils.filepath_creater import join_filepath
+from optinist.api.snakemake.smk_utils import smk_input, smk_output, smk_conda
+from optinist.routers.model import FILETYPE
 
 configfile: join_filepath([DIRPATH.ROOT_DIR, 'config.yaml'])
 
-for rule, details in config["rules"].items():
-    include: details["rule_file"]
 
 rule all:
-    input: [join_filepath([DIRPATH.OUTPUT_DIR, x]) for x in config["last_output"]]
+    input: [join_filepath([DIRPATH.OUTPUT_DIR, x]) for x in config["last_output"]]
+
+
+for rule_name, details in config["rules"].items():
+    if details["type"] in [FILETYPE.IMAGE, FILETYPE.CSV, FILETYPE.BEHAVIOR, FILETYPE.HDF5]:
+        rule:
+            input:
+                smk_input(details)
+            output:
+                smk_output(details)
+            params:
+                name = details
+            script:
+                f"{DIRPATH.ROOT_DIR}/rules/scripts/data.py"
+    else:
+        rule:
+            input:
+                smk_input(details)
+            output:
+                smk_output(details)
+            params:
+                name = details
+            conda:
+                smk_conda(details)
+            script:
+                f"{DIRPATH.ROOT_DIR}/rules/scripts/func.py"

optinist/api/snakemake/smk.py

@@ -7,7 +7,6 @@
 
 @dataclass
 class Rule:
-    rule_file: str
     input: list
     return_arg: Union[str, Dict[str, str]]
     params: dict

optinist/api/snakemake/smk_utils.py

@@ -0,0 +1,50 @@
+from optinist.api.dir_path import DIRPATH
+from optinist.api.utils.filepath_creater import join_filepath
+from optinist.routers.model import FILETYPE
+from optinist.wrappers import wrapper_dict
+
+
+def smk_input(details):
+    if details["type"] in [FILETYPE.IMAGE]:
+        return [
+            join_filepath([DIRPATH.INPUT_DIR, x])
+            for x in details["input"]
+        ]
+    elif details["type"] in [FILETYPE.CSV, FILETYPE.BEHAVIOR, FILETYPE.HDF5]:
+        return join_filepath([DIRPATH.INPUT_DIR, details["input"]])
+    else:
+        return [
+            join_filepath([DIRPATH.OUTPUT_DIR, x])
+            for x in details["input"]
+        ]
+
+
+def smk_output(details):
+    return join_filepath([
+        DIRPATH.OUTPUT_DIR,
+        details["output"]
+    ])
+
+
+def smk_conda(details):
+    if details["type"] in [FILETYPE.IMAGE, FILETYPE.CSV, FILETYPE.BEHAVIOR, FILETYPE.HDF5]:
+        return None
+
+    wrapper = _dict2leaf(
+        wrapper_dict,
+        details["path"].split('/')
+    )
+
+    if "conda" in wrapper:
+        env_filename = wrapper['conda']
+        return f"{DIRPATH.ROOT_DIR}/rules/envs/{env_filename}"
+    else:
+        return None
+
+
+def _dict2leaf(root_dict: dict, path_list):
+    path = path_list.pop(0)
+    if len(path_list) > 0:
+        return _dict2leaf(root_dict[path], path_list)
+    else:
+        return root_dict[path]

optinist/api/snakemake/snakemake_reader.py

@@ -5,7 +5,6 @@ class RuleConfigReader:
     @classmethod
     def read(cls, rule):
         return Rule(
-            rule_file=rule["rule_file"],
             input=rule["input"],
             return_arg=rule["return_arg"],
             params=rule["params"],

optinist/api/snakemake/snakemake_setfile.py

@@ -19,7 +19,6 @@ def image(cls, unique_id: str, node: Node, edgeDict: Dict[str, Edge], nwbfile):
         )
 
         return Rule(
-            rule_file=f"rules/smk/image.smk",
             input=node.data.path,
             return_arg=return_name,
             params=node.data.param,
@@ -39,7 +38,6 @@ def csv(cls, unique_id, node: Node, edgeDict: Dict[str, Edge], nwbfile, nodeType
         )
 
         return Rule(
-            rule_file=f"rules/smk/{nodeType}.smk",
             input=node.data.path,
             return_arg=return_name,
             params=node.data.param,
@@ -59,7 +57,6 @@ def hdf5(cls, unique_id, node: Node, edgeDict: Dict[str, Edge], nwbfile):
         )
 
         return Rule(
-            rule_file=f"rules/smk/hdf5.smk",
             input=node.data.path,
             return_arg=return_name,
             params=node.data.param,
@@ -99,7 +96,6 @@ def algo(cls, unique_id, node: Node, edgeDict: Dict[str, Edge], nodeDict: Dict[s
         algo_output = get_pickle_file(unique_id, node.id, node.data.label)
 
         return Rule(
-            rule_file=f"rules/smk/{node.data.path}.smk",
             input=algo_input,
             return_arg=return_arg_names,
             params=params,

optinist/rules/scripts/data.py

@@ -6,33 +6,25 @@
 from optinist.rules.scripts.file_writer import FileWriter
 from optinist.api.snakemake.snakemake_reader import RuleConfigReader
 from optinist.routers.model import FILETYPE
-from optinist.api.dir_path import DIRPATH
-from optinist.api.utils.filepath_creater import join_filepath
 
 
 if __name__ == '__main__':
     last_output = snakemake.config["last_output"]
 
-    for rule_config in snakemake.config["rules"].values():
-        rule_config = RuleConfigReader.read(rule_config)
-
-
-        if rule_config.type in [FILETYPE.IMAGE]:
-            rule_config.input = [
-                join_filepath([DIRPATH.INPUT_DIR, x])
-                for x in rule_config.input
-            ]
-        elif rule_config.type in [FILETYPE.CSV, FILETYPE.BEHAVIOR, FILETYPE.HDF5]:
-            rule_config.input = join_filepath([DIRPATH.INPUT_DIR, rule_config.input])
-
-        rule_config.output = join_filepath([DIRPATH.OUTPUT_DIR, rule_config.output])
-
-        if rule_config.type in [FILETYPE.CSV, FILETYPE.BEHAVIOR]:
-            outputfile = FileWriter.csv(rule_config, rule_config.type)
-            PickleWriter.write(rule_config.output, outputfile)
-        elif rule_config.type == FILETYPE.IMAGE:
-            outputfile = FileWriter.image(rule_config)
-            PickleWriter.write(rule_config.output, outputfile)
-        elif rule_config.type == FILETYPE.HDF5:
-            outputfile = FileWriter.hdf5(rule_config)
-            PickleWriter.write(rule_config.output, outputfile)
+    rule_config = RuleConfigReader.read(snakemake.params.name)
+    if rule_config.type in [FILETYPE.IMAGE]:
+        rule_config.input = snakemake.input
+    elif rule_config.type in [FILETYPE.CSV, FILETYPE.BEHAVIOR, FILETYPE.HDF5]:
+        rule_config.input = snakemake.input[0]
+
+    rule_config.output = snakemake.output[0]
+
+    if rule_config.type in [FILETYPE.CSV, FILETYPE.BEHAVIOR]:
+        outputfile = FileWriter.csv(rule_config, rule_config.type)
+        PickleWriter.write(rule_config.output, outputfile)
+    elif rule_config.type == FILETYPE.IMAGE:
+        outputfile = FileWriter.image(rule_config)
+        PickleWriter.write(rule_config.output, outputfile)
+    elif rule_config.type == FILETYPE.HDF5:
+        outputfile = FileWriter.hdf5(rule_config)
+        PickleWriter.write(rule_config.output, outputfile)

optinist/rules/scripts/func.py

@@ -13,14 +13,9 @@
         for x in snakemake.config["last_output"]
     ]
 
-    for rule_config in snakemake.config["rules"].values():
-        rule_config = RuleConfigReader.read(rule_config)
+    rule_config = RuleConfigReader.read(snakemake.params.name)
 
-        rule_config.input = [
-            join_filepath([DIRPATH.OUTPUT_DIR, x])
-            for x in rule_config.input
-        ]
-        rule_config.output = join_filepath([DIRPATH.OUTPUT_DIR, rule_config.output])
+    rule_config.input = snakemake.input
+    rule_config.output = snakemake.output[0]
 
-        if rule_config.type == snakemake.params.name:  
-            Runner.run(rule_config, last_output)
+    Runner.run(rule_config, last_output)