feat: add gitfile option to make it possible to use local git repos w…

…hen importing modules (snakemake#1376)

* feat: add gitfile option to make it possible to use local git repos when importing modules.

* refactor: clean up code.

* fix: provide path/url when retrieving schema.

* style: make black formatting happy.

* test: add test cases for using a local git repo

* Update sourcecache.py

* Update sourcecache.py

* Update sourcecache.py

* Update sourcecache.py

* Update sourcecache.py

* Update sourcecache.py

* Update sourcecache.py

* Update sourcecache.py

* Update sourcecache.py

* Update sourcecache.py

* Update sourcecache.py

* Update sourcecache.py

* fix: update tests

* make black formatting happy

* add missing expected-results

* fix failing tests

* feat: make it possible to use relative repo path

* style: make black happy

Co-authored-by: Johannes Köster <johannes.koester@tu-dortmund.de>

snakemake/sourcecache.py

@@ -149,14 +149,21 @@ def __init__(
         self.path = path
 
     def get_path_or_uri(self):
-        return "git+file://{}/{}@{}".format(self.repo_path, self.path, self.ref)
+        return "git+file://{}/{}@{}".format(
+            os.path.abspath(self.repo_path), self.path, self.ref
+        )
 
     def join(self, path):
+        path = os.path.normpath("/".join((self.path, path)))
+        if ON_WINDOWS:
+            # convert back to URL separators
+            # (win specific separators are introduced by normpath above)
+            path = path.replace("\\", "/")
         return LocalGitFile(
             self.repo_path,
-            "/".join((self.path, path)),
+            path,
             tag=self.tag,
-            ref=self.ref,
+            ref=self._ref,
             commit=self.commit,
         )
 
@@ -166,7 +173,7 @@ def get_basedir(self):
             path=os.path.dirname(self.path),
             tag=self.tag,
             commit=self.commit,
-            ref=self.ref,
+            ref=self._ref,
         )
 
     def is_persistently_cacheable(self):

snakemake/utils.py

@@ -61,7 +61,7 @@ def validate(data, schema, set_default=True):
 
     if isinstance(schemafile, LocalSourceFile) and not schemafile.isabs() and workflow:
         # if workflow object is not available this has not been started from a workflow
-        schemafile = workflow.current_basedir.join(schemafile)
+        schemafile = workflow.current_basedir.join(schemafile.get_path_or_uri())
 
     source = (
         workflow.sourcecache.open(schemafile)

snakemake/workflow.py

@@ -257,6 +257,7 @@ def __init__(
         _globals["gather"] = Gather()
         _globals["github"] = sourcecache.GithubFile
         _globals["gitlab"] = sourcecache.GitlabFile
+        _globals["gitfile"] = sourcecache.LocalGitFile
 
         self.vanilla_globals = dict(_globals)
         self.modifier_stack = [WorkflowModifier(self, globals=_globals)]

tests/common.py

@@ -16,6 +16,7 @@
 import pytest
 import glob
 import subprocess
+import tarfile
 
 from snakemake import snakemake
 from snakemake.shell import shell
@@ -92,6 +93,12 @@ def get_expected_files(results_dir):
     ]
 
 
+def untar_folder(tar_file, output_path):
+    if not os.path.isdir(output_path):
+        with tarfile.open(tar_file) as tar:
+            tar.extractall(path=output_path)
+
+
 def run(
     path,
     shouldfail=False,

tests/test_module_local_git/Snakefile

@@ -0,0 +1,21 @@
+from snakemake.utils import min_version
+
+min_version("6.1.1")
+
+configfile: "config.yaml"
+
+import os
+
+module_path = os.path.join(os.getcwd(), "repo/module")
+
+module local_git_module:
+    snakefile:
+        gitfile(module_path, path="workflow/Snakefile", tag="v0.0.1")
+    config:
+        config
+
+use rule * from local_git_module as local_git_module_*
+
+rule all:
+    input:
+        "test.txt"

tests/test_module_local_git/Snakefile_main_missing_rule_and_schema

@@ -0,0 +1,20 @@
+from snakemake.utils import min_version
+import os
+
+min_version("6.1.1")
+
+configfile: "config.yaml"
+
+module_path = os.path.join(os.getcwd(), "repo/module")
+
+module local_git_module:
+    snakefile:
+        gitfile(module_path, path="workflow/Snakefile", tag="main")
+    config:
+        config
+
+use rule * from local_git_module as local_git_module_*
+
+rule all:
+    input:
+        "test.txt"

tests/test_module_local_git/Snakefile_missing_rule

@@ -0,0 +1,20 @@
+from snakemake.utils import min_version
+import os
+
+min_version("6.1.1")
+
+configfile: "config.yaml"
+
+module_path = os.path.join(os.getcwd(), "repo/module")
+
+module local_git_module:
+    snakefile:
+        gitfile(module_path, path="workflow/Snakefile", tag="e0.0.1")
+    config:
+        config
+
+use rule * from local_git_module as local_git_module_*
+
+rule all:
+    input:
+        "test.txt"

tests/test_module_local_git/Snakefile_missing_schema

@@ -0,0 +1,20 @@
+from snakemake.utils import min_version
+import os
+
+min_version("6.1.1")
+
+configfile: "config.yaml"
+
+module_path = os.path.join(os.getcwd(), "repo/module")
+
+module local_git_module:
+    snakefile:
+        gitfile(module_path, path="workflow/Snakefile", tag="e0.0.2")
+    config:
+        config
+
+use rule * from local_git_module as local_git_module_*
+
+rule all:
+    input:
+        "test.txt"

tests/test_module_local_git/Snakefile_relative

@@ -0,0 +1,17 @@
+from snakemake.utils import min_version
+
+min_version("6.1.1")
+
+configfile: "config.yaml"
+
+module local_git_module:
+    snakefile:
+        gitfile("repo/module", path="workflow/Snakefile", tag="v0.0.1")
+    config:
+        config
+
+use rule * from local_git_module as local_git_module_*
+
+rule all:
+    input:
+        "test.txt"

tests/test_module_local_git/config.yaml

@@ -0,0 +1,3 @@
+samples: samples.tsv
+
+units: units.tsv

tests/test_module_local_git/samples.tsv

@@ -0,0 +1,2 @@
+sample_name	alias	group	platform	purity
+a	a	a	ILLUMINA	

tests/test_module_local_git/units.tsv

@@ -0,0 +1,2 @@
+sample_name	unit_name	fq1	fq2	sra	adapters
+a	a	data/a.1.fq	data/a.2.fq		

tests/tests.py

@@ -1667,6 +1667,66 @@ def test_module_complex2():
     run(dpath("test_module_complex2"), dryrun=True)
 
 
+@skip_on_windows
+def test_module_use_local_git_repo():
+    untar_folder(
+        dpath("test_module_local_git/module.tar.gz"),
+        dpath("test_module_local_git/repo/module"),
+    )
+    run(dpath("test_module_local_git"), dryrun=True)
+
+
+@skip_on_windows
+def test_module_use_local_relative_git_repo():
+    untar_folder(
+        dpath("test_module_local_git/module.tar.gz"),
+        dpath("test_module_local_git/repo/module"),
+    )
+    run(dpath("test_module_local_git"), snakefile="Snakefile_relative", dryrun=True)
+
+
+@skip_on_windows
+def test_module_use_local_git_repo_missing_rule():
+    untar_folder(
+        dpath("test_module_local_git/module.tar.gz"),
+        dpath("test_module_local_git/repo/module"),
+    )
+    run(
+        dpath("test_module_local_git"),
+        snakefile="Snakefile_missing_rule",
+        dryrun=True,
+        shouldfail=True,
+    )
+
+
+@skip_on_windows
+def test_module_use_local_git_repo_missing_schema():
+    untar_folder(
+        dpath("test_module_local_git/module.tar.gz"),
+        dpath("test_module_local_git/repo/module"),
+    )
+    run(
+        dpath("test_module_local_git"),
+        snakefile="Snakefile_missing_schema",
+        dryrun=True,
+        shouldfail=True,
+    )
+
+
+@skip_on_windows
+def test_module_use_local_git_repo_missing_rule_and_schema():
+    untar_folder(
+        dpath("test_module_local_git/module.tar.gz"),
+        dpath("test_module_local_git/repo/module"),
+    )
+    run(
+        dpath("test_module_local_git"),
+        snakefile="Snakefile_main_missing_rule_and_schema",
+        dryrun=True,
+        shouldfail=True,
+    )
+
+
 @skip_on_windows
 def test_module_no_prefixing_modified_paths():
     run(