Merge pull request #883 from maxplanck-ie/condaDir

Conda dir

.github/workflows/linux.yml

@@ -95,5 +95,5 @@ jobs:
       run: |
         micromamba activate snakePipes_CI
         conda config --set channel_priority flexible
-        snakePipes createEnvs --force --only ${{matrix.envs}}
+        snakePipes createEnvs --autodetectCondaEnvDir --force --only ${{matrix.envs}}
 

.github/workflows/osx.yml

@@ -46,5 +46,5 @@ jobs:
         run: |
           micromamba activate snakePipes_CI
           conda config --set channel_priority flexible
-          snakePipes createEnvs --force --only ${{matrix.envs}}
+          snakePipes createEnvs --force --autodetectCondaEnvDir --only ${{matrix.envs}}
      

bin/snakePipes

@@ -36,21 +36,16 @@ def parse_arguments():
         help="Prints the location in which each conda environment is actually stored.",
     )
 
-    mex = createEnvsParser.add_mutually_exclusive_group()
-    mex.add_argument(
-        "--keepCondaDir",
+
+    createEnvsParser.add_argument(
+        "--autodetectCondaEnvDir",
         action="store_true",
-        help="If specified, the `snakemakeOptions:` setting in "
-        "the workflow `defaults.yaml` files will NOT be overwritten. "
-        "This is typically unwise and only expert users should specify this.",
-    )
-    mex.add_argument(
-        "--condaDir",
-        help="If specified, use this as the base directory for the "
-        "created conda environments. This will ignore what is already "
-        "in the workflow-specific yaml files and where conda is installed.",
+        help="If specified, this will set condaEnvDir to system conda prefix,"
+       "and will overwrite the condaEnvDir entry in defaults.yaml ."
+       "Use with caution."
     )
 
+
     createEnvsParser.add_argument(
         "--only",
         nargs="+",
@@ -117,6 +112,13 @@ def parse_arguments():
         default=defaults["snakemakeOptions"],
     )
 
+    configParser.add_argument(
+        "--condaEnvDir",
+        help="If specified, use this as the base directory for the "
+        "created conda environments. This will ignore what is already "
+        "in the workflow-specific yaml files and where conda is installed.",
+    )
+
     configParser.add_argument(
         "--organismsDir",
         help="The directory where global organism YAML files are to be stored. Both "
@@ -254,19 +256,13 @@ def envInfo():
     For each environment yaml file print where its conda env is actually located
     """
     baseDir = os.path.dirname(snakePipes.__file__)
-    condaDir = os.environ.get("CONDA_PREFIX")
-    if "envs" in condaDir:
-        condaDir = os.path.dirname(condaDir)
-    else:
-        condaDir = os.path.join(condaDir, "envs")
 
     f = open(os.path.join(baseDir, "shared/defaults.yaml"))
     cf = yaml.load(f, Loader=yaml.FullLoader)
     f.close()
 
-    _ = cf["snakemakeOptions"].split(" ")
-    idx = _.index("--conda-prefix")
-    condaEnvDir = _[idx + 1]
+    condaEnvDir=cf["condaEnvDir"]
+
 
     for env in cof.set_env_yamls().values():
         # Hash the file ala snakemake
@@ -307,36 +303,16 @@ def createCondaEnvs(args):
     """
     baseDir = os.path.dirname(snakePipes.__file__)
 
-    condaDir = os.environ.get("CONDA_PREFIX")
-    rootDir = condaDir
-    if "envs" in condaDir:
-        condaDir = os.path.dirname(condaDir)
-    else:
-        condaDir = os.path.join(condaDir, "envs")
-
     f = open(os.path.join(baseDir, "shared/defaults.yaml"))
     cf = yaml.load(f, Loader=yaml.FullLoader)
     f.close()
-    _ = cf["snakemakeOptions"].split(" ")
-    try:
-        idx = _.index("--conda-prefix")
-    except:
-        idx = len(_)
-        _.extend(["--conda-prefix", condaDir])
-
-    condaEnvDir = _[idx + 1]
-    if args.condaDir:
-        condaDirUse = args.condaDir
-        _[idx + 1] = condaDirUse
-    elif args.keepCondaDir:
-        condaDirUse = _[idx + 1]
-    else:
-        condaDirUse = condaDir
-        _[idx + 1] = condaDirUse
-    cf["snakemakeOptions"] = " ".join(_)
+    condaEnvDir=cf["condaEnvDir"]
+    condaDirUse=condaEnvDir
 
+    if args.autodetectCondaEnvDir:
+        condaDirUse=detectCondaDir()
     # rewrite defaults.yaml
-    cof.write_configfile(os.path.join(baseDir, "shared/defaults.yaml"), cf)
+        cof.write_configfile(os.path.join(baseDir, "shared/defaults.yaml"), cf)
 
     for envName, env in cof.set_env_yamls().items():
         if args.only is not None and envName not in args.only:
@@ -362,12 +338,7 @@ def createCondaEnvs(args):
             "--file",
             os.path.join(baseDir, "shared/rules", env),
         ]
-        if "--conda-prefix" in cf["snakemakeOptions"] and (
-            args.condaDir or args.keepCondaDir
-        ):
-            cmd += ["--prefix", os.path.join(condaDirUse, h)]
-        else:
-            cmd += ["--name", h]
+        cmd += ["--prefix", os.path.join(condaDirUse, h)]
 
         # Don't actually create the env if either --info is set or it already exists and --force is NOT set
         if not args.info:
@@ -388,6 +359,15 @@ def createCondaEnvs(args):
     if args.noSitePackages and not args.info:
         fixSitePy(rootDir)
 
+def detectCondaDir():
+    "Detect the default conda folder."
+    condaDir = os.environ.get("CONDA_PREFIX")
+    if "envs" in condaDir:
+        condaDir = os.path.dirname(condaDir)
+    else:
+        condaDir = os.path.join(condaDir, "envs")
+    return(condaDir)
+
 
 def updateConfig(args):
     """Update the global defaults"""
@@ -400,6 +380,7 @@ def updateConfig(args):
     if args.configMode == "manual":
         d = {
             "snakemakeOptions": args.snakemakeOptions,
+            "condaEnvDir": args.condaEnvDir,
             "organismsDir": args.organismsDir,
             "clusterConfig": args.clusterConfig,
             "tempDir": args.tempDir,

docs/content/News.rst

@@ -1,6 +1,16 @@
 snakePipes News
 ===============
 
+snakePipes x.x.x
+----------------
+* Changed the behaviour of snakePipes createEnvs - it is no longer possible to set condaEnvDir with this function. It is required to set it with snakePipes config beforhand, instead. To ingore what's in the defaults.yaml and overwrite the condaEnvDir value with default system conda prefix, use '--autodetectCondaEnvDir'.
+* Snakemake options in the defaults.yaml are now an empty string. The required arguments '--use-conda --conda-prefix' have been directly added to the command string. condaEnvDir is parsed from defaults.yaml, requiring running snakePipes config first.
+
+* Fixes #819
+
+
+
+
 snakePipes 2.7.2
 ----------------
 * STAR version has been updated to 2.7.10b. 2.7.10a was returning segmentation fault on MAC.

snakePipes/common_functions.py

@@ -632,15 +632,16 @@ def commonYAMLandLogs(baseDir, workflowDir, defaults, args, callingScript):
         args.snakemakeOptions += " --notemp"
 
     snakemake_cmd = """
-                    TMPDIR={tempDir} PYTHONNOUSERSITE=True {snakemake} {snakemakeOptions} --latency-wait {latency_wait} --snakefile {snakefile} --jobs {maxJobs} --directory {workingdir} --configfile {configFile} --keep-going
+                    TMPDIR={tempDir} PYTHONNOUSERSITE=True {snakemake} {snakemakeOptions} --latency-wait {latency_wait} --snakefile {snakefile} --jobs {maxJobs} --directory {workingdir} --configfile {configFile} --keep-going --use-conda --conda-prefix {condaEnvDir}
                     """.format(snakemake=os.path.join(snakemake_path, "snakemake"),
                                latency_wait=cluster_config["snakemake_latency_wait"],
                                snakefile=os.path.join(workflowDir, "Snakefile"),
                                maxJobs=args.maxJobs,
                                workingdir=args.workingdir,
                                snakemakeOptions=str(args.snakemakeOptions or ''),
                                tempDir=cfg["tempDir"],
-                               configFile=os.path.join(args.outdir, '{}.config.yaml'.format(workflowName))).split()
+                               configFile=os.path.join(args.outdir, '{}.config.yaml'.format(workflowName)),
+                               condaEnvDir=cfg["condaEnvDir"]).split()
 
     if args.verbose:
         snakemake_cmd.append("--printshellcmds")

snakePipes/shared/defaults.yaml

@@ -7,7 +7,8 @@
 # permitted here.
 ################################################################################
 #
-snakemakeOptions: ' --use-conda --conda-prefix /package/mamba/envs/ '
+condaEnvDir: '/package/mamba/envs/'
+snakemakeOptions: ''
 organismsDir: 'shared/organisms'
 clusterConfig: 'shared/cluster.yaml'
 tempDir: /data/extended/