Merge pull request #53 from pepkit/dev

v0.5 release

MANIFEST.in

@@ -1,5 +1,5 @@
 include divvy/default_config/*
-include divvy/default_config/submit_templates/*
+include divvy/default_config/divvy_templates/*
 include requirements/*
 include README.md
 include docs/img/divvy_logo.svg

divvy/__init__.py

@@ -9,11 +9,12 @@
 
 import logmuse
 from ._version import __version__
-from .compute import ComputingConfiguration
+from .compute import ComputingConfiguration, select_divvy_config
 from .const import *
 from .utils import write_submit_script
 
 __classes__ = ["ComputingConfiguration"]
-__all__ = __classes__ + [write_submit_script.__name__]
+__functions__ = ["select_divvy_config"]
+__all__ = __classes__ + __functions__ + [write_submit_script.__name__]
 
 logmuse.init_logger("divvy")

divvy/_version.py

@@ -1 +1 @@
-__version__ = "0.4.1"
+__version__ = "0.5.0"

divvy/const.py

@@ -1,16 +1,17 @@
 """ Package constants """
-
-__author__ = "Vince Reuter"
-__email__ = "vreuter@virginia.edu"
-
+import os
 
 # Compute-related
-COMPUTE_SETTINGS_VARNAME = ["DIVCFG", "PEPENV"]
+COMPUTE_SETTINGS_VARNAME = ["DIVCFG"]
 DEFAULT_COMPUTE_RESOURCES_NAME = "default"
 OLD_COMPUTE_KEY = "compute"
 NEW_COMPUTE_KEY = "compute_packages"
+DEFAULT_CONFIG_FILEPATH = os.path.join(
+        os.path.dirname(__file__),
+        "default_config",
+        "divvy_config.yaml")
 COMPUTE_CONSTANTS = [
     "COMPUTE_SETTINGS_VARNAME", "DEFAULT_COMPUTE_RESOURCES_NAME",
-    "NEW_COMPUTE_KEY"]
+    "NEW_COMPUTE_KEY", "DEFAULT_CONFIG_FILEPATH"]
 
 __all__ = COMPUTE_CONSTANTS + ["DEFAULT_COMPUTE_RESOURCES_NAME"]

divvy/default_config/divvy_config.yaml

@@ -1,26 +1,37 @@
 # Use this to change your cluster manager (SLURM, SGE, LFS, etc).
 # Relative paths are relative to this compute environment configuration file.
 # Compute resource parameters fill the submission_template file's fields.
+adapters:
+  CODE: looper.command
+  JOBNAME: looper.job_name
+  CORES: compute.cores
+  LOGFILE: looper.log_file
+  TIME: compute.time
+  MEM: compute.mem
+  DOCKER_ARGS: compute.docker_args
+  DOCKER_IMAGE: compute.docker_image
+  SINGULARITY_IMAGE: compute.singularity_image
+  SINGULARITY_ARGS: compute.singularity_args
 compute_packages:
   default:
-    submission_template: submit_templates/localhost_template.sub
+    submission_template: divvy_templates/localhost_template.sub
     submission_command: sh
   local:
-    submission_template: submit_templates/localhost_template.sub
+    submission_template: divvy_templates/localhost_template.sub
     submission_command: sh
   slurm:
-    submission_template: submit_templates/slurm_template.sub
+    submission_template: divvy_templates/slurm_template.sub
     submission_command: sbatch
   singularity:
-    submission_template: submit_templates/localhost_singularity_template.sub
+    submission_template: divvy_templates/localhost_singularity_template.sub
     submission_command: sh
     singularity_args: ""
   singularity_slurm:
-    submission_template: submit_templates/slurm_singularity_template.sub
+    submission_template: divvy_templates/slurm_singularity_template.sub
     submission_command: sbatch
     singularity_args: ""
   docker:
-    submission_template: submit_templates/localhost_docker_template.sub
+    submission_template: divvy_templates/localhost_docker_template.sub
     submission_command: sh
     docker_args: |
       --user=$(id -u):$(id -g) \

divvy/utils.py

@@ -64,7 +64,6 @@ def parse_config_file(conf_file):
     return env_settings
 
 
-
 def write_submit_script(fp, content, data):
     """
     Write a submission script by populating a template with data.
@@ -86,12 +85,16 @@ def write_submit_script(fp, content, data):
         _LOGGER.warning("> Warning: %d submission template variables are not "
                         "populated: '%s'", len(keys_left), str(keys_left))
 
-    outdir = os.path.dirname(fp)
-    if outdir and not os.path.isdir(outdir):
-        os.makedirs(outdir)
-    with open(fp, 'w') as f:
-        f.write(content)
-    return fp
+    if not fp:
+        print(content)
+        return content
+    else:
+        outdir = os.path.dirname(fp)
+        if outdir and not os.path.isdir(outdir):
+            os.makedirs(outdir)
+        with open(fp, 'w') as f:
+            f.write(content)
+        return fp
 
 
 def get_first_env_var(ev):

docs/README.md

@@ -4,23 +4,25 @@
 
 ## What is `divvy`?
 
-`Divvy` is a computing resource configuration manager. It organizes your computing resources and populates job submission templates. It makes it easy for users to toggle among any computing resource (laptop, cluster, cloud). `Divvy` provides both an interactive python API and a command-line interface.
+<img src="img/divvy-merge.svg" style="float:right; padding-left: 25px; padding-right: 5px; width:300px">
 
 
+`Divvy` allows you to populate job submission scripts by integrating job-specific settings with separately configured computing environment settings. Divvy *makes software portable*, so users may easily toggle among any computing resource (laptop, cluster, cloud). 
+
 ## What makes `divvy` better?
 
 <img src="img/nodivvy.svg" style="float:left; padding-left: 5px; padding-right: 25px">
-Divvy *makes compute-heavy software portable*, so it works on any computing environment, from laptop to cloud.
 
-Many bioinformatics tools require a particular compute resource setup. For example, one pipeline requires SLURM, another requires AWS, and yet another just runs directly on your laptop. This makes it difficult to transfer to different environments. For tools that can run in multiple environments, each one must be configured separately.
+
+tools require a particular compute resource setup. For example, one pipeline requires SLURM, another requires AWS, and yet another just runs directly on your laptop. This makes it difficult to transfer to different environments. For tools that can run in multiple environments, each one must be configured separately.
 
 <hr>
 
 <img src="img/divvy-connect.svg" style="float:left; padding-left: 5px; padding-right: 25px">
 
 Instead, `divvy`-compatible tools can run on any computing resource. **Users configure their computing environment once, and all divvy-compatible tools will use this same configuration.**
 
-Divvy reads a standard configuration file describing available compute resources and then uses a simple Jinja-like template system to write custom job submission scripts. Computing resources are organized as *compute packages*, which users select, populate with values, and build scripts for compute jobs. 
+Divvy reads a standard configuration file describing available compute resources and then uses a simple template system to write custom job submission scripts. Computing resources are organized as *compute packages*, which users select, populate with values, and build scripts for compute jobs. 
 
 <br clear="all"/>
 
@@ -49,14 +51,13 @@ local
 slurm
 ```
 
-<img src="img/divvy-merge.svg" style="float:right; padding-left: 25px; padding-right: 5px">
 
 Divvy will take variables from a file or the command line, merge these with environment settings to create a specific job script. Write a submission script from the command line:
 
 ```{console}
 divvy write --package slurm \
 	--settings myjob.yaml \
-	--sample sample1 \
+	--compute sample=sample1 \
 	--outfile submit_script.txt
 ```
 

docs/adapters.md

@@ -0,0 +1,18 @@
+# Adapters make template variables flexible
+
+Starting with `divvy v0.5.0` the configuration file can include an `adapters` section, which is used to provide a set of variable mappings that `divvy` uses to populate the submission templates.
+
+This makes the connection with `divvy` and client software more flexible and more elegant, since the source of the data does not need to follow any particular naming scheme, any mapping can be used and adapted to work with any `divvy` templates.
+
+## Example
+
+```yaml
+adapters:
+  CODE: namespace.command
+  LOGFILE: namespace1.log_file
+  JOBNAME: user_settings.program.job_name
+  CORES: processors_number
+...
+```
+
+As you can see in the example `adapters` section above, each adapter is a key-value pair that maps a `divvy` template variable to a target value. The target values can use namespaces (nested mapping).

docs/changelog.md

@@ -1,5 +1,25 @@
+This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. 
+
 # Changelog
 
+## [0.5.0] -- 2020-05-19
+### Added
+- adapters support, see [#47](https://github.com/pepkit/divvy/issues/47) for detailed explanation
+- `select_divvy_config` function
+- `divvy submit` command
+- added new `--compute` CLI argument
+- `divvy write` can now be run without an outfile, which just prints the template to stdout.
+
+### Changed
+- Instead of passing extra variables as CLI args, you now must explicitly pass them to the `--compute` arg.
+- The config file is now passed with a positional argument, instead of with `--config`.
+- Made all one-char CLI args lowercase (`-P` to `-p`, `-S` to `-s`).
+- Renamed default templates subfolder from submit_templates to divvy_templates
+- removed `$PEPENV` from environment variables that may point to divvy computing configuration file. `$DIVCFG` is the only one now.
+
+### Removed
+- `config_file`, `no_env_error` and `no_compute_exception` from `ComputingConfiguration` class constructor
+
 ## [0.4.1] -- 2020-03-20
 ### Fixed
 - `NameError` in `divvy init`; [#44](https://github.com/pepkit/divvy/issues/44)

docs/configuration.md

@@ -4,11 +4,11 @@ At the heart of `divvy` is a the *divvy configuration file*, or `DIVCFG` for sho
 
 ## Configuration file priority lookup
 
-When `divvy` starts, it checks a few places for the `DIVCFG` file. First, the user may may specify a `DIVCFG` file when invoking `divvy` either from the command line (with `--config`) or from within python. If the file is not provided, `divvy` will next look file in the `$DIVCFG` environment variable. If it cannot find one there, then it will load a default configuration file with a few basic compute packages. We recommend setting the `DIVCFG` environment variable as the most convenient use case.
+When `divvy` starts, it checks a few places for the `DIVCFG` file. First, the user may may specify a `DIVCFG` file when invoking `divvy` either from the command line or from within python. If the file is not provided, `divvy` will next look file in the `$DIVCFG` environment variable. If it cannot find one there, then it will load a default configuration file with a few basic compute packages. We recommend setting the `DIVCFG` environment variable as the most convenient use case.
 
 ## Customizing your configuration file
 
-The easiest way to customize your computing configuration is to edit the default configuration file. To get a fresh copy of the default configuration, use `divvy init -c custom_divvy_config.yaml`. This will create for you a config file along with a folder containing all the default templates.
+The easiest way to customize your computing configuration is to edit the default configuration file. To get a fresh copy of the default configuration, use `divvy init custom_divvy_config.yaml`. This will create for you a config file along with a folder containing all the default templates.
 
 Here is an example `divvy` configuration file: