CWL Support 1

README.rst

@@ -218,6 +218,19 @@ For more information see the documentation for the `brew
 and `brew_env
 <http://planemo.readthedocs.org/en/latest/commands.html#brew_env-command>`__ commands.
 
+--------------------------
+Common Workflow Language
+--------------------------
+
+Planemo includes highly experimental support for running a subset of valid
+`Common Workflow Language`_ (CWL) tools using a fork of Galaxy enhanced to run
+CWL tools.
+
+::
+
+    % planemo project_init --template cwl_draft2_spec
+    % planemo serve --cwl cwl_draft2_spec/cat1-tool.cwl
+
 .. _Galaxy: http://galaxyproject.org/
 .. _GitHub: https://github.com/
 .. _Docker: https://www.docker.com/
@@ -228,3 +241,4 @@ and `brew_env
 .. _`tools-devteam`: https://github.com/galaxyproject/tools-devteam
 .. _`tools-iuc`: https://github.com/galaxyproject/tools-iuc
 .. _Publishing to the Tool Shed: http://planemo.readthedocs.org/en/latest/publishing.html
+.. _Common Workfow Language: http://common-workflow-language.github.io

planemo/commands/cmd_serve.py

@@ -8,6 +8,8 @@
 @click.command('serve')
 @options.optional_tools_arg(multiple=True)
 @options.galaxy_serve_options()
+@options.enable_cwl_option()
+@options.galaxy_cwl_root_option()
 @pass_context
 def cli(ctx, paths, **kwds):
     """Launch a Galaxy instance with the specified tool in the tool panel.

planemo/galaxy_config.py

@@ -92,6 +92,8 @@
 DOWNLOADABLE_MIGRATION_VERSIONS = [127, 120, 117]
 LATEST_URL = DOWNLOADS_URL + "latest.sqlite"
 
+PIP_INSTALL_CMD = "[ -d .venv ] && . .venv/bin/activate && pip install %s"
+
 FAILED_TO_FIND_GALAXY_EXCEPTION = (
     "Failed to find Galaxy root directory - please explicitly specify one "
     "with --galaxy_root."
@@ -185,6 +187,7 @@ def config_join(*args):
             database_auto_migrate="True",
             cleanup_job="never",
             master_api_key="${master_api_key}",
+            enable_beta_tool_formats="True",
             id_secret="${id_secret}",
             log_level="${log_level}",
             debug="${debug}",
@@ -381,11 +384,15 @@ def _check_galaxy(ctx, **kwds):
 
 
 def _find_galaxy_root(ctx, **kwds):
-    galaxy_root = kwds.get("galaxy_root", None)
+    root_prop = "galaxy_root"
+    cwl = kwds.get("cwl", False)
+    if cwl:
+        root_prop = "cwl_galaxy_root"
+    galaxy_root = kwds.get(root_prop, None)
     if galaxy_root:
         return galaxy_root
-    elif ctx.global_config.get("galaxy_root", None):
-        return ctx.global_config["galaxy_root"]
+    elif ctx.global_config.get(root_prop, None):
+        return ctx.global_config[root_prop]
     else:
         par_dir = os.getcwd()
         while True:
@@ -489,16 +496,17 @@ def _install_galaxy(ctx, config_directory, kwds):
 
 
 def _install_galaxy_via_download(config_directory, kwds):
-    command = galaxy_run.DOWNLOAD_GALAXY + "; tar -zxvf dev | tail"
-    _install_with_command(config_directory, command)
+    branch = _galaxy_branch(kwds)
+    tar_cmd = "tar -zxvf %s" % branch
+    command = galaxy_run.DOWNLOAD_GALAXY + "; %s | tail" % tar_cmd
+    _install_with_command(config_directory, command, kwds)
 
 
 def _install_galaxy_via_git(ctx, config_directory, kwds):
-    _ensure_galaxy_repository_available(ctx)
-    workspace = ctx.workspace
-    gx_repo = os.path.join(workspace, "gx_repo")
-    command = git.command_clone(ctx, gx_repo, "galaxy-dev")
-    _install_with_command(config_directory, command)
+    gx_repo = _ensure_galaxy_repository_available(ctx, kwds)
+    branch = _galaxy_branch(kwds)
+    command = git.command_clone(ctx, gx_repo, "galaxy-dev", branch=branch)
+    _install_with_command(config_directory, command, kwds)
 
 
 def _build_eggs_cache(ctx, env, kwds):
@@ -511,27 +519,50 @@ def _build_eggs_cache(ctx, env, kwds):
     env["GALAXY_EGGS_PATH"] = eggs_path
 
 
-def _install_with_command(config_directory, command):
+def _galaxy_branch(kwds):
+    # TODO: implement generic --branch argument
+    cwl = kwds.get("cwl", False)
+    branch = "cwl" if cwl else "dev"
+    return branch
+
+
+def _install_with_command(config_directory, command, kwds):
+    # TODO: --watchdog
+    pip_installs = []
+    if kwds.get("cwl", False):
+        pip_installs.append("cwltool")
+    if pip_installs:
+        pip_install_command = PIP_INSTALL_CMD % " ".join(pip_installs)
+    else:
+        pip_install_command = ""
     install_cmds = [
         "cd %s" % config_directory,
         command,
         "cd galaxy-dev",
         "type virtualenv >/dev/null 2>&1 && virtualenv .venv",
+        pip_install_command,
         galaxy_run.ACTIVATE_COMMAND,
     ]
-    shell(";".join(install_cmds))
+    shell(";".join([c for c in install_cmds if c]))
 
 
-def _ensure_galaxy_repository_available(ctx):
+def _ensure_galaxy_repository_available(ctx, kwds):
     workspace = ctx.workspace
+    cwl = kwds.get("cwl", False)
     gx_repo = os.path.join(workspace, "gx_repo")
+    if cwl:
+        gx_repo += "_cwl"
     if os.path.exists(gx_repo):
         # Attempt fetch - but don't fail if not interweb, etc...
         shell("git --git-dir %s fetch >/dev/null 2>&1" % gx_repo)
     else:
-        remote_repo = "https://github.com/galaxyproject/galaxy"
+        if cwl:
+            remote_repo = "https://github.com/common-workflow-language/galaxy"
+        else:
+            remote_repo = "https://github.com/galaxyproject/galaxy"
         command = git.command_clone(ctx, remote_repo, gx_repo, bare=True)
         shell(command)
+    return gx_repo
 
 
 def _build_env_for_galaxy(properties, template_args):

planemo/git.py

@@ -4,13 +4,17 @@
 from planemo import io
 
 
-def command_clone(ctx, src, dest, bare=False):
+def command_clone(ctx, src, dest, bare=False, branch=None):
     """ Take in ctx to allow more configurability down the road.
     """
     bare_arg = ""
     if bare:
         bare_arg = "--bare"
-    return "git clone %s '%s' '%s'" % (bare_arg, src, dest)
+    branch_arg = ""
+    if branch is not None:
+        branch_arg = "--branch '%s'" % branch
+    cmd = "git clone %s %s '%s' '%s'" % (bare_arg, branch_arg, src, dest)
+    return cmd
 
 
 def clone(*args, **kwds):

planemo/options.py

@@ -42,6 +42,17 @@ def galaxy_root_option():
     )
 
 
+def galaxy_cwl_root_option():
+    return click.option(
+        "--cwl_galaxy_root",
+        type=click.Path(exists=True, file_okay=False, resolve_path=True),
+        help=("Root of development galaxy directory to execute command with"
+              " (must be branch of Galaxy with CWL support, this option"
+              " is experimental and will be replaced with --galaxy_root when"
+              " and if CWL support is merged into Galaxy."),
+    )
+
+
 def galaxy_port_option():
     return click.option(
         "--port",
@@ -78,6 +89,16 @@ def dependency_resolvers_option():
     )
 
 
+def enable_cwl_option():
+    return click.option(
+        "--cwl",
+        is_flag=True,
+        help=("Configure Galaxy for use with CWL tool."
+              " (this option is experimental and will be replaced when"
+              " and if CWL support is merged into Galaxy."),
+    )
+
+
 def brew_dependency_resolution():
     return click.option(
         "--brew_dependency_resolution",

project_templates/cwl_draft2_spec/add-lines-wf.cwl

@@ -0,0 +1,34 @@
+#!/usr/bin/env cwl-runner
+class: Workflow
+inputs:
+    - id: "#file1"
+      type: File
+    - id: "#file2"
+      type: File
+outputs:
+    - id: "#count_output"
+      type: int
+      connect: {"source": "#step2_output"}
+steps:
+  - id: "#step1"
+    run: count-lines4-wf.cwl
+    inputs:
+      - { param: "count-lines4-wf.cwl#file1", connect: {"source": "#file1"} }
+      - { param: "count-lines4-wf.cwl#file2", connect: {"source": "#file2"} }
+    outputs:
+      - { id: "#step1_output", param: "count-lines4-wf.cwl#count_output" }
+
+  - id: "#step2"
+    class: ExpressionTool
+    inputs:
+      - id: "#a"
+        type:
+          type: array
+          items: int
+        connect: {source: "#step1_output"}
+    outputs:
+      - id: "#step2_output"
+        type: int
+    script:
+      class: JavascriptExpression
+      script: "{return {'step2_output': ($job.a[0] + $job.a[1])}; }"

project_templates/cwl_draft2_spec/binding-test.cwl

@@ -0,0 +1,19 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+
+inputs:
+  - id: "#reference"
+    type: File
+    inputBinding: { position: 2 }
+
+  - id: "#reads"
+    type:
+      type: array
+      items: File
+      inputBinding: { prefix: "-YYY" }
+    inputBinding: { position: 3, prefix: "-XXX" }
+
+outputs: []
+
+baseCommand: ["bwa", "mem"]

project_templates/cwl_draft2_spec/bwa-mem-job.json

@@ -0,0 +1,29 @@
+{
+    "reference": {
+        "class": "File",
+        "path": "rabix/tests/test-files/chr20.fa",
+        "size": 123,
+        "checksum": "sha1$hash"
+    },
+    "reads": [
+        {
+            "class": "File",
+            "path": "rabix/tests/test-files/example_human_Illumina.pe_1.fastq"
+        },
+        {
+            "class": "File",
+            "path": "rabix/tests/test-files/example_human_Illumina.pe_2.fastq"
+        }
+    ],
+    "min_std_max_min": [
+        1,
+        2,
+        3,
+        4
+    ],
+    "minimum_seed_length": 3,
+    "allocatedResources": {
+        "cpu": 4,
+        "mem": 5000
+    }
+}

project_templates/cwl_draft2_spec/bwa-mem-tool.cwl

@@ -0,0 +1,48 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+requirements:
+  - import: node-engine.cwl
+
+hints:
+  - class: DockerRequirement
+    dockerPull: images.sbgenomics.com/rabix/bwa
+    dockerImageId: 9d3b9b0359cf
+
+inputs:
+  - id: "#reference"
+    type: File
+    inputBinding: { position: 2 }
+
+  - id: "#reads"
+    type:
+      type: array
+      items: File
+    inputBinding: { position: 3 }
+
+  - id: "#minimum_seed_length"
+    type: int
+    inputBinding: { position: 1, prefix: "-m" }
+
+  - id: "#min_std_max_min"
+    type: { type: array, items: int }
+    inputBinding:
+      position: 1
+      prefix: "-I"
+      itemSeparator: ","
+
+outputs:
+  - id: "#sam"
+    type: "File"
+    outputBinding: { "glob": "output.sam" }
+
+baseCommand: ["bwa", "mem"]
+
+arguments:
+  - valueFrom:
+      engine: "node-engine.cwl"
+      script: "$job.allocatedResources.cpu"
+    position: 1
+    prefix: "-t"
+
+stdout: "output.sam"

project_templates/cwl_draft2_spec/cat-job.json

@@ -0,0 +1,6 @@
+{
+    "file1": {
+        "class": "File",
+        "path": "hello.txt"
+    }
+}

project_templates/cwl_draft2_spec/cat-n-job.json

@@ -0,0 +1,7 @@
+{
+    "file1": {
+        "class": "File",
+        "path": "hello.txt"
+    },
+    "numbering": true
+}

project_templates/cwl_draft2_spec/cat1-tool.cwl

@@ -0,0 +1,28 @@
+#!/usr/bin/env cwl-runner
+{
+    "class": "CommandLineTool",
+    "description": "Print the contents of a file to stdout using 'cat' running in a docker container.",
+    "hints": [
+        {
+          "class": "DockerRequirement",
+          "dockerPull": "debian:wheezy"
+        }
+    ],
+    "inputs": [
+        {
+            "id": "#file1",
+            "type": "File",
+            "inputBinding": {"position": 1}
+        },
+        {
+            "id": "#numbering",
+            "type": ["null", "boolean"],
+            "inputBinding": {
+                "position": 0,
+                "prefix": "-n"
+            }
+        }
+    ],
+    "outputs": [],
+    "baseCommand": "cat"
+}