From c31d4ae436dc27828f273f79f349870d99439ffb Mon Sep 17 00:00:00 2001 From: Thibault Jeandet Date: Tue, 29 Mar 2016 11:07:21 -0400 Subject: [PATCH 01/30] version bump --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 8edff10..eaad136 100644 --- a/build.sbt +++ b/build.sbt @@ -10,7 +10,7 @@ organization := "org.broadinstitute" scalaVersion := "2.11.7" // Upcoming release, or current if we're on the master branch -git.baseVersion := "0.4" +git.baseVersion := "0.5" // Shorten the git commit hash git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) } From 5bca4b709093d2227fefe031b80de4ef4ea14fbf Mon Sep 17 00:00:00 2001 From: Geraldine Van der Auwera Date: Wed, 13 Jul 2016 18:02:11 -0400 Subject: [PATCH 02/30] Update WDL site URL in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8803dd5..55a2b6d 100644 --- a/README.md +++ b/README.md @@ -167,4 +167,4 @@ $ java -jar wdltool.jar highlight test.wdl html # Getting Started with WDL -For many examples on how to use WDL see [the WDL site](https://github.com/broadinstitute/wdl/tree/develop#getting-started-with-wdl) +For documentation and many examples on how to use WDL see [the WDL website](https://software.broadinstitute.org/wdl/). From 8fd1f4ae317e562b346d8fb9e25bf50624f224c6 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Thu, 22 Sep 2016 13:57:31 -0400 Subject: [PATCH 03/30] Updated wdl4s to latest version. Updated build to match similar wdl4s build.sbt updates. --- build.sbt | 5 ++--- project/build.properties | 1 + src/main/scala/wdltool/Main.scala | 8 +++---- src/test/scala/wdltool/MainSpec.scala | 40 +++++++++++++++++------------------ 4 files changed, 26 insertions(+), 28 deletions(-) create mode 100644 project/build.properties diff --git a/build.sbt b/build.sbt index eaad136..d88275d 100644 --- a/build.sbt +++ b/build.sbt @@ -1,13 +1,12 @@ import com.typesafe.sbt.GitPlugin.autoImport._ import sbt.Keys._ import sbtassembly.MergeStrategy -import com.typesafe.sbt.SbtGit.GitCommand name := "wdltool" organization := "org.broadinstitute" -scalaVersion := "2.11.7" +scalaVersion := "2.11.8" // Upcoming release, or current if we're on the master branch git.baseVersion := "0.5" @@ -30,7 +29,7 @@ resolvers ++= Seq( ) libraryDependencies ++= Seq( - "org.broadinstitute" %% "wdl4s" % "0.4", + "org.broadinstitute" %% "wdl4s" % "0.5", //---------- Test libraries -------------------// "org.scalatest" %% "scalatest" % "2.2.5" % Test ) diff --git a/project/build.properties b/project/build.properties new file mode 100644 index 0000000..35c88ba --- /dev/null +++ b/project/build.properties @@ -0,0 +1 @@ +sbt.version=0.13.12 diff --git a/src/main/scala/wdltool/Main.scala b/src/main/scala/wdltool/Main.scala index 59c9943..d25aaca 100644 --- a/src/main/scala/wdltool/Main.scala +++ b/src/main/scala/wdltool/Main.scala @@ -1,9 +1,9 @@ package wdltool -import java.io.{File => JFile} +import java.nio.file.Paths import wdl4s.formatter.{AnsiSyntaxHighlighter, HtmlSyntaxHighlighter, SyntaxFormatter} -import wdl4s.{WdlNamespace, NamespaceWithWorkflow, AstTools} +import wdl4s.{AstTools, NamespaceWithWorkflow, WdlNamespace} import spray.json._ import scala.util.{Failure, Success, Try} @@ -68,14 +68,14 @@ object Main extends App { def parse(args: Seq[String]): Termination = { continueIf(args.length == 1) { - SuccessfulTermination(AstTools.getAst(new JFile(args.head)).toPrettyString) + SuccessfulTermination(AstTools.getAst(Paths.get(args.head)).toPrettyString) } } private[this] def continueIf(valid: => Boolean)(block: => Termination): Termination = if (valid) block else BadUsageTermination private[this] def loadWdl(path: String)(f: WdlNamespace => Termination): Termination = { - Try(WdlNamespace.load(new JFile(path))) match { + Try(WdlNamespace.load(Paths.get(path))) match { case Success(namespace) => f(namespace) case Failure(t) => UnsuccessfulTermination(t.getMessage) } diff --git a/src/test/scala/wdltool/MainSpec.scala b/src/test/scala/wdltool/MainSpec.scala index 01c69c9..d568a0b 100644 --- a/src/test/scala/wdltool/MainSpec.scala +++ b/src/test/scala/wdltool/MainSpec.scala @@ -1,10 +1,9 @@ package wdltool -import java.nio.file.{Paths, Path} import better.files._ import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -import wdltool.SampleWdl.{EmptyWorkflow, EmptyTask, EmptyInvalid, ThreeStep} -import MainSpec._ +import wdltool.MainSpec._ +import wdltool.SampleWdl.{EmptyInvalid, EmptyTask, EmptyWorkflow, ThreeStep} class MainSpec extends FlatSpec with Matchers with BeforeAndAfterAll { @@ -137,48 +136,47 @@ object MainSpec { */ case class WdlAndInputs(sampleWdl: SampleWdl, optionsJson: String = "{}") { // Track all the temporary files we create, and delete them after the test. - private var tempFiles = Vector.empty[Path] + private var tempFiles = Vector.empty[File] - lazy val wdlPath: Path = { - val path = File.newTemp(s"${sampleWdl.name}.", ".wdl").path + lazy val wdlFile = { + val path = File.newTemporaryFile(s"${sampleWdl.name}.", ".wdl") tempFiles :+= path path write sampleWdl.wdlSource("") path } - lazy val wdl = wdlPath.fullPath + lazy val wdl = wdlFile.pathAsString - lazy val inputsPath = { - val path = swapExt(wdlPath, ".wdl", ".inputs") + lazy val inputsFile = { + val path = swapExt(wdlFile, ".wdl", ".inputs") tempFiles :+= path path write sampleWdl.wdlJson path } - lazy val inputs = inputsPath.fullPath + lazy val inputs = inputsFile.pathAsString - lazy val optionsPath = { - val path = swapExt(wdlPath, ".wdl", ".options") + lazy val optionsFile = { + val path = swapExt(wdlFile, ".wdl", ".options") tempFiles :+= path path write optionsJson path } - lazy val options = optionsPath.fullPath + lazy val options = optionsFile.pathAsString - lazy val metadataPath = { - val path = swapExt(wdlPath, ".wdl", ".metadata.json") + lazy val metadataFile = { + val path = swapExt(wdlFile, ".wdl", ".metadata.json") tempFiles :+= path - path.toAbsolutePath + path } - lazy val metadata = metadataPath.fullPath + lazy val metadata = metadataFile.pathAsString - def deleteTempFiles() = tempFiles.foreach(_.delete(ignoreIOExceptions = true)) + def deleteTempFiles() = tempFiles.foreach(_.delete(swallowIOExceptions = true)) } - def swapExt(filePath: Path, oldExt: String, newExt: String): Path = { - Paths.get(filePath.toString.stripSuffix(oldExt) + newExt) + def swapExt(filePath: File, oldExt: String, newExt: String): File = { + File(filePath.toString.stripSuffix(oldExt) + newExt) } } - From 40c85cb196955e69cf8bb1a9872586bf3c3bc571 Mon Sep 17 00:00:00 2001 From: Thibault Jeandet Date: Thu, 22 Sep 2016 15:29:39 -0400 Subject: [PATCH 04/30] bump version to 0.6 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index d88275d..ccae611 100644 --- a/build.sbt +++ b/build.sbt @@ -9,7 +9,7 @@ organization := "org.broadinstitute" scalaVersion := "2.11.8" // Upcoming release, or current if we're on the master branch -git.baseVersion := "0.5" +git.baseVersion := "0.6" // Shorten the git commit hash git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) } From ca3d7212c780744a3879fec4d90cea824dedecac Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Wed, 12 Oct 2016 11:44:22 -0400 Subject: [PATCH 05/30] Update WDL4s version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index ccae611..cced536 100644 --- a/build.sbt +++ b/build.sbt @@ -29,7 +29,7 @@ resolvers ++= Seq( ) libraryDependencies ++= Seq( - "org.broadinstitute" %% "wdl4s" % "0.5", + "org.broadinstitute" %% "wdl4s" % "0.6", //---------- Test libraries -------------------// "org.scalatest" %% "scalatest" % "2.2.5" % Test ) From 31037eb1774888d681c349cc46b81f16ae7a88f3 Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Wed, 12 Oct 2016 11:51:52 -0400 Subject: [PATCH 06/30] update wdltool version from 0.6 -> 0.7 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index cced536..3f4b32f 100644 --- a/build.sbt +++ b/build.sbt @@ -9,7 +9,7 @@ organization := "org.broadinstitute" scalaVersion := "2.11.8" // Upcoming release, or current if we're on the master branch -git.baseVersion := "0.6" +git.baseVersion := "0.7" // Shorten the git commit hash git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) } From 1143dd66c15dc3313ba957a71aa88f84e7bcbe67 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Fri, 18 Nov 2016 20:15:47 -0500 Subject: [PATCH 07/30] Fix pullapprove. Adding ruchim and kcibul to pullapprove. Removing scottfrazer from pullapprove to fix "'scottfrazer' is not a collaborator of this repo" error. --- .pullapprove.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pullapprove.yml b/.pullapprove.yml index 7eee40d..ceb8dc6 100644 --- a/.pullapprove.yml +++ b/.pullapprove.yml @@ -7,7 +7,8 @@ reviewers: members: - cjllanwarne - Horneth - - scottfrazer - mcovarr - geoffjentry - kshakir + - kcibul + - ruchim From e5f8c9573e9a8a21dbe8c0b33f9eeba03411a497 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Mon, 21 Nov 2016 16:07:33 -0500 Subject: [PATCH 08/30] Publish `SNAPSHOT`s obfuscated as `SNAP` releases. Fixed build issues related to latest wdl4s. --- build.sbt | 21 +++++++++++++-------- src/main/scala/wdltool/Main.scala | 6 +++--- src/test/scala/wdltool/MainSpec.scala | 2 +- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/build.sbt b/build.sbt index 3f4b32f..defbc74 100644 --- a/build.sbt +++ b/build.sbt @@ -8,16 +8,21 @@ organization := "org.broadinstitute" scalaVersion := "2.11.8" -// Upcoming release, or current if we're on the master branch -git.baseVersion := "0.7" +lazy val versionSettings = Seq( + // Upcoming release, or current if we're on the master branch + git.baseVersion := "0.7", -// Shorten the git commit hash -git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) } + // Shorten the git commit hash + git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) }, -// Travis will deploy tagged releases, add -SNAPSHOT for all local builds -git.gitUncommittedChanges := true + // Travis will deploy tagged releases, add -SNAPSHOT for all local builds + git.gitUncommittedChanges := true, -versionWithGit + // For now, obfuscate SNAPSHOTs from sbt's developers: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241 + git.uncommittedSignifier := Option("SNAP") +) + +versionWithGit ++ versionSettings assemblyJarName in assembly := "wdltool-" + git.baseVersion.value + ".jar" @@ -29,7 +34,7 @@ resolvers ++= Seq( ) libraryDependencies ++= Seq( - "org.broadinstitute" %% "wdl4s" % "0.6", + "org.broadinstitute" %% "wdl4s" % "0.7-4a9e61e-SNAP", //---------- Test libraries -------------------// "org.scalatest" %% "scalatest" % "2.2.5" % Test ) diff --git a/src/main/scala/wdltool/Main.scala b/src/main/scala/wdltool/Main.scala index d25aaca..7f39d9a 100644 --- a/src/main/scala/wdltool/Main.scala +++ b/src/main/scala/wdltool/Main.scala @@ -3,7 +3,7 @@ package wdltool import java.nio.file.Paths import wdl4s.formatter.{AnsiSyntaxHighlighter, HtmlSyntaxHighlighter, SyntaxFormatter} -import wdl4s.{AstTools, NamespaceWithWorkflow, WdlNamespace} +import wdl4s.{AstTools, WdlNamespace, WdlNamespaceWithWorkflow} import spray.json._ import scala.util.{Failure, Success, Try} @@ -57,7 +57,7 @@ object Main extends App { loadWdl(args.head) { namespace => import wdl4s.types.WdlTypeJsonFormatter._ val msg = namespace match { - case x: NamespaceWithWorkflow => x.workflow.inputs.toJson.prettyPrint + case x: WdlNamespaceWithWorkflow => x.workflow.inputs.toJson.prettyPrint case _ => "WDL does not have a local workflow" } @@ -75,7 +75,7 @@ object Main extends App { private[this] def continueIf(valid: => Boolean)(block: => Termination): Termination = if (valid) block else BadUsageTermination private[this] def loadWdl(path: String)(f: WdlNamespace => Termination): Termination = { - Try(WdlNamespace.load(Paths.get(path))) match { + Try(WdlNamespace.loadUsingPath(Paths.get(path), None, None)) match { case Success(namespace) => f(namespace) case Failure(t) => UnsuccessfulTermination(t.getMessage) } diff --git a/src/test/scala/wdltool/MainSpec.scala b/src/test/scala/wdltool/MainSpec.scala index d568a0b..d20317b 100644 --- a/src/test/scala/wdltool/MainSpec.scala +++ b/src/test/scala/wdltool/MainSpec.scala @@ -35,7 +35,7 @@ class MainSpec extends FlatSpec with Matchers with BeforeAndAfterAll { testWdl(ThreeStep) { wdlAndInputs => val res = Main.dispatchCommand(Seq("parse", wdlAndInputs.wdl)) assert(res.isInstanceOf[SuccessfulTermination]) - res.output should include("(Document:") + res.output should include("(Namespace:") } } From 8075e9e431c078405edb8706b1ded6dda9ed80d1 Mon Sep 17 00:00:00 2001 From: geoffjentry Date: Mon, 28 Nov 2016 22:09:24 -0500 Subject: [PATCH 09/30] Update to version of wdl4s which allows for meta in workflow --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index defbc74..2f1c44d 100644 --- a/build.sbt +++ b/build.sbt @@ -34,7 +34,7 @@ resolvers ++= Seq( ) libraryDependencies ++= Seq( - "org.broadinstitute" %% "wdl4s" % "0.7-4a9e61e-SNAP", + "org.broadinstitute" %% "wdl4s" % "0.7-799567f-SNAP", //---------- Test libraries -------------------// "org.scalatest" %% "scalatest" % "2.2.5" % Test ) From 0da4aaacad2b1c8c6415a57962c613e942147d92 Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Fri, 2 Dec 2016 17:31:35 -0500 Subject: [PATCH 10/30] Update WdlTool version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 2f1c44d..44e002d 100644 --- a/build.sbt +++ b/build.sbt @@ -10,7 +10,7 @@ scalaVersion := "2.11.8" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch - git.baseVersion := "0.7", + git.baseVersion := "0.8", // Shorten the git commit hash git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) }, From 58ab47922c03f61a184092d6461ce2a27ff95d9c Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 6 Dec 2016 11:22:42 -0500 Subject: [PATCH 11/30] Added a graph print feature --- README.md | 103 ++++++++++++++++++++++++++++++++ build.sbt | 2 +- src/main/scala/wdltool/GraphPrint.scala | 77 ++++++++++++++++++++++++ src/main/scala/wdltool/Main.scala | 32 +++++++++- 4 files changed, 210 insertions(+), 4 deletions(-) create mode 100644 src/main/scala/wdltool/GraphPrint.scala diff --git a/README.md b/README.md index 55a2b6d..6ff656f 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,109 @@ $ java -jar wdltool.jar highlight test.wdl html } ``` +## graph + +The syntax of the graph command is: +``` +wdltool graph [--all] wdlFile.wdl +``` + +Given a WDL file input, command generates the data-flow graph through the system in `.dot` format. + +For example the fork-join WDL: +``` +task mkFile { + command { + for i in `seq 1 1000` + do + echo $i + done + } + output { + File numbers = stdout() + } + runtime {docker: "ubuntu:latest"} +} + +task grep { + String pattern + File in_file + command { + grep '${pattern}' ${in_file} | wc -l + } + output { + Int count = read_int(stdout()) + } + runtime {docker: "ubuntu:latest"} +} + +task wc { + File in_file + command { + cat ${in_file} | wc -l + } + output { + Int count = read_int(stdout()) + } + runtime {docker: "ubuntu:latest"} +} + +task join { + Int grepCount + Int wcCount + command { + expr ${wcCount} / ${grepCount} + } + output { + Int proportion = read_int(stdout()) + } + runtime {docker: "ubuntu:latest"} +} + +workflow forkjoin { + call mkFile + call grep { input: in_file = mkFile.numbers } + call wc { input: in_file=mkFile.numbers } + call join { input: wcCount = wc.count, grepCount = grep.count } + output { + join.proportion + } +} +``` + +Produces the DAG: +``` +digraph forkjoin { + "call forkjoin.mkFile" -> "call forkjoin.wc" + "call forkjoin.mkFile" -> "call forkjoin.grep" + "call forkjoin.wc" -> "call forkjoin.join" + "call forkjoin.grep" -> "call forkjoin.join" +} +``` + +### The --all flag + +If this flag is set, all WDL graph nodes become nodes in the generated DAG, even if they are not "executed". Typically this will mean task declarations and call outputs. +For example in the above example, with `--all` you would get: + +``` +digraph forkjoin { + "call forkjoin.grep" -> "String forkjoin.grep.pattern" + "call forkjoin.grep" -> "output { forkjoin.grep.count = read_int(stdout()) }" + "call forkjoin.grep" -> "File forkjoin.grep.in_file" + "call forkjoin.wc" -> "output { forkjoin.wc.count = read_int(stdout()) }" + "call forkjoin.grep" -> "call forkjoin.join" + "call forkjoin.wc" -> "File forkjoin.wc.in_file" + "call forkjoin.mkFile" -> "call forkjoin.grep" + "call forkjoin.join" -> "output { forkjoin.join.proportion = read_int(stdout()) }" + "call forkjoin.join" -> "Int forkjoin.join.wcCount" + "call forkjoin.wc" -> "call forkjoin.join" + "call forkjoin.mkFile" -> "output { forkjoin.mkFile.numbers = stdout() }" + "call forkjoin.mkFile" -> "call forkjoin.wc" + "call forkjoin.join" -> "Int forkjoin.join.grepCount" +} +``` + # Getting Started with WDL For documentation and many examples on how to use WDL see [the WDL website](https://software.broadinstitute.org/wdl/). diff --git a/build.sbt b/build.sbt index 44e002d..a081bde 100644 --- a/build.sbt +++ b/build.sbt @@ -34,7 +34,7 @@ resolvers ++= Seq( ) libraryDependencies ++= Seq( - "org.broadinstitute" %% "wdl4s" % "0.7-799567f-SNAP", + "org.broadinstitute" %% "wdl4s" % "0.8-020c10c-SNAP", //---------- Test libraries -------------------// "org.scalatest" %% "scalatest" % "2.2.5" % Test ) diff --git a/src/main/scala/wdltool/GraphPrint.scala b/src/main/scala/wdltool/GraphPrint.scala new file mode 100644 index 0000000..9587468 --- /dev/null +++ b/src/main/scala/wdltool/GraphPrint.scala @@ -0,0 +1,77 @@ +package wdltool + +import java.nio.file.{Files, Paths} + +import wdl4s.{CallOutput, Declaration, If, Scatter, _} +import scala.collection.JavaConverters._ + +object GraphPrint { + + case class WorkflowDigraph(workflowName: String, digraph: Set[String]) + + def generateWorkflowDigraph(file: String, allNodesMode: Boolean): WorkflowDigraph = { + val namespace = WdlNamespaceWithWorkflow.load(Files.readAllLines(Paths.get(file)).asScala.mkString(System.lineSeparator()), Seq(WdlNamespace.fileResolver _)) + + val digraph = if (allNodesMode) { + listAllGraphNodes(namespace) + } else { + val executables = GraphPrint.listExecutableGraphNodes(namespace.workflow) + listAllGraphNodes(namespace, graphNode => executables.contains(graphNode)) + } + + WorkflowDigraph(namespace.workflow.unqualifiedName, digraph) + } + + private def defaultFilter: GraphNode => Boolean = _ => true + + private def listAllGraphNodes(namespace: WdlNamespaceWithWorkflow, filter: GraphNode => Boolean = defaultFilter): Set[String] = { + + val graphNodes = namespace.descendants collect { + case g: GraphNode if filter(g) => g + } + + graphNodes flatMap { graphNode => + val name = graphName(graphNode) + val initialSet: Set[String] = graphNode match { + case c: Call => Set(s""""${dotSafe(name)}"""") + case _ => Set.empty + } + val upstreamLinks = graphNode.upstream collect { + case upstream if filter(upstream) => + val upstreamName = graphName(upstream) + s""""${dotSafe(upstreamName)}" -> "${dotSafe(name)}"""" + } + + initialSet ++ upstreamLinks + } + } + + private def listExecutableGraphNodes(s: Scope): Set[GraphNode] = { + s.children.toSet flatMap { child: Scope => child match { + case call: Call => Set[GraphNode](call) + case scatter: Scatter => Set[GraphNode](scatter) ++ listExecutableGraphNodes(scatter) + case i: If => Set[GraphNode](i) ++ listExecutableGraphNodes(i) + case declaration: Declaration => Set[GraphNode](declaration) + case _ => Set.empty[GraphNode] + }} + } + + + private def dotSafe(s: String) = s.replaceAllLiterally("\"", "\\\"") + + private def graphName(g: GraphNode): String = g match { + case d: Declaration => + val exprString = d.expression.map(e => " = " + e.toWdlString).getOrElse("") + s"${d.wdlType.toWdlString} ${d.fullyQualifiedName}$exprString" + case c: Call => + s"call ${c.fullyQualifiedName}" + case i: If => + s"if (${i.condition.toWdlString})" + case s: Scatter => + s"scatter (${s.item} in ${s.collection.toWdlString})" + case c: CallOutput => + val exprString = c.expression.map(e => " = " + e.toWdlString).getOrElse("") + s"output { ${c.fullyQualifiedName}$exprString }" + case other => s"${other.getClass.getSimpleName}: ${other.fullyQualifiedName}" + } +} diff --git a/src/main/scala/wdltool/Main.scala b/src/main/scala/wdltool/Main.scala index 7f39d9a..04b9854 100644 --- a/src/main/scala/wdltool/Main.scala +++ b/src/main/scala/wdltool/Main.scala @@ -3,9 +3,10 @@ package wdltool import java.nio.file.Paths import wdl4s.formatter.{AnsiSyntaxHighlighter, HtmlSyntaxHighlighter, SyntaxFormatter} -import wdl4s.{AstTools, WdlNamespace, WdlNamespaceWithWorkflow} +import wdl4s._ import spray.json._ + import scala.util.{Failure, Success, Try} object Main extends App { @@ -33,6 +34,7 @@ object Main extends App { case Some(x) if x == Actions.Highlight => highlight(args.tail) case Some(x) if x == Actions.Inputs => inputs(args.tail) case Some(x) if x == Actions.Parse => parse(args.tail) + case Some(x) if x == Actions.Graph => graph(args.tail) case _ => BadUsageTermination } } @@ -72,6 +74,23 @@ object Main extends App { } } + def graph(args: Seq[String]): Termination = { + continueIf(args.length == 1 || (args.length == 2 && args.head.equals("--all"))) { + + val (file, allNodesMode) = + if (args.size == 1) (args.head, false) + else (args(1), true) + + val workflowDigraph = GraphPrint.generateWorkflowDigraph(file, allNodesMode) + + val result = s"""|digraph ${workflowDigraph.workflowName} { + | ${workflowDigraph.digraph.mkString(System.lineSeparator + " ")} + |} + |""" + SuccessfulTermination(result.stripMargin) + } + } + private[this] def continueIf(valid: => Boolean)(block: => Termination): Termination = if (valid) block else BadUsageTermination private[this] def loadWdl(path: String)(f: WdlNamespace => Termination): Termination = { @@ -88,7 +107,7 @@ object Main extends App { } yield action object Actions extends Enumeration { - val Parse, Validate, Highlight, Inputs = Value + val Parse, Validate, Highlight, Inputs, Graph = Value } val UsageMessage = """ @@ -119,7 +138,14 @@ object Main extends App { | abstract syntax tree if it is valid, and a syntax error | otherwise. Note that higher-level AST checks are not done | via this sub-command and the 'validate' subcommand should - | be used for full validation + | be used for full validation. + |graph [--all] + | + | Reads a WDL file against the grammar and prints out a + | .dot of the DAG if it is valid, and a syntax error + | otherwise. + | Use [--all] to show all graph nodes in the WDL spec, + | even the non-executable nodes. """.stripMargin val termination = dispatchCommand(args) From 99c7d2208f22c7b6b76eedba6f79149b495f0754 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Thu, 5 Jan 2017 13:36:29 -0500 Subject: [PATCH 12/30] Updated wdl4s to a release version. --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index a081bde..a65c713 100644 --- a/build.sbt +++ b/build.sbt @@ -34,7 +34,7 @@ resolvers ++= Seq( ) libraryDependencies ++= Seq( - "org.broadinstitute" %% "wdl4s" % "0.8-020c10c-SNAP", + "org.broadinstitute" %% "wdl4s" % "0.8", //---------- Test libraries -------------------// "org.scalatest" %% "scalatest" % "2.2.5" % Test ) From 3e4f92d16f8dbd4e276aa8b832edd19c55971243 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Thu, 5 Jan 2017 13:40:19 -0500 Subject: [PATCH 13/30] Version bump --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index a65c713..7c2483c 100644 --- a/build.sbt +++ b/build.sbt @@ -10,7 +10,7 @@ scalaVersion := "2.11.8" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch - git.baseVersion := "0.8", + git.baseVersion := "0.9", // Shorten the git commit hash git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) }, From 7f4b9839fe136acb0410d649d0a9ea53a40b5354 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Mon, 23 Jan 2017 13:05:08 -0500 Subject: [PATCH 14/30] Disabled type annotation inspections in IntelliJ. Synced common scala .gitignore across repositories. --- .gitignore | 15 +++++++++++---- .idea/inspectionProfiles/Project_Default.xml | 6 ++++++ 2 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 .idea/inspectionProfiles/Project_Default.xml diff --git a/.gitignore b/.gitignore index fa01827..387acc5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,12 @@ -tags -.idea/ -target -.artifactory +# common scala config *~ +.DS_Store +.artifactory +.idea/* +!/.idea/inspectionProfiles/ +.idea/inspectionProfiles/* +!/.idea/inspectionProfiles/Project_Default.xml +target + +# custom config +tags diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..b15d7f2 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file From ac6ebeb204a791ae417a56c508a4add499a7c111 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Wed, 8 Feb 2017 15:53:36 -0500 Subject: [PATCH 15/30] Added katevoss and alphabetized pullapprove --- .pullapprove.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.pullapprove.yml b/.pullapprove.yml index ceb8dc6..7dd4c51 100644 --- a/.pullapprove.yml +++ b/.pullapprove.yml @@ -5,10 +5,11 @@ author_approval: ignored reviewers: required: 2 members: - - cjllanwarne - Horneth - - mcovarr + - cjllanwarne - geoffjentry - - kshakir + - katevoss - kcibul + - kshakir + - mcovarr - ruchim From 51e9098be1b4bd277a27d26b2121482c267dbf22 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Mon, 13 Feb 2017 15:05:07 -0500 Subject: [PATCH 16/30] new wdl4s dependency --- build.sbt | 2 +- src/main/scala/wdltool/GraphPrint.scala | 3 ++- src/main/scala/wdltool/Main.scala | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 7c2483c..76d358a 100644 --- a/build.sbt +++ b/build.sbt @@ -34,7 +34,7 @@ resolvers ++= Seq( ) libraryDependencies ++= Seq( - "org.broadinstitute" %% "wdl4s" % "0.8", + "org.broadinstitute" %% "wdl4s" % "0.10-ae6c153-SNAP", //---------- Test libraries -------------------// "org.scalatest" %% "scalatest" % "2.2.5" % Test ) diff --git a/src/main/scala/wdltool/GraphPrint.scala b/src/main/scala/wdltool/GraphPrint.scala index 9587468..6bb12ff 100644 --- a/src/main/scala/wdltool/GraphPrint.scala +++ b/src/main/scala/wdltool/GraphPrint.scala @@ -10,7 +10,8 @@ object GraphPrint { case class WorkflowDigraph(workflowName: String, digraph: Set[String]) def generateWorkflowDigraph(file: String, allNodesMode: Boolean): WorkflowDigraph = { - val namespace = WdlNamespaceWithWorkflow.load(Files.readAllLines(Paths.get(file)).asScala.mkString(System.lineSeparator()), Seq(WdlNamespace.fileResolver _)) + // It's ok to use .get here, we're happy to throw an exception and crash the program! + val namespace = WdlNamespaceWithWorkflow.load(Files.readAllLines(Paths.get(file)).asScala.mkString(System.lineSeparator()), Seq(WdlNamespace.fileResolver _)).get val digraph = if (allNodesMode) { listAllGraphNodes(namespace) diff --git a/src/main/scala/wdltool/Main.scala b/src/main/scala/wdltool/Main.scala index 04b9854..28bdaef 100644 --- a/src/main/scala/wdltool/Main.scala +++ b/src/main/scala/wdltool/Main.scala @@ -94,7 +94,7 @@ object Main extends App { private[this] def continueIf(valid: => Boolean)(block: => Termination): Termination = if (valid) block else BadUsageTermination private[this] def loadWdl(path: String)(f: WdlNamespace => Termination): Termination = { - Try(WdlNamespace.loadUsingPath(Paths.get(path), None, None)) match { + WdlNamespace.loadUsingPath(Paths.get(path), None, None) match { case Success(namespace) => f(namespace) case Failure(t) => UnsuccessfulTermination(t.getMessage) } From 476113771613af28231915f3e0b19ff02c67b529 Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Fri, 3 Mar 2017 16:04:52 -0500 Subject: [PATCH 17/30] Update to using WDL4s version without snap --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 76d358a..5519c4e 100644 --- a/build.sbt +++ b/build.sbt @@ -34,7 +34,7 @@ resolvers ++= Seq( ) libraryDependencies ++= Seq( - "org.broadinstitute" %% "wdl4s" % "0.10-ae6c153-SNAP", + "org.broadinstitute" %% "wdl4s" % "0.10", //---------- Test libraries -------------------// "org.scalatest" %% "scalatest" % "2.2.5" % Test ) From caa8f1d0092cc020e8b1d7a889b16eaef68261e3 Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Fri, 3 Mar 2017 16:10:26 -0500 Subject: [PATCH 18/30] Update develop to the next version of Wdltool --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 5519c4e..ed92493 100644 --- a/build.sbt +++ b/build.sbt @@ -10,7 +10,7 @@ scalaVersion := "2.11.8" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch - git.baseVersion := "0.9", + git.baseVersion := "0.10", // Shorten the git commit hash git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) }, From 5e1be25fed6e9edceaa38cb87b5d0775de0c5c35 Mon Sep 17 00:00:00 2001 From: Thibault Jeandet Date: Mon, 13 Mar 2017 13:52:38 -0400 Subject: [PATCH 19/30] make build.sbt more script friendly --- build.sbt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index ed92493..589a2a5 100644 --- a/build.sbt +++ b/build.sbt @@ -8,6 +8,8 @@ organization := "org.broadinstitute" scalaVersion := "2.11.8" +val wdl4sV = "0.10" + lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch git.baseVersion := "0.10", @@ -34,7 +36,7 @@ resolvers ++= Seq( ) libraryDependencies ++= Seq( - "org.broadinstitute" %% "wdl4s" % "0.10", + "org.broadinstitute" %% "wdl4s" % wdl4sV, //---------- Test libraries -------------------// "org.scalatest" %% "scalatest" % "2.2.5" % Test ) From a8f2b9f1504f419513bd372bf413a376c34c8fa5 Mon Sep 17 00:00:00 2001 From: geoffjentry Date: Sat, 25 Mar 2017 13:50:25 -0400 Subject: [PATCH 20/30] Update to using scala 2.12 --- build.sbt | 6 +++--- project/plugins.sbt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 589a2a5..41ad0bb 100644 --- a/build.sbt +++ b/build.sbt @@ -6,9 +6,9 @@ name := "wdltool" organization := "org.broadinstitute" -scalaVersion := "2.11.8" +scalaVersion := "2.12.1" -val wdl4sV = "0.10" +val wdl4sV = "0.10-f7a345e-SNAP" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch @@ -38,7 +38,7 @@ resolvers ++= Seq( libraryDependencies ++= Seq( "org.broadinstitute" %% "wdl4s" % wdl4sV, //---------- Test libraries -------------------// - "org.scalatest" %% "scalatest" % "2.2.5" % Test + "org.scalatest" %% "scalatest" % "3.0.1" % Test ) val customMergeStrategy: String => MergeStrategy = { diff --git a/project/plugins.sbt b/project/plugins.sbt index 20755cc..cadbd87 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,7 +2,7 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "0.8.5") addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.0") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.0.4") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.0") addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.0.0") From f4badc720856bfc65f07c98f5b2be3e22e51872b Mon Sep 17 00:00:00 2001 From: jsotobroad Date: Mon, 3 Apr 2017 14:29:23 -0400 Subject: [PATCH 21/30] Adding jsotobroad to pullapprover members list --- .pullapprove.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pullapprove.yml b/.pullapprove.yml index 7dd4c51..3ab4b6b 100644 --- a/.pullapprove.yml +++ b/.pullapprove.yml @@ -8,6 +8,7 @@ reviewers: - Horneth - cjllanwarne - geoffjentry + - jsotobroad - katevoss - kcibul - kshakir From 582ea2476f58f96f8344f6cdc4f1a093c1709441 Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Tue, 11 Apr 2017 16:10:47 -0400 Subject: [PATCH 22/30] Update wdltool version from 0.10 to 0.11 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 41ad0bb..8591e0e 100644 --- a/build.sbt +++ b/build.sbt @@ -12,7 +12,7 @@ val wdl4sV = "0.10-f7a345e-SNAP" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch - git.baseVersion := "0.10", + git.baseVersion := "0.11", // Shorten the git commit hash git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) }, From 31fe3c5fd998ef86a7e159e9b2bbd77be0204d4f Mon Sep 17 00:00:00 2001 From: Thibault Jeandet Date: Tue, 30 May 2017 13:53:23 -0400 Subject: [PATCH 23/30] Update wdltool version from 0.11 to 0.12 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 8591e0e..38f6cd6 100644 --- a/build.sbt +++ b/build.sbt @@ -12,7 +12,7 @@ val wdl4sV = "0.10-f7a345e-SNAP" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch - git.baseVersion := "0.11", + git.baseVersion := "0.12", // Shorten the git commit hash git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) }, From 08c00dc0b8484aee0c8fb144bc81e8ef40f0358c Mon Sep 17 00:00:00 2001 From: Thibault Jeandet Date: Tue, 30 May 2017 15:16:21 -0400 Subject: [PATCH 24/30] update wdl4s version to 0.12 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 38f6cd6..d393765 100644 --- a/build.sbt +++ b/build.sbt @@ -8,7 +8,7 @@ organization := "org.broadinstitute" scalaVersion := "2.12.1" -val wdl4sV = "0.10-f7a345e-SNAP" +val wdl4sV = "0.12" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch From 01f9c7fe49ce31f88e38aa76302a1218f55a78bb Mon Sep 17 00:00:00 2001 From: Thibault Jeandet Date: Tue, 30 May 2017 15:18:43 -0400 Subject: [PATCH 25/30] update version to upcoming 0.13 version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index d393765..f7e9992 100644 --- a/build.sbt +++ b/build.sbt @@ -12,7 +12,7 @@ val wdl4sV = "0.12" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch - git.baseVersion := "0.12", + git.baseVersion := "0.13", // Shorten the git commit hash git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) }, From 6056347f1f5954dffe23f3e0e7775005e4474a56 Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Wed, 7 Jun 2017 17:43:48 -0400 Subject: [PATCH 26/30] jfrog --- build.sbt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index f7e9992..176f72f 100644 --- a/build.sbt +++ b/build.sbt @@ -8,7 +8,7 @@ organization := "org.broadinstitute" scalaVersion := "2.12.1" -val wdl4sV = "0.12" +val wdl4sV = "0.13-4804734-SNAP" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch @@ -31,8 +31,8 @@ assemblyJarName in assembly := "wdltool-" + git.baseVersion.value + ".jar" logLevel in assembly := Level.Info resolvers ++= Seq( - "Broad Artifactory Releases" at "https://artifactory.broadinstitute.org/artifactory/libs-release/", - "Broad Artifactory Snapshots" at "https://artifactory.broadinstitute.org/artifactory/libs-snapshot/" + "Broad Artifactory Releases" at "https://broadinstitute.jfrog.io/broadinstitute/libs-release/", + "Broad Artifactory Snapshots" at "https://broadinstitute.jfrog.io/broadinstitute/libs-snapshot/" ) libraryDependencies ++= Seq( From 20205f15d86b0f0dcb5b04f119adb2036a97622a Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Fri, 30 Jun 2017 13:27:39 -0400 Subject: [PATCH 27/30] Update wdltool version to 0.13 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 176f72f..2e1c112 100644 --- a/build.sbt +++ b/build.sbt @@ -8,7 +8,7 @@ organization := "org.broadinstitute" scalaVersion := "2.12.1" -val wdl4sV = "0.13-4804734-SNAP" +val wdl4sV = "0.13" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch From 7cb6b1d1b957175ab126e08c13425f5794708c79 Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Fri, 30 Jun 2017 13:27:46 -0400 Subject: [PATCH 28/30] Update wdltool version from 0.13 to 0.14 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 2e1c112..07efa1e 100644 --- a/build.sbt +++ b/build.sbt @@ -12,7 +12,7 @@ val wdl4sV = "0.13" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch - git.baseVersion := "0.13", + git.baseVersion := "0.14", // Shorten the git commit hash git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) }, From 9757a16ec2df118be4f1fc82f92b83a6d73dd794 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Sat, 22 Jul 2017 10:41:07 -0400 Subject: [PATCH 29/30] Better graphs for scatters using subgraphs --- build.sbt | 20 +++++++- src/main/scala/wdltool/GraphPrint.scala | 86 ++++++++++++++++++++------------- src/main/scala/wdltool/Main.scala | 13 ++--- src/test/scala/wdltool/SampleWdl.scala | 8 +-- 4 files changed, 81 insertions(+), 46 deletions(-) diff --git a/build.sbt b/build.sbt index 07efa1e..66eaed0 100644 --- a/build.sbt +++ b/build.sbt @@ -8,7 +8,7 @@ organization := "org.broadinstitute" scalaVersion := "2.12.1" -val wdl4sV = "0.13" +val wdl4sV = "0.14-7c693a3-SNAP" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch @@ -35,11 +35,27 @@ resolvers ++= Seq( "Broad Artifactory Snapshots" at "https://broadinstitute.jfrog.io/broadinstitute/libs-snapshot/" ) +lazy val catsDependencies = List( + "org.typelevel" %% "cats" % "0.9.0", + "org.typelevel" %% "kittens" % "1.0.0-M10", + "com.github.benhutchison" %% "mouse" % "0.9" +) map (_ + /* + Exclude test framework cats-laws and its transitive dependency scalacheck. + If sbt detects scalacheck, it tries to run it. + Explicitly excluding the two problematic artifacts instead of including the three (or four?). + https://github.com/typelevel/cats/tree/v0.7.2#getting-started + Re "_2.12", see also: https://github.com/sbt/sbt/issues/1518 + */ + exclude("org.typelevel", "cats-laws_2.12") + exclude("org.typelevel", "cats-kernel-laws_2.12") + ) + libraryDependencies ++= Seq( "org.broadinstitute" %% "wdl4s" % wdl4sV, //---------- Test libraries -------------------// "org.scalatest" %% "scalatest" % "3.0.1" % Test -) +) ++ catsDependencies val customMergeStrategy: String => MergeStrategy = { case x if Assembly.isConfigFile(x) => diff --git a/src/main/scala/wdltool/GraphPrint.scala b/src/main/scala/wdltool/GraphPrint.scala index 6bb12ff..2125c8a 100644 --- a/src/main/scala/wdltool/GraphPrint.scala +++ b/src/main/scala/wdltool/GraphPrint.scala @@ -1,71 +1,89 @@ package wdltool import java.nio.file.{Files, Paths} +import java.util.concurrent.atomic.AtomicInteger + +import wdl4s.wdl.{CallOutput, Declaration, If, Scatter, WdlCall, _} +import wdl4s.wdl.WdlGraphNode -import wdl4s.{CallOutput, Declaration, If, Scatter, _} import scala.collection.JavaConverters._ +import cats.implicits._ +import cats.derived.monoid._, legacy._ + object GraphPrint { - case class WorkflowDigraph(workflowName: String, digraph: Set[String]) + final case class WorkflowDigraph(workflowName: String, digraph: NodesAndLinks) + final case class NodesAndLinks(nodes: Set[String], links: Set[String]) def generateWorkflowDigraph(file: String, allNodesMode: Boolean): WorkflowDigraph = { // It's ok to use .get here, we're happy to throw an exception and crash the program! val namespace = WdlNamespaceWithWorkflow.load(Files.readAllLines(Paths.get(file)).asScala.mkString(System.lineSeparator()), Seq(WdlNamespace.fileResolver _)).get - val digraph = if (allNodesMode) { - listAllGraphNodes(namespace) - } else { - val executables = GraphPrint.listExecutableGraphNodes(namespace.workflow) - listAllGraphNodes(namespace, graphNode => executables.contains(graphNode)) - } + val digraph = listAllGraphNodes(namespace.workflow) WorkflowDigraph(namespace.workflow.unqualifiedName, digraph) } - private def defaultFilter: GraphNode => Boolean = _ => true + private val clusterCount: AtomicInteger = new AtomicInteger(0) + + + private def listAllGraphNodes(scope: Scope): NodesAndLinks = { + + val callsAndDeclarations: Set[WdlGraphNode] = (scope.children collect { + case w: WdlGraphNode if isCallOrDeclaration(w) => w + }).toSet - private def listAllGraphNodes(namespace: WdlNamespaceWithWorkflow, filter: GraphNode => Boolean = defaultFilter): Set[String] = { + val subGraphs: Set[WdlGraphNode] = (scope.children collect { + case s: Scatter => s + case i: If => i + }).toSet - val graphNodes = namespace.descendants collect { - case g: GraphNode if filter(g) => g + def upstreamLinks(wdlGraphNode: WdlGraphNode, graphNodeName: String, suffix: String = ""): Set[String] = wdlGraphNode.upstream collect { + case upstream: WdlGraphNode if isCallOrDeclaration(upstream) => + val upstreamName = graphName(upstream) + s""""$upstreamName" -> "$graphNodeName" $suffix""" } - graphNodes flatMap { graphNode => + val thisLevelNodesAndLinks: NodesAndLinks = callsAndDeclarations foldMap { graphNode => val name = graphName(graphNode) val initialSet: Set[String] = graphNode match { - case c: Call => Set(s""""${dotSafe(name)}"""") + case w: WdlGraphNode if isCallOrDeclaration(w) => Set(s""""$name"""") case _ => Set.empty } - val upstreamLinks = graphNode.upstream collect { - case upstream if filter(upstream) => - val upstreamName = graphName(upstream) - s""""${dotSafe(upstreamName)}" -> "${dotSafe(name)}"""" - } - initialSet ++ upstreamLinks + val fromStart = if (graphNode.upstream.isEmpty) Set(s""""start" -> "$name"""") else Set.empty + + NodesAndLinks(initialSet, upstreamLinks(graphNode, name) ++ fromStart) } - } - private def listExecutableGraphNodes(s: Scope): Set[GraphNode] = { - s.children.toSet flatMap { child: Scope => child match { - case call: Call => Set[GraphNode](call) - case scatter: Scatter => Set[GraphNode](scatter) ++ listExecutableGraphNodes(scatter) - case i: If => Set[GraphNode](i) ++ listExecutableGraphNodes(i) - case declaration: Declaration => Set[GraphNode](declaration) - case _ => Set.empty[GraphNode] - }} + val subGraphNodesAndLinks: NodesAndLinks = subGraphs foldMap { wdlGraphNode => + val clusterName = "cluster_" + clusterCount.getAndIncrement() + val subGraphName = graphName(wdlGraphNode) + val subNodes = listAllGraphNodes(wdlGraphNode) + val scope = s""" + |subgraph $clusterName { + | ${subNodes.nodes.mkString(sep="\n ")} + | "$subGraphName" [shape=plaintext] + |} + """.stripMargin + + NodesAndLinks(Set(scope), subNodes.links ++ upstreamLinks(wdlGraphNode, subGraphName, s"[lhead=$clusterName]")) + } + + thisLevelNodesAndLinks |+| subGraphNodesAndLinks } + private def isCallOrDeclaration(w: WdlGraphNode): Boolean = w.isInstanceOf[WdlCall] || w.isInstanceOf[Declaration] private def dotSafe(s: String) = s.replaceAllLiterally("\"", "\\\"") - private def graphName(g: GraphNode): String = g match { + private def graphName(g: WdlGraphNode): String = dotSafe(g match { case d: Declaration => val exprString = d.expression.map(e => " = " + e.toWdlString).getOrElse("") - s"${d.wdlType.toWdlString} ${d.fullyQualifiedName}$exprString" - case c: Call => - s"call ${c.fullyQualifiedName}" + s"${d.wdlType.toWdlString} ${d.unqualifiedName}$exprString" + case c: WdlCall => + s"call ${c.unqualifiedName}" case i: If => s"if (${i.condition.toWdlString})" case s: Scatter => @@ -74,5 +92,5 @@ object GraphPrint { val exprString = c.expression.map(e => " = " + e.toWdlString).getOrElse("") s"output { ${c.fullyQualifiedName}$exprString }" case other => s"${other.getClass.getSimpleName}: ${other.fullyQualifiedName}" - } + }) } diff --git a/src/main/scala/wdltool/Main.scala b/src/main/scala/wdltool/Main.scala index 28bdaef..da4231b 100644 --- a/src/main/scala/wdltool/Main.scala +++ b/src/main/scala/wdltool/Main.scala @@ -2,12 +2,11 @@ package wdltool import java.nio.file.Paths -import wdl4s.formatter.{AnsiSyntaxHighlighter, HtmlSyntaxHighlighter, SyntaxFormatter} -import wdl4s._ +import wdl4s.wdl.formatter.{AnsiSyntaxHighlighter, HtmlSyntaxHighlighter, SyntaxFormatter} +import wdl4s.wdl._ import spray.json._ - -import scala.util.{Failure, Success, Try} +import scala.util.{Failure, Success} object Main extends App { sealed trait Termination { @@ -57,7 +56,7 @@ object Main extends App { def inputs(args: Seq[String]): Termination = { continueIf(args.length == 1) { loadWdl(args.head) { namespace => - import wdl4s.types.WdlTypeJsonFormatter._ + import wdl4s.wdl.types.WdlTypeJsonFormatter._ val msg = namespace match { case x: WdlNamespaceWithWorkflow => x.workflow.inputs.toJson.prettyPrint case _ => "WDL does not have a local workflow" @@ -84,7 +83,9 @@ object Main extends App { val workflowDigraph = GraphPrint.generateWorkflowDigraph(file, allNodesMode) val result = s"""|digraph ${workflowDigraph.workflowName} { - | ${workflowDigraph.digraph.mkString(System.lineSeparator + " ")} + | compound=true; + | ${workflowDigraph.digraph.links.mkString(System.lineSeparator + " ")} + | ${workflowDigraph.digraph.nodes.mkString(System.lineSeparator + " ")} |} |""" SuccessfulTermination(result.stripMargin) diff --git a/src/test/scala/wdltool/SampleWdl.scala b/src/test/scala/wdltool/SampleWdl.scala index d2179b9..4326449 100644 --- a/src/test/scala/wdltool/SampleWdl.scala +++ b/src/test/scala/wdltool/SampleWdl.scala @@ -4,13 +4,13 @@ import java.io.{FileWriter, File} import java.nio.file.{Files, Path} import spray.json._ -import wdl4s._ -import wdl4s.values._ +import wdl4s.wdl._ +import wdl4s.wdl.values._ import scala.language.postfixOps trait SampleWdl { - def wdlSource(runtime: String = ""): WdlSource + def wdlSource(runtime: String = ""): WorkflowSource def rawInputs: WorkflowRawInputs def name = getClass.getSimpleName.stripSuffix("$") @@ -35,7 +35,7 @@ trait SampleWdl { def read(value: JsValue) = throw new NotImplementedError(s"Reading JSON not implemented: $value") } - def wdlJson: WdlJson = rawInputs.toJson.prettyPrint + def wdlJson: WorkflowJson = rawInputs.toJson.prettyPrint def createFileArray(base: Path): Unit = { createFile("f1", base, "line1\nline2\n") From af5b296189c6bc794364d721f44928319176bac4 Mon Sep 17 00:00:00 2001 From: Phil Shapiro Date: Mon, 31 Jul 2017 14:14:27 -0400 Subject: [PATCH 30/30] - fix issue where an error, such as a validation failure, wouldn't result in a non-zero exit code --- src/main/scala/wdltool/Main.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/wdltool/Main.scala b/src/main/scala/wdltool/Main.scala index da4231b..d2d0c9d 100644 --- a/src/main/scala/wdltool/Main.scala +++ b/src/main/scala/wdltool/Main.scala @@ -157,5 +157,5 @@ object Main extends App { case BadUsageTermination => Console.err.println(UsageMessage) } - termination.returnCode + sys.exit(termination.returnCode) }