From 01f9c7fe49ce31f88e38aa76302a1218f55a78bb Mon Sep 17 00:00:00 2001 From: Thibault Jeandet Date: Tue, 30 May 2017 15:18:43 -0400 Subject: [PATCH 1/6] update version to upcoming 0.13 version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index d393765..f7e9992 100644 --- a/build.sbt +++ b/build.sbt @@ -12,7 +12,7 @@ val wdl4sV = "0.12" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch - git.baseVersion := "0.12", + git.baseVersion := "0.13", // Shorten the git commit hash git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) }, From 6056347f1f5954dffe23f3e0e7775005e4474a56 Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Wed, 7 Jun 2017 17:43:48 -0400 Subject: [PATCH 2/6] jfrog --- build.sbt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index f7e9992..176f72f 100644 --- a/build.sbt +++ b/build.sbt @@ -8,7 +8,7 @@ organization := "org.broadinstitute" scalaVersion := "2.12.1" -val wdl4sV = "0.12" +val wdl4sV = "0.13-4804734-SNAP" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch @@ -31,8 +31,8 @@ assemblyJarName in assembly := "wdltool-" + git.baseVersion.value + ".jar" logLevel in assembly := Level.Info resolvers ++= Seq( - "Broad Artifactory Releases" at "https://artifactory.broadinstitute.org/artifactory/libs-release/", - "Broad Artifactory Snapshots" at "https://artifactory.broadinstitute.org/artifactory/libs-snapshot/" + "Broad Artifactory Releases" at "https://broadinstitute.jfrog.io/broadinstitute/libs-release/", + "Broad Artifactory Snapshots" at "https://broadinstitute.jfrog.io/broadinstitute/libs-snapshot/" ) libraryDependencies ++= Seq( From 20205f15d86b0f0dcb5b04f119adb2036a97622a Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Fri, 30 Jun 2017 13:27:39 -0400 Subject: [PATCH 3/6] Update wdltool version to 0.13 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 176f72f..2e1c112 100644 --- a/build.sbt +++ b/build.sbt @@ -8,7 +8,7 @@ organization := "org.broadinstitute" scalaVersion := "2.12.1" -val wdl4sV = "0.13-4804734-SNAP" +val wdl4sV = "0.13" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch From 7cb6b1d1b957175ab126e08c13425f5794708c79 Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Fri, 30 Jun 2017 13:27:46 -0400 Subject: [PATCH 4/6] Update wdltool version from 0.13 to 0.14 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 2e1c112..07efa1e 100644 --- a/build.sbt +++ b/build.sbt @@ -12,7 +12,7 @@ val wdl4sV = "0.13" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch - git.baseVersion := "0.13", + git.baseVersion := "0.14", // Shorten the git commit hash git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) }, From 9757a16ec2df118be4f1fc82f92b83a6d73dd794 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Sat, 22 Jul 2017 10:41:07 -0400 Subject: [PATCH 5/6] Better graphs for scatters using subgraphs --- build.sbt | 20 +++++++- src/main/scala/wdltool/GraphPrint.scala | 86 ++++++++++++++++++++------------- src/main/scala/wdltool/Main.scala | 13 ++--- src/test/scala/wdltool/SampleWdl.scala | 8 +-- 4 files changed, 81 insertions(+), 46 deletions(-) diff --git a/build.sbt b/build.sbt index 07efa1e..66eaed0 100644 --- a/build.sbt +++ b/build.sbt @@ -8,7 +8,7 @@ organization := "org.broadinstitute" scalaVersion := "2.12.1" -val wdl4sV = "0.13" +val wdl4sV = "0.14-7c693a3-SNAP" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch @@ -35,11 +35,27 @@ resolvers ++= Seq( "Broad Artifactory Snapshots" at "https://broadinstitute.jfrog.io/broadinstitute/libs-snapshot/" ) +lazy val catsDependencies = List( + "org.typelevel" %% "cats" % "0.9.0", + "org.typelevel" %% "kittens" % "1.0.0-M10", + "com.github.benhutchison" %% "mouse" % "0.9" +) map (_ + /* + Exclude test framework cats-laws and its transitive dependency scalacheck. + If sbt detects scalacheck, it tries to run it. + Explicitly excluding the two problematic artifacts instead of including the three (or four?). + https://github.com/typelevel/cats/tree/v0.7.2#getting-started + Re "_2.12", see also: https://github.com/sbt/sbt/issues/1518 + */ + exclude("org.typelevel", "cats-laws_2.12") + exclude("org.typelevel", "cats-kernel-laws_2.12") + ) + libraryDependencies ++= Seq( "org.broadinstitute" %% "wdl4s" % wdl4sV, //---------- Test libraries -------------------// "org.scalatest" %% "scalatest" % "3.0.1" % Test -) +) ++ catsDependencies val customMergeStrategy: String => MergeStrategy = { case x if Assembly.isConfigFile(x) => diff --git a/src/main/scala/wdltool/GraphPrint.scala b/src/main/scala/wdltool/GraphPrint.scala index 6bb12ff..2125c8a 100644 --- a/src/main/scala/wdltool/GraphPrint.scala +++ b/src/main/scala/wdltool/GraphPrint.scala @@ -1,71 +1,89 @@ package wdltool import java.nio.file.{Files, Paths} +import java.util.concurrent.atomic.AtomicInteger + +import wdl4s.wdl.{CallOutput, Declaration, If, Scatter, WdlCall, _} +import wdl4s.wdl.WdlGraphNode -import wdl4s.{CallOutput, Declaration, If, Scatter, _} import scala.collection.JavaConverters._ +import cats.implicits._ +import cats.derived.monoid._, legacy._ + object GraphPrint { - case class WorkflowDigraph(workflowName: String, digraph: Set[String]) + final case class WorkflowDigraph(workflowName: String, digraph: NodesAndLinks) + final case class NodesAndLinks(nodes: Set[String], links: Set[String]) def generateWorkflowDigraph(file: String, allNodesMode: Boolean): WorkflowDigraph = { // It's ok to use .get here, we're happy to throw an exception and crash the program! val namespace = WdlNamespaceWithWorkflow.load(Files.readAllLines(Paths.get(file)).asScala.mkString(System.lineSeparator()), Seq(WdlNamespace.fileResolver _)).get - val digraph = if (allNodesMode) { - listAllGraphNodes(namespace) - } else { - val executables = GraphPrint.listExecutableGraphNodes(namespace.workflow) - listAllGraphNodes(namespace, graphNode => executables.contains(graphNode)) - } + val digraph = listAllGraphNodes(namespace.workflow) WorkflowDigraph(namespace.workflow.unqualifiedName, digraph) } - private def defaultFilter: GraphNode => Boolean = _ => true + private val clusterCount: AtomicInteger = new AtomicInteger(0) + + + private def listAllGraphNodes(scope: Scope): NodesAndLinks = { + + val callsAndDeclarations: Set[WdlGraphNode] = (scope.children collect { + case w: WdlGraphNode if isCallOrDeclaration(w) => w + }).toSet - private def listAllGraphNodes(namespace: WdlNamespaceWithWorkflow, filter: GraphNode => Boolean = defaultFilter): Set[String] = { + val subGraphs: Set[WdlGraphNode] = (scope.children collect { + case s: Scatter => s + case i: If => i + }).toSet - val graphNodes = namespace.descendants collect { - case g: GraphNode if filter(g) => g + def upstreamLinks(wdlGraphNode: WdlGraphNode, graphNodeName: String, suffix: String = ""): Set[String] = wdlGraphNode.upstream collect { + case upstream: WdlGraphNode if isCallOrDeclaration(upstream) => + val upstreamName = graphName(upstream) + s""""$upstreamName" -> "$graphNodeName" $suffix""" } - graphNodes flatMap { graphNode => + val thisLevelNodesAndLinks: NodesAndLinks = callsAndDeclarations foldMap { graphNode => val name = graphName(graphNode) val initialSet: Set[String] = graphNode match { - case c: Call => Set(s""""${dotSafe(name)}"""") + case w: WdlGraphNode if isCallOrDeclaration(w) => Set(s""""$name"""") case _ => Set.empty } - val upstreamLinks = graphNode.upstream collect { - case upstream if filter(upstream) => - val upstreamName = graphName(upstream) - s""""${dotSafe(upstreamName)}" -> "${dotSafe(name)}"""" - } - initialSet ++ upstreamLinks + val fromStart = if (graphNode.upstream.isEmpty) Set(s""""start" -> "$name"""") else Set.empty + + NodesAndLinks(initialSet, upstreamLinks(graphNode, name) ++ fromStart) } - } - private def listExecutableGraphNodes(s: Scope): Set[GraphNode] = { - s.children.toSet flatMap { child: Scope => child match { - case call: Call => Set[GraphNode](call) - case scatter: Scatter => Set[GraphNode](scatter) ++ listExecutableGraphNodes(scatter) - case i: If => Set[GraphNode](i) ++ listExecutableGraphNodes(i) - case declaration: Declaration => Set[GraphNode](declaration) - case _ => Set.empty[GraphNode] - }} + val subGraphNodesAndLinks: NodesAndLinks = subGraphs foldMap { wdlGraphNode => + val clusterName = "cluster_" + clusterCount.getAndIncrement() + val subGraphName = graphName(wdlGraphNode) + val subNodes = listAllGraphNodes(wdlGraphNode) + val scope = s""" + |subgraph $clusterName { + | ${subNodes.nodes.mkString(sep="\n ")} + | "$subGraphName" [shape=plaintext] + |} + """.stripMargin + + NodesAndLinks(Set(scope), subNodes.links ++ upstreamLinks(wdlGraphNode, subGraphName, s"[lhead=$clusterName]")) + } + + thisLevelNodesAndLinks |+| subGraphNodesAndLinks } + private def isCallOrDeclaration(w: WdlGraphNode): Boolean = w.isInstanceOf[WdlCall] || w.isInstanceOf[Declaration] private def dotSafe(s: String) = s.replaceAllLiterally("\"", "\\\"") - private def graphName(g: GraphNode): String = g match { + private def graphName(g: WdlGraphNode): String = dotSafe(g match { case d: Declaration => val exprString = d.expression.map(e => " = " + e.toWdlString).getOrElse("") - s"${d.wdlType.toWdlString} ${d.fullyQualifiedName}$exprString" - case c: Call => - s"call ${c.fullyQualifiedName}" + s"${d.wdlType.toWdlString} ${d.unqualifiedName}$exprString" + case c: WdlCall => + s"call ${c.unqualifiedName}" case i: If => s"if (${i.condition.toWdlString})" case s: Scatter => @@ -74,5 +92,5 @@ object GraphPrint { val exprString = c.expression.map(e => " = " + e.toWdlString).getOrElse("") s"output { ${c.fullyQualifiedName}$exprString }" case other => s"${other.getClass.getSimpleName}: ${other.fullyQualifiedName}" - } + }) } diff --git a/src/main/scala/wdltool/Main.scala b/src/main/scala/wdltool/Main.scala index 28bdaef..da4231b 100644 --- a/src/main/scala/wdltool/Main.scala +++ b/src/main/scala/wdltool/Main.scala @@ -2,12 +2,11 @@ package wdltool import java.nio.file.Paths -import wdl4s.formatter.{AnsiSyntaxHighlighter, HtmlSyntaxHighlighter, SyntaxFormatter} -import wdl4s._ +import wdl4s.wdl.formatter.{AnsiSyntaxHighlighter, HtmlSyntaxHighlighter, SyntaxFormatter} +import wdl4s.wdl._ import spray.json._ - -import scala.util.{Failure, Success, Try} +import scala.util.{Failure, Success} object Main extends App { sealed trait Termination { @@ -57,7 +56,7 @@ object Main extends App { def inputs(args: Seq[String]): Termination = { continueIf(args.length == 1) { loadWdl(args.head) { namespace => - import wdl4s.types.WdlTypeJsonFormatter._ + import wdl4s.wdl.types.WdlTypeJsonFormatter._ val msg = namespace match { case x: WdlNamespaceWithWorkflow => x.workflow.inputs.toJson.prettyPrint case _ => "WDL does not have a local workflow" @@ -84,7 +83,9 @@ object Main extends App { val workflowDigraph = GraphPrint.generateWorkflowDigraph(file, allNodesMode) val result = s"""|digraph ${workflowDigraph.workflowName} { - | ${workflowDigraph.digraph.mkString(System.lineSeparator + " ")} + | compound=true; + | ${workflowDigraph.digraph.links.mkString(System.lineSeparator + " ")} + | ${workflowDigraph.digraph.nodes.mkString(System.lineSeparator + " ")} |} |""" SuccessfulTermination(result.stripMargin) diff --git a/src/test/scala/wdltool/SampleWdl.scala b/src/test/scala/wdltool/SampleWdl.scala index d2179b9..4326449 100644 --- a/src/test/scala/wdltool/SampleWdl.scala +++ b/src/test/scala/wdltool/SampleWdl.scala @@ -4,13 +4,13 @@ import java.io.{FileWriter, File} import java.nio.file.{Files, Path} import spray.json._ -import wdl4s._ -import wdl4s.values._ +import wdl4s.wdl._ +import wdl4s.wdl.values._ import scala.language.postfixOps trait SampleWdl { - def wdlSource(runtime: String = ""): WdlSource + def wdlSource(runtime: String = ""): WorkflowSource def rawInputs: WorkflowRawInputs def name = getClass.getSimpleName.stripSuffix("$") @@ -35,7 +35,7 @@ trait SampleWdl { def read(value: JsValue) = throw new NotImplementedError(s"Reading JSON not implemented: $value") } - def wdlJson: WdlJson = rawInputs.toJson.prettyPrint + def wdlJson: WorkflowJson = rawInputs.toJson.prettyPrint def createFileArray(base: Path): Unit = { createFile("f1", base, "line1\nline2\n") From af5b296189c6bc794364d721f44928319176bac4 Mon Sep 17 00:00:00 2001 From: Phil Shapiro Date: Mon, 31 Jul 2017 14:14:27 -0400 Subject: [PATCH 6/6] - fix issue where an error, such as a validation failure, wouldn't result in a non-zero exit code --- src/main/scala/wdltool/Main.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/wdltool/Main.scala b/src/main/scala/wdltool/Main.scala index da4231b..d2d0c9d 100644 --- a/src/main/scala/wdltool/Main.scala +++ b/src/main/scala/wdltool/Main.scala @@ -157,5 +157,5 @@ object Main extends App { case BadUsageTermination => Console.err.println(UsageMessage) } - termination.returnCode + sys.exit(termination.returnCode) }