jfrog

build.sbt

@@ -8,11 +8,11 @@ organization := "org.broadinstitute"
 
 scalaVersion := "2.12.1"
 
-val wdl4sV = "0.12"
+val wdl4sV = "0.14-7c693a3-SNAP"
 
 lazy val versionSettings = Seq(
   // Upcoming release, or current if we're on the master branch
-  git.baseVersion := "0.12",
+  git.baseVersion := "0.14",
 
   // Shorten the git commit hash
   git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) },
@@ -31,15 +31,31 @@ assemblyJarName in assembly := "wdltool-" + git.baseVersion.value + ".jar"
 logLevel in assembly := Level.Info
 
 resolvers ++= Seq(
-  "Broad Artifactory Releases" at "https://artifactory.broadinstitute.org/artifactory/libs-release/",
-  "Broad Artifactory Snapshots" at "https://artifactory.broadinstitute.org/artifactory/libs-snapshot/"
+  "Broad Artifactory Releases" at "https://broadinstitute.jfrog.io/broadinstitute/libs-release/",
+  "Broad Artifactory Snapshots" at "https://broadinstitute.jfrog.io/broadinstitute/libs-snapshot/"
 )
 
+lazy val catsDependencies = List(
+  "org.typelevel" %% "cats" % "0.9.0",
+  "org.typelevel" %% "kittens" % "1.0.0-M10",
+  "com.github.benhutchison" %% "mouse" % "0.9"
+) map (_
+  /*
+  Exclude test framework cats-laws and its transitive dependency scalacheck.
+  If sbt detects scalacheck, it tries to run it.
+  Explicitly excluding the two problematic artifacts instead of including the three (or four?).
+  https://github.com/typelevel/cats/tree/v0.7.2#getting-started
+  Re "_2.12", see also: https://github.com/sbt/sbt/issues/1518
+   */
+  exclude("org.typelevel", "cats-laws_2.12")
+  exclude("org.typelevel", "cats-kernel-laws_2.12")
+  )
+
 libraryDependencies ++= Seq(
   "org.broadinstitute" %% "wdl4s" % wdl4sV,
   //---------- Test libraries -------------------//
   "org.scalatest" %% "scalatest" % "3.0.1" % Test
-)
+) ++ catsDependencies
 
 val customMergeStrategy: String => MergeStrategy = {
   case x if Assembly.isConfigFile(x) =>

src/main/scala/wdltool/GraphPrint.scala

@@ -1,71 +1,89 @@
 package wdltool
 
 import java.nio.file.{Files, Paths}
+import java.util.concurrent.atomic.AtomicInteger
+
+import wdl4s.wdl.{CallOutput, Declaration, If, Scatter, WdlCall, _}
+import wdl4s.wdl.WdlGraphNode
 
-import wdl4s.{CallOutput, Declaration, If, Scatter, _}
 import scala.collection.JavaConverters._
 
+import cats.implicits._
+import cats.derived.monoid._, legacy._
+
 object GraphPrint {
 
-  case class WorkflowDigraph(workflowName: String, digraph: Set[String])
+  final case class WorkflowDigraph(workflowName: String, digraph: NodesAndLinks)
+  final case class NodesAndLinks(nodes: Set[String], links: Set[String])
 
   def generateWorkflowDigraph(file: String, allNodesMode: Boolean): WorkflowDigraph = {
     // It's ok to use .get here, we're happy to throw an exception and crash the program!
     val namespace = WdlNamespaceWithWorkflow.load(Files.readAllLines(Paths.get(file)).asScala.mkString(System.lineSeparator()), Seq(WdlNamespace.fileResolver _)).get
 
-    val digraph = if (allNodesMode) {
-      listAllGraphNodes(namespace)
-    } else {
-      val executables = GraphPrint.listExecutableGraphNodes(namespace.workflow)
-      listAllGraphNodes(namespace, graphNode => executables.contains(graphNode))
-    }
+    val digraph = listAllGraphNodes(namespace.workflow)
 
     WorkflowDigraph(namespace.workflow.unqualifiedName, digraph)
   }
 
-  private def defaultFilter: GraphNode => Boolean = _ => true
+  private val clusterCount: AtomicInteger = new AtomicInteger(0)
+
+
+  private def listAllGraphNodes(scope: Scope): NodesAndLinks = {
+
+    val callsAndDeclarations: Set[WdlGraphNode] = (scope.children collect {
+      case w: WdlGraphNode if isCallOrDeclaration(w) => w
+    }).toSet
 
-  private def listAllGraphNodes(namespace: WdlNamespaceWithWorkflow, filter: GraphNode => Boolean = defaultFilter): Set[String] = {
+    val subGraphs: Set[WdlGraphNode] = (scope.children collect {
+      case s: Scatter => s
+      case i: If => i
+    }).toSet
 
-    val graphNodes = namespace.descendants collect {
-      case g: GraphNode if filter(g) => g
+    def upstreamLinks(wdlGraphNode: WdlGraphNode, graphNodeName: String, suffix: String = ""): Set[String] = wdlGraphNode.upstream collect {
+      case upstream: WdlGraphNode if isCallOrDeclaration(upstream) =>
+        val upstreamName = graphName(upstream)
+        s""""$upstreamName" -> "$graphNodeName" $suffix"""
     }
 
-    graphNodes flatMap { graphNode =>
+    val thisLevelNodesAndLinks: NodesAndLinks = callsAndDeclarations foldMap { graphNode =>
       val name = graphName(graphNode)
       val initialSet: Set[String] = graphNode match {
-        case c: Call => Set(s""""${dotSafe(name)}"""")
+        case w: WdlGraphNode if isCallOrDeclaration(w) => Set(s""""$name"""")
         case _ => Set.empty
       }
-      val upstreamLinks = graphNode.upstream collect {
-        case upstream if filter(upstream) =>
-          val upstreamName = graphName(upstream)
-          s""""${dotSafe(upstreamName)}" -> "${dotSafe(name)}""""
-      }
 
-      initialSet ++ upstreamLinks
+      val fromStart = if (graphNode.upstream.isEmpty) Set(s""""start" -> "$name"""") else Set.empty
+
+      NodesAndLinks(initialSet, upstreamLinks(graphNode, name) ++ fromStart)
     }
-  }
 
-  private def listExecutableGraphNodes(s: Scope): Set[GraphNode] = {
-    s.children.toSet flatMap { child: Scope => child match {
-      case call: Call => Set[GraphNode](call)
-      case scatter: Scatter => Set[GraphNode](scatter) ++ listExecutableGraphNodes(scatter)
-      case i: If => Set[GraphNode](i) ++ listExecutableGraphNodes(i)
-      case declaration: Declaration => Set[GraphNode](declaration)
-      case _ => Set.empty[GraphNode]
-    }}
+    val subGraphNodesAndLinks: NodesAndLinks = subGraphs foldMap { wdlGraphNode =>
+      val clusterName = "cluster_" + clusterCount.getAndIncrement()
+      val subGraphName = graphName(wdlGraphNode)
+      val subNodes = listAllGraphNodes(wdlGraphNode)
+      val scope = s"""
+         |subgraph $clusterName {
+         |  ${subNodes.nodes.mkString(sep="\n  ")}
+         |  "$subGraphName" [shape=plaintext]
+         |}
+      """.stripMargin
+
+      NodesAndLinks(Set(scope), subNodes.links ++ upstreamLinks(wdlGraphNode, subGraphName, s"[lhead=$clusterName]"))
+    }
+
+    thisLevelNodesAndLinks |+| subGraphNodesAndLinks
   }
 
+  private def isCallOrDeclaration(w: WdlGraphNode): Boolean = w.isInstanceOf[WdlCall] || w.isInstanceOf[Declaration]
 
   private def dotSafe(s: String) = s.replaceAllLiterally("\"", "\\\"")
 
-  private def graphName(g: GraphNode): String = g match {
+  private def graphName(g: WdlGraphNode): String = dotSafe(g match {
     case d: Declaration =>
       val exprString = d.expression.map(e => " = " + e.toWdlString).getOrElse("")
-      s"${d.wdlType.toWdlString} ${d.fullyQualifiedName}$exprString"
-    case c: Call =>
-      s"call ${c.fullyQualifiedName}"
+      s"${d.wdlType.toWdlString} ${d.unqualifiedName}$exprString"
+    case c: WdlCall =>
+      s"call ${c.unqualifiedName}"
     case i: If =>
       s"if (${i.condition.toWdlString})"
     case s: Scatter =>
@@ -74,5 +92,5 @@ object GraphPrint {
       val exprString = c.expression.map(e => " = " + e.toWdlString).getOrElse("")
       s"output { ${c.fullyQualifiedName}$exprString }"
     case other => s"${other.getClass.getSimpleName}: ${other.fullyQualifiedName}"
-  }
+  })
 }

src/main/scala/wdltool/Main.scala

@@ -2,12 +2,11 @@ package wdltool
 
 import java.nio.file.Paths
 
-import wdl4s.formatter.{AnsiSyntaxHighlighter, HtmlSyntaxHighlighter, SyntaxFormatter}
-import wdl4s._
+import wdl4s.wdl.formatter.{AnsiSyntaxHighlighter, HtmlSyntaxHighlighter, SyntaxFormatter}
+import wdl4s.wdl._
 import spray.json._
 
-
-import scala.util.{Failure, Success, Try}
+import scala.util.{Failure, Success}
 
 object Main extends App {
   sealed trait Termination {
@@ -57,7 +56,7 @@ object Main extends App {
   def inputs(args: Seq[String]): Termination = {
     continueIf(args.length == 1) {
       loadWdl(args.head) { namespace =>
-        import wdl4s.types.WdlTypeJsonFormatter._
+        import wdl4s.wdl.types.WdlTypeJsonFormatter._
         val msg = namespace match {
           case x: WdlNamespaceWithWorkflow => x.workflow.inputs.toJson.prettyPrint
           case _ => "WDL does not have a local workflow"
@@ -84,7 +83,9 @@ object Main extends App {
       val workflowDigraph = GraphPrint.generateWorkflowDigraph(file, allNodesMode)
 
       val result = s"""|digraph ${workflowDigraph.workflowName} {
-                       |  ${workflowDigraph.digraph.mkString(System.lineSeparator + "  ")}
+                       |  compound=true;
+                       |  ${workflowDigraph.digraph.links.mkString(System.lineSeparator + "  ")}
+                       |  ${workflowDigraph.digraph.nodes.mkString(System.lineSeparator + "  ")}
                        |}
                        |"""
       SuccessfulTermination(result.stripMargin)
@@ -156,5 +157,5 @@ object Main extends App {
     case BadUsageTermination => Console.err.println(UsageMessage)
   }
 
-  termination.returnCode
+  sys.exit(termination.returnCode)
 }

src/test/scala/wdltool/SampleWdl.scala

@@ -4,13 +4,13 @@ import java.io.{FileWriter, File}
 import java.nio.file.{Files, Path}
 import spray.json._
 
-import wdl4s._
-import wdl4s.values._
+import wdl4s.wdl._
+import wdl4s.wdl.values._
 
 import scala.language.postfixOps
 
 trait SampleWdl {
-  def wdlSource(runtime: String = ""): WdlSource
+  def wdlSource(runtime: String = ""): WorkflowSource
   def rawInputs: WorkflowRawInputs
 
   def name = getClass.getSimpleName.stripSuffix("$")
@@ -35,7 +35,7 @@ trait SampleWdl {
     def read(value: JsValue) = throw new NotImplementedError(s"Reading JSON not implemented: $value")
   }
 
-  def wdlJson: WdlJson = rawInputs.toJson.prettyPrint
+  def wdlJson: WorkflowJson = rawInputs.toJson.prettyPrint
 
   def createFileArray(base: Path): Unit = {
     createFile("f1", base, "line1\nline2\n")