Update WDL site URL in README

.gitignore

@@ -1,5 +1,12 @@
-tags
-.idea/
-target
-.artifactory
+# common scala config
 *~
+.DS_Store
+.artifactory
+.idea/*
+!/.idea/inspectionProfiles/
+.idea/inspectionProfiles/*
+!/.idea/inspectionProfiles/Project_Default.xml
+target
+
+# custom config
+tags

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="TypeAnnotation" enabled="false" level="WARNING" enabled_by_default="false" />
+  </profile>
+</component>

.pullapprove.yml

@@ -5,9 +5,12 @@ author_approval: ignored
 reviewers:
     required: 2
     members:
-        - cjllanwarne
         - Horneth
-        - scottfrazer
-        - mcovarr
+        - cjllanwarne
         - geoffjentry
+        - jsotobroad
+        - katevoss
+        - kcibul
         - kshakir
+        - mcovarr
+        - ruchim

README.md

@@ -165,6 +165,109 @@ $ java -jar wdltool.jar highlight test.wdl html
 }
 ```
 
+## graph 
+ 
+The syntax of the graph command is:
+```
+wdltool graph [--all] wdlFile.wdl
+```
+
+Given a WDL file input, command generates the data-flow graph through the system in `.dot` format.
+
+For example the fork-join WDL:
+```
+task mkFile {
+  command {
+    for i in `seq 1 1000`
+    do
+      echo $i
+    done
+  }
+  output {
+    File numbers = stdout()
+  }
+  runtime {docker: "ubuntu:latest"}
+}
+
+task grep {
+  String pattern
+  File in_file
+  command {
+    grep '${pattern}' ${in_file} | wc -l
+  }
+  output {
+    Int count = read_int(stdout())
+  }
+  runtime {docker: "ubuntu:latest"}
+}
+
+task wc {
+  File in_file
+  command {
+    cat ${in_file} | wc -l
+  }
+  output {
+    Int count = read_int(stdout())
+  }
+  runtime {docker: "ubuntu:latest"}
+}
+
+task join {
+  Int grepCount
+  Int wcCount
+  command {
+    expr ${wcCount} / ${grepCount}
+  }
+  output {
+    Int proportion = read_int(stdout())
+  }
+  runtime {docker: "ubuntu:latest"}
+}
+
+workflow forkjoin {
+  call mkFile
+  call grep { input: in_file = mkFile.numbers }
+  call wc { input: in_file=mkFile.numbers }
+  call join { input: wcCount = wc.count, grepCount = grep.count }
+  output {
+    join.proportion
+  }
+}
+```
+
+Produces the DAG:
+```
+digraph forkjoin {
+  "call forkjoin.mkFile" -> "call forkjoin.wc"
+  "call forkjoin.mkFile" -> "call forkjoin.grep"
+  "call forkjoin.wc" -> "call forkjoin.join"
+  "call forkjoin.grep" -> "call forkjoin.join"
+}
+```
+
+### The --all flag
+
+If this flag is set, all WDL graph nodes become nodes in the generated DAG, even if they are not "executed". Typically this will mean task declarations and call outputs. 
+For example in the above example, with `--all` you would get:
+
+```
+digraph forkjoin {
+  "call forkjoin.grep" -> "String forkjoin.grep.pattern"
+  "call forkjoin.grep" -> "output { forkjoin.grep.count = read_int(stdout()) }"
+  "call forkjoin.grep" -> "File forkjoin.grep.in_file"
+  "call forkjoin.wc" -> "output { forkjoin.wc.count = read_int(stdout()) }"
+  "call forkjoin.grep" -> "call forkjoin.join"
+  "call forkjoin.wc" -> "File forkjoin.wc.in_file"
+  "call forkjoin.mkFile" -> "call forkjoin.grep"
+  "call forkjoin.join" -> "output { forkjoin.join.proportion = read_int(stdout()) }"
+  "call forkjoin.join" -> "Int forkjoin.join.wcCount"
+  "call forkjoin.wc" -> "call forkjoin.join"
+  "call forkjoin.mkFile" -> "output { forkjoin.mkFile.numbers = stdout() }"
+  "call forkjoin.mkFile" -> "call forkjoin.wc"
+  "call forkjoin.join" -> "Int forkjoin.join.grepCount"
+}
+```
+
 # Getting Started with WDL
 
-For many examples on how to use WDL see [the WDL site](https://github.com/broadinstitute/wdl/tree/develop#getting-started-with-wdl)
+For documentation and many examples on how to use WDL see [the WDL website](https://software.broadinstitute.org/wdl/).

build.sbt

@@ -1,39 +1,61 @@
 import com.typesafe.sbt.GitPlugin.autoImport._
 import sbt.Keys._
 import sbtassembly.MergeStrategy
-import com.typesafe.sbt.SbtGit.GitCommand
 
 name := "wdltool"
 
 organization := "org.broadinstitute"
 
-scalaVersion := "2.11.7"
+scalaVersion := "2.12.1"
 
-// Upcoming release, or current if we're on the master branch
-git.baseVersion := "0.4"
+val wdl4sV = "0.14-7c693a3-SNAP"
 
-// Shorten the git commit hash
-git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) }
+lazy val versionSettings = Seq(
+  // Upcoming release, or current if we're on the master branch
+  git.baseVersion := "0.14",
 
-// Travis will deploy tagged releases, add -SNAPSHOT for all local builds
-git.gitUncommittedChanges := true
+  // Shorten the git commit hash
+  git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) },
 
-versionWithGit
+  // Travis will deploy tagged releases, add -SNAPSHOT for all local builds
+  git.gitUncommittedChanges := true,
+
+  // For now, obfuscate SNAPSHOTs from sbt's developers: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241
+  git.uncommittedSignifier := Option("SNAP")
+)
+
+versionWithGit ++ versionSettings
 
 assemblyJarName in assembly := "wdltool-" + git.baseVersion.value + ".jar"
 
 logLevel in assembly := Level.Info
 
 resolvers ++= Seq(
-  "Broad Artifactory Releases" at "https://artifactory.broadinstitute.org/artifactory/libs-release/",
-  "Broad Artifactory Snapshots" at "https://artifactory.broadinstitute.org/artifactory/libs-snapshot/"
+  "Broad Artifactory Releases" at "https://broadinstitute.jfrog.io/broadinstitute/libs-release/",
+  "Broad Artifactory Snapshots" at "https://broadinstitute.jfrog.io/broadinstitute/libs-snapshot/"
 )
 
+lazy val catsDependencies = List(
+  "org.typelevel" %% "cats" % "0.9.0",
+  "org.typelevel" %% "kittens" % "1.0.0-M10",
+  "com.github.benhutchison" %% "mouse" % "0.9"
+) map (_
+  /*
+  Exclude test framework cats-laws and its transitive dependency scalacheck.
+  If sbt detects scalacheck, it tries to run it.
+  Explicitly excluding the two problematic artifacts instead of including the three (or four?).
+  https://github.com/typelevel/cats/tree/v0.7.2#getting-started
+  Re "_2.12", see also: https://github.com/sbt/sbt/issues/1518
+   */
+  exclude("org.typelevel", "cats-laws_2.12")
+  exclude("org.typelevel", "cats-kernel-laws_2.12")
+  )
+
 libraryDependencies ++= Seq(
-  "org.broadinstitute" %% "wdl4s" % "0.4",
+  "org.broadinstitute" %% "wdl4s" % wdl4sV,
   //---------- Test libraries -------------------//
-  "org.scalatest" %% "scalatest" % "2.2.5" % Test
-)
+  "org.scalatest" %% "scalatest" % "3.0.1" % Test
+) ++ catsDependencies
 
 val customMergeStrategy: String => MergeStrategy = {
   case x if Assembly.isConfigFile(x) =>

project/build.properties

@@ -0,0 +1 @@
+sbt.version=0.13.12

project/plugins.sbt

@@ -2,7 +2,7 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "0.8.5")
 
 addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.0")
 
-addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.0.4")
+addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.0")
 
 addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.0.0")
 

src/main/scala/wdltool/GraphPrint.scala

@@ -0,0 +1,96 @@
+package wdltool
+
+import java.nio.file.{Files, Paths}
+import java.util.concurrent.atomic.AtomicInteger
+
+import wdl4s.wdl.{CallOutput, Declaration, If, Scatter, WdlCall, _}
+import wdl4s.wdl.WdlGraphNode
+
+import scala.collection.JavaConverters._
+
+import cats.implicits._
+import cats.derived.monoid._, legacy._
+
+object GraphPrint {
+
+  final case class WorkflowDigraph(workflowName: String, digraph: NodesAndLinks)
+  final case class NodesAndLinks(nodes: Set[String], links: Set[String])
+
+  def generateWorkflowDigraph(file: String, allNodesMode: Boolean): WorkflowDigraph = {
+    // It's ok to use .get here, we're happy to throw an exception and crash the program!
+    val namespace = WdlNamespaceWithWorkflow.load(Files.readAllLines(Paths.get(file)).asScala.mkString(System.lineSeparator()), Seq(WdlNamespace.fileResolver _)).get
+
+    val digraph = listAllGraphNodes(namespace.workflow)
+
+    WorkflowDigraph(namespace.workflow.unqualifiedName, digraph)
+  }
+
+  private val clusterCount: AtomicInteger = new AtomicInteger(0)
+
+
+  private def listAllGraphNodes(scope: Scope): NodesAndLinks = {
+
+    val callsAndDeclarations: Set[WdlGraphNode] = (scope.children collect {
+      case w: WdlGraphNode if isCallOrDeclaration(w) => w
+    }).toSet
+
+    val subGraphs: Set[WdlGraphNode] = (scope.children collect {
+      case s: Scatter => s
+      case i: If => i
+    }).toSet
+
+    def upstreamLinks(wdlGraphNode: WdlGraphNode, graphNodeName: String, suffix: String = ""): Set[String] = wdlGraphNode.upstream collect {
+      case upstream: WdlGraphNode if isCallOrDeclaration(upstream) =>
+        val upstreamName = graphName(upstream)
+        s""""$upstreamName" -> "$graphNodeName" $suffix"""
+    }
+
+    val thisLevelNodesAndLinks: NodesAndLinks = callsAndDeclarations foldMap { graphNode =>
+      val name = graphName(graphNode)
+      val initialSet: Set[String] = graphNode match {
+        case w: WdlGraphNode if isCallOrDeclaration(w) => Set(s""""$name"""")
+        case _ => Set.empty
+      }
+
+      val fromStart = if (graphNode.upstream.isEmpty) Set(s""""start" -> "$name"""") else Set.empty
+
+      NodesAndLinks(initialSet, upstreamLinks(graphNode, name) ++ fromStart)
+    }
+
+    val subGraphNodesAndLinks: NodesAndLinks = subGraphs foldMap { wdlGraphNode =>
+      val clusterName = "cluster_" + clusterCount.getAndIncrement()
+      val subGraphName = graphName(wdlGraphNode)
+      val subNodes = listAllGraphNodes(wdlGraphNode)
+      val scope = s"""
+         |subgraph $clusterName {
+         |  ${subNodes.nodes.mkString(sep="\n  ")}
+         |  "$subGraphName" [shape=plaintext]
+         |}
+      """.stripMargin
+
+      NodesAndLinks(Set(scope), subNodes.links ++ upstreamLinks(wdlGraphNode, subGraphName, s"[lhead=$clusterName]"))
+    }
+
+    thisLevelNodesAndLinks |+| subGraphNodesAndLinks
+  }
+
+  private def isCallOrDeclaration(w: WdlGraphNode): Boolean = w.isInstanceOf[WdlCall] || w.isInstanceOf[Declaration]
+
+  private def dotSafe(s: String) = s.replaceAllLiterally("\"", "\\\"")
+
+  private def graphName(g: WdlGraphNode): String = dotSafe(g match {
+    case d: Declaration =>
+      val exprString = d.expression.map(e => " = " + e.toWdlString).getOrElse("")
+      s"${d.wdlType.toWdlString} ${d.unqualifiedName}$exprString"
+    case c: WdlCall =>
+      s"call ${c.unqualifiedName}"
+    case i: If =>
+      s"if (${i.condition.toWdlString})"
+    case s: Scatter =>
+      s"scatter (${s.item} in ${s.collection.toWdlString})"
+    case c: CallOutput =>
+      val exprString = c.expression.map(e => " = " + e.toWdlString).getOrElse("")
+      s"output { ${c.fullyQualifiedName}$exprString }"
+    case other => s"${other.getClass.getSimpleName}: ${other.fullyQualifiedName}"
+  })
+}