diff --git a/.gitignore b/.gitignore index fa01827..387acc5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,12 @@ -tags -.idea/ -target -.artifactory +# common scala config *~ +.DS_Store +.artifactory +.idea/* +!/.idea/inspectionProfiles/ +.idea/inspectionProfiles/* +!/.idea/inspectionProfiles/Project_Default.xml +target + +# custom config +tags diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..b15d7f2 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.pullapprove.yml b/.pullapprove.yml index ceb8dc6..3ab4b6b 100644 --- a/.pullapprove.yml +++ b/.pullapprove.yml @@ -5,10 +5,12 @@ author_approval: ignored reviewers: required: 2 members: - - cjllanwarne - Horneth - - mcovarr + - cjllanwarne - geoffjentry - - kshakir + - jsotobroad + - katevoss - kcibul + - kshakir + - mcovarr - ruchim diff --git a/build.sbt b/build.sbt index a65c713..66eaed0 100644 --- a/build.sbt +++ b/build.sbt @@ -6,11 +6,13 @@ name := "wdltool" organization := "org.broadinstitute" -scalaVersion := "2.11.8" +scalaVersion := "2.12.1" + +val wdl4sV = "0.14-7c693a3-SNAP" lazy val versionSettings = Seq( // Upcoming release, or current if we're on the master branch - git.baseVersion := "0.8", + git.baseVersion := "0.14", // Shorten the git commit hash git.gitHeadCommit := git.gitHeadCommit.value map { _.take(7) }, @@ -29,15 +31,31 @@ assemblyJarName in assembly := "wdltool-" + git.baseVersion.value + ".jar" logLevel in assembly := Level.Info resolvers ++= Seq( - "Broad Artifactory Releases" at "https://artifactory.broadinstitute.org/artifactory/libs-release/", - "Broad Artifactory Snapshots" at "https://artifactory.broadinstitute.org/artifactory/libs-snapshot/" + "Broad Artifactory Releases" at "https://broadinstitute.jfrog.io/broadinstitute/libs-release/", + "Broad Artifactory Snapshots" at "https://broadinstitute.jfrog.io/broadinstitute/libs-snapshot/" ) +lazy val catsDependencies = List( + "org.typelevel" %% "cats" % "0.9.0", + "org.typelevel" %% "kittens" % "1.0.0-M10", + "com.github.benhutchison" %% "mouse" % "0.9" +) map (_ + /* + Exclude test framework cats-laws and its transitive dependency scalacheck. + If sbt detects scalacheck, it tries to run it. + Explicitly excluding the two problematic artifacts instead of including the three (or four?). + https://github.com/typelevel/cats/tree/v0.7.2#getting-started + Re "_2.12", see also: https://github.com/sbt/sbt/issues/1518 + */ + exclude("org.typelevel", "cats-laws_2.12") + exclude("org.typelevel", "cats-kernel-laws_2.12") + ) + libraryDependencies ++= Seq( - "org.broadinstitute" %% "wdl4s" % "0.8", + "org.broadinstitute" %% "wdl4s" % wdl4sV, //---------- Test libraries -------------------// - "org.scalatest" %% "scalatest" % "2.2.5" % Test -) + "org.scalatest" %% "scalatest" % "3.0.1" % Test +) ++ catsDependencies val customMergeStrategy: String => MergeStrategy = { case x if Assembly.isConfigFile(x) => diff --git a/project/plugins.sbt b/project/plugins.sbt index 20755cc..cadbd87 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,7 +2,7 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "0.8.5") addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.0") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.0.4") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.0") addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.0.0") diff --git a/src/main/scala/wdltool/GraphPrint.scala b/src/main/scala/wdltool/GraphPrint.scala index 9587468..2125c8a 100644 --- a/src/main/scala/wdltool/GraphPrint.scala +++ b/src/main/scala/wdltool/GraphPrint.scala @@ -1,70 +1,89 @@ package wdltool import java.nio.file.{Files, Paths} +import java.util.concurrent.atomic.AtomicInteger + +import wdl4s.wdl.{CallOutput, Declaration, If, Scatter, WdlCall, _} +import wdl4s.wdl.WdlGraphNode -import wdl4s.{CallOutput, Declaration, If, Scatter, _} import scala.collection.JavaConverters._ +import cats.implicits._ +import cats.derived.monoid._, legacy._ + object GraphPrint { - case class WorkflowDigraph(workflowName: String, digraph: Set[String]) + final case class WorkflowDigraph(workflowName: String, digraph: NodesAndLinks) + final case class NodesAndLinks(nodes: Set[String], links: Set[String]) def generateWorkflowDigraph(file: String, allNodesMode: Boolean): WorkflowDigraph = { - val namespace = WdlNamespaceWithWorkflow.load(Files.readAllLines(Paths.get(file)).asScala.mkString(System.lineSeparator()), Seq(WdlNamespace.fileResolver _)) + // It's ok to use .get here, we're happy to throw an exception and crash the program! + val namespace = WdlNamespaceWithWorkflow.load(Files.readAllLines(Paths.get(file)).asScala.mkString(System.lineSeparator()), Seq(WdlNamespace.fileResolver _)).get - val digraph = if (allNodesMode) { - listAllGraphNodes(namespace) - } else { - val executables = GraphPrint.listExecutableGraphNodes(namespace.workflow) - listAllGraphNodes(namespace, graphNode => executables.contains(graphNode)) - } + val digraph = listAllGraphNodes(namespace.workflow) WorkflowDigraph(namespace.workflow.unqualifiedName, digraph) } - private def defaultFilter: GraphNode => Boolean = _ => true + private val clusterCount: AtomicInteger = new AtomicInteger(0) + + + private def listAllGraphNodes(scope: Scope): NodesAndLinks = { + + val callsAndDeclarations: Set[WdlGraphNode] = (scope.children collect { + case w: WdlGraphNode if isCallOrDeclaration(w) => w + }).toSet - private def listAllGraphNodes(namespace: WdlNamespaceWithWorkflow, filter: GraphNode => Boolean = defaultFilter): Set[String] = { + val subGraphs: Set[WdlGraphNode] = (scope.children collect { + case s: Scatter => s + case i: If => i + }).toSet - val graphNodes = namespace.descendants collect { - case g: GraphNode if filter(g) => g + def upstreamLinks(wdlGraphNode: WdlGraphNode, graphNodeName: String, suffix: String = ""): Set[String] = wdlGraphNode.upstream collect { + case upstream: WdlGraphNode if isCallOrDeclaration(upstream) => + val upstreamName = graphName(upstream) + s""""$upstreamName" -> "$graphNodeName" $suffix""" } - graphNodes flatMap { graphNode => + val thisLevelNodesAndLinks: NodesAndLinks = callsAndDeclarations foldMap { graphNode => val name = graphName(graphNode) val initialSet: Set[String] = graphNode match { - case c: Call => Set(s""""${dotSafe(name)}"""") + case w: WdlGraphNode if isCallOrDeclaration(w) => Set(s""""$name"""") case _ => Set.empty } - val upstreamLinks = graphNode.upstream collect { - case upstream if filter(upstream) => - val upstreamName = graphName(upstream) - s""""${dotSafe(upstreamName)}" -> "${dotSafe(name)}"""" - } - initialSet ++ upstreamLinks + val fromStart = if (graphNode.upstream.isEmpty) Set(s""""start" -> "$name"""") else Set.empty + + NodesAndLinks(initialSet, upstreamLinks(graphNode, name) ++ fromStart) } - } - private def listExecutableGraphNodes(s: Scope): Set[GraphNode] = { - s.children.toSet flatMap { child: Scope => child match { - case call: Call => Set[GraphNode](call) - case scatter: Scatter => Set[GraphNode](scatter) ++ listExecutableGraphNodes(scatter) - case i: If => Set[GraphNode](i) ++ listExecutableGraphNodes(i) - case declaration: Declaration => Set[GraphNode](declaration) - case _ => Set.empty[GraphNode] - }} + val subGraphNodesAndLinks: NodesAndLinks = subGraphs foldMap { wdlGraphNode => + val clusterName = "cluster_" + clusterCount.getAndIncrement() + val subGraphName = graphName(wdlGraphNode) + val subNodes = listAllGraphNodes(wdlGraphNode) + val scope = s""" + |subgraph $clusterName { + | ${subNodes.nodes.mkString(sep="\n ")} + | "$subGraphName" [shape=plaintext] + |} + """.stripMargin + + NodesAndLinks(Set(scope), subNodes.links ++ upstreamLinks(wdlGraphNode, subGraphName, s"[lhead=$clusterName]")) + } + + thisLevelNodesAndLinks |+| subGraphNodesAndLinks } + private def isCallOrDeclaration(w: WdlGraphNode): Boolean = w.isInstanceOf[WdlCall] || w.isInstanceOf[Declaration] private def dotSafe(s: String) = s.replaceAllLiterally("\"", "\\\"") - private def graphName(g: GraphNode): String = g match { + private def graphName(g: WdlGraphNode): String = dotSafe(g match { case d: Declaration => val exprString = d.expression.map(e => " = " + e.toWdlString).getOrElse("") - s"${d.wdlType.toWdlString} ${d.fullyQualifiedName}$exprString" - case c: Call => - s"call ${c.fullyQualifiedName}" + s"${d.wdlType.toWdlString} ${d.unqualifiedName}$exprString" + case c: WdlCall => + s"call ${c.unqualifiedName}" case i: If => s"if (${i.condition.toWdlString})" case s: Scatter => @@ -73,5 +92,5 @@ object GraphPrint { val exprString = c.expression.map(e => " = " + e.toWdlString).getOrElse("") s"output { ${c.fullyQualifiedName}$exprString }" case other => s"${other.getClass.getSimpleName}: ${other.fullyQualifiedName}" - } + }) } diff --git a/src/main/scala/wdltool/Main.scala b/src/main/scala/wdltool/Main.scala index 04b9854..d2d0c9d 100644 --- a/src/main/scala/wdltool/Main.scala +++ b/src/main/scala/wdltool/Main.scala @@ -2,12 +2,11 @@ package wdltool import java.nio.file.Paths -import wdl4s.formatter.{AnsiSyntaxHighlighter, HtmlSyntaxHighlighter, SyntaxFormatter} -import wdl4s._ +import wdl4s.wdl.formatter.{AnsiSyntaxHighlighter, HtmlSyntaxHighlighter, SyntaxFormatter} +import wdl4s.wdl._ import spray.json._ - -import scala.util.{Failure, Success, Try} +import scala.util.{Failure, Success} object Main extends App { sealed trait Termination { @@ -57,7 +56,7 @@ object Main extends App { def inputs(args: Seq[String]): Termination = { continueIf(args.length == 1) { loadWdl(args.head) { namespace => - import wdl4s.types.WdlTypeJsonFormatter._ + import wdl4s.wdl.types.WdlTypeJsonFormatter._ val msg = namespace match { case x: WdlNamespaceWithWorkflow => x.workflow.inputs.toJson.prettyPrint case _ => "WDL does not have a local workflow" @@ -84,7 +83,9 @@ object Main extends App { val workflowDigraph = GraphPrint.generateWorkflowDigraph(file, allNodesMode) val result = s"""|digraph ${workflowDigraph.workflowName} { - | ${workflowDigraph.digraph.mkString(System.lineSeparator + " ")} + | compound=true; + | ${workflowDigraph.digraph.links.mkString(System.lineSeparator + " ")} + | ${workflowDigraph.digraph.nodes.mkString(System.lineSeparator + " ")} |} |""" SuccessfulTermination(result.stripMargin) @@ -94,7 +95,7 @@ object Main extends App { private[this] def continueIf(valid: => Boolean)(block: => Termination): Termination = if (valid) block else BadUsageTermination private[this] def loadWdl(path: String)(f: WdlNamespace => Termination): Termination = { - Try(WdlNamespace.loadUsingPath(Paths.get(path), None, None)) match { + WdlNamespace.loadUsingPath(Paths.get(path), None, None) match { case Success(namespace) => f(namespace) case Failure(t) => UnsuccessfulTermination(t.getMessage) } @@ -156,5 +157,5 @@ object Main extends App { case BadUsageTermination => Console.err.println(UsageMessage) } - termination.returnCode + sys.exit(termination.returnCode) } diff --git a/src/test/scala/wdltool/SampleWdl.scala b/src/test/scala/wdltool/SampleWdl.scala index d2179b9..4326449 100644 --- a/src/test/scala/wdltool/SampleWdl.scala +++ b/src/test/scala/wdltool/SampleWdl.scala @@ -4,13 +4,13 @@ import java.io.{FileWriter, File} import java.nio.file.{Files, Path} import spray.json._ -import wdl4s._ -import wdl4s.values._ +import wdl4s.wdl._ +import wdl4s.wdl.values._ import scala.language.postfixOps trait SampleWdl { - def wdlSource(runtime: String = ""): WdlSource + def wdlSource(runtime: String = ""): WorkflowSource def rawInputs: WorkflowRawInputs def name = getClass.getSimpleName.stripSuffix("$") @@ -35,7 +35,7 @@ trait SampleWdl { def read(value: JsValue) = throw new NotImplementedError(s"Reading JSON not implemented: $value") } - def wdlJson: WdlJson = rawInputs.toJson.prettyPrint + def wdlJson: WorkflowJson = rawInputs.toJson.prettyPrint def createFileArray(base: Path): Unit = { createFile("f1", base, "line1\nline2\n")