From 308f087287f57585de89c7c41f371db031fb2d7f Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Fri, 16 Jun 2017 11:58:52 -0400 Subject: [PATCH] Command line parsing redo. --- CHANGELOG.md | 3 + README.md | 202 ++++++++++++++++++++- project/Dependencies.scala | 3 +- src/bin/travis/testCentaurJes.sh | 2 +- src/main/scala/cromwell/CommandLineParser.scala | 107 +++++++++++ src/main/scala/cromwell/CromwellCommandLine.scala | 154 ---------------- src/main/scala/cromwell/CromwellEntryPoint.scala | 192 ++++++++++++++++++++ src/main/scala/cromwell/Main.scala | 151 --------------- .../scala/cromwell/CromwellCommandLineSpec.scala | 106 ++++++----- 9 files changed, 570 insertions(+), 350 deletions(-) create mode 100644 src/main/scala/cromwell/CommandLineParser.scala delete mode 100644 src/main/scala/cromwell/CromwellCommandLine.scala create mode 100644 src/main/scala/cromwell/CromwellEntryPoint.scala delete mode 100644 src/main/scala/cromwell/Main.scala diff --git a/CHANGELOG.md b/CHANGELOG.md index f613fd567..5b79db90d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ * Request timeouts for HTTP requests on the REST API now return a 503 status code instead of 500. The response for a request timeout is no longer in JSON format. +* Command line usage has been extensively revised for Cromwell 29. Please see the +[README](https://github.com/broadinstitute/cromwell#command-line-usage) for details. + ## 28 ### Bug Fixes diff --git a/README.md b/README.md index 57a03b777..3b589e977 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management * [Building](#building) * [Installing](#installing) * [Upgrading from 0.19 to 0.21](#upgrading-from-019-to-021) -* [**NEW** Command Line Usage](http://gatkforums.broadinstitute.org/wdl/discussion/8782/command-line-cromwell) (on the WDL/Cromwell Website) +* [Command Line Usage](#command-line-usage) * [Getting Started with WDL](#getting-started-with-wdl) * [WDL Support](#wdl-support) * [Configuring Cromwell](#configuring-cromwell) @@ -138,6 +138,206 @@ OS X users can install Cromwell with Homebrew: `brew install cromwell`. See the [migration document](MIGRATION.md) for more details. +# Command Line Usage + +For built-in documentation of Cromwell command line usage, run the Cromwell JAR file with no arguments: + +``` +$ java -jar cromwell-.jar +``` + +For example, `$ java -jar cromwell-29.jar`. You will get a usage message like the following: + +``` +cromwell 29 +Usage: java -jar /path/to/cromwell.jar [server|run] [options] ... + + --help Cromwell - Workflow Execution Engine + --version +Command: server +Starts a web server on port 8000. See the web server documentation for more details about the API endpoints. +Command: run [options] workflow-source +Run the workflow and print out the outputs in JSON format. + workflow-source Workflow source file. + -i, --inputs Workflow inputs file. + -o, --options Workflow options file. + -t, --type Workflow type. + -v, --type-version + Workflow type version. + -l, --labels Workflow labels file. + -p, --imports A directory or zipfile to search for workflow imports. + -m, --metadata-output + An optional directory path to output metadata. +``` + +## --version + +The `--version` option prints the version of Cromwell and exits. + +## --help + +The `--help` option prints the full help text above and exits. + +## server + +The `server` command runs Cromwell as a web server. No arguments are accepted. +See the documentation for Cromwell's REST endpoints [here](#rest-api). + +## run + +The `run` command executes a single workflow in Cromwell. + +### workflow-source +The `run` command requires a single argument for the workflow source file. + +### --inputs +An optional file of workflow inputs. Although optional, it is a best practice to use an inputs file to satisfy workflow +requirements rather than hardcoding inputs directly into a workflow source file. + +### --options +An optional file of workflow options. Some options are global (supported by all backends), while others are backend-specific. +See the [workflow options](#workflow-options) documentation for more details. + +### --type +An optional parameter to specify the language for the workflow source. Any value specified for this parameter is currently +ignored and internally the value `WDL` is used. + +### --type-version +An optional parameter to specify the version of the language for the workflow source. Currently any specified value is ignored. + +### --labels +An optional parameter to specify a file of JSON key-value label pairs to associate with the workflow. + +### --imports +You have the option of importing WDL workflows or tasks to use within your workflow, known as sub-workflows. +If you use sub-workflows within your primary workflow then you must include a zip file with the WDL import files. + +For example, say you have a directory of WDL files: + +``` +wdl_library +└──cgrep.wdl +└──ps.wdl +└──wc.wdl +``` + +If you zip that directory into `wdl_library.zip`, then you can reference and use these WDLs within your primary WDL. + +This could be your primary WDL: + +``` +import "ps.wdl" as ps +import "cgrep.wdl" +import "wc.wdl" as wordCount + +workflow my_wf { + +call ps.ps as getStatus +call cgrep.cgrep { input: str = getStatus.x } +call wordCount { input: str = ... } + +} +``` + +Then to run this WDL without any inputs, workflow options, or metadata files, you would enter: + +`$ java -jar cromwell-.jar run my_wf.wdl --imports /path/to/wdl_library.zip` + +### --metadata-output + +You can include a path where Cromwell will write the workflow metadata JSON, such as start/end timestamps, status, inputs, and outputs. By default, Cromwell does not write workflow metadata. + +This example includes a metadata path called `/path/to/my_wf.metadata`: + +``` +$ java -jar cromwell-.jar run my_wf.wdl --metadata-output /path/to/my_wf.metadata +``` + +Again, Cromwell is very verbose. Here is the metadata output in my_wf.metadata: + +``` +{ + "workflowName": "my_wf", + "submittedFiles": { + "inputs": "{\"my_wf.hello.addressee\":\"m'Lord\"}", + "workflow": "\ntask hello {\n String addressee\n command {\n echo \"Hello ${addressee}!\"\n }\n output {\n String salutation = read_string(stdout())\n }\n runtime {\n +\n }\n}\n\nworkflow my_wf {\n call hello\n output {\n hello.salutation\n }\n}\n", + "options": "{\n\n}" + }, + "calls": { + "my_wf.hello": [ + { + "executionStatus": "Done", + "stdout": "/Users/jdoe/Documents/cromwell-executions/my_wf/cd0fe94a-984e-4a19-ab4c-8f7f07038068/call-hello/execution/stdout", + "backendStatus": "Done", + "shardIndex": -1, + "outputs": { + "salutation": "Hello m'Lord!" + }, + "runtimeAttributes": { + "continueOnReturnCode": "0", + "failOnStderr": "false" + }, + "callCaching": { + "allowResultReuse": false, + "effectiveCallCachingMode": "CallCachingOff" + }, + "inputs": { + "addressee": "m'Lord" + }, + "returnCode": 0, + "jobId": "28955", + "backend": "Local", + "end": "2017-04-19T10:53:25.045-04:00", + "stderr": "/Users/jdoe/Documents/cromwell-executions/my_wf/cd0fe94a-984e-4a19-ab4c-8f7f07038068/call-hello/execution/stderr", + "callRoot": "/Users/jdoe/Documents/cromwell-executions/my_wf/cd0fe94a-984e-4a19-ab4c-8f7f07038068/call-hello", + "attempt": 1, + "executionEvents": [ + { + "startTime": "2017-04-19T10:53:23.570-04:00", + "description": "PreparingJob", + "endTime": "2017-04-19T10:53:23.573-04:00" + }, + { + "startTime": "2017-04-19T10:53:23.569-04:00", + "description": "Pending", + "endTime": "2017-04-19T10:53:23.570-04:00" + }, + { + "startTime": "2017-04-19T10:53:25.040-04:00", + "description": "UpdatingJobStore", + "endTime": "2017-04-19T10:53:25.045-04:00" + }, + { + "startTime": "2017-04-19T10:53:23.570-04:00", + "description": "RequestingExecutionToken", + "endTime": "2017-04-19T10:53:23.570-04:00" + }, + { + "startTime": "2017-04-19T10:53:23.573-04:00", + "description": "RunningJob", + "endTime": "2017-04-19T10:53:25.040-04:00" + } + ], + "start": "2017-04-19T10:53:23.569-04:00" + } + ] + }, + "outputs": { + "my_wf.hello.salutation": "Hello m'Lord!" + }, + "workflowRoot": "/Users/jdoe/Documents/cromwell-executions/my_wf/cd0fe94a-984e-4a19-ab4c-8f7f07038068", + "id": "cd0fe94a-984e-4a19-ab4c-8f7f07038068", + "inputs": { + "my_wf.hello.addressee": "m'Lord" + }, + "submission": "2017-04-19T10:53:19.565-04:00", + "status": "Succeeded", + "end": "2017-04-19T10:53:25.063-04:00", + "start": "2017-04-19T10:53:23.535-04:00" +} +``` + # Getting Started with WDL For many examples on how to use WDL see [the WDL site](https://github.com/broadinstitute/wdl#getting-started-with-wdl) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 86a34562e..eec5ddb6a 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -127,7 +127,8 @@ object Dependencies { "com.google.guava" % "guava" % "22.0", "com.google.auth" % "google-auth-library-oauth2-http" % "0.7.0", "com.typesafe.akka" %% "akka-stream-testkit" % akkaV, - "com.chuusai" %% "shapeless" % "2.3.2" + "com.chuusai" %% "shapeless" % "2.3.2", + "com.github.scopt" %% "scopt" % "3.6.0" ) ++ baseDependencies ++ googleApiClientDependencies ++ // TODO: We're not using the "F" in slf4j. Core only supports logback, specifically the WorkflowLogger. slf4jBindingDependencies diff --git a/src/bin/travis/testCentaurJes.sh b/src/bin/travis/testCentaurJes.sh index aa398e2b2..26ef67663 100755 --- a/src/bin/travis/testCentaurJes.sh +++ b/src/bin/travis/testCentaurJes.sh @@ -110,7 +110,7 @@ fi JAR_GCS_PATH=gs://cloud-cromwell-dev/travis-centaur/${CROMWELL_JAR} gsutil cp target/scala-2.12/cromwell-*.jar "${JAR_GCS_PATH}" -java -Dconfig.file=./jes.conf -jar target/scala-2.12/cromwell-*.jar run src/bin/travis/resources/centaur.wdl src/bin/travis/resources/centaur.inputs | tee log.txt +java -Dconfig.file=./jes.conf -jar target/scala-2.12/cromwell-*.jar run src/bin/travis/resources/centaur.wdl --inputs src/bin/travis/resources/centaur.inputs | tee log.txt EXIT_CODE="${PIPESTATUS[0]}" # The perl code below is to remove our lovely color highlighting diff --git a/src/main/scala/cromwell/CommandLineParser.scala b/src/main/scala/cromwell/CommandLineParser.scala new file mode 100644 index 000000000..da909505e --- /dev/null +++ b/src/main/scala/cromwell/CommandLineParser.scala @@ -0,0 +1,107 @@ +package cromwell + +import com.typesafe.config.ConfigFactory +import cromwell.core.path.{DefaultPathBuilder, Path} +import scopt.OptionParser + +object CommandLineParser extends App { + + sealed trait Command + case object Run extends Command + case object Server extends Command + + case class CommandLineArguments(command: Option[Command] = None, + workflowSource: Option[Path] = None, + workflowInputs: Option[Path] = None, + workflowOptions: Option[Path] = None, + workflowType: Option[String] = Option("WDL"), + workflowTypeVersion: Option[String] = Option("v2.0-draft"), + workflowLabels: Option[Path] = None, + imports: Option[Path] = None, + metadataOutput: Option[Path] = None + ) + + lazy val cromwellVersion = ConfigFactory.load("cromwell-version.conf").getConfig("version").getString("cromwell") + + case class ParserAndCommand(parser: OptionParser[CommandLineArguments], command: Option[Command]) + + // cromwell 29 + // Usage: java -jar /path/to/cromwell.jar [server|run] [options] ... + // + // --help Cromwell - Workflow Execution Engine + // --version + // Command: server + // Starts a web server on port 8000. See the web server documentation for more details about the API endpoints. + // Command: run [options] workflow-source + // Run the workflow and print out the outputs in JSON format. + // workflow-source Workflow source file. + // -i, --inputs Workflow inputs file. + // -o, --options Workflow options file. + // -t, --type Workflow type. + // -v, --type-version + // Workflow type version. + // -l, --labels Workflow labels file. + // -p, --imports A directory or zipfile to search for workflow imports. + // -m, --metadata-output + // An optional directory path to output metadata. + + def buildParser(): scopt.OptionParser[CommandLineArguments] = { + new scopt.OptionParser[CommandLineArguments]("java -jar /path/to/cromwell.jar") { + head("cromwell", cromwellVersion) + + help("help").text("Cromwell - Workflow Execution Engine") + + version("version") + + cmd("server").action((_, c) => c.copy(command = Option(Server))).text( + "Starts a web server on port 8000. See the web server documentation for more details about the API endpoints.") + + cmd("run"). + action((_, c) => c.copy(command = Option(Run))). + text("Run the workflow and print out the outputs in JSON format."). + children( + arg[String]("workflow-source").text("Workflow source file.").required(). + action((s, c) => c.copy(workflowSource = Option(DefaultPathBuilder.get(s)))), + opt[String]('i', "inputs").text("Workflow inputs file."). + action((s, c) => + c.copy(workflowInputs = Option(DefaultPathBuilder.get(s)))), + opt[String]('o', "options").text("Workflow options file."). + action((s, c) => + c.copy(workflowOptions = Option(DefaultPathBuilder.get(s)))), + opt[String]('t', "type").text("Workflow type."). + action((s, c) => + c.copy(workflowType = Option(s))), + opt[String]('v', "type-version").text("Workflow type version."). + action((s, c) => + c.copy(workflowTypeVersion = Option(s))), + opt[String]('l', "labels").text("Workflow labels file."). + action((s, c) => + c.copy(workflowLabels = Option(DefaultPathBuilder.get(s)))), + opt[String]('p', "imports").text( + "A directory or zipfile to search for workflow imports."). + action((s, c) => + c.copy(imports = Option(DefaultPathBuilder.get(s)))), + opt[String]('m', "metadata-output").text( + "An optional directory path to output metadata."). + action((s, c) => + c.copy(metadataOutput = Option(DefaultPathBuilder.get(s)))) + ) + } + } + + def runCromwell(args: CommandLineArguments): Unit = { + args.command match { + case Some(Run) => CromwellEntryPoint.runSingle(args) + case Some(Server) => CromwellEntryPoint.runServer() + case None => parser.showUsage() + } + } + + val parser = buildParser() + + val parsedArgs = parser.parse(args, CommandLineArguments()) + parsedArgs match { + case Some(pa) => runCromwell(pa) + case None => parser.showUsage() + } +} diff --git a/src/main/scala/cromwell/CromwellCommandLine.scala b/src/main/scala/cromwell/CromwellCommandLine.scala deleted file mode 100644 index 77b82a83b..000000000 --- a/src/main/scala/cromwell/CromwellCommandLine.scala +++ /dev/null @@ -1,154 +0,0 @@ -package cromwell - -import cats.data.Validated._ -import cats.syntax.cartesian._ -import cats.syntax.validated._ -import cromwell.core.path.{DefaultPathBuilder, Path} -import cromwell.core.{WorkflowSourceFilesCollection, WorkflowSourceFilesWithDependenciesZip, WorkflowSourceFilesWithoutImports} -import lenthall.exception.MessageAggregation -import lenthall.validation.ErrorOr._ -import org.slf4j.LoggerFactory - -import scala.util.{Failure, Success, Try} - -sealed abstract class CromwellCommandLine -case object UsageAndExit extends CromwellCommandLine -case object RunServer extends CromwellCommandLine -case object VersionAndExit extends CromwellCommandLine - -object CromwellCommandLine { - def apply(args: Seq[String]): CromwellCommandLine = { - args.headOption match { - case Some("server") if args.size == 1 => RunServer - case Some("run") if args.size >= 2 && args.size <= 6 => RunSingle(args.tail) - case Some("-version") if args.size == 1 => VersionAndExit - case _ => UsageAndExit - } - } -} - -// We cannot initialize the logging until after we parse the command line in Main.scala. So we have to bundle up and pass back this information, just for logging. -case class SingleRunPathParameters(wdlPath: Path, inputsPath: Option[Path], optionsPath: Option[Path], metadataPath: Option[Path], importPath: Option[Path], labelsPath: Option[Path]) { - def logMe(log: org.slf4j.Logger) = { - log.info(s" WDL file: $wdlPath") - inputsPath foreach { i => log.info(s" Inputs: $i") } - optionsPath foreach { o => log.info(s" Workflow Options: $o") } - metadataPath foreach { m => log.info(s" Workflow Metadata Output: $m") } - importPath foreach { i => log.info(s" WDL import bundle: $i") } - labelsPath foreach { o => log.info(s" Custom labels: $o") } - } -} - -final case class RunSingle(sourceFiles: WorkflowSourceFilesCollection, paths: SingleRunPathParameters) extends CromwellCommandLine - -object RunSingle { - - lazy val Log = LoggerFactory.getLogger("cromwell") - - def apply(args: Seq[String]): RunSingle = { - val pathParameters = SingleRunPathParameters( - wdlPath = DefaultPathBuilder.get(args.head).toAbsolutePath, - inputsPath = argPath(args, 1, Option("inputs"), checkDefaultExists = false), - optionsPath = argPath(args, 2, Option("options")), - metadataPath = argPath(args, 3, None), - importPath = argPath(args, 4, None), - labelsPath = argPath(args, 5, None) - ) - - val wdl = readContent("WDL file", pathParameters.wdlPath) - val inputsJson = readJson("Inputs", pathParameters.inputsPath) - val optionsJson = readJson("Workflow Options", pathParameters.optionsPath) - val labelsJson = readJson("Labels", pathParameters.labelsPath) - - val sourceFileCollection = pathParameters.importPath match { - case Some(p) => (wdl |@| inputsJson |@| optionsJson |@| labelsJson) map { (w, i, o, l) => - WorkflowSourceFilesWithDependenciesZip.apply( - workflowSource = w, - workflowType = Option("WDL"), - workflowTypeVersion = None, - inputsJson = i, - workflowOptionsJson = o, - labelsJson = l, - importsZip = p.loadBytes) - } - case None => (wdl |@| inputsJson |@| optionsJson |@| labelsJson) map { (w, i, o, l) => - WorkflowSourceFilesWithoutImports.apply( - workflowSource = w, - workflowType = Option("WDL"), - workflowTypeVersion = None, - inputsJson = i, - workflowOptionsJson = o, - labelsJson = l - ) - } - } - - val runSingle: ErrorOr[RunSingle] = for { - sources <- sourceFileCollection - _ <- writeableMetadataPath(pathParameters.metadataPath) - } yield RunSingle(sources, pathParameters) - - runSingle match { - case Valid(r) => r - case Invalid(nel) => throw new RuntimeException with MessageAggregation { - override def exceptionContext: String = "ERROR: Unable to run Cromwell:" - override def errorMessages: Traversable[String] = nel.toList - } - } - } - - private def writeableMetadataPath(path: Option[Path]): ErrorOr[Unit] = { - path match { - case Some(p) if !metadataPathIsWriteable(p) => s"Unable to write to metadata directory: $p".invalidNel - case _ => ().validNel - } - } - - /** Read the path to a string. */ - private def readContent(inputDescription: String, path: Path): ErrorOr[String] = { - if (!path.exists) { - s"$inputDescription does not exist: $path".invalidNel - } else if (!path.isReadable) { - s"$inputDescription is not readable: $path".invalidNel - } else path.contentAsString.validNel - } - - /** Read the path to a string, unless the path is None, in which case returns "{}". */ - private def readJson(inputDescription: String, pathOption: Option[Path]): ErrorOr[String] = { - pathOption match { - case Some(path) => readContent(inputDescription, path) - case None => "{}".validNel - } - } - - private def metadataPathIsWriteable(metadataPath: Path): Boolean = { - Try(metadataPath.createIfNotExists(createParents = true).append("")) match { - case Success(_) => true - case Failure(_) => false - } - } - - /** - * Retrieve the arg at index as path, or return some default. Args specified as "-" will be returned as None. - * - * @param args The run command arguments, with the wdl path at arg.head. - * @param index The index of the path we're looking for. - * @param defaultExt The default extension to use if the argument was not specified at all. - * @param checkDefaultExists If true, verify that our computed default file exists before using it. - * @return The argument as a Path resolved as a sibling to the wdl path. - */ - private def argPath(args: Seq[String], index: Int, defaultExt: Option[String], - checkDefaultExists: Boolean = true): Option[Path] = { - - // To return a default, swap the extension, and then maybe check if the file exists. - def defaultPath = defaultExt - .map(ext => DefaultPathBuilder.get(args.head).swapExt("wdl", ext)) - .filter(path => !checkDefaultExists || path.exists) - .map(_.pathAsString) - - // Return the path for the arg index, or the default, but remove "-" paths. - for { - path <- args.lift(index) orElse defaultPath filterNot (_ == "-") - } yield DefaultPathBuilder.get(path).toAbsolutePath - } -} diff --git a/src/main/scala/cromwell/CromwellEntryPoint.scala b/src/main/scala/cromwell/CromwellEntryPoint.scala new file mode 100644 index 000000000..780cfb7a3 --- /dev/null +++ b/src/main/scala/cromwell/CromwellEntryPoint.scala @@ -0,0 +1,192 @@ +package cromwell + +import cats.data.Validated._ +import cats.syntax.cartesian._ +import cats.syntax.validated._ +import com.typesafe.config.ConfigFactory +import cromwell.CommandLineParser._ +import cromwell.core.path.Path +import cromwell.core.{WorkflowSourceFilesCollection, WorkflowSourceFilesWithDependenciesZip, WorkflowSourceFilesWithoutImports} +import cromwell.engine.workflow.SingleWorkflowRunnerActor +import cromwell.engine.workflow.SingleWorkflowRunnerActor.RunWorkflow +import cromwell.server.{CromwellServer, CromwellSystem} +import lenthall.exception.MessageAggregation +import lenthall.validation.ErrorOr._ +import org.slf4j.LoggerFactory + +import scala.collection.JavaConverters._ +import scala.concurrent.duration.{Duration, _} +import scala.concurrent.{Await, Future, TimeoutException} +import scala.language.postfixOps +import scala.util.{Failure, Success, Try} + + +object CromwellEntryPoint { + + /** + * Run Cromwell in server mode. + */ + def runServer() = { + val system = buildCromwellSystem(Server) + waitAndExit(CromwellServer.run, system) + } + + /** + * Run a single workflow using the successfully parsed but as yet not validated arguments. + */ + def runSingle(args: CommandLineArguments): Unit = { + val cromwellSystem = buildCromwellSystem(Run) + implicit val actorSystem = cromwellSystem.actorSystem + + val sources = validateRunArguments(args) + val runnerProps = SingleWorkflowRunnerActor.props(sources, args.metadataOutput)(cromwellSystem.materializer) + + val runner = cromwellSystem.actorSystem.actorOf(runnerProps, "SingleWorkflowRunnerActor") + + import cromwell.util.PromiseActor.EnhancedActorRef + waitAndExit(_ => runner.askNoTimeout(RunWorkflow), cromwellSystem) + } + + private def buildCromwellSystem(command: Command): CromwellSystem = { + initLogging(command) + lazy val Log = LoggerFactory.getLogger("cromwell") + Try { + new CromwellSystem {} + } recoverWith { + case t: Throwable => + Log.error("Failed to instantiate Cromwell System. Shutting down Cromwell.") + Log.error(t.getMessage) + System.exit(1) + Failure(t) + } get + } + /** + * If a cromwell server is going to be run, makes adjustments to the default logback configuration. + * Overwrites LOG_MODE system property used in our logback.xml, _before_ the logback classes load. + * Restored from similar functionality in + * https://github.com/broadinstitute/cromwell/commit/2e3f45b#diff-facc2160a82442932c41026c9a1e4b2bL28 + * TODO: Logback is configurable programmatically. We don't have to overwrite system properties like this. + * + * Also copies variables from config/system/environment/defaults over to the system properties. + * Fixes issue where users are trying to specify Java properties as environment variables. + */ + private def initLogging(command: Command): Unit = { + val logbackSetting = command match { + case Server => "STANDARD" + case Run => "PRETTY" + } + + val defaultProps = Map( + "LOG_MODE" -> logbackSetting, + "LOG_LEVEL" -> "INFO" + ) + + val config = ConfigFactory.load + .withFallback(ConfigFactory.systemEnvironment()) + .withFallback(ConfigFactory.parseMap(defaultProps.asJava, "Defaults")) + + val props = sys.props + defaultProps.keys foreach { key => + props += key -> config.getString(key) + } + + /* + We've possibly copied values from the environment, or our defaults, into the system properties. + Make sure that the next time one uses the ConfigFactory that our updated system properties are loaded. + */ + ConfigFactory.invalidateCaches() + } + + private def waitAndExit(runner: CromwellSystem => Future[Any], workflowManagerSystem: CromwellSystem): Unit = { + val futureResult = runner(workflowManagerSystem) + Await.ready(futureResult, Duration.Inf) + + try { + Await.ready(workflowManagerSystem.shutdownActorSystem(), 30 seconds) + } catch { + case _: TimeoutException => Console.err.println("Timed out trying to shutdown actor system") + case other: Exception => Console.err.println(s"Unexpected error trying to shutdown actor system: ${other.getMessage}") + } + + val returnCode = futureResult.value.get match { + case Success(_) => 0 + case Failure(e) => + Console.err.println(e.getMessage) + 1 + } + + sys.exit(returnCode) + } + + def validateRunArguments(args: CommandLineArguments): WorkflowSourceFilesCollection = { + + val workflowSource = readContent("Workflow source", args.workflowSource.get) + val inputsJson = readJson("Workflow inputs", args.workflowInputs) + val optionsJson = readJson("Workflow options", args.workflowOptions) + val labelsJson = readJson("Workflow labels", args.workflowLabels) + + val sourceFileCollection = args.imports match { + case Some(p) => (workflowSource |@| inputsJson |@| optionsJson |@| labelsJson) map { (w, i, o, l) => + WorkflowSourceFilesWithDependenciesZip.apply( + workflowSource = w, + workflowType = Option("WDL"), + workflowTypeVersion = None, + inputsJson = i, + workflowOptionsJson = o, + labelsJson = l, + importsZip = p.loadBytes) + } + case None => (workflowSource |@| inputsJson |@| optionsJson |@| labelsJson) map { (w, i, o, l) => + WorkflowSourceFilesWithoutImports.apply( + workflowSource = w, + workflowType = Option("WDL"), + workflowTypeVersion = None, + inputsJson = i, + workflowOptionsJson = o, + labelsJson = l + ) + } + } + + val sourceFiles = for { + sources <- sourceFileCollection + _ <- writeableMetadataPath(args.metadataOutput) + } yield sources + + sourceFiles match { + case Valid(r) => r + case Invalid(nel) => throw new RuntimeException with MessageAggregation { + override def exceptionContext: String = "ERROR: Unable to run Cromwell:" + override def errorMessages: Traversable[String] = nel.toList + } + } + } + + private def writeableMetadataPath(path: Option[Path]): ErrorOr[Unit] = { + path match { + case Some(p) if !metadataPathIsWriteable(p) => s"Unable to write to metadata directory: $p".invalidNel + case _ => ().validNel + } + } + + /** Read the path to a string. */ + private def readContent(inputDescription: String, path: Path): ErrorOr[String] = { + if (!path.exists) { + s"$inputDescription does not exist: $path".invalidNel + } else if (!path.isReadable) { + s"$inputDescription is not readable: $path".invalidNel + } else path.contentAsString.validNel + } + + /** Read the path to a string, unless the path is None, in which case returns "{}". */ + private def readJson(inputDescription: String, pathOption: Option[Path]): ErrorOr[String] = { + pathOption match { + case Some(path) => readContent(inputDescription, path) + case None => "{}".validNel + } + } + + private def metadataPathIsWriteable(metadataPath: Path): Boolean = + Try(metadataPath.createIfNotExists(createParents = true).append("")).isSuccess + +} diff --git a/src/main/scala/cromwell/Main.scala b/src/main/scala/cromwell/Main.scala deleted file mode 100644 index cf438bc8c..000000000 --- a/src/main/scala/cromwell/Main.scala +++ /dev/null @@ -1,151 +0,0 @@ -package cromwell - -import com.typesafe.config.ConfigFactory -import cromwell.engine.workflow.SingleWorkflowRunnerActor -import cromwell.engine.workflow.SingleWorkflowRunnerActor.RunWorkflow -import cromwell.server.{CromwellServer, CromwellSystem} -import cromwell.util.PromiseActor -import org.slf4j.LoggerFactory - -import scala.collection.JavaConverters._ -import scala.concurrent.duration._ -import scala.concurrent.{Await, Future, TimeoutException} -import scala.language.postfixOps -import scala.util.{Failure, Success, Try} - -object Main extends App { - val CommandLine = CromwellCommandLine(args) - initLogging(CommandLine) - - lazy val Log = LoggerFactory.getLogger("cromwell") - lazy val CromwellSystem: CromwellSystem = Try { - new CromwellSystem {} - } recoverWith { - case t: Throwable => - Log.error("Failed to instantiate Cromwell System. Shutting down Cromwell.") - Log.error(t.getMessage) - System.exit(1) - Failure(t) - } get - - CommandLine match { - case UsageAndExit => usageAndExit() - case VersionAndExit => versionAndExit() - case RunServer => waitAndExit(CromwellServer.run(CromwellSystem), CromwellSystem) - case r: RunSingle => runWorkflow(r) - } - - /** - * If a cromwell server is going to be run, makes adjustments to the default logback configuration. - * Overwrites LOG_MODE system property used in our logback.xml, _before_ the logback classes load. - * Restored from similar functionality in - * https://github.com/broadinstitute/cromwell/commit/2e3f45b#diff-facc2160a82442932c41026c9a1e4b2bL28 - * TODO: Logback is configurable programmatically. We don't have to overwrite system properties like this. - * - * Also copies variables from config/system/environment/defaults over to the system properties. - * Fixes issue where users are trying to specify Java properties as environment variables. - */ - private def initLogging(commandLine: CromwellCommandLine): Unit = { - val defaultLogMode = commandLine match { - case RunServer => "STANDARD" - case _ => "PRETTY" - } - - val defaultProps = Map("LOG_MODE" -> defaultLogMode, "LOG_LEVEL" -> "INFO") - - val config = ConfigFactory.load - .withFallback(ConfigFactory.systemEnvironment()) - .withFallback(ConfigFactory.parseMap(defaultProps.asJava, "Defaults")) - - val props = sys.props - defaultProps.keys foreach { key => - props += key -> config.getString(key) - } - - /* - We've possibly copied values from the environment, or our defaults, into the system properties. - Make sure that the next time one uses the ConfigFactory that our updated system properties are loaded. - */ - ConfigFactory.invalidateCaches() - } - - private def runWorkflow(commandLine: RunSingle): Unit = { - implicit val actorSystem = CromwellSystem.actorSystem - - Log.info(s"RUN sub-command") - commandLine.paths.logMe(Log) - val runnerProps = SingleWorkflowRunnerActor.props(commandLine.sourceFiles, commandLine.paths.metadataPath)(CromwellSystem.materializer) - - val runner = CromwellSystem.actorSystem.actorOf(runnerProps, "SingleWorkflowRunnerActor") - - import PromiseActor.EnhancedActorRef - - waitAndExit(runner.askNoTimeout(RunWorkflow), CromwellSystem) - } - - private def waitAndExit(futureResult: Future[Any], workflowManagerSystem: CromwellSystem): Unit = { - Await.ready(futureResult, Duration.Inf) - - try { - Await.ready(workflowManagerSystem.shutdownActorSystem(), 30 seconds) - } catch { - case timeout: TimeoutException => Console.err.println("Timed out trying to shutdown actor system") - case other: Exception => Console.err.println(s"Unexpected error trying to shutdown actor system: ${other.getMessage}") - } - - val returnCode = futureResult.value.get match { - case Success(_) => 0 - case Failure(e) => - Console.err.println(e.getMessage) - 1 - } - - sys.exit(returnCode) - } - - def usageAndExit(): Unit = { - println( - """ - |java -jar cromwell.jar - | - |Actions: - |run [] [] - | [] [] [] - | - | Given a WDL file and JSON file containing the value of the - | workflow inputs, this will run the workflow locally and - | print out the outputs in JSON format. The workflow - | options file specifies some runtime configuration for the - | workflow (see README for details). The workflow metadata - | output is an optional file path to output the metadata. The - | directory of WDL files is optional. However, it is required - | if the primary workflow imports workflows that are outside - | of the root directory of the Cromwell project. - | - | Use a single dash ("-") to skip optional files. Ex: - | run noinputs.wdl - - metadata.json - - | - |server - | - | Starts a web server on port 8000. See the web server - | documentation for more details about the API endpoints. - | - |-version - | - | Returns the version of the Cromwell engine. - | - """.stripMargin) - - System.exit(0) - } - - def versionAndExit(): Unit = { - val versionConf = ConfigFactory.load("cromwell-version.conf").getConfig("version") - println( - s""" - |cromwell: ${versionConf.getString("cromwell")} - """.stripMargin - ) - System.exit(0) - } -} diff --git a/src/test/scala/cromwell/CromwellCommandLineSpec.scala b/src/test/scala/cromwell/CromwellCommandLineSpec.scala index 162f1689e..bfe5b4b5c 100644 --- a/src/test/scala/cromwell/CromwellCommandLineSpec.scala +++ b/src/test/scala/cromwell/CromwellCommandLineSpec.scala @@ -1,86 +1,107 @@ package cromwell +import cromwell.CommandLineParser.{CommandLineArguments, Run, Server} +import cromwell.CromwellCommandLineSpec.WdlAndInputs import cromwell.core.path.{DefaultPathBuilder, Path} import cromwell.util.SampleWdl import cromwell.util.SampleWdl.{FileClobber, FilePassingWorkflow, ThreeStep} -import org.scalatest.{FlatSpec, Matchers} +import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers} import scala.util.Try -class CromwellCommandLineSpec extends FlatSpec with Matchers { - import CromwellCommandLineSpec._ +class CromwellCommandLineSpec extends FlatSpec with Matchers with BeforeAndAfter { + + var parser: scopt.OptionParser[CommandLineArguments] = _ behavior of "CromwellCommandLine" - it should "UsageAndExit with no arguments" in { - CromwellCommandLine(List.empty[String]) shouldBe UsageAndExit + before { + parser = CommandLineParser.buildParser() } - it should "RunServer when specified" in { - CromwellCommandLine(List("server")) shouldBe RunServer + it should "fail to parse with no arguments" in { + parser.parse(Array.empty[String], CommandLineArguments()).get.command shouldBe None } - it should "UsageAndExit when supplying an argument to server" in { - CromwellCommandLine(List("server", "foo")) shouldBe UsageAndExit + it should "run server when specified" in { + parser.parse(Array("server"), CommandLineArguments()).get.command shouldBe Some(Server) } - it should "UsageAndExit with no arguments to run" in { - CromwellCommandLine(List("run")) shouldBe UsageAndExit + it should "fail to parse when supplying an argument to server" in { + parser.parse(Array("server", "foo"), CommandLineArguments()) shouldBe None } - it should "fail with too many arguments to run" in { - CromwellCommandLine(List("run", "bork", "bork", "bork", "bork", "bork", "blerg")) + it should "fail to parse with no arguments to run" in { + parser.parse(Array("run"), CommandLineArguments()) shouldBe None } - it should "VersionAndExit when the `-version` flag is passed" in { - CromwellCommandLine(List("-version")) shouldBe VersionAndExit + it should "fail to parse with too many arguments to run" in { + parser.parse(Array("run", "forrest", "run"), CommandLineArguments()) shouldBe None } - it should "RunSingle when supplying wdl and inputs" in { - CromwellCommandLine(List("run", ThreeStepWithoutOptions.wdl, ThreeStepWithoutOptions.inputs)) shouldBe a [RunSingle] + // --version exits the JVM which is not great in a test suite. Haven't figure out a way to test this yet. + // it should "handle version output when the `-version` flag is passed" in { + // // I don't see a way to see that --version is printing just the version, but this at least confirms a `None` + // // output that should generate a usage and version. + // parser.parse(Array("--version"), CommandLineArguments()) shouldBe None + // } + + it should "run single when supplying wdl and inputs" in { + val optionsLast = parser.parse(Array("run", "3step.wdl", "--inputs", "3step.inputs"), CommandLineArguments()).get + optionsLast.command shouldBe Some(Run) + optionsLast.workflowSource.get.pathAsString shouldBe "3step.wdl" + optionsLast.workflowInputs.get.pathAsString shouldBe "3step.inputs" + + val optionsFirst = parser.parse(Array("run", "--inputs", "3step.inputs", "3step.wdl"), CommandLineArguments()).get + optionsFirst.command shouldBe Some(Run) + optionsFirst.workflowSource.get.pathAsString shouldBe "3step.wdl" + optionsFirst.workflowInputs.get.pathAsString shouldBe "3step.inputs" } - it should "RunSingle with default inputs when only supplying wdl" in { - val ccl = CromwellCommandLine(List("run", ThreeStepWithoutOptions.wdl)).asInstanceOf[RunSingle] - ccl.sourceFiles.inputsJson shouldBe ThreeStepInputs + it should "run single when supplying wdl and inputs and options" in { + val optionsLast = parser.parse(Array("run", "3step.wdl", "--inputs", "3step.inputs", "--options", "3step.options"), CommandLineArguments()).get + optionsLast.command shouldBe Some(Run) + optionsLast.workflowSource.get.pathAsString shouldBe "3step.wdl" + optionsLast.workflowInputs.get.pathAsString shouldBe "3step.inputs" + optionsLast.workflowOptions.get.pathAsString shouldBe "3step.options" + + val optionsFirst = parser.parse(Array("run", "--inputs", "3step.inputs", "--options", "3step.options", "3step.wdl"), CommandLineArguments()).get + optionsFirst.command shouldBe Some(Run) + optionsFirst.workflowSource.get.pathAsString shouldBe "3step.wdl" + optionsFirst.workflowInputs.get.pathAsString shouldBe "3step.inputs" + optionsFirst.workflowOptions.get.pathAsString shouldBe "3step.options" } - it should "RunSingle with defaults if you use dashes" in { - val ccl = CromwellCommandLine(List("run", ThreeStepWithoutOptions.wdl, "-", "-", "-")).asInstanceOf[RunSingle] - ccl.sourceFiles.inputsJson shouldBe "{}" - ccl.sourceFiles.workflowOptionsJson shouldBe "{}" - } + it should "fail if input files do not exist" in { + val parsedArgs = parser.parse(Array("run", "3step.wdl", "--inputs", "3step.inputs", "--options", "3step.options"), CommandLineArguments()).get + val validation = Try(CromwellEntryPoint.validateRunArguments(parsedArgs)) - it should "RunSingle with options, if passed in" in { - val threeStep = WdlAndInputs(ThreeStep, optionsJson = """{ foobar bad json! }""") - val ccl = CromwellCommandLine(List("run", threeStep.wdl, threeStep.inputs, threeStep.options)).asInstanceOf[RunSingle] - ccl.sourceFiles.workflowOptionsJson shouldBe threeStep.optionsJson - } - - it should "fail if inputs path does not exist" in { - val ccl = Try(CromwellCommandLine(List("run", ThreeStepWithoutOptions.wdl, "/some/path/that/doesnt/exit"))) - ccl.isFailure shouldBe true - ccl.failed.get.getMessage should include("Inputs does not exist") + validation.isFailure shouldBe true + validation.failed.get.getMessage should include("Workflow source does not exist") + validation.failed.get.getMessage should include("Workflow inputs does not exist") + validation.failed.get.getMessage should include("Workflow options does not exist") } - it should "fail if inputs path is not writeable" in { + it should "fail if inputs path is not readable" in { val threeStep = WdlAndInputs(ThreeStep) + val parsedArgs = parser.parse(Array("run", threeStep.wdl, "--inputs", threeStep.inputs), CommandLineArguments()).get threeStep.inputsFile setPermissions Set.empty - val ccl = Try(CromwellCommandLine(List("run", threeStep.wdl, threeStep.inputs))) + val ccl = Try(CromwellEntryPoint.validateRunArguments(parsedArgs)) ccl.isFailure shouldBe true - ccl.failed.get.getMessage should include("Inputs is not readable") + ccl.failed.get.getMessage should include("Workflow inputs is not readable") } - it should "fail if metadata path is not writeable" in { + it should "fail if metadata output path is not writeable" in { val threeStep = WdlAndInputs(ThreeStep) + val parsedArgs = parser.parse(Array("run", threeStep.wdl, "--inputs", threeStep.inputs, "--metadata-output", threeStep.metadata), CommandLineArguments()).get threeStep.metadataFile write "foo" threeStep.metadataFile setPermissions Set.empty - val ccl = Try(CromwellCommandLine(List("run", threeStep.wdl, threeStep.inputs, "-", threeStep.metadata))) + val ccl = Try(CromwellEntryPoint.validateRunArguments(parsedArgs)) ccl.isFailure shouldBe true ccl.failed.get.getMessage should include("Unable to write to metadata directory:") } - it should "run if imports directory is a .zip file" in { + it should "run if the imports path is a .zip file" in { val wdlDir = DefaultPathBuilder.createTempDirectory("wdlDirectory") val filePassing = DefaultPathBuilder.createTempFile("filePassing", ".wdl", Option(wdlDir)) @@ -91,7 +112,8 @@ class CromwellCommandLineSpec extends FlatSpec with Matchers { val zippedDir = wdlDir.zip() val zippedPath = zippedDir.pathAsString - val ccl = Try(CromwellCommandLine(List("run", filePassing.pathAsString, "-", "-", "-", zippedPath))) + val parsedArgs = parser.parse(Array("run", filePassing.pathAsString, "--imports", zippedPath), CommandLineArguments()).get + val ccl = Try(CromwellEntryPoint.validateRunArguments(parsedArgs)) ccl.isFailure shouldBe false zippedDir.delete(swallowIOExceptions = true)