From 1c26f759c8bd22b2336ec76450401e31df6c46ef Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Fri, 9 Jun 2017 14:58:33 -0400 Subject: [PATCH] Remove some user-facing WDL specificity. --- .travis.yml | 2 +- CHANGELOG.md | 8 +++ README.md | 60 ++++++++--------- .../scala/cromwell/backend/BackendSpec.scala | 4 +- .../core/WorkflowSourceFilesCollection.scala | 14 ++-- .../test/scala/cromwell/util/SampleWdl.scala | 66 +++++++++---------- .../scala/cromwell/api/CromwellClient.scala | 4 +- .../src/main/resources/swagger/cromwell.yaml | 4 +- .../engine/workflow/WorkflowActor.scala | 35 ++++++---- .../MaterializeWorkflowDescriptorActor.scala | 4 +- .../workflowstore/SqlWorkflowStore.scala | 4 +- .../WorkflowStoreSubmitActor.scala | 2 +- .../webservice/CromwellApiHandler.scala | 2 +- .../webservice/CromwellApiService.scala | 41 ++++++++---- .../scala/cromwell/ArrayWorkflowSpec.scala | 2 +- .../scala/cromwell/CromwellTestKitSpec.scala | 2 +- .../cromwell/DeclarationWorkflowSpec.scala | 2 +- .../test/scala/cromwell/MapWorkflowSpec.scala | 2 +- .../scala/cromwell/RestartWorkflowSpec.scala | 2 +- .../cromwell/SimpleWorkflowActorSpec.scala | 2 +- .../cromwell/engine/WorkflowAbortSpec.scala | 2 +- .../engine/WorkflowStoreActorSpec.scala | 2 +- .../engine/workflow/WorkflowActorSpec.scala | 6 +- ...terializeWorkflowDescriptorActorSpec.scala | 32 ++++----- .../execution/ExecutionStoreBenchmark.scala | 2 +- .../SubWorkflowStoreSpec.scala | 2 +- .../webservice/CromwellApiServiceSpec.scala | 12 ++-- .../scala/cromwell/CromwellCommandLine.scala | 4 +- .../cromwell/CromwellCommandLineSpec.scala | 6 +- ...JesAsyncBackendJobExecutionActorSpec.scala | 18 ++--- .../backend/impl/jes/JesCallPathsSpec.scala | 6 +- .../impl/jes/JesInitializationActorSpec.scala | 2 +- .../impl/jes/JesWorkflowPathsSpec.scala | 2 +- .../impl/sfs/config/ConfigWdlNamespace.scala | 10 +-- .../spark/SparkJobExecutionActorSpec.scala | 4 +- .../spark/SparkRuntimeAttributesSpec.scala | 4 +- 36 files changed, 204 insertions(+), 172 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1f20afb5613..f49645b0701 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ before_install: - openssl aes-256-cbc -K "$encrypted_5ebd3ff04788_key" -iv "$encrypted_5ebd3ff04788_iv" -in src/bin/travis/resources/jesConf.tar.enc -out jesConf.tar -d || true env: global: - - CENTAUR_BRANCH=mlc_workflow_type + - CENTAUR_BRANCH=mlc_wdl_b_gone matrix: # Setting this variable twice will cause the 'script' section to run twice with the respective env var invoked - BUILD_TYPE=sbt diff --git a/CHANGELOG.md b/CHANGELOG.md index d6b4a093854..b8a9771e497 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Cromwell Change Log +## 28 + +### Workflow Submission + +The workflow submission parameters `wdlSource` and `wdlDependencies` have been deprecated in favor of `workflowSource` and +`workflowDependencies` respectively. The older names are still supported in Cromwell 28 with deprecation warnings but will +be removed in a future version of Cromwell. + ## 27 ### Migration diff --git a/README.md b/README.md index f79ced33309..8e0638d93b1 100644 --- a/README.md +++ b/README.md @@ -361,7 +361,7 @@ system { Or, via `-Dsystem.abort-jobs-on-terminate=true` command line option. -By default, this value is false when running `java -jar cromwell.jar server`, and true when running `java -jar cromwell.jar run `. +By default, this value is false when running `java -jar cromwell.jar server`, and true when running `java -jar cromwell.jar run `. # Security @@ -477,7 +477,7 @@ When Cromwell runs a workflow, it first creates a directory `//call-`. This is the ``. For example, having a `stdout` and `stderr` file is common among both backends and they both write a shell script file to the `` as well. See the descriptions below for details about backend-specific files that are written to these directories. -An example of a workflow output directory for a three-step WDL file might look like this: +An example of a workflow output directory for a three-step workflow might look like this: ``` cromwell-executions/ @@ -1227,7 +1227,7 @@ nativeSpecs attribute needs to be specified as String. ## Spark Backend -This backend adds support for execution of spark jobs in a workflow using the existing wdl format. +This backend adds support for execution of spark jobs in a workflow. It supports the following Spark deploy modes: @@ -1301,7 +1301,7 @@ Supported runtime attributes for a Spark Job is as follows: * appMainClass ( Spark app/job entry point) * numberOfExecutors ( Specific to cluster deploy mode) -Sample usage : +Sample usage: ```wdl task sparkjob_with_yarn_cluster { @@ -1327,8 +1327,8 @@ Supported File Systems as follows: * Network File System * Distributed file system -### Sample Wdl -Next, create a Wdl, and it's json input like so: +### Sample WDL +Next, create a WDL, and its json input like so: ```wdl task sparkjob_with_yarn_cluster { @@ -1682,7 +1682,7 @@ Valid keys and their meanings: * **google_project** - (JES backend only) Specifies which google project to execute this workflow. * **refresh_token** - (JES backend only) Only used if `localizeWithRefreshToken` is specified in the [configuration file](#configuring-cromwell). * **auth_bucket** - (JES backend only) defaults to the the value in **jes_gcs_root**. This should represent a GCS URL that only Cromwell can write to. The Cromwell account is determined by the `google.authScheme` (and the corresponding `google.userAuth` and `google.serviceAuth`) - * **monitoring_script** - (JES backend only) Specifies a GCS URL to a script that will be invoked prior to the WDL command being run. For example, if the value for monitoring_script is "gs://bucket/script.sh", it will be invoked as `./script.sh > monitoring.log &`. The value `monitoring.log` file will be automatically de-localized. + * **monitoring_script** - (JES backend only) Specifies a GCS URL to a script that will be invoked prior to the user command being run. For example, if the value for monitoring_script is "gs://bucket/script.sh", it will be invoked as `./script.sh > monitoring.log &`. The value `monitoring.log` file will be automatically de-localized. # Labels @@ -1757,12 +1757,12 @@ Cromwell also accepts two [workflow option](#workflow-options) related to call c Docker tags are a convenient way to point to a version of an image (ubuntu:14.04), or even the latest version (ubuntu:latest). For that purpose, tags are mutable, meaning that the image they point to can change, while the tag name stays the same. -While this is very convenient in some cases, using mutable, or "floating" tags in WDL affects the reproducibility of the WDL file: the same WDL using "ubuntu:latest" run now, and a year, or even a month from now will actually run with different docker images. +While this is very convenient in some cases, using mutable, or "floating" tags in tasks affects the reproducibility of a workflow: the same workflow using "ubuntu:latest" run now, and a year, or even a month from now will actually run with different docker images. This has an even bigger impact when Call Caching is turned on in Cromwell, and could lead to unpredictable behaviors if a tag is updated in the middle of a workflow or even a scatter for example. Docker provides another way of identifying an image version, using the specific digest of the image. The digest is guaranteed to be different if 2 images have different byte content. For more information see https://docs.docker.com/registry/spec/api/#/content-digests A docker image with digest can be referenced as follows : **ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950** The above image refers to a specific image of ubuntu, that does not depend on a floating tag. -A WDL containing this Docker image run now and a year from now will run in the exact same container. +A workflow containing this Docker image run now and a year from now will run in the exact same container. In order to remove unpredictable behaviors, Cromwell takes the following approach regarding floating docker tags. @@ -1834,7 +1834,7 @@ When running a job on the Config (Shared Filesystem) backend, Cromwell provides ``` # Imports -Import statements inside of a WDL file are supported by Cromwell when running in Server mode as well as Single Workflow Runner Mode. +Import statements inside of a workflow file are supported by Cromwell when running in Server mode as well as Single Workflow Runner Mode. In Single Workflow Runner Mode, you pass in a zip file which includes the WDL files referenced by the import statements. Cromwell requires the zip file to be passed in as a command line argument, as explained by the section [run](#run). @@ -1843,7 +1843,7 @@ For example, given a workflow `wf.wdl` and an imports directory `WdlImports.zip` java -jar cromwell.jar wf.wdl wf.inputs - - WdlImports.zip ``` -In Server Mode, you pass in a zip file using the parameter `wdlDependencies` via the [POST /api/workflows/:version](#post-apiworkflowsversion) endpoint. +In Server Mode, you pass in a zip file using the parameter `workflowDependencies` via the [POST /api/workflows/:version](#post-apiworkflowsversion) endpoint. # Sub Workflows @@ -2304,7 +2304,7 @@ It's also possible to set the URL query parameter `expandSubWorkflows` to `true` # REST API -The `server` subcommand on the executable JAR will start an HTTP server which can accept WDL files to run as well as check status and output of existing workflows. +The `server` subcommand on the executable JAR will start an HTTP server which can accept workflow files to run as well as check status and output of existing workflows. The following sub-sections define which HTTP Requests the web server can accept and what they will return. Example HTTP requests are given in [HTTPie](https://github.com/jkbrzt/httpie) and [cURL](https://curl.haxx.se/) @@ -2316,12 +2316,12 @@ All web server requests include an API version in the url. The current version i This endpoint accepts a POST request with a `multipart/form-data` encoded body. The form fields that may be included are: -* `wdlSource` - *Required* Contains the WDL file to submit for execution. -* `workflowInputs` - *Optional* JSON file containing the inputs. A skeleton file can be generated from [wdltool](https://github.com/broadinstitute/wdltool) using the "inputs" subcommand. +* `workflowSource` - *Required* Contains the workflow source file to submit for execution. +* `workflowInputs` - *Optional* JSON file containing the inputs. For WDL workflows a skeleton file can be generated from [wdltool](https://github.com/broadinstitute/wdltool) using the "inputs" subcommand. * `workflowInputs_n` - *Optional* Where `n` is an integer. JSON file containing the 'n'th set of auxiliary inputs. * `workflowOptions` - *Optional* JSON file containing options for this workflow execution. See the [run](#run) CLI sub-command for some more information about this. * `customLabels` - *Optional* JSON file containing a set of custom labels to apply to this workflow. See [Labels](#labels) for the expected format. -* `wdlDependencies` - *Optional* ZIP file containing WDL files that are used to resolve import statements. +* `workflowDependencies` - *Optional* ZIP file containing workflow source files that are used to resolve import statements. Regarding the workflowInputs parameter, in case of key conflicts between multiple input JSON files, higher values of x in workflowInputs_x override lower values. For example, an input specified in workflowInputs_3 will override an input with the same name in workflowInputs or workflowInputs_2. Similarly, an input key specified in workflowInputs_5 will override an identical input key in any other input file. @@ -2332,13 +2332,13 @@ Additionally, although Swagger has a limit of 5 JSON input files, the REST endpo cURL: ``` -$ curl -v "localhost:8000/api/workflows/v1" -F wdlSource=@src/main/resources/3step.wdl -F workflowInputs=@test.json +$ curl -v "localhost:8000/api/workflows/v1" -F workflowSource=@src/main/resources/3step.wdl -F workflowInputs=@test.json ``` HTTPie: ``` -$ http --print=hbHB --form POST localhost:8000/api/workflows/v1 wdlSource=@src/main/resources/3step.wdl workflowInputs@inputs.json +$ http --print=hbHB --form POST localhost:8000/api/workflows/v1 workflowSource=@src/main/resources/3step.wdl workflowInputs@inputs.json ``` Request: @@ -2354,7 +2354,7 @@ Host: localhost:8000 User-Agent: HTTPie/0.9.2 --64128d499e9e4616adea7d281f695dca -Content-Disposition: form-data; name="wdlSource" +Content-Disposition: form-data; name="workflowSource" task ps { command { @@ -2424,13 +2424,13 @@ To specify workflow options as well: cURL: ``` -$ curl -v "localhost:8000/api/workflows/v1" -F wdlSource=@wdl/jes0.wdl -F workflowInputs=@wdl/jes0.json -F workflowOptions=@options.json +$ curl -v "localhost:8000/api/workflows/v1" -F workflowSource=@wdl/jes0.wdl -F workflowInputs=@wdl/jes0.json -F workflowOptions=@options.json ``` HTTPie: ``` -http --print=HBhb --form POST http://localhost:8000/api/workflows/v1 wdlSource=@wdl/jes0.wdl workflowInputs@wdl/jes0.json workflowOptions@options.json +http --print=HBhb --form POST http://localhost:8000/api/workflows/v1 workflowSource=@wdl/jes0.wdl workflowInputs@wdl/jes0.json workflowOptions@options.json ``` Request (some parts truncated for brevity): @@ -2446,7 +2446,7 @@ Host: localhost:8000 User-Agent: HTTPie/0.9.2 --f3fd038395644de596c460257626edd7 -Content-Disposition: form-data; name="wdlSource" +Content-Disposition: form-data; name="workflowSource" task x { ... } task y { ... } @@ -2482,28 +2482,28 @@ Content-Disposition: form-data; name="workflowOptions"; filename="options.json" This endpoint accepts a POST request with a `multipart/form-data` encoded body. The form fields that may be included are: -* `wdlSource` - *Required* Contains the WDL file to submit for +* `workflowSource` - *Required* Contains the workflow source file to submit for execution. * `workflowInputs` - *Required* JSON file containing the inputs in a -JSON array. A skeleton file for a single inputs json element can be +JSON array. For WDL workflows a skeleton file for a single inputs json element can be generated from [wdltool](https://github.com/broadinstitute/wdltool) using the "inputs" subcommand. The orderded endpoint responses will contain one workflow submission response for each input, respectively. * `workflowOptions` - *Optional* JSON file containing options for this workflow execution. See the [run](#run) CLI sub-command for some more information about this. -* `wdlDependencies` - *Optional* ZIP file containing WDL files that are used to resolve import statements. Applied equally to all workflowInput sets. +* `workflowDependencies` - *Optional* ZIP file containing workflow source files that are used to resolve import statements. Applied equally to all workflowInput sets. cURL: ``` -$ curl -v "localhost:8000/api/workflows/v1/batch" -F wdlSource=@src/main/resources/3step.wdl -F workflowInputs=@test_array.json +$ curl -v "localhost:8000/api/workflows/v1/batch" -F workflowSource=@src/main/resources/3step.wdl -F workflowInputs=@test_array.json ``` HTTPie: ``` -$ http --print=hbHB --form POST localhost:8000/api/workflows/v1/batch wdlSource=@src/main/resources/3step.wdl workflowInputs@inputs_array.json +$ http --print=hbHB --form POST localhost:8000/api/workflows/v1/batch workflowSource=@src/main/resources/3step.wdl workflowInputs@inputs_array.json ``` Request: @@ -2519,7 +2519,7 @@ Host: localhost:8000 User-Agent: HTTPie/0.9.2 --64128d499e9e4616adea7d281f695dcb -Content-Disposition: form-data; name="wdlSource" +Content-Disposition: form-data; name="workflowSource" task ps { command { @@ -2600,13 +2600,13 @@ To specify workflow options as well: cURL: ``` -$ curl -v "localhost:8000/api/workflows/v1/batch" -F wdlSource=@wdl/jes0.wdl -F workflowInputs=@wdl/jes0_array.json -F workflowOptions=@options.json +$ curl -v "localhost:8000/api/workflows/v1/batch" -F workflowSource=@wdl/jes0.wdl -F workflowInputs=@wdl/jes0_array.json -F workflowOptions=@options.json ``` HTTPie: ``` -http --print=HBhb --form POST http://localhost:8000/api/workflows/v1/batch wdlSource=@wdl/jes0.wdl workflowInputs@wdl/jes0_array.json workflowOptions@options.json +http --print=HBhb --form POST http://localhost:8000/api/workflows/v1/batch workflowSource=@wdl/jes0.wdl workflowInputs@wdl/jes0_array.json workflowOptions@options.json ``` Request (some parts truncated for brevity): @@ -2622,7 +2622,7 @@ Host: localhost:8000 User-Agent: HTTPie/0.9.2 --f3fd038395644de596c460257626edd8 -Content-Disposition: form-data; name="wdlSource" +Content-Disposition: form-data; name="workflowSource" task x { ... } task y { ... } diff --git a/backend/src/test/scala/cromwell/backend/BackendSpec.scala b/backend/src/test/scala/cromwell/backend/BackendSpec.scala index bc0f36a75d4..0e20c15cd3f 100644 --- a/backend/src/test/scala/cromwell/backend/BackendSpec.scala +++ b/backend/src/test/scala/cromwell/backend/BackendSpec.scala @@ -23,13 +23,13 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito { executeJobAndAssertOutputs(backend, workflow.expectedResponse) } - def buildWorkflowDescriptor(wdl: WdlSource, + def buildWorkflowDescriptor(workflowSource: WdlSource, inputs: Map[String, WdlValue] = Map.empty, options: WorkflowOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue])), runtime: String = "") = { BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime), Seq.empty[ImportResolver]).get.workflow, // Get ok, this is a test! + WdlNamespaceWithWorkflow.load(workflowSource.replaceAll("RUNTIME", runtime), Seq.empty[ImportResolver]).get.workflow, // Get ok, this is a test! inputs, options, Labels.empty diff --git a/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala b/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala index 1b9b2743eae..5025690af29 100644 --- a/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala +++ b/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala @@ -7,7 +7,7 @@ import wdl4s.{WdlJson, WdlSource} */ sealed trait WorkflowSourceFilesCollection { - def wdlSource: WdlSource + def workflowSource: WdlSource def inputsJson: WdlJson def workflowOptionsJson: WorkflowOptionsJson def labelsJson: WdlJson @@ -26,7 +26,7 @@ sealed trait WorkflowSourceFilesCollection { } object WorkflowSourceFilesCollection { - def apply(wdlSource: WdlSource, + def apply(workflowSource: WdlSource, workflowType: Option[WorkflowType], workflowTypeVersion: Option[WorkflowTypeVersion], inputsJson: WdlJson, @@ -34,25 +34,25 @@ object WorkflowSourceFilesCollection { labelsJson: WdlJson, importsFile: Option[Array[Byte]]): WorkflowSourceFilesCollection = importsFile match { case Some(imports) => - WorkflowSourceFilesWithDependenciesZip(wdlSource, workflowType, workflowTypeVersion, inputsJson, workflowOptionsJson, labelsJson, imports) + WorkflowSourceFilesWithDependenciesZip(workflowSource, workflowType, workflowTypeVersion, inputsJson, workflowOptionsJson, labelsJson, imports) case None => - WorkflowSourceFilesWithoutImports(wdlSource, workflowType, workflowTypeVersion, inputsJson, workflowOptionsJson, labelsJson) + WorkflowSourceFilesWithoutImports(workflowSource, workflowType, workflowTypeVersion, inputsJson, workflowOptionsJson, labelsJson) } } -final case class WorkflowSourceFilesWithoutImports(wdlSource: WdlSource, +final case class WorkflowSourceFilesWithoutImports(workflowSource: WdlSource, workflowType: Option[WorkflowType], workflowTypeVersion: Option[WorkflowTypeVersion], inputsJson: WdlJson, workflowOptionsJson: WorkflowOptionsJson, labelsJson: WdlJson) extends WorkflowSourceFilesCollection -final case class WorkflowSourceFilesWithDependenciesZip(wdlSource: WdlSource, +final case class WorkflowSourceFilesWithDependenciesZip(workflowSource: WdlSource, workflowType: Option[WorkflowType], workflowTypeVersion: Option[WorkflowTypeVersion], inputsJson: WdlJson, workflowOptionsJson: WorkflowOptionsJson, labelsJson: WdlJson, importsZip: Array[Byte]) extends WorkflowSourceFilesCollection { - override def toString = s"WorkflowSourceFilesWithDependenciesZip($wdlSource, $inputsJson, $workflowOptionsJson, $labelsJson, <>)" + override def toString = s"WorkflowSourceFilesWithDependenciesZip($workflowSource, $inputsJson, $workflowOptionsJson, $labelsJson, <>)" } diff --git a/core/src/test/scala/cromwell/util/SampleWdl.scala b/core/src/test/scala/cromwell/util/SampleWdl.scala index 7d20ecdf596..06bc9828303 100644 --- a/core/src/test/scala/cromwell/util/SampleWdl.scala +++ b/core/src/test/scala/cromwell/util/SampleWdl.scala @@ -13,14 +13,14 @@ import wdl4s.values._ import scala.language.postfixOps trait SampleWdl extends TestFileUtil { - def wdlSource(runtime: String = ""): WdlSource + def workflowSource(runtime: String = ""): WdlSource def asWorkflowSources(runtime: String = "", workflowOptions: String = "{}", labels: String = "{}", workflowType: Option[String] = Option("WDL"), workflowTypeVersion: Option[String] = None) = { WorkflowSourceFilesWithoutImports( - wdlSource = wdlSource(runtime), + workflowSource = workflowSource(runtime), inputsJson = wdlJson, workflowOptionsJson = workflowOptions, labelsJson = labels, @@ -74,7 +74,7 @@ trait SampleWdl extends TestFileUtil { object SampleWdl { object HelloWorld extends SampleWdl { - override def wdlSource(runtime: String = "") = + override def workflowSource(runtime: String = "") = s""" |task hello { | String addressee @@ -99,7 +99,7 @@ object SampleWdl { } object HelloWorldWithoutWorkflow extends SampleWdl { - override def wdlSource(runtime: String = "") = + override def workflowSource(runtime: String = "") = s""" |task hello { | String addressee @@ -119,7 +119,7 @@ object SampleWdl { } object GoodbyeWorld extends SampleWdl { - override def wdlSource(runtime: String = "") = + override def workflowSource(runtime: String = "") = """ |task goodbye { | command { @@ -140,7 +140,7 @@ object SampleWdl { } object EmptyString extends SampleWdl { - override def wdlSource(runtime: String = "") = + override def workflowSource(runtime: String = "") = s""" |task hello { | command { @@ -182,13 +182,13 @@ object SampleWdl { object EmptyWorkflow extends SampleWdl { - override def wdlSource(runtime: String = "") = "workflow empty_workflow {}" + override def workflowSource(runtime: String = "") = "workflow empty_workflow {}" val rawInputs = Map.empty[String, Any] } object CoercionNotDefined extends SampleWdl { - override def wdlSource(runtime: String = "") = { + override def workflowSource(runtime: String = "") = { s""" |task summary { | String bfile @@ -219,7 +219,7 @@ object SampleWdl { } trait ThreeStepTemplate extends SampleWdl { - override def wdlSource(runtime: String = "") = sourceString().replaceAll("RUNTIME", runtime) + override def workflowSource(runtime: String = "") = sourceString().replaceAll("RUNTIME", runtime) private val outputSectionPlaceholder = "OUTPUTSECTIONPLACEHOLDER" def sourceString(outputsSection: String = "") = { val withPlaceholders = @@ -280,7 +280,7 @@ object SampleWdl { object ThreeStep extends ThreeStepTemplate object ThreeStepWithOutputsSection extends ThreeStepTemplate { - override def wdlSource(runtime: String = "") = sourceString(outputsSection = + override def workflowSource(runtime: String = "") = sourceString(outputsSection = """ |output { | cgrep.count @@ -290,7 +290,7 @@ object SampleWdl { } object ThreeStepWithInputsInTheOutputsSection extends ThreeStepTemplate { - override def wdlSource(runtime: String = "") = sourceString(outputsSection = + override def workflowSource(runtime: String = "") = sourceString(outputsSection = """ |output { | cgrep.pattern @@ -305,7 +305,7 @@ object SampleWdl { object WorkflowOutputsWithFiles extends SampleWdl { // ASCII art from http://www.chris.com/ascii/joan/www.geocities.com/SoHo/7373/flag.html with pipes // replaced by exclamation points to keep stripMargin from removing the flagpole. - override def wdlSource(runtime: String = "") = + override def workflowSource(runtime: String = "") = """ task A { command { @@ -372,7 +372,7 @@ object SampleWdl { } object WorkflowScatterOutputsWithFileArrays extends SampleWdl { - override def wdlSource(runtime: String = "") = + override def workflowSource(runtime: String = "") = """ |task A { | command { @@ -399,7 +399,7 @@ object SampleWdl { object DeclarationsWorkflow extends SampleWdl { - override def wdlSource(runtime: String): WdlSource = + override def workflowSource(runtime: String): WdlSource = s""" |task cat { | File file @@ -453,7 +453,7 @@ object SampleWdl { } trait ZeroOrMorePostfixQuantifier extends SampleWdl { - override def wdlSource(runtime: String): WdlSource = + override def workflowSource(runtime: String): WdlSource = s""" |task hello { | Array[String] person @@ -484,7 +484,7 @@ object SampleWdl { } trait OneOrMorePostfixQuantifier extends SampleWdl { - override def wdlSource(runtime: String): WdlSource = + override def workflowSource(runtime: String): WdlSource = s""" |task hello { | Array[String]+ person @@ -511,7 +511,7 @@ object SampleWdl { } object CurrentDirectory extends SampleWdl { - override def wdlSource(runtime: String): String = + override def workflowSource(runtime: String): String = """ |task whereami { | command { @@ -532,7 +532,7 @@ object SampleWdl { } object ArrayIO extends SampleWdl { - override def wdlSource(runtime: String = "") = + override def workflowSource(runtime: String = "") = s""" |task serialize { | Array[String] strs @@ -559,7 +559,7 @@ object SampleWdl { createFileArray(catRootDir) def cleanup() = cleanupFileArray(catRootDir) - override def wdlSource(runtime: String = "") = + override def workflowSource(runtime: String = "") = s""" |task cat { | Array[File]+ files @@ -584,7 +584,7 @@ object SampleWdl { createFileArray(catRootDir) def cleanup() = cleanupFileArray(catRootDir) - override def wdlSource(runtime: String = "") = + override def workflowSource(runtime: String = "") = s""" |task write_map { | Map[File, String] file_to_name @@ -673,7 +673,7 @@ object SampleWdl { |} """.stripMargin - override def wdlSource(runtime: String = "") = + override def workflowSource(runtime: String = "") = s"""$tasks | |workflow w { @@ -691,7 +691,7 @@ object SampleWdl { } object SiblingsScatterWdl extends ScatterWdl { - override def wdlSource(runtime: String = "") = + override def workflowSource(runtime: String = "") = s"""$tasks | |workflow w { @@ -712,7 +712,7 @@ object SampleWdl { } object SimpleScatterWdl extends SampleWdl { - override def wdlSource(runtime: String = "") = + override def workflowSource(runtime: String = "") = s"""task echo_int { | Int int | command {echo $${int}} @@ -735,7 +735,7 @@ object SampleWdl { } object SimpleScatterWdlWithOutputs extends SampleWdl { - override def wdlSource(runtime: String = "") = + override def workflowSource(runtime: String = "") = s"""task echo_int { | Int int | command {echo $${int}} @@ -760,7 +760,7 @@ object SampleWdl { } case class PrepareScatterGatherWdl(salt: String = UUID.randomUUID().toString) extends SampleWdl { - override def wdlSource(runtime: String = "") = { + override def workflowSource(runtime: String = "") = { s""" |# |# Goal here is to split up the input file into files of 1 line each (in the prepare) then in parallel call wc -w on each newly created file and count the words into another file then in the gather, sum the results of each parallel call to come up with @@ -828,7 +828,7 @@ object SampleWdl { } object FileClobber extends SampleWdl { - override def wdlSource(runtime: String = "") = + override def workflowSource(runtime: String = "") = s"""task read_line { | File in | command { cat $${in} } @@ -853,7 +853,7 @@ object SampleWdl { } object FilePassingWorkflow extends SampleWdl { - override def wdlSource(runtime: String): WdlSource = + override def workflowSource(runtime: String): WdlSource = s"""task a { | File in | String out_name = "out" @@ -893,7 +893,7 @@ object SampleWdl { * different */ case class CallCachingWorkflow(salt: String) extends SampleWdl { - override def wdlSource(runtime: String): WdlSource = + override def workflowSource(runtime: String): WdlSource = s"""task a { | File in | String out_name = "out" @@ -945,7 +945,7 @@ object SampleWdl { |k3\tv3 """.stripMargin.trim - override def wdlSource(runtime: String): WdlSource = + override def workflowSource(runtime: String): WdlSource = s""" |task a { | Array[String] array @@ -981,7 +981,7 @@ object SampleWdl { } object ArrayOfArrays extends SampleWdl { - override def wdlSource(runtime: String = "") = + override def workflowSource(runtime: String = "") = s"""task subtask { | Array[File] a | command { @@ -1021,7 +1021,7 @@ object SampleWdl { } object CallCachingHashingWdl extends SampleWdl { - override def wdlSource(runtime: String): WdlSource = + override def workflowSource(runtime: String): WdlSource = s"""task t { | Int a | Float b @@ -1059,7 +1059,7 @@ object SampleWdl { } object ExpressionsInInputs extends SampleWdl { - override def wdlSource(runtime: String = "") = + override def workflowSource(runtime: String = "") = s"""task echo { | String inString | command { @@ -1089,7 +1089,7 @@ object SampleWdl { } object WorkflowFailSlow extends SampleWdl { - override def wdlSource(runtime: String = "") = + override def workflowSource(runtime: String = "") = s""" task shouldCompleteFast { | Int a diff --git a/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala b/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala index c96cbe6b9b4..f7c35496ef6 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala @@ -37,7 +37,7 @@ class CromwellClient(val cromwellUrl: URL, val apiVersion: String)(implicit acto import cromwell.api.model.LabelsJsonFormatter._ val sourceBodyParts = Map( - "wdlSource" -> Option(workflowSubmission.wdl), + "workflowSource" -> Option(workflowSubmission.wdl), "workflowType" -> workflowSubmission.workflowType, "workflowTypeVersion" -> workflowSubmission.workflowTypeVersion, "workflowInputs" -> workflowSubmission.inputsJson, @@ -45,7 +45,7 @@ class CromwellClient(val cromwellUrl: URL, val apiVersion: String)(implicit acto "customLabels" -> Option(workflowSubmission.customLabels.toJson.toString) ) collect { case (name, Some(source: String)) => Multipart.FormData.BodyPart(name, HttpEntity(MediaTypes.`application/json`, ByteString(source))) } val zipBodyParts = Map( - "wdlDependencies" -> workflowSubmission.zippedImports + "workflowDependencies" -> workflowSubmission.zippedImports ) collect { case (name, Some(file)) => Multipart.FormData.BodyPart.fromPath(name, MediaTypes.`application/zip`, file.path) } val multipartFormData = Multipart.FormData((sourceBodyParts ++ zipBodyParts).toSeq : _*) diff --git a/engine/src/main/resources/swagger/cromwell.yaml b/engine/src/main/resources/swagger/cromwell.yaml index cceeffb732a..df4dd5cb405 100644 --- a/engine/src/main/resources/swagger/cromwell.yaml +++ b/engine/src/main/resources/swagger/cromwell.yaml @@ -98,7 +98,7 @@ paths: required: false type: file in: formData - - name: wdlDependencies + - name: workflowDependencies description: Zip file containing WDL dependencies required: false type: file @@ -149,7 +149,7 @@ paths: required: false type: file in: formData - - name: wdlDependencies + - name: workflowDependencies description: Zip file containing WDL dependencies required: false type: file diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala index d2dcc7f5b30..efe5bade34c 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala @@ -20,7 +20,6 @@ import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor._ import cromwell.engine.workflow.lifecycle.execution.{WorkflowExecutionActor, WorkflowMetadataHelper} import cromwell.subworkflowstore.SubWorkflowStoreActor.WorkflowComplete import cromwell.webservice.EngineStatsActor -import wdl4s.{LocallyQualifiedName => _} import scala.concurrent.Future import scala.util.Failure @@ -135,7 +134,7 @@ object WorkflowActor { def props(workflowId: WorkflowId, startMode: StartMode, - wdlSource: WorkflowSourceFilesCollection, + workflowSourceFilesCollection: WorkflowSourceFilesCollection, conf: Config, ioActor: ActorRef, serviceRegistryActor: ActorRef, @@ -149,12 +148,22 @@ object WorkflowActor { backendSingletonCollection: BackendSingletonCollection, serverMode: Boolean): Props = { Props( - new WorkflowActor(workflowId, startMode, wdlSource, conf, - ioActor = ioActor, serviceRegistryActor = serviceRegistryActor, workflowLogCopyRouter = workflowLogCopyRouter, - jobStoreActor = jobStoreActor, subWorkflowStoreActor = subWorkflowStoreActor, - callCacheReadActor = callCacheReadActor, callCacheWriteActor = callCacheWriteActor, - dockerHashActor = dockerHashActor, jobTokenDispenserActor = jobTokenDispenserActor, - backendSingletonCollection, serverMode)).withDispatcher(EngineDispatcher) + new WorkflowActor( + workflowId = workflowId, + startMode = startMode, + workflowSourceFilesCollection = workflowSourceFilesCollection, + conf = conf, + ioActor = ioActor, + serviceRegistryActor = serviceRegistryActor, + workflowLogCopyRouter = workflowLogCopyRouter, + jobStoreActor = jobStoreActor, + subWorkflowStoreActor = subWorkflowStoreActor, + callCacheReadActor = callCacheReadActor, + callCacheWriteActor = callCacheWriteActor, + dockerHashActor = dockerHashActor, + jobTokenDispenserActor = jobTokenDispenserActor, + backendSingletonCollection = backendSingletonCollection, + serverMode = serverMode)).withDispatcher(EngineDispatcher) } } @@ -163,7 +172,7 @@ object WorkflowActor { */ class WorkflowActor(val workflowId: WorkflowId, startMode: StartMode, - workflowSources: WorkflowSourceFilesCollection, + workflowSourceFilesCollection: WorkflowSourceFilesCollection, conf: Config, ioActor: ActorRef, override val serviceRegistryActor: ActorRef, @@ -195,7 +204,7 @@ class WorkflowActor(val workflowId: WorkflowId, val actor = context.actorOf(MaterializeWorkflowDescriptorActor.props(serviceRegistryActor, workflowId, importLocalFilesystem = !serverMode), "MaterializeWorkflowDescriptorActor") pushWorkflowStart(workflowId) - actor ! MaterializeWorkflowDescriptorCommand(workflowSources, conf) + actor ! MaterializeWorkflowDescriptorCommand(workflowSourceFilesCollection, conf) goto(MaterializingWorkflowDescriptorState) using stateData.copy(currentLifecycleStateActor = Option(actor)) case Event(AbortWorkflowCommand, stateData) => goto(WorkflowAbortedState) } @@ -307,7 +316,7 @@ class WorkflowActor(val workflowId: WorkflowId, * being recreated so that in case MaterializeWorkflowDescriptor fails, the workflow logs can still * be copied by accessing the workflow options outside of the EngineWorkflowDescriptor. */ - def bruteForceWorkflowOptions: WorkflowOptions = WorkflowOptions.fromJsonString(workflowSources.workflowOptionsJson).getOrElse(WorkflowOptions.fromJsonString("{}").get) + def bruteForceWorkflowOptions: WorkflowOptions = WorkflowOptions.fromJsonString(workflowSourceFilesCollection.workflowOptionsJson).getOrElse(WorkflowOptions.fromJsonString("{}").get) val system = context.system val ec = context.system.dispatcher def bruteForcePathBuilders: Future[List[PathBuilder]] = EngineFilesystems.pathBuildersForWorkflow(bruteForceWorkflowOptions)(system, ec) @@ -344,8 +353,8 @@ class WorkflowActor(val workflowId: WorkflowId, private def finalizationSucceeded(data: WorkflowActorData) = { val finalState = data.lastStateReached match { case StateCheckpoint(WorkflowAbortingState, None) => WorkflowAbortedState - case StateCheckpoint(state, Some(failures)) => WorkflowFailedState - case StateCheckpoint(state, None) => WorkflowSucceededState + case StateCheckpoint(_, Some(_)) => WorkflowFailedState + case StateCheckpoint(_, None) => WorkflowSucceededState } goto(finalState) using data.copy(currentLifecycleStateActor = None) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala index 27811381a65..2404c456ce7 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala @@ -406,7 +406,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, } else { List(WdlNamespace.directoryResolver(importsDirFile)) } - val results = WdlNamespaceWithWorkflow.load(w.wdlSource, importResolvers) + val results = WdlNamespaceWithWorkflow.load(w.workflowSource, importResolvers) importsDir.delete(swallowIOExceptions = true) results match { case Success(ns) => validateWorkflowNameLengths(ns) @@ -435,7 +435,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, } else { List.empty } - WdlNamespaceWithWorkflow.load(w.wdlSource, importResolvers) match { + WdlNamespaceWithWorkflow.load(w.workflowSource, importResolvers) match { case Failure(e) => s"Unable to load namespace from workflow: ${e.getMessage}".invalidNel case Success(namespace) => validateWorkflowNameLengths(namespace) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala index aa4efe3ceb9..f757bdcf52d 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala @@ -55,7 +55,7 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase) extends Workf private def fromWorkflowStoreEntry(workflowStoreEntry: WorkflowStoreEntry): WorkflowToStart = { val sources = WorkflowSourceFilesCollection( - wdlSource = workflowStoreEntry.workflowDefinition.toRawString, + workflowSource = workflowStoreEntry.workflowDefinition.toRawString, workflowType = Option(workflowStoreEntry.workflowType), workflowTypeVersion = workflowStoreEntry.workflowTypeVersion, inputsJson = workflowStoreEntry.workflowInputs.toRawString, @@ -75,7 +75,7 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase) extends Workf WorkflowStoreEntry( workflowExecutionUuid = WorkflowId.randomId().toString, - workflowDefinition = workflowSourceFiles.wdlSource.toClobOption, + workflowDefinition = workflowSourceFiles.workflowSource.toClobOption, // TODO ensure safety workflowType = workflowSourceFiles.workflowType.get, workflowTypeVersion = workflowSourceFiles.workflowTypeVersion, diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreSubmitActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreSubmitActor.scala index a1f54e76878..4c4c63edd18 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreSubmitActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreSubmitActor.scala @@ -82,7 +82,7 @@ final case class WorkflowStoreSubmitActor(store: WorkflowStore, serviceRegistryA MetadataEvent.empty(MetadataKey(id, None, WorkflowMetadataKeys.Outputs)), MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.Status), MetadataValue(WorkflowSubmitted)), - MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Workflow), MetadataValue(sourceFiles.wdlSource)), + MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Workflow), MetadataValue(sourceFiles.workflowSource)), MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Inputs), MetadataValue(sourceFiles.inputsJson)), MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Options), MetadataValue(sourceFiles.workflowOptionsJson)) ) diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala index b0dc71b5c62..0a1c03c2af2 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala @@ -70,7 +70,7 @@ class CromwellApiHandler(requestHandlerActor: ActorRef) extends Actor with Workf case ApiHandlerWorkflowSubmitBatch(sources) => requestHandlerActor ! WorkflowStoreActor.BatchSubmitWorkflows(sources.map(w => WorkflowSourceFilesCollection( - wdlSource = w.wdlSource, + workflowSource = w.workflowSource, workflowType = w.workflowType, workflowTypeVersion = w.workflowTypeVersion, inputsJson = w.inputsJson, diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala index 06d343d2636..bc6ecfe52af 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala @@ -1,18 +1,18 @@ package cromwell.webservice import akka.actor._ -import cats.data.Validated.{Invalid, Valid} import cats.data.NonEmptyList -import cats.syntax.validated._ +import cats.data.Validated.{Invalid, Valid} import cats.syntax.cartesian._ +import cats.syntax.validated._ import com.typesafe.config.{Config, ConfigFactory} -import cromwell.core.{WorkflowId, WorkflowOptions, WorkflowOptionsJson, WorkflowSourceFilesCollection} +import cromwell.core._ import cromwell.engine.backend.BackendConfiguration import cromwell.services.metadata.MetadataService._ import cromwell.webservice.WorkflowJsonSupport._ import cromwell.webservice.metadata.MetadataBuilderActor -import cromwell.core._ import lenthall.validation.ErrorOr.ErrorOr +import org.slf4j.LoggerFactory import spray.http.MediaTypes._ import spray.http._ import spray.httpx.SprayJsonSupport._ @@ -136,7 +136,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator { case class PartialWorkflowSources ( - wdlSource: Option[WdlSource], + workflowSource: Option[WdlSource], workflowType: Option[WorkflowType], workflowTypeVersion: Option[WorkflowTypeVersion], workflowInputs: Vector[WdlJson], @@ -146,8 +146,11 @@ trait CromwellApiService extends HttpService with PerRequestCreator { zippedImports: Option[Array[Byte]]) object PartialWorkflowSources { + + val log = LoggerFactory.getLogger(classOf[PartialWorkflowSources]) + def empty = PartialWorkflowSources( - wdlSource = None, + workflowSource = None, // TODO do not hardcode, especially not out here at the boundary layer good gravy workflowType = Option("WDL"), workflowTypeVersion = None, @@ -181,17 +184,17 @@ trait CromwellApiService extends HttpService with PerRequestCreator { def validateOptions(options: Option[WorkflowOptionsJson]): ErrorOr[WorkflowOptions] = WorkflowOptions.fromJsonString(options.getOrElse("{}")).tryToErrorOr leftMap { _ map { i => s"Invalid workflow options provided: $i" } } - def validateWdlSource(partialSource: PartialWorkflowSources): ErrorOr[WdlJson] = partialSource.wdlSource match { + def validateWorkflowSources(partialSource: PartialWorkflowSources): ErrorOr[WdlJson] = partialSource.workflowSource match { case Some(src) => src.validNel case _ => s"Incomplete workflow submission: $partialSource".invalidNel } partialSources match { case Valid(partialSource) => - (validateWdlSource(partialSource) |@| validateInputs(partialSource) |@| validateOptions(partialSource.workflowOptions)) map { - case (wdlSource, wfInputs, wfOptions) => + (validateWorkflowSources(partialSource) |@| validateInputs(partialSource) |@| validateOptions(partialSource.workflowOptions)) map { + case (wfSource, wfInputs, wfOptions) => wfInputs.map(inputsJson => WorkflowSourceFilesCollection( - wdlSource = wdlSource, + workflowSource = wfSource, workflowType = partialSource.workflowType, workflowTypeVersion = partialSource.workflowTypeVersion, inputsJson = inputsJson, @@ -203,12 +206,23 @@ trait CromwellApiService extends HttpService with PerRequestCreator { } } + def deprecationWarning(out: String, in: String): Unit = { + val warning = + s""" + |The '$out' parameter name has been deprecated in favor of '$in'. + |Support for '$out' will be removed from future versions of Cromwell. + |Please switch to using '$in' in future submissions. + """.stripMargin + log.warn(warning) + } + def fromSubmitRoute(formData: MultipartFormData, allowNoInputs: Boolean): Try[Seq[WorkflowSourceFilesCollection]] = { val partialSources = Try(formData.fields.foldLeft(PartialWorkflowSources.empty) { (partialSources: PartialWorkflowSources, bodyPart: BodyPart) => val name = bodyPart.name lazy val data = bodyPart.entity.data - if (name.contains("wdlSource")) { - partialSources.copy(wdlSource = Option(data.asString)) + if (name.contains("wdlSource") || name.contains("workflowSource")) { + if (name.contains("wdlSource")) deprecationWarning(out = "wdlSource", in = "workflowSource") + partialSources.copy(workflowSource = Option(data.asString)) } else if (name.contains("workflowType")) { partialSources.copy(workflowType = Option(data.asString)) } else if (name.contains("workflowTypeVersion")) { @@ -220,7 +234,8 @@ trait CromwellApiService extends HttpService with PerRequestCreator { partialSources.copy(workflowInputsAux = partialSources.workflowInputsAux + (index -> data.asString)) } else if (name.contains("workflowOptions")) { partialSources.copy(workflowOptions = Option(data.asString)) - } else if (name.contains("wdlDependencies")) { + } else if (name.contains("wdlDependencies") || name.contains("workflowDependencies")) { + if (name.contains("wdlDependencies")) deprecationWarning(out = "wdlDependencies", in = "workflowDependencies") partialSources.copy(zippedImports = Option(data.toByteArray)) } else if (name.contains("customLabels")) { partialSources.copy(customLabels = Option(data.asString)) diff --git a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala index 103a8b62cf5..92430bde7a6 100644 --- a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala @@ -10,7 +10,7 @@ import wdl4s.{ImportResolver, WdlNamespaceWithWorkflow} class ArrayWorkflowSpec extends CromwellTestKitWordSpec { val tmpDir = DefaultPathBuilder.createTempDirectory("ArrayWorkflowSpec") - val ns = WdlNamespaceWithWorkflow.load(SampleWdl.ArrayLiteral(tmpDir).wdlSource(), Seq.empty[ImportResolver]).get + val ns = WdlNamespaceWithWorkflow.load(SampleWdl.ArrayLiteral(tmpDir).workflowSource(), Seq.empty[ImportResolver]).get val expectedArray = WdlArray(WdlArrayType(WdlFileType), Seq(WdlFile("f1"), WdlFile("f2"), WdlFile("f3"))) "A static Array[File] declaration" should { diff --git a/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala index b17c934cf44..3a117350346 100644 --- a/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala +++ b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala @@ -359,7 +359,7 @@ abstract class CromwellTestKitSpec(val twms: TestWorkflowManagerSystem = default patienceConfig: PatienceConfig = defaultPatience)(implicit ec: ExecutionContext): Map[FullyQualifiedName, WdlValue] = { val rootActor = buildCromwellRootActor(config) val sources = WorkflowSourceFilesWithoutImports( - wdlSource = sampleWdl.wdlSource(runtime), + workflowSource = sampleWdl.workflowSource(runtime), workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = sampleWdl.wdlJson, diff --git a/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala b/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala index ca2b56de7e6..e687ff73e24 100644 --- a/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala @@ -9,7 +9,7 @@ import org.scalatest.{Matchers, WordSpecLike} class DeclarationWorkflowSpec extends Matchers with WordSpecLike { "A workflow with declarations in it" should { "compute inputs properly" in { - WdlNamespaceWithWorkflow.load(SampleWdl.DeclarationsWorkflow.wdlSource(runtime=""), Seq.empty[ImportResolver]).get.workflow.inputs shouldEqual Map( + WdlNamespaceWithWorkflow.load(SampleWdl.DeclarationsWorkflow.workflowSource(runtime=""), Seq.empty[ImportResolver]).get.workflow.inputs shouldEqual Map( "two_step.cat.file" -> WorkflowInput("two_step.cat.file", WdlFileType), "two_step.cgrep.str_decl" -> WorkflowInput("two_step.cgrep.str_decl", WdlStringType), "two_step.cgrep.pattern" -> WorkflowInput("two_step.cgrep.pattern", WdlStringType), diff --git a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala index a3ac386ce41..66c829153bb 100644 --- a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala @@ -13,7 +13,7 @@ import scala.util.{Success, Try} class MapWorkflowSpec extends CromwellTestKitWordSpec { private val pwd = DefaultPathBuilder.get(".") private val sampleWdl = SampleWdl.MapLiteral(pwd) - val ns = WdlNamespaceWithWorkflow.load(sampleWdl.wdlSource(), Seq.empty[ImportResolver]).get + val ns = WdlNamespaceWithWorkflow.load(sampleWdl.workflowSource(), Seq.empty[ImportResolver]).get val expectedMap = WdlMap(WdlMapType(WdlFileType, WdlStringType), Map( WdlFile("f1") -> WdlString("alice"), WdlFile("f2") -> WdlString("bob"), diff --git a/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala b/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala index 07e32d55f37..acb0afbb911 100644 --- a/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala @@ -11,7 +11,7 @@ class RestartWorkflowSpec extends CromwellTestKitWordSpec with WorkflowDescripto val actorSystem = ActorSystem("RestartWorkflowSpec", ConfigFactory.parseString(CromwellTestKitSpec.ConfigText)) //val localBackend = new OldStyleLocalBackend(CromwellTestkitSpec.DefaultLocalBackendConfigEntry, actorSystem) val sources = WorkflowSourceFilesWithoutImports( - wdlSource = + workflowSource = """task a {command{}} |workflow w { | call a diff --git a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala index 3e2adb05403..1c80e1a8aed 100644 --- a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala @@ -35,7 +35,7 @@ class SimpleWorkflowActorSpec extends CromwellTestKitWordSpec with BeforeAndAfte workflowId: WorkflowId, matchers: Matcher*): TestableWorkflowActorAndMetadataPromise = { val workflowSources = WorkflowSourceFilesWithoutImports( - wdlSource = sampleWdl.wdlSource(), + workflowSource = sampleWdl.workflowSource(), workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = rawInputsOverride, diff --git a/engine/src/test/scala/cromwell/engine/WorkflowAbortSpec.scala b/engine/src/test/scala/cromwell/engine/WorkflowAbortSpec.scala index 759adfdcb18..6e0580d62d4 100644 --- a/engine/src/test/scala/cromwell/engine/WorkflowAbortSpec.scala +++ b/engine/src/test/scala/cromwell/engine/WorkflowAbortSpec.scala @@ -14,7 +14,7 @@ class WorkflowAbortSpec extends CromwellTestKitWordSpec { // val waitThreshold = 10 // // // Start the workflow: -// val workflowId = messageAndWait[WorkflowId](SubmitWorkflow(TripleSleep.wdlSource(), TripleSleep.wdlJson, TripleSleep.rawInputs)) +// val workflowId = messageAndWait[WorkflowId](SubmitWorkflow(TripleSleep.workflowSource(), TripleSleep.wdlJson, TripleSleep.rawInputs)) // // def waitForStarted(currentAttempt: Int): Unit = { // val status = messageAndWait[Option[WorkflowState]](WorkflowStatus(workflowId)) diff --git a/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala b/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala index b9e23f18f7c..bcdb9f06dd3 100644 --- a/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala @@ -125,7 +125,7 @@ class WorkflowStoreActorSpec extends CromwellTestKitWordSpec with Matchers with workflowNel.toList.foreach { case WorkflowToStart(id, sources, state) => insertedIds.contains(id) should be(true) - sources.wdlSource should be(optionedSourceFiles.wdlSource) + sources.workflowSource should be(optionedSourceFiles.workflowSource) sources.inputsJson should be(optionedSourceFiles.inputsJson) state should be(WorkflowStoreState.Submitted) diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala index 86d180a0ae2..23134cba14b 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala @@ -35,8 +35,8 @@ class WorkflowActorSpec extends CromwellTestKitWordSpec with WorkflowDescriptorB var currentWorkflowId: WorkflowId = _ val currentLifecycleActor = TestProbe() - val wdlSources = ThreeStep.asWorkflowSources(workflowOptions = mockWorkflowOptions) - val descriptor = createMaterializedEngineWorkflowDescriptor(WorkflowId.randomId(), workflowSources = wdlSources) + val workflowSources = ThreeStep.asWorkflowSources(workflowOptions = mockWorkflowOptions) + val descriptor = createMaterializedEngineWorkflowDescriptor(WorkflowId.randomId(), workflowSources = workflowSources) val supervisorProbe = TestProbe() val deathwatch = TestProbe() val finalizationProbe = TestProbe() @@ -55,7 +55,7 @@ class WorkflowActorSpec extends CromwellTestKitWordSpec with WorkflowDescriptorB finalizationProbe = finalizationProbe, workflowId = currentWorkflowId, startMode = StartNewWorkflow, - workflowSources = wdlSources, + workflowSources = workflowSources, conf = ConfigFactory.load, ioActor = system.actorOf(SimpleIoActor.props), serviceRegistryActor = mockServiceRegistryActor, diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala index cf62b342f8a..f0bfeec9cf4 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala @@ -43,8 +43,8 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitWordSpec wit val badCustomLabelsFile="""{ "Label1": "valu£1", "--label2": "valuevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevalue" }""" val validInputsJson = HelloWorld.rawInputs.toJson.toString() - val wdlSourceWithDocker = HelloWorld.wdlSource(""" runtime { docker: "ubuntu:latest" } """) - val wdlSourceNoDocker = HelloWorld.wdlSource(""" runtime { } """) + val workflowSourceWithDocker = HelloWorld.workflowSource(""" runtime { docker: "ubuntu:latest" } """) + val workflowSourceNoDocker = HelloWorld.workflowSource(""" runtime { } """) val Timeout = 10.second.dilated val NoBehaviorActor = system.actorOf(Props.empty) @@ -59,7 +59,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitWordSpec wit "accept valid WDL, inputs and options files" in { val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false)) val sources = WorkflowSourceFilesWithoutImports( - wdlSource = wdlSourceNoDocker, + workflowSource = workflowSourceNoDocker, workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = validInputsJson, @@ -117,7 +117,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitWordSpec wit """.stripMargin val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false)) val sources = WorkflowSourceFilesWithoutImports( - wdlSource = wdl, + workflowSource = wdl, workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = "{}", @@ -170,7 +170,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitWordSpec wit // Run the test: val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, cromwellBackends, importLocalFilesystem = false)) val sources = WorkflowSourceFilesWithoutImports( - wdlSource = wdl, + workflowSource = wdl, workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = "{}", @@ -211,7 +211,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitWordSpec wit val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false)) val sources = WorkflowSourceFilesWithoutImports( - wdlSource = wdl, + workflowSource = wdl, workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = "{}", @@ -236,7 +236,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitWordSpec wit "reject an invalid WDL source" in { val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false)) val sources = WorkflowSourceFilesWithoutImports( - wdlSource = unstructuredFile, + workflowSource = unstructuredFile, workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = validInputsJson, @@ -266,7 +266,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitWordSpec wit """.stripMargin val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false)) val sources = WorkflowSourceFilesWithoutImports( - wdlSource = noWorkflowWdl, + workflowSource = noWorkflowWdl, workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = validInputsJson, @@ -297,7 +297,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitWordSpec wit """.stripMargin val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false)) val badWdlSources = WorkflowSourceFilesWithoutImports( - wdlSource = noWorkflowWdl, + workflowSource = noWorkflowWdl, workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = validInputsJson, @@ -322,7 +322,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitWordSpec wit "reject an invalid options file" in { val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false)) val sources = WorkflowSourceFilesWithoutImports( - wdlSource = wdlSourceNoDocker, + workflowSource = workflowSourceNoDocker, workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = validInputsJson, @@ -346,7 +346,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitWordSpec wit "reject an unstructured labels file" in { val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false)) val sources = WorkflowSourceFilesWithoutImports( - wdlSource = wdlSourceNoDocker, + workflowSource = workflowSourceNoDocker, workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = validInputsJson, @@ -372,7 +372,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitWordSpec wit "reject invalid labels" in { val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false)) val sources = WorkflowSourceFilesWithoutImports( - wdlSource = wdlSourceNoDocker, + workflowSource = workflowSourceNoDocker, workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = validInputsJson, @@ -401,7 +401,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitWordSpec wit "reject an invalid workflow inputs file" in { val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false)) val sources = WorkflowSourceFilesWithoutImports( - wdlSource = wdlSourceNoDocker, + workflowSource = workflowSourceNoDocker, workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = unstructuredFile, @@ -426,7 +426,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitWordSpec wit val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false)) val noInputsJson = "{}" val badOptionsSources = WorkflowSourceFilesWithoutImports( - wdlSource = wdlSourceNoDocker, + workflowSource = workflowSourceNoDocker, workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = noInputsJson, @@ -458,7 +458,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitWordSpec wit """.stripMargin val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false)) val sources = WorkflowSourceFilesWithoutImports( - wdlSource = wdl, + workflowSource = wdl, workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = "{}", @@ -493,7 +493,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitWordSpec wit val jsonInput = Map("foo.bad_one" -> "\"gs://this/is/a/bad/gcs/path.txt", "foo.good_one" -> "\"/local/path/is/ok.txt", "foo.bad_two" -> "\"gs://another/bad/gcs/path.txt").toJson.toString val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false)) val sources = WorkflowSourceFilesWithoutImports( - wdlSource = wdl, + workflowSource = wdl, workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = jsonInput, diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStoreBenchmark.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStoreBenchmark.scala index 091bd3c88a4..00a6a2ef2d2 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStoreBenchmark.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStoreBenchmark.scala @@ -22,7 +22,7 @@ object ExecutionStoreBenchmark extends Bench[Double] { lazy val reporter = new LoggingReporter[Double] lazy val persistor = Persistor.None - val wdl = WdlNamespaceWithWorkflow.load(SampleWdl.PrepareScatterGatherWdl().wdlSource(), Seq.empty).get + val wdl = WdlNamespaceWithWorkflow.load(SampleWdl.PrepareScatterGatherWdl().workflowSource(), Seq.empty).get val prepareCall: TaskCall = wdl.workflow.findCallByName("do_prepare").get.asInstanceOf[TaskCall] val scatterCall: TaskCall = wdl.workflow.findCallByName("do_scatter").get.asInstanceOf[TaskCall] diff --git a/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala b/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala index eb668499a54..2dec5f06a92 100644 --- a/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala +++ b/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala @@ -47,7 +47,7 @@ class SubWorkflowStoreSpec extends CromwellTestKitWordSpec with Matchers with Mo } workflowStoreService ! SubmitWorkflow(WorkflowSourceFilesWithoutImports( - wdlSource = "", + workflowSource = "", workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = "{}", diff --git a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala index bbd926fc92f..7acc1bc7ec6 100644 --- a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala @@ -138,7 +138,7 @@ class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Match behavior of "REST API submission endpoint" it should "return 201 for a successful workflow submission " in { - val bodyParts: Map[String, BodyPart] = Map("wdlSource" -> BodyPart(HelloWorld.wdlSource()), "workflowInputs" -> BodyPart(HelloWorld.rawInputs.toJson.toString())) + val bodyParts: Map[String, BodyPart] = Map("workflowSource" -> BodyPart(HelloWorld.workflowSource()), "workflowInputs" -> BodyPart(HelloWorld.rawInputs.toJson.toString())) Post(s"/workflows/$version", MultipartFormData(bodyParts)) ~> cromwellApiService.submitRoute ~> check { @@ -156,7 +156,7 @@ class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Match } it should "return 400 for an unrecognized form data request parameter " in { - val bodyParts: Map[String, BodyPart] = Map("incorrectParameter" -> BodyPart(HelloWorld.wdlSource())) + val bodyParts: Map[String, BodyPart] = Map("incorrectParameter" -> BodyPart(HelloWorld.workflowSource())) Post(s"/workflows/$version", MultipartFormData(bodyParts)) ~> cromwellApiService.submitRoute ~> check { @@ -182,7 +182,7 @@ class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Match |} |""".stripMargin - val bodyParts = Map("wdlSource" -> BodyPart(HelloWorld.wdlSource()), "workflowOptions" -> BodyPart(options)) + val bodyParts = Map("workflowSource" -> BodyPart(HelloWorld.workflowSource()), "workflowOptions" -> BodyPart(options)) Post(s"/workflows/$version", MultipartFormData(bodyParts)) ~> cromwellApiService.submitRoute ~> @@ -198,7 +198,7 @@ class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Match |{"read_from_cache": "true" |""".stripMargin - val bodyParts = Map("wdlSource" -> BodyPart(HelloWorld.wdlSource()), "workflowOptions" -> BodyPart(options)) + val bodyParts = Map("workflowSource" -> BodyPart(HelloWorld.workflowSource()), "workflowOptions" -> BodyPart(options)) Post(s"/workflows/$version", MultipartFormData(bodyParts)) ~> cromwellApiService.submitRoute ~> @@ -225,7 +225,7 @@ class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Match behavior of "REST API batch submission endpoint" it should "return 200 for a successful workflow submission " in { val inputs = HelloWorld.rawInputs.toJson - val bodyParts = Map("wdlSource" -> BodyPart(HelloWorld.wdlSource()), "workflowInputs" -> BodyPart(s"[$inputs, $inputs]")) + val bodyParts = Map("workflowSource" -> BodyPart(HelloWorld.workflowSource()), "workflowInputs" -> BodyPart(s"[$inputs, $inputs]")) Post(s"/workflows/$version/batch", MultipartFormData(bodyParts)) ~> cromwellApiService.submitBatchRoute ~> @@ -247,7 +247,7 @@ class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Match } it should "return 400 for an submission with no inputs" in { - val bodyParts = Map("wdlSource" -> BodyPart(HelloWorld.wdlSource())) + val bodyParts = Map("workflowSource" -> BodyPart(HelloWorld.workflowSource())) Post(s"/workflows/$version/batch", MultipartFormData(bodyParts)) ~> cromwellApiService.submitBatchRoute ~> diff --git a/src/main/scala/cromwell/CromwellCommandLine.scala b/src/main/scala/cromwell/CromwellCommandLine.scala index 494e9621d6e..77b82a83bdf 100644 --- a/src/main/scala/cromwell/CromwellCommandLine.scala +++ b/src/main/scala/cromwell/CromwellCommandLine.scala @@ -63,7 +63,7 @@ object RunSingle { val sourceFileCollection = pathParameters.importPath match { case Some(p) => (wdl |@| inputsJson |@| optionsJson |@| labelsJson) map { (w, i, o, l) => WorkflowSourceFilesWithDependenciesZip.apply( - wdlSource = w, + workflowSource = w, workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = i, @@ -73,7 +73,7 @@ object RunSingle { } case None => (wdl |@| inputsJson |@| optionsJson |@| labelsJson) map { (w, i, o, l) => WorkflowSourceFilesWithoutImports.apply( - wdlSource = w, + workflowSource = w, workflowType = Option("WDL"), workflowTypeVersion = None, inputsJson = i, diff --git a/src/test/scala/cromwell/CromwellCommandLineSpec.scala b/src/test/scala/cromwell/CromwellCommandLineSpec.scala index bd6da383add..162f1689e92 100644 --- a/src/test/scala/cromwell/CromwellCommandLineSpec.scala +++ b/src/test/scala/cromwell/CromwellCommandLineSpec.scala @@ -85,8 +85,8 @@ class CromwellCommandLineSpec extends FlatSpec with Matchers { val filePassing = DefaultPathBuilder.createTempFile("filePassing", ".wdl", Option(wdlDir)) val fileClobber = DefaultPathBuilder.createTempFile("fileClobber", ".wdl", Option(wdlDir)) - filePassing write FilePassingWorkflow.wdlSource() - fileClobber write FileClobber.wdlSource() + filePassing write FilePassingWorkflow.workflowSource() + fileClobber write FileClobber.workflowSource() val zippedDir = wdlDir.zip() val zippedPath = zippedDir.pathAsString @@ -113,7 +113,7 @@ object CromwellCommandLineSpec { lazy val wdlFile = { val file = DefaultPathBuilder.createTempFile(s"${sampleWdl.name}.", ".wdl") tempFiles :+= file - file write sampleWdl.wdlSource() + file write sampleWdl.workflowSource() } lazy val wdl = wdlFile.pathAsString diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala index 903794cfabb..29ad7338b95 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala @@ -398,7 +398,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow, + WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).workflowSource, Seq.empty[ImportResolver]).get.workflow, inputs, NoOptions, Labels.empty @@ -438,7 +438,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend TestActorRef[TestableJesJobExecutionActor] = { val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(sampleWdl.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow, + WdlNamespaceWithWorkflow.load(sampleWdl.asWorkflowSources(DockerAndDiskRuntime).workflowSource, Seq.empty[ImportResolver]).get.workflow, inputs, NoOptions, Labels.empty @@ -499,7 +499,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow, + WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).workflowSource, Seq.empty[ImportResolver]).get.workflow, inputs, NoOptions, Labels.empty @@ -528,7 +528,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow, + WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).workflowSource, Seq.empty[ImportResolver]).get.workflow, inputs, NoOptions, Labels.empty @@ -573,7 +573,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow, + WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).workflowSource, Seq.empty[ImportResolver]).get.workflow, Map.empty, NoOptions, Labels.empty @@ -606,7 +606,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend it should "create a JesFileInput for the monitoring script, when specified" in { val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow, + WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).workflowSource, Seq.empty[ImportResolver]).get.workflow, Map.empty, WorkflowOptions.fromJsonString("""{"monitoring_script": "gs://path/to/script"}""").get, Labels.empty @@ -628,7 +628,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend it should "not create a JesFileInput for the monitoring script, when not specified" in { val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow, + WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).workflowSource, Seq.empty[ImportResolver]).get.workflow, Map.empty, NoOptions, Labels.empty @@ -650,7 +650,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId(UUID.fromString("e6236763-c518-41d0-9688-432549a8bf7c")), WdlNamespaceWithWorkflow.load( - SampleWdl.HelloWorld.asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).wdlSource, Seq.empty[ImportResolver]).get.workflow, + SampleWdl.HelloWorld.asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).workflowSource, Seq.empty[ImportResolver]).get.workflow, Map.empty, WorkflowOptions.fromJsonString(""" {"jes_gcs_root": "gs://path/to/gcs_root"} """).get, Labels.empty @@ -682,7 +682,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId(UUID.fromString("e6236763-c518-41d0-9688-432549a8bf7d")), WdlNamespaceWithWorkflow.load( - new SampleWdl.ScatterWdl().asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).wdlSource, Seq.empty[ImportResolver]).get.workflow, + new SampleWdl.ScatterWdl().asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).workflowSource, Seq.empty[ImportResolver]).get.workflow, Map.empty, WorkflowOptions.fromJsonString(""" {"jes_gcs_root": "gs://path/to/gcs_root"} """).get, Labels.empty diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala index ca93bec6184..73a1cf85f98 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala @@ -18,7 +18,7 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with it should "map the correct filenames" in { GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials() - val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.wdlSource()) + val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.workflowSource()) val jobDescriptorKey = firstJobDescriptorKey(workflowDescriptor) val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) val workflowPaths = JesWorkflowPaths(workflowDescriptor, NoCredentials.getInstance(), NoCredentials.getInstance(), jesConfiguration) @@ -34,7 +34,7 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with it should "map the correct paths" in { GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials() - val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.wdlSource()) + val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.workflowSource()) val jobDescriptorKey = firstJobDescriptorKey(workflowDescriptor) val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) val workflowPaths = JesWorkflowPaths(workflowDescriptor, NoCredentials.getInstance(), NoCredentials.getInstance(), jesConfiguration) @@ -54,7 +54,7 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with it should "map the correct call context" in { GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials() - val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.wdlSource()) + val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.workflowSource()) val jobDescriptorKey = firstJobDescriptorKey(workflowDescriptor) val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) val workflowPaths = JesWorkflowPaths(workflowDescriptor, NoCredentials.getInstance(), NoCredentials.getInstance(), jesConfiguration) diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala index 6432ce3c160..d9970d368d7 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala @@ -224,7 +224,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe private def buildJesInitializationTestingBits(backendConfig: Config = dockerBackendConfig): TestingBits = { val workflowOptions = WorkflowOptions.fromMap(Map("refresh_token" -> "mytoken")).get - val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.wdlSource(), options = workflowOptions) + val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.workflowSource(), options = workflowOptions) val calls = workflowDescriptor.workflow.taskCalls val backendConfigurationDescriptor = BackendConfigurationDescriptor(backendConfig, globalConfig) val jesConfiguration = new JesConfiguration(backendConfigurationDescriptor) diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala index df529491e6b..e8faa6951e3 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala @@ -17,7 +17,7 @@ class JesWorkflowPathsSpec extends TestKitSuite with FlatSpecLike with Matchers it should "map the correct paths" in { GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials() - val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.wdlSource()) + val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.workflowSource()) val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) val workflowPaths = JesWorkflowPaths(workflowDescriptor, NoCredentials.getInstance(), NoCredentials.getInstance(), jesConfiguration)(system) diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala index 565819e4807..27564506411 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala @@ -33,7 +33,7 @@ class ConfigWdlNamespace(backendConfig: Config) { private val checkAliveSourceOption = checkAliveCommandOption.map(makeWdlSource( CheckAliveTask, _, jobIdRuntimeAttributes)) - private val wdlSource = + private val workflowSource = s""" |${submitSourceOption getOrElse ""} |${submitDockerSourceOption getOrElse ""} @@ -45,9 +45,9 @@ class ConfigWdlNamespace(backendConfig: Config) { * The wdl namespace containing the submit, kill, and check alive tasks. */ val wdlNamespace = { - WdlNamespace.loadUsingSource(wdlSource, None, None) match { + WdlNamespace.loadUsingSource(workflowSource, None, None) match { case Success(ns) => ns - case Failure(f) => throw new RuntimeException(s"Error parsing generated wdl:\n$wdlSource".stripMargin, f) + case Failure(f) => throw new RuntimeException(s"Error parsing generated wdl:\n$workflowSource".stripMargin, f) } } @@ -73,8 +73,8 @@ object ConfigWdlNamespace { } private def makeTask(taskName: String, command: String, declarations: String): Task = { - val wdlSource = makeWdlSource(taskName, command, declarations) - val wdlNamespace = WdlNamespace.loadUsingSource(wdlSource, None, None).get + val workflowSource = makeWdlSource(taskName, command, declarations) + val wdlNamespace = WdlNamespace.loadUsingSource(workflowSource, None, None).get wdlNamespace.findTask(taskName).getOrElse(throw new RuntimeException(s"Couldn't find task $taskName")) } diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala index 0c61145ea64..c5663dd3689 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala @@ -442,8 +442,8 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") () } - private def prepareJob(wdlSource: WdlSource = helloWorldWdl, runtimeString: String = passOnStderr, inputFiles: Option[Map[String, WdlValue]] = None, isCluster: Boolean = false): TestJobDescriptor = { - val backendWorkflowDescriptor = buildWorkflowDescriptor(wdl = wdlSource, inputs = inputFiles.getOrElse(Map.empty), runtime = runtimeString) + private def prepareJob(workflowSource: WdlSource = helloWorldWdl, runtimeString: String = passOnStderr, inputFiles: Option[Map[String, WdlValue]] = None, isCluster: Boolean = false): TestJobDescriptor = { + val backendWorkflowDescriptor = buildWorkflowDescriptor(workflowSource = workflowSource, inputs = inputFiles.getOrElse(Map.empty), runtime = runtimeString) val backendConfigurationDescriptor = if (isCluster) BackendConfigurationDescriptor(backendClusterConfig, ConfigFactory.load) else BackendConfigurationDescriptor(backendClientConfig, ConfigFactory.load) val jobDesc = jobDescriptorFromSingleCallWorkflow(backendWorkflowDescriptor, inputFiles.getOrElse(Map.empty), WorkflowOptions.empty, Set.empty) val jobPaths = if (isCluster) JobPathsWithDocker(jobDesc.key, backendWorkflowDescriptor, backendClusterConfig) else JobPathsWithDocker(jobDesc.key, backendWorkflowDescriptor, backendClientConfig) diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala index d7c0340d8a7..d3c7e0789cc 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala @@ -98,8 +98,8 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { ) } - private def createRuntimeAttributes(wdlSource: WdlSource, runtimeAttributes: String) = { - val workflowDescriptor = buildWorkflowDescriptor(wdlSource, runtime = runtimeAttributes) + private def createRuntimeAttributes(workflowSource: WdlSource, runtimeAttributes: String) = { + val workflowDescriptor = buildWorkflowDescriptor(workflowSource, runtime = runtimeAttributes) def createLookup(call: Call): ScopedLookupFunction = { val knownInputs = workflowDescriptor.knownValues