From e56c9e8a0b33f5f99c864c4de73283c4c64a0af7 Mon Sep 17 00:00:00 2001 From: "Francisco M. Casares" Date: Thu, 22 Sep 2016 16:24:46 -0700 Subject: [PATCH 001/375] Added configurable default working and output folders. (#1471) --- core/src/main/resources/reference.conf | 13 ++++++++----- .../impl/htcondor/HtCondorJobExecutionActor.scala | 7 +++++-- .../htcondor/HtCondorJobExecutionActorSpec.scala | 22 +++++++++++++--------- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 795103bad..02490fb28 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -228,13 +228,16 @@ backend { # # #Placeholders: # #1. Working directory. - # #2. Inputs volumes. - # #3. Output volume. - # #4. Docker image. - # #5. Job command. + # #2. Working directory volume. + # #3. Inputs volumes. + # #4. Output volume. + # #5. Docker image. + # #6. Job command. # docker { # #Allow soft links in dockerized jobs - # cmd = "docker run -w %s %s %s --rm %s %s" + # cmd = "docker run -w %s %s %s %s --rm %s %s" + # defaultWorkingDir = "/workingDir/" + # defaultOutputDir = "/output/" # } # # cache { diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala index f30111b3d..870b160cb 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala @@ -319,12 +319,15 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor log.debug("{} List of input volumes: {}", tag, dockerInputDataVol.mkString(",")) val dockerCmd = configurationDescriptor.backendConfig.getString("docker.cmd") + val defaultWorkingDir = configurationDescriptor.backendConfig.getString("docker.defaultWorkingDir") + val defaultOutputDir = configurationDescriptor.backendConfig.getString("docker.defaultOutputDir") val dockerVolume = "-v %s:%s" val dockerVolumeInputs = s"$dockerVolume:ro" // `v.get` is safe below since we filtered the list earlier with only defined elements val inputVolumes = dockerInputDataVol.distinct.map(v => dockerVolumeInputs.format(v, v)).mkString(" ") - val outputVolume = dockerVolume.format(executionDir.toAbsolutePath.toString, runtimeAttributes.dockerOutputDir.getOrElse(executionDir.toAbsolutePath.toString)) - val cmd = dockerCmd.format(runtimeAttributes.dockerWorkingDir.getOrElse(executionDir.toAbsolutePath.toString), inputVolumes, outputVolume, runtimeAttributes.dockerImage.get, jobCmd.get) + val outputVolume = dockerVolume.format(executionDir.toAbsolutePath.toString, runtimeAttributes.dockerOutputDir.getOrElse(defaultOutputDir)) + val workingDir = dockerVolume.format(executionDir.toAbsolutePath.toString, runtimeAttributes.dockerWorkingDir.getOrElse(defaultWorkingDir)) + val cmd = dockerCmd.format(runtimeAttributes.dockerWorkingDir.getOrElse(defaultWorkingDir), workingDir, inputVolumes, outputVolume, runtimeAttributes.dockerImage.get, jobCmd.get) log.debug("{} Docker command line to be used for task execution: {}.", tag, cmd) cmd } diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala index e44e66b4a..8a18ddf0d 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala @@ -102,7 +102,9 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc | root = "local-cromwell-executions" | | docker { - | cmd = "docker run -w %s %s %s --rm %s %s" + | cmd = "docker run -w %s %s %s %s --rm %s %s" + | defaultWorkingDir = "/workingDir/" + | defaultOutputDir = "/output/" | } | | filesystems { @@ -280,8 +282,8 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc """ |runtime { | docker: "ubuntu/latest" - | dockerWorkingDir: "/workingDir" - | dockerOutputDir: "/outputDir" + | dockerWorkingDir: "/workingDir/" + | dockerOutputDir: "/outputDir/" |} """.stripMargin val jsonInputFile = createCannedFile("testFile", "some content").pathAsString @@ -313,9 +315,10 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc val bashScript = Source.fromFile(jobPaths.script.toFile).getLines.mkString - assert(bashScript.contains("docker run -w /workingDir -v")) + assert(bashScript.contains("docker run -w /workingDir/ -v")) + assert(bashScript.contains(":/workingDir/")) assert(bashScript.contains(":ro")) - assert(bashScript.contains("/call-hello/execution:/outputDir --rm ubuntu/latest echo")) + assert(bashScript.contains("/call-hello/execution:/outputDir/ --rm ubuntu/latest echo")) cleanUpJob(jobPaths) } @@ -356,8 +359,8 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc """ |runtime { | docker: "ubuntu/latest" - | dockerWorkingDir: "/workingDir" - | dockerOutputDir: "/outputDir" + | dockerWorkingDir: "/workingDir/" + | dockerOutputDir: "/outputDir/" |} """.stripMargin @@ -396,10 +399,11 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc val bashScript = Source.fromFile(jobPaths.script.toFile).getLines.mkString - assert(bashScript.contains("docker run -w /workingDir -v")) + assert(bashScript.contains("docker run -w /workingDir/ -v")) + assert(bashScript.contains(":/workingDir/")) assert(bashScript.contains(tempDir1.toAbsolutePath.toString)) assert(bashScript.contains(tempDir2.toAbsolutePath.toString)) - assert(bashScript.contains("/call-hello/execution:/outputDir --rm ubuntu/latest echo")) + assert(bashScript.contains("/call-hello/execution:/outputDir/ --rm ubuntu/latest echo")) cleanUpJob(jobPaths) } From d6874976269d8295ece402e44112d80ae159287b Mon Sep 17 00:00:00 2001 From: Jeff Gentry Date: Fri, 23 Sep 2016 12:46:31 -0400 Subject: [PATCH 002/375] Fix incorrect URL --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3c5efc009..fd1477d50 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management * [GET /api/workflows/:version/:id/metadata](#get-apiworkflowsversionidmetadata) * [POST /api/workflows/:version/:id/abort](#post-apiworkflowsversionidabort) * [GET /api/workflows/:version/backends](#get-apiworkflowsversionbackends) - * [GET /api/workflows/:version/stats](#get-apiworkflowsversionstats) + * [GET /api/engine/:version/stats](#get-apiengineversionstats) * [Error handling](#error-handling) * [Developer](#developer) * [Generating table of contents on Markdown files](#generating-table-of-contents-on-markdown-files) @@ -2501,18 +2501,18 @@ Server: spray-can/1.3.3 } ``` -## GET /api/workflows/:version/stats +## GET /api/engine/:version/stats This endpoint returns some basic statistics on the current state of the engine. At the moment that includes the number of running workflows and the number of active jobs. cURL: ``` -$ curl http://localhost:8000/api/workflows/v1/stats +$ curl http://localhost:8000/api/engine/v1/stats ``` HTTPie: ``` -$ http http://localhost:8000/api/workflows/v1/stats +$ http http://localhost:8000/api/engine/v1/stats ``` Response: From fb258d8fee4366665e72016682d963f00ee0a351 Mon Sep 17 00:00:00 2001 From: Himanshu Jain Date: Fri, 23 Sep 2016 11:31:18 -0700 Subject: [PATCH 003/375] [HtCondor] pushing metadata events from htcondor backend (#1460) * pushing metadata events from htcondor backend * adding metadata event for command failed as well. --- .../impl/htcondor/HtCondorJobExecutionActor.scala | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala index 870b160cb..46678ae25 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala @@ -2,6 +2,7 @@ package cromwell.backend.impl.htcondor import java.nio.file.attribute.PosixFilePermission import java.nio.file.{FileSystems, Files, Path, Paths} +import java.util.UUID import akka.actor.{ActorRef, Props} import better.files.File @@ -13,6 +14,7 @@ import cromwell.backend.io.JobPaths import cromwell.backend.sfs.{SharedFileSystem, SharedFileSystemExpressionFunctions} import cromwell.core.{JobOutput, JobOutputs, LocallyQualifiedName} import cromwell.services.keyvalue.KeyValueServiceActor._ +import cromwell.services.metadata.CallMetadataKeys import org.apache.commons.codec.digest.DigestUtils import wdl4s._ import wdl4s.parser.MemoryUnit @@ -299,10 +301,26 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor } private def resolveJobCommand(localizedInputs: CallInputs): Try[String] = { - if (runtimeAttributes.dockerImage.isDefined) + val command = if (runtimeAttributes.dockerImage.isDefined) { modifyCommandForDocker(call.task.instantiateCommand(localizedInputs, callEngineFunction, identity), localizedInputs) - else + } else { call.task.instantiateCommand(localizedInputs, callEngineFunction, identity) + } + command match { + case Success(cmd) => tellMetadata(Map("command" -> cmd)) + case Failure(ex) => + log.error("{} failed to resolve command due to exception:{}", tag, ex) + tellMetadata(Map(s"${CallMetadataKeys.Failures}[${UUID.randomUUID().toString}]" -> ex.getMessage)) + } + command + } + + /** + * Fire and forget data to the metadata service + */ + private def tellMetadata(metadataKeyValues: Map[String, Any]): Unit = { + import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter + serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), metadataKeyValues) } private def modifyCommandForDocker(jobCmd: Try[String], localizedInputs: CallInputs): Try[String] = { From 32dcba9d104a48624c17a716e6ed0c66f2f73046 Mon Sep 17 00:00:00 2001 From: Kristian Cibulskis Date: Fri, 23 Sep 2016 12:12:08 -0400 Subject: [PATCH 004/375] added to entrypoint --- project/Settings.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Settings.scala b/project/Settings.scala index cfb0032d3..e3f949e32 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -57,7 +57,7 @@ object Settings { from("openjdk:8") expose(8000) add(artifact, artifactTargetPath) - entryPoint("java", "-jar", artifactTargetPath) + entryPoint("/bin/bash", "-c", "java -jar " + artifactTargetPath + " $CROMWELL_ARGS") } }, buildOptions in docker := BuildOptions( From b6f2a2b7abcde50e906c18da53d75f028dac85ab Mon Sep 17 00:00:00 2001 From: Kristian Cibulskis Date: Fri, 23 Sep 2016 13:44:21 -0400 Subject: [PATCH 005/375] fixed regression for docker run --- project/Settings.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Settings.scala b/project/Settings.scala index e3f949e32..785122cba 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -57,7 +57,7 @@ object Settings { from("openjdk:8") expose(8000) add(artifact, artifactTargetPath) - entryPoint("/bin/bash", "-c", "java -jar " + artifactTargetPath + " $CROMWELL_ARGS") + entryPoint("/bin/bash", "-c", "java -jar " + artifactTargetPath + " ${CROMWELL_ARGS} ${*}", "--") } }, buildOptions in docker := BuildOptions( From 33c3fd63bd2ac029f06b6f145e1db1cffe7bc129 Mon Sep 17 00:00:00 2001 From: Kristian Cibulskis Date: Mon, 26 Sep 2016 13:24:12 -0400 Subject: [PATCH 006/375] addressed PR comments --- project/Settings.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/project/Settings.scala b/project/Settings.scala index 785122cba..d9fd8bb36 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -57,6 +57,11 @@ object Settings { from("openjdk:8") expose(8000) add(artifact, artifactTargetPath) + + // If you use the 'exec' form for an entry point, shell processing is not performed and + // environment variable substitution does not occur. Thus we have to /bin/bash here + // and pass along any subsequent command line arguments + // See https://docs.docker.com/engine/reference/builder/#/entrypoint entryPoint("/bin/bash", "-c", "java -jar " + artifactTargetPath + " ${CROMWELL_ARGS} ${*}", "--") } }, From afc702d4c4baa62b2851367a64bc35b29fe45a55 Mon Sep 17 00:00:00 2001 From: Ruchi Date: Mon, 26 Sep 2016 15:55:51 -0400 Subject: [PATCH 007/375] enable caching for local centaur (#1478) * enable caching for local_centaur * test branch rm_convertToStandardSpec --- src/bin/travis/resources/local_centaur.conf | 18 ++++++++++++++++++ src/bin/travis/testCentaurLocal.sh | 3 ++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 src/bin/travis/resources/local_centaur.conf diff --git a/src/bin/travis/resources/local_centaur.conf b/src/bin/travis/resources/local_centaur.conf new file mode 100644 index 000000000..68ba866bf --- /dev/null +++ b/src/bin/travis/resources/local_centaur.conf @@ -0,0 +1,18 @@ +akka { + loggers = ["akka.event.slf4j.Slf4jLogger"] + logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" +} + +spray.can { + server { + request-timeout = 40s + } + client { + request-timeout = 40s + connecting-timeout = 40s + } +} + +call-caching { + enabled = true +} diff --git a/src/bin/travis/testCentaurLocal.sh b/src/bin/travis/testCentaurLocal.sh index 203d87c18..fe77a1347 100755 --- a/src/bin/travis/testCentaurLocal.sh +++ b/src/bin/travis/testCentaurLocal.sh @@ -31,6 +31,7 @@ set -e sbt assembly CROMWELL_JAR=$(find "$(pwd)/target/scala-2.11" -name "cromwell-*.jar") +LOCAL_CONF="$(pwd)/src/bin/travis/resources/local_centaur.conf" git clone https://github.com/broadinstitute/centaur.git cd centaur -./test_cromwell.sh -j"${CROMWELL_JAR}" +./test_cromwell.sh -j"${CROMWELL_JAR}" -c${LOCAL_CONF} From c239aad1712f3cb20e9d9c319b04f08710b12e40 Mon Sep 17 00:00:00 2001 From: Thib Date: Mon, 26 Sep 2016 16:03:51 -0400 Subject: [PATCH 008/375] Update develop version to 0.22 (#1486) --- project/Version.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Version.scala b/project/Version.scala index 99e00f9f6..e6de0a5cd 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -4,7 +4,7 @@ import sbt._ object Version { // Upcoming release, or current if we're on the master branch - val cromwellVersion = "0.20" + val cromwellVersion = "0.22" // Adapted from SbtGit.versionWithGit def cromwellVersionWithGit: Seq[Setting[_]] = From 4c8dd90902f1f913181ac377a04a19dd94003688 Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Fri, 2 Sep 2016 17:21:31 -0400 Subject: [PATCH 009/375] meow meow meow meow --- README.md | 4 +- .../main/scala/cromwell/backend/MemorySize.scala | 26 ++-- .../ContinueOnReturnCodeValidation.scala | 20 ++- .../backend/validation/CpuValidation.scala | 5 +- .../backend/validation/DockerValidation.scala | 5 +- .../validation/FailOnStderrValidation.scala | 6 +- .../backend/validation/MemoryValidation.scala | 15 +- .../validation/RuntimeAttributesDefault.scala | 8 +- .../validation/RuntimeAttributesValidation.scala | 23 +-- .../ValidatedRuntimeAttributesBuilder.scala | 35 ++--- .../validation/RuntimeAttributesDefaultSpec.scala | 12 +- .../RuntimeAttributesValidationSpec.scala | 169 ++++++++++----------- build.sbt | 1 + core/src/main/scala/cromwell/core/ConfigUtil.scala | 26 ++-- core/src/main/scala/cromwell/core/ErrorOr.scala | 22 +++ .../main/scala/cromwell/core/WorkflowState.scala | 7 +- core/src/main/scala/cromwell/core/package.scala | 7 +- .../database/slick/CallCachingSlickDatabase.scala | 2 +- .../database/slick/MetadataSlickDatabase.scala | 2 +- .../tables/CallCachingHashEntryComponent.scala | 4 +- .../slick/tables/MetadataEntryComponent.scala | 5 +- .../database/sql/CallCachingSqlDatabase.scala | 2 +- .../database/sql/MetadataSqlDatabase.scala | 2 +- .../scala/cromwell/engine/EngineFilesystems.scala | 7 +- .../engine/workflow/WorkflowManagerActor.scala | 7 +- .../MaterializeWorkflowDescriptorActor.scala | 75 +++++---- .../execution/WorkflowExecutionActor.scala | 11 +- .../execution/callcaching/CallCache.scala | 5 +- .../workflow/workflowstore/SqlWorkflowStore.scala | 4 +- .../workflow/workflowstore/WorkflowStore.scala | 2 +- .../workflowstore/WorkflowStoreActor.scala | 14 +- .../scala/cromwell/webservice/ApiDataModels.scala | 6 +- .../cromwell/webservice/CromwellApiHandler.scala | 4 +- .../cromwell/webservice/CromwellApiService.scala | 11 +- .../webservice/metadata/IndexedJsonValue.scala | 13 +- .../webservice/metadata/MetadataBuilderActor.scala | 11 +- .../cromwell/engine/WorkflowStoreActorSpec.scala | 26 ++-- .../workflow/lifecycle/CachingConfigSpec.scala | 8 +- .../workflowstore/InMemoryWorkflowStore.scala | 4 +- .../filesystems/gcs/GoogleConfiguration.scala | 29 ++-- .../scala/cromwell/filesystems/gcs/package.scala | 2 - project/Dependencies.scala | 11 +- .../cromwell/services/metadata/MetadataQuery.scala | 3 +- .../services/metadata/MetadataService.scala | 2 +- .../services/metadata/WorkflowQueryKey.scala | 50 +++--- .../metadata/WorkflowQueryParameters.scala | 29 ++-- .../metadata/impl/MetadataDatabaseAccess.scala | 13 +- .../metadata/WorkflowQueryParametersSpec.scala | 77 +++++----- src/main/scala/cromwell/CromwellCommandLine.scala | 28 ++-- .../impl/htcondor/HtCondorRuntimeAttributes.scala | 31 ++-- .../cromwell/backend/impl/jes/JesAttributes.scala | 22 +-- .../backend/impl/jes/JesRuntimeAttributes.scala | 45 +++--- .../backend/impl/jes/io/JesAttachedDisk.scala | 29 ++-- .../backend/impl/jes/JesAttributesSpec.scala | 2 +- .../PrimitiveRuntimeAttributesValidation.scala | 11 +- .../sfs/GcsWorkflowFileSystemProvider.scala | 5 +- .../impl/spark/SparkRuntimeAttributes.scala | 24 +-- 57 files changed, 535 insertions(+), 494 deletions(-) create mode 100644 core/src/main/scala/cromwell/core/ErrorOr.scala diff --git a/README.md b/README.md index fd1477d50..1800278e9 100644 --- a/README.md +++ b/README.md @@ -101,8 +101,8 @@ There is a [Cromwell gitter channel](https://gitter.im/broadinstitute/cromwell) The following is the toolchain used for development of Cromwell. Other versions may work, but these are recommended. -* [Scala 2.11.7](http://www.scala-lang.org/news/2.11.7/) -* [SBT 0.13.8](https://github.com/sbt/sbt/releases/tag/v0.13.8) +* [Scala 2.11.8](http://www.scala-lang.org/news/2.11.8/) +* [SBT 0.13.12](https://github.com/sbt/sbt/releases/tag/v0.13.12) * [Java 8](http://www.oracle.com/technetwork/java/javase/overview/java8-2100321.html) # Building diff --git a/backend/src/main/scala/cromwell/backend/MemorySize.scala b/backend/src/main/scala/cromwell/backend/MemorySize.scala index 52e6364a9..b207174b6 100644 --- a/backend/src/main/scala/cromwell/backend/MemorySize.scala +++ b/backend/src/main/scala/cromwell/backend/MemorySize.scala @@ -1,10 +1,16 @@ package cromwell.backend -import wdl4s.parser.MemoryUnit + +import cats.data.Validated._ +import cats.syntax.cartesian._ +import cats.syntax.validated._ +import cromwell.core.ErrorOr._ +import mouse.string._ import scala.language.postfixOps import scala.util.{Failure, Success, Try} -import scalaz.Scalaz._ +import wdl4s.parser.MemoryUnit + object MemorySize { val memoryPattern = """(\d+(?:\.\d+)?)\s*(\w+)""".r @@ -12,18 +18,18 @@ object MemorySize { def parse(unparsed: String): Try[MemorySize] = { unparsed match { case memoryPattern(amountString, unitString) => - val amount = amountString.parseDouble leftMap { + val amount: ErrorOr[Double] = amountString.parseDouble leftMap { _.getMessage - } toValidationNel - val unit = MemoryUnit.values find { + } toValidatedNel + val unit: ErrorOr[MemoryUnit] = MemoryUnit.values find { _.suffixes.contains(unitString) } match { - case Some(s) => s.successNel[String] - case None => s"$unitString is an invalid memory unit".failureNel + case Some(s) => s.validNel + case None => s"$unitString is an invalid memory unit".invalidNel } - (amount |@| unit) { (a, u) => new MemorySize(a, u) } match { - case scalaz.Success(memorySize) => Success(memorySize) - case scalaz.Failure(nel) => Failure(new UnsupportedOperationException(nel.list.toList.mkString("\n"))) + (amount |@| unit) map { (a, u) => new MemorySize(a, u) } match { + case Valid(memorySize) => Success(memorySize) + case Invalid(nel) => Failure(new UnsupportedOperationException(nel.toList.mkString("\n"))) } case _ => Failure(new UnsupportedOperationException(s"$unparsed should be of the form 'X Unit' where X is a number, e.g. 8 GB")) } diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala index 977c21618..81f9f1c89 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala @@ -1,12 +1,16 @@ package cromwell.backend.validation +import cats.data.Validated.{Invalid, Valid} +import cats.instances.list._ +import cats.syntax.traverse._ +import cats.syntax.validated._ import cromwell.backend.validation.RuntimeAttributesValidation._ import cromwell.core._ +import cromwell.core.ErrorOr._ import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType} import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString} import scala.util.Try -import scalaz.Scalaz._ /** * Validates the "continueOnReturnCode" runtime attribute a Boolean, a String 'true' or 'false', or an Array[Int], @@ -41,14 +45,14 @@ class ContinueOnReturnCodeValidation extends RuntimeAttributesValidation[Continu override def coercion = ContinueOnReturnCode.validWdlTypes override def validateValue = { - case WdlBoolean(value) => ContinueOnReturnCodeFlag(value).successNel - case WdlString(value) if Try(value.toBoolean).isSuccess => ContinueOnReturnCodeFlag(value.toBoolean).successNel - case WdlInteger(value) => ContinueOnReturnCodeSet(Set(value)).successNel + case WdlBoolean(value) => ContinueOnReturnCodeFlag(value).validNel + case WdlString(value) if Try(value.toBoolean).isSuccess => ContinueOnReturnCodeFlag(value.toBoolean).validNel + case WdlInteger(value) => ContinueOnReturnCodeSet(Set(value)).validNel case WdlArray(wdlType, seq) => val errorOrInts: ErrorOr[List[Int]] = (seq.toList map validateInt).sequence[ErrorOr, Int] errorOrInts match { - case scalaz.Success(ints) => ContinueOnReturnCodeSet(ints.toSet).successNel - case scalaz.Failure(_) => failureWithMessage + case Valid(ints) => ContinueOnReturnCodeSet(ints.toSet).validNel + case Invalid(_) => failureWithMessage } } @@ -56,8 +60,8 @@ class ContinueOnReturnCodeValidation extends RuntimeAttributesValidation[Continu case _: WdlBoolean => true case WdlString(value) if Try(value.toBoolean).isSuccess => true case _: WdlInteger => true - case WdlArray(WdlArrayType(WdlStringType), elements) => elements.forall(validateInt(_).isSuccess) - case WdlArray(WdlArrayType(WdlIntegerType), elements) => elements.forall(validateInt(_).isSuccess) + case WdlArray(WdlArrayType(WdlStringType), elements) => elements.forall(validateInt(_).isValid) + case WdlArray(WdlArrayType(WdlIntegerType), elements) => elements.forall(validateInt(_).isValid) } override protected def failureMessage = missingMessage diff --git a/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala b/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala index 051f63326..43303b6cb 100644 --- a/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala @@ -1,10 +1,9 @@ package cromwell.backend.validation +import cats.syntax.validated._ import wdl4s.types.WdlIntegerType import wdl4s.values.WdlInteger -import scalaz.Scalaz._ - /** * Validates the "cpu" runtime attribute an Integer greater than 0, returning the value as an `Int`. * @@ -39,7 +38,7 @@ class CpuValidation extends RuntimeAttributesValidation[Int] { override protected def validateValue = { case wdlValue if WdlIntegerType.coerceRawValue(wdlValue).isSuccess => WdlIntegerType.coerceRawValue(wdlValue).get match { - case WdlInteger(value) => if (value.toInt <= 0) wrongAmountMsg.failureNel else value.toInt.successNel + case WdlInteger(value) => if (value.toInt <= 0) wrongAmountMsg.invalidNel else value.toInt.validNel } } diff --git a/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala b/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala index 97129d391..12e090fcf 100644 --- a/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala @@ -1,10 +1,9 @@ package cromwell.backend.validation +import cats.syntax.validated._ import wdl4s.types.WdlStringType import wdl4s.values.WdlString -import scalaz.Scalaz._ - /** * Validates the "docker" runtime attribute as a String, returning it as `String`. * @@ -32,7 +31,7 @@ class DockerValidation extends RuntimeAttributesValidation[String] { override def coercion = Seq(WdlStringType) override protected def validateValue = { - case WdlString(value) => value.successNel + case WdlString(value) => value.validNel } override protected def failureMessage = missingMessage diff --git a/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala b/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala index 662499ddf..6c75a40cf 100644 --- a/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala @@ -1,10 +1,10 @@ package cromwell.backend.validation +import cats.syntax.validated._ import wdl4s.types.{WdlBooleanType, WdlStringType} import wdl4s.values.{WdlBoolean, WdlString} import scala.util.Try -import scalaz.Scalaz._ /** * Validates the "failOnStderr" runtime attribute as a Boolean or a String 'true' or 'false', returning the value as a @@ -36,8 +36,8 @@ class FailOnStderrValidation extends RuntimeAttributesValidation[Boolean] { override def coercion = Seq(WdlBooleanType, WdlStringType) override protected def validateValue = { - case WdlBoolean(value) => value.successNel - case WdlString(value) if Try(value.toBoolean).isSuccess => value.toBoolean.successNel + case WdlBoolean(value) => value.validNel + case WdlString(value) if Try(value.toBoolean).isSuccess => value.toBoolean.validNel } override def validateExpression = { diff --git a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala index d5a2c8367..81e8f3ea7 100644 --- a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala @@ -1,13 +1,12 @@ package cromwell.backend.validation +import cats.syntax.validated._ import cromwell.backend.MemorySize -import cromwell.core._ +import cromwell.core.ErrorOr._ import wdl4s.parser.MemoryUnit import wdl4s.types.{WdlIntegerType, WdlStringType} import wdl4s.values.{WdlInteger, WdlString} -import scalaz.Scalaz._ - /** * Validates the "memory" runtime attribute as an Integer or String with format '8 GB', returning the value as a * `MemorySize`. @@ -39,11 +38,11 @@ object MemoryValidation { private[validation] def validateMemoryString(value: String): ErrorOr[MemorySize] = { MemorySize.parse(value) match { case scala.util.Success(memorySize: MemorySize) if memorySize.amount > 0 => - memorySize.to(MemoryUnit.GB).successNel + memorySize.to(MemoryUnit.GB).validNel case scala.util.Success(memorySize: MemorySize) => - wrongAmountFormat.format(memorySize.amount).failureNel + wrongAmountFormat.format(memorySize.amount).invalidNel case scala.util.Failure(throwable) => - missingFormat.format(throwable.getMessage).failureNel + missingFormat.format(throwable.getMessage).invalidNel } } @@ -52,9 +51,9 @@ object MemoryValidation { private[validation] def validateMemoryInteger(value: Int): ErrorOr[MemorySize] = { if (value <= 0) - wrongAmountFormat.format(value).failureNel + wrongAmountFormat.format(value).invalidNel else - MemorySize(value, MemoryUnit.Bytes).to(MemoryUnit.GB).successNel + MemorySize(value, MemoryUnit.Bytes).to(MemoryUnit.GB).validNel } } diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala index 0a2ed3b5c..7a080170e 100644 --- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala +++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala @@ -1,13 +1,13 @@ package cromwell.backend.validation -import cromwell.core.{OptionNotFoundException, EvaluatedRuntimeAttributes, WorkflowOptions} +import cats.data.ValidatedNel +import cats.syntax.validated._ +import cromwell.core.{EvaluatedRuntimeAttributes, OptionNotFoundException, WorkflowOptions} import wdl4s.types.WdlType import wdl4s.util.TryUtil import wdl4s.values.WdlValue import scala.util.{Failure, Try} -import scalaz.Scalaz._ -import scalaz.ValidationNel object RuntimeAttributesDefault { @@ -35,5 +35,5 @@ object RuntimeAttributesDefault { }) } - def noValueFoundFor[A](attribute: String): ValidationNel[String, A] = s"Can't find an attribute value for key $attribute".failureNel + def noValueFoundFor[A](attribute: String): ValidatedNel[String, A] = s"Can't find an attribute value for key $attribute".invalidNel } diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala index 47e90edae..9e56c71be 100644 --- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala @@ -1,8 +1,10 @@ package cromwell.backend.validation +import cats.syntax.validated._ import cromwell.backend.wdl.OnlyPureFunctions import cromwell.backend.{MemorySize, RuntimeAttributeDefinition} import cromwell.core._ +import cromwell.core.ErrorOr._ import org.slf4j.Logger import wdl4s.WdlExpression import wdl4s.WdlExpression._ @@ -10,7 +12,6 @@ import wdl4s.types.{WdlBooleanType, WdlIntegerType, WdlType} import wdl4s.values._ import scala.util.{Failure, Success} -import scalaz.Scalaz._ object RuntimeAttributesValidation { @@ -48,22 +49,22 @@ object RuntimeAttributesValidation { missingValidationMessage: String): ErrorOr[T] = { valueOption match { case Some(value) => - validation.validateValue.applyOrElse(value, (_: Any) => missingValidationMessage.failureNel) + validation.validateValue.applyOrElse(value, (_: Any) => missingValidationMessage.invalidNel) case None => onMissingValue } } def validateInt(value: WdlValue): ErrorOr[Int] = { WdlIntegerType.coerceRawValue(value) match { - case scala.util.Success(WdlInteger(i)) => i.intValue.successNel - case _ => s"Could not coerce ${value.valueString} into an integer".failureNel + case scala.util.Success(WdlInteger(i)) => i.intValue.validNel + case _ => s"Could not coerce ${value.valueString} into an integer".invalidNel } } def validateBoolean(value: WdlValue): ErrorOr[Boolean] = { WdlBooleanType.coerceRawValue(value) match { - case scala.util.Success(WdlBoolean(b)) => b.booleanValue.successNel - case _ => s"Could not coerce ${value.valueString} into a boolean".failureNel + case scala.util.Success(WdlBoolean(b)) => b.booleanValue.validNel + case _ => s"Could not coerce ${value.valueString} into a boolean".invalidNel } } @@ -238,7 +239,7 @@ trait RuntimeAttributesValidation[ValidatedType] { /** * Validates the wdl value. * - * @return The validated value or an error, wrapped in a scalaz validation. + * @return The validated value or an error, wrapped in a cats validation. */ protected def validateValue: PartialFunction[WdlValue, ErrorOr[ValidatedType]] @@ -282,7 +283,7 @@ trait RuntimeAttributesValidation[ValidatedType] { * * @return Wrapped failureMessage. */ - protected final lazy val failureWithMessage: ErrorOr[ValidatedType] = failureMessage.failureNel + protected final lazy val failureWithMessage: ErrorOr[ValidatedType] = failureMessage.invalidNel /** * Runs this validation on the value matching key. @@ -313,7 +314,7 @@ trait RuntimeAttributesValidation[ValidatedType] { */ def validateOptionalExpression(wdlExpressionMaybe: Option[WdlValue]): Boolean = { wdlExpressionMaybe match { - case None => staticDefaultOption.isDefined || validateNone.isSuccess + case None => staticDefaultOption.isDefined || validateNone.isValid case Some(wdlExpression: WdlExpression) => /* TODO: BUG: @@ -384,7 +385,7 @@ trait OptionalRuntimeAttributesValidation[ValidatedType] extends RuntimeAttribut * This method is the same as `validateValue`, but allows the implementor to not have to wrap the response in an * `Option`. * - * @return The validated value or an error, wrapped in a scalaz validation. + * @return The validated value or an error, wrapped in a cats validation. */ protected def validateOption: PartialFunction[WdlValue, ErrorOr[ValidatedType]] @@ -394,5 +395,5 @@ trait OptionalRuntimeAttributesValidation[ValidatedType] extends RuntimeAttribut override def apply(wdlValue: WdlValue) = validateOption.apply(wdlValue).map(Option.apply) } - override final protected lazy val validateNone = None.successNel[String] + override final protected lazy val validateNone = None.validNel[String] } diff --git a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala index 22c632520..59e8cec26 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala @@ -1,14 +1,13 @@ package cromwell.backend.validation +import cats.data.Validated._ +import cats.instances.list._ import cromwell.backend.RuntimeAttributeDefinition -import cromwell.core._ import lenthall.exception.MessageAggregation +import cromwell.core.ErrorOr._ import org.slf4j.Logger -import wdl4s.types.WdlType import wdl4s.values.WdlValue -import scalaz.{Failure, Success} - final case class ValidatedRuntimeAttributes(attributes: Map[String, Any]) /** @@ -54,32 +53,26 @@ trait ValidatedRuntimeAttributesBuilder { val runtimeAttributesErrorOr: ErrorOr[ValidatedRuntimeAttributes] = validate(attrs) runtimeAttributesErrorOr match { - case Success(runtimeAttributes) => runtimeAttributes - case Failure(nel) => throw new RuntimeException with MessageAggregation { + case Valid(runtimeAttributes) => runtimeAttributes + case Invalid(nel) => throw new RuntimeException with MessageAggregation { override def exceptionContext: String = "Runtime attribute validation failed" - override def errorMessages: Traversable[String] = nel.list.toList + override def errorMessages: Traversable[String] = nel.toList } } } private def validate(values: Map[String, WdlValue]): ErrorOr[ValidatedRuntimeAttributes] = { - val validationsForValues: Seq[RuntimeAttributesValidation[_]] = validations ++ unsupportedExtraValidations - val errorsOrValuesMap: Seq[(String, ErrorOr[Any])] = - validationsForValues.map(validation => validation.key -> validation.validate(values)) - - import scalaz.Scalaz._ + val validationsForValues = validations ++ unsupportedExtraValidations + val listOfKeysToErrorOrAnys: List[(String, ErrorOr[Any])] = + validationsForValues.map(validation => validation.key -> validation.validate(values)).toList - val emptyResult: ErrorOr[List[(String, Any)]] = List.empty[(String, Any)].success - val validationResult = errorsOrValuesMap.foldLeft(emptyResult) { (agg, errorOrValue) => - agg +++ { - errorOrValue match { - case (key, Success(value)) => List(key -> value).success - case (key, Failure(nel)) => nel.failure - } - } + val listOfErrorOrKeysToAnys: List[ErrorOr[(String, Any)]] = listOfKeysToErrorOrAnys map { + case (key, errorOrAny) => errorOrAny map { any => (key, any) } } - validationResult.map(result => ValidatedRuntimeAttributes(result.toMap)) + import cats.syntax.traverse._ + val errorOrListOfKeysToAnys: ErrorOr[List[(String, Any)]] = listOfErrorOrKeysToAnys.sequence[ErrorOr, (String, Any)] + errorOrListOfKeysToAnys map { listOfKeysToAnys => ValidatedRuntimeAttributes(listOfKeysToAnys.toMap) } } } diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala index 14898d92d..d1df9d790 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala @@ -1,11 +1,11 @@ package cromwell.backend.validation +import cromwell.backend.validation.RuntimeAttributesDefault._ import cromwell.core.WorkflowOptions -import org.scalatest.{Matchers, FlatSpec} +import org.scalatest.{FlatSpec, Matchers} import spray.json._ -import cromwell.backend.validation.RuntimeAttributesDefault._ import wdl4s.types._ -import wdl4s.values.{WdlArray, WdlInteger, WdlBoolean, WdlString} +import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString} class RuntimeAttributesDefaultSpec extends FlatSpec with Matchers { @@ -103,8 +103,8 @@ class RuntimeAttributesDefaultSpec extends FlatSpec with Matchers { ) } - "noValueFoundFor" should "provide a failureNel for missing values" in { - import scalaz.Scalaz._ - noValueFoundFor("myKey") shouldBe "Can't find an attribute value for key myKey".failureNel + "noValueFoundFor" should "provide an invalidNel for missing values" in { + import cats.syntax.validated._ + noValueFoundFor("myKey") shouldBe "Can't find an attribute value for key myKey".invalidNel } } diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala index 6c914f161..22180bb77 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala @@ -1,150 +1,149 @@ package cromwell.backend.validation +import cats.data.Validated.{Invalid, Valid} +import cats.syntax.validated._ import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} -import wdl4s.parser.MemoryUnit -import wdl4s.types.{WdlIntegerType, WdlStringType, WdlArrayType} +import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType} import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString} -import scalaz.Scalaz._ - class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with BeforeAndAfterAll { "RuntimeAttributesValidation" should { "return success when tries to validate a valid Docker entry" in { val dockerValue = Some(WdlString("someImage")) val result = RuntimeAttributesValidation.validateDocker(dockerValue, - "Failed to get Docker mandatory key from runtime attributes".failureNel) + "Failed to get Docker mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x.get == "someImage") - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x.get == "someImage") + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return success (based on defined HoF) when tries to validate a docker entry but it does not contain a value" in { val dockerValue = None - val result = RuntimeAttributesValidation.validateDocker(dockerValue, None.successNel) + val result = RuntimeAttributesValidation.validateDocker(dockerValue, None.validNel) result match { - case scalaz.Success(x) => assert(!x.isDefined) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x.isEmpty) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return failure (based on defined HoF) when tries to validate a docker entry but it does not contain a value" in { val dockerValue = None val result = RuntimeAttributesValidation.validateDocker(dockerValue, - "Failed to get Docker mandatory key from runtime attributes".failureNel) + "Failed to get Docker mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Failed to get Docker mandatory key from runtime attributes") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Failed to get Docker mandatory key from runtime attributes") } } "return failure when there is an invalid docker runtime attribute defined" in { val dockerValue = Some(WdlInteger(1)) val result = RuntimeAttributesValidation.validateDocker(dockerValue, - "Failed to get Docker mandatory key from runtime attributes".failureNel) + "Failed to get Docker mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Expecting docker runtime attribute to be a String") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting docker runtime attribute to be a String") } } "return success when tries to validate a failOnStderr boolean entry" in { val failOnStderrValue = Some(WdlBoolean(true)) val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, - "Failed to get failOnStderr mandatory key from runtime attributes".failureNel) + "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return success when tries to validate a failOnStderr 'true' string entry" in { val failOnStderrValue = Some(WdlString("true")) val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, - "Failed to get failOnStderr mandatory key from runtime attributes".failureNel) + "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return success when tries to validate a failOnStderr 'false' string entry" in { val failOnStderrValue = Some(WdlString("false")) val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, - "Failed to get failOnStderr mandatory key from runtime attributes".failureNel) + "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(!x) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(!x) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return failure when there is an invalid failOnStderr runtime attribute defined" in { val failOnStderrValue = Some(WdlInteger(1)) val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, - "Failed to get failOnStderr mandatory key from runtime attributes".failureNel) + "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'") } } "return success (based on defined HoF) when tries to validate a failOnStderr entry but it does not contain a value" in { val failOnStderrValue = None - val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, true.successNel) + val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, true.validNel) result match { - case scalaz.Success(x) => assert(x) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return success when tries to validate a continueOnReturnCode boolean entry" in { val continueOnReturnCodeValue = Some(WdlBoolean(true)) val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".failureNel) + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x == ContinueOnReturnCodeFlag(true)) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x == ContinueOnReturnCodeFlag(true)) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return success when tries to validate a continueOnReturnCode 'true' string entry" in { val continueOnReturnCodeValue = Some(WdlString("true")) val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".failureNel) + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x == ContinueOnReturnCodeFlag(true)) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x == ContinueOnReturnCodeFlag(true)) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return success when tries to validate a continueOnReturnCode 'false' string entry" in { val continueOnReturnCodeValue = Some(WdlString("false")) val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".failureNel) + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x == ContinueOnReturnCodeFlag(false)) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x == ContinueOnReturnCodeFlag(false)) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return success when tries to validate a continueOnReturnCode int entry" in { val continueOnReturnCodeValue = Some(WdlInteger(12)) val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".failureNel) + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x == ContinueOnReturnCodeSet(Set(12))) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x == ContinueOnReturnCodeSet(Set(12))) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return failure when there is an invalid continueOnReturnCode runtime attribute defined" in { val continueOnReturnCodeValue = Some(WdlString("yes")) val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".failureNel) + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]") } } @@ -152,29 +151,29 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be "return success when there is a valid integer array in continueOnReturnCode runtime attribute" in { val continueOnReturnCodeValue = Some(WdlArray(WdlArrayType(WdlIntegerType), Seq(WdlInteger(1), WdlInteger(2)))) val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".failureNel) + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x == ContinueOnReturnCodeSet(Set(1, 2))) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x == ContinueOnReturnCodeSet(Set(1, 2))) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return failure when there is an invalid array in continueOnReturnCode runtime attribute" in { val continueOnReturnCodeValue = Some(WdlArray(WdlArrayType(WdlStringType), Seq(WdlString("one"), WdlString("two")))) val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".failureNel) + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]") } } "return success (based on defined HoF) when tries to validate a continueOnReturnCode entry but it does not contain a value" in { val continueOnReturnCodeValue = None - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, ContinueOnReturnCodeFlag(false).successNel) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, ContinueOnReturnCodeFlag(false).validNel) result match { - case scalaz.Success(x) => assert(x == ContinueOnReturnCodeFlag(false)) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x == ContinueOnReturnCodeFlag(false)) + case Invalid(e) => fail(e.toList.mkString(" ")) } } @@ -182,20 +181,20 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be val expectedGb = 1 val memoryValue = Some(WdlInteger(1000000000)) val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".failureNel) + "Failed to get memory mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x.amount == expectedGb) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x.amount == expectedGb) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return failure when tries to validate an invalid Integer memory entry" in { val memoryValue = Some(WdlInteger(-1)) val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".failureNel) + "Failed to get memory mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Expecting memory runtime attribute value greater than 0 but got -1") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting memory runtime attribute value greater than 0 but got -1") } } @@ -203,80 +202,80 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be val expectedGb = 2 val memoryValue = Some(WdlString("2 GB")) val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".failureNel) + "Failed to get memory mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x.amount == expectedGb) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x.amount == expectedGb) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return failure when tries to validate an invalid size in String memory entry" in { val memoryValue = Some(WdlString("0 GB")) val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".failureNel) + "Failed to get memory mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Expecting memory runtime attribute value greater than 0 but got 0.0") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting memory runtime attribute value greater than 0 but got 0.0") } } "return failure when tries to validate an invalid String memory entry" in { val memoryValue = Some(WdlString("value")) val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".failureNel) + "Failed to get memory mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: value should be of the form 'X Unit' where X is a number, e.g. 8 GB") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: value should be of the form 'X Unit' where X is a number, e.g. 8 GB") } } "return failure when tries to validate an invalid memory entry" in { val memoryValue = Some(WdlBoolean(true)) val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".failureNel) + "Failed to get memory mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: Not supported WDL type value") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: Not supported WDL type value") } } "return failure when tries to validate a non-provided memory entry" in { val memoryValue = None val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".failureNel) + "Failed to get memory mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Failed to get memory mandatory key from runtime attributes") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Failed to get memory mandatory key from runtime attributes") } } "return success when tries to validate a valid cpu entry" in { val cpuValue = Some(WdlInteger(1)) val result = RuntimeAttributesValidation.validateCpu(cpuValue, - "Failed to get cpu mandatory key from runtime attributes".failureNel) + "Failed to get cpu mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x == 1) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x == 1) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return failure when tries to validate an invalid cpu entry" in { val cpuValue = Some(WdlInteger(-1)) val result = RuntimeAttributesValidation.validateCpu(cpuValue, - "Failed to get cpu mandatory key from runtime attributes".failureNel) + "Failed to get cpu mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Expecting cpu runtime attribute value greater than 0") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting cpu runtime attribute value greater than 0") } } "return failure when tries to validate a non-provided cpu entry" in { val cpuValue = None val result = RuntimeAttributesValidation.validateMemory(cpuValue, - "Failed to get cpu mandatory key from runtime attributes".failureNel) + "Failed to get cpu mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Failed to get cpu mandatory key from runtime attributes") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Failed to get cpu mandatory key from runtime attributes") } } } diff --git a/build.sbt b/build.sbt index 26934f881..8e666691f 100644 --- a/build.sbt +++ b/build.sbt @@ -8,6 +8,7 @@ lazy val core = (project in file("core")) lazy val gcsFileSystem = (project in file("filesystems/gcs")) .settings(gcsFileSystemSettings:_*) .withTestSettings + .dependsOn(core) lazy val databaseSql = (project in file("database/sql")) .settings(databaseSqlSettings:_*) diff --git a/core/src/main/scala/cromwell/core/ConfigUtil.scala b/core/src/main/scala/cromwell/core/ConfigUtil.scala index 0d90f3e2e..9f22c330e 100644 --- a/core/src/main/scala/cromwell/core/ConfigUtil.scala +++ b/core/src/main/scala/cromwell/core/ConfigUtil.scala @@ -2,13 +2,13 @@ package cromwell.core import java.net.URL +import cats.data.ValidatedNel +import cats.syntax.validated._ import com.typesafe.config.{Config, ConfigException, ConfigValue} import org.slf4j.LoggerFactory import scala.collection.JavaConversions._ import scala.reflect.{ClassTag, classTag} -import scalaz.Scalaz._ -import scalaz._ object ConfigUtil { @@ -30,21 +30,21 @@ object ConfigUtil { /** * Validates that the value for this key is a well formed URL. */ - def validateURL(key: String): ValidationNel[String, URL] = key.validateAny { url => + def validateURL(key: String): ValidatedNel[String, URL] = key.validateAny { url => new URL(config.getString(url)) } - def validateString(key: String): ValidationNel[String, String] = try { - config.getString(key).successNel + def validateString(key: String): ValidatedNel[String, String] = try { + config.getString(key).validNel } catch { - case e: ConfigException.Missing => s"Could not find key: $key".failureNel + case e: ConfigException.Missing => s"Could not find key: $key".invalidNel } - def validateConfig(key: String): ValidationNel[String, Config] = try { - config.getConfig(key).successNel + def validateConfig(key: String): ValidatedNel[String, Config] = try { + config.getConfig(key).validNel } catch { - case e: ConfigException.Missing => "Could not find key: $key".failureNel - case e: ConfigException.WrongType => s"key $key cannot be parsed to a Config".failureNel + case e: ConfigException.Missing => "Could not find key: $key".invalidNel + case e: ConfigException.WrongType => s"key $key cannot be parsed to a Config".invalidNel } } @@ -58,10 +58,10 @@ object ConfigUtil { * @tparam O return type of validationFunction * @tparam E Restricts the subtype of Exception that should be caught during validation */ - def validateAny[O, E <: Exception: ClassTag](validationFunction: I => O): ValidationNel[String, O] = try { - validationFunction(value).successNel + def validateAny[O, E <: Exception: ClassTag](validationFunction: I => O): ValidatedNel[String, O] = try { + validationFunction(value).validNel } catch { - case e if classTag[E].runtimeClass.isInstance(e) => e.getMessage.failureNel + case e if classTag[E].runtimeClass.isInstance(e) => e.getMessage.invalidNel } } diff --git a/core/src/main/scala/cromwell/core/ErrorOr.scala b/core/src/main/scala/cromwell/core/ErrorOr.scala new file mode 100644 index 000000000..cd344f8ac --- /dev/null +++ b/core/src/main/scala/cromwell/core/ErrorOr.scala @@ -0,0 +1,22 @@ +package cromwell.core + +import cats.data.Validated.{Invalid, Valid} +import cats.data.{NonEmptyList, Validated} + +object ErrorOr { + type ErrorOr[A] = Validated[NonEmptyList[String], A] + + implicit class ShortCircuitingFlatMap[A](val fa: ErrorOr[A]) extends AnyVal { + /** + * Not consistent with `Applicative#ap` but useful in for comprehensions. + * + * @see http://typelevel.org/cats/tut/validated.html#of-flatmaps-and-xors + */ + def flatMap[B](f: A => ErrorOr[B]): ErrorOr[B] = { + fa match { + case Valid(v) => f(v) + case i @ Invalid(_) => i + } + } + } +} diff --git a/core/src/main/scala/cromwell/core/WorkflowState.scala b/core/src/main/scala/cromwell/core/WorkflowState.scala index b5066d292..41ac2bb97 100644 --- a/core/src/main/scala/cromwell/core/WorkflowState.scala +++ b/core/src/main/scala/cromwell/core/WorkflowState.scala @@ -1,11 +1,12 @@ package cromwell.core -import scalaz.Semigroup +import cats.Semigroup + sealed trait WorkflowState { def isTerminal: Boolean protected def ordinal: Int - def append(that: WorkflowState): WorkflowState = if (this.ordinal > that.ordinal) this else that + def combine(that: WorkflowState): WorkflowState = if (this.ordinal > that.ordinal) this else that } object WorkflowState { @@ -15,7 +16,7 @@ object WorkflowState { throw new NoSuchElementException(s"No such WorkflowState: $str")) implicit val WorkflowStateSemigroup = new Semigroup[WorkflowState] { - override def append(f1: WorkflowState, f2: => WorkflowState): WorkflowState = f1.append(f2) + override def combine(f1: WorkflowState, f2: WorkflowState): WorkflowState = f1.combine(f2) } implicit val WorkflowStateOrdering = Ordering.by { self: WorkflowState => self.ordinal } diff --git a/core/src/main/scala/cromwell/core/package.scala b/core/src/main/scala/cromwell/core/package.scala index b218a2dfb..ee1448640 100644 --- a/core/src/main/scala/cromwell/core/package.scala +++ b/core/src/main/scala/cromwell/core/package.scala @@ -1,16 +1,13 @@ package cromwell -import lenthall.exception.ThrowableAggregation import java.nio.file.Path -import wdl4s.values.{SymbolHash, WdlValue} - -import scalaz._ +import lenthall.exception.ThrowableAggregation +import wdl4s.values.WdlValue package object core { case class CallContext(root: Path, stdout: String, stderr: String) - type ErrorOr[+A] = ValidationNel[String, A] type LocallyQualifiedName = String type FullyQualifiedName = String type WorkflowOutputs = Map[FullyQualifiedName, JobOutput] diff --git a/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala index 1193f4a21..51ea7c3a4 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala @@ -1,10 +1,10 @@ package cromwell.database.slick +import cats.data.NonEmptyList import cromwell.database.sql._ import cromwell.database.sql.joins.CallCachingJoin import scala.concurrent.{ExecutionContext, Future} -import scalaz.NonEmptyList trait CallCachingSlickDatabase extends CallCachingSqlDatabase { this: SlickDatabase => diff --git a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala index 2be8f0069..7a0725ffe 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala @@ -2,11 +2,11 @@ package cromwell.database.slick import java.sql.Timestamp +import cats.data.NonEmptyList import cromwell.database.sql.MetadataSqlDatabase import cromwell.database.sql.tables.{MetadataEntry, WorkflowMetadataSummaryEntry} import scala.concurrent.{ExecutionContext, Future} -import scalaz._ trait MetadataSlickDatabase extends MetadataSqlDatabase { this: SlickDatabase with SummaryStatusSlickDatabase => diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala index 21d2d4c9b..b82ec957c 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala @@ -1,7 +1,7 @@ package cromwell.database.slick.tables +import cats.data.NonEmptyList import cromwell.database.sql.tables.CallCachingHashEntry -import scalaz._ trait CallCachingHashEntryComponent { @@ -61,7 +61,7 @@ trait CallCachingHashEntryComponent { Rep[Boolean] = { hashKeyHashValues. map(existsCallCachingEntryIdHashKeyHashValue(callCachingEntryId)). - list.toList.reduce(_ && _) + toList.reduce(_ && _) } /** diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala index 42f79fdf2..90c1898fc 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala @@ -2,10 +2,9 @@ package cromwell.database.slick.tables import java.sql.Timestamp +import cats.data.NonEmptyList import cromwell.database.sql.tables.MetadataEntry -import scalaz._ - trait MetadataEntryComponent { this: DriverComponent => @@ -146,7 +145,7 @@ trait MetadataEntryComponent { private[this] def metadataEntryHasMetadataKeysLike(metadataEntry: MetadataEntries, metadataKeys: NonEmptyList[String]): Rep[Boolean] = { - metadataKeys.list.toList.map(metadataEntry.metadataKey like _).reduce(_ || _) + metadataKeys.toList.map(metadataEntry.metadataKey like _).reduce(_ || _) } private[this] def metadataEntryHasEmptyJobKey(metadataEntry: MetadataEntries, diff --git a/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala index 12f9b1c77..abf262ae3 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala @@ -1,9 +1,9 @@ package cromwell.database.sql +import cats.data.NonEmptyList import cromwell.database.sql.joins.CallCachingJoin import scala.concurrent.{ExecutionContext, Future} -import scalaz.NonEmptyList trait CallCachingSqlDatabase { def addCallCaching(callCachingJoin: CallCachingJoin)(implicit ec: ExecutionContext): Future[Unit] diff --git a/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala index 596d84749..3f8b83a14 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala @@ -2,10 +2,10 @@ package cromwell.database.sql import java.sql.Timestamp +import cats.data.NonEmptyList import cromwell.database.sql.tables.{MetadataEntry, WorkflowMetadataSummaryEntry} import scala.concurrent.{ExecutionContext, Future} -import scalaz.NonEmptyList trait MetadataSqlDatabase { this: SqlDatabase => diff --git a/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala b/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala index e05df2bb6..e18bd3dae 100644 --- a/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala +++ b/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala @@ -2,6 +2,7 @@ package cromwell.engine import java.nio.file.{FileSystem, FileSystems} +import cats.data.Validated.{Invalid, Valid} import com.typesafe.config.ConfigFactory import cromwell.core.WorkflowOptions import cromwell.engine.backend.EnhancedWorkflowOptions._ @@ -17,10 +18,10 @@ object EngineFilesystems { private val googleConf: GoogleConfiguration = GoogleConfiguration(config) private val googleAuthMode = config.getStringOption("engine.filesystems.gcs.auth") map { confMode => googleConf.auth(confMode) match { - case scalaz.Success(mode) => mode - case scalaz.Failure(errors) => throw new RuntimeException() with MessageAggregation { + case Valid(mode) => mode + case Invalid(errors) => throw new RuntimeException() with MessageAggregation { override def exceptionContext: String = s"Failed to create authentication mode for $confMode" - override def errorMessages: Traversable[String] = errors.list.toList + override def errorMessages: Traversable[String] = errors.toList } } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala index e7879c803..78d34439c 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala @@ -1,10 +1,10 @@ package cromwell.engine.workflow -import java.util.UUID import akka.actor.FSM.{CurrentState, SubscribeTransitionCallBack, Transition} import akka.actor._ import akka.event.Logging +import cats.data.NonEmptyList import com.typesafe.config.{Config, ConfigFactory} import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.{WorkflowAborted, WorkflowId} @@ -19,7 +19,6 @@ import lenthall.config.ScalaConfig.EnhancedScalaConfig import scala.concurrent.duration._ import scala.concurrent.{Await, Promise} import scala.language.postfixOps -import scalaz.NonEmptyList object WorkflowManagerActor { val DefaultMaxWorkflowsToRun = 5000 @@ -66,7 +65,7 @@ object WorkflowManagerActor { def withAddition(entries: NonEmptyList[WorkflowIdToActorRef]): WorkflowManagerData = { val entryTuples = entries map { e => e.workflowId -> e.workflowActor } - this.copy(workflows = workflows ++ entryTuples.list.toList) + this.copy(workflows = workflows ++ entryTuples.toList) } def without(id: WorkflowId): WorkflowManagerData = this.copy(workflows = workflows - id) @@ -144,7 +143,7 @@ class WorkflowManagerActor(config: Config, stay() case Event(WorkflowStoreActor.NewWorkflowsToStart(newWorkflows), stateData) => val newSubmissions = newWorkflows map submitWorkflow - log.info("Retrieved {} workflows from the WorkflowStoreActor", newSubmissions.size) + log.info("Retrieved {} workflows from the WorkflowStoreActor", newSubmissions.toList.size) scheduleNextNewWorkflowPoll() stay() using stateData.withAddition(newSubmissions) case Event(SubscribeToWorkflowCommand(id), data) => diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala index 833001588..d79d97998 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala @@ -3,10 +3,16 @@ package cromwell.engine.workflow.lifecycle import java.nio.file.FileSystem import akka.actor.{ActorRef, FSM, LoggingFSM, Props} +import cats.data.NonEmptyList +import cats.data.Validated._ +import cats.instances.list._ +import cats.syntax.cartesian._ +import cats.syntax.traverse._ +import cats.syntax.validated._ +import cromwell.core._ import com.typesafe.config.Config import com.typesafe.scalalogging.LazyLogging import cromwell.backend.BackendWorkflowDescriptor -import cromwell.core._ import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.WorkflowOptions.{ReadFromCache, WorkflowOption, WriteToCache} import cromwell.core.callcaching._ @@ -15,8 +21,9 @@ import cromwell.engine._ import cromwell.engine.backend.CromwellBackends import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.{MaterializeWorkflowDescriptorActorData, MaterializeWorkflowDescriptorActorState} import cromwell.services.metadata.MetadataService._ -import cromwell.services.metadata.{MetadataValue, MetadataKey, MetadataEvent} +import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} import lenthall.config.ScalaConfig.EnhancedScalaConfig +import cromwell.core.ErrorOr._ import spray.json.{JsObject, _} import wdl4s._ import wdl4s.expression.NoFunctions @@ -24,8 +31,6 @@ import wdl4s.values.{WdlString, WdlValue} import scala.language.postfixOps import scala.util.{Failure, Success, Try} -import scalaz.Scalaz._ -import scalaz.Validation.FlatMap._ object MaterializeWorkflowDescriptorActor { @@ -79,9 +84,9 @@ object MaterializeWorkflowDescriptorActor { def readOptionalOption(option: WorkflowOption): ErrorOr[Boolean] = { workflowOptions.getBoolean(option.name) match { - case Success(x) => x.successNel - case Failure(_: OptionNotFoundException) => true.successNel - case Failure(t) => t.getMessage.failureNel + case Success(x) => x.validNel + case Failure(_: OptionNotFoundException) => true.validNel + case Failure(t) => t.getMessage.invalidNel } } @@ -90,7 +95,7 @@ object MaterializeWorkflowDescriptorActor { val readFromCache = readOptionalOption(ReadFromCache) val writeToCache = readOptionalOption(WriteToCache) - (readFromCache |@| writeToCache) { + (readFromCache |@| writeToCache) map { case (false, false) => CallCachingOff case (true, false) => CallCachingActivity(ReadCache) case (false, true) => CallCachingActivity(WriteCache) @@ -98,7 +103,7 @@ object MaterializeWorkflowDescriptorActor { } } else { - CallCachingOff.successNel + CallCachingOff.validNel } } } @@ -116,10 +121,10 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor when(ReadyToMaterializeState) { case Event(MaterializeWorkflowDescriptorCommand(workflowSourceFiles, conf), _) => buildWorkflowDescriptor(workflowId, workflowSourceFiles, conf) match { - case scalaz.Success(descriptor) => + case Valid(descriptor) => sender() ! MaterializeWorkflowDescriptorSuccessResponse(descriptor) goto(MaterializationSuccessfulState) - case scalaz.Failure(error) => + case Invalid(error) => sender() ! MaterializeWorkflowDescriptorFailureResponse( new IllegalArgumentException with ExceptionWithErrors { val message = s"Workflow input processing failed." @@ -157,7 +162,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor conf: Config): ErrorOr[EngineWorkflowDescriptor] = { val namespaceValidation = validateNamespace(sourceFiles.wdlSource) val workflowOptionsValidation = validateWorkflowOptions(sourceFiles.workflowOptionsJson) - (namespaceValidation |@| workflowOptionsValidation) { + (namespaceValidation |@| workflowOptionsValidation) map { (_, _) } flatMap { case (namespace, workflowOptions) => pushWfNameMetadataService(namespace.workflow.unqualifiedName) @@ -185,7 +190,8 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor val failureModeValidation = validateWorkflowFailureMode(workflowOptions, conf) val backendAssignmentsValidation = validateBackendAssignments(namespace.workflow.calls, workflowOptions, defaultBackendName) val callCachingModeValidation = validateCallCachingMode(workflowOptions, conf) - (rawInputsValidation |@| failureModeValidation |@| backendAssignmentsValidation |@| callCachingModeValidation ) { + + (rawInputsValidation |@| failureModeValidation |@| backendAssignmentsValidation |@| callCachingModeValidation ) map { (_, _, _, _) } flatMap { case (rawInputs, failureMode, backendAssignments, callCachingMode) => buildWorkflowDescriptor(id, namespace, rawInputs, backendAssignments, workflowOptions, failureMode, engineFilesystems, callCachingMode) @@ -203,13 +209,16 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor def checkTypes(inputs: Map[FullyQualifiedName, WdlValue]): ErrorOr[Map[FullyQualifiedName, WdlValue]] = { val allDeclarations = namespace.workflow.scopedDeclarations ++ namespace.workflow.calls.flatMap(_.scopedDeclarations) - inputs.map({ case (k, v) => + val list: List[ErrorOr[(FullyQualifiedName, WdlValue)]] = inputs.map({ case (k, v) => allDeclarations.find(_.fullyQualifiedName == k) match { case Some(decl) if decl.wdlType.coerceRawValue(v).isFailure => - s"Invalid right-side type of '$k'. Expecting ${decl.wdlType.toWdlString}, got ${v.wdlType.toWdlString}".failureNel - case _ => (k, v).successNel[String] + s"Invalid right-side type of '$k'. Expecting ${decl.wdlType.toWdlString}, got ${v.wdlType.toWdlString}".invalidNel + case _ => (k, v).validNel[String] } - }).toList.sequence[ErrorOr, (FullyQualifiedName, WdlValue)].map(_.toMap) + }).toList + + val validatedInputs: ErrorOr[List[(FullyQualifiedName, WdlValue)]] = list.sequence[ErrorOr, (FullyQualifiedName, WdlValue)] + validatedInputs.map(_.toMap) } for { @@ -256,8 +265,8 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor case Success(backendMap) => val backendMapAsString = backendMap.map({case (k, v) => s"${k.fullyQualifiedName} -> $v"}).mkString(", ") workflowLogger.info(s"Call-to-Backend assignments: $backendMapAsString") - backendMap.successNel - case Failure(t) => t.getMessage.failureNel + backendMap.validNel + case Failure(t) => t.getMessage.invalidNel } } @@ -284,40 +293,40 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor coercedInputs: WorkflowCoercedInputs, engineFileSystems: List[FileSystem]): ErrorOr[WorkflowCoercedInputs] = { namespace.staticWorkflowDeclarationsRecursive(coercedInputs, new WdlFunctions(engineFileSystems)) match { - case Success(d) => d.successNel - case Failure(e) => s"Workflow has invalid declarations: ${e.getMessage}".failureNel + case Success(d) => d.validNel + case Failure(e) => s"Workflow has invalid declarations: ${e.getMessage}".invalidNel } } private def validateNamespace(source: WdlSource): ErrorOr[NamespaceWithWorkflow] = { try { - NamespaceWithWorkflow.load(source).successNel + NamespaceWithWorkflow.load(source).validNel } catch { - case e: Exception => s"Unable to load namespace from workflow: ${e.getMessage}".failureNel + case e: Exception => s"Unable to load namespace from workflow: ${e.getMessage}".invalidNel } } private def validateRawInputs(json: WdlJson): ErrorOr[Map[String, JsValue]] = { Try(json.parseJson) match { - case Success(JsObject(inputs)) => inputs.successNel - case Failure(reason: Throwable) => s"Workflow contains invalid inputs JSON: ${reason.getMessage}".failureNel - case _ => s"Workflow inputs JSON cannot be parsed to JsObject: $json".failureNel + case Success(JsObject(inputs)) => inputs.validNel + case Failure(reason: Throwable) => s"Workflow contains invalid inputs JSON: ${reason.getMessage}".invalidNel + case _ => s"Workflow inputs JSON cannot be parsed to JsObject: $json".invalidNel } } private def validateCoercedInputs(rawInputs: Map[String, JsValue], namespace: NamespaceWithWorkflow): ErrorOr[WorkflowCoercedInputs] = { namespace.coerceRawInputs(rawInputs) match { - case Success(r) => r.successNel - case Failure(e: ExceptionWithErrors) => scalaz.Failure(e.errors) - case Failure(e) => e.getMessage.failureNel + case Success(r) => r.validNel + case Failure(e: ExceptionWithErrors) => Invalid(e.errors) + case Failure(e) => e.getMessage.invalidNel } } private def validateWorkflowOptions(workflowOptions: WdlJson): ErrorOr[WorkflowOptions] = { WorkflowOptions.fromJsonString(workflowOptions) match { - case Success(opts) => opts.successNel - case Failure(e) => s"Workflow contains invalid options JSON: ${e.getMessage}".failureNel + case Success(opts) => opts.validNel + case Failure(e) => s"Workflow contains invalid options JSON: ${e.getMessage}".invalidNel } } @@ -329,8 +338,8 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor } modeString flatMap WorkflowFailureMode.tryParse match { - case Success(mode) => mode.successNel - case Failure(t) => t.getMessage.failureNel + case Success(mode) => mode.validNel + case Failure(t) => t.getMessage.invalidNel } } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index 750a215b5..139f5d6c5 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -4,8 +4,9 @@ import java.time.OffsetDateTime import akka.actor.SupervisorStrategy.{Escalate, Stop} import akka.actor._ +import cats.data.NonEmptyList import com.typesafe.config.ConfigFactory -import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, FailedRetryableResponse, FailedNonRetryableResponse, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, FailedNonRetryableResponse, FailedRetryableResponse, SucceededResponse} import cromwell.backend.BackendLifecycleActor.AbortJobCommand import cromwell.backend.{AllBackendInitializationData, BackendJobDescriptor, BackendJobDescriptorKey} import cromwell.core.Dispatcher.EngineDispatcher @@ -17,10 +18,10 @@ import cromwell.core.WorkflowOptions.WorkflowFailureMode import cromwell.core._ import cromwell.core.logging.WorkflowLogging import cromwell.engine.backend.CromwellBackends -import cromwell.engine.workflow.lifecycle.{EngineLifecycleActorAbortCommand, EngineLifecycleActorAbortedResponse} import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.JobRunning import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor.BackendJobPreparationFailed import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.WorkflowExecutionActorState +import cromwell.engine.workflow.lifecycle.{EngineLifecycleActorAbortCommand, EngineLifecycleActorAbortedResponse} import cromwell.engine.{ContinueWhilePossible, EngineWorkflowDescriptor} import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata._ @@ -34,8 +35,6 @@ import wdl4s.{Scope, _} import scala.annotation.tailrec import scala.language.postfixOps import scala.util.{Failure, Random, Success, Try} -import scalaz.NonEmptyList -import scalaz.Scalaz._ object WorkflowExecutionActor { @@ -130,7 +129,7 @@ object WorkflowExecutionActor { } case class WorkflowExecutionException[T <: Throwable](exceptions: NonEmptyList[T]) extends ThrowableAggregation { - override val throwables = exceptions.list.toList + override val throwables = exceptions.toList override val exceptionContext = s"WorkflowExecutionActor" } @@ -592,7 +591,7 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, ejeaRef ! EngineJobExecutionActor.Execute Success(WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Starting))) case None => - throw WorkflowExecutionException(new Exception(s"Could not get BackendLifecycleActor for backend $backendName").wrapNel) + throw WorkflowExecutionException(NonEmptyList.of(new Exception(s"Could not get BackendLifecycleActor for backend $backendName"))) } } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala index 99e0fbd44..d4c144d9a 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala @@ -1,5 +1,6 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching +import cats.data.NonEmptyList import cromwell.backend.BackendJobExecutionActor.SucceededResponse import cromwell.core.ExecutionIndex.IndexEnhancedIndex import cromwell.core.WorkflowId @@ -12,8 +13,6 @@ import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashing import scala.concurrent.{ExecutionContext, Future} import scala.language.postfixOps -import scalaz.Scalaz._ -import scalaz._ final case class MetaInfoId(id: Int) @@ -64,7 +63,7 @@ class CallCache(database: CallCachingSqlDatabase) { } def fetchMetaInfoIdsMatchingHashes(callCacheHashes: CallCacheHashes)(implicit ec: ExecutionContext): Future[Set[MetaInfoId]] = { - metaInfoIdsMatchingHashes(callCacheHashes.hashes.toList.toNel.get) + metaInfoIdsMatchingHashes(NonEmptyList.fromListUnsafe(callCacheHashes.hashes.toList)) } private def metaInfoIdsMatchingHashes(hashKeyValuePairs: NonEmptyList[HashResult]) diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala index 63a12a154..7056137c3 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala @@ -2,6 +2,7 @@ package cromwell.engine.workflow.workflowstore import java.time.OffsetDateTime +import cats.data.NonEmptyList import cromwell.core.{WorkflowId, WorkflowSourceFiles} import cromwell.database.sql.SqlConverters._ import cromwell.database.sql.WorkflowStoreSqlDatabase @@ -9,7 +10,6 @@ import cromwell.database.sql.tables.WorkflowStoreEntry import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState import scala.concurrent.{ExecutionContext, Future} -import scalaz.NonEmptyList case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase) extends WorkflowStore { override def initialize(implicit ec: ExecutionContext): Future[Unit] = { @@ -42,7 +42,7 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase) extends Workf val returnValue = asStoreEntries map { workflowStore => WorkflowId.fromString(workflowStore.workflowExecutionUuid) } // The results from the Future aren't useful, so on completion map it into the precalculated return value instead. Magic! - sqlDatabase.addWorkflowStoreEntries(asStoreEntries.list.toList) map { _ => returnValue } + sqlDatabase.addWorkflowStoreEntries(asStoreEntries.toList) map { _ => returnValue } } private def fromWorkflowStoreEntry(workflowStoreEntry: WorkflowStoreEntry): WorkflowToStart = { diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala index f24cd99cb..e3d7b44be 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala @@ -1,10 +1,10 @@ package cromwell.engine.workflow.workflowstore +import cats.data.NonEmptyList import cromwell.core.{WorkflowId, WorkflowSourceFiles} import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState import scala.concurrent.{ExecutionContext, Future} -import scalaz.NonEmptyList trait WorkflowStore { diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala index 3655b6938..6355b64c5 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala @@ -3,19 +3,19 @@ package cromwell.engine.workflow.workflowstore import java.time.OffsetDateTime import akka.actor.{ActorLogging, ActorRef, LoggingFSM, Props} +import cats.data.NonEmptyList import cromwell.core.{WorkflowId, WorkflowMetadataKeys, WorkflowSourceFiles} import cromwell.engine.workflow.WorkflowManagerActor import cromwell.engine.workflow.WorkflowManagerActor.WorkflowNotFoundException import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._ import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState -import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} import cromwell.services.metadata.MetadataService.{MetadataPutAcknowledgement, PutMetadataAction} +import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} import org.apache.commons.lang3.exception.ExceptionUtils import scala.concurrent.{ExecutionContext, Future} import scala.language.postfixOps import scala.util.{Failure, Success} -import scalaz.NonEmptyList case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor: ActorRef) extends LoggingFSM[WorkflowStoreActorState, WorkflowStoreActorData] with ActorLogging { @@ -70,7 +70,7 @@ case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor: ActorR private def startNewWork(command: WorkflowStoreActorCommand, sndr: ActorRef, nextData: WorkflowStoreActorData) = { val work: Future[Any] = command match { case cmd @ SubmitWorkflow(sourceFiles) => - store.add(NonEmptyList(sourceFiles)) map { ids => + store.add(NonEmptyList.of(sourceFiles)) map { ids => val id = ids.head registerSubmissionWithMetadataService(id, sourceFiles) sndr ! WorkflowSubmittedToStore(id) @@ -78,15 +78,15 @@ case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor: ActorR } case cmd @ BatchSubmitWorkflows(sources) => store.add(sources) map { ids => - val assignedSources = ids.zip(sources) + val assignedSources = ids.toList.zip(sources.toList) assignedSources foreach { case (id, sourceFiles) => registerSubmissionWithMetadataService(id, sourceFiles) } sndr ! WorkflowsBatchSubmittedToStore(ids) - log.info("Workflows {} submitted.", ids.list.toList.mkString(", ")) + log.info("Workflows {} submitted.", ids.toList.mkString(", ")) } case cmd @ FetchRunnableWorkflows(n) => newWorkflowMessage(n) map { nwm => nwm match { - case NewWorkflowsToStart(workflows) => log.info("{} new workflows fetched", workflows.size) + case NewWorkflowsToStart(workflows) => log.info("{} new workflows fetched", workflows.toList.size) case NoNewWorkflowsToStart => log.debug("No workflows fetched") case _ => log.error("Unexpected response from newWorkflowMessage({}): {}", n, nwm) } @@ -145,7 +145,7 @@ case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor: ActorR } yield restartableWorkflows ++ submittedWorkflows runnableWorkflows map { - case x :: xs => NewWorkflowsToStart(NonEmptyList.nels(x, xs: _*)) + case x :: xs => NewWorkflowsToStart(NonEmptyList.of(x, xs: _*)) case _ => NoNewWorkflowsToStart } } diff --git a/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala b/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala index c910ecce9..b39c2fc51 100644 --- a/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala +++ b/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala @@ -2,7 +2,7 @@ package cromwell.webservice import spray.json._ import wdl4s.values.WdlValue -import wdl4s.{FullyQualifiedName, ExceptionWithErrors} +import wdl4s.{ExceptionWithErrors, FullyQualifiedName} import scala.language.postfixOps @@ -19,14 +19,12 @@ case class CallOutputResponse(id: String, callFqn: String, outputs: Map[FullyQua case class WorkflowMetadataQueryParameters(outputs: Boolean = true, timings: Boolean = true) object APIResponse { - import WorkflowJsonSupport._ - import spray.httpx.SprayJsonSupport._ private def constructFailureResponse(status: String, ex: Throwable) ={ ex match { case exceptionWithErrors: ExceptionWithErrors => FailureResponse(status, exceptionWithErrors.message, - Option(JsArray(exceptionWithErrors.errors.list.toList.map(JsString(_)).toVector))) + Option(JsArray(exceptionWithErrors.errors.toList.map(JsString(_)).toVector))) case e: Throwable => FailureResponse(status, e.getMessage, None) } } diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala index d7afa5834..0aae04dac 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala @@ -2,6 +2,7 @@ package cromwell.webservice import akka.actor.{Actor, ActorRef, Props} import akka.event.Logging +import cats.data.NonEmptyList import com.typesafe.config.ConfigFactory import cromwell.core._ import cromwell.engine.workflow.WorkflowManagerActor @@ -13,7 +14,6 @@ import spray.http.{StatusCodes, Uri} import spray.httpx.SprayJsonSupport._ import scala.language.postfixOps -import scalaz.NonEmptyList object CromwellApiHandler { def props(requestHandlerActor: ActorRef): Props = { @@ -71,6 +71,6 @@ class CromwellApiHandler(requestHandlerActor: ActorRef) extends Actor with Workf case WorkflowStoreActor.WorkflowsBatchSubmittedToStore(ids) => val responses = ids map { id => WorkflowSubmitResponse(id.toString, WorkflowSubmitted.toString) } - context.parent ! RequestComplete(StatusCodes.OK, responses.list.toList) + context.parent ! RequestComplete(StatusCodes.OK, responses.toList) } } diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala index 170ed3020..8e7c55168 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala @@ -1,6 +1,7 @@ package cromwell.webservice import akka.actor._ +import cats.data.NonEmptyList import cromwell.core.{WorkflowId, WorkflowSourceFiles} import cromwell.engine.backend.BackendConfiguration import cromwell.services.metadata.MetadataService._ @@ -13,8 +14,6 @@ import spray.httpx.SprayJsonSupport._ import spray.json._ import spray.routing._ -import scalaz.NonEmptyList - trait SwaggerService extends SwaggerUiResourceHttpService { override def swaggerServiceName = "cromwell" @@ -127,7 +126,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator { import spray.json._ workflowInputs.parseJson match { case JsArray(Seq(x, xs@_*)) => - val nelInputses = NonEmptyList.nels(x, xs: _*) + val nelInputses = NonEmptyList.of(x, xs: _*) val sources = nelInputses.map(inputs => WorkflowSourceFiles(wdlSource, inputs.compactPrint, workflowOptions.getOrElse("{}"))) perRequest(requestContext, CromwellApiHandler.props(workflowStoreActor), CromwellApiHandler.ApiHandlerWorkflowSubmitBatch(sources)) case JsArray(_) => failBadRequest(new RuntimeException("Nothing was submitted")) @@ -158,10 +157,8 @@ trait CromwellApiService extends HttpService with PerRequestCreator { def metadataRoute = path("workflows" / Segment / Segment / "metadata") { (version, possibleWorkflowId) => parameterMultiMap { parameters => - // import scalaz_ & Scalaz._ add too many slow implicits, on top of the spray and json implicits - import scalaz.syntax.std.list._ - val includeKeysOption = parameters.getOrElse("includeKey", List.empty).toNel - val excludeKeysOption = parameters.getOrElse("excludeKey", List.empty).toNel + val includeKeysOption = NonEmptyList.fromList(parameters.getOrElse("includeKey", List.empty)) + val excludeKeysOption = NonEmptyList.fromList(parameters.getOrElse("excludeKey", List.empty)) (includeKeysOption, excludeKeysOption) match { case (Some(_), Some(_)) => failBadRequest(new IllegalArgumentException("includeKey and excludeKey may not be specified together")) diff --git a/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala b/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala index 4f1ba9bcc..c1969342c 100644 --- a/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala +++ b/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala @@ -2,30 +2,29 @@ package cromwell.webservice.metadata import java.time.OffsetDateTime +import cats.{Monoid, Semigroup} +import cats.instances.map._ import spray.json._ -import scalaz.{Monoid, Semigroup} -// This is useful, do not remove -import scalaz.Scalaz._ private object IndexedJsonValue { private implicit val dateTimeOrdering: Ordering[OffsetDateTime] = scala.Ordering.fromLessThan(_ isBefore _) private val timestampedJsValueOrdering: Ordering[TimestampedJsValue] = scala.Ordering.by(_.timestamp) implicit val TimestampedJsonMonoid: Monoid[TimestampedJsValue] = new Monoid[TimestampedJsValue] { - def append(f1: TimestampedJsValue, f2: => TimestampedJsValue): TimestampedJsValue = { + def combine(f1: TimestampedJsValue, f2: TimestampedJsValue): TimestampedJsValue = { (f1, f2) match { case (o1: TimestampedJsObject, o2: TimestampedJsObject) => val sg = implicitly[Semigroup[Map[String, TimestampedJsValue]]] - TimestampedJsObject(sg.append(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) + TimestampedJsObject(sg.combine(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) case (o1: TimestampedJsList, o2: TimestampedJsList) => val sg = implicitly[Semigroup[Map[Int, TimestampedJsValue]]] - TimestampedJsList(sg.append(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) + TimestampedJsList(sg.combine(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) case (o1, o2) => timestampedJsValueOrdering.max(o1, o2) } } - override def zero: TimestampedJsValue = TimestampedJsObject(Map.empty, OffsetDateTime.now) + override def empty: TimestampedJsValue = TimestampedJsObject(Map.empty, OffsetDateTime.now) } } diff --git a/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala b/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala index 35ea1aaa5..6d007dddc 100644 --- a/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala +++ b/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala @@ -3,6 +3,9 @@ package cromwell.webservice.metadata import java.time.OffsetDateTime import akka.actor.{ActorRef, LoggingFSM, Props} +import cromwell.webservice.metadata.IndexedJsonValue._ +import cats.instances.list._ +import cats.syntax.foldable._ import cromwell.core.Dispatcher.ApiDispatcher import cromwell.core.ExecutionIndex.ExecutionIndex import cromwell.core.{WorkflowId, WorkflowMetadataKeys, WorkflowState} @@ -10,7 +13,6 @@ import cromwell.services.ServiceRegistryActor.ServiceRegistryFailure import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata._ import cromwell.webservice.PerRequest.{RequestComplete, RequestCompleteWithHeaders} -import cromwell.webservice.metadata.IndexedJsonValue._ import cromwell.webservice.metadata.MetadataBuilderActor.{Idle, MetadataBuilderActorState, WaitingForMetadataService} import cromwell.webservice.{APIResponse, WorkflowJsonSupport} import org.slf4j.LoggerFactory @@ -21,8 +23,7 @@ import spray.json._ import scala.collection.immutable.TreeMap import scala.language.postfixOps import scala.util.{Failure, Success, Try} -import scalaz.std.list._ -import scalaz.syntax.foldable._ + object MetadataBuilderActor { sealed trait MetadataBuilderActorState @@ -133,8 +134,8 @@ object MetadataBuilderActor { /** Sort events by timestamp, transform them into TimestampedJsValues, and merge them together. */ private def eventsToIndexedJson(events: Seq[MetadataEvent]): TimestampedJsValue = { // The `List` has a `Foldable` instance defined in scope, and because the `List`'s elements have a `Monoid` instance - // defined in scope, `suml` can derive a sane `TimestampedJsValue` value even if the `List` of events is empty. - events.toList map { e => keyValueToIndexedJson(e.key.key, e.value, e.offsetDateTime) } suml + // defined in scope, `combineAll` can derive a sane `TimestampedJsValue` value even if the `List` of events is empty. + events.toList map { e => keyValueToIndexedJson(e.key.key, e.value, e.offsetDateTime) } combineAll } private def eventsToAttemptMetadata(attempt: Int, events: Seq[MetadataEvent]) = { diff --git a/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala b/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala index 43764e705..18460e765 100644 --- a/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala @@ -1,5 +1,6 @@ package cromwell.engine +import cats.data.NonEmptyList import cromwell.CromwellTestkitSpec import cromwell.core.WorkflowId import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._ @@ -9,7 +10,6 @@ import org.scalatest.Matchers import scala.concurrent.duration._ import scala.language.postfixOps -import scalaz.NonEmptyList class WorkflowStoreActorSpec extends CromwellTestkitSpec with Matchers { val helloWorldSourceFiles = HelloWorld.asWorkflowSources() @@ -42,25 +42,25 @@ class WorkflowStoreActorSpec extends CromwellTestkitSpec with Matchers { "return 3 IDs for a batch submission of 3" in { val store = new InMemoryWorkflowStore val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestkitSpec.ServiceRegistryActorInstance)) - storeActor ! BatchSubmitWorkflows(NonEmptyList(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles)) + storeActor ! BatchSubmitWorkflows(NonEmptyList.of(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles)) expectMsgPF(10 seconds) { - case WorkflowsBatchSubmittedToStore(ids) => ids.size shouldBe 3 + case WorkflowsBatchSubmittedToStore(ids) => ids.toList.size shouldBe 3 } } "fetch exactly N workflows" in { val store = new InMemoryWorkflowStore val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestkitSpec.ServiceRegistryActorInstance)) - storeActor ! BatchSubmitWorkflows(NonEmptyList(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles)) - val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.list.toList + storeActor ! BatchSubmitWorkflows(NonEmptyList.of(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles)) + val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.toList storeActor ! FetchRunnableWorkflows(2) expectMsgPF(10 seconds) { case NewWorkflowsToStart(workflowNel) => - workflowNel.size shouldBe 2 - checkDistinctIds(workflowNel.list.toList) shouldBe true - workflowNel.foreach { + workflowNel.toList.size shouldBe 2 + checkDistinctIds(workflowNel.toList) shouldBe true + workflowNel map { case WorkflowToStart(id, sources, state) => insertedIds.contains(id) shouldBe true sources shouldBe helloWorldSourceFiles @@ -72,16 +72,16 @@ class WorkflowStoreActorSpec extends CromwellTestkitSpec with Matchers { "return only the remaining workflows if N is larger than size" in { val store = new InMemoryWorkflowStore val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestkitSpec.ServiceRegistryActorInstance)) - storeActor ! BatchSubmitWorkflows(NonEmptyList(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles)) - val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.list.toList + storeActor ! BatchSubmitWorkflows(NonEmptyList.of(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles)) + val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.toList storeActor ! FetchRunnableWorkflows(100) expectMsgPF(10 seconds) { case NewWorkflowsToStart(workflowNel) => - workflowNel.size shouldBe 3 - checkDistinctIds(workflowNel.list.toList) shouldBe true - workflowNel.foreach { + workflowNel.toList.size shouldBe 3 + checkDistinctIds(workflowNel.toList) shouldBe true + workflowNel map { case WorkflowToStart(id, sources, state) => insertedIds.contains(id) shouldBe true sources shouldBe helloWorldSourceFiles diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/CachingConfigSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/CachingConfigSpec.scala index 50c0218e7..4015f01de 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/CachingConfigSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/CachingConfigSpec.scala @@ -1,12 +1,12 @@ package cromwell.engine.workflow.lifecycle +import cats.data.Validated.{Invalid, Valid} import com.typesafe.config.{Config, ConfigFactory} import cromwell.core.WorkflowOptions import cromwell.core.callcaching.CallCachingMode import org.scalatest.{FlatSpec, Matchers} import scala.collection.JavaConverters._ -import scalaz.{Failure => ScalazFailure, Success => ScalazSuccess} import scala.util.{Success, Try} class CachingConfigSpec extends FlatSpec with Matchers { @@ -62,9 +62,9 @@ class CachingConfigSpec extends FlatSpec with Matchers { combinations foreach { case (config, Success(wfOptions)) => MaterializeWorkflowDescriptorActor.validateCallCachingMode(wfOptions, config) match { - case ScalazSuccess(activity) => verificationFunction(activity) - case ScalazFailure(errors) => - val errorsList = errors.list.toList.mkString(", ") + case Valid(activity) => verificationFunction(activity) + case Invalid(errors) => + val errorsList = errors.toList.mkString(", ") fail(s"Failure generating Call Config Mode: $errorsList") } case x => fail(s"Unexpected test tuple: $x") diff --git a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala index 9ee29439d..a24be2d32 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala @@ -1,10 +1,10 @@ package cromwell.engine.workflow.workflowstore +import cats.data.NonEmptyList import cromwell.core.{WorkflowId, WorkflowSourceFiles} import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState import scala.concurrent.{ExecutionContext, Future} -import scalaz.NonEmptyList class InMemoryWorkflowStore extends WorkflowStore { @@ -16,7 +16,7 @@ class InMemoryWorkflowStore extends WorkflowStore { */ override def add(sources: NonEmptyList[WorkflowSourceFiles])(implicit ec: ExecutionContext): Future[NonEmptyList[WorkflowId]] = { val submittedWorkflows = sources map { SubmittedWorkflow(WorkflowId.randomId(), _, WorkflowStoreState.Submitted) } - workflowStore = workflowStore ++ submittedWorkflows.list.toList + workflowStore = workflowStore ++ submittedWorkflows.toList Future.successful(submittedWorkflows map { _.id }) } diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala index 8c4e559ae..9315b51cc 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala @@ -1,16 +1,19 @@ package cromwell.filesystems.gcs +import cats.data.Validated._ +import cats.instances.list._ +import cats.syntax.cartesian._ +import cats.syntax.traverse._ +import cats.syntax.validated._ import com.google.api.services.storage.StorageScopes import com.typesafe.config.Config import lenthall.config.ConfigValidationException import lenthall.config.ValidatedConfig._ +import cromwell.core.ErrorOr._ import org.slf4j.LoggerFactory import scala.collection.JavaConverters._ import scala.language.postfixOps -import scalaz.Scalaz._ -import scalaz.Validation.FlatMap._ -import scalaz._ final case class GoogleConfiguration private (applicationName: String, authsByName: Map[String, GoogleAuthMode]) { @@ -19,8 +22,8 @@ final case class GoogleConfiguration private (applicationName: String, authsByNa authsByName.get(name) match { case None => val knownAuthNames = authsByName.keys.mkString(", ") - s"`google` configuration stanza does not contain an auth named '$name'. Known auth names: $knownAuthNames".failureNel - case Some(a) => a.successNel + s"`google` configuration stanza does not contain an auth named '$name'. Known auth names: $knownAuthNames".invalidNel + case Some(a) => a.validNel } } } @@ -56,7 +59,7 @@ object GoogleConfiguration { cfg => RefreshTokenMode(name, cfg.getString("client-id"), cfg.getString("client-secret")) } - def applicationDefaultAuth(name: String) = ApplicationDefaultMode(name, GoogleScopes).successNel[String] + def applicationDefaultAuth(name: String): ErrorOr[GoogleAuthMode] = ApplicationDefaultMode(name, GoogleScopes).validNel val name = authConfig.getString("name") val scheme = authConfig.getString("scheme") @@ -65,7 +68,7 @@ object GoogleConfiguration { case "user_account" => userAccountAuth(authConfig, name) case "refresh_token" => refreshTokenAuth(authConfig, name) case "application_default" => applicationDefaultAuth(name) - case wut => s"Unsupported authentication scheme: $wut".failureNel + case wut => s"Unsupported authentication scheme: $wut".invalidNel } } @@ -75,20 +78,20 @@ object GoogleConfiguration { def uniqueAuthNames(list: List[GoogleAuthMode]): ErrorOr[Unit] = { val duplicateAuthNames = list.groupBy(_.name) collect { case (n, as) if as.size > 1 => n } if (duplicateAuthNames.nonEmpty) { - ("Duplicate auth names: " + duplicateAuthNames.mkString(", ")).failureNel + ("Duplicate auth names: " + duplicateAuthNames.mkString(", ")).invalidNel } else { - ().successNel + ().validNel } } - (appName |@| errorOrAuthList) { (_, _) } flatMap { case (name, list) => + (appName |@| errorOrAuthList) map { (_, _) } flatMap { case (name, list) => uniqueAuthNames(list) map { _ => GoogleConfiguration(name, list map { a => a.name -> a } toMap) } } match { - case Success(r) => r - case Failure(f) => - val errorMessages = f.list.toList.mkString(", ") + case Valid(r) => r + case Invalid(f) => + val errorMessages = f.toList.mkString(", ") log.error(errorMessages) throw new ConfigValidationException("Google", errorMessages) } diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/package.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/package.scala index 19140a069..0ec2c0316 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/package.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/package.scala @@ -1,8 +1,6 @@ package cromwell.filesystems -import scalaz.ValidationNel package object gcs { - type ErrorOr[+A] = ValidationNel[String, A] type RefreshToken = String } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index cf6b04238..5b632d157 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,8 +1,8 @@ import sbt._ object Dependencies { - lazy val lenthallV = "0.18" - lazy val wdl4sV = "0.5" + lazy val lenthallV = "0.19-98b3f3a-SNAPSHOT" + lazy val wdl4sV = "0.6-2964173-SNAPSHOT" lazy val sprayV = "1.3.3" /* spray-json is an independent project from the "spray suite" @@ -16,13 +16,14 @@ object Dependencies { lazy val slickV = "3.1.1" lazy val googleClientApiV = "1.20.0" lazy val betterFilesV = "2.16.0" - lazy val scalazCoreV = "7.2.5" + lazy val catsV = "0.7.2" // Internal collections of dependencies private val baseDependencies = List( "org.broadinstitute" %% "lenthall" % lenthallV, - "org.scalaz" %% "scalaz-core" % scalazCoreV, + "org.typelevel" %% "cats" % catsV, + "com.github.benhutchison" %% "mouse" % "0.5", "org.scalatest" %% "scalatest" % "3.0.0" % Test, "org.specs2" %% "specs2" % "3.7" % Test ) @@ -120,7 +121,7 @@ object Dependencies { "org.webjars" % "swagger-ui" % "2.1.1", "commons-codec" % "commons-codec" % "1.10", "commons-io" % "commons-io" % "2.5", - "org.scalaz" %% "scalaz-core" % scalazCoreV, + "org.typelevel" %% "cats" % catsV, "com.github.pathikrit" %% "better-files" % betterFilesV, "io.swagger" % "swagger-parser" % "1.0.22" % Test, "org.yaml" % "snakeyaml" % "1.17" % Test diff --git a/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala b/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala index 979a6474c..374169d05 100644 --- a/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala +++ b/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala @@ -2,12 +2,11 @@ package cromwell.services.metadata import java.time.OffsetDateTime +import cats.data.NonEmptyList import cromwell.core.WorkflowId import org.slf4j.LoggerFactory import wdl4s.values.{WdlBoolean, WdlFloat, WdlInteger, WdlValue} -import scalaz.NonEmptyList - case class MetadataJobKey(callFqn: String, index: Option[Int], attempt: Int) case class MetadataKey(workflowId: WorkflowId, jobKey: Option[MetadataJobKey], key: String) diff --git a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala index cce4f6eb1..b440b9f9e 100644 --- a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala +++ b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala @@ -3,12 +3,12 @@ package cromwell.services.metadata import java.time.OffsetDateTime import akka.actor.{ActorRef, DeadLetterSuppression} +import cats.data.NonEmptyList import cromwell.core.{JobKey, WorkflowId, WorkflowState} import cromwell.services.ServiceRegistryActor.ServiceRegistryMessage import wdl4s.values._ import scala.language.postfixOps -import scalaz.NonEmptyList object MetadataService { diff --git a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala index 0cdba09bd..3bf9fce2b 100644 --- a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala +++ b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala @@ -2,12 +2,14 @@ package cromwell.services.metadata import java.time.OffsetDateTime -import cromwell.core.{ErrorOr, WorkflowId, WorkflowState} +import cats.instances.list._ +import cats.syntax.traverse._ +import cats.syntax.validated._ +import cromwell.core.{WorkflowId, WorkflowState} +import cromwell.core.ErrorOr._ import scala.language.postfixOps import scala.util.{Success, Try} -import scalaz.Scalaz._ -import scalaz.ValidationNel object WorkflowQueryKey { val ValidKeys = Set(StartDate, EndDate, Name, Id, Status, Page, PageSize) map { _.name } @@ -35,38 +37,38 @@ object WorkflowQueryKey { case object Name extends SeqStringWorkflowQueryKey { override val name = "Name" - override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[Seq[String]] = { + override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[List[String]] = { val values = valuesFromMap(grouped).toList val nels = values map { - case Patterns.WorkflowName(n) => n.successNel[String] - case v => v.failureNel + case Patterns.WorkflowName(n) => n.validNel[String] + case v => v.invalidNel[String] } - sequenceListOfValidationNels(s"Name values do not match allowed workflow naming pattern", nels) + sequenceListOfValidatedNels(s"Name values do not match allowed workflow naming pattern", nels) } } case object Id extends SeqStringWorkflowQueryKey { override val name = "Id" - override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[Seq[String]] = { + override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[List[String]] = { val values = valuesFromMap(grouped).toList val nels = values map { v => - if (Try(WorkflowId.fromString(v.toLowerCase.capitalize)).isSuccess) v.successNel[String] else v.failureNel + if (Try(WorkflowId.fromString(v.toLowerCase.capitalize)).isSuccess) v.validNel[String] else v.invalidNel[String] } - sequenceListOfValidationNels(s"Id values do match allowed workflow id pattern", nels) + sequenceListOfValidatedNels(s"Id values do match allowed workflow id pattern", nels) } } case object Status extends SeqStringWorkflowQueryKey { override val name = "Status" - override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[Seq[String]] = { + override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[List[String]] = { val values = valuesFromMap(grouped).toList val nels = values map { v => - if (Try(WorkflowState.fromString(v.toLowerCase.capitalize)).isSuccess) v.successNel[String] else v.failureNel + if (Try(WorkflowState.fromString(v.toLowerCase.capitalize)).isSuccess) v.validNel[String] else v.invalidNel[String] } - sequenceListOfValidationNels("Unrecognized status values", nels) + sequenceListOfValidatedNels("Unrecognized status values", nels) } } } @@ -83,12 +85,12 @@ sealed trait DateTimeWorkflowQueryKey extends WorkflowQueryKey[Option[OffsetDate override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[Option[OffsetDateTime]] = { valuesFromMap(grouped) match { case vs if vs.size > 1 => - s"Found ${vs.size} values for key '$name' but at most one is allowed.".failureNel - case Nil => None.successNel + s"Found ${vs.size} values for key '$name' but at most one is allowed.".invalidNel[Option[OffsetDateTime]] + case Nil => None.validNel[String] case v :: Nil => Try(OffsetDateTime.parse(v)) match { - case Success(dt) => Option(dt).successNel - case _ => s"Value given for $displayName does not parse as a datetime: $v".failureNel + case Success(dt) => Option(dt).validNel[String] + case _ => s"Value given for $displayName does not parse as a datetime: $v".invalidNel[Option[OffsetDateTime]] } } } @@ -97,11 +99,11 @@ sealed trait DateTimeWorkflowQueryKey extends WorkflowQueryKey[Option[OffsetDate sealed trait SeqStringWorkflowQueryKey extends WorkflowQueryKey[Seq[String]] { /** `sequence` the `List[ErrorOr[String]]` to a single `ErrorOr[List[String]]` */ - protected def sequenceListOfValidationNels(prefix: String, errorOrList: List[ErrorOr[String]]): ErrorOr[List[String]] = { + protected def sequenceListOfValidatedNels(prefix: String, errorOrList: List[ErrorOr[String]]): ErrorOr[List[String]] = { val errorOr = errorOrList.sequence[ErrorOr, String] // With a leftMap, prepend an error message to the concatenated error values if there are error values. - // This turns the ValidationNel into a Validation, force it back to a ValidationNel with toValidationNel. - errorOr.leftMap(prefix + ": " + _.list.toList.mkString(", ")).toValidationNel + // This turns the ValidatedNel into a Validated, force it back to a ValidatedNel with toValidationNel. + errorOr.leftMap(prefix + ": " + _.toList.mkString(", ")).toValidatedNel } } @@ -109,12 +111,12 @@ sealed trait IntWorkflowQueryKey extends WorkflowQueryKey[Option[Int]] { override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[Option[Int]] = { valuesFromMap(grouped) match { case vs if vs.size > 1 => - s"Found ${vs.size} values for key '$name' but at most one is allowed.".failureNel - case Nil => None.successNel + s"Found ${vs.size} values for key '$name' but at most one is allowed.".invalidNel[Option[Int]] + case Nil => None.validNel case v :: Nil => Try(v.toInt) match { - case Success(intVal) => if (intVal > 0) Option(intVal).successNel else s"Integer value not greater than 0".failureNel - case _ => s"Value given for $displayName does not parse as a integer: $v".failureNel + case Success(intVal) => if (intVal > 0) Option(intVal).validNel else s"Integer value not greater than 0".invalidNel[Option[Int]] + case _ => s"Value given for $displayName does not parse as a integer: $v".invalidNel[Option[Int]] } } } diff --git a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala index a7952d1b4..669bde613 100644 --- a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala +++ b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala @@ -2,13 +2,14 @@ package cromwell.services.metadata import java.time.OffsetDateTime +import cats.data.Validated._ +import cats.syntax.cartesian._ +import cats.syntax.validated._ import cromwell.core.WorkflowId import cromwell.services.metadata.WorkflowQueryKey._ +import cromwell.core.ErrorOr._ import scala.language.postfixOps -import scalaz.Scalaz._ -import scalaz.{Name => _, _} - case class WorkflowQueryParameters private(statuses: Set[String], names: Set[String], @@ -20,7 +21,7 @@ case class WorkflowQueryParameters private(statuses: Set[String], object WorkflowQueryParameters { - private def validateStartBeforeEnd(start: Option[OffsetDateTime], end: Option[OffsetDateTime]): ValidationNel[String, Unit] = { + private def validateStartBeforeEnd(start: Option[OffsetDateTime], end: Option[OffsetDateTime]): ErrorOr[Unit] = { // Invert the notion of success/failure here to only "successfully" generate an error message if // both start and end dates have been specified and start is after end. val startAfterEndError = for { @@ -30,10 +31,10 @@ object WorkflowQueryParameters { } yield s"Specified start date is after specified end date: start: $s, end: $e" // If the Option is defined this represents a failure, if it's empty this is a success. - startAfterEndError map { _.failureNel } getOrElse ().successNel + startAfterEndError map { _.invalidNel } getOrElse ().validNel } - private def validateOnlyRecognizedKeys(rawParameters: Seq[(String, String)]): ValidationNel[String, Unit] = { + private def validateOnlyRecognizedKeys(rawParameters: Seq[(String, String)]): ErrorOr[Unit] = { // Create a map of keys by canonical capitalization (capitalized first letter, lowercase everything else). // The values are the keys capitalized as actually given to the API, which is what will be used in any // error messages. @@ -43,8 +44,8 @@ object WorkflowQueryParameters { keysByCanonicalCapitalization.keys.toSet -- WorkflowQueryKey.ValidKeys match { case set if set.nonEmpty => val unrecognized = set flatMap keysByCanonicalCapitalization - ("Unrecognized query keys: " + unrecognized.mkString(", ")).failureNel - case _ => ().successNel + ("Unrecognized query keys: " + unrecognized.mkString(", ")).invalidNel + case _ => ().validNel } } @@ -52,7 +53,7 @@ object WorkflowQueryParameters { * Run the validation logic over the specified raw parameters, creating a `WorkflowQueryParameters` if all * validation succeeds, otherwise accumulate all validation messages within the `ValidationNel`. */ - private [metadata] def runValidation(rawParameters: Seq[(String, String)]): ValidationNel[String, WorkflowQueryParameters] = { + private [metadata] def runValidation(rawParameters: Seq[(String, String)]): ErrorOr[WorkflowQueryParameters] = { val onlyRecognizedKeys = validateOnlyRecognizedKeys(rawParameters) @@ -68,11 +69,11 @@ object WorkflowQueryParameters { // Only validate start before end if both of the individual date parsing validations have already succeeded. val startBeforeEnd = (startDate, endDate) match { - case (Success(s), Success(e)) => validateStartBeforeEnd(s, e) - case _ => ().successNel[String] + case (Valid(s), Valid(e)) => validateStartBeforeEnd(s, e) + case _ => ().validNel[String] } - (onlyRecognizedKeys |@| startBeforeEnd |@| statuses |@| names |@| ids |@| startDate |@| endDate |@| page |@| pageSize) { + (onlyRecognizedKeys |@| startBeforeEnd |@| statuses |@| names |@| ids |@| startDate |@| endDate |@| page |@| pageSize) map { case (_, _, status, name, uuid, start, end, _page, _pageSize) => val workflowId = uuid map WorkflowId.fromString WorkflowQueryParameters(status.toSet, name.toSet, workflowId.toSet, start, end, _page, _pageSize) @@ -81,8 +82,8 @@ object WorkflowQueryParameters { def apply(rawParameters: Seq[(String, String)]): WorkflowQueryParameters = { runValidation(rawParameters) match { - case Success(queryParameters) => queryParameters - case Failure(x) => throw new IllegalArgumentException(x.list.toList.mkString("\n")) + case Valid(queryParameters) => queryParameters + case Invalid(x) => throw new IllegalArgumentException(x.toList.mkString("\n")) } } } diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala index dfd14c94a..92ba34589 100644 --- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala +++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala @@ -2,6 +2,9 @@ package cromwell.services.metadata.impl import java.time.OffsetDateTime +import cats.Semigroup +import cats.data.NonEmptyList +import cats.syntax.semigroup._ import cromwell.core.{WorkflowId, WorkflowMetadataKeys, WorkflowState} import cromwell.database.sql.SqlConverters._ import cromwell.database.sql.tables.{MetadataEntry, WorkflowMetadataSummaryEntry} @@ -10,14 +13,12 @@ import cromwell.services.metadata.MetadataService.{QueryMetadata, WorkflowQueryR import cromwell.services.metadata._ import scala.concurrent.{ExecutionContext, Future} -import scalaz.Scalaz._ -import scalaz.{NonEmptyList, Semigroup} object MetadataDatabaseAccess { private lazy val WorkflowMetadataSummarySemigroup = new Semigroup[WorkflowMetadataSummaryEntry] { - override def append(summary1: WorkflowMetadataSummaryEntry, - summary2: => WorkflowMetadataSummaryEntry): WorkflowMetadataSummaryEntry = { + override def combine(summary1: WorkflowMetadataSummaryEntry, + summary2: WorkflowMetadataSummaryEntry): WorkflowMetadataSummaryEntry = { // Resolve the status if both `this` and `that` have defined statuses. This will evaluate to `None` // if one or both of the statuses is not defined. val resolvedStatus = for { @@ -131,7 +132,7 @@ trait MetadataDatabaseAccess { (implicit ec: ExecutionContext): Future[Seq[MetadataEvent]] = { val uuid = id.id.toString databaseInterface.queryMetadataEntriesLikeMetadataKeys( - uuid, s"${WorkflowMetadataKeys.Outputs}:%".wrapNel, requireEmptyJobKey = true). + uuid, NonEmptyList.of(s"${WorkflowMetadataKeys.Outputs}:%"), requireEmptyJobKey = true). map(metadataToMetadataEvents(id)) } @@ -139,7 +140,7 @@ trait MetadataDatabaseAccess { (implicit ec: ExecutionContext): Future[Seq[MetadataEvent]] = { import cromwell.services.metadata.CallMetadataKeys._ - val keys = NonEmptyList(Stdout, Stderr, BackendLogsPrefix + ":%") + val keys = NonEmptyList.of(Stdout, Stderr, BackendLogsPrefix + ":%") databaseInterface.queryMetadataEntriesLikeMetadataKeys(id.id.toString, keys, requireEmptyJobKey = false) map metadataToMetadataEvents(id) } diff --git a/services/src/test/scala/cromwell/services/metadata/WorkflowQueryParametersSpec.scala b/services/src/test/scala/cromwell/services/metadata/WorkflowQueryParametersSpec.scala index 50ae75fe0..161d5b198 100644 --- a/services/src/test/scala/cromwell/services/metadata/WorkflowQueryParametersSpec.scala +++ b/services/src/test/scala/cromwell/services/metadata/WorkflowQueryParametersSpec.scala @@ -2,10 +2,9 @@ package cromwell.services.metadata import java.time.OffsetDateTime +import cats.data.Validated._ import cromwell.services.metadata.WorkflowQueryKey._ -import org.scalatest.{WordSpec, Matchers} - -import scalaz.{Name => _, _} +import org.scalatest.{Matchers, WordSpec} class WorkflowQueryParametersSpec extends WordSpec with Matchers { @@ -17,13 +16,13 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { "be accepted if empty" in { val result = WorkflowQueryParameters.runValidation(Seq.empty) result match { - case Success(r) => + case Valid(r) => r.startDate should be('empty) r.endDate should be('empty) r.names should be('empty) r.statuses should be('empty) - case Failure(fs) => - throw new RuntimeException(fs.list.toList.mkString(", ")) + case Invalid(fs) => + throw new RuntimeException(fs.toList.mkString(", ")) } } @@ -38,13 +37,13 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { ) val result = WorkflowQueryParameters.runValidation(rawParameters) result match { - case Success(r) => + case Valid(r) => r.startDate.get.toInstant should equal(OffsetDateTime.parse(StartDateString).toInstant) r.endDate.get.toInstant should equal(OffsetDateTime.parse(EndDateString).toInstant) r.names should be(Set("my_workflow", "my_other_workflow")) r.statuses should be(Set("Succeeded", "Running")) - case Failure(fs) => - throw new RuntimeException(fs.list.toList.mkString(", ")) + case Invalid(fs) => + throw new RuntimeException(fs.toList.mkString(", ")) } } @@ -55,11 +54,11 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { ) val result = WorkflowQueryParameters.runValidation(rawParameters) result match { - case Success(r) => + case Valid(r) => throw new RuntimeException(s"Unexpected success: $r") - case Failure(fs) => - fs.list.toList should have size 1 - fs.list.toList.head should include("Unrecognized query keys: Bogosity") + case Invalid(fs) => + fs.toList should have size 1 + fs.toList.head should include("Unrecognized query keys: Bogosity") } } @@ -71,11 +70,11 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { ) val result = WorkflowQueryParameters.runValidation(rawParameters) result match { - case Success(r) => + case Valid(r) => throw new RuntimeException(s"Unexpected success: $r") - case Failure(fs) => - fs.list.toList should have size 1 - fs.list.toList.head should include("Specified start date is after specified end date") + case Invalid(fs) => + fs.toList should have size 1 + fs.toList.head should include("Specified start date is after specified end date") } } @@ -88,11 +87,11 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { ) val result = WorkflowQueryParameters.runValidation(rawParameters) result match { - case Success(r) => + case Valid(r) => throw new RuntimeException(s"Unexpected success: $r") - case Failure(fs) => - fs.list.toList should have size 1 - fs.list.toList.head should include("Name values do not match allowed workflow naming pattern") + case Invalid(fs) => + fs.toList should have size 1 + fs.toList.head should include("Name values do not match allowed workflow naming pattern") } } @@ -104,11 +103,11 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { ) val result = WorkflowQueryParameters.runValidation(rawParameters) result match { - case Success(r) => + case Valid(r) => throw new RuntimeException(s"Unexpected success: $r") - case Failure(fs) => - fs.list.toList should have size 1 - fs.list.toList.head should include("does not parse as a datetime") + case Invalid(fs) => + fs.toList should have size 1 + fs.toList.head should include("does not parse as a datetime") } } @@ -119,11 +118,11 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { ) val result = WorkflowQueryParameters.runValidation(rawParameters) result match { - case Success(r) => + case Valid(r) => throw new RuntimeException(s"Unexpected success: $r") - case Failure(fs) => - fs.list.toList should have size 1 - fs.list.toList.head should include("at most one is allowed") + case Invalid(fs) => + fs.toList should have size 1 + fs.toList.head should include("at most one is allowed") } } @@ -135,11 +134,11 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { ) val result = WorkflowQueryParameters.runValidation(rawParameters) result match { - case Success(r) => + case Valid(r) => throw new RuntimeException(s"Unexpected success: $r") - case Failure(fs) => - fs.list.toList should have size 1 - fs.list.toList.head should be("Unrecognized status values: Moseying") + case Invalid(fs) => + fs.toList should have size 1 + fs.toList.head should be("Unrecognized status values: Moseying") } } @@ -152,13 +151,13 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { ) val result = WorkflowQueryParameters.runValidation(rawParameters) result match { - case Success(r) => + case Valid(r) => throw new RuntimeException(s"Unexpected success: $r") - case Failure(fs) => - fs.list.toList should have size 3 - fs.list.toList find { _ == "Unrecognized status values: Moseying" } getOrElse fail - fs.list.toList find { _ contains "does not parse as a datetime" } getOrElse fail - fs.list.toList find { _ contains "Name values do not match allowed workflow naming pattern" } getOrElse fail + case Invalid(fs) => + fs.toList should have size 3 + fs.toList find { _ == "Unrecognized status values: Moseying" } getOrElse fail + fs.toList find { _ contains "does not parse as a datetime" } getOrElse fail + fs.toList find { _ contains "Name values do not match allowed workflow naming pattern" } getOrElse fail } } } diff --git a/src/main/scala/cromwell/CromwellCommandLine.scala b/src/main/scala/cromwell/CromwellCommandLine.scala index 192242a5e..c52ebbd66 100644 --- a/src/main/scala/cromwell/CromwellCommandLine.scala +++ b/src/main/scala/cromwell/CromwellCommandLine.scala @@ -3,12 +3,15 @@ package cromwell import java.nio.file.{Files, Path, Paths} import better.files._ -import cromwell.core.{ErrorOr, WorkflowSourceFiles} +import cats.data.Validated._ +import cats.syntax.cartesian._ +import cats.syntax.validated._ +import cromwell.core.WorkflowSourceFiles import cromwell.util.FileUtil._ import lenthall.exception.MessageAggregation +import cromwell.core.ErrorOr._ import scala.util.{Failure, Success, Try} -import scalaz.Scalaz._ sealed abstract class CromwellCommandLine case object UsageAndExit extends CromwellCommandLine @@ -40,44 +43,43 @@ object RunSingle { val inputsJson = readJson("Inputs", inputsPath) val optionsJson = readJson("Workflow Options", optionsPath) - val sourceFiles = (wdl |@| inputsJson |@| optionsJson) { WorkflowSourceFiles.apply } + val sourceFiles = (wdl |@| inputsJson |@| optionsJson) map { WorkflowSourceFiles.apply } - import scalaz.Validation.FlatMap._ val runSingle = for { sources <- sourceFiles _ <- writeableMetadataPath(metadataPath) } yield RunSingle(wdlPath, sources, inputsPath, optionsPath, metadataPath) runSingle match { - case scalaz.Success(r) => r - case scalaz.Failure(nel) => throw new RuntimeException with MessageAggregation { + case Valid(r) => r + case Invalid(nel) => throw new RuntimeException with MessageAggregation { override def exceptionContext: String = "ERROR: Unable to run Cromwell:" - override def errorMessages: Traversable[String] = nel.list.toList + override def errorMessages: Traversable[String] = nel.toList } } } private def writeableMetadataPath(path: Option[Path]): ErrorOr[Unit] = { path match { - case Some(p) if !metadataPathIsWriteable(p) => s"Unable to write to metadata directory: $p".failureNel - case otherwise => ().successNel + case Some(p) if !metadataPathIsWriteable(p) => s"Unable to write to metadata directory: $p".invalidNel + case otherwise => ().validNel } } /** Read the path to a string. */ private def readContent(inputDescription: String, path: Path): ErrorOr[String] = { if (!Files.exists(path)) { - s"$inputDescription does not exist: $path".failureNel + s"$inputDescription does not exist: $path".invalidNel } else if (!Files.isReadable(path)) { - s"$inputDescription is not readable: $path".failureNel - } else File(path).contentAsString.successNel + s"$inputDescription is not readable: $path".invalidNel + } else File(path).contentAsString.validNel } /** Read the path to a string, unless the path is None, in which case returns "{}". */ private def readJson(inputDescription: String, pathOption: Option[Path]): ErrorOr[String] = { pathOption match { case Some(path) => readContent(inputDescription, path) - case None => "{}".successNel + case None => "{}".validNel } } diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala index 1ddb35c35..f8dd9a595 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala @@ -1,17 +1,20 @@ package cromwell.backend.impl.htcondor + +import cats.data.Validated.{Invalid, Valid} +import cats.syntax.cartesian._ +import cats.syntax.validated._ import cromwell.backend.MemorySize import cromwell.backend.validation.ContinueOnReturnCode import cromwell.backend.validation.RuntimeAttributesDefault._ import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.backend.validation.RuntimeAttributesValidation._ import cromwell.core._ +import cromwell.core.ErrorOr._ import lenthall.exception.MessageAggregation import wdl4s.types.{WdlIntegerType, WdlStringType, WdlBooleanType, WdlType} import wdl4s.values.{WdlString, WdlBoolean, WdlInteger, WdlValue} -import scalaz.Scalaz._ -import scalaz._ object HtCondorRuntimeAttributes { private val FailOnStderrDefaultValue = false @@ -48,39 +51,39 @@ object HtCondorRuntimeAttributes { val defaultFromOptions = workflowOptionsDefault(options, coercionMap).get val withDefaultValues = withDefaults(attrs, List(defaultFromOptions, staticDefaults)) - val docker = validateDocker(withDefaultValues.get(DockerKey), None.successNel) - val dockerWorkingDir = validateDockerWorkingDir(withDefaultValues.get(DockerWorkingDirKey), None.successNel) - val dockerOutputDir = validateDockerOutputDir(withDefaultValues.get(DockerOutputDirKey), None.successNel) + val docker = validateDocker(withDefaultValues.get(DockerKey), None.validNel) + val dockerWorkingDir = validateDockerWorkingDir(withDefaultValues.get(DockerWorkingDirKey), None.validNel) + val dockerOutputDir = validateDockerOutputDir(withDefaultValues.get(DockerOutputDirKey), None.validNel) val failOnStderr = validateFailOnStderr(withDefaultValues.get(FailOnStderrKey), noValueFoundFor(FailOnStderrKey)) val continueOnReturnCode = validateContinueOnReturnCode(withDefaultValues.get(ContinueOnReturnCodeKey), noValueFoundFor(ContinueOnReturnCodeKey)) val cpu = validateCpu(withDefaultValues.get(CpuKey), noValueFoundFor(CpuKey)) val memory = validateMemory(withDefaultValues.get(MemoryKey), noValueFoundFor(MemoryKey)) val disk = validateDisk(withDefaultValues.get(DiskKey), noValueFoundFor(DiskKey)) - (continueOnReturnCode |@| docker |@| dockerWorkingDir |@| dockerOutputDir |@| failOnStderr |@| cpu |@| memory |@| disk) { + (continueOnReturnCode |@| docker |@| dockerWorkingDir |@| dockerOutputDir |@| failOnStderr |@| cpu |@| memory |@| disk) map { new HtCondorRuntimeAttributes(_, _, _, _, _, _, _, _) } match { - case Success(x) => x - case Failure(nel) => throw new RuntimeException with MessageAggregation { + case Valid(x) => x + case Invalid(nel) => throw new RuntimeException with MessageAggregation { override def exceptionContext: String = "Runtime attribute validation failed" - override def errorMessages: Traversable[String] = nel.list.toList + override def errorMessages: Traversable[String] = nel.toList } } } private def validateDockerWorkingDir(dockerWorkingDir: Option[WdlValue], onMissingKey: => ErrorOr[Option[String]]): ErrorOr[Option[String]] = { dockerWorkingDir match { - case Some(WdlString(s)) => Some(s).successNel + case Some(WdlString(s)) => Some(s).validNel case None => onMissingKey - case _ => s"Expecting $DockerWorkingDirKey runtime attribute to be a String".failureNel + case _ => s"Expecting $DockerWorkingDirKey runtime attribute to be a String".invalidNel } } private def validateDockerOutputDir(dockerOutputDir: Option[WdlValue], onMissingKey: => ErrorOr[Option[String]]): ErrorOr[Option[String]] = { dockerOutputDir match { - case Some(WdlString(s)) => Some(s).successNel + case Some(WdlString(s)) => Some(s).validNel case None => onMissingKey - case _ => s"Expecting $DockerOutputDirKey runtime attribute to be a String".failureNel + case _ => s"Expecting $DockerOutputDirKey runtime attribute to be a String".invalidNel } } @@ -90,7 +93,7 @@ object HtCondorRuntimeAttributes { value match { case Some(i: WdlInteger) => parseMemoryInteger(i) case Some(s: WdlString) => parseMemoryString(s) - case Some(_) => String.format(diskWrongFormatMsg, "Not supported WDL type value").failureNel + case Some(_) => String.format(diskWrongFormatMsg, "Not supported WDL type value").invalidNel case None => onMissingKey } } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala index fc10cff9b..8fd05c348 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala @@ -2,18 +2,20 @@ package cromwell.backend.impl.jes import java.net.URL +import cats.data._ +import cats.data.Validated._ +import cats.syntax.cartesian._ +import cats.syntax.validated._ import com.typesafe.config.Config import cromwell.backend.impl.jes.JesImplicits.GoogleAuthWorkflowOptions -import cromwell.core.{ErrorOr, WorkflowOptions} +import cromwell.core.WorkflowOptions import cromwell.filesystems.gcs.{GoogleAuthMode, GoogleConfiguration} import lenthall.config.ScalaConfig._ import lenthall.config.ValidatedConfig._ +import cromwell.core.ErrorOr._ import wdl4s.ExceptionWithErrors import scala.language.postfixOps -import scalaz.Scalaz._ -import scalaz.Validation.FlatMap._ -import scalaz._ case class JesAttributes(project: String, genomicsAuth: GoogleAuthMode, @@ -44,22 +46,22 @@ object JesAttributes { def apply(googleConfig: GoogleConfiguration, backendConfig: Config): JesAttributes = { backendConfig.warnNotRecognized(jesKeys, context) - val project: ErrorOr[String] = backendConfig.validateString("project") - val executionBucket: ErrorOr[String] = backendConfig.validateString("root") + val project: ValidatedNel[String, String] = backendConfig.validateString("project") + val executionBucket: ValidatedNel[String, String] = backendConfig.validateString("root") val endpointUrl: ErrorOr[URL] = backendConfig.validateURL("genomics.endpoint-url") val maxPollingInterval: Int = backendConfig.getIntOption("maximum-polling-interval").getOrElse(600) val genomicsAuthName: ErrorOr[String] = backendConfig.validateString("genomics.auth") val gcsFilesystemAuthName: ErrorOr[String] = backendConfig.validateString("filesystems.gcs.auth") - (project |@| executionBucket |@| endpointUrl |@| genomicsAuthName |@| gcsFilesystemAuthName) { + (project |@| executionBucket |@| endpointUrl |@| genomicsAuthName |@| gcsFilesystemAuthName) map { (_, _, _, _, _) } flatMap { case (p, b, u, genomicsName, gcsName) => - (googleConfig.auth(genomicsName) |@| googleConfig.auth(gcsName)) { case (genomicsAuth, gcsAuth) => + (googleConfig.auth(genomicsName) |@| googleConfig.auth(gcsName)) map { case (genomicsAuth, gcsAuth) => JesAttributes(p, genomicsAuth, gcsAuth, b, u, maxPollingInterval) } } match { - case Success(r) => r - case Failure(f) => + case Valid(r) => r + case Invalid(f) => throw new IllegalArgumentException with ExceptionWithErrors { override val message = "Jes Configuration is not valid: Errors" override val errors = f diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala index 5e4a42ea5..19891ee61 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala @@ -1,18 +1,21 @@ package cromwell.backend.impl.jes +import cats.data.Validated._ +import cats.syntax.cartesian._ +import cats.syntax.validated._ import cromwell.backend.MemorySize -import cromwell.backend.impl.jes.io.{JesWorkingDisk, JesAttachedDisk} +import cromwell.backend.impl.jes.io.{JesAttachedDisk, JesWorkingDisk} +import cromwell.backend.validation.RuntimeAttributesDefault._ import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.backend.validation.RuntimeAttributesValidation._ import cromwell.backend.validation._ import cromwell.core._ import lenthall.exception.MessageAggregation +import cromwell.core.ErrorOr._ +import mouse.string._ import org.slf4j.Logger import wdl4s.types._ import wdl4s.values._ -import cromwell.backend.validation.RuntimeAttributesDefault._ -import scalaz._ -import Scalaz._ case class JesRuntimeAttributes(cpu: Int, zones: Vector[String], @@ -98,28 +101,28 @@ object JesRuntimeAttributes { val noAddress = validateNoAddress(attrs(NoAddressKey)) val bootDiskSize = validateBootDisk(attrs(BootDiskSizeKey)) val disks = validateLocalDisks(attrs(DisksKey)) - (cpu |@| zones |@| preemptible |@| bootDiskSize |@| memory |@| disks |@| docker |@| failOnStderr |@| continueOnReturnCode |@| noAddress) { + (cpu |@| zones |@| preemptible |@| bootDiskSize |@| memory |@| disks |@| docker |@| failOnStderr |@| continueOnReturnCode |@| noAddress) map { new JesRuntimeAttributes(_, _, _, _, _, _, _, _, _, _) } match { - case Success(x) => x - case Failure(nel) => throw new RuntimeException with MessageAggregation { + case Valid(x) => x + case Invalid(nel) => throw new RuntimeException with MessageAggregation { override def exceptionContext: String = "Runtime attribute validation failed" - override def errorMessages: Traversable[String] = nel.list.toList + override def errorMessages: Traversable[String] = nel.toList } } } private def validateZone(zoneValue: WdlValue): ErrorOr[Vector[String]] = { zoneValue match { - case WdlString(s) => s.split("\\s+").toVector.successNel + case WdlString(s) => s.split("\\s+").toVector.validNel case WdlArray(wdlType, value) if wdlType.memberType == WdlStringType => - value.map(_.valueString).toVector.successNel - case _ => s"Expecting $ZonesKey runtime attribute to be either a whitespace separated String or an Array[String]".failureNel + value.map(_.valueString).toVector.validNel + case _ => s"Expecting $ZonesKey runtime attribute to be either a whitespace separated String or an Array[String]".invalidNel } } private def contextualizeFailure[T](validation: ErrorOr[T], key: String): ErrorOr[T] = { - validation.leftMap[String](errors => s"Failed to validate $key runtime attribute: " + errors.toList.mkString(",")).toValidationNel + validation.leftMap[String](errors => s"Failed to validate $key runtime attribute: " + errors.toList.mkString(",")).toValidatedNel } private def validatePreemptible(preemptible: WdlValue): ErrorOr[Int] = { @@ -133,23 +136,23 @@ object JesRuntimeAttributes { private def validateBootDisk(diskSize: WdlValue): ErrorOr[Int] = diskSize match { case x if WdlIntegerType.isCoerceableFrom(x.wdlType) => WdlIntegerType.coerceRawValue(x) match { - case scala.util.Success(x: WdlInteger) => x.value.intValue.successNel - case scala.util.Success(unhandled) => s"Coercion was expected to create an Integer but instead got $unhandled".failureNel - case scala.util.Failure(t) => s"Expecting $BootDiskSizeKey runtime attribute to be an Integer".failureNel + case scala.util.Success(x: WdlInteger) => x.value.intValue.validNel + case scala.util.Success(unhandled) => s"Coercion was expected to create an Integer but instead got $unhandled".invalidNel + case scala.util.Failure(t) => s"Expecting $BootDiskSizeKey runtime attribute to be an Integer".invalidNel } } private def validateLocalDisks(value: WdlValue): ErrorOr[Seq[JesAttachedDisk]] = { - val nels = value match { + val nels: Seq[ErrorOr[JesAttachedDisk]] = value match { case WdlString(s) => s.split(",\\s*").toSeq.map(validateLocalDisk) case WdlArray(wdlType, seq) if wdlType.memberType == WdlStringType => seq.map(_.valueString).map(validateLocalDisk) case _ => - Seq(s"Expecting $DisksKey runtime attribute to be a comma separated String or Array[String]".failureNel[JesAttachedDisk]) + Seq(s"Expecting $DisksKey runtime attribute to be a comma separated String or Array[String]".invalidNel) } - val emptyDiskNel = Vector.empty[JesAttachedDisk].successNel[String] - val disksNel = nels.foldLeft(emptyDiskNel)((acc, v) => (acc |@| v) { (a, v) => a :+ v }) + val emptyDiskNel = Vector.empty[JesAttachedDisk].validNel[String] + val disksNel = nels.foldLeft(emptyDiskNel)((acc, v) => (acc |@| v) map { (a, v) => a :+ v }) disksNel map { case disks if disks.exists(_.name == JesWorkingDisk.Name) => disks @@ -159,8 +162,8 @@ object JesRuntimeAttributes { private def validateLocalDisk(disk: String): ErrorOr[JesAttachedDisk] = { JesAttachedDisk.parse(disk) match { - case scala.util.Success(localDisk) => localDisk.successNel - case scala.util.Failure(ex) => ex.getMessage.failureNel + case scala.util.Success(localDisk) => localDisk.validNel + case scala.util.Failure(ex) => ex.getMessage.invalidNel } } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala index 32b558b53..848b188c1 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala @@ -2,14 +2,17 @@ package cromwell.backend.impl.jes.io import java.nio.file.{Path, Paths} +import cats.data.Validated._ +import cats.syntax.cartesian._ +import cats.syntax.validated._ import com.google.api.services.genomics.model.Disk -import cromwell.core.ErrorOr +import cromwell.core.ErrorOr._ +import mouse.string._ import wdl4s.ExceptionWithErrors import wdl4s.values._ import scala.util.Try -import scalaz.Scalaz._ -import scalaz._ + object JesAttachedDisk { val Identifier = "[a-zA-Z0-9-_]+" @@ -19,21 +22,21 @@ object JesAttachedDisk { val MountedDiskPattern = s"""($Directory)\\s+($Integer)\\s+($Identifier)""".r def parse(s: String): Try[JesAttachedDisk] = { - val validation = s match { + val validation: ErrorOr[JesAttachedDisk] = s match { case WorkingDiskPattern(sizeGb, diskType) => - (validateLong(sizeGb) |@| validateDiskType(diskType)) { (s, dt) => + (validateLong(sizeGb) |@| validateDiskType(diskType)) map { (s, dt) => JesWorkingDisk(dt, s.toInt) } case MountedDiskPattern(mountPoint, sizeGb, diskType) => - (validateLong(sizeGb) |@| validateDiskType(diskType)) { (s, dt) => + (validateLong(sizeGb) |@| validateDiskType(diskType)) map { (s, dt) => JesEmptyMountedDisk(dt, s.toInt, Paths.get(mountPoint)) } - case _ => s"Disk strings should be of the format 'local-disk SIZE TYPE' or '/mount/point SIZE TYPE'".failureNel + case _ => s"Disk strings should be of the format 'local-disk SIZE TYPE' or '/mount/point SIZE TYPE'".invalidNel } Try(validation match { - case Success(localDisk) => localDisk - case Failure(nels) => + case Valid(localDisk) => localDisk + case Invalid(nels) => throw new UnsupportedOperationException with ExceptionWithErrors { val message = "" val errors = nels @@ -43,18 +46,18 @@ object JesAttachedDisk { private def validateDiskType(diskTypeName: String): ErrorOr[DiskType] = { DiskType.values().find(_.diskTypeName == diskTypeName) match { - case Some(diskType) => diskType.successNel[String] + case Some(diskType) => diskType.validNel case None => val diskTypeNames = DiskType.values.map(_.diskTypeName).mkString(", ") - s"Disk TYPE $diskTypeName should be one of $diskTypeNames".failureNel + s"Disk TYPE $diskTypeName should be one of $diskTypeNames".invalidNel } } private def validateLong(value: String): ErrorOr[Long] = { try { - value.toLong.successNel + value.toLong.validNel } catch { - case _: IllegalArgumentException => s"$value not convertible to a Long".failureNel[Long] + case _: IllegalArgumentException => s"$value not convertible to a Long".invalidNel } } } diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala index 020be850b..17c33d0de 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala @@ -52,7 +52,7 @@ class JesAttributesSpec extends FlatSpec with Matchers { val exception = intercept[IllegalArgumentException with ExceptionWithErrors] { JesAttributes(googleConfig, nakedConfig) } - val errorsList = exception.errors.list.toList + val errorsList = exception.errors.toList errorsList should contain("Could not find key: project") errorsList should contain("Could not find key: root") errorsList should contain("Could not find key: genomics.auth") diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/PrimitiveRuntimeAttributesValidation.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/PrimitiveRuntimeAttributesValidation.scala index deec1bcf9..16fed1f3d 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/PrimitiveRuntimeAttributesValidation.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/PrimitiveRuntimeAttributesValidation.scala @@ -1,11 +1,10 @@ package cromwell.backend.impl.sfs.config +import cats.syntax.validated._ import cromwell.backend.validation.RuntimeAttributesValidation import wdl4s.types._ import wdl4s.values.{WdlBoolean, WdlFloat, WdlInteger, WdlString} -import scalaz.Scalaz._ - /** * Validates one of the wdl primitive types: Boolean, Float, Integer, or String. WdlFile is not supported. * @@ -23,7 +22,7 @@ class BooleanRuntimeAttributesValidation(override val key: String) extends override val wdlType = WdlBooleanType override protected def validateValue = { - case WdlBoolean(value) => value.successNel + case WdlBoolean(value) => value.validNel } } @@ -31,7 +30,7 @@ class FloatRuntimeAttributesValidation(override val key: String) extends Primiti override val wdlType = WdlFloatType override protected def validateValue = { - case WdlFloat(value) => value.successNel + case WdlFloat(value) => value.validNel } } @@ -39,7 +38,7 @@ class IntRuntimeAttributesValidation(override val key: String) extends Primitive override val wdlType = WdlIntegerType override protected def validateValue = { - case WdlInteger(value) => value.toInt.successNel + case WdlInteger(value) => value.toInt.validNel } } @@ -47,6 +46,6 @@ class StringRuntimeAttributesValidation(override val key: String) extends Primit override val wdlType = WdlStringType override protected def validateValue = { - case WdlString(value) => value.successNel + case WdlString(value) => value.validNel } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala index 35cbd97d6..4cde3b7c9 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala @@ -1,5 +1,6 @@ package cromwell.backend.sfs +import cats.data.Validated.{Invalid, Valid} import cromwell.backend.wfs.{WorkflowFileSystemProvider, WorkflowFileSystemProviderParams} import cromwell.filesystems.gcs.GoogleAuthMode.GoogleAuthOptions import cromwell.filesystems.gcs.{GcsFileSystem, GcsFileSystemProvider, GoogleConfiguration} @@ -20,8 +21,8 @@ object GcsWorkflowFileSystemProvider extends WorkflowFileSystemProvider { val googleAuthModeValidation = googleConfig.auth(gcsAuthName) val gcsAuthMode = googleAuthModeValidation match { - case scalaz.Success(googleAuthMode) => googleAuthMode - case scalaz.Failure(errors) => + case Valid(googleAuthMode) => googleAuthMode + case Invalid(errors) => throw new ValidationException("Could not create gcs filesystem from configuration", errors) } diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkRuntimeAttributes.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkRuntimeAttributes.scala index 1a951f4b7..778b85a86 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkRuntimeAttributes.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkRuntimeAttributes.scala @@ -1,16 +1,18 @@ package cromwell.backend.impl.spark +import cats.data.Validated.{Invalid, Valid} +import cats.syntax.cartesian._ +import cats.syntax.validated._ import cromwell.backend.MemorySize import cromwell.backend.validation.RuntimeAttributesDefault._ import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.backend.validation.RuntimeAttributesValidation._ import cromwell.core._ +import cromwell.core.ErrorOr._ import lenthall.exception.MessageAggregation import wdl4s.types.{WdlBooleanType, WdlIntegerType, WdlStringType, WdlType} import wdl4s.values.{WdlBoolean, WdlInteger, WdlString, WdlValue} -import scalaz.Scalaz._ -import scalaz._ object SparkRuntimeAttributes { private val FailOnStderrDefaultValue = false @@ -46,33 +48,33 @@ object SparkRuntimeAttributes { val executorCores = validateCpu(withDefaultValues.get(ExecutorCoresKey), noValueFoundFor(ExecutorCoresKey)) val executorMemory = validateMemory(withDefaultValues.get(ExecutorMemoryKey), noValueFoundFor(ExecutorMemoryKey)) - val numberOfExecutors = validateNumberOfExecutors(withDefaultValues.get(NumberOfExecutorsKey), None.successNel) + val numberOfExecutors = validateNumberOfExecutors(withDefaultValues.get(NumberOfExecutorsKey), None.validNel) val appMainCLass = validateAppEntryPoint(withDefaultValues(AppMainClassKey)) - (executorCores |@| executorMemory |@| numberOfExecutors |@| appMainCLass |@| failOnStderr) { + (executorCores |@| executorMemory |@| numberOfExecutors |@| appMainCLass |@| failOnStderr) map { new SparkRuntimeAttributes(_, _, _, _, _) } match { - case Success(x) => x - case Failure(nel) => throw new RuntimeException with MessageAggregation { + case Valid(x) => x + case Invalid(nel) => throw new RuntimeException with MessageAggregation { override def exceptionContext: String = "Runtime attribute validation failed" - override def errorMessages: Traversable[String] = nel.list.toList + override def errorMessages: Traversable[String] = nel.toList } } } private def validateNumberOfExecutors(numOfExecutors: Option[WdlValue], onMissingKey: => ErrorOr[Option[Int]]): ErrorOr[Option[Int]] = { numOfExecutors match { - case Some(i: WdlInteger) => Option(i.value.intValue()).successNel + case Some(i: WdlInteger) => Option(i.value.intValue()).validNel case None => onMissingKey - case _ => s"Expecting $NumberOfExecutorsKey runtime attribute to be an Integer".failureNel + case _ => s"Expecting $NumberOfExecutorsKey runtime attribute to be an Integer".invalidNel } } private def validateAppEntryPoint(mainClass: WdlValue): ErrorOr[String] = { WdlStringType.coerceRawValue(mainClass) match { - case scala.util.Success(WdlString(s)) => s.successNel - case _ => s"Could not coerce $AppMainClassKey into a String".failureNel + case scala.util.Success(WdlString(s)) => s.validNel + case _ => s"Could not coerce $AppMainClassKey into a String".invalidNel } } } From 5198253cb6e1958cc498caa844c7b5bf16d4e52b Mon Sep 17 00:00:00 2001 From: Thib Date: Fri, 30 Sep 2016 14:03:37 -0400 Subject: [PATCH 010/375] Hashing strategies v2 - Closes #1483 (#1508) * follow symlinks for hashing and localization and introduce some hashing strategies --- core/src/main/resources/reference.conf | 16 +++ .../src/main/scala/cromwell/core/PathFactory.scala | 4 + core/src/main/scala/cromwell/util/FileUtil.scala | 13 ++ .../sfs/config/ConfigAsyncJobExecutionActor.scala | 4 +- .../ConfigBackendLifecycleActorFactory.scala | 12 +- .../impl/sfs/config/ConfigHashingStrategy.scala | 74 +++++++++++ .../sfs/BackgroundAsyncJobExecutionActor.scala | 10 +- .../cromwell/backend/sfs/SharedFileSystem.scala | 67 +++++----- .../SharedFileSystemAsyncJobExecutionActor.scala | 6 +- .../sfs/config/ConfigHashingStrategySpec.scala | 144 +++++++++++++++++++++ 10 files changed, 303 insertions(+), 47 deletions(-) create mode 100644 supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala create mode 100644 supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 02490fb28..9bf2807ea 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -161,6 +161,22 @@ backend { localization: [ "hard-link", "soft-link", "copy" ] + + caching { + duplication-strategy: [ + "hard-link", "soft-link", "copy" + ] + + # Possible values: file, path + # "file" will compute an md5 hash of the file content. + # "path" will compute an md5 hash of the file path. This strategy will only be effective if the duplication-strategy (above) is set to "soft-link", + # in order to allow for the original file path to be hashed. + hashing-strategy: "file" + + # When true, will check if a sibling file with the same name and the .md5 extension exists, and if it does, use the content of this file as a hash. + # If false or the md5 does not exist, will proceed with the above-defined hashing strategy. + check-sibling-md5: false + } } } } diff --git a/core/src/main/scala/cromwell/core/PathFactory.scala b/core/src/main/scala/cromwell/core/PathFactory.scala index a7889227b..499578107 100644 --- a/core/src/main/scala/cromwell/core/PathFactory.scala +++ b/core/src/main/scala/cromwell/core/PathFactory.scala @@ -3,6 +3,8 @@ package cromwell.core import java.io.Writer import java.nio.file.{FileSystem, Path} +import better.files.File + import scala.collection.immutable.Queue import scala.util.{Success, Failure, Try} @@ -30,6 +32,8 @@ trait PathFactory { }) } + def buildFile(rawString: String, fileSystems: List[FileSystem]): File = File(buildPath(rawString, fileSystems)) + private def hasWrongScheme(rawString: String, fileSystem: FileSystem): Boolean = { schemeMatcher.findFirstMatchIn(rawString) match { case Some(m) => m.group(1) != fileSystem.provider().getScheme diff --git a/core/src/main/scala/cromwell/util/FileUtil.scala b/core/src/main/scala/cromwell/util/FileUtil.scala index 3c0744755..28cd9d072 100644 --- a/core/src/main/scala/cromwell/util/FileUtil.scala +++ b/core/src/main/scala/cromwell/util/FileUtil.scala @@ -5,6 +5,7 @@ import java.nio.file.Path import better.files._ import wdl4s.values.Hashable +import scala.annotation.tailrec import scala.util.{Failure, Success, Try} object FileUtil { @@ -23,4 +24,16 @@ object FileUtil { implicit class EnhancedFile(val file: Path) extends AnyVal with Hashable { def md5Sum: String = File(file).md5.toLowerCase // toLowerCase for backwards compatibility } + + @tailrec + final private def followSymlinks(file: better.files.File): better.files.File = { + file.symbolicLink match { + case Some(target) => followSymlinks(target) + case None => file + } + } + + implicit class EnvenBetterFile(val file: better.files.File) extends AnyVal { + def followSymlinks = FileUtil.followSymlinks(file) + } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala index 014faa1b0..cf6fcea8c 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala @@ -48,12 +48,12 @@ sealed trait ConfigAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecut * @param taskName The name of the task to retrieve from the precomputed wdl namespace. * @param inputs The customized inputs to this task. */ - def writeTaskScript(script: Path, taskName: String, inputs: CallInputs): Unit = { + def writeTaskScript(script: File, taskName: String, inputs: CallInputs): Unit = { val task = configInitializationData.wdlNamespace.findTask(taskName). getOrElse(throw new RuntimeException(s"Unable to find task $taskName")) val command = task.instantiateCommand(inputs, NoFunctions).get jobLogger.info(s"executing: $command") - File(script).write( + script.write( s"""|#!/bin/bash |$command |""".stripMargin) diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala index e37d9416a..a92e7fdb5 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala @@ -5,6 +5,7 @@ import cromwell.backend.impl.sfs.config.ConfigConstants._ import cromwell.backend.sfs._ import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, RuntimeAttributeDefinition} import lenthall.config.ScalaConfig._ +import org.slf4j.LoggerFactory /** * Builds a backend by reading the job control from the config. @@ -14,6 +15,11 @@ import lenthall.config.ScalaConfig._ class ConfigBackendLifecycleActorFactory(val configurationDescriptor: BackendConfigurationDescriptor) extends SharedFileSystemBackendLifecycleActorFactory { + lazy val logger = LoggerFactory.getLogger(getClass) + lazy val hashingStrategy = { + configurationDescriptor.backendConfig.getConfigOption("filesystems.local.caching") map ConfigHashingStrategy.apply getOrElse ConfigHashingStrategy.defaultStrategy + } + override def initializationActorClass = classOf[ConfigInitializationActor] override def asyncJobExecutionActorClass: Class[_ <: ConfigAsyncJobExecutionActor] = { @@ -32,6 +38,10 @@ class ConfigBackendLifecycleActorFactory(val configurationDescriptor: BackendCon initializationData.runtimeAttributesBuilder.definitions.toSet } - override lazy val fileHashingFunction: Option[FileHashingFunction] = Option(FileHashingFunction(ConfigBackendFileHashing.getMd5Result)) + override lazy val fileHashingFunction: Option[FileHashingFunction] = { + logger.debug(hashingStrategy.toString) + Option(FileHashingFunction(hashingStrategy.getHash)) + } + override lazy val fileHashingActorCount: Int = 5 } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala new file mode 100644 index 000000000..21719550e --- /dev/null +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala @@ -0,0 +1,74 @@ +package cromwell.backend.impl.sfs.config + +import akka.event.LoggingAdapter +import better.files.File +import com.typesafe.config.Config +import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest +import cromwell.util.TryWithResource._ +import cromwell.util.FileUtil._ +import lenthall.config.ScalaConfig._ +import org.apache.commons.codec.digest.DigestUtils +import org.slf4j.LoggerFactory + +import scala.util.Try + +object ConfigHashingStrategy { + val logger = LoggerFactory.getLogger(getClass) + val defaultStrategy = HashFileStrategy(false) + + def apply(hashingConfig: Config): ConfigHashingStrategy = { + val checkSiblingMd5 = hashingConfig.getBooleanOr("check-sibling-md5", default = false) + + hashingConfig.getStringOr("hashing-strategy", "file") match { + case "path" => HashPathStrategy(checkSiblingMd5) + case "file" => HashFileStrategy(checkSiblingMd5) + case what => + logger.warn(s"Unrecognized hashing strategy $what.") + HashPathStrategy(checkSiblingMd5) + } + } +} + +abstract class ConfigHashingStrategy { + def checkSiblingMd5: Boolean + protected def hash(file: File): Try[String] + protected def description: String + + protected lazy val checkSiblingMessage = if (checkSiblingMd5) "Check first for sibling md5 and if not found " else "" + + def getHash(request: SingleFileHashRequest, log: LoggingAdapter): Try[String] = { + val file = File(request.file.valueString).followSymlinks + + if (checkSiblingMd5) { + precomputedMd5(file) match { + case Some(md5) => Try(md5.contentAsString) + case None => hash(file) + } + } else hash(file) + } + + private def precomputedMd5(file: File): Option[File] = { + val md5 = file.sibling(s"${file.name}.md5") + if (md5.exists) Option(md5) else None + } + + override def toString = { + s"Call caching hashing strategy: $checkSiblingMessage$description." + } +} + +final case class HashPathStrategy(checkSiblingMd5: Boolean) extends ConfigHashingStrategy { + override def hash(file: File): Try[String] = { + Try(DigestUtils.md5Hex(file.path.toAbsolutePath.toString)) + } + + override val description = "hash file path" +} + +final case class HashFileStrategy(checkSiblingMd5: Boolean) extends ConfigHashingStrategy { + override protected def hash(file: File): Try[String] = { + tryWithResource(() => file.newInputStream) { DigestUtils.md5Hex } + } + + override val description = "hash file content" +} diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala index 41173d811..c17ebe269 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala @@ -13,10 +13,10 @@ trait BackgroundAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecution val stdout = pathPlusSuffix(jobPaths.stdout, "background") val stderr = pathPlusSuffix(jobPaths.stderr, "background") val argv = Seq("/bin/bash", backgroundScript) - new ProcessRunner(argv, stdout, stderr) + new ProcessRunner(argv, stdout.path, stderr.path) } - private def writeBackgroundScript(backgroundScript: Path, backgroundCommand: String): Unit = { + private def writeBackgroundScript(backgroundScript: File, backgroundCommand: String): Unit = { /* Run the `backgroundCommand` in the background. Redirect the stdout and stderr to the appropriate files. While not necessary, mark the job as not receiving any stdin by pointing it at /dev/null. @@ -35,7 +35,7 @@ trait BackgroundAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecution & | send the entire compound command, including the || to the background $! | a variable containing the previous background command's process id (PID) */ - File(backgroundScript).write( + backgroundScript.write( s"""|#!/bin/bash |$backgroundCommand \\ | > ${jobPaths.stdout} \\ @@ -63,11 +63,11 @@ trait BackgroundAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecution SharedFileSystemCommand("/bin/bash", killScript) } - private def writeKillScript(killScript: Path, job: SharedFileSystemJob): Unit = { + private def writeKillScript(killScript: File, job: SharedFileSystemJob): Unit = { /* Use pgrep to find the children of a process, and recursively kill the children before killing the parent. */ - File(killScript).write( + killScript.write( s"""|#!/bin/bash |kill_children() { | local pid=$$1 diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala index 1ac20f284..3d81722c6 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala @@ -1,6 +1,6 @@ package cromwell.backend.sfs -import java.nio.file.{FileSystem, Files, Path, Paths} +import java.nio.file.{FileSystem, Path, Paths} import com.typesafe.config.Config import cromwell.backend.io.JobPaths @@ -29,51 +29,46 @@ object SharedFileSystem { } } - type PathsPair = (Path, Path) - type DuplicationStrategy = (Path, Path) => Try[Unit] + case class PairOfFiles(src: File, dst: File) + type DuplicationStrategy = (File, File) => Try[Unit] /** * Return a `Success` result if the file has already been localized, otherwise `Failure`. */ - private def localizePathAlreadyLocalized(originalPath: Path, executionPath: Path): Try[Unit] = { - if (File(executionPath).exists) Success(Unit) else Failure(new RuntimeException(s"$originalPath doesn't exists")) + private def localizePathAlreadyLocalized(originalPath: File, executionPath: File): Try[Unit] = { + if (executionPath.exists) Success(()) else Failure(new RuntimeException(s"$originalPath doesn't exists")) } - private def localizePathViaCopy(originalPath: Path, executionPath: Path): Try[Unit] = { - File(executionPath).parent.createDirectories() + private def localizePathViaCopy(originalPath: File, executionPath: File): Try[Unit] = { + executionPath.parent.createDirectories() val executionTmpPath = pathPlusSuffix(executionPath, ".tmp") - Try(File(originalPath).copyTo(executionTmpPath, overwrite = true).moveTo(executionPath, overwrite = true)) + Try(originalPath.copyTo(executionTmpPath, overwrite = true).moveTo(executionPath, overwrite = true)) } - private def localizePathViaHardLink(originalPath: Path, executionPath: Path): Try[Unit] = { - File(executionPath).parent.createDirectories() - Try(Files.createLink(executionPath, originalPath)) + private def localizePathViaHardLink(originalPath: File, executionPath: File): Try[Unit] = { + executionPath.parent.createDirectories() + // link.linkTo(target) returns target, + // however we want to return the link, not the target, so map the result back to executionPath + Try(executionPath.linkTo(originalPath, symbolic = false)) map { _ => executionPath } } - /** - * TODO: The 'call' parameter here represents the call statement in WDL that references this path. - * We're supposed to not use symbolic links if the call uses Docker. However, this is currently a - * bit incorrect because multiple calls can reference the same path if that path is in a declaration. - * - * The symbolic link will only fail in the Docker case if a Call uses the file directly and not - * indirectly through one of its input expressions - */ - - private def localizePathViaSymbolicLink(originalPath: Path, executionPath: Path): Try[Unit] = { - if (File(originalPath).isDirectory) Failure(new UnsupportedOperationException("Cannot localize directory with symbolic links")) + private def localizePathViaSymbolicLink(originalPath: File, executionPath: File): Try[Unit] = { + if (originalPath.isDirectory) Failure(new UnsupportedOperationException("Cannot localize directory with symbolic links")) else { - File(executionPath).parent.createDirectories() - Try(Files.createSymbolicLink(executionPath, originalPath.toAbsolutePath)) + executionPath.parent.createDirectories() + Try(executionPath.linkTo(originalPath, symbolic = true)) map { _ => executionPath } } } - private def duplicate(description: String, source: Path, dest: Path, strategies: Stream[DuplicationStrategy]) = { - strategies.map(_ (source, dest)).find(_.isSuccess) getOrElse { + private def duplicate(description: String, source: File, dest: File, strategies: Stream[DuplicationStrategy]) = { + import cromwell.util.FileUtil._ + + strategies.map(_ (source.followSymlinks, dest)).find(_.isSuccess) getOrElse { Failure(new UnsupportedOperationException(s"Could not $description $source -> $dest")) } } - def pathPlusSuffix(path: Path, suffix: String) = path.resolveSibling(s"${File(path).name}.$suffix") + def pathPlusSuffix(path: File, suffix: String) = path.sibling(s"${path.name}.$suffix") } trait SharedFileSystem extends PathFactory { @@ -88,7 +83,7 @@ trait SharedFileSystem extends PathFactory { lazy val Localizers = createStrategies(LocalizationStrategies, docker = false) lazy val DockerLocalizers = createStrategies(LocalizationStrategies, docker = true) - lazy val CachingStrategies = getConfigStrategies("caching") + lazy val CachingStrategies = getConfigStrategies("caching.duplication-strategy") lazy val Cachers = createStrategies(CachingStrategies, docker = false) private def getConfigStrategies(configPath: String): Seq[String] = { @@ -163,17 +158,17 @@ trait SharedFileSystem extends PathFactory { * Transform an original input path to a path in the call directory. * The new path matches the original path, it only "moves" the root to be the call directory. */ - def toCallPath(path: String): Try[PathsPair] = Try { - val src = buildPath(path, filesystems) + def toCallPath(path: String): Try[PairOfFiles] = Try { + val src = buildFile(path, filesystems) // Strip out potential prefix protocol - val localInputPath = stripProtocolScheme(src) - val dest = if (File(inputsRoot).isParentOf(localInputPath)) localInputPath + val localInputPath = stripProtocolScheme(src.path) + val dest = if (File(inputsRoot).isParentOf(localInputPath)) File(localInputPath) else { // Concatenate call directory with absolute input path - Paths.get(inputsRoot.toString, localInputPath.toString) + File(Paths.get(inputsRoot.toString, localInputPath.toString)) } - (src, dest) + PairOfFiles(src, dest) } // Optional function to adjust the path to "docker path" if the call runs in docker @@ -195,7 +190,7 @@ trait SharedFileSystem extends PathFactory { * @param wdlValue WdlValue to localize * @return localized wdlValue */ - private def localizeWdlValue(toDestPath: (String => Try[PathsPair]), strategies: Stream[DuplicationStrategy]) + private def localizeWdlValue(toDestPath: (String => Try[PairOfFiles]), strategies: Stream[DuplicationStrategy]) (wdlValue: WdlValue): Try[WdlValue] = { def adjustArray(t: WdlArrayType, inputArray: Seq[WdlValue]): Try[WdlArray] = { @@ -216,7 +211,7 @@ trait SharedFileSystem extends PathFactory { def adjustFile(path: String) = { toDestPath(path) flatMap { - case (src, dst) => duplicate("localize", src, dst, strategies) map { _ => WdlFile(dst.toString) } + case PairOfFiles(src, dst) => duplicate("localize", src, dst, strategies) map { _ => WdlFile(dst.toString) } } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala index 3a925aad9..6aa913658 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala @@ -286,12 +286,12 @@ trait SharedFileSystemAsyncJobExecutionActor val argv = checkAliveArgs(job).argv val stdout = pathPlusSuffix(jobPaths.stdout, "check") val stderr = pathPlusSuffix(jobPaths.stderr, "check") - val checkAlive = new ProcessRunner(argv, stdout, stderr) + val checkAlive = new ProcessRunner(argv, stdout.path, stderr.path) checkAlive.run() == 0 } def tryKill(job: SharedFileSystemJob): Unit = { - val returnCodeTmp = File(pathPlusSuffix(jobPaths.returnCode, "kill")) + val returnCodeTmp = pathPlusSuffix(jobPaths.returnCode, "kill") returnCodeTmp.write(s"$SIGTERM\n") try { returnCodeTmp.moveTo(jobPaths.returnCode) @@ -303,7 +303,7 @@ trait SharedFileSystemAsyncJobExecutionActor val argv = killArgs(job).argv val stdout = pathPlusSuffix(jobPaths.stdout, "kill") val stderr = pathPlusSuffix(jobPaths.stderr, "kill") - val killer = new ProcessRunner(argv, stdout, stderr) + val killer = new ProcessRunner(argv, stdout.path, stderr.path) killer.run() } diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala new file mode 100644 index 000000000..17ea69fc7 --- /dev/null +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala @@ -0,0 +1,144 @@ +package cromwell.backend.impl.sfs.config + +import java.util.UUID + +import akka.event.LoggingAdapter +import better.files._ +import com.typesafe.config.{ConfigFactory, ConfigValueFactory} +import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest +import org.apache.commons.codec.digest.DigestUtils +import org.scalatest.prop.TableDrivenPropertyChecks +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} +import org.specs2.mock.Mockito +import wdl4s.values.WdlFile + +import scala.util.Success + +class ConfigHashingStrategySpec extends FlatSpec with Matchers with TableDrivenPropertyChecks with Mockito with BeforeAndAfterAll { + + behavior of "ConfigHashingStrategy" + + val steak = "Steak" + val steakHash = DigestUtils.md5Hex(steak) + val file = File.newTemporaryFile() + val symLinksDir = File.newTemporaryDirectory("sym-dir") + val pathHash = DigestUtils.md5Hex(file.pathAsString) + val md5File = file.sibling(s"${file.name}.md5") + // Not the md5 value of "Steak". This is intentional so we can verify which hash is used depending on the strategy + val md5FileHash = "103508832bace55730c8ee8d89c1a45f" + + override def beforeAll() = { + file.write(steak) + } + + private def randomName(): String = UUID.randomUUID().toString + + def mockRequest(withSibling: Boolean, symlink: Boolean = false) = { + if (withSibling && md5File.notExists) md5File.write(md5FileHash) + val request = mock[SingleFileHashRequest] + val requestFile = if (symlink) { + val symLink : File = symLinksDir./(s"symlink-${randomName()}") + symLink.symbolicLinkTo(file) + symLink + } else file + + request.file returns WdlFile(requestFile.pathAsString) + + request + } + + def makeStrategy(strategy: String, checkSibling: Option[Boolean] = None) = { + val conf = ConfigFactory.parseString(s"""hashing-strategy: "$strategy"""") + ConfigHashingStrategy( + checkSibling map { check => conf.withValue("check-sibling-md5", ConfigValueFactory.fromAnyRef(check)) } getOrElse conf + ) + } + + it should "create a path hashing strategy from config" in { + val defaultSibling = makeStrategy("path") + defaultSibling.isInstanceOf[HashPathStrategy] shouldBe true + defaultSibling.checkSiblingMd5 shouldBe false + + val checkSibling = makeStrategy("path", Option(true)) + + checkSibling.isInstanceOf[HashPathStrategy] shouldBe true + checkSibling.checkSiblingMd5 shouldBe true + checkSibling.toString shouldBe "Call caching hashing strategy: Check first for sibling md5 and if not found hash file path." + + val dontCheckSibling = makeStrategy("path", Option(false)) + + dontCheckSibling.isInstanceOf[HashPathStrategy] shouldBe true + dontCheckSibling.checkSiblingMd5 shouldBe false + dontCheckSibling.toString shouldBe "Call caching hashing strategy: hash file path." + } + + it should "have a path hashing strategy and use md5 sibling file when appropriate" in { + val table = Table( + ("check", "withMd5", "expected"), + (true, true, md5FileHash), + (false, true, pathHash), + (true, false, pathHash), + (false, false, pathHash) + ) + + forAll(table) { (check, withMd5, expected) => + md5File.delete(swallowIOExceptions = true) + val checkSibling = makeStrategy("path", Option(check)) + + checkSibling.getHash(mockRequest(withMd5, symlink = false), mock[LoggingAdapter]) shouldBe Success(expected) + + val symLinkRequest: SingleFileHashRequest = mockRequest(withMd5, symlink = true) + val symlink = File(symLinkRequest.file.valueString) + + symlink.isSymbolicLink shouldBe true + DigestUtils.md5Hex(symlink.pathAsString) should not be expected + checkSibling.getHash(symLinkRequest, mock[LoggingAdapter]) shouldBe Success(expected) + } + } + + it should "create a file hashing strategy from config" in { + val defaultSibling = makeStrategy("file") + defaultSibling.isInstanceOf[HashFileStrategy] shouldBe true + defaultSibling.checkSiblingMd5 shouldBe false + + val checkSibling = makeStrategy("file", Option(true)) + + checkSibling.isInstanceOf[HashFileStrategy] shouldBe true + checkSibling.checkSiblingMd5 shouldBe true + checkSibling.toString shouldBe "Call caching hashing strategy: Check first for sibling md5 and if not found hash file content." + + val dontCheckSibling = makeStrategy("file", Option(false)) + + dontCheckSibling.isInstanceOf[HashFileStrategy] shouldBe true + dontCheckSibling.checkSiblingMd5 shouldBe false + dontCheckSibling.toString shouldBe "Call caching hashing strategy: hash file content." + } + + it should "have a file hashing strategy and use md5 sibling file when appropriate" in { + val table = Table( + ("check", "withMd5", "expected"), + (true, true, md5FileHash), + (false, true, steakHash), + (true, false, steakHash), + (false, false, steakHash) + ) + + forAll(table) { (check, withMd5, expected) => + md5File.delete(swallowIOExceptions = true) + val checkSibling = makeStrategy("file", Option(check)) + + checkSibling.getHash(mockRequest(withMd5, symlink = false), mock[LoggingAdapter]) shouldBe Success(expected) + + val symLinkRequest: SingleFileHashRequest = mockRequest(withMd5, symlink = true) + val symlink = File(symLinkRequest.file.valueString) + + symlink.isSymbolicLink shouldBe true + checkSibling.getHash(symLinkRequest, mock[LoggingAdapter]) shouldBe Success(expected) + } + } + + override def afterAll() = { + file.delete(true) + md5File.delete(true) + } +} From 108ff10affe38379683e355a73eef776718b8fdb Mon Sep 17 00:00:00 2001 From: Kristian Cibulskis Date: Thu, 29 Sep 2016 15:58:43 -0400 Subject: [PATCH 011/375] adding support for JAVA_OPTS env variable and symlink jar to /app/cromwell.jar --- project/Settings.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project/Settings.scala b/project/Settings.scala index d9fd8bb36..3038ffed3 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -57,12 +57,13 @@ object Settings { from("openjdk:8") expose(8000) add(artifact, artifactTargetPath) + runRaw(s"ln -s $artifactTargetPath /app/cromwell.jar") // If you use the 'exec' form for an entry point, shell processing is not performed and // environment variable substitution does not occur. Thus we have to /bin/bash here // and pass along any subsequent command line arguments // See https://docs.docker.com/engine/reference/builder/#/entrypoint - entryPoint("/bin/bash", "-c", "java -jar " + artifactTargetPath + " ${CROMWELL_ARGS} ${*}", "--") + entryPoint("/bin/bash", "-c", "java ${JAVA_OPTS} -jar /app/cromwell.jar ${CROMWELL_ARGS} ${*}", "--") } }, buildOptions in docker := BuildOptions( From 11bc336aa65050350f9e5e34730730833658d082 Mon Sep 17 00:00:00 2001 From: Thib Date: Tue, 4 Oct 2016 13:07:22 -0400 Subject: [PATCH 012/375] Clone pipeline resources and strip out mount point for runtime Closes #1501 (#1507) * clone pipeline resources and strip out mount point for runtime * PR comments --- .../jes/src/main/scala/cromwell/backend/impl/jes/Run.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala index caf87e024..b24f7affe 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala @@ -54,9 +54,15 @@ object Run { .setInputParameters(jesParameters.collect({ case i: JesInput => i.toGooglePipelineParameter }).toVector.asJava) .setOutputParameters(jesParameters.collect({ case i: JesFileOutput => i.toGooglePipelineParameter }).toVector.asJava) + // disks cannot have mount points at runtime, so set them null + val runtimePipelineResources = { + val resources = pipelineInfoBuilder.build(commandLine, runtimeAttributes).resources + val disksWithoutMountPoint = resources.getDisks.asScala map { _.setMountPoint(null) } + resources.setDisks(disksWithoutMountPoint.asJava) + } + def runPipeline: String = { - val runtimeResources = new PipelineResources().set(NoAddressFieldName, runtimeAttributes.noAddress) - val rpargs = new RunPipelineArgs().setProjectId(projectId).setServiceAccount(JesServiceAccount).setResources(runtimeResources) + val rpargs = new RunPipelineArgs().setProjectId(projectId).setServiceAccount(JesServiceAccount).setResources(runtimePipelineResources) rpargs.setInputs(jesParameters.collect({ case i: JesInput => i.name -> i.toGoogleRunParameter }).toMap.asJava) logger.debug(s"Inputs:\n${stringifyMap(rpargs.getInputs.asScala.toMap)}") From b34264e3957c03469a27e8081c13e12189143879 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 20 Sep 2016 15:46:46 -0400 Subject: [PATCH 013/375] Added an execution token dispenser to limit concurrent calls --- .../backend/BackendJobExecutionActor.scala | 5 +- .../backend/BackendLifecycleActorFactory.scala | 3 + .../backend/callcaching/CacheHitDuplicating.scala | 2 +- .../scala/cromwell/core/JobExecutionToken.scala | 11 + .../engine/backend/BackendConfiguration.scala | 5 +- .../cromwell/engine/workflow/WorkflowActor.scala | 9 +- .../engine/workflow/WorkflowManagerActor.scala | 16 +- .../execution/EngineJobExecutionActor.scala | 35 ++- .../execution/WorkflowExecutionActor.scala | 7 +- .../tokens/JobExecutionTokenDispenserActor.scala | 133 +++++++++ .../engine/workflow/tokens/TokenPool.scala | 50 ++++ .../scala/cromwell/server/CromwellRootActor.scala | 5 +- .../scala/cromwell/SimpleWorkflowActorSpec.scala | 4 +- .../mock/DefaultBackendJobExecutionActor.scala | 2 +- .../RetryableBackendLifecycleActorFactory.scala | 2 +- .../workflow/SingleWorkflowRunnerActorSpec.scala | 5 +- .../engine/workflow/WorkflowActorSpec.scala | 6 +- .../execution/WorkflowExecutionActorSpec.scala | 7 +- .../lifecycle/execution/ejea/EjeaPendingSpec.scala | 32 +-- .../ejea/EjeaRequestingExecutionTokenSpec.scala | 54 ++++ .../lifecycle/execution/ejea/PerTestHelper.scala | 12 +- .../JobExecutionTokenDispenserActorSpec.scala | 311 +++++++++++++++++++++ .../workflow/tokens/TokenGrabbingActor.scala | 38 +++ .../impl/htcondor/HtCondorBackendFactory.scala | 2 +- .../impl/jes/JesBackendLifecycleActorFactory.scala | 2 +- .../ConfigBackendLifecycleActorFactory.scala | 8 +- .../backend/impl/spark/SparkBackendFactory.scala | 2 +- 27 files changed, 710 insertions(+), 58 deletions(-) create mode 100644 core/src/main/scala/cromwell/core/JobExecutionToken.scala create mode 100644 engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala create mode 100644 engine/src/main/scala/cromwell/engine/workflow/tokens/TokenPool.scala create mode 100644 engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRequestingExecutionTokenSpec.scala create mode 100644 engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala create mode 100644 engine/src/test/scala/cromwell/engine/workflow/tokens/TokenGrabbingActor.scala diff --git a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala index c7f712eba..60cdc1e02 100644 --- a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala @@ -24,8 +24,9 @@ object BackendJobExecutionActor { sealed trait BackendJobExecutionResponse extends BackendJobExecutionActorResponse { def jobKey: BackendJobDescriptorKey } case class SucceededResponse(jobKey: BackendJobDescriptorKey, returnCode: Option[Int], jobOutputs: JobOutputs, jobDetritusFiles: Option[Map[String, String]], executionEvents: Seq[ExecutionEvent]) extends BackendJobExecutionResponse case class AbortedResponse(jobKey: BackendJobDescriptorKey) extends BackendJobExecutionResponse - case class FailedNonRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobExecutionResponse - case class FailedRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobExecutionResponse + sealed trait BackendJobFailedResponse extends BackendJobExecutionResponse { def throwable: Throwable; def returnCode: Option[Int] } + case class FailedNonRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse + case class FailedRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse } /** diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala index 6c48329ce..0101da9f0 100644 --- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala +++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala @@ -7,6 +7,7 @@ import com.typesafe.config.Config import cromwell.backend.callcaching.FileHashingActor import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction import cromwell.backend.io.WorkflowPaths +import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.core.{ExecutionStore, OutputStore} import wdl4s.Call import wdl4s.expression.WdlStandardLibraryFunctions @@ -52,4 +53,6 @@ trait BackendLifecycleActorFactory { lazy val fileHashingActorCount: Int = 50 def fileHashingActorProps: Props = FileHashingActor.props(fileHashingFunction) + + def jobExecutionTokenType: JobExecutionTokenType = JobExecutionTokenType("Default", None) } diff --git a/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala b/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala index addbe0163..c199de6d4 100644 --- a/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala +++ b/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala @@ -48,7 +48,7 @@ trait CacheHitDuplicating { private def lookupSourceCallRootPath(sourceJobDetritusFiles: Map[String, String]): Path = { sourceJobDetritusFiles.get(JobPaths.CallRootPathKey).map(getPath).getOrElse(throw new RuntimeException( - s"The call detritus files for source cache hit aren't found for call ${jobDescriptor.call.fullyQualifiedName}") + s"${JobPaths.CallRootPathKey} wasn't found for call ${jobDescriptor.call.fullyQualifiedName}") ) } diff --git a/core/src/main/scala/cromwell/core/JobExecutionToken.scala b/core/src/main/scala/cromwell/core/JobExecutionToken.scala new file mode 100644 index 000000000..460d36e6c --- /dev/null +++ b/core/src/main/scala/cromwell/core/JobExecutionToken.scala @@ -0,0 +1,11 @@ +package cromwell.core + +import java.util.UUID + +import cromwell.core.JobExecutionToken.JobExecutionTokenType + +case class JobExecutionToken(jobExecutionTokenType: JobExecutionTokenType, id: UUID) + +object JobExecutionToken { + case class JobExecutionTokenType(backend: String, maxPoolSize: Option[Int]) +} diff --git a/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala b/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala index 866e11062..fcd6da9ce 100644 --- a/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala +++ b/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala @@ -1,6 +1,5 @@ package cromwell.engine.backend -import akka.actor.ActorSystem import com.typesafe.config.{Config, ConfigFactory} import cromwell.backend.{BackendConfigurationDescriptor, BackendLifecycleActorFactory} import lenthall.config.ScalaConfig._ @@ -11,8 +10,8 @@ import scala.util.{Failure, Success, Try} case class BackendConfigurationEntry(name: String, lifecycleActorFactoryClass: String, config: Config) { def asBackendLifecycleActorFactory: BackendLifecycleActorFactory = { Class.forName(lifecycleActorFactoryClass) - .getConstructor(classOf[BackendConfigurationDescriptor]) - .newInstance(asBackendConfigurationDescriptor) + .getConstructor(classOf[String], classOf[BackendConfigurationDescriptor]) + .newInstance(name, asBackendConfigurationDescriptor) .asInstanceOf[BackendLifecycleActorFactory] } diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala index 3759136f8..f137883fd 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala @@ -140,9 +140,10 @@ object WorkflowActor { serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, - callCacheReadActor: ActorRef): Props = { + callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef): Props = { Props(new WorkflowActor(workflowId, startMode, wdlSource, conf, serviceRegistryActor, workflowLogCopyRouter, - jobStoreActor, callCacheReadActor)).withDispatcher(EngineDispatcher) + jobStoreActor, callCacheReadActor, jobTokenDispenserActor)).withDispatcher(EngineDispatcher) } } @@ -156,7 +157,8 @@ class WorkflowActor(val workflowId: WorkflowId, serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, - callCacheReadActor: ActorRef) + callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef) extends LoggingFSM[WorkflowActorState, WorkflowActorData] with WorkflowLogging with PathFactory { implicit val ec = context.dispatcher @@ -204,6 +206,7 @@ class WorkflowActor(val workflowId: WorkflowId, serviceRegistryActor, jobStoreActor, callCacheReadActor, + jobTokenDispenserActor, initializationData, restarting = restarting), name = s"WorkflowExecutionActor-$workflowId") diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala index 78d34439c..9c797bb76 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala @@ -43,9 +43,10 @@ object WorkflowManagerActor { serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, - callCacheReadActor: ActorRef): Props = { + callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef): Props = { Props(new WorkflowManagerActor( - workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor) + workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) ).withDispatcher(EngineDispatcher) } @@ -82,15 +83,17 @@ class WorkflowManagerActor(config: Config, val serviceRegistryActor: ActorRef, val workflowLogCopyRouter: ActorRef, val jobStoreActor: ActorRef, - val callCacheReadActor: ActorRef) + val callCacheReadActor: ActorRef, + val jobTokenDispenserActor: ActorRef) extends LoggingFSM[WorkflowManagerState, WorkflowManagerData] { def this(workflowStore: ActorRef, serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, - callCacheReadActor: ActorRef) = this( - ConfigFactory.load, workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor) + callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef) = this( + ConfigFactory.load, workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) private val maxWorkflowsRunning = config.getConfig("system").getIntOr("max-concurrent-workflows", default=DefaultMaxWorkflowsToRun) private val maxWorkflowsToLaunch = config.getConfig("system").getIntOr("max-workflow-launch-count", default=DefaultMaxWorkflowsToLaunch) @@ -259,7 +262,7 @@ class WorkflowManagerActor(config: Config, } val wfProps = WorkflowActor.props(workflowId, startMode, workflow.sources, config, serviceRegistryActor, - workflowLogCopyRouter, jobStoreActor, callCacheReadActor) + workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) val wfActor = context.actorOf(wfProps, name = s"WorkflowActor-$workflowId") wfActor ! SubscribeTransitionCallBack(self) @@ -272,4 +275,3 @@ class WorkflowManagerActor(config: Config, context.system.scheduler.scheduleOnce(newWorkflowPollRate, self, RetrieveNewWorkflows)(context.dispatcher) } } - diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala index 26aa93891..4ff275a35 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala @@ -18,6 +18,7 @@ import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor.{Backend import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CacheMiss, CallCacheHashes, HashError} import cromwell.engine.workflow.lifecycle.execution.callcaching.FetchCachedResultsActor.{CachedOutputLookupFailed, CachedOutputLookupSucceeded} import cromwell.engine.workflow.lifecycle.execution.callcaching._ +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed, JobExecutionTokenRequest, JobExecutionTokenReturn} import cromwell.jobstore.JobStoreActor._ import cromwell.jobstore.{Pending => _, _} import cromwell.services.SingletonServicesStore @@ -25,6 +26,7 @@ import cromwell.services.metadata.MetadataService.PutMetadataAction import cromwell.services.metadata.{MetadataEvent, MetadataJobKey, MetadataKey, MetadataValue} import wdl4s.TaskOutput +import scala.concurrent.ExecutionContext import scala.util.{Failure, Success, Try} class EngineJobExecutionActor(replyTo: ActorRef, @@ -36,6 +38,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, serviceRegistryActor: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef, backendName: String, callCachingMode: CallCachingMode) extends LoggingFSM[EngineJobExecutionActorState, EJEAData] with WorkflowLogging { @@ -51,8 +54,12 @@ class EngineJobExecutionActor(replyTo: ActorRef, // For tests: private[execution] def checkEffectiveCallCachingMode = effectiveCallCachingMode + private[execution] var executionToken: Option[JobExecutionToken] = None + private val effectiveCallCachingKey = "Effective call caching mode" + implicit val ec: ExecutionContext = context.dispatcher + log.debug(s"$tag: $effectiveCallCachingKey: $effectiveCallCachingMode") writeCallCachingModeToMetadata() @@ -62,6 +69,13 @@ class EngineJobExecutionActor(replyTo: ActorRef, // When Pending, the FSM always has NoData when(Pending) { case Event(Execute, NoData) => + requestExecutionToken() + goto(RequestingExecutionToken) + } + + when(RequestingExecutionToken) { + case Event(JobExecutionTokenDispensed(jobExecutionToken), NoData) => + executionToken = Option(jobExecutionToken) if (restarting) { val jobStoreKey = jobDescriptorKey.toJobStoreKey(workflowId) jobStoreActor ! QueryJobCompletion(jobStoreKey, jobDescriptorKey.call.task.outputs) @@ -69,6 +83,9 @@ class EngineJobExecutionActor(replyTo: ActorRef, } else { prepareJob() } + case Event(JobExecutionTokenDenied(positionInQueue), NoData) => + log.debug("Token denied so cannot start yet. Currently position {} in the queue", positionInQueue) + stay() } // When CheckingJobStore, the FSM always has NoData @@ -146,7 +163,10 @@ class EngineJobExecutionActor(replyTo: ActorRef, saveJobCompletionToJobStore(data.withSuccessResponse(response)) case Event(response: BackendJobExecutionResponse, data: ResponsePendingData) => // This matches all response types other than `SucceededResponse`. - log.error("{}: Failed copying cache results, falling back to running job.", jobDescriptorKey) + response match { + case f: BackendJobFailedResponse =>log.error("{}: Failed copying cache results, falling back to running job: {}", jobDescriptorKey, f.throwable) + case _ => // + } runJob(data) // Hashes arrive: @@ -217,8 +237,17 @@ class EngineJobExecutionActor(replyTo: ActorRef, stay } + private def requestExecutionToken(): Unit = { + jobTokenDispenserActor ! JobExecutionTokenRequest(factory.jobExecutionTokenType) + } + + private def returnExecutionToken(): Unit = { + executionToken foreach { jobTokenDispenserActor ! JobExecutionTokenReturn(_) } + } + private def forwardAndStop(response: Any): State = { replyTo forward response + returnExecutionToken() tellEventMetadata() context stop self stay() @@ -226,6 +255,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, private def respondAndStop(response: Any): State = { replyTo ! response + returnExecutionToken() tellEventMetadata() context stop self stay() @@ -393,6 +423,7 @@ object EngineJobExecutionActor { /** States */ sealed trait EngineJobExecutionActorState case object Pending extends EngineJobExecutionActorState + case object RequestingExecutionToken extends EngineJobExecutionActorState case object CheckingJobStore extends EngineJobExecutionActorState case object CheckingCallCache extends EngineJobExecutionActorState case object FetchingCachedOutputsFromDatabase extends EngineJobExecutionActorState @@ -417,6 +448,7 @@ object EngineJobExecutionActor { serviceRegistryActor: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef, backendName: String, callCachingMode: CallCachingMode) = { Props(new EngineJobExecutionActor( @@ -429,6 +461,7 @@ object EngineJobExecutionActor { serviceRegistryActor = serviceRegistryActor, jobStoreActor = jobStoreActor, callCacheReadActor = callCacheReadActor, + jobTokenDispenserActor = jobTokenDispenserActor, backendName = backendName: String, callCachingMode = callCachingMode)).withDispatcher(EngineDispatcher) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index 139f5d6c5..8e6c93e6c 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -138,10 +138,11 @@ object WorkflowExecutionActor { serviceRegistryActor: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef, initializationData: AllBackendInitializationData, restarting: Boolean): Props = { Props(WorkflowExecutionActor(workflowId, workflowDescriptor, serviceRegistryActor, jobStoreActor, - callCacheReadActor, initializationData, restarting)).withDispatcher(EngineDispatcher) + callCacheReadActor, jobTokenDispenserActor, initializationData, restarting)).withDispatcher(EngineDispatcher) } private implicit class EnhancedExecutionStore(val executionStore: ExecutionStore) extends AnyVal { @@ -227,7 +228,6 @@ object WorkflowExecutionActor { } toMap } } - } final case class WorkflowExecutionActor(workflowId: WorkflowId, @@ -235,6 +235,7 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, serviceRegistryActor: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef, initializationData: AllBackendInitializationData, restarting: Boolean) extends LoggingFSM[WorkflowExecutionActorState, WorkflowExecutionActorData] with WorkflowLogging { @@ -585,7 +586,7 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, val ejeaName = s"${workflowDescriptor.id}-EngineJobExecutionActor-${jobKey.tag}" val ejeaProps = EngineJobExecutionActor.props( self, jobKey, data, factory, initializationData.get(backendName), restarting, serviceRegistryActor, - jobStoreActor, callCacheReadActor, backendName, workflowDescriptor.callCachingMode) + jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendName, workflowDescriptor.callCachingMode) val ejeaRef = context.actorOf(ejeaProps, ejeaName) pushNewJobMetadata(jobKey, backendName) ejeaRef ! EngineJobExecutionActor.Execute diff --git a/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala b/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala new file mode 100644 index 000000000..4d5e460bb --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala @@ -0,0 +1,133 @@ +package cromwell.engine.workflow.tokens + +import akka.actor.{Actor, ActorLogging, ActorRef, Props, Terminated} +import cromwell.core.JobExecutionToken +import JobExecutionToken._ +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor._ +import cromwell.engine.workflow.tokens.TokenPool.TokenPoolPop + +import scala.collection.immutable.Queue +import scala.language.postfixOps + +class JobExecutionTokenDispenserActor extends Actor with ActorLogging { + + /** + * Lazily created token pool. We only create a pool for a token type when we need it + */ + var tokenPools: Map[JobExecutionTokenType, TokenPool] = Map.empty + var tokenAssignments: Map[ActorRef, JobExecutionToken] = Map.empty + + override def receive: Actor.Receive = { + case JobExecutionTokenRequest(tokenType) => sendTokenRequestResult(sender, tokenType) + case JobExecutionTokenReturn(token) => unassign(sender, token) + case Terminated(terminee) => onTerminate(terminee) + } + + private def sendTokenRequestResult(sndr: ActorRef, tokenType: JobExecutionTokenType): Unit = { + if (tokenAssignments.contains(sndr)) { + sndr ! JobExecutionTokenDispensed(tokenAssignments(sndr)) + } else { + context.watch(sndr) + val updatedTokenPool = getTokenPool(tokenType).pop() match { + case TokenPoolPop(newTokenPool, Some(token)) => + assignAndSendToken(sndr, token) + newTokenPool + case TokenPoolPop(sizedTokenPoolAndQueue: SizedTokenPoolAndActorQueue, None) => + val (poolWithActorEnqueued, positionInQueue) = sizedTokenPoolAndQueue.enqueue(sndr) + sndr ! JobExecutionTokenDenied(positionInQueue) + poolWithActorEnqueued + case TokenPoolPop(someOtherTokenPool, None) => + //If this has happened, somebody's been playing around in this class and not covered this case: + throw new RuntimeException(s"Unexpected token pool type didn't return a token: ${someOtherTokenPool.getClass.getSimpleName}") + } + + tokenPools += tokenType -> updatedTokenPool + } + } + + private def getTokenPool(tokenType: JobExecutionTokenType): TokenPool = tokenPools.getOrElse(tokenType, createNewPool(tokenType)) + + private def createNewPool(tokenType: JobExecutionTokenType): TokenPool = { + val newPool = TokenPool(tokenType) match { + case s: SizedTokenPool => SizedTokenPoolAndActorQueue(s, Queue.empty) + case anythingElse => anythingElse + } + tokenPools += tokenType -> newPool + newPool + } + + private def assignAndSendToken(actor: ActorRef, token: JobExecutionToken) = { + tokenAssignments += actor -> token + actor ! JobExecutionTokenDispensed(token) + } + + private def unassign(actor: ActorRef, token: JobExecutionToken) = { + if (tokenAssignments.contains(actor) && tokenAssignments(actor) == token) { + tokenAssignments -= actor + + val pool = getTokenPool(token.jobExecutionTokenType) match { + case SizedTokenPoolAndActorQueue(innerPool, queue) if queue.nonEmpty => + val (nextInLine, newQueue) = queue.dequeue + assignAndSendToken(nextInLine, token) + SizedTokenPoolAndActorQueue(innerPool, newQueue) + case other => + other.push(token) + } + + tokenPools += token.jobExecutionTokenType -> pool + context.unwatch(actor) + } else { + log.error("Job execution token returned from incorrect actor: {}", token) + } + } + + private def onTerminate(terminee: ActorRef) = { + tokenAssignments.get(terminee) match { + case Some(token) => + log.error("Actor {} stopped without returning its Job Execution Token. Reclaiming it!", terminee) + self.tell(msg = JobExecutionTokenReturn(token), sender = terminee) + case None => + log.debug("Actor {} stopped while we were still watching it... but it doesn't have a token. Removing it from any queues if necessary", terminee) + tokenPools = tokenPools map { + case (tokenType, SizedTokenPoolAndActorQueue(pool, queue)) => tokenType -> SizedTokenPoolAndActorQueue(pool, queue.filterNot(_ == terminee)) + case (tokenType, other) => tokenType -> other + } + } + context.unwatch(terminee) + } +} + +object JobExecutionTokenDispenserActor { + + def props = Props(new JobExecutionTokenDispenserActor) + + case class JobExecutionTokenRequest(jobExecutionTokenType: JobExecutionTokenType) + case class JobExecutionTokenReturn(jobExecutionToken: JobExecutionToken) + + sealed trait JobExecutionTokenRequestResult + case class JobExecutionTokenDispensed(jobExecutionToken: JobExecutionToken) extends JobExecutionTokenRequestResult + case class JobExecutionTokenDenied(positionInQueue: Integer) extends JobExecutionTokenRequestResult + + case class SizedTokenPoolAndActorQueue(sizedPool: SizedTokenPool, queue: Queue[ActorRef]) extends TokenPool { + override def currentLoans = sizedPool.currentLoans + override def push(jobExecutionToken: JobExecutionToken) = SizedTokenPoolAndActorQueue(sizedPool.push(jobExecutionToken), queue) + override def pop() = { + val underlyingPop = sizedPool.pop() + TokenPoolPop(SizedTokenPoolAndActorQueue(underlyingPop.newTokenPool.asInstanceOf[SizedTokenPool], queue), underlyingPop.poppedItem) + } + + /** + * Enqueues an actor (or just finds its current position) + * + * @return The actor's position in the queue + */ + def enqueue(actor: ActorRef): (SizedTokenPoolAndActorQueue, Int) = { + queue.indexOf(actor) match { + case -1 => + val newQueue = queue :+ actor + (SizedTokenPoolAndActorQueue(sizedPool, newQueue), newQueue.size - 1) // Convert from 1-indexed to 0-indexed + case index => (this, index) + } + } + } +} diff --git a/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenPool.scala b/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenPool.scala new file mode 100644 index 000000000..ee378856c --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenPool.scala @@ -0,0 +1,50 @@ +package cromwell.engine.workflow.tokens + +import java.util.UUID + +import cromwell.core.JobExecutionToken +import JobExecutionToken.JobExecutionTokenType +import cromwell.engine.workflow.tokens.TokenPool.TokenPoolPop + +import scala.language.postfixOps + +trait TokenPool { + def currentLoans: Set[JobExecutionToken] + def pop(): TokenPoolPop + def push(jobExecutionToken: JobExecutionToken): TokenPool +} + +object TokenPool { + + case class TokenPoolPop(newTokenPool: TokenPool, poppedItem: Option[JobExecutionToken]) + + def apply(tokenType: JobExecutionTokenType): TokenPool = { + tokenType.maxPoolSize map { ps => + val pool = (1 to ps toList) map { _ => JobExecutionToken(tokenType, UUID.randomUUID()) } + SizedTokenPool(pool, Set.empty) + } getOrElse { + InfiniteTokenPool(tokenType, Set.empty) + } + } +} + +final case class SizedTokenPool(pool: List[JobExecutionToken], override val currentLoans: Set[JobExecutionToken]) extends TokenPool { + + override def pop(): TokenPoolPop = pool match { + case head :: tail => TokenPoolPop(SizedTokenPool(tail, currentLoans + head), Option(head)) + case Nil => TokenPoolPop(SizedTokenPool(List.empty, currentLoans), None) + } + + + override def push(token: JobExecutionToken): SizedTokenPool = { + if (currentLoans.contains(token)) { SizedTokenPool(pool :+ token, currentLoans - token) } else this + } +} + +final case class InfiniteTokenPool(tokenType: JobExecutionTokenType, override val currentLoans: Set[JobExecutionToken]) extends TokenPool { + override def pop() = { + val newToken = JobExecutionToken(tokenType, UUID.randomUUID()) + TokenPoolPop(InfiniteTokenPool(tokenType, currentLoans + newToken), Option(newToken)) + } + override def push(token: JobExecutionToken): InfiniteTokenPool = if (currentLoans.contains(token)) { InfiniteTokenPool(tokenType, currentLoans - token) } else this +} diff --git a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala index 3289e9df4..9e8c04c17 100644 --- a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala +++ b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala @@ -8,6 +8,7 @@ import com.typesafe.config.ConfigFactory import cromwell.engine.workflow.WorkflowManagerActor import cromwell.engine.workflow.lifecycle.CopyWorkflowLogsActor import cromwell.engine.workflow.lifecycle.execution.callcaching.{CallCache, CallCacheReadActor} +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor import cromwell.engine.workflow.workflowstore.{SqlWorkflowStore, WorkflowStore, WorkflowStoreActor} import cromwell.jobstore.{JobStore, JobStoreActor, SqlJobStore} import cromwell.services.{ServiceRegistryActor, SingletonServicesStore} @@ -48,9 +49,11 @@ import lenthall.config.ScalaConfig.EnhancedScalaConfig .props(CallCacheReadActor.props(callCache)), "CallCacheReadActor") + lazy val jobExecutionTokenDispenserActor = context.actorOf(JobExecutionTokenDispenserActor.props) + lazy val workflowManagerActor = context.actorOf( WorkflowManagerActor.props( - workflowStoreActor, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor), + workflowStoreActor, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobExecutionTokenDispenserActor), "WorkflowManagerActor") override def receive = { diff --git a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala index 8925f1142..f2dd4db13 100644 --- a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala @@ -10,6 +10,7 @@ import cromwell.SimpleWorkflowActorSpec._ import cromwell.core.{WorkflowId, WorkflowSourceFiles} import cromwell.engine.workflow.WorkflowActor import cromwell.engine.workflow.WorkflowActor._ +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor import cromwell.util.SampleWdl import cromwell.util.SampleWdl.HelloWorld.Addressee import org.scalatest.BeforeAndAfter @@ -42,7 +43,8 @@ class SimpleWorkflowActorSpec extends CromwellTestkitSpec with BeforeAndAfter { serviceRegistryActor = watchActor, workflowLogCopyRouter = system.actorOf(Props.empty, s"workflow-copy-log-router-$workflowId-${UUID.randomUUID()}"), jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props), - callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props)), + callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props), + jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props)), supervisor = supervisor.ref, name = s"workflow-actor-$workflowId" ) diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala b/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala index 4b6f55e4a..6893f6baf 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala @@ -22,7 +22,7 @@ case class DefaultBackendJobExecutionActor(override val jobDescriptor: BackendJo override def abort(): Unit = () } -class DefaultBackendLifecycleActorFactory(configurationDescriptor: BackendConfigurationDescriptor) +class DefaultBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala index ec527db26..74e900c6d 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala @@ -5,7 +5,7 @@ import cromwell.backend._ import wdl4s.Call import wdl4s.expression.{NoFunctions, WdlStandardLibraryFunctions} -class RetryableBackendLifecycleActorFactory(configurationDescriptor: BackendConfigurationDescriptor) +class RetryableBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala index 46e4c00a8..5c0a48253 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala @@ -12,6 +12,7 @@ import cromwell.CromwellTestkitSpec._ import cromwell.core.WorkflowSourceFiles import cromwell.engine.workflow.SingleWorkflowRunnerActor.RunWorkflow import cromwell.engine.workflow.SingleWorkflowRunnerActorSpec._ +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreActor} import cromwell.util.SampleWdl import cromwell.util.SampleWdl.{ExpressionsInInputs, GoodbyeWorld, ThreeStep} @@ -55,6 +56,7 @@ abstract class SingleWorkflowRunnerActorSpec extends CromwellTestkitSpec { private val workflowStore = system.actorOf(WorkflowStoreActor.props(new InMemoryWorkflowStore, dummyServiceRegistryActor)) private val jobStore = system.actorOf(AlwaysHappyJobStoreActor.props) private val callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props) + private val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props) def workflowManagerActor(): ActorRef = { @@ -63,7 +65,8 @@ abstract class SingleWorkflowRunnerActorSpec extends CromwellTestkitSpec { dummyServiceRegistryActor, dummyLogCopyRouter, jobStore, - callCacheReadActor)), "WorkflowManagerActor") + callCacheReadActor, + jobTokenDispenserActor)), "WorkflowManagerActor") } def createRunnerActor(sampleWdl: SampleWdl = ThreeStep, managerActor: => ActorRef = workflowManagerActor(), diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala index cc3581234..6a0245528 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala @@ -52,7 +52,8 @@ class WorkflowActorSpec extends CromwellTestkitSpec with WorkflowDescriptorBuild serviceRegistryActor = mockServiceRegistryActor, workflowLogCopyRouter = TestProbe().ref, jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props), - callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props) + callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props), + jobTokenDispenserActor = TestProbe().ref ), supervisor = supervisorProbe.ref) actor.setState(stateName = state, stateData = WorkflowActorData(Option(currentLifecycleActor.ref), Option(descriptor), @@ -155,7 +156,8 @@ class MockWorkflowActor(val finalizationProbe: TestProbe, serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, - callCacheReadActor: ActorRef) extends WorkflowActor(workflowId, startMode, workflowSources, conf, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor) { + callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef) extends WorkflowActor(workflowId, startMode, workflowSources, conf, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) { override def makeFinalizationActor(workflowDescriptor: EngineWorkflowDescriptor, executionStore: ExecutionStore, outputStore: OutputStore) = finalizationProbe.ref } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala index 3d9ef1938..639d8f250 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala @@ -8,6 +8,7 @@ import cromwell.core.WorkflowId import cromwell.engine.backend.{BackendConfigurationEntry, CromwellBackends} import cromwell.engine.workflow.WorkflowDescriptorBuilder import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.ExecuteWorkflowCommand +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor import cromwell.services.ServiceRegistryActor import cromwell.services.metadata.MetadataService import cromwell.util.SampleWdl @@ -49,6 +50,7 @@ class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter val metadataWatcherProps = Props(MetadataWatchActor(metadataSuccessPromise, requiredMetadataMatchers: _*)) val serviceRegistryActor = system.actorOf(ServiceRegistryActor.props(ConfigFactory.load(), overrides = Map(MetadataService.MetadataServiceName -> metadataWatcherProps))) val jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props) + val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props) val MockBackendConfigEntry = BackendConfigurationEntry( name = "Mock", lifecycleActorFactoryClass = "cromwell.engine.backend.mock.RetryableBackendLifecycleActorFactory", @@ -62,7 +64,7 @@ class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter val workflowExecutionActor = system.actorOf( WorkflowExecutionActor.props(workflowId, engineWorkflowDescriptor, serviceRegistryActor, jobStoreActor, - callCacheReadActor.ref, AllBackendInitializationData.empty, restarting = false), + callCacheReadActor.ref, jobTokenDispenserActor, AllBackendInitializationData.empty, restarting = false), "WorkflowExecutionActor") EventFilter.info(pattern = ".*Final Outputs", occurrences = 1).intercept { @@ -82,6 +84,7 @@ class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter val serviceRegistry = mockServiceRegistryActor val jobStore = system.actorOf(AlwaysHappyJobStoreActor.props) val callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props) + val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props) val MockBackendConfigEntry = BackendConfigurationEntry( name = "Mock", @@ -94,7 +97,7 @@ class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter val engineWorkflowDescriptor = createMaterializedEngineWorkflowDescriptor(workflowId, SampleWdl.SimpleScatterWdl.asWorkflowSources(runtime = runtimeSection)) val workflowExecutionActor = system.actorOf( WorkflowExecutionActor.props(workflowId, engineWorkflowDescriptor, serviceRegistry, jobStore, - callCacheReadActor, AllBackendInitializationData.empty, restarting = false), + callCacheReadActor, jobTokenDispenserActor, AllBackendInitializationData.empty, restarting = false), "WorkflowExecutionActor") val scatterLog = "Starting calls: scatter0.inside_scatter:0:1, scatter0.inside_scatter:1:1, scatter0.inside_scatter:2:1, scatter0.inside_scatter:3:1, scatter0.inside_scatter:4:1" diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPendingSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPendingSpec.scala index 9fc78f1f7..2810753cc 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPendingSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPendingSpec.scala @@ -1,8 +1,7 @@ package cromwell.engine.workflow.lifecycle.execution.ejea -import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{CheckingJobStore, EngineJobExecutionActorState, Execute, Pending, PreparingJob} -import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor -import cromwell.jobstore.JobStoreActor.QueryJobCompletion +import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.JobExecutionTokenRequest import org.scalatest.concurrent.Eventually class EjeaPendingSpec extends EngineJobExecutionActorSpec with CanValidateJobStoreKey with Eventually { @@ -11,29 +10,16 @@ class EjeaPendingSpec extends EngineJobExecutionActorSpec with CanValidateJobSto "An EJEA in the Pending state" should { - CallCachingModes foreach { mode => - s"check against the Job Store if restarting is true ($mode)" in { - ejea = helper.buildEJEA(restarting = true) + List(false, true) foreach { restarting => + s"wait for the Execute signal then request an execution token (with restarting=$restarting)" in { + ejea = helper.buildEJEA(restarting = restarting) ejea ! Execute - helper.jobStoreProbe.expectMsgPF(max = awaitTimeout, hint = "Awaiting job store lookup") { - case QueryJobCompletion(jobKey, taskOutputs) => - validateJobStoreKey(jobKey) - taskOutputs should be(helper.task.outputs) - } - helper.bjeaProbe.expectNoMsg(awaitAlmostNothing) - helper.jobHashingInitializations shouldBe NothingYet - ejea.stateName should be(CheckingJobStore) - } - - - s"bypass the Job Store and start preparing the job for running or call caching ($mode)" in { - ejea = helper.buildEJEA(restarting = false) - ejea ! Execute + helper.jobTokenDispenserProbe.expectMsgClass(max = awaitTimeout, classOf[JobExecutionTokenRequest]) - helper.jobPreparationProbe.expectMsg(max = awaitTimeout, hint = "Awaiting job preparation", JobPreparationActor.Start) - helper.jobStoreProbe.expectNoMsg(awaitAlmostNothing) - ejea.stateName should be(PreparingJob) + helper.jobPreparationProbe.msgAvailable should be(false) + helper.jobStoreProbe.msgAvailable should be(false) + ejea.stateName should be(RequestingExecutionToken) } } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRequestingExecutionTokenSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRequestingExecutionTokenSpec.scala new file mode 100644 index 000000000..83ae08835 --- /dev/null +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRequestingExecutionTokenSpec.scala @@ -0,0 +1,54 @@ +package cromwell.engine.workflow.lifecycle.execution.ejea + +import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ +import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed} +import cromwell.jobstore.JobStoreActor.QueryJobCompletion +import org.scalatest.concurrent.Eventually + +class EjeaRequestingExecutionTokenSpec extends EngineJobExecutionActorSpec with CanValidateJobStoreKey with Eventually { + + override implicit val stateUnderTest: EngineJobExecutionActorState = RequestingExecutionToken + + "An EJEA in the RequestingExecutionToken state" should { + + List(true, false) foreach { restarting => + s"do nothing when denied a token (with restarting=$restarting)" in { + ejea = helper.buildEJEA(restarting = restarting) + ejea ! JobExecutionTokenDenied(1) // 1 is arbitrary. Doesn't matter what position in the queue we are. + + helper.jobTokenDispenserProbe.expectNoMsg(max = awaitAlmostNothing) + helper.jobPreparationProbe.msgAvailable should be(false) + helper.jobStoreProbe.msgAvailable should be(false) + + ejea.stateName should be(RequestingExecutionToken) + } + } + + CallCachingModes foreach { mode => + s"check against the Job Store if restarting is true ($mode)" in { + ejea = helper.buildEJEA(restarting = true) + ejea ! JobExecutionTokenDispensed(helper.executionToken) + + helper.jobStoreProbe.expectMsgPF(max = awaitTimeout, hint = "Awaiting job store lookup") { + case QueryJobCompletion(jobKey, taskOutputs) => + validateJobStoreKey(jobKey) + taskOutputs should be(helper.task.outputs) + } + helper.bjeaProbe.expectNoMsg(awaitAlmostNothing) + helper.jobHashingInitializations shouldBe NothingYet + ejea.stateName should be(CheckingJobStore) + } + + + s"bypass the Job Store and start preparing the job for running or call caching ($mode)" in { + ejea = helper.buildEJEA(restarting = false) + ejea ! JobExecutionTokenDispensed(helper.executionToken) + + helper.jobPreparationProbe.expectMsg(max = awaitTimeout, hint = "Awaiting job preparation", JobPreparationActor.Start) + helper.jobStoreProbe.expectNoMsg(awaitAlmostNothing) + ejea.stateName should be(PreparingJob) + } + } + } +} diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala index 6fd15b90a..80ae87fa9 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala @@ -1,11 +1,14 @@ package cromwell.engine.workflow.lifecycle.execution.ejea +import java.util.UUID + import akka.actor.{ActorRef, ActorSystem, Props} import akka.testkit.{TestFSMRef, TestProbe} import cromwell.backend.BackendJobExecutionActor.SucceededResponse import cromwell.backend.{BackendInitializationData, BackendJobDescriptor, BackendJobDescriptorKey, BackendLifecycleActorFactory, BackendWorkflowDescriptor} +import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.core.callcaching.{CallCachingActivity, CallCachingMode, CallCachingOff} -import cromwell.core.{ExecutionStore, OutputStore, WorkflowId} +import cromwell.core.{ExecutionStore, JobExecutionToken, OutputStore, WorkflowId} import cromwell.engine.EngineWorkflowDescriptor import cromwell.engine.workflow.lifecycle.execution.{EngineJobExecutionActor, WorkflowExecutionActorData} import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{EJEAData, EngineJobExecutionActorState} @@ -31,6 +34,8 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock val jobIndex = Some(1) val jobAttempt = 1 + val executionToken = JobExecutionToken(JobExecutionTokenType("test", None), UUID.randomUUID()) + val task = mock[Task] task.declarations returns Seq.empty task.runtimeAttributes returns RuntimeAttributes(Map.empty) @@ -70,6 +75,7 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock val callCacheReadActorProbe = TestProbe() val callCacheHitCopyingProbe = TestProbe() val jobPreparationProbe = TestProbe() + val jobTokenDispenserProbe = TestProbe() def buildFactory() = new BackendLifecycleActorFactory { @@ -113,6 +119,7 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock serviceRegistryActor = serviceRegistryProbe.ref, jobStoreActor = jobStoreProbe.ref, callCacheReadActor = callCacheReadActorProbe.ref, + jobTokenDispenserActor = jobTokenDispenserProbe.ref, backendName = "NOT USED", callCachingMode = callCachingMode )), parentProbe.ref, s"EngineJobExecutionActorSpec-$workflowId") @@ -133,8 +140,9 @@ private[ejea] class MockEjea(helper: PerTestHelper, serviceRegistryActor: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef, backendName: String, - callCachingMode: CallCachingMode) extends EngineJobExecutionActor(replyTo, jobDescriptorKey, executionData, factory, initializationData, restarting, serviceRegistryActor, jobStoreActor, callCacheReadActor, backendName, callCachingMode) { + callCachingMode: CallCachingMode) extends EngineJobExecutionActor(replyTo, jobDescriptorKey, executionData, factory, initializationData, restarting, serviceRegistryActor, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendName, callCachingMode) { override def makeFetchCachedResultsActor(cacheHit: CacheHit, taskOutputs: Seq[TaskOutput]) = helper.fetchCachedResultsActorCreations = helper.fetchCachedResultsActorCreations.foundOne((cacheHit, taskOutputs)) override def initializeJobHashing(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity) = helper.jobHashingInitializations = helper.jobHashingInitializations.foundOne((jobDescriptor, activity)) diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala new file mode 100644 index 000000000..543fc9473 --- /dev/null +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala @@ -0,0 +1,311 @@ +package cromwell.engine.workflow.tokens + +import java.util.UUID + +import akka.actor.{ ActorRef, ActorSystem, Kill, PoisonPill} +import org.scalatest._ +import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} +import cromwell.core.JobExecutionToken.JobExecutionTokenType +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed, JobExecutionTokenRequest, JobExecutionTokenReturn} +import JobExecutionTokenDispenserActorSpec._ +import cromwell.core.JobExecutionToken +import cromwell.engine.workflow.tokens.TokenGrabbingActor.StoppingSupervisor +import org.scalatest.concurrent.Eventually + +import scala.concurrent.duration._ + +class JobExecutionTokenDispenserActorSpec extends TestKit(ActorSystem("JETDASpec")) with ImplicitSender with FlatSpecLike with Matchers with BeforeAndAfter with BeforeAndAfterAll with Eventually { + + val MaxWaitTime = 100.milliseconds + implicit val pc: PatienceConfig = PatienceConfig(MaxWaitTime) + + behavior of "JobExecutionTokenDispenserActor" + + it should "dispense an infinite token correctly" in { + actorRefUnderTest ! JobExecutionTokenRequest(TestInfiniteTokenType) + expectMsgPF(max = MaxWaitTime, hint = "token dispensed message") { + case JobExecutionTokenDispensed(token) => + token.jobExecutionTokenType should be(TestInfiniteTokenType) + } + } + + it should "accept return of an infinite token correctly" in { + actorRefUnderTest ! JobExecutionTokenRequest(TestInfiniteTokenType) + expectMsgPF(max = MaxWaitTime, hint = "token dispensed message") { + case JobExecutionTokenDispensed(token) => + actorRefUnderTest ! JobExecutionTokenReturn(token) + } + } + + it should "dispense indefinitely for an infinite token type" in { + var currentSet: Set[JobExecutionToken] = Set.empty + 100 indexedTimes { i => + val sender = TestProbe() + actorRefUnderTest.tell(msg = JobExecutionTokenRequest(TestInfiniteTokenType), sender = sender.ref) + sender.expectMsgPF(max = MaxWaitTime, hint = "token dispensed message") { + case JobExecutionTokenDispensed(token) => + token.jobExecutionTokenType should be(TestInfiniteTokenType) + currentSet.contains(token) should be(false) + currentSet += token + } + } + } + + it should "dispense a limited token correctly" in { + + actorRefUnderTest ! JobExecutionTokenRequest(LimitedTo5Tokens) + expectMsgPF(max = MaxWaitTime, hint = "token dispensed message") { + case JobExecutionTokenDispensed(token) => token.jobExecutionTokenType should be(LimitedTo5Tokens) + } + } + + it should "accept return of a limited token type correctly" in { + actorRefUnderTest ! JobExecutionTokenRequest(LimitedTo5Tokens) + expectMsgPF(max = MaxWaitTime, hint = "token dispensed message") { + case JobExecutionTokenDispensed(token) => actorRefUnderTest ! JobExecutionTokenReturn(token) + } + } + + it should "limit the dispensing of a limited token type" in { + + var currentTokens: Map[TestProbe, JobExecutionToken] = Map.empty + val dummyActors = (0 until 100 map { i => i -> TestProbe("dummy_" + i) }).toMap + + // Dispense the first 5: + 5 indexedTimes { i => + val sndr = dummyActors(i) + actorRefUnderTest.tell(msg = JobExecutionTokenRequest(LimitedTo5Tokens), sender = sndr.ref) + sndr.expectMsgPF(max = MaxWaitTime, hint = "token dispensed message") { + case JobExecutionTokenDispensed(token) => + token.jobExecutionTokenType should be(LimitedTo5Tokens) + currentTokens.values.toList.contains(token) should be(false) // Check we didn't already get this token + currentTokens += sndr -> token + } + } + + // Queue the next 95: + 95 indexedTimes { i => + val sndr = dummyActors(5 + i) + actorRefUnderTest.tell(msg = JobExecutionTokenRequest(LimitedTo5Tokens), sender = sndr.ref) + sndr.expectMsgPF(max = MaxWaitTime, hint = "token denied message") { + case JobExecutionTokenDenied(positionInQueue) => + positionInQueue should be(i) + } + } + + // It should allow queued actors to check their position in the queue: + 95 indexedTimes { i => + val sndr = dummyActors(5 + i) + actorRefUnderTest.tell(msg = JobExecutionTokenRequest(LimitedTo5Tokens), sender = sndr.ref) + sndr.expectMsgPF(max = MaxWaitTime, hint = "token denied message") { + case JobExecutionTokenDenied(positionInQueue) => + positionInQueue should be(i) + } + } + + // It should release tokens as soon as they're available (while there's still a queue...): + 95 indexedTimes { i => + val returner = dummyActors(i) + val nextInLine = dummyActors(i + 5) + val tokenBeingReturned = currentTokens(returner) + actorRefUnderTest.tell(msg = JobExecutionTokenReturn(tokenBeingReturned), sender = returner.ref) + currentTokens -= returner + nextInLine.expectMsgPF(max = MaxWaitTime, hint = s"token dispensed message to the next in line actor (#${i + 5})") { + case JobExecutionTokenDispensed(token) => + token should be(tokenBeingReturned) // It just gets immediately passed out again! + currentTokens += nextInLine -> token + } + } + + // Double-check the queue state: when we request a token now, we should still be denied: + actorRefUnderTest ! JobExecutionTokenRequest(LimitedTo5Tokens) + expectMsgClass(classOf[JobExecutionTokenDenied]) + + //And finally, silently release the remaining tokens: + 5 indexedTimes { i => + val returner = dummyActors(i + 95) + val tokenBeingReturned = currentTokens(returner) + actorRefUnderTest.tell(msg = JobExecutionTokenReturn(tokenBeingReturned), sender = returner.ref) + currentTokens -= returner + } + + // And we should have gotten our own token by now: + expectMsgClass(classOf[JobExecutionTokenDispensed]) + + // Check we didn't get anything else in the meanwhile: + msgAvailable should be(false) + dummyActors.values foreach { testProbe => testProbe.msgAvailable should be(false) } + } + + it should "resend the same token to an actor which already has one" in { + actorRefUnderTest ! JobExecutionTokenRequest(LimitedTo5Tokens) + val firstResponse = expectMsgClass(classOf[JobExecutionTokenDispensed]) + + 5 indexedTimes { i => + actorRefUnderTest ! JobExecutionTokenRequest(LimitedTo5Tokens) + expectMsg(MaxWaitTime, s"same token again (attempt ${i + 1})", firstResponse) // Always the same + } + } + + + // Incidentally, also covers: it should "not be fooled if the wrong actor returns a token" + it should "not be fooled by a doubly-returned token" in { + var currentTokens: Map[TestProbe, JobExecutionToken] = Map.empty + val dummyActors = (0 until 7 map { i => i -> TestProbe("dummy_" + i) }).toMap + + // Set up by taking all 5 tokens out, and then adding 2 to the queue: + 5 indexedTimes { i => + val sndr = dummyActors(i) + actorRefUnderTest.tell(msg = JobExecutionTokenRequest(LimitedTo5Tokens), sender = sndr.ref) + currentTokens += dummyActors(i) -> sndr.expectMsgClass(classOf[JobExecutionTokenDispensed]).jobExecutionToken + } + 2 indexedTimes { i => + val sndr = dummyActors(5 + i) + actorRefUnderTest.tell(msg = JobExecutionTokenRequest(LimitedTo5Tokens), sender = sndr.ref) + sndr.expectMsgClass(classOf[JobExecutionTokenDenied]) + } + + // The first time we return a token, the next in line should be given it: + val returningActor = dummyActors(0) + val nextInLine1 = dummyActors(5) + val nextInLine2 = dummyActors(6) + val tokenBeingReturned = currentTokens(returningActor) + currentTokens -= returningActor + actorRefUnderTest.tell(msg = JobExecutionTokenReturn(tokenBeingReturned), sender = returningActor.ref) + val tokenPassedOn = nextInLine1.expectMsgClass(classOf[JobExecutionTokenDispensed]).jobExecutionToken + tokenPassedOn should be(tokenBeingReturned) + currentTokens += nextInLine1 -> tokenPassedOn + + // But the next time, nothing should happen because the wrong actor is returning the token: + actorRefUnderTest.tell(msg = JobExecutionTokenReturn(tokenBeingReturned), sender = returningActor.ref) + nextInLine2.expectNoMsg(MaxWaitTime) + } + + it should "not be fooled if an actor returns a token which doesn't exist" in { + var currentTokens: Map[TestProbe, JobExecutionToken] = Map.empty + val dummyActors = (0 until 6 map { i => i -> TestProbe("dummy_" + i) }).toMap + + // Set up by taking all 5 tokens out, and then adding 2 to the queue: + 5 indexedTimes { i => + val sndr = dummyActors(i) + actorRefUnderTest.tell(msg = JobExecutionTokenRequest(LimitedTo5Tokens), sender = sndr.ref) + currentTokens += dummyActors(i) -> sndr.expectMsgClass(classOf[JobExecutionTokenDispensed]).jobExecutionToken + } + 1 indexedTimes { i => + val sndr = dummyActors(5 + i) + actorRefUnderTest.tell(msg = JobExecutionTokenRequest(LimitedTo5Tokens), sender = sndr.ref) + sndr.expectMsgClass(classOf[JobExecutionTokenDenied]) + } + + actorRefUnderTest.tell(msg = JobExecutionTokenReturn(JobExecutionToken(LimitedTo5Tokens, UUID.randomUUID())), sender = dummyActors(0).ref) + dummyActors(5).expectNoMsg(MaxWaitTime) + } + + val actorDeathMethods: List[(String, ActorRef => Unit)] = List( + ("external_stop", (a: ActorRef) => system.stop(a)), + ("internal_stop", (a: ActorRef) => a ! TokenGrabbingActor.InternalStop), + ("poison_pill", (a: ActorRef) => a ! PoisonPill), + ("kill_message", (a: ActorRef) => a ! Kill), + ("throw_exception", (a: ActorRef) => a ! TokenGrabbingActor.ThrowException) + ) + + actorDeathMethods foreach { case (name, stopMethod) => + it should s"recover tokens lost to actors which are $name before they hand back their token" in { + var currentTokens: Map[TestActorRef[TokenGrabbingActor], JobExecutionToken] = Map.empty + var tokenGrabbingActors: Map[Int, TestActorRef[TokenGrabbingActor]] = Map.empty + val grabberSupervisor = TestActorRef(new StoppingSupervisor()) + + // Set up by taking all 5 tokens out, and then adding 2 to the queue: + 5 indexedTimes { i => + val newGrabbingActor = TestActorRef[TokenGrabbingActor](TokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"grabber_${name}_" + i) + tokenGrabbingActors += i -> newGrabbingActor + eventually { + newGrabbingActor.underlyingActor.token.isDefined should be(true) + } + currentTokens += newGrabbingActor -> newGrabbingActor.underlyingActor.token.get + } + + val unassignedActorIndex = 5 + val newGrabbingActor = TestActorRef(new TokenGrabbingActor(actorRefUnderTest, LimitedTo5Tokens), s"grabber_${name}_" + unassignedActorIndex) + tokenGrabbingActors += unassignedActorIndex -> newGrabbingActor + eventually { + newGrabbingActor.underlyingActor.rejections should be(1) + } + + val actorToStop = tokenGrabbingActors(0) + val actorToStopsToken = currentTokens(actorToStop) + val nextInLine = tokenGrabbingActors(unassignedActorIndex) + + val deathwatch = TestProbe() + deathwatch watch actorToStop + stopMethod(actorToStop) + deathwatch.expectTerminated(actorToStop) + eventually { nextInLine.underlyingActor.token should be(Some(actorToStopsToken)) } + } + } + + it should "skip over dead actors when assigning tokens to the actor queue" in { + var currentTokens: Map[TestActorRef[TokenGrabbingActor], JobExecutionToken] = Map.empty + var tokenGrabbingActors: Map[Int, TestActorRef[TokenGrabbingActor]] = Map.empty + val grabberSupervisor = TestActorRef(new StoppingSupervisor()) + + // Set up by taking all 5 tokens out, and then adding 2 to the queue: + 5 indexedTimes { i => + val newGrabbingActor = TestActorRef[TokenGrabbingActor](TokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"skip_test_" + i) + tokenGrabbingActors += i -> newGrabbingActor + eventually { + newGrabbingActor.underlyingActor.token.isDefined should be(true) + } + currentTokens += newGrabbingActor -> newGrabbingActor.underlyingActor.token.get + } + 2 indexedTimes { i => + val index = i + 5 + val newGrabbingActor = TestActorRef[TokenGrabbingActor](TokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"skip_test_" + index) + tokenGrabbingActors += index -> newGrabbingActor + eventually { + newGrabbingActor.underlyingActor.rejections should be(1) + } + } + + val returningActor = tokenGrabbingActors(0) + val returnedToken = currentTokens(returningActor) + val nextInLine1 = tokenGrabbingActors(5) + val nextInLine2 = tokenGrabbingActors(6) + + // First, kill off the actor which would otherwise be first in line: + val deathwatch = TestProbe() + deathwatch watch nextInLine1 + nextInLine1 ! PoisonPill + deathwatch.expectTerminated(nextInLine1) + + // Now, stop one of the workers unexpectedly and check that the released token goes to the right place: + actorRefUnderTest.tell(msg = JobExecutionTokenReturn(returnedToken), sender = returningActor) + eventually { nextInLine2.underlyingActor.token should be(Some(returnedToken)) } // Some is OK. This is the **expected** value! + } + + var actorRefUnderTest: TestActorRef[JobExecutionTokenDispenserActor] = _ + + before { + actorRefUnderTest = TestActorRef(new JobExecutionTokenDispenserActor()) + + } + after { + actorRefUnderTest = null + } + + override def afterAll = { + TestKit.shutdownActorSystem(system) + } +} + +object JobExecutionTokenDispenserActorSpec { + + implicit class intWithTimes(n: Int) { + def times(f: => Unit) = 1 to n foreach { _ => f } + def indexedTimes(f: Int => Unit) = 0 until n foreach { i => f(i) } + } + + val TestInfiniteTokenType = JobExecutionTokenType("infinite", maxPoolSize = None) + def limitedTokenType(limit: Int) = JobExecutionTokenType(s"$limit-limit", maxPoolSize = Option(limit)) + val LimitedTo5Tokens = limitedTokenType(5) +} diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/TokenGrabbingActor.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/TokenGrabbingActor.scala new file mode 100644 index 000000000..06eff8583 --- /dev/null +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/TokenGrabbingActor.scala @@ -0,0 +1,38 @@ +package cromwell.engine.workflow.tokens + +import akka.actor.{Actor, ActorRef, Props, SupervisorStrategy} +import cromwell.core.JobExecutionToken +import cromwell.core.JobExecutionToken.JobExecutionTokenType +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed, JobExecutionTokenRequest} +import cromwell.engine.workflow.tokens.TokenGrabbingActor.{InternalStop, ThrowException} + +/** + * Grabs a token and doesn't let it go! + */ +class TokenGrabbingActor(tokenDispenser: ActorRef, tokenType: JobExecutionTokenType) extends Actor { + + var token: Option[JobExecutionToken] = None + var rejections = 0 + + def receive = { + case JobExecutionTokenDispensed(dispensedToken) => token = Option(dispensedToken) + case JobExecutionTokenDenied(positionInQueue) => rejections += 1 + case ThrowException => throw new RuntimeException("Test exception (don't be scared by the stack trace, it's deliberate!)") + case InternalStop => context.stop(self) + } + + tokenDispenser ! JobExecutionTokenRequest(tokenType) +} + +object TokenGrabbingActor { + + def props(tokenDispenserActor: ActorRef, tokenType: JobExecutionTokenType) = Props(new TokenGrabbingActor(tokenDispenserActor, tokenType)) + + case object ThrowException + case object InternalStop + + class StoppingSupervisor extends Actor { + override val supervisorStrategy = SupervisorStrategy.stoppingStrategy + override def receive = Actor.emptyBehavior + } +} diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala index bd4ce3add..7faf6baed 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala @@ -13,7 +13,7 @@ import wdl4s.expression.WdlStandardLibraryFunctions import scala.util.{Failure, Success, Try} -case class HtCondorBackendFactory(configurationDescriptor: BackendConfigurationDescriptor) +case class HtCondorBackendFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory with StrictLogging { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala index 4028166cc..3f498ff71 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala @@ -15,7 +15,7 @@ import wdl4s.expression.WdlStandardLibraryFunctions import scala.language.postfixOps -case class JesBackendLifecycleActorFactory(configurationDescriptor: BackendConfigurationDescriptor) +case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory { import JesBackendLifecycleActorFactory._ diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala index a92e7fdb5..93ad1fe3a 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala @@ -4,6 +4,7 @@ import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction import cromwell.backend.impl.sfs.config.ConfigConstants._ import cromwell.backend.sfs._ import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, RuntimeAttributeDefinition} +import cromwell.core.JobExecutionToken.JobExecutionTokenType import lenthall.config.ScalaConfig._ import org.slf4j.LoggerFactory @@ -12,7 +13,7 @@ import org.slf4j.LoggerFactory * * @param configurationDescriptor The config information. */ -class ConfigBackendLifecycleActorFactory(val configurationDescriptor: BackendConfigurationDescriptor) +class ConfigBackendLifecycleActorFactory(name: String, val configurationDescriptor: BackendConfigurationDescriptor) extends SharedFileSystemBackendLifecycleActorFactory { lazy val logger = LoggerFactory.getLogger(getClass) @@ -44,4 +45,9 @@ class ConfigBackendLifecycleActorFactory(val configurationDescriptor: BackendCon } override lazy val fileHashingActorCount: Int = 5 + + override val jobExecutionTokenType: JobExecutionTokenType = { + val concurrentJobLimit = configurationDescriptor.backendConfig.getIntOption("concurrent-job-limit") + JobExecutionTokenType(name, concurrentJobLimit) + } } diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala index d69446519..ba09b6cb9 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala @@ -8,7 +8,7 @@ import cromwell.core.CallContext import wdl4s.Call import wdl4s.expression.WdlStandardLibraryFunctions -case class SparkBackendFactory(configurationDescriptor: BackendConfigurationDescriptor, actorSystem: ActorSystem) extends BackendLifecycleActorFactory { +case class SparkBackendFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor, actorSystem: ActorSystem) extends BackendLifecycleActorFactory { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], serviceRegistryActor: ActorRef): Option[Props] = { Option(SparkInitializationActor.props(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor)) } From b5eeb30bbc33b296dca6ec8d9c02e047261c8eea Mon Sep 17 00:00:00 2001 From: Ruchi Date: Wed, 5 Oct 2016 10:09:06 -0400 Subject: [PATCH 014/375] allowing for multiple inputs (#1511) change submitRoute to accept multiple inputs + test overrides occur for conflict input keys --- engine/src/main/resources/swagger/cromwell.yaml | 22 +++++++++++++++++- .../cromwell/webservice/CromwellApiService.scala | 27 ++++++++++++++++++++-- .../webservice/CromwellApiServiceSpec.scala | 15 +++++++++++- 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/engine/src/main/resources/swagger/cromwell.yaml b/engine/src/main/resources/swagger/cromwell.yaml index cdbce1392..7960bb937 100644 --- a/engine/src/main/resources/swagger/cromwell.yaml +++ b/engine/src/main/resources/swagger/cromwell.yaml @@ -64,7 +64,27 @@ paths: type: file in: formData - name: workflowInputs - description: WDL Inputs JSON + description: WDL Inputs JSON, 1 + required: false + type: file + in: formData + - name: workflowInputs_2 + description: WDL Inputs JSON, 2 + required: false + type: file + in: formData + - name: workflowInputs_3 + description: WDL Inputs JSON, 3 + required: false + type: file + in: formData + - name: workflowInputs_4 + description: WDL Inputs JSON, 4 + required: false + type: file + in: formData + - name: workflowInputs_5 + description: WDL Inputs JSON, 5 required: false type: file in: formData diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala index 8e7c55168..6f66407fd 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala @@ -1,6 +1,7 @@ package cromwell.webservice import akka.actor._ +import java.lang.Throwable._ import cats.data.NonEmptyList import cromwell.core.{WorkflowId, WorkflowSourceFiles} import cromwell.engine.backend.BackendConfiguration @@ -25,6 +26,23 @@ trait CromwellApiService extends HttpService with PerRequestCreator { val workflowStoreActor: ActorRef val serviceRegistryActor: ActorRef + def toMap(someInput: Option[String]): Map[String, JsValue] = { + import spray.json._ + someInput match { + case Some(inputs: String) => inputs.parseJson match { + case JsObject(inputMap) => inputMap + case _ => + throw new RuntimeException(s"Submitted inputs couldn't be processed, please check for syntactical errors") + } + case None => Map.empty + } + } + + def mergeMaps(allInputs: Seq[Option[String]]): JsObject = { + val convertToMap = allInputs.map(x => toMap(x)) + JsObject(convertToMap reduce (_ ++ _)) + } + def metadataBuilderProps: Props = MetadataBuilderActor.props(serviceRegistryActor) def handleMetadataRequest(message: AnyRef): Route = { @@ -109,9 +127,14 @@ trait CromwellApiService extends HttpService with PerRequestCreator { def submitRoute = path("workflows" / Segment) { version => post { - formFields("wdlSource", "workflowInputs".?, "workflowOptions".?) { (wdlSource, workflowInputs, workflowOptions) => + formFields("wdlSource", "workflowInputs".?, "workflowInputs_2".?, "workflowInputs_3".?, + "workflowInputs_4".?, "workflowInputs_5".?, "workflowOptions".?) { + (wdlSource, workflowInputs, workflowInputs_2, workflowInputs_3, workflowInputs_4, workflowInputs_5, workflowOptions) => requestContext => - val workflowSourceFiles = WorkflowSourceFiles(wdlSource, workflowInputs.getOrElse("{}"), workflowOptions.getOrElse("{}")) + //The order of addition allows for the expected override of colliding keys. + val wfInputs = mergeMaps(Seq(workflowInputs, workflowInputs_2, workflowInputs_3, workflowInputs_4, workflowInputs_5)).toString + + val workflowSourceFiles = WorkflowSourceFiles(wdlSource, wfInputs, workflowOptions.getOrElse("{}")) perRequest(requestContext, CromwellApiHandler.props(workflowStoreActor), CromwellApiHandler.ApiHandlerWorkflowSubmit(workflowSourceFiles)) } } diff --git a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala index d6dc8919d..87db769c9 100644 --- a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala @@ -15,7 +15,8 @@ import cromwell.server.{CromwellServerActor, CromwellSystem} import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata._ import cromwell.services.metadata.impl.MetadataSummaryRefreshActor.MetadataSummarySuccess -import cromwell.util.SampleWdl.HelloWorld +import cromwell.util.SampleWdl.DeclarationsWorkflow._ +import cromwell.util.SampleWdl.{ExpressionsInInputs, DeclarationsWorkflow, ThreeStep, HelloWorld} import org.scalatest.concurrent.{PatienceConfiguration, ScalaFutures} import org.scalatest.{FlatSpec, Matchers} import org.specs2.mock.Mockito @@ -256,6 +257,18 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala } } } + it should "succesfully merge and override multiple input files" in { + + val input1 = Map("wf.a1" -> "hello", "wf.a2" -> "world").toJson.toString + val input2 = Map.empty.toJson.toString + val overrideInput1 = Map("wf.a2" -> "universe").toJson.toString + val allInputs = mergeMaps(Seq(Option(input1), Option(input2), Option(overrideInput1))) + + check { + allInputs.fields.keys should contain allOf("wf.a1", "wf.a2") + allInputs.fields("wf.a2") should be(JsString("universe")) + } + } behavior of "REST API batch submission endpoint" it should "return 200 for a successful workflow submission " in { From 72a3cfde8a1993970ef0362b63180ac6bddf0b61 Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Fri, 30 Sep 2016 17:12:56 -0400 Subject: [PATCH 015/375] Lots of cool compiler flags. Closes #1282. --- .../BackendWorkflowInitializationActor.scala | 2 +- .../backend/RuntimeAttributeDefinition.scala | 1 - .../async/AsyncBackendJobExecutionActor.scala | 3 + .../src/main/scala/cromwell/backend/backend.scala | 53 ++++++++ .../src/main/scala/cromwell/backend/package.scala | 54 +------- .../backend/validation/MemoryValidation.scala | 2 +- .../cromwell/backend/wdl/ReadLikeFunctions.scala | 1 - .../scala/cromwell/backend/io/TestWorkflows.scala | 14 +-- core/src/main/scala/cromwell/core/ConfigUtil.scala | 2 +- .../main/scala/cromwell/core/ExecutionStore.scala | 1 - core/src/main/scala/cromwell/core/PathCopier.scala | 1 + .../src/main/scala/cromwell/core/PathFactory.scala | 4 +- core/src/main/scala/cromwell/core/core.scala | 12 ++ .../cromwell/core/logging/LoggerWrapper.scala | 10 +- core/src/main/scala/cromwell/core/package.scala | 9 -- .../test/scala/cromwell/core/retry/RetrySpec.scala | 4 - core/src/test/scala/cromwell/util/SampleWdl.scala | 137 +++++++++++---------- .../scala/cromwell/util/TryWithResourceSpec.scala | 2 +- .../database/migration/WdlTransformation.scala | 1 - .../metadata/table/symbol/QueryPaginator.scala | 3 +- .../table/symbol/SymbolTableMigration.scala | 1 - .../restart/table/JobStoreSimpletonMigration.scala | 2 - .../database/slick/JobKeyValueSlickDatabase.scala | 6 +- .../database/slick/JobStoreSlickDatabase.scala | 7 +- .../database/slick/MetadataSlickDatabase.scala | 12 +- .../cromwell/database/slick/SlickDatabase.scala | 2 +- .../slick/SummaryStatusSlickDatabase.scala | 4 +- .../slick/WorkflowStoreSlickDatabase.scala | 9 +- .../scala/cromwell/database/sql/SqlDatabase.scala | 2 +- .../cromwell/engine/backend/CromwellBackends.scala | 1 - engine/src/main/scala/cromwell/engine/engine.scala | 25 ++++ .../src/main/scala/cromwell/engine/package.scala | 22 ---- .../workflow/SingleWorkflowRunnerActor.scala | 8 +- .../cromwell/engine/workflow/WorkflowActor.scala | 1 - .../engine/workflow/WorkflowManagerActor.scala | 10 +- .../workflow/lifecycle/CopyWorkflowLogsActor.scala | 6 +- .../MaterializeWorkflowDescriptorActor.scala | 1 - .../lifecycle/WorkflowFinalizationActor.scala | 1 - .../lifecycle/WorkflowInitializationActor.scala | 1 - .../execution/EngineJobExecutionActor.scala | 14 ++- .../execution/WorkflowExecutionActor.scala | 4 +- .../execution/WorkflowExecutionActorData.scala | 1 - .../execution/callcaching/CallCache.scala | 1 - .../execution/callcaching/CallCacheReadActor.scala | 11 +- .../engine/workflow/lifecycle/lifecycle.scala | 7 ++ .../engine/workflow/lifecycle/package.scala | 8 -- .../tokens/JobExecutionTokenDispenserActor.scala | 7 +- .../workflowstore/WorkflowStoreActor.scala | 1 - .../engine/workflow/workflowstore/package.scala | 18 --- .../workflow/workflowstore/workflowstore_.scala | 15 +++ .../cromwell/jobstore/JobStoreWriterActor.scala | 2 +- .../main/scala/cromwell/jobstore/jobstore_.scala | 10 ++ .../src/main/scala/cromwell/jobstore/package.scala | 8 +- .../scala/cromwell/server/CromwellServer.scala | 13 +- .../scala/cromwell/server/CromwellSystem.scala | 8 +- .../scala/cromwell/webservice/ApiDataModels.scala | 1 - .../cromwell/webservice/CromwellApiHandler.scala | 19 ++- .../cromwell/webservice/CromwellApiService.scala | 5 +- .../scala/cromwell/webservice/PerRequest.scala | 13 +- .../webservice/metadata/IndexedJsonValue.scala | 34 ++--- .../webservice/metadata/MetadataBuilderActor.scala | 17 +-- .../main/scala/cromwell/webservice/package.scala | 37 ------ .../scala/cromwell/webservice/webservice_.scala | 39 ++++++ .../scala/cromwell/ArrayOfArrayCoercionSpec.scala | 1 - .../test/scala/cromwell/ArrayWorkflowSpec.scala | 3 - .../scala/cromwell/CallCachingWorkflowSpec.scala | 16 +-- engine/src/test/scala/cromwell/CromwellSpec.scala | 1 - .../test/scala/cromwell/CromwellTestkitSpec.scala | 10 +- .../scala/cromwell/DeclarationWorkflowSpec.scala | 1 - .../scala/cromwell/FilePassingWorkflowSpec.scala | 4 +- .../src/test/scala/cromwell/MapWorkflowSpec.scala | 1 - .../test/scala/cromwell/MetadataWatchActor.scala | 1 + .../MultipleFilesWithSameNameWorkflowSpec.scala | 3 +- .../scala/cromwell/OptionalParamWorkflowSpec.scala | 5 +- .../cromwell/PostfixQuantifierWorkflowSpec.scala | 1 - .../test/scala/cromwell/RestartWorkflowSpec.scala | 4 +- .../test/scala/cromwell/ScatterWorkflowSpec.scala | 2 - .../scala/cromwell/SimpleWorkflowActorSpec.scala | 1 - .../cromwell/WdlFunctionsAtWorkflowLevelSpec.scala | 6 +- .../test/scala/cromwell/WorkflowOutputsSpec.scala | 1 - .../cromwell/engine/WorkflowManagerActorSpec.scala | 3 +- .../workflow/SingleWorkflowRunnerActorSpec.scala | 18 +-- .../engine/workflow/WorkflowActorSpec.scala | 1 - .../workflow/lifecycle/CachingConfigSpec.scala | 12 +- .../MaterializeWorkflowDescriptorActorSpec.scala | 7 +- .../execution/ejea/EjeaCheckingCallCacheSpec.scala | 1 + .../execution/ejea/EjeaCheckingJobStoreSpec.scala | 13 +- .../ejea/EngineJobExecutionActorSpec.scala | 4 +- .../ejea/EngineJobExecutionActorSpecUtil.scala | 3 +- .../lifecycle/execution/ejea/ExpectOne.scala | 6 +- .../JobExecutionTokenDispenserActorSpec.scala | 2 +- .../cromwell/jobstore/JobStoreServiceSpec.scala | 2 - .../cromwell/jobstore/JobStoreWriterSpec.scala | 3 + .../webservice/CromwellApiServiceSpec.scala | 7 +- .../webservice/MetadataBuilderActorSpec.scala | 15 ++- .../cromwell/filesystems/gcs/GcsFileSystem.scala | 1 - .../filesystems/gcs/GcsFileSystemProvider.scala | 30 +++-- .../cromwell/filesystems/gcs/GoogleAuthMode.scala | 2 +- .../filesystems/gcs/GoogleConfiguration.scala | 1 - .../cromwell/filesystems/gcs/NioGcsPath.scala | 2 +- .../filesystems/gcs/GoogleConfigurationSpec.scala | 1 - project/Settings.scala | 26 +++- .../services/metadata/MetadataService.scala | 1 - .../services/metadata/WorkflowQueryKey.scala | 1 - .../metadata/WorkflowQueryParameters.scala | 1 - .../metadata/impl/MetadataDatabaseAccess.scala | 3 +- .../metadata/impl/MetadataServiceActor.scala | 10 +- .../impl/MetadataSummaryRefreshActor.scala | 1 - .../cromwell/services/metadata/metadata.scala | 39 ++++++ .../scala/cromwell/services/metadata/package.scala | 37 ------ src/main/scala/cromwell/Main.scala | 2 - .../scala/cromwell/CromwellCommandLineSpec.scala | 1 - .../impl/htcondor/HtCondorJobExecutionActor.scala | 28 +++-- .../backend/impl/htcondor/HtCondorWrapper.scala | 4 +- .../htcondor/HtCondorInitializationActorSpec.scala | 4 +- .../htcondor/HtCondorJobExecutionActorSpec.scala | 13 +- .../htcondor/HtCondorRuntimeAttributesSpec.scala | 12 +- .../backend/impl/jes/GenomicsFactory.scala | 1 - .../jes/JesAsyncBackendJobExecutionActor.scala | 8 +- .../cromwell/backend/impl/jes/JesAttributes.scala | 3 - .../impl/jes/JesBackendLifecycleActorFactory.scala | 1 - .../backend/impl/jes/JesFinalizationActor.scala | 5 +- .../backend/impl/jes/JesInitializationActor.scala | 6 +- .../impl/jes/JesJobCachingActorHelper.scala | 2 +- .../backend/impl/jes/JesJobExecutionActor.scala | 1 - .../backend/impl/jes/JesRuntimeAttributes.scala | 1 - .../main/scala/cromwell/backend/impl/jes/Run.scala | 7 +- .../backend/impl/jes/io/JesAttachedDisk.scala | 1 - .../cromwell/backend/impl/jes/io/package.scala | 2 - .../jes/JesAsyncBackendJobExecutionActorSpec.scala | 19 +-- .../impl/jes/JesInitializationActorSpec.scala | 4 +- .../impl/jes/JesRuntimeAttributesSpec.scala | 5 +- .../sfs/config/ConfigAsyncJobExecutionActor.scala | 1 + .../sfs/BackgroundAsyncJobExecutionActor.scala | 2 + .../scala/cromwell/backend/sfs/ProcessRunner.scala | 4 +- .../cromwell/backend/sfs/SharedFileSystem.scala | 13 +- .../SharedFileSystemAsyncJobExecutionActor.scala | 1 + .../sfs/SharedFileSystemCacheHitCopyingActor.scala | 6 +- .../sfs/SharedFileSystemInitializationActor.scala | 2 +- .../sfs/config/ConfigHashingStrategySpec.scala | 2 + .../SharedFileSystemInitializationActorSpec.scala | 4 +- ...stemValidatedRuntimeAttributesBuilderSpec.scala | 9 +- .../backend/impl/spark/SparkClusterProcess.scala | 5 +- .../impl/spark/SparkInitializationActor.scala | 4 +- .../impl/spark/SparkJobExecutionActor.scala | 10 +- .../cromwell/backend/impl/spark/SparkProcess.scala | 1 - .../impl/spark/SparkClusterProcessSpec.scala | 2 - .../impl/spark/SparkJobExecutionActorSpec.scala | 10 +- .../impl/spark/SparkRuntimeAttributesSpec.scala | 12 +- 149 files changed, 648 insertions(+), 614 deletions(-) create mode 100644 backend/src/main/scala/cromwell/backend/backend.scala create mode 100644 core/src/main/scala/cromwell/core/core.scala create mode 100644 engine/src/main/scala/cromwell/engine/engine.scala create mode 100644 engine/src/main/scala/cromwell/engine/workflow/lifecycle/lifecycle.scala delete mode 100644 engine/src/main/scala/cromwell/engine/workflow/lifecycle/package.scala delete mode 100644 engine/src/main/scala/cromwell/engine/workflow/workflowstore/package.scala create mode 100644 engine/src/main/scala/cromwell/engine/workflow/workflowstore/workflowstore_.scala create mode 100644 engine/src/main/scala/cromwell/jobstore/jobstore_.scala create mode 100644 engine/src/main/scala/cromwell/webservice/webservice_.scala create mode 100644 services/src/main/scala/cromwell/services/metadata/metadata.scala diff --git a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala index 7f0a273db..f98234ce5 100644 --- a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala @@ -142,7 +142,7 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w def badRuntimeAttrsForTask(task: Task) = { runtimeAttributeValidators map { case (attributeName, validator) => val value = task.runtimeAttributes.attrs.get(attributeName) orElse defaultRuntimeAttribute(attributeName) - attributeName -> (value, validator(value)) + attributeName -> ((value, validator(value))) } collect { case (name, (value, false)) => s"Task ${task.name} has an invalid runtime attribute $name = ${value map { _.valueString} getOrElse "!! NOT FOUND !!"}" } diff --git a/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala b/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala index a3b4d9255..238309143 100644 --- a/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala +++ b/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala @@ -39,7 +39,6 @@ object RuntimeAttributeDefinition { def addDefaultsToAttributes(runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition], workflowOptions: WorkflowOptions) (specifiedAttributes: Map[LocallyQualifiedName, WdlValue]): Map[LocallyQualifiedName, WdlValue] = { import WdlValueJsonFormatter._ - import spray.json._ // IGNORE INTELLIJ - this *is* required (unless it isn't any more, who will ever know...!) def isUnspecifiedAttribute(name: String) = !specifiedAttributes.contains(name) diff --git a/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala index 40747c346..bbbfbf82b 100644 --- a/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala @@ -70,7 +70,10 @@ trait AsyncBackendJobExecutionActor { this: Actor with ActorLogging => case IssuePollRequest(handle) => robustPoll(handle) case PollResponseReceived(handle) if handle.isDone => self ! Finish(handle) case PollResponseReceived(handle) => + // This should stash the Cancellable someplace so it can be cancelled once polling is complete. + // -Ywarn-value-discard context.system.scheduler.scheduleOnce(pollBackOff.backoffMillis.millis, self, IssuePollRequest(handle)) + () case Finish(SuccessfulExecutionHandle(outputs, returnCode, jobDetritusFiles, executionEvents, resultsClonedFrom)) => completionPromise.success(SucceededResponse(jobDescriptor.key, Some(returnCode), outputs, Option(jobDetritusFiles), executionEvents)) context.stop(self) diff --git a/backend/src/main/scala/cromwell/backend/backend.scala b/backend/src/main/scala/cromwell/backend/backend.scala new file mode 100644 index 000000000..8ac55a347 --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/backend.scala @@ -0,0 +1,53 @@ +package cromwell.backend + +import com.typesafe.config.Config +import cromwell.core.WorkflowOptions.WorkflowOption +import cromwell.core.{JobKey, WorkflowId, WorkflowOptions} +import wdl4s.values.WdlValue +import wdl4s.{Call, NamespaceWithWorkflow, _} + +import scala.util.Try + +/** + * For uniquely identifying a job which has been or will be sent to the backend. + */ +case class BackendJobDescriptorKey(call: Call, index: Option[Int], attempt: Int) extends JobKey { + def scope = call + private val indexString = index map { _.toString } getOrElse "NA" + val tag = s"${call.fullyQualifiedName}:$indexString:$attempt" + val isShard = index.isDefined + def mkTag(workflowId: WorkflowId) = s"$workflowId:$this" +} + +/** + * For passing to a BackendWorkflowActor for job execution or recovery + */ +case class BackendJobDescriptor(workflowDescriptor: BackendWorkflowDescriptor, + key: BackendJobDescriptorKey, + runtimeAttributes: Map[LocallyQualifiedName, WdlValue], + inputs: Map[LocallyQualifiedName, WdlValue]) { + val call = key.call + override val toString = s"${key.mkTag(workflowDescriptor.id)}" +} + +/** + * For passing to a BackendActor construction time + */ +case class BackendWorkflowDescriptor(id: WorkflowId, + workflowNamespace: NamespaceWithWorkflow, + inputs: Map[FullyQualifiedName, WdlValue], + workflowOptions: WorkflowOptions) { + override def toString: String = s"[BackendWorkflowDescriptor id=${id.shortString} workflowName=${workflowNamespace.workflow.unqualifiedName}]" + def getWorkflowOption(key: WorkflowOption) = workflowOptions.get(key).toOption +} + +/** + * For passing to a BackendActor construction time + */ +case class BackendConfigurationDescriptor(backendConfig: Config, globalConfig: Config) + +final case class AttemptedLookupResult(name: String, value: Try[WdlValue]) { + def toPair = name -> value +} + +case class PreemptedException(msg: String) extends Exception(msg) diff --git a/backend/src/main/scala/cromwell/backend/package.scala b/backend/src/main/scala/cromwell/backend/package.scala index e5bb64474..3bad6f61f 100644 --- a/backend/src/main/scala/cromwell/backend/package.scala +++ b/backend/src/main/scala/cromwell/backend/package.scala @@ -1,66 +1,14 @@ package cromwell -import com.typesafe.config.Config -import cromwell.core.WorkflowOptions.WorkflowOption -import cromwell.core.{JobKey, WorkflowId, WorkflowOptions} -import cromwell.util.JsonFormatting.WdlValueJsonFormatter -import wdl4s._ -import wdl4s.expression.WdlStandardLibraryFunctions -import wdl4s.util.TryUtil import wdl4s.values.WdlValue import scala.language.postfixOps -import scala.util.{Success, Try} +import scala.util.Success package object backend { - - /** - * For uniquely identifying a job which has been or will be sent to the backend. - */ - case class BackendJobDescriptorKey(call: Call, index: Option[Int], attempt: Int) extends JobKey { - def scope = call - private val indexString = index map { _.toString } getOrElse "NA" - val tag = s"${call.fullyQualifiedName}:$indexString:$attempt" - val isShard = index.isDefined - def mkTag(workflowId: WorkflowId) = s"$workflowId:$this" - } - - /** - * For passing to a BackendWorkflowActor for job execution or recovery - */ - case class BackendJobDescriptor(workflowDescriptor: BackendWorkflowDescriptor, - key: BackendJobDescriptorKey, - runtimeAttributes: Map[LocallyQualifiedName, WdlValue], - inputs: Map[LocallyQualifiedName, WdlValue]) { - val call = key.call - override val toString = s"${key.mkTag(workflowDescriptor.id)}" - } - - /** - * For passing to a BackendActor construction time - */ - case class BackendWorkflowDescriptor(id: WorkflowId, - workflowNamespace: NamespaceWithWorkflow, - inputs: Map[FullyQualifiedName, WdlValue], - workflowOptions: WorkflowOptions) { - override def toString: String = s"[BackendWorkflowDescriptor id=${id.shortString} workflowName=${workflowNamespace.workflow.unqualifiedName}]" - def getWorkflowOption(key: WorkflowOption) = workflowOptions.get(key).toOption - } - - /** - * For passing to a BackendActor construction time - */ - case class BackendConfigurationDescriptor(backendConfig: Config, globalConfig: Config) - - final case class AttemptedLookupResult(name: String, value: Try[WdlValue]) { - def toPair = name -> value - } - implicit class AugmentedAttemptedLookupSequence(s: Seq[AttemptedLookupResult]) { def toLookupMap: Map[String, WdlValue] = s collect { case AttemptedLookupResult(name, Success(value)) => (name, value) } toMap } - - case class PreemptedException(msg: String) extends Exception(msg) } diff --git a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala index 81e8f3ea7..17ba9fb66 100644 --- a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala @@ -53,7 +53,7 @@ object MemoryValidation { if (value <= 0) wrongAmountFormat.format(value).invalidNel else - MemorySize(value, MemoryUnit.Bytes).to(MemoryUnit.GB).validNel + MemorySize(value.toDouble, MemoryUnit.Bytes).to(MemoryUnit.GB).validNel } } diff --git a/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala b/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala index fd2aae6ea..1f06e10d8 100644 --- a/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala +++ b/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala @@ -6,7 +6,6 @@ import wdl4s.parser.MemoryUnit import wdl4s.types.{WdlArrayType, WdlFileType, WdlObjectType, WdlStringType} import wdl4s.values._ -import scala.language.postfixOps import scala.util.{Failure, Success, Try} trait ReadLikeFunctions extends FileSystems { this: WdlStandardLibraryFunctions => diff --git a/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala b/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala index 6905fa0f1..34497e2ef 100644 --- a/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala +++ b/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala @@ -10,11 +10,11 @@ object TestWorkflows { expectedResponse: BackendJobExecutionResponse) val HelloWorld = - """ + s""" |task hello { | String addressee = "you " | command { - | echo "Hello ${addressee}!" + | echo "Hello $${addressee}!" | } | output { | String salutation = read_string(stdout()) @@ -45,14 +45,14 @@ object TestWorkflows { """.stripMargin val InputFiles = - """ + s""" |task localize { | File inputFileFromJson | File inputFileFromCallInputs | command { - | cat ${inputFileFromJson} + | cat $${inputFileFromJson} | echo "" - | cat ${inputFileFromCallInputs} + | cat $${inputFileFromCallInputs} | } | output { | Array[String] out = read_lines(stdout()) @@ -82,11 +82,11 @@ object TestWorkflows { """.stripMargin val Scatter = - """ + s""" |task scattering { | Int intNumber | command { - | echo ${intNumber} + | echo $${intNumber} | } | output { | Int out = read_string(stdout()) diff --git a/core/src/main/scala/cromwell/core/ConfigUtil.scala b/core/src/main/scala/cromwell/core/ConfigUtil.scala index 9f22c330e..881fec686 100644 --- a/core/src/main/scala/cromwell/core/ConfigUtil.scala +++ b/core/src/main/scala/cromwell/core/ConfigUtil.scala @@ -43,7 +43,7 @@ object ConfigUtil { def validateConfig(key: String): ValidatedNel[String, Config] = try { config.getConfig(key).validNel } catch { - case e: ConfigException.Missing => "Could not find key: $key".invalidNel + case e: ConfigException.Missing => s"Could not find key: $key".invalidNel case e: ConfigException.WrongType => s"key $key cannot be parsed to a Config".invalidNel } diff --git a/core/src/main/scala/cromwell/core/ExecutionStore.scala b/core/src/main/scala/cromwell/core/ExecutionStore.scala index 3512e68ef..1632061ce 100644 --- a/core/src/main/scala/cromwell/core/ExecutionStore.scala +++ b/core/src/main/scala/cromwell/core/ExecutionStore.scala @@ -2,7 +2,6 @@ package cromwell.core import cromwell.core.ExecutionStatus._ -import scala.language.postfixOps object ExecutionStore { def empty = ExecutionStore(Map.empty) diff --git a/core/src/main/scala/cromwell/core/PathCopier.scala b/core/src/main/scala/cromwell/core/PathCopier.scala index f7ec545ce..c941ea890 100644 --- a/core/src/main/scala/cromwell/core/PathCopier.scala +++ b/core/src/main/scala/cromwell/core/PathCopier.scala @@ -24,5 +24,6 @@ object PathCopier { def copy(sourceFilePath: Path, destinationFilePath: Path): Unit = { Option(File(destinationFilePath).parent).foreach(_.createDirectories()) File(sourceFilePath).copyTo(destinationFilePath, overwrite = true) + () } } diff --git a/core/src/main/scala/cromwell/core/PathFactory.scala b/core/src/main/scala/cromwell/core/PathFactory.scala index 499578107..f85a05c95 100644 --- a/core/src/main/scala/cromwell/core/PathFactory.scala +++ b/core/src/main/scala/cromwell/core/PathFactory.scala @@ -80,7 +80,7 @@ trait PathWriter { * * @param string Line to add to the logs. */ - def writeWithNewline(string: String) { + def writeWithNewline(string: String): Unit = { writer.write(string) writer.write("\n") } @@ -108,7 +108,7 @@ case class TailedWriter(path: Path, tailedSize: Int) extends PathWriter { * * @param string Line to add to the logs. */ - override def writeWithNewline(string: String) { + override def writeWithNewline(string: String): Unit = { tailedLines :+= string while (tailedLines.size > tailedSize) { tailedLines = tailedLines.takeRight(tailedSize) diff --git a/core/src/main/scala/cromwell/core/core.scala b/core/src/main/scala/cromwell/core/core.scala new file mode 100644 index 000000000..ec1a3babe --- /dev/null +++ b/core/src/main/scala/cromwell/core/core.scala @@ -0,0 +1,12 @@ +package cromwell.core + +import java.nio.file.Path + +import lenthall.exception.ThrowableAggregation +import wdl4s.values.WdlValue + + +case class CallContext(root: Path, stdout: String, stderr: String) +case class JobOutput(wdlValue: WdlValue) +class CromwellFatalException(exception: Throwable) extends Exception(exception) +case class CromwellAggregatedException(throwables: Seq[Throwable], exceptionContext: String = "") extends ThrowableAggregation diff --git a/core/src/main/scala/cromwell/core/logging/LoggerWrapper.scala b/core/src/main/scala/cromwell/core/logging/LoggerWrapper.scala index ee04700e5..196762aa1 100644 --- a/core/src/main/scala/cromwell/core/logging/LoggerWrapper.scala +++ b/core/src/main/scala/cromwell/core/logging/LoggerWrapper.scala @@ -89,7 +89,7 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { slf4jLoggers.foreach(_.error(formatted, t)) } - override def error(pattern: String, arguments: AnyRef*) { + override def error(pattern: String, arguments: AnyRef*): Unit = { lazy val formatted: String = format(pattern) varargsAkkaLog(pattern, arguments) @@ -110,7 +110,7 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { slf4jLoggers.foreach(_.error(formatted, arg1, arg2: Any)) } - def error(t: Throwable, pattern: String, arguments: Any*) { + def error(t: Throwable, pattern: String, arguments: Any*): Unit = { lazy val formatted: String = format(pattern) akkaLogger.foreach(_.error(t, formatted, arguments)) @@ -131,7 +131,7 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { slf4jLoggers.foreach(_.debug(formatted, t)) } - override def debug(pattern: String, arguments: AnyRef*) { + override def debug(pattern: String, arguments: AnyRef*): Unit = { lazy val formatted: String = format(pattern) varargsAkkaLog(pattern, arguments) @@ -160,7 +160,7 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { slf4jLoggers.foreach(_.trace(format(msg), t)) } - override def trace(pattern: String, arguments: AnyRef*) { + override def trace(pattern: String, arguments: AnyRef*): Unit = { slf4jLoggers.foreach(_.trace(format(pattern), arguments:_*)) } @@ -186,7 +186,7 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { slf4jLoggers.foreach(_.info(formatted, t)) } - override def info(pattern: String, arguments: AnyRef*) { + override def info(pattern: String, arguments: AnyRef*): Unit = { lazy val formatted: String = format(pattern) varargsAkkaLog(pattern, arguments) diff --git a/core/src/main/scala/cromwell/core/package.scala b/core/src/main/scala/cromwell/core/package.scala index ee1448640..3334bfa24 100644 --- a/core/src/main/scala/cromwell/core/package.scala +++ b/core/src/main/scala/cromwell/core/package.scala @@ -1,22 +1,13 @@ package cromwell -import java.nio.file.Path - -import lenthall.exception.ThrowableAggregation import wdl4s.values.WdlValue package object core { - case class CallContext(root: Path, stdout: String, stderr: String) - type LocallyQualifiedName = String type FullyQualifiedName = String type WorkflowOutputs = Map[FullyQualifiedName, JobOutput] type WorkflowOptionsJson = String - case class JobOutput(wdlValue: WdlValue) type JobOutputs = Map[LocallyQualifiedName, JobOutput] type HostInputs = Map[String, WdlValue] type EvaluatedRuntimeAttributes = Map[String, WdlValue] - - class CromwellFatalException(exception: Throwable) extends Exception(exception) - case class CromwellAggregatedException(throwables: Seq[Throwable], exceptionContext: String = "") extends ThrowableAggregation } diff --git a/core/src/test/scala/cromwell/core/retry/RetrySpec.scala b/core/src/test/scala/cromwell/core/retry/RetrySpec.scala index 3e5c03886..27f24076c 100644 --- a/core/src/test/scala/cromwell/core/retry/RetrySpec.scala +++ b/core/src/test/scala/cromwell/core/retry/RetrySpec.scala @@ -7,7 +7,6 @@ import org.scalatest.time.{Millis, Seconds, Span} import org.scalatest.{FlatSpecLike, Matchers} import scala.concurrent.Future -import scala.concurrent.duration._ class RetrySpec extends TestKitSuite("retry-spec") with FlatSpecLike with Matchers with ScalaFutures { class TransientException extends Exception @@ -33,9 +32,6 @@ class RetrySpec extends TestKitSuite("retry-spec") with FlatSpecLike with Matche work: MockWork, isTransient: Throwable => Boolean = Retry.throwableToFalse, isFatal: Throwable => Boolean = Retry.throwableToFalse): Future[Int] = { - implicit val ec = system.dispatcher - - val backoff = SimpleExponentialBackoff(1.millis, 2.millis, 1) withRetry( f = work.doIt, diff --git a/core/src/test/scala/cromwell/util/SampleWdl.scala b/core/src/test/scala/cromwell/util/SampleWdl.scala index cdc3fde5f..dc2598cf1 100644 --- a/core/src/test/scala/cromwell/util/SampleWdl.scala +++ b/core/src/test/scala/cromwell/util/SampleWdl.scala @@ -24,6 +24,7 @@ trait SampleWdl extends TestFileUtil { createFile("f1", base, "line1\nline2\n") createFile("f2", base, "line3\nline4\n") createFile("f3", base, "line5\n") + () } def cleanupFileArray(base: Path) = { @@ -61,11 +62,11 @@ object SampleWdl { object HelloWorld extends SampleWdl { override def wdlSource(runtime: String = "") = - """ + s""" |task hello { | String addressee | command { - | echo "Hello ${addressee}!" + | echo "Hello $${addressee}!" | } | output { | String salutation = read_string(stdout()) @@ -86,11 +87,11 @@ object SampleWdl { object HelloWorldWithoutWorkflow extends SampleWdl { override def wdlSource(runtime: String = "") = - """ + s""" |task hello { | String addressee | command { - | echo "Hello ${addressee}!" + | echo "Hello $${addressee}!" | } | output { | String salutation = read_string(stdout()) @@ -127,7 +128,7 @@ object SampleWdl { object EmptyString extends SampleWdl { override def wdlSource(runtime: String = "") = - """ + s""" |task hello { | command { | echo "Hello!" @@ -141,7 +142,7 @@ object SampleWdl { |task goodbye { | String emptyInputString | command { - | echo "${emptyInputString}" + | echo "$${emptyInputString}" | } | output { | String empty = read_string(stdout()) @@ -174,11 +175,11 @@ object SampleWdl { object CoercionNotDefined extends SampleWdl { override def wdlSource(runtime: String = "") = { - """ + s""" |task summary { | String bfile | command { - | ~/plink --bfile ${bfile} --missing --hardy --out foo --allow-no-sex + | ~/plink --bfile $${bfile} --missing --hardy --out foo --allow-no-sex | } | output { | File hwe = "foo.hwe" @@ -208,7 +209,7 @@ object SampleWdl { private val outputSectionPlaceholder = "OUTPUTSECTIONPLACEHOLDER" def sourceString(outputsSection: String = "") = { val withPlaceholders = - """ + s""" |task ps { | command { | ps @@ -224,7 +225,7 @@ object SampleWdl { | File in_file | | command { - | grep '${pattern}' ${in_file} | wc -l + | grep '$${pattern}' $${in_file} | wc -l | } | output { | Int count = read_int(stdout()) @@ -235,7 +236,7 @@ object SampleWdl { |task wc { | File in_file | command { - | cat ${in_file} | wc -l + | cat $${in_file} | wc -l | } | output { | Int count = read_int(stdout()) @@ -385,12 +386,12 @@ object SampleWdl { object DeclarationsWorkflow extends SampleWdl { override def wdlSource(runtime: String): WdlSource = - """ + s""" |task cat { | File file | String? flags | command { - | cat ${flags} ${file} + | cat $${flags} $${file} | } | output { | File procs = stdout() @@ -402,7 +403,7 @@ object SampleWdl { | String pattern | File in_file | command { - | grep '${pattern}' ${in_file} | wc -l + | grep '$${pattern}' $${in_file} | wc -l | } | output { | Int count = read_int(stdout()) @@ -439,11 +440,11 @@ object SampleWdl { trait ZeroOrMorePostfixQuantifier extends SampleWdl { override def wdlSource(runtime: String): WdlSource = - """ + s""" |task hello { | Array[String] person | command { - | echo "hello ${sep = "," person}" + | echo "hello $${sep = "," person}" | } | output { | String greeting = read_string(stdout()) @@ -470,11 +471,11 @@ object SampleWdl { trait OneOrMorePostfixQuantifier extends SampleWdl { override def wdlSource(runtime: String): WdlSource = - """ + s""" |task hello { | Array[String]+ person | command { - | echo "hello ${sep = "," person}" + | echo "hello $${sep = "," person}" | } | output { | String greeting = read_string(stdout()) @@ -518,11 +519,11 @@ object SampleWdl { object ArrayIO extends SampleWdl { override def wdlSource(runtime: String = "") = - """task concat_files { + s"""task concat_files { | String? flags | Array[File]+ files | command { - | cat ${default = "-s" flags} ${sep = " " files} + | cat $${default = "-s" flags} $${sep = " " files} | } | output { | File concatenated = stdout() @@ -534,7 +535,7 @@ object SampleWdl { | String pattern | File root | command { - | find ${root} ${"-name " + pattern} + | find $${root} $${"-name " + pattern} | } | output { | Array[String] results = read_lines(stdout()) @@ -545,7 +546,7 @@ object SampleWdl { |task count_lines { | Array[File]+ files | command { - | cat ${sep = ' ' files} | wc -l + | cat $${sep = ' ' files} | wc -l | } | output { | Int count = read_int(stdout()) @@ -556,7 +557,7 @@ object SampleWdl { |task serialize { | Array[String] strs | command { - | cat ${write_lines(strs)} + | cat $${write_lines(strs)} | } | output { | String contents = read_string(stdout()) @@ -599,11 +600,11 @@ object SampleWdl { def cleanup() = cleanupFileArray(catRootDir) override def wdlSource(runtime: String = "") = - """ + s""" |task cat { | Array[File]+ files | command { - | cat -s ${sep = ' ' files} + | cat -s $${sep = ' ' files} | } | output { | Array[String] lines = read_lines(stdout()) @@ -624,11 +625,11 @@ object SampleWdl { def cleanup() = cleanupFileArray(catRootDir) override def wdlSource(runtime: String = "") = - """ + s""" |task write_map { | Map[File, String] file_to_name | command { - | cat ${write_map(file_to_name)} + | cat $${write_map(file_to_name)} | } | output { | String contents = read_string(stdout()) @@ -639,7 +640,7 @@ object SampleWdl { | command <<< | python <>> | output { @@ -658,7 +659,7 @@ object SampleWdl { } class ScatterWdl extends SampleWdl { - val tasks = """task A { + val tasks = s"""task A { | command { | echo -n -e "jeff\nchris\nmiguel\nthibault\nkhalid\nscott" | } @@ -671,7 +672,7 @@ object SampleWdl { |task B { | String B_in | command { - | python -c "print(len('${B_in}'))" + | python -c "print(len('$${B_in}'))" | } | RUNTIME | output { @@ -682,7 +683,7 @@ object SampleWdl { |task C { | Int C_in | command { - | python -c "print(${C_in}*100)" + | python -c "print($${C_in}*100)" | } | RUNTIME | output { @@ -693,7 +694,7 @@ object SampleWdl { |task D { | Array[Int] D_in | command { - | python -c "print(${sep = '+' D_in})" + | python -c "print($${sep = '+' D_in})" | } | RUNTIME | output { @@ -752,9 +753,9 @@ object SampleWdl { object SimpleScatterWdl extends SampleWdl { override def wdlSource(runtime: String = "") = - """task echo_int { + s"""task echo_int { | Int int - | command {echo ${int}} + | command {echo $${int}} | output {Int out = read_int(stdout())} | RUNTIME_PLACEHOLDER |} @@ -775,9 +776,9 @@ object SampleWdl { object SimpleScatterWdlWithOutputs extends SampleWdl { override def wdlSource(runtime: String = "") = - """task echo_int { + s"""task echo_int { | Int int - | command {echo ${int}} + | command {echo $${int}} | output {Int out = read_int(stdout())} |} | @@ -800,7 +801,7 @@ object SampleWdl { case class PrepareScatterGatherWdl(salt: String = UUID.randomUUID().toString) extends SampleWdl { override def wdlSource(runtime: String = "") = { - """ + s""" |# |# Goal here is to split up the input file into files of 1 line each (in the prepare) then in parallel call wc -w on each newly created file and count the words into another file then in the gather, sum the results of each parallel call to come up with |# the word-count for the fil @@ -809,7 +810,7 @@ object SampleWdl { |task do_prepare { | File input_file | command { - | split -l 1 ${input_file} temp_ && ls -1 temp_?? > files.list + | split -l 1 $${input_file} temp_ && ls -1 temp_?? > files.list | } | output { | Array[File] split_files = read_lines("files.list") @@ -821,8 +822,8 @@ object SampleWdl { | String salt | File input_file | command { - | # ${salt} - | wc -w ${input_file} > output.txt + | # $${salt} + | wc -w $${input_file} > output.txt | } | output { | File count_file = "output.txt" @@ -833,7 +834,7 @@ object SampleWdl { |task do_gather { | Array[File] input_files | command <<< - | cat ${sep = ' ' input_files} | awk '{s+=$1} END {print s}' + | cat $${sep = ' ' input_files} | awk '{s+=$$1} END {print s}' | >>> | output { | Int sum = read_int(stdout()) @@ -867,9 +868,9 @@ object SampleWdl { object FileClobber extends SampleWdl { override def wdlSource(runtime: String = "") = - """task read_line { + s"""task read_line { | File in - | command { cat ${in} } + | command { cat $${in} } | output { String out = read_string(stdout()) } |} | @@ -892,18 +893,18 @@ object SampleWdl { object FilePassingWorkflow extends SampleWdl { override def wdlSource(runtime: String): WdlSource = - """task a { + s"""task a { | File in | String out_name = "out" | | command { - | cat ${in} > ${out_name} + | cat $${in} > $${out_name} | } | RUNTIME | output { | File out = "out" - | File out_interpolation = "${out_name}" - | String contents = read_string("${out_name}") + | File out_interpolation = "$${out_name}" + | String contents = read_string("$${out_name}") | } |} | @@ -932,21 +933,21 @@ object SampleWdl { */ case class CallCachingWorkflow(salt: String) extends SampleWdl { override def wdlSource(runtime: String): WdlSource = - """task a { + s"""task a { | File in | String out_name = "out" | String salt | | command { - | # ${salt} + | # $${salt} | echo "Something" - | cat ${in} > ${out_name} + | cat $${in} > $${out_name} | } | RUNTIME | output { | File out = "out" - | File out_interpolation = "${out_name}" - | String contents = read_string("${out_name}") + | File out_interpolation = "$${out_name}" + | String contents = read_string("$${out_name}") | Array[String] stdoutContent = read_lines(stdout()) | } |} @@ -984,13 +985,13 @@ object SampleWdl { """.stripMargin.trim override def wdlSource(runtime: String): WdlSource = - """ + s""" |task a { | Array[String] array | Map[String, String] map | | command { - | echo ${sep = ' ' array} > concat + | echo $${sep = ' ' array} > concat | } | output { | String x = read_string("concat") @@ -1020,10 +1021,10 @@ object SampleWdl { object ArrayOfArrays extends SampleWdl { override def wdlSource(runtime: String = "") = - """task subtask { + s"""task subtask { | Array[File] a | command { - | cat ${sep = " " a} + | cat $${sep = " " a} | } | output { | String concatenated = read_string(stdout()) @@ -1060,17 +1061,17 @@ object SampleWdl { object CallCachingHashingWdl extends SampleWdl { override def wdlSource(runtime: String): WdlSource = - """task t { + s"""task t { | Int a | Float b | String c | File d | | command { - | echo "${a}" > a - | echo "${b}" > b - | echo "${c}" > c - | cat ${d} > d + | echo "$${a}" > a + | echo "$${b}" > b + | echo "$${c}" > c + | cat $${d} > d | } | output { | Int w = read_int("a") + 2 @@ -1098,10 +1099,10 @@ object SampleWdl { object ExpressionsInInputs extends SampleWdl { override def wdlSource(runtime: String = "") = - """task echo { + s"""task echo { | String inString | command { - | echo ${inString} + | echo $${inString} | } | | output { @@ -1128,11 +1129,11 @@ object SampleWdl { object WorkflowFailSlow extends SampleWdl { override def wdlSource(runtime: String = "") = - """ + s""" task shouldCompleteFast { | Int a | command { - | echo "The number was: ${a}" + | echo "The number was: $${a}" | } | output { | Int echo = a @@ -1142,7 +1143,7 @@ task shouldCompleteFast { |task shouldCompleteSlow { | Int a | command { - | echo "The number was: ${a}" + | echo "The number was: $${a}" | # More than 1 so this should finish second | sleep 2 | } @@ -1154,7 +1155,7 @@ task shouldCompleteFast { |task failMeSlowly { | Int a | command { - | echo "The number was: ${a}" + | echo "The number was: $${a}" | # Less than 2 so this should finish first | sleep 1 | ./NOOOOOO @@ -1168,7 +1169,7 @@ task shouldCompleteFast { | Int a | Int b | command { - | echo "You can't fight in here - this is the war room ${a + b}" + | echo "You can't fight in here - this is the war room $${a + b}" | } | output { | Int echo = a diff --git a/core/src/test/scala/cromwell/util/TryWithResourceSpec.scala b/core/src/test/scala/cromwell/util/TryWithResourceSpec.scala index 8aa73d339..77046fc27 100644 --- a/core/src/test/scala/cromwell/util/TryWithResourceSpec.scala +++ b/core/src/test/scala/cromwell/util/TryWithResourceSpec.scala @@ -9,7 +9,7 @@ class TryWithResourceSpec extends FlatSpec with Matchers { behavior of "tryWithResource" it should "catch instantiation errors" in { - val triedMyBest = tryWithResource(() => throw InstantiationException) { _ => 5 } + val triedMyBest = tryWithResource(() => if (1 == 1) throw InstantiationException else null) { _ => 5 } triedMyBest should be(Failure(InstantiationException)) } diff --git a/database/migration/src/main/scala/cromwell/database/migration/WdlTransformation.scala b/database/migration/src/main/scala/cromwell/database/migration/WdlTransformation.scala index 22766d9c7..84f38a7db 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/WdlTransformation.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/WdlTransformation.scala @@ -8,7 +8,6 @@ import org.apache.commons.codec.binary.Base64 import org.apache.commons.io.IOUtils import wdl4s.types.{WdlPrimitiveType, WdlType} -import scala.language.postfixOps import scala.util.Try private [migration] object WdlTransformation { diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/QueryPaginator.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/QueryPaginator.scala index e0d90d3d4..f7929cea4 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/QueryPaginator.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/QueryPaginator.scala @@ -2,7 +2,6 @@ package cromwell.database.migration.metadata.table.symbol import java.sql.{PreparedStatement, ResultSet} -import liquibase.database.jvm.JdbcConnection class QueryPaginator(statement: PreparedStatement, batchSize: Int, @@ -17,5 +16,5 @@ class QueryPaginator(statement: PreparedStatement, statement.executeQuery() } - def hasNext(): Boolean = cursor <= count + def hasNext: Boolean = cursor <= count } diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/SymbolTableMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/SymbolTableMigration.scala index 04a7bbaac..a400866a9 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/SymbolTableMigration.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/SymbolTableMigration.scala @@ -15,7 +15,6 @@ import wdl4s.WdlExpression import wdl4s.types.WdlType import wdl4s.values.WdlValue -import scala.language.postfixOps import scala.util.{Failure, Success, Try} object SymbolTableMigration { diff --git a/database/migration/src/main/scala/cromwell/database/migration/restart/table/JobStoreSimpletonMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/restart/table/JobStoreSimpletonMigration.scala index c48e76943..9287add50 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/restart/table/JobStoreSimpletonMigration.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/restart/table/JobStoreSimpletonMigration.scala @@ -5,8 +5,6 @@ import cromwell.database.migration.WdlTransformation._ import liquibase.database.jvm.JdbcConnection import wdl4s.types.WdlType -import scala.language.postfixOps - class JobStoreSimpletonMigration extends AbstractRestartMigration { override val description = "WORKFLOW_EXECUTION + EXECUTION + SYMBOL + JOB_STORE -> JOB_STORE_RESULT_SIMPLETON" diff --git a/database/sql/src/main/scala/cromwell/database/slick/JobKeyValueSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/JobKeyValueSlickDatabase.scala index c9f40e436..41d9c2237 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/JobKeyValueSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/JobKeyValueSlickDatabase.scala @@ -19,12 +19,12 @@ trait JobKeyValueSlickDatabase extends JobKeyValueSqlDatabase { } else { for { updateCount <- dataAccess. - storeValuesForJobKeyAndStoreKey( + storeValuesForJobKeyAndStoreKey(( jobKeyValueEntry.workflowExecutionUuid, jobKeyValueEntry.callFullyQualifiedName, jobKeyValueEntry.jobIndex, jobKeyValueEntry.jobAttempt, - jobKeyValueEntry.storeKey). + jobKeyValueEntry.storeKey)). update(jobKeyValueEntry.storeValue) _ <- updateCount match { case 0 => dataAccess.jobKeyValueEntryIdsAutoInc += jobKeyValueEntry @@ -39,7 +39,7 @@ trait JobKeyValueSlickDatabase extends JobKeyValueSqlDatabase { jobRetryAttempt: Int, storeKey: String) (implicit ec: ExecutionContext): Future[Option[String]] = { val action = dataAccess. - storeValuesForJobKeyAndStoreKey(workflowExecutionUuid, callFqn, jobScatterIndex, jobRetryAttempt, storeKey). + storeValuesForJobKeyAndStoreKey((workflowExecutionUuid, callFqn, jobScatterIndex, jobRetryAttempt, storeKey)). result.headOption runTransaction(action) } diff --git a/database/sql/src/main/scala/cromwell/database/slick/JobStoreSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/JobStoreSlickDatabase.scala index 8794ca8bd..829ddd86a 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/JobStoreSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/JobStoreSlickDatabase.scala @@ -1,9 +1,12 @@ package cromwell.database.slick +import cats.instances.future._ +import cats.syntax.functor._ import cromwell.database.sql.JobStoreSqlDatabase import cromwell.database.sql.joins.JobStoreJoin import scala.concurrent.{ExecutionContext, Future} +import scala.language.postfixOps trait JobStoreSlickDatabase extends JobStoreSqlDatabase { this: SlickDatabase => @@ -21,7 +24,7 @@ trait JobStoreSlickDatabase extends JobStoreSqlDatabase { override def addJobStores(jobStoreJoins: Seq[JobStoreJoin]) (implicit ec: ExecutionContext): Future[Unit] = { val action = DBIO.sequence(jobStoreJoins map addJobStore) - runTransaction(action) map { _ => () } + runTransaction(action) void } override def queryJobStores(workflowExecutionUuid: String, callFqn: String, jobScatterIndex: Int, @@ -30,7 +33,7 @@ trait JobStoreSlickDatabase extends JobStoreSqlDatabase { val action = for { jobStoreEntryOption <- dataAccess. - jobStoreEntriesForJobKey(workflowExecutionUuid, callFqn, jobScatterIndex, jobScatterAttempt).result.headOption + jobStoreEntriesForJobKey((workflowExecutionUuid, callFqn, jobScatterIndex, jobScatterAttempt)).result.headOption jobStoreSimpletonEntries <- jobStoreEntryOption match { case Some(jobStoreEntry) => dataAccess.jobStoreSimpletonEntriesForJobStoreEntryId(jobStoreEntry.jobStoreEntryId.get).result diff --git a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala index 7a0725ffe..efc27b1f6 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala @@ -34,7 +34,7 @@ trait MetadataSlickDatabase extends MetadataSqlDatabase { metadataKey: String) (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = { val action = - dataAccess.metadataEntriesForWorkflowExecutionUuidAndMetadataKey(workflowExecutionUuid, metadataKey).result + dataAccess.metadataEntriesForWorkflowExecutionUuidAndMetadataKey((workflowExecutionUuid, metadataKey)).result runTransaction(action) } @@ -44,7 +44,7 @@ trait MetadataSlickDatabase extends MetadataSqlDatabase { jobAttempt: Int) (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = { val action = dataAccess. - metadataEntriesForJobKey(workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt).result + metadataEntriesForJobKey((workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt)).result runTransaction(action) } @@ -54,8 +54,8 @@ trait MetadataSlickDatabase extends MetadataSqlDatabase { jobIndex: Option[Int], jobAttempt: Int) (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = { - val action = dataAccess.metadataEntriesForJobKeyAndMetadataKey( - workflowUuid, metadataKey, callFullyQualifiedName, jobIndex, jobAttempt).result + val action = dataAccess.metadataEntriesForJobKeyAndMetadataKey(( + workflowUuid, metadataKey, callFullyQualifiedName, jobIndex, jobAttempt)).result runTransaction(action) } @@ -121,8 +121,8 @@ trait MetadataSlickDatabase extends MetadataSqlDatabase { previousMetadataEntryIdOption <- getSummaryStatusEntryMaximumId( "WORKFLOW_METADATA_SUMMARY_ENTRY", "METADATA_ENTRY") previousMetadataEntryId = previousMetadataEntryIdOption.getOrElse(0L) - metadataEntries <- dataAccess.metadataEntriesForIdGreaterThanOrEqual( - previousMetadataEntryId + 1L, metadataKey1, metadataKey2, metadataKey3, metadataKey4).result + metadataEntries <- dataAccess.metadataEntriesForIdGreaterThanOrEqual(( + previousMetadataEntryId + 1L, metadataKey1, metadataKey2, metadataKey3, metadataKey4)).result metadataByWorkflowUuid = metadataEntries.groupBy(_.workflowExecutionUuid) _ <- DBIO.sequence(metadataByWorkflowUuid map updateWorkflowMetadataSummaryEntry(buildUpdatedSummary)) maximumMetadataEntryId = previousOrMaximum(previousMetadataEntryId, metadataEntries.map(_.metadataEntryId.get)) diff --git a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala index 752248cb1..fbfbece8a 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala @@ -104,7 +104,7 @@ class SlickDatabase(override val originalDatabaseConfig: Config) extends SqlData protected[this] def assertUpdateCount(description: String, updates: Int, expected: Int): DBIO[Unit] = { if (updates == expected) { - DBIO.successful(Unit) + DBIO.successful(()) } else { DBIO.failed(new RuntimeException(s"$description expected update count $expected, got $updates")) } diff --git a/database/sql/src/main/scala/cromwell/database/slick/SummaryStatusSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SummaryStatusSlickDatabase.scala index 1bd388b66..6903a29e6 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/SummaryStatusSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/SummaryStatusSlickDatabase.scala @@ -12,7 +12,7 @@ trait SummaryStatusSlickDatabase { private[slick] def getSummaryStatusEntryMaximumId(summaryTableName: String, summarizedTableName: String) (implicit ec: ExecutionContext): DBIO[Option[Long]] = { dataAccess. - maximumIdForSummaryTableNameSummarizedTableName(summaryTableName, summarizedTableName). + maximumIdForSummaryTableNameSummarizedTableName((summaryTableName, summarizedTableName)). result.headOption } @@ -28,7 +28,7 @@ trait SummaryStatusSlickDatabase { } else { for { updateCount <- dataAccess. - maximumIdForSummaryTableNameSummarizedTableName(summaryTableName, summarizedTableName). + maximumIdForSummaryTableNameSummarizedTableName((summaryTableName, summarizedTableName)). update(maximumId) _ <- updateCount match { case 0 => diff --git a/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala index e514822c2..fa1a9a5fc 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala @@ -1,9 +1,12 @@ package cromwell.database.slick +import cats.instances.future._ +import cats.syntax.functor._ import cromwell.database.sql.WorkflowStoreSqlDatabase import cromwell.database.sql.tables.WorkflowStoreEntry import scala.concurrent.{ExecutionContext, Future} +import scala.language.postfixOps trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase { this: SlickDatabase => @@ -13,19 +16,19 @@ trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase { override def updateWorkflowState(queryWorkflowState: String, updateWorkflowState: String) (implicit ec: ExecutionContext): Future[Unit] = { val action = dataAccess.workflowStateForWorkflowState(queryWorkflowState).update(updateWorkflowState) - runTransaction(action) map { _ => () } + runTransaction(action) void } override def addWorkflowStoreEntries(workflowStoreEntries: Iterable[WorkflowStoreEntry]) (implicit ec: ExecutionContext): Future[Unit] = { val action = dataAccess.workflowStoreEntryIdsAutoInc ++= workflowStoreEntries - runTransaction(action) map { _ => () } + runTransaction(action) void } override def queryWorkflowStoreEntries(limit: Int, queryWorkflowState: String, updateWorkflowState: String) (implicit ec: ExecutionContext): Future[Seq[WorkflowStoreEntry]] = { val action = for { - workflowStoreEntries <- dataAccess.workflowStoreEntriesForWorkflowState(queryWorkflowState, limit).result + workflowStoreEntries <- dataAccess.workflowStoreEntriesForWorkflowState((queryWorkflowState, limit.toLong)).result _ <- DBIO.sequence(workflowStoreEntries map updateWorkflowStateForWorkflowExecutionUuid(updateWorkflowState)) } yield workflowStoreEntries runTransaction(action) diff --git a/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala index c90e76e3b..c6c29479b 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala @@ -30,7 +30,7 @@ object SqlDatabase { */ def withUniqueSchema(config: Config, urlKey: String): Config = { val urlValue = config.getString(urlKey) - if (urlValue.contains("${uniqueSchema}")) { + if (urlValue.contains(s"$${uniqueSchema}")) { // Config wasn't updating with a simple withValue/withFallback. // So instead, do a bit of extra work to insert the generated schema name in the url. val schema = UUID.randomUUID().toString diff --git a/engine/src/main/scala/cromwell/engine/backend/CromwellBackends.scala b/engine/src/main/scala/cromwell/engine/backend/CromwellBackends.scala index b9d86ee65..9b90e6277 100644 --- a/engine/src/main/scala/cromwell/engine/backend/CromwellBackends.scala +++ b/engine/src/main/scala/cromwell/engine/backend/CromwellBackends.scala @@ -2,7 +2,6 @@ package cromwell.engine.backend import cromwell.backend.BackendLifecycleActorFactory -import scala.language.postfixOps import scala.util.{Failure, Success, Try} /** diff --git a/engine/src/main/scala/cromwell/engine/engine.scala b/engine/src/main/scala/cromwell/engine/engine.scala new file mode 100644 index 000000000..7a65770b1 --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/engine.scala @@ -0,0 +1,25 @@ +package cromwell.engine + +import java.time.OffsetDateTime + +import wdl4s._ + +import scala.util.{Failure, Success, Try} + +final case class AbortFunction(function: () => Unit) +final case class AbortRegistrationFunction(register: AbortFunction => Unit) + +final case class FailureEventEntry(failure: String, timestamp: OffsetDateTime) +final case class CallAttempt(fqn: FullyQualifiedName, attempt: Int) + +object WorkflowFailureMode { + def tryParse(mode: String): Try[WorkflowFailureMode] = { + val modes = Seq(ContinueWhilePossible, NoNewCalls) + modes find { _.toString.equalsIgnoreCase(mode) } map { Success(_) } getOrElse Failure(new Exception(s"Invalid workflow failure mode: $mode")) + } +} +sealed trait WorkflowFailureMode { + def allowNewCallsAfterFailure: Boolean +} +case object ContinueWhilePossible extends WorkflowFailureMode { override val allowNewCallsAfterFailure = true } +case object NoNewCalls extends WorkflowFailureMode { override val allowNewCallsAfterFailure = false } \ No newline at end of file diff --git a/engine/src/main/scala/cromwell/engine/package.scala b/engine/src/main/scala/cromwell/engine/package.scala index 4ea1c964b..6b2df1cc7 100644 --- a/engine/src/main/scala/cromwell/engine/package.scala +++ b/engine/src/main/scala/cromwell/engine/package.scala @@ -1,22 +1,11 @@ package cromwell -import java.time.OffsetDateTime - import cromwell.core.JobOutput import wdl4s._ import wdl4s.values.WdlValue -import scala.language.implicitConversions -import scala.util.{Failure, Success, Try} - package object engine { - final case class AbortFunction(function: () => Unit) - final case class AbortRegistrationFunction(register: AbortFunction => Unit) - - final case class FailureEventEntry(failure: String, timestamp: OffsetDateTime) - final case class CallAttempt(fqn: FullyQualifiedName, attempt: Int) - implicit class EnhancedFullyQualifiedName(val fqn: FullyQualifiedName) extends AnyVal { def scopeAndVariableName: (String, String) = { val array = fqn.split("\\.(?=[^\\.]+$)") @@ -30,15 +19,4 @@ package object engine { } } - object WorkflowFailureMode { - def tryParse(mode: String): Try[WorkflowFailureMode] = { - val modes = Seq(ContinueWhilePossible, NoNewCalls) - modes find { _.toString.equalsIgnoreCase(mode) } map { Success(_) } getOrElse Failure(new Exception(s"Invalid workflow failure mode: $mode")) - } - } - sealed trait WorkflowFailureMode { - def allowNewCallsAfterFailure: Boolean - } - case object ContinueWhilePossible extends WorkflowFailureMode { override val allowNewCallsAfterFailure = true } - case object NoNewCalls extends WorkflowFailureMode { override val allowNewCallsAfterFailure = false } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala index d7605a2b9..8abb0874c 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala @@ -6,6 +6,8 @@ import java.util.UUID import akka.actor.FSM.{CurrentState, Transition} import akka.actor._ import better.files._ +import cats.instances.try_._ +import cats.syntax.functor._ import cromwell.core.retry.SimpleExponentialBackoff import cromwell.core.{ExecutionStore => _, _} import cromwell.engine.workflow.SingleWorkflowRunnerActor._ @@ -80,7 +82,9 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: } private def schedulePollRequest(): Unit = { + // -Ywarn-value-discard should stash Cancellable to cancel context.system.scheduler.scheduleOnce(backoff.backoffMillis.millis, self, IssuePollRequest) + () } private def requestStatus(): Unit = { @@ -148,7 +152,7 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: data.terminalState foreach { state => log.info(s"$Tag workflow finished with status '$state'.") } data.failures foreach { e => log.error(e, e.getMessage) } - val message = data.terminalState collect { case WorkflowSucceeded => () } getOrElse Status.Failure(data.failures.head) + val message: Any = data.terminalState collect { case WorkflowSucceeded => () } getOrElse Status.Failure(data.failures.head) data.replyTo foreach { _ ! message } stay() } @@ -192,6 +196,6 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: log.info(s"$Tag writing metadata to $path") path.createIfNotExists(asDirectory = false, createParents = true).write(metadata.prettyPrint) } - } + } void } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala index f137883fd..684c81cce 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala @@ -22,7 +22,6 @@ import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} import cromwell.webservice.EngineStatsActor -import scala.language.postfixOps import scala.util.Random object WorkflowActor { diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala index 9c797bb76..1fa7fdb0e 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala @@ -18,7 +18,6 @@ import lenthall.config.ScalaConfig.EnhancedScalaConfig import scala.concurrent.duration._ import scala.concurrent.{Await, Promise} -import scala.language.postfixOps object WorkflowManagerActor { val DefaultMaxWorkflowsToRun = 5000 @@ -106,13 +105,13 @@ class WorkflowManagerActor(config: Config, private var abortingWorkflowToReplyTo = Map.empty[WorkflowId, ActorRef] - override def preStart() { + override def preStart(): Unit = { addShutdownHook() // Starts the workflow polling cycle self ! RetrieveNewWorkflows } - private def addShutdownHook(): Unit = { + private def addShutdownHook() = { // Only abort jobs on SIGINT if the config explicitly sets backend.abortJobsOnTerminate = true. val abortJobsOnTerminate = config.getConfig("system").getBooleanOr("abort-jobs-on-terminate", default = false) @@ -121,7 +120,7 @@ class WorkflowManagerActor(config: Config, sys.addShutdownHook { logger.info(s"$tag: Received shutdown signal. Aborting all running workflows...") self ! AbortAllWorkflowsCommand - Await.ready(donePromise.future, Duration.Inf) + Await.result(donePromise.future, Duration.Inf) } } } @@ -242,6 +241,7 @@ class WorkflowManagerActor(config: Config, case _ -> Done => logger.info(s"$tag All workflows finished. Stopping self.") donePromise.trySuccess(()) + () case fromState -> toState => logger.debug(s"$tag transitioning from $fromState to $toState") } @@ -271,7 +271,7 @@ class WorkflowManagerActor(config: Config, WorkflowIdToActorRef(workflowId, wfActor) } - private def scheduleNextNewWorkflowPoll(): Unit = { + private def scheduleNextNewWorkflowPoll() = { context.system.scheduler.scheduleOnce(newWorkflowPollRate, self, RetrieveNewWorkflows)(context.dispatcher) } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala index ff3df1b20..aa25fdfb8 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala @@ -30,11 +30,13 @@ class CopyWorkflowLogsActor(serviceRegistryActor: ActorRef) with ActorLogging with PathFactory { - def copyAndClean(src: Path, dest: Path): Unit = { + def copyAndClean(src: Path, dest: Path) = { File(dest).parent.createDirectories() File(src).copyTo(dest, overwrite = true) - if (WorkflowLogger.isTemporary) File(src).delete() + if (WorkflowLogger.isTemporary) { + File(src).delete() + } } override def receive = { diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala index d79d97998..67e93faf2 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala @@ -3,7 +3,6 @@ package cromwell.engine.workflow.lifecycle import java.nio.file.FileSystem import akka.actor.{ActorRef, FSM, LoggingFSM, Props} -import cats.data.NonEmptyList import cats.data.Validated._ import cats.instances.list._ import cats.syntax.cartesian._ diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala index 7c8d3748c..9614696e2 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala @@ -10,7 +10,6 @@ import cromwell.engine.backend.CromwellBackends import cromwell.engine.workflow.lifecycle.WorkflowFinalizationActor._ import cromwell.engine.workflow.lifecycle.WorkflowLifecycleActor._ -import scala.language.postfixOps import scala.util.{Failure, Success, Try} object WorkflowFinalizationActor { diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala index fc18939f6..2fd5d75aa 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala @@ -11,7 +11,6 @@ import cromwell.engine.backend.CromwellBackends import cromwell.engine.workflow.lifecycle.WorkflowInitializationActor._ import cromwell.engine.workflow.lifecycle.WorkflowLifecycleActor._ -import scala.language.postfixOps import scala.util.{Failure, Success, Try} object WorkflowInitializationActor { diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala index 4ff275a35..64f3eec7a 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala @@ -298,15 +298,20 @@ class EngineJobExecutionActor(replyTo: ActorRef, callCacheReadActor, factory.runtimeAttributeDefinitions(initializationData), backendName, activity) context.actorOf(props, s"ejha_for_$jobDescriptor") + () } - def makeFetchCachedResultsActor(cacheHit: CacheHit, taskOutputs: Seq[TaskOutput]): Unit = context.actorOf(FetchCachedResultsActor.props(cacheHit, self, new CallCache(SingletonServicesStore.databaseInterface))) - def fetchCachedResults(data: ResponsePendingData, taskOutputs: Seq[TaskOutput], cacheHit: CacheHit) = { + def makeFetchCachedResultsActor(cacheHit: CacheHit, taskOutputs: Seq[TaskOutput]): Unit = { + context.actorOf(FetchCachedResultsActor.props(cacheHit, self, new CallCache(SingletonServicesStore.databaseInterface))) + () + } + + private def fetchCachedResults(data: ResponsePendingData, taskOutputs: Seq[TaskOutput], cacheHit: CacheHit) = { makeFetchCachedResultsActor(cacheHit, taskOutputs) goto(FetchingCachedOutputsFromDatabase) } - def makeBackendCopyCacheHit(cacheHit: CacheHit, wdlValueSimpletons: Seq[WdlValueSimpleton], jobDetritusFiles: Map[String,String], returnCode: Option[Int], data: ResponsePendingData) = { + private def makeBackendCopyCacheHit(cacheHit: CacheHit, wdlValueSimpletons: Seq[WdlValueSimpleton], jobDetritusFiles: Map[String,String], returnCode: Option[Int], data: ResponsePendingData) = { factory.cacheHitCopyingActorProps match { case Some(propsMaker) => val backendCacheHitCopyingActorProps = propsMaker(data.jobDescriptor, initializationData, serviceRegistryActor) @@ -323,7 +328,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, } } - def runJob(data: ResponsePendingData) = { + private def runJob(data: ResponsePendingData) = { val backendJobExecutionActor = context.actorOf(data.bjeaProps, buildJobExecutionActorName(data.jobDescriptor)) val message = if (restarting) RecoverJobCommand else ExecuteJobCommand backendJobExecutionActor ! message @@ -342,6 +347,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, protected def createSaveCacheResultsActor(hashes: CallCacheHashes, success: SucceededResponse): Unit = { val callCache = new CallCache(SingletonServicesStore.databaseInterface) context.actorOf(CallCacheWriteActor.props(callCache, workflowId, hashes, success), s"CallCacheWriteActor-$tag") + () } private def saveCacheResults(hashes: CallCacheHashes, data: SucceededResponseData) = { diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index 8e6c93e6c..a3ef978ec 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -145,7 +145,7 @@ object WorkflowExecutionActor { callCacheReadActor, jobTokenDispenserActor, initializationData, restarting)).withDispatcher(EngineDispatcher) } - private implicit class EnhancedExecutionStore(val executionStore: ExecutionStore) extends AnyVal { + implicit class EnhancedExecutionStore(val executionStore: ExecutionStore) extends AnyVal { // Convert the store to a `List` before `collect`ing to sidestep expensive and pointless hashing of `Scope`s when // assembling the result. def runnableScopes = executionStore.store.toList collect { case entry if isRunnable(entry) => entry._1 } @@ -208,7 +208,7 @@ object WorkflowExecutionActor { } } - private implicit class EnhancedOutputStore(val outputStore: OutputStore) extends AnyVal { + implicit class EnhancedOutputStore(val outputStore: OutputStore) extends AnyVal { /** * Try to generate output for a collector call, by collecting outputs for all of its shards. * It's fail-fast on shard output retrieval diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala index 985d9c812..599c8f1b4 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala @@ -8,7 +8,6 @@ import cromwell.engine.{EngineWorkflowDescriptor, WdlFunctions} import cromwell.util.JsonFormatting.WdlValueJsonFormatter import wdl4s.Scope -import scala.language.postfixOps object WorkflowExecutionDiff { def empty = WorkflowExecutionDiff(Map.empty) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala index d4c144d9a..16ecc89f0 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala @@ -12,7 +12,6 @@ import cromwell.database.sql.tables.{CallCachingDetritusEntry, CallCachingEntry, import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes import scala.concurrent.{ExecutionContext, Future} -import scala.language.postfixOps final case class MetaInfoId(id: Int) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala index 2080ae8e8..229fc47a2 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala @@ -15,10 +15,10 @@ import scala.concurrent.ExecutionContext */ class CallCacheReadActor(cache: CallCache) extends Actor with ActorLogging { - implicit val ec: ExecutionContext = context.dispatcher + private implicit val ec: ExecutionContext = context.dispatcher - var requestQueue: List[RequestTuple] = List.empty - var currentRequester: Option[ActorRef] = None + private var requestQueue: List[RequestTuple] = List.empty + private var currentRequester: Option[ActorRef] = None override def receive: Receive = { case CacheLookupRequest(callCacheHashes) => @@ -30,7 +30,7 @@ class CallCacheReadActor(cache: CallCache) extends Actor with ActorLogging { log.error("Unexpected message type to CallCacheReadActor: " + other.getClass.getSimpleName) } - private def runRequest(callCacheHashes: CallCacheHashes) = { + private def runRequest(callCacheHashes: CallCacheHashes): Unit = { val response = cache.fetchMetaInfoIdsMatchingHashes(callCacheHashes) map { CacheResultMatchesForHashes(callCacheHashes.hashes, _) } recover { @@ -38,6 +38,7 @@ class CallCacheReadActor(cache: CallCache) extends Actor with ActorLogging { } response.pipeTo(self) + () } private def cycleRequestQueue() = requestQueue match { @@ -49,7 +50,7 @@ class CallCacheReadActor(cache: CallCache) extends Actor with ActorLogging { currentRequester = None } - private def receiveNewRequest(callCacheHashes: CallCacheHashes) = currentRequester match { + private def receiveNewRequest(callCacheHashes: CallCacheHashes): Unit = currentRequester match { case Some(x) => requestQueue :+= RequestTuple(sender, callCacheHashes) case None => currentRequester = Option(sender) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/lifecycle.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/lifecycle.scala new file mode 100644 index 000000000..bb0d8fa80 --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/lifecycle.scala @@ -0,0 +1,7 @@ +package cromwell.engine.workflow.lifecycle + +case object EngineLifecycleActorAbortCommand + +trait EngineLifecycleStateCompleteResponse + +trait EngineLifecycleActorAbortedResponse extends EngineLifecycleStateCompleteResponse diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/package.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/package.scala deleted file mode 100644 index 1f240dc7b..000000000 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/package.scala +++ /dev/null @@ -1,8 +0,0 @@ -package cromwell.engine.workflow - -package object lifecycle { - case object EngineLifecycleActorAbortCommand - - trait EngineLifecycleStateCompleteResponse - trait EngineLifecycleActorAbortedResponse extends EngineLifecycleStateCompleteResponse -} diff --git a/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala b/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala index 4d5e460bb..b2afd6b5e 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala @@ -7,7 +7,6 @@ import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor._ import cromwell.engine.workflow.tokens.TokenPool.TokenPoolPop import scala.collection.immutable.Queue -import scala.language.postfixOps class JobExecutionTokenDispenserActor extends Actor with ActorLogging { @@ -61,7 +60,7 @@ class JobExecutionTokenDispenserActor extends Actor with ActorLogging { actor ! JobExecutionTokenDispensed(token) } - private def unassign(actor: ActorRef, token: JobExecutionToken) = { + private def unassign(actor: ActorRef, token: JobExecutionToken): Unit = { if (tokenAssignments.contains(actor) && tokenAssignments(actor) == token) { tokenAssignments -= actor @@ -76,12 +75,13 @@ class JobExecutionTokenDispenserActor extends Actor with ActorLogging { tokenPools += token.jobExecutionTokenType -> pool context.unwatch(actor) + () } else { log.error("Job execution token returned from incorrect actor: {}", token) } } - private def onTerminate(terminee: ActorRef) = { + private def onTerminate(terminee: ActorRef): Unit = { tokenAssignments.get(terminee) match { case Some(token) => log.error("Actor {} stopped without returning its Job Execution Token. Reclaiming it!", terminee) @@ -94,6 +94,7 @@ class JobExecutionTokenDispenserActor extends Actor with ActorLogging { } } context.unwatch(terminee) + () } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala index 6355b64c5..24cb3a6a7 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala @@ -14,7 +14,6 @@ import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} import org.apache.commons.lang3.exception.ExceptionUtils import scala.concurrent.{ExecutionContext, Future} -import scala.language.postfixOps import scala.util.{Failure, Success} case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor: ActorRef) diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/package.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/package.scala deleted file mode 100644 index e8f90bd74..000000000 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/package.scala +++ /dev/null @@ -1,18 +0,0 @@ -package cromwell.engine.workflow - -import cromwell.core.{WorkflowId, WorkflowSourceFiles} -import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState - -package object workflowstore { - - sealed trait WorkflowStoreState {def isStartable: Boolean} - - object WorkflowStoreState { - case object Running extends WorkflowStoreState { override def isStartable = false } - sealed trait StartableState extends WorkflowStoreState { override def isStartable = true } - case object Submitted extends StartableState - case object Restartable extends StartableState - } - - final case class WorkflowToStart(id: WorkflowId, sources: WorkflowSourceFiles, state: StartableState) -} diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/workflowstore_.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/workflowstore_.scala new file mode 100644 index 000000000..0d9481c47 --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/workflowstore_.scala @@ -0,0 +1,15 @@ +package cromwell.engine.workflow.workflowstore + +import cromwell.core.{WorkflowId, WorkflowSourceFiles} +import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState + +sealed trait WorkflowStoreState {def isStartable: Boolean} + +object WorkflowStoreState { + case object Running extends WorkflowStoreState { override def isStartable = false } + sealed trait StartableState extends WorkflowStoreState { override def isStartable = true } + case object Submitted extends StartableState + case object Restartable extends StartableState +} + +final case class WorkflowToStart(id: WorkflowId, sources: WorkflowSourceFiles, state: StartableState) diff --git a/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala b/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala index cf7b37acb..9bb680b0c 100644 --- a/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala +++ b/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala @@ -83,7 +83,7 @@ object JobStoreWriterData { case class JobStoreWriterData(currentOperation: List[(ActorRef, JobStoreWriterCommand)], nextOperation: List[(ActorRef, JobStoreWriterCommand)]) { def isEmpty = nextOperation.isEmpty && currentOperation.isEmpty - def withNewOperation(sender: ActorRef, command: JobStoreWriterCommand) = this.copy(nextOperation = this.nextOperation :+ (sender, command)) + def withNewOperation(sender: ActorRef, command: JobStoreWriterCommand) = this.copy(nextOperation = this.nextOperation :+ ((sender, command))) def rolledOver = JobStoreWriterData(this.nextOperation, List.empty) } diff --git a/engine/src/main/scala/cromwell/jobstore/jobstore_.scala b/engine/src/main/scala/cromwell/jobstore/jobstore_.scala new file mode 100644 index 000000000..7fbdd0107 --- /dev/null +++ b/engine/src/main/scala/cromwell/jobstore/jobstore_.scala @@ -0,0 +1,10 @@ +package cromwell.jobstore + +import cromwell.core.{WorkflowId, _} + +case class JobStoreKey(workflowId: WorkflowId, callFqn: String, index: Option[Int], attempt: Int) + +sealed trait JobResult +case class JobResultSuccess(returnCode: Option[Int], jobOutputs: JobOutputs) extends JobResult +case class JobResultFailure(returnCode: Option[Int], reason: Throwable, retryable: Boolean) extends JobResult + diff --git a/engine/src/main/scala/cromwell/jobstore/package.scala b/engine/src/main/scala/cromwell/jobstore/package.scala index 2e82109e0..f96ca4041 100644 --- a/engine/src/main/scala/cromwell/jobstore/package.scala +++ b/engine/src/main/scala/cromwell/jobstore/package.scala @@ -1,14 +1,8 @@ package cromwell -import cromwell.core.{JobKey, JobOutputs, WorkflowId} +import cromwell.core.{JobKey, WorkflowId} package object jobstore { - case class JobStoreKey(workflowId: WorkflowId, callFqn: String, index: Option[Int], attempt: Int) - - sealed trait JobResult - case class JobResultSuccess(returnCode: Option[Int], jobOutputs: JobOutputs) extends JobResult - case class JobResultFailure(returnCode: Option[Int], reason: Throwable, retryable: Boolean) extends JobResult - implicit class EnhancedJobKey(val jobKey: JobKey) extends AnyVal { def toJobStoreKey(workflowId: WorkflowId): JobStoreKey = JobStoreKey(workflowId, jobKey.scope.fullyQualifiedName, jobKey.index, jobKey.attempt) } diff --git a/engine/src/main/scala/cromwell/server/CromwellServer.scala b/engine/src/main/scala/cromwell/server/CromwellServer.scala index c3772f3f7..0dd8ac59f 100644 --- a/engine/src/main/scala/cromwell/server/CromwellServer.scala +++ b/engine/src/main/scala/cromwell/server/CromwellServer.scala @@ -4,20 +4,17 @@ import java.util.concurrent.TimeoutException import akka.actor.Props import akka.util.Timeout -import com.typesafe.config.{Config, ConfigFactory} -import cromwell.services.ServiceRegistryActor +import com.typesafe.config.Config +import cromwell.webservice.WorkflowJsonSupport._ import cromwell.webservice.{APIResponse, CromwellApiService, SwaggerService} +import lenthall.config.ScalaConfig._ import lenthall.spray.SprayCanHttpService._ -import spray.http.HttpHeaders.`Content-Type` -import spray.http.MediaTypes._ -import spray.http.{ContentType, MediaTypes, _} import lenthall.spray.WrappedRoute._ -import lenthall.config.ScalaConfig._ -import cromwell.webservice.WorkflowJsonSupport._ +import spray.http.{ContentType, MediaTypes, _} import spray.json._ -import scala.concurrent.{Await, Future} import scala.concurrent.duration._ +import scala.concurrent.{Await, Future} import scala.util.{Failure, Success} // Note that as per the language specification, this is instantiated lazily and only used when necessary (i.e. server mode) diff --git a/engine/src/main/scala/cromwell/server/CromwellSystem.scala b/engine/src/main/scala/cromwell/server/CromwellSystem.scala index d125b8c9c..648850f71 100644 --- a/engine/src/main/scala/cromwell/server/CromwellSystem.scala +++ b/engine/src/main/scala/cromwell/server/CromwellSystem.scala @@ -1,10 +1,12 @@ package cromwell.server -import akka.actor.ActorSystem +import akka.actor.{ActorSystem, Terminated} import com.typesafe.config.ConfigFactory import cromwell.engine.backend.{BackendConfiguration, CromwellBackends} import org.slf4j.LoggerFactory +import scala.concurrent.Future + trait CromwellSystem { protected def systemName = "cromwell-system" protected def newActorSystem(): ActorSystem = ActorSystem(systemName) @@ -12,7 +14,9 @@ trait CromwellSystem { val logger = LoggerFactory.getLogger(getClass.getName) implicit final lazy val actorSystem = newActorSystem() - def shutdownActorSystem(): Unit = actorSystem.terminate() + def shutdownActorSystem(): Future[Terminated] = { + actorSystem.terminate() + } CromwellBackends.initBackends(BackendConfiguration.AllBackendEntries) } diff --git a/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala b/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala index b39c2fc51..dee173ca3 100644 --- a/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala +++ b/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala @@ -4,7 +4,6 @@ import spray.json._ import wdl4s.values.WdlValue import wdl4s.{ExceptionWithErrors, FullyQualifiedName} -import scala.language.postfixOps case class WorkflowStatusResponse(id: String, status: String) diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala index 0aae04dac..a2441abb4 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala @@ -13,7 +13,6 @@ import cromwell.webservice.metadata.WorkflowQueryPagination import spray.http.{StatusCodes, Uri} import spray.httpx.SprayJsonSupport._ -import scala.language.postfixOps object CromwellApiHandler { def props(requestHandlerActor: ActorRef): Props = { @@ -43,34 +42,34 @@ class CromwellApiHandler(requestHandlerActor: ActorRef) extends Actor with Workf val conf = ConfigFactory.load() def callNotFound(callFqn: String, id: WorkflowId) = { - RequestComplete(StatusCodes.NotFound, APIResponse.error( - new RuntimeException(s"Call $callFqn not found for workflow '$id'."))) + RequestComplete((StatusCodes.NotFound, APIResponse.error( + new RuntimeException(s"Call $callFqn not found for workflow '$id'.")))) } private def error(t: Throwable)(f: Throwable => RequestComplete[_]): Unit = context.parent ! f(t) override def receive = { case ApiHandlerEngineStats => requestHandlerActor ! WorkflowManagerActor.EngineStatsCommand - case stats: EngineStatsActor.EngineStats => context.parent ! RequestComplete(StatusCodes.OK, stats) + case stats: EngineStatsActor.EngineStats => context.parent ! RequestComplete((StatusCodes.OK, stats)) case ApiHandlerWorkflowAbort(id, manager) => requestHandlerActor ! WorkflowStoreActor.AbortWorkflow(id, manager) case WorkflowStoreActor.WorkflowAborted(id) => - context.parent ! RequestComplete(StatusCodes.OK, WorkflowAbortResponse(id.toString, WorkflowAborted.toString)) + context.parent ! RequestComplete((StatusCodes.OK, WorkflowAbortResponse(id.toString, WorkflowAborted.toString))) case WorkflowStoreActor.WorkflowAbortFailed(_, e) => error(e) { - case _: IllegalStateException => RequestComplete(StatusCodes.Forbidden, APIResponse.error(e)) - case _: WorkflowNotFoundException => RequestComplete(StatusCodes.NotFound, APIResponse.error(e)) - case _ => RequestComplete(StatusCodes.InternalServerError, APIResponse.error(e)) + case _: IllegalStateException => RequestComplete((StatusCodes.Forbidden, APIResponse.error(e))) + case _: WorkflowNotFoundException => RequestComplete((StatusCodes.NotFound, APIResponse.error(e))) + case _ => RequestComplete((StatusCodes.InternalServerError, APIResponse.error(e))) } case ApiHandlerWorkflowSubmit(source) => requestHandlerActor ! WorkflowStoreActor.SubmitWorkflow(source) case WorkflowStoreActor.WorkflowSubmittedToStore(id) => - context.parent ! RequestComplete(StatusCodes.Created, WorkflowSubmitResponse(id.toString, WorkflowSubmitted.toString)) + context.parent ! RequestComplete((StatusCodes.Created, WorkflowSubmitResponse(id.toString, WorkflowSubmitted.toString))) case ApiHandlerWorkflowSubmitBatch(sources) => requestHandlerActor ! WorkflowStoreActor.BatchSubmitWorkflows(sources) case WorkflowStoreActor.WorkflowsBatchSubmittedToStore(ids) => val responses = ids map { id => WorkflowSubmitResponse(id.toString, WorkflowSubmitted.toString) } - context.parent ! RequestComplete(StatusCodes.OK, responses.toList) + context.parent ! RequestComplete((StatusCodes.OK, responses.toList)) } } diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala index 6f66407fd..54ddf2786 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala @@ -1,7 +1,7 @@ package cromwell.webservice import akka.actor._ -import java.lang.Throwable._ +import java.lang.Throwable import cats.data.NonEmptyList import cromwell.core.{WorkflowId, WorkflowSourceFiles} import cromwell.engine.backend.BackendConfiguration @@ -51,7 +51,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator { } private def failBadRequest(exception: Exception, statusCode: StatusCode = StatusCodes.BadRequest) = respondWithMediaType(`application/json`) { - complete(statusCode, APIResponse.fail(exception).toJson.prettyPrint) + complete((statusCode, APIResponse.fail(exception).toJson.prettyPrint)) } val workflowRoutes = queryRoute ~ queryPostRoute ~ workflowOutputsRoute ~ submitRoute ~ submitBatchRoute ~ @@ -155,6 +155,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator { case JsArray(_) => failBadRequest(new RuntimeException("Nothing was submitted")) case _ => reject } + () } } } diff --git a/engine/src/main/scala/cromwell/webservice/PerRequest.scala b/engine/src/main/scala/cromwell/webservice/PerRequest.scala index 0ae86447a..f4fb9e876 100644 --- a/engine/src/main/scala/cromwell/webservice/PerRequest.scala +++ b/engine/src/main/scala/cromwell/webservice/PerRequest.scala @@ -63,7 +63,7 @@ trait PerRequest extends Actor { OneForOneStrategy() { case e => system.log.error(e, "error processing request: " + r.request.uri) - r.complete(InternalServerError, e.getMessage) + r.complete((InternalServerError, e.getMessage)) Stop } } @@ -85,13 +85,13 @@ object PerRequest { case class RequestCompleteWithHeaders[T](response: T, headers: HttpHeader*)(implicit val marshaller: ToResponseMarshaller[T]) extends PerRequestMessage /** allows for pattern matching with extraction of marshaller */ - private object RequestComplete_ { - def unapply[T](requestComplete: RequestComplete[T]) = Some((requestComplete.response, requestComplete.marshaller)) + object RequestComplete_ { + def unapply[T](requestComplete: RequestComplete[T]) = Option((requestComplete.response, requestComplete.marshaller)) } /** allows for pattern matching with extraction of marshaller */ - private object RequestCompleteWithHeaders_ { - def unapply[T](requestComplete: RequestCompleteWithHeaders[T]) = Some((requestComplete.response, requestComplete.headers, requestComplete.marshaller)) + object RequestCompleteWithHeaders_ { + def unapply[T](requestComplete: RequestCompleteWithHeaders[T]) = Option((requestComplete.response, requestComplete.headers, requestComplete.marshaller)) } case class WithProps(r: RequestContext, props: Props, message: AnyRef, timeout: Duration, name: String) extends PerRequest { @@ -108,8 +108,9 @@ trait PerRequestCreator { def perRequest(r: RequestContext, props: Props, message: AnyRef, timeout: Duration = 1 minutes, - name: String = PerRequestCreator.endpointActorName) = { + name: String = PerRequestCreator.endpointActorName): Unit = { actorRefFactory.actorOf(Props(WithProps(r, props, message, timeout, name)).withDispatcher(ApiDispatcher), name) + () } } diff --git a/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala b/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala index c1969342c..d9ed77438 100644 --- a/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala +++ b/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala @@ -7,29 +7,29 @@ import cats.instances.map._ import spray.json._ -private object IndexedJsonValue { +object IndexedJsonValue { private implicit val dateTimeOrdering: Ordering[OffsetDateTime] = scala.Ordering.fromLessThan(_ isBefore _) private val timestampedJsValueOrdering: Ordering[TimestampedJsValue] = scala.Ordering.by(_.timestamp) - implicit val TimestampedJsonMonoid: Monoid[TimestampedJsValue] = new Monoid[TimestampedJsValue] { - def combine(f1: TimestampedJsValue, f2: TimestampedJsValue): TimestampedJsValue = { - (f1, f2) match { - case (o1: TimestampedJsObject, o2: TimestampedJsObject) => - val sg = implicitly[Semigroup[Map[String, TimestampedJsValue]]] - TimestampedJsObject(sg.combine(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) - case (o1: TimestampedJsList, o2: TimestampedJsList) => - val sg = implicitly[Semigroup[Map[Int, TimestampedJsValue]]] - TimestampedJsList(sg.combine(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) - case (o1, o2) => timestampedJsValueOrdering.max(o1, o2) - } - } - - override def empty: TimestampedJsValue = TimestampedJsObject(Map.empty, OffsetDateTime.now) - } + implicit val TimestampedJsonMonoid: Monoid[TimestampedJsValue] = new Monoid[TimestampedJsValue] { + def combine(f1: TimestampedJsValue, f2: TimestampedJsValue): TimestampedJsValue = { + (f1, f2) match { + case (o1: TimestampedJsObject, o2: TimestampedJsObject) => + val sg = implicitly[Semigroup[Map[String, TimestampedJsValue]]] + TimestampedJsObject(sg.combine(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) + case (o1: TimestampedJsList, o2: TimestampedJsList) => + val sg = implicitly[Semigroup[Map[Int, TimestampedJsValue]]] + TimestampedJsList(sg.combine(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) + case (o1, o2) => timestampedJsValueOrdering.max(o1, o2) + } + } + + override def empty: TimestampedJsValue = TimestampedJsObject(Map.empty, OffsetDateTime.now) + } } /** Customized version of Json data structure, to account for timestamped values and lazy array creation */ -private sealed trait TimestampedJsValue { +sealed trait TimestampedJsValue { def toJson: JsValue def timestamp: OffsetDateTime } diff --git a/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala b/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala index 6d007dddc..0653be425 100644 --- a/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala +++ b/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala @@ -215,36 +215,37 @@ class MetadataBuilderActor(serviceRegistryActor: ActorRef) extends LoggingFSM[Me when(WaitingForMetadataService) { case Event(MetadataLookupResponse(query, metadata), _) => - context.parent ! RequestComplete(StatusCodes.OK, processMetadataResponse(query, metadata)) + context.parent ! RequestComplete((StatusCodes.OK, processMetadataResponse(query, metadata))) allDone case Event(StatusLookupResponse(w, status), _) => - context.parent ! RequestComplete(StatusCodes.OK, processStatusResponse(w, status)) + context.parent ! RequestComplete((StatusCodes.OK, processStatusResponse(w, status))) allDone case Event(failure: ServiceRegistryFailure, _) => val response = APIResponse.fail(new RuntimeException("Can't find metadata service")) - context.parent ! RequestComplete(StatusCodes.InternalServerError, response) + context.parent ! RequestComplete((StatusCodes.InternalServerError, response)) allDone case Event(WorkflowQuerySuccess(uri: Uri, response, metadata), _) => + import WorkflowJsonSupport._ context.parent ! RequestCompleteWithHeaders(response, generateLinkHeaders(uri, metadata):_*) allDone case Event(failure: WorkflowQueryFailure, _) => - context.parent ! RequestComplete(StatusCodes.BadRequest, APIResponse.fail(failure.reason)) + context.parent ! RequestComplete((StatusCodes.BadRequest, APIResponse.fail(failure.reason))) allDone case Event(WorkflowOutputsResponse(id, events), _) => // Add in an empty output event if there aren't already any output events. val hasOutputs = events exists { _.key.key.startsWith(WorkflowMetadataKeys.Outputs + ":") } val updatedEvents = if (hasOutputs) events else MetadataEvent.empty(MetadataKey(id, None, WorkflowMetadataKeys.Outputs)) +: events - context.parent ! RequestComplete(StatusCodes.OK, workflowMetadataResponse(id, updatedEvents, includeCallsIfEmpty = false)) + context.parent ! RequestComplete((StatusCodes.OK, workflowMetadataResponse(id, updatedEvents, includeCallsIfEmpty = false))) allDone case Event(LogsResponse(w, l), _) => - context.parent ! RequestComplete(StatusCodes.OK, workflowMetadataResponse(w, l, includeCallsIfEmpty = false)) + context.parent ! RequestComplete((StatusCodes.OK, workflowMetadataResponse(w, l, includeCallsIfEmpty = false))) allDone case Event(failure: MetadataServiceFailure, _) => - context.parent ! RequestComplete(StatusCodes.InternalServerError, APIResponse.error(failure.reason)) + context.parent ! RequestComplete((StatusCodes.InternalServerError, APIResponse.error(failure.reason))) allDone case Event(unexpectedMessage, stateData) => val response = APIResponse.fail(new RuntimeException(s"MetadataBuilderActor $tag(WaitingForMetadataService, $stateData) got an unexpected message: $unexpectedMessage")) - context.parent ! RequestComplete(StatusCodes.InternalServerError, response) + context.parent ! RequestComplete((StatusCodes.InternalServerError, response)) context stop self stay() } diff --git a/engine/src/main/scala/cromwell/webservice/package.scala b/engine/src/main/scala/cromwell/webservice/package.scala index 1fd8ec1d4..5b22ab108 100644 --- a/engine/src/main/scala/cromwell/webservice/package.scala +++ b/engine/src/main/scala/cromwell/webservice/package.scala @@ -1,42 +1,5 @@ package cromwell package object webservice { - case class QueryParameter(key: String, value: String) type QueryParameters = Seq[QueryParameter] - - object Patterns { - val WorkflowName = """ - (?x) # Turn on comments and whitespace insensitivity. - - ( # Begin capture. - - [a-zA-Z][a-zA-Z0-9_]* # WDL identifier naming pattern of an initial alpha character followed by zero - # or more alphanumeric or underscore characters. - - ) # End capture. - """.trim.r - - val CallFullyQualifiedName = """ - (?x) # Turn on comments and whitespace insensitivity. - - ( # Begin outer capturing group for FQN. - - (?:[a-zA-Z][a-zA-Z0-9_]*) # Inner noncapturing group for top-level workflow name. This is the WDL - # identifier naming pattern of an initial alpha character followed by zero - # or more alphanumeric or underscore characters. - - (?:\.[a-zA-Z][a-zA-Z0-9_]*){1} # Inner noncapturing group for call name, a literal dot followed by a WDL - # identifier. Currently this is quantified to {1} since the call name is - # mandatory and nested workflows are not supported. This could be changed - # to + or a different quantifier if these assumptions change. - - ) # End outer capturing group for FQN. - - - (?: # Begin outer noncapturing group for shard. - \. # Literal dot. - (\d+) # Captured shard digits. - )? # End outer optional noncapturing group for shard. - """.trim.r // The trim is necessary as (?x) must be at the beginning of the regex. - } } diff --git a/engine/src/main/scala/cromwell/webservice/webservice_.scala b/engine/src/main/scala/cromwell/webservice/webservice_.scala new file mode 100644 index 000000000..d68ba0bdb --- /dev/null +++ b/engine/src/main/scala/cromwell/webservice/webservice_.scala @@ -0,0 +1,39 @@ +package cromwell.webservice + +case class QueryParameter(key: String, value: String) + +object Patterns { + val WorkflowName = """ + (?x) # Turn on comments and whitespace insensitivity. + + ( # Begin capture. + + [a-zA-Z][a-zA-Z0-9_]* # WDL identifier naming pattern of an initial alpha character followed by zero + # or more alphanumeric or underscore characters. + + ) # End capture. + """.trim.r + + val CallFullyQualifiedName = """ + (?x) # Turn on comments and whitespace insensitivity. + + ( # Begin outer capturing group for FQN. + + (?:[a-zA-Z][a-zA-Z0-9_]*) # Inner noncapturing group for top-level workflow name. This is the WDL + # identifier naming pattern of an initial alpha character followed by zero + # or more alphanumeric or underscore characters. + + (?:\.[a-zA-Z][a-zA-Z0-9_]*){1} # Inner noncapturing group for call name, a literal dot followed by a WDL + # identifier. Currently this is quantified to {1} since the call name is + # mandatory and nested workflows are not supported. This could be changed + # to + or a different quantifier if these assumptions change. + + ) # End outer capturing group for FQN. + + + (?: # Begin outer noncapturing group for shard. + \. # Literal dot. + (\d+) # Captured shard digits. + )? # End outer optional noncapturing group for shard. + """.trim.r // The trim is necessary as (?x) must be at the beginning of the regex. +} diff --git a/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala b/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala index 06f437de6..00a530f31 100644 --- a/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala +++ b/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala @@ -5,7 +5,6 @@ import wdl4s.types.{WdlArrayType, WdlStringType} import wdl4s.values.{WdlArray, WdlString} import cromwell.util.SampleWdl -import scala.language.postfixOps class ArrayOfArrayCoercionSpec extends CromwellTestkitSpec { "A workflow that has an Array[Array[File]] input " should { diff --git a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala index dada6157f..843796c1a 100644 --- a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala @@ -1,7 +1,6 @@ package cromwell import java.nio.file.Files -import java.util.UUID import akka.testkit._ import better.files._ @@ -11,7 +10,6 @@ import wdl4s.expression.NoFunctions import wdl4s.types.{WdlArrayType, WdlFileType, WdlStringType} import wdl4s.values.{WdlArray, WdlFile, WdlInteger, WdlString} -import scala.language.postfixOps class ArrayWorkflowSpec extends CromwellTestkitSpec { val tmpDir = Files.createTempDirectory("ArrayWorkflowSpec") @@ -65,7 +63,6 @@ class ArrayWorkflowSpec extends CromwellTestkitSpec { ) ) ) - val uuid = UUID.randomUUID() val pwd = File(".") val sampleWdl = SampleWdl.ArrayLiteral(pwd.path) runWdlAndAssertOutputs( diff --git a/engine/src/test/scala/cromwell/CallCachingWorkflowSpec.scala b/engine/src/test/scala/cromwell/CallCachingWorkflowSpec.scala index 5718ad337..cea47fe8d 100644 --- a/engine/src/test/scala/cromwell/CallCachingWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/CallCachingWorkflowSpec.scala @@ -5,15 +5,11 @@ import java.util.UUID import akka.testkit._ import com.typesafe.config.ConfigFactory import cromwell.CallCachingWorkflowSpec._ -import cromwell.core.Tags.DockerTest -import cromwell.core.Tags._ -import cromwell.engine.workflow.WorkflowManagerActor -import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreActor} +import cromwell.core.Tags.{DockerTest, _} import cromwell.util.SampleWdl import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType} import wdl4s.values.{WdlArray, WdlFile, WdlInteger, WdlString} -import scala.language.postfixOps class CallCachingWorkflowSpec extends CromwellTestkitSpec { def cacheHitMessageForCall(name: String) = s"Call Caching: Cache hit. Using UUID\\(.{8}\\):$name\\.*" @@ -146,11 +142,11 @@ class CallCachingWorkflowSpec extends CromwellTestkitSpec { FIXME: This test had been constructing a custom WorkflowManagerActor. I don't believe this is still necessary but this test is being ignored so I'm not sure */ - val workflowId = runWdlAndAssertOutputs( - sampleWdl = SampleWdl.CallCachingWorkflow(UUID.randomUUID().toString), - eventFilter = EventFilter.info(pattern = cacheHitMessageForCall("a"), occurrences = 1), - expectedOutputs = expectedOutputs, - config = CallCachingWorkflowSpec.callCachingConfig) +// val workflowId = runWdlAndAssertOutputs( +// sampleWdl = SampleWdl.CallCachingWorkflow(UUID.randomUUID().toString), +// eventFilter = EventFilter.info(pattern = cacheHitMessageForCall("a"), occurrences = 1), +// expectedOutputs = expectedOutputs, +// config = CallCachingWorkflowSpec.callCachingConfig) // val status = messageAndWait[WorkflowManagerStatusSuccess](WorkflowStatus(workflowId)).state // status shouldEqual WorkflowSucceeded diff --git a/engine/src/test/scala/cromwell/CromwellSpec.scala b/engine/src/test/scala/cromwell/CromwellSpec.scala index f1ff598d7..da1720f63 100644 --- a/engine/src/test/scala/cromwell/CromwellSpec.scala +++ b/engine/src/test/scala/cromwell/CromwellSpec.scala @@ -1,7 +1,6 @@ package cromwell import com.typesafe.config.ConfigFactory -import org.scalatest.Tag object CromwellSpec { val BackendConfText = diff --git a/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala b/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala index c6f6b3045..c4aec5879 100644 --- a/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala +++ b/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala @@ -3,7 +3,7 @@ package cromwell import java.nio.file.Paths import java.util.UUID -import akka.actor.{Actor, ActorRef, ActorSystem, Props} +import akka.actor.{Actor, ActorRef, ActorSystem, Props, Terminated} import akka.pattern.ask import akka.testkit._ import com.typesafe.config.{Config, ConfigFactory} @@ -36,7 +36,7 @@ import wdl4s.types._ import wdl4s.values._ import scala.concurrent.duration._ -import scala.concurrent.{Await, ExecutionContext} +import scala.concurrent.{Await, ExecutionContext, Future} import scala.language.postfixOps import scala.util.matching.Regex @@ -127,7 +127,8 @@ object CromwellTestkitSpec { * Do NOT shut down the test actor system inside the normal flow. * The actor system will be externally shutdown outside the block. */ - override def shutdownActorSystem() = {} + // -Ywarn-value-discard + override def shutdownActorSystem(): Future[Terminated] = { Future.successful(null) } def shutdownTestActorSystem() = super.shutdownActorSystem() } @@ -276,7 +277,7 @@ object CromwellTestkitSpec { abstract class CromwellTestkitSpec(val twms: TestWorkflowManagerSystem = new CromwellTestkitSpec.TestWorkflowManagerSystem()) extends TestKit(twms.actorSystem) with DefaultTimeout with ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll with ScalaFutures with OneInstancePerTest with Eventually { - override protected def afterAll() = { twms.shutdownTestActorSystem() } + override protected def afterAll() = { twms.shutdownTestActorSystem(); () } implicit val defaultPatience = PatienceConfig(timeout = Span(30, Seconds), interval = Span(100, Millis)) implicit val ec = system.dispatcher @@ -407,6 +408,7 @@ abstract class CromwellTestkitSpec(val twms: TestWorkflowManagerSystem = new Cro } getWorkflowState(workflowId, serviceRegistryActor) should equal (expectedState) + () } private def getWorkflowOutputsFromMetadata(id: WorkflowId, serviceRegistryActor: ActorRef): Map[FullyQualifiedName, WdlValue] = { diff --git a/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala b/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala index 9671706ef..d829b84eb 100644 --- a/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala @@ -5,7 +5,6 @@ import wdl4s.{NamespaceWithWorkflow, WorkflowInput} import cromwell.util.SampleWdl import org.scalatest.{Matchers, WordSpecLike} -import scala.language.postfixOps class DeclarationWorkflowSpec extends Matchers with WordSpecLike { "A workflow with declarations in it" should { diff --git a/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala b/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala index 3ddffcba7..1aaafaf2f 100644 --- a/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala @@ -1,12 +1,10 @@ package cromwell import akka.testkit._ -import wdl4s.values.{WdlFile, WdlString} import cromwell.util.SampleWdl +import wdl4s.values.{WdlFile, WdlString} -import scala.concurrent.ExecutionContext import scala.concurrent.duration._ -import scala.language.postfixOps class FilePassingWorkflowSpec extends CromwellTestkitSpec { "A workflow that passes files between tasks" should { diff --git a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala index 1176a4f99..9a00c115e 100644 --- a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala @@ -8,7 +8,6 @@ import wdl4s.expression.{NoFunctions, WdlFunctions} import wdl4s.types.{WdlFileType, WdlIntegerType, WdlMapType, WdlStringType} import wdl4s.values._ -import scala.language.postfixOps import scala.util.{Success, Try} class MapWorkflowSpec extends CromwellTestkitSpec { diff --git a/engine/src/test/scala/cromwell/MetadataWatchActor.scala b/engine/src/test/scala/cromwell/MetadataWatchActor.scala index cc334b05f..691c4efc5 100644 --- a/engine/src/test/scala/cromwell/MetadataWatchActor.scala +++ b/engine/src/test/scala/cromwell/MetadataWatchActor.scala @@ -19,6 +19,7 @@ final case class MetadataWatchActor(promise: Promise[Unit], matchers: Matcher*) unsatisfiedMatchers = unsatisfiedMatchers.filterNot { m => m.matches(events) } if (unsatisfiedMatchers.isEmpty) { promise.trySuccess(()) + () } case PutMetadataAction(_) => // Superfluous message. Ignore case _ => throw new Exception("Invalid message to MetadataWatchActor") diff --git a/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala b/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala index ca7e2720b..f0b7a70af 100644 --- a/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala @@ -1,10 +1,9 @@ package cromwell import akka.testkit._ -import wdl4s.values.{WdlString, WdlFile} import cromwell.util.SampleWdl +import wdl4s.values.WdlString -import scala.language.postfixOps class MultipleFilesWithSameNameWorkflowSpec extends CromwellTestkitSpec { "A workflow with two file inputs that have the same name" should { diff --git a/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala b/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala index af3d68c29..919008315 100644 --- a/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala @@ -5,17 +5,16 @@ import wdl4s.WdlNamespace import wdl4s.expression.NoFunctions import wdl4s.values.{WdlFile, WdlString} -import scala.language.postfixOps class OptionalParamWorkflowSpec extends Matchers with WordSpecLike { "A workflow with an optional parameter that has a prefix inside the tag" should { "not include that prefix if no value is specified" in { - val wf = """ + val wf = s""" |task find { | String? pattern | File root | command { - | find ${root} ${"-name " + pattern} + | find $${root} $${"-name " + pattern} | } |} | diff --git a/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala b/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala index 8daf9be69..8530dd6d2 100644 --- a/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala @@ -4,7 +4,6 @@ import akka.testkit._ import wdl4s.values.WdlString import cromwell.util.SampleWdl -import scala.language.postfixOps class PostfixQuantifierWorkflowSpec extends CromwellTestkitSpec { "A task which contains a parameter with a zero-or-more postfix quantifier" should { diff --git a/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala b/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala index b999e3623..6b706fbc6 100644 --- a/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala @@ -23,8 +23,8 @@ class RestartWorkflowSpec extends CromwellTestkitSpec with WorkflowDescriptorBui "RestartWorkflowSpec" should { "restart a call in Running state" taggedAs PostMVP ignore { - val id = WorkflowId.randomId() - val descriptor = createMaterializedEngineWorkflowDescriptor(id, sources) +// val id = WorkflowId.randomId() +// val descriptor = createMaterializedEngineWorkflowDescriptor(id, sources) // val a = ExecutionDatabaseKey("w.a", Option(-1), 1) // val b = ExecutionDatabaseKey("w.b", Option(-1), 1) // diff --git a/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala b/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala index 035004669..0d8847a27 100644 --- a/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala @@ -6,8 +6,6 @@ import wdl4s.types.{WdlArrayType, WdlFileType, WdlIntegerType, WdlStringType} import wdl4s.values.{WdlArray, WdlFile, WdlInteger, WdlString} import cromwell.util.SampleWdl -import scala.language.postfixOps - class ScatterWorkflowSpec extends CromwellTestkitSpec { "A workflow with a stand-alone scatter block in it" should { "run properly" in { diff --git a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala index f2dd4db13..34ec73895 100644 --- a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala @@ -17,7 +17,6 @@ import org.scalatest.BeforeAndAfter import scala.concurrent.duration._ import scala.concurrent.{Await, Promise} -import scala.language.postfixOps object SimpleWorkflowActorSpec { diff --git a/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala b/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala index 4a13be95d..72c618fca 100644 --- a/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala +++ b/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala @@ -1,12 +1,10 @@ package cromwell import akka.testkit._ -import wdl4s.types.{WdlMapType, WdlStringType, WdlArrayType} -import wdl4s.values.{WdlMap, WdlArray, WdlString} -import cromwell.core.Tags.DockerTest import cromwell.util.SampleWdl +import wdl4s.types.{WdlMapType, WdlStringType} +import wdl4s.values.{WdlMap, WdlString} -import scala.language.postfixOps class WdlFunctionsAtWorkflowLevelSpec extends CromwellTestkitSpec { val outputMap = WdlMap(WdlMapType(WdlStringType, WdlStringType), Map( diff --git a/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala b/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala index bbdd67a6e..18df31795 100644 --- a/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala +++ b/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala @@ -4,7 +4,6 @@ import akka.testkit._ import cromwell.util.SampleWdl import cromwell.CromwellTestkitSpec.AnyValueIsFine -import scala.language.postfixOps class WorkflowOutputsSpec extends CromwellTestkitSpec { "Workflow outputs" should { diff --git a/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala b/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala index 9ecff4ba5..55faea29f 100644 --- a/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala @@ -4,7 +4,6 @@ import cromwell.CromwellTestkitSpec import cromwell.engine.workflow.WorkflowDescriptorBuilder import cromwell.util.SampleWdl -import scala.language.postfixOps class WorkflowManagerActorSpec extends CromwellTestkitSpec with WorkflowDescriptorBuilder { override implicit val actorSystem = system @@ -15,7 +14,7 @@ class WorkflowManagerActorSpec extends CromwellTestkitSpec with WorkflowDescript val outputs = runWdl(sampleWdl = SampleWdl.CurrentDirectory) val outputName = "whereami.whereami.pwd" - val salutation = outputs.get(outputName).get + val salutation = outputs(outputName) val actualOutput = salutation.valueString.trim actualOutput should endWith("/call-whereami/execution") } diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala index 5c0a48253..73e8bfa7b 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala @@ -22,7 +22,6 @@ import spray.json._ import scala.concurrent.Await import scala.concurrent.duration.Duration -import scala.language.postfixOps import scala.util._ /** @@ -79,6 +78,7 @@ abstract class SingleWorkflowRunnerActorSpec extends CromwellTestkitSpec { val actorRef = createRunnerActor(sampleWdl, managerActor, outputFile) val futureResult = actorRef ? RunWorkflow Await.ready(futureResult, Duration.Inf) + () } } @@ -103,7 +103,7 @@ class SingleWorkflowRunnerActorWithMetadataSpec extends SingleWorkflowRunnerActo super.afterAll() } - private def doTheTest(wdlFile: SampleWdl, expectedCalls: TableFor3[String, Int, Int], workflowInputs: Int, workflowOutputs: Int) = { + private def doTheTest(wdlFile: SampleWdl, expectedCalls: TableFor3[String, Long, Long], workflowInputs: Long, workflowOutputs: Long) = { val testStart = OffsetDateTime.now within(TimeoutDuration) { singleWorkflowActor( @@ -153,18 +153,18 @@ class SingleWorkflowRunnerActorWithMetadataSpec extends SingleWorkflowRunnerActo "successfully run a workflow outputting metadata" in { val expectedCalls = Table( ("callName", "numInputs", "numOutputs"), - ("three_step.wc", 1, 1), - ("three_step.ps", 0, 1), - ("three_step.cgrep", 2, 1)) + ("three_step.wc", 1L, 1L), + ("three_step.ps", 0L, 1L), + ("three_step.cgrep", 2L, 1L)) - doTheTest(ThreeStep, expectedCalls, 1, 3) + doTheTest(ThreeStep, expectedCalls, 1L, 3L) } "run a workflow outputting metadata with no remaining input expressions" in { val expectedCalls = Table( ("callName", "numInputs", "numOutputs"), - ("wf.echo", 1, 1), - ("wf.echo2", 1, 1)) - doTheTest(ExpressionsInInputs, expectedCalls, 2, 2) + ("wf.echo", 1L, 1L), + ("wf.echo2", 1L, 1L)) + doTheTest(ExpressionsInInputs, expectedCalls, 2L, 2L) } } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala index 6a0245528..860ac6dab 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala @@ -66,7 +66,6 @@ class WorkflowActorSpec extends CromwellTestkitSpec with WorkflowDescriptorBuild "WorkflowActor" should { "run Finalization actor if Initialization fails" in { - val workflowId = WorkflowId.randomId() val actor = createWorkflowActor(InitializingWorkflowState) deathwatch watch actor actor ! WorkflowInitializationFailedResponse(Seq(new Exception("Materialization Failed"))) diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/CachingConfigSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/CachingConfigSpec.scala index 4015f01de..499e6f0c2 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/CachingConfigSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/CachingConfigSpec.scala @@ -4,7 +4,7 @@ import cats.data.Validated.{Invalid, Valid} import com.typesafe.config.{Config, ConfigFactory} import cromwell.core.WorkflowOptions import cromwell.core.callcaching.CallCachingMode -import org.scalatest.{FlatSpec, Matchers} +import org.scalatest.{Assertion, FlatSpec, Matchers} import scala.collection.JavaConverters._ import scala.util.{Success, Try} @@ -51,13 +51,13 @@ class CachingConfigSpec extends FlatSpec with Matchers { val writeCacheOffCombinations = allCombinations -- writeCacheOnCombinations val readCacheOffCombinations = allCombinations -- readCacheOnCombinations - validateCallCachingMode("write cache on options", writeCacheOnCombinations) { mode => mode.writeToCache should be(true) } - validateCallCachingMode("read cache on options", readCacheOnCombinations) { mode => mode.readFromCache should be(true) } - validateCallCachingMode("write cache off options", writeCacheOffCombinations) { mode => mode.writeToCache should be(false) } - validateCallCachingMode("read cache off options", readCacheOffCombinations) { mode => mode.readFromCache should be(false) } + validateCallCachingMode("write cache on options", writeCacheOnCombinations) { _.writeToCache should be(true) } + validateCallCachingMode("read cache on options", readCacheOnCombinations) { _.readFromCache should be(true) } + validateCallCachingMode("write cache off options", writeCacheOffCombinations) { _.writeToCache should be(false) } + validateCallCachingMode("read cache off options", readCacheOffCombinations) { _.readFromCache should be(false) } - private def validateCallCachingMode(testName: String, combinations: Set[(Config, Try[WorkflowOptions])])(verificationFunction: CallCachingMode => Unit) = { + private def validateCallCachingMode(testName: String, combinations: Set[(Config, Try[WorkflowOptions])])(verificationFunction: CallCachingMode => Assertion) = { it should s"correctly identify $testName" in { combinations foreach { case (config, Success(wfOptions)) => diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala index 9414647bd..970bc7156 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala @@ -15,7 +15,6 @@ import spray.json._ import wdl4s.values.{WdlInteger, WdlString} import scala.concurrent.duration._ -import scala.language.postfixOps class MaterializeWorkflowDescriptorActorSpec extends CromwellTestkitSpec with BeforeAndAfter with MockitoSugar { @@ -65,9 +64,9 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestkitSpec with Be wfDesc.id shouldBe workflowId wfDesc.name shouldBe "hello" wfDesc.namespace.tasks.size shouldBe 1 - wfDesc.workflowInputs.head shouldBe ("hello.hello.addressee", WdlString("world")) - wfDesc.backendDescriptor.inputs.head shouldBe ("hello.hello.addressee", WdlString("world")) - wfDesc.getWorkflowOption(WorkflowOptions.WriteToCache) shouldBe Some("true") + wfDesc.workflowInputs.head shouldBe (("hello.hello.addressee", WdlString("world"))) + wfDesc.backendDescriptor.inputs.head shouldBe (("hello.hello.addressee", WdlString("world"))) + wfDesc.getWorkflowOption(WorkflowOptions.WriteToCache) shouldBe Option("true") wfDesc.getWorkflowOption(WorkflowOptions.ReadFromCache) shouldBe None // Default backend assignment is "Local": wfDesc.backendAssignments foreach { diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala index 498180780..d9de4432b 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala @@ -46,5 +46,6 @@ class EjeaCheckingCallCacheSpec extends EngineJobExecutionActorSpec with Eventua private def createCheckingCallCacheEjea(restarting: Boolean = false): Unit = { ejea = helper.buildEJEA(restarting = restarting, callCachingMode = CallCachingActivity(ReadCache)) ejea.setStateInline(state = CheckingCallCache, data = ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, None)) + () } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala index e5a8ff36e..a8f2bdf46 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala @@ -1,15 +1,12 @@ package cromwell.engine.workflow.lifecycle.execution.ejea -import akka.testkit.TestProbe -import cromwell.backend.BackendJobDescriptorKey -import cromwell.backend.BackendJobExecutionActor.{FailedNonRetryableResponse, FailedRetryableResponse, RecoverJobCommand, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{FailedNonRetryableResponse, FailedRetryableResponse, SucceededResponse} import cromwell.core._ -import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{CheckingJobStore, JobRunning, NoData, PreparingJob} -import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor.BackendJobPreparationFailed -import cromwell.jobstore.{JobResultFailure, JobResultSuccess} -import cromwell.jobstore.JobStoreActor.{JobComplete, JobNotComplete} -import EngineJobExecutionActorSpec.EnhancedTestEJEA +import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{CheckingJobStore, NoData, PreparingJob} import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor +import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec.EnhancedTestEJEA +import cromwell.jobstore.JobStoreActor.{JobComplete, JobNotComplete} +import cromwell.jobstore.{JobResultFailure, JobResultSuccess} class EjeaCheckingJobStoreSpec extends EngineJobExecutionActorSpec { diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala index 87f2c5131..e8eaeec1a 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala @@ -29,8 +29,8 @@ trait EngineJobExecutionActorSpec extends CromwellTestkitSpec // The default values for these are "null". The helper is created in "before", the ejea is up to the test cases - var helper: PerTestHelper = _ - var ejea: TestFSMRef[EngineJobExecutionActorState, EJEAData, MockEjea] = _ + private[ejea] var helper: PerTestHelper = _ + private[ejea] var ejea: TestFSMRef[EngineJobExecutionActorState, EJEAData, MockEjea] = _ implicit def stateUnderTest: EngineJobExecutionActorState before { diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala index 5a2e7bff1..60cc183a9 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala @@ -31,7 +31,7 @@ private[ejea] trait CanExpectCacheWrites extends Eventually { self: EngineJobExe creation._2 should be(expectedResponse) case _ => fail("Expected exactly one cache write actor creation.") } - + () } } @@ -46,6 +46,7 @@ private[ejea] trait CanExpectJobStoreWrites extends CanValidateJobStoreKey { sel ejea.stateName should be(UpdatingJobStore) ejea.stateData should be(expectedData) } + () } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/ExpectOne.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/ExpectOne.scala index bf8049098..7ddfd340f 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/ExpectOne.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/ExpectOne.scala @@ -1,10 +1,10 @@ package cromwell.engine.workflow.lifecycle.execution.ejea private[ejea] sealed trait ExpectOne[+A] { - def checkIt(block: A => Unit): Unit = throw new IllegalStateException("An ExpectOne must have exactly one element for checkIt to work") + def checkIt(block: A => Any): Unit = throw new IllegalStateException("An ExpectOne must have exactly one element for checkIt to work") def hasExactlyOne: Boolean def foundOne[B >: A](theFoundOne: B) = this match { - case NothingYet => new GotOne(theFoundOne) + case NothingYet => GotOne(theFoundOne) case GotOne(theOriginalOne) => GotTooMany(List(theOriginalOne, theFoundOne)) case GotTooMany(theOnes) => GotTooMany(theOnes :+ theFoundOne) } @@ -15,7 +15,7 @@ private[ejea] case object NothingYet extends ExpectOne[scala.Nothing] { } private[ejea] case class GotOne[+A](theOne: A) extends ExpectOne[A] { - override def checkIt(block: A => Unit): Unit = block(theOne) + override def checkIt(block: A => Any): Unit = { block(theOne); () } override def hasExactlyOne = true } diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala index 543fc9473..b71c6a48f 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala @@ -302,7 +302,7 @@ object JobExecutionTokenDispenserActorSpec { implicit class intWithTimes(n: Int) { def times(f: => Unit) = 1 to n foreach { _ => f } - def indexedTimes(f: Int => Unit) = 0 until n foreach { i => f(i) } + def indexedTimes(f: Int => Any) = 0 until n foreach { i => f(i) } } val TestInfiniteTokenType = JobExecutionTokenType("infinite", maxPoolSize = None) diff --git a/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala b/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala index e1853e7e1..fa1cc5067 100644 --- a/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala +++ b/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala @@ -1,6 +1,5 @@ package cromwell.jobstore -import com.typesafe.config.ConfigFactory import cromwell.CromwellTestkitSpec import cromwell.backend.BackendJobDescriptorKey import cromwell.core.{JobOutput, WorkflowId} @@ -25,7 +24,6 @@ class JobStoreServiceSpec extends CromwellTestkitSpec with Matchers with Mockito "JobStoreService" should { "work" in { - val config = ConfigFactory.parseString("{}") lazy val jobStore: JobStore = new SqlJobStore(SingletonServicesStore.databaseInterface) val jobStoreService = system.actorOf(JobStoreActor.props(jobStore)) diff --git a/engine/src/test/scala/cromwell/jobstore/JobStoreWriterSpec.scala b/engine/src/test/scala/cromwell/jobstore/JobStoreWriterSpec.scala index 5699f61ed..a6e59679a 100644 --- a/engine/src/test/scala/cromwell/jobstore/JobStoreWriterSpec.scala +++ b/engine/src/test/scala/cromwell/jobstore/JobStoreWriterSpec.scala @@ -35,12 +35,14 @@ class JobStoreWriterSpec extends CromwellTestkitSpec with Matchers with BeforeAn key.callFqn shouldBe "call.fqn" key.index shouldBe None result shouldBe successResult + () } private def assertDb(totalWritesCalled: Int, jobCompletionsRecorded: Int, workflowCompletionsRecorded: Int): Unit = { database.totalWritesCalled shouldBe totalWritesCalled database.jobCompletionsRecorded shouldBe jobCompletionsRecorded database.workflowCompletionsRecorded shouldBe workflowCompletionsRecorded + () } private def assertReceived(expectedJobStoreWriteAcks: Int): Unit = { @@ -51,6 +53,7 @@ class JobStoreWriterSpec extends CromwellTestkitSpec with Matchers with BeforeAn case message => fail(s"Unexpected response message: $message") } jobStoreWriter.underlyingActor.stateName shouldBe Pending + () } "JobStoreWriter" should { diff --git a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala index 87db769c9..9904835dc 100644 --- a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala @@ -16,7 +16,7 @@ import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata._ import cromwell.services.metadata.impl.MetadataSummaryRefreshActor.MetadataSummarySuccess import cromwell.util.SampleWdl.DeclarationsWorkflow._ -import cromwell.util.SampleWdl.{ExpressionsInInputs, DeclarationsWorkflow, ThreeStep, HelloWorld} +import cromwell.util.SampleWdl.HelloWorld import org.scalatest.concurrent.{PatienceConfiguration, ScalaFutures} import org.scalatest.{FlatSpec, Matchers} import org.specs2.mock.Mockito @@ -80,6 +80,7 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala import akka.pattern.ask val putResult = serviceRegistryActor.ask(PutMetadataAction(events))(timeout) putResult.futureValue(PatienceConfiguration.Timeout(timeout.duration)) shouldBe a[MetadataPutAcknowledgement] + () } def forceSummary(): Unit = { @@ -242,7 +243,7 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala behavior of "REST API submission endpoint" it should "return 201 for a successful workflow submission " in { - Post("/workflows/$version", FormData(Seq("wdlSource" -> HelloWorld.wdlSource(), "workflowInputs" -> HelloWorld.rawInputs.toJson.toString()))) ~> + Post(s"/workflows/$version", FormData(Seq("wdlSource" -> HelloWorld.wdlSource(), "workflowInputs" -> HelloWorld.rawInputs.toJson.toString()))) ~> submitRoute ~> check { assertResult( @@ -274,7 +275,7 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala it should "return 200 for a successful workflow submission " in { val inputs = HelloWorld.rawInputs.toJson - Post("/workflows/$version/batch", + Post(s"/workflows/$version/batch", FormData(Seq("wdlSource" -> HelloWorld.wdlSource(), "workflowInputs" -> s"[$inputs, $inputs]"))) ~> submitBatchRoute ~> check { diff --git a/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala b/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala index 08c1e5b16..5ae82221b 100644 --- a/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala @@ -5,9 +5,8 @@ import java.util.UUID import akka.testkit._ import cromwell.core.{TestKitSuite, WorkflowId} +import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata._ -import MetadataService._ -import cromwell.services._ import cromwell.webservice.PerRequest.RequestComplete import cromwell.webservice.metadata.MetadataBuilderActor import org.scalatest.prop.TableDrivenPropertyChecks @@ -52,12 +51,12 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik val workflowA = WorkflowId.randomId() val workflowACalls = List( - Option(new MetadataJobKey("callB", Some(1), 3)), - Option(new MetadataJobKey("callB", None, 1)), - Option(new MetadataJobKey("callB", Some(1), 2)), - Option(new MetadataJobKey("callA", None, 1)), - Option(new MetadataJobKey("callB", Some(1), 1)), - Option(new MetadataJobKey("callB", Some(0), 1)), + Option(MetadataJobKey("callB", Some(1), 3)), + Option(MetadataJobKey("callB", None, 1)), + Option(MetadataJobKey("callB", Some(1), 2)), + Option(MetadataJobKey("callA", None, 1)), + Option(MetadataJobKey("callB", Some(1), 1)), + Option(MetadataJobKey("callB", Some(0), 1)), None ) val workflowAEvents = workflowACalls map { makeEvent(workflowA, _) } diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystem.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystem.scala index 2aad4b57b..215b18935 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystem.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystem.scala @@ -6,7 +6,6 @@ import java.nio.file.attribute.UserPrincipalLookupService import java.nio.file.spi.FileSystemProvider import java.util.{Collections, Set => JSet} -import scala.language.postfixOps case class NotAGcsPathException(path: String) extends IllegalArgumentException(s"$path is not a valid GCS path.") diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala index 7199d58ec..fd40a5403 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala @@ -11,6 +11,8 @@ import java.util import java.util.Collections import java.util.concurrent.{AbstractExecutorService, TimeUnit} +import cats.instances.try_._ +import cats.syntax.functor._ import com.google.api.client.googleapis.json.GoogleJsonResponseException import com.google.api.client.googleapis.media.MediaHttpUploader import com.google.api.services.storage.Storage @@ -47,7 +49,7 @@ object GcsFileSystemProvider { if retries > 0 && (ex.getStatusCode == 404 || ex.getStatusCode == 500) => // FIXME remove this sleep - Thread.sleep(retryInterval.toMillis.toInt) + Thread.sleep(retryInterval.toMillis) withRetry(f, retries - 1) case Failure(ex) => throw ex } @@ -102,12 +104,13 @@ class GcsFileSystemProvider private[gcs](storageClient: Try[Storage], val execut lazy val defaultFileSystem: GcsFileSystem = GcsFileSystem(this) - private def exists(path: Path) = path match { + private def exists(path: Path): Unit = path match { case gcsPath: NioGcsPath => - Try(withRetry(client.objects.get(gcsPath.bucket, gcsPath.objectName).execute)) recover { + val attempt: Try[Any] = Try(withRetry(client.objects.get(gcsPath.bucket, gcsPath.objectName).execute)) recover { case ex: GoogleJsonResponseException if ex.getStatusCode == 404 => if (!gcsPath.isDirectory) throw new FileNotFoundException(path.toString) - } get + } + attempt.void.get case _ => throw new FileNotFoundException(path.toString) } @@ -173,12 +176,13 @@ class GcsFileSystemProvider private[gcs](storageClient: Try[Storage], val execut override def copy(source: Path, target: Path, options: CopyOption*): Unit = { (source, target) match { case (s: NioGcsPath, d: NioGcsPath) => - def innerCopy = { + def innerCopy(): Unit = { val storageObject = client.objects.get(s.bucket, s.objectName).execute client.objects.copy(s.bucket, s.objectName, d.bucket, d.objectName, storageObject).execute + () } - withRetry(innerCopy) + withRetry(innerCopy()) case _ => throw new UnsupportedOperationException(s"Can only copy from GCS to GCS: $source or $target is not a GCS path") } } @@ -186,7 +190,10 @@ class GcsFileSystemProvider private[gcs](storageClient: Try[Storage], val execut override def delete(path: Path): Unit = { path match { case gcs: NioGcsPath => try { - withRetry(client.objects.delete(gcs.bucket, gcs.objectName).execute()) + withRetry { + client.objects.delete(gcs.bucket, gcs.objectName).execute() + () + } } catch { case ex: GoogleJsonResponseException if ex.getStatusCode == 404 => throw new NoSuchFileException(path.toString) } @@ -204,12 +211,13 @@ class GcsFileSystemProvider private[gcs](storageClient: Try[Storage], val execut override def move(source: Path, target: Path, options: CopyOption*): Unit = { (source, target) match { case (s: NioGcsPath, d: NioGcsPath) => - def moveInner = { + def moveInner(): Unit = { val storageObject = client.objects.get(s.bucket, s.objectName).execute client.objects.rewrite(s.bucket, s.objectName, d.bucket, d.objectName, storageObject).execute + () } - withRetry(moveInner) + withRetry(moveInner()) case _ => throw new UnsupportedOperationException(s"Can only move from GCS to GCS: $source or $target is not a GCS path") } } @@ -219,7 +227,7 @@ class GcsFileSystemProvider private[gcs](storageClient: Try[Storage], val execut case _ => notAGcsPath(path) } - override def checkAccess(path: Path, modes: AccessMode*): Unit = exists(path) + override def checkAccess(path: Path, modes: AccessMode*): Unit = { exists(path); () } override def createDirectory(dir: Path, attrs: FileAttribute[_]*): Unit = {} override def getFileSystem(uri: URI): FileSystem = defaultFileSystem @@ -239,7 +247,7 @@ class GcsFileSystemProvider private[gcs](storageClient: Try[Storage], val execut // Contains a Seq corresponding to the current page of objects, plus a token for the next page of objects, if any. case class ListPageResult(objects: Seq[StorageObject], nextPageToken: Option[String]) - def requestListPage(pageToken: Option[String] = None): ListPageResult = { + def requestListPage(pageToken: Option[String]): ListPageResult = { val objects = withRetry(listRequest.setPageToken(pageToken.orNull).execute()) ListPageResult(objects.getItems.asScala, Option(objects.getNextPageToken)) } diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleAuthMode.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleAuthMode.scala index c18a85a25..2930cc911 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleAuthMode.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleAuthMode.scala @@ -161,7 +161,7 @@ final case class RefreshTokenMode(name: String, clientId: String, clientSecret: /** * Throws if the refresh token is not specified. */ - override def assertWorkflowOptions(options: GoogleAuthOptions) = getToken(options) + override def assertWorkflowOptions(options: GoogleAuthOptions): Unit = { getToken(options); () } private def getToken(options: GoogleAuthOptions): String = { options.get(RefreshTokenOptionKey).getOrElse(throw new IllegalArgumentException(s"Missing parameters in workflow options: $RefreshTokenOptionKey")) diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala index 9315b51cc..9c5579839 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala @@ -13,7 +13,6 @@ import cromwell.core.ErrorOr._ import org.slf4j.LoggerFactory import scala.collection.JavaConverters._ -import scala.language.postfixOps final case class GoogleConfiguration private (applicationName: String, authsByName: Map[String, GoogleAuthMode]) { diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/NioGcsPath.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/NioGcsPath.scala index 672b64cf0..65e148f77 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/NioGcsPath.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/NioGcsPath.scala @@ -7,7 +7,7 @@ import java.nio.file._ import java.util import scala.collection.JavaConverters._ -import scala.language.{implicitConversions, postfixOps} +import scala.language.postfixOps import scala.util.Try object NioGcsPath { diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala index b1d44feaa..3eeeaf568 100644 --- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala +++ b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala @@ -4,7 +4,6 @@ import com.typesafe.config.{ConfigException, ConfigFactory} import lenthall.config.ConfigValidationException import org.scalatest.{FlatSpec, Matchers} -import scala.language.postfixOps class GoogleConfigurationSpec extends FlatSpec with Matchers { diff --git a/project/Settings.scala b/project/Settings.scala index 3038ffed3..627ac080c 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -23,13 +23,31 @@ object Settings { https://github.com/sbt/sbt-assembly/issues/69 https://github.com/scala/pickling/issues/10 + + Other fancy flags from + + http://blog.threatstack.com/useful-scalac-options-for-better-scala-development-part-1 + + and + + https://tpolecat.github.io/2014/04/11/scalac-flags.html + */ val compilerSettings = List( - "-deprecation", - "-unchecked", + "-Xlint", "-feature", - "-Xmax-classfile-name", - "200" + "-Xmax-classfile-name", "200", + "-target:jvm-1.8", + "-encoding", "UTF-8", + "-unchecked", + "-deprecation", + "-Xfuture", + "-Yno-adapted-args", + "-Ywarn-dead-code", + "-Ywarn-numeric-widen", + "-Ywarn-value-discard", + "-Ywarn-unused", + "-Ywarn-unused-import" ) lazy val assemblySettings = Seq( diff --git a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala index b440b9f9e..672f68580 100644 --- a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala +++ b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala @@ -8,7 +8,6 @@ import cromwell.core.{JobKey, WorkflowId, WorkflowState} import cromwell.services.ServiceRegistryActor.ServiceRegistryMessage import wdl4s.values._ -import scala.language.postfixOps object MetadataService { diff --git a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala index 3bf9fce2b..435c14df2 100644 --- a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala +++ b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala @@ -8,7 +8,6 @@ import cats.syntax.validated._ import cromwell.core.{WorkflowId, WorkflowState} import cromwell.core.ErrorOr._ -import scala.language.postfixOps import scala.util.{Success, Try} object WorkflowQueryKey { diff --git a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala index 669bde613..f66173657 100644 --- a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala +++ b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala @@ -9,7 +9,6 @@ import cromwell.core.WorkflowId import cromwell.services.metadata.WorkflowQueryKey._ import cromwell.core.ErrorOr._ -import scala.language.postfixOps case class WorkflowQueryParameters private(statuses: Set[String], names: Set[String], diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala index 92ba34589..76476d3fd 100644 --- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala +++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala @@ -38,7 +38,8 @@ object MetadataDatabaseAccess { def baseSummary(workflowUuid: String) = WorkflowMetadataSummaryEntry(workflowUuid, None, None, None, None, None) - private implicit class MetadatumEnhancer(val metadatum: MetadataEntry) extends AnyVal { + // If visibility is made `private`, there's a bogus warning about this being unused. + implicit class MetadatumEnhancer(val metadatum: MetadataEntry) extends AnyVal { def toSummary: WorkflowMetadataSummaryEntry = { val base = baseSummary(metadatum.workflowExecutionUuid) metadatum.metadataKey match { diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala index 48a85b71e..ab9e20f29 100644 --- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala +++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala @@ -1,6 +1,5 @@ package cromwell.services.metadata.impl -import java.time.OffsetDateTime import java.util.UUID import akka.actor.{Actor, ActorLogging, ActorRef, Props} @@ -13,7 +12,6 @@ import cromwell.services.metadata.impl.MetadataSummaryRefreshActor.{MetadataSumm import lenthall.config.ScalaConfig._ import scala.concurrent.duration.{Duration, FiniteDuration} -import scala.language.postfixOps import scala.util.{Failure, Success, Try} object MetadataServiceActor { @@ -37,8 +35,8 @@ case class MetadataServiceActor(serviceConfig: Config, globalConfig: Config) summaryActor foreach { _ => self ! RefreshSummary } - private def scheduleSummary = { - MetadataSummaryRefreshInterval map { context.system.scheduler.scheduleOnce(_, self, RefreshSummary)(context.dispatcher, self) } + private def scheduleSummary(): Unit = { + MetadataSummaryRefreshInterval foreach { context.system.scheduler.scheduleOnce(_, self, RefreshSummary)(context.dispatcher, self) } } private def buildSummaryActor: Option[ActorRef] = { @@ -76,9 +74,9 @@ case class MetadataServiceActor(serviceConfig: Config, globalConfig: Config) case v: ValidateWorkflowIdAndExecute => validateWorkflowId(v) case action: ReadAction => readActor forward action case RefreshSummary => summaryActor foreach { _ ! SummarizeMetadata(sender()) } - case MetadataSummarySuccess => scheduleSummary + case MetadataSummarySuccess => scheduleSummary() case MetadataSummaryFailure(t) => log.error(t, "Error summarizing metadata") - scheduleSummary + scheduleSummary() } } diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala index ad176567e..007755fa6 100644 --- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala +++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala @@ -1,6 +1,5 @@ package cromwell.services.metadata.impl -import java.time.OffsetDateTime import akka.actor.{ActorRef, LoggingFSM, Props} import com.typesafe.config.ConfigFactory diff --git a/services/src/main/scala/cromwell/services/metadata/metadata.scala b/services/src/main/scala/cromwell/services/metadata/metadata.scala new file mode 100644 index 000000000..9e621cfad --- /dev/null +++ b/services/src/main/scala/cromwell/services/metadata/metadata.scala @@ -0,0 +1,39 @@ +package cromwell.services.metadata + +case class QueryParameter(key: String, value: String) + +object Patterns { + val WorkflowName = """ + (?x) # Turn on comments and whitespace insensitivity. + + ( # Begin capture. + + [a-zA-Z][a-zA-Z0-9_]* # WDL identifier naming pattern of an initial alpha character followed by zero + # or more alphanumeric or underscore characters. + + ) # End capture. + """.trim.r + + val CallFullyQualifiedName = """ + (?x) # Turn on comments and whitespace insensitivity. + + ( # Begin outer capturing group for FQN. + + (?:[a-zA-Z][a-zA-Z0-9_]*) # Inner noncapturing group for top-level workflow name. This is the WDL + # identifier naming pattern of an initial alpha character followed by zero + # or more alphanumeric or underscore characters. + + (?:\.[a-zA-Z][a-zA-Z0-9_]*){1} # Inner noncapturing group for call name, a literal dot followed by a WDL + # identifier. Currently this is quantified to {1} since the call name is + # mandatory and nested workflows are not supported. This could be changed + # to + or a different quantifier if these assumptions change. + + ) # End outer capturing group for FQN. + + + (?: # Begin outer noncapturing group for shard. + \. # Literal dot. + (\d+) # Captured shard digits. + )? # End outer optional noncapturing group for shard. + """.trim.r // The trim is necessary as (?x) must be at the beginning of the regex. +} diff --git a/services/src/main/scala/cromwell/services/metadata/package.scala b/services/src/main/scala/cromwell/services/metadata/package.scala index a6ed193a2..f35408296 100644 --- a/services/src/main/scala/cromwell/services/metadata/package.scala +++ b/services/src/main/scala/cromwell/services/metadata/package.scala @@ -1,42 +1,5 @@ package cromwell.services package object metadata { - case class QueryParameter(key: String, value: String) type QueryParameters = Seq[QueryParameter] - - object Patterns { - val WorkflowName = """ - (?x) # Turn on comments and whitespace insensitivity. - - ( # Begin capture. - - [a-zA-Z][a-zA-Z0-9_]* # WDL identifier naming pattern of an initial alpha character followed by zero - # or more alphanumeric or underscore characters. - - ) # End capture. - """.trim.r - - val CallFullyQualifiedName = """ - (?x) # Turn on comments and whitespace insensitivity. - - ( # Begin outer capturing group for FQN. - - (?:[a-zA-Z][a-zA-Z0-9_]*) # Inner noncapturing group for top-level workflow name. This is the WDL - # identifier naming pattern of an initial alpha character followed by zero - # or more alphanumeric or underscore characters. - - (?:\.[a-zA-Z][a-zA-Z0-9_]*){1} # Inner noncapturing group for call name, a literal dot followed by a WDL - # identifier. Currently this is quantified to {1} since the call name is - # mandatory and nested workflows are not supported. This could be changed - # to + or a different quantifier if these assumptions change. - - ) # End outer capturing group for FQN. - - - (?: # Begin outer noncapturing group for shard. - \. # Literal dot. - (\d+) # Captured shard digits. - )? # End outer optional noncapturing group for shard. - """.trim.r // The trim is necessary as (?x) must be at the beginning of the regex. - } } diff --git a/src/main/scala/cromwell/Main.scala b/src/main/scala/cromwell/Main.scala index a54a982de..74a5405c1 100644 --- a/src/main/scala/cromwell/Main.scala +++ b/src/main/scala/cromwell/Main.scala @@ -10,7 +10,6 @@ import org.slf4j.LoggerFactory import scala.collection.JavaConverters._ import scala.concurrent.duration._ import scala.concurrent.{Await, Future} -import scala.language.postfixOps import scala.util.{Failure, Success} object Main extends App { @@ -74,7 +73,6 @@ object Main extends App { val runner = CromwellSystem.actorSystem.actorOf(runnerProps, "SingleWorkflowRunnerActor") import PromiseActor.EnhancedActorRef - import scala.concurrent.ExecutionContext.Implicits.global val promise = runner.askNoTimeout(RunWorkflow) waitAndExit(promise, CromwellSystem) diff --git a/src/test/scala/cromwell/CromwellCommandLineSpec.scala b/src/test/scala/cromwell/CromwellCommandLineSpec.scala index ec85f3ab7..42f03fdfc 100644 --- a/src/test/scala/cromwell/CromwellCommandLineSpec.scala +++ b/src/test/scala/cromwell/CromwellCommandLineSpec.scala @@ -6,7 +6,6 @@ import cromwell.util.SampleWdl import cromwell.util.SampleWdl.ThreeStep import org.scalatest.{FlatSpec, Matchers} -import scala.language.postfixOps import scala.util.Try class CromwellCommandLineSpec extends FlatSpec with Matchers { diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala index 46678ae25..400c6a55f 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala @@ -1,18 +1,16 @@ package cromwell.backend.impl.htcondor +import java.nio.file.FileSystems import java.nio.file.attribute.PosixFilePermission -import java.nio.file.{FileSystems, Files, Path, Paths} import java.util.UUID import akka.actor.{ActorRef, Props} -import better.files.File -import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse, FailedNonRetryableResponse, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, FailedNonRetryableResponse, SucceededResponse} import cromwell.backend._ import cromwell.backend.impl.htcondor.caching.CacheActor._ import cromwell.backend.impl.htcondor.caching.localization.CachedResultLocalization import cromwell.backend.io.JobPaths import cromwell.backend.sfs.{SharedFileSystem, SharedFileSystemExpressionFunctions} -import cromwell.core.{JobOutput, JobOutputs, LocallyQualifiedName} import cromwell.services.keyvalue.KeyValueServiceActor._ import cromwell.services.metadata.CallMetadataKeys import org.apache.commons.codec.digest.DigestUtils @@ -20,12 +18,11 @@ import wdl4s._ import wdl4s.parser.MemoryUnit import wdl4s.types.{WdlArrayType, WdlFileType} import wdl4s.util.TryUtil -import wdl4s.values.{WdlArray, WdlFile, WdlSingleFile, WdlValue} +import wdl4s.values.WdlArray import scala.concurrent.{Future, Promise} import scala.sys.process.ProcessLogger import scala.util.{Failure, Success, Try} -import scala.language.postfixOps object HtCondorJobExecutionActor { val HtCondorJobIdKey = "htCondor_job_id" @@ -110,12 +107,15 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor case JobExecutionResponse(resp) => log.debug("{}: Completing job [{}] with response: [{}]", tag, jobDescriptor.key, resp) executionResponse trySuccess resp + () case TrackTaskStatus(id) => // Avoid the redundant status check if the response is already completed (e.g. in case of abort) if (!executionResponse.isCompleted) trackTask(id) // Messages received from Caching actor - case ExecutionResultFound(succeededResponse) => executionResponse trySuccess localizeCachedResponse(succeededResponse) + case ExecutionResultFound(succeededResponse) => + executionResponse trySuccess localizeCachedResponse(succeededResponse) + () case ExecutionResultNotFound => prepareAndExecute() case ExecutionResultStored(hash) => log.debug("{} Cache entry was stored for Job with hash {}.", tag, hash) case ExecutionResultAlreadyExist => log.warning("{} Cache entry for hash {} already exist.", tag, jobHash) @@ -127,9 +127,13 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor case KvKeyLookupFailed(_) => log.debug("{} Job id not found. Falling back to execute.", tag) execute + // -Ywarn-value-discard + () case KvFailure(_, e) => log.error("{} Failure attempting to look up HtCondor job id. Exception message: {}. Falling back to execute.", tag, e.getMessage) execute + // -Ywarn-value-discard + () } /** @@ -223,6 +227,7 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor import scala.concurrent.duration._ // Job is still running in HtCondor. Check back again after `pollingInterval` seconds context.system.scheduler.scheduleOnce(pollingInterval.seconds, self, TrackTaskStatus(jobIdentifier)) + () case Success(Some(rc)) if runtimeAttributes.continueOnReturnCode.continueFor(rc) => self ! JobExecutionResponse(processSuccess(rc)) case Success(Some(rc)) => self ! JobExecutionResponse(FailedNonRetryableResponse(jobDescriptor.key, new IllegalStateException("Job exited with invalid return code: " + rc), Option(rc))) @@ -254,7 +259,7 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor log.error(ex.getCause, errMsg) throw new IllegalStateException(errMsg, ex.getCause) } - val str = Seq(cmd, + val str = Seq[Any](cmd, runtimeAttributes.failOnStderr, runtimeAttributes.dockerImage.getOrElse(""), runtimeAttributes.dockerWorkingDir.getOrElse(""), @@ -292,6 +297,7 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor ) cmds.generateSubmitFile(submitFilePath, attributes) // This writes the condor submit file + () } catch { case ex: Exception => @@ -352,11 +358,11 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor } private def prepareAndExecute(): Unit = { - Try { + try { createExecutionFolderAndScript() executeTask() - } recover { - case exception => self ! JobExecutionResponse(FailedNonRetryableResponse(jobDescriptor.key, exception, None)) + } catch { + case e: Exception => self ! JobExecutionResponse(FailedNonRetryableResponse(jobDescriptor.key, e, None)) } } diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala index 27364b8f2..76a81c08a 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala @@ -8,7 +8,6 @@ import cromwell.backend.impl.htcondor import cromwell.core.PathFactory.{EnhancedPath, FlushingAndClosingWriter} import cromwell.core.{TailedWriter, UntailedWriter} -import scala.language.postfixOps import scala.sys.process._ object JobStatus { @@ -62,6 +61,7 @@ class HtCondorCommands extends StrictLogging { |$instantiatedCommand |echo $$? > rc |""".stripMargin) + () } def generateSubmitFile(path: Path, attributes: Map[String, Any]): String = { @@ -84,7 +84,7 @@ class HtCondorProcess extends StrictLogging { private val stdout = new StringBuilder private val stderr = new StringBuilder - def processLogger: ProcessLogger = ProcessLogger(stdout append _, stderr append _) + def processLogger: ProcessLogger = ProcessLogger(s => { stdout append s; () }, s => { stderr append s; () }) def processStdout: String = stdout.toString().trim def processStderr: String = stderr.toString().trim def commandList(command: String): Seq[String] = Seq("/bin/bash",command) diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala index 45da26644..9e71d1a14 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala @@ -16,11 +16,11 @@ class HtCondorInitializationActorSpec extends TestKitSuite("HtCondorInitializati import BackendSpec._ val HelloWorld = - """ + s""" |task hello { | String addressee = "you" | command { - | echo "Hello ${addressee}!" + | echo "Hello $${addressee}!" | } | output { | String salutation = read_string(stdout()) diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala index 8a18ddf0d..589554e94 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala @@ -60,12 +60,12 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc """.stripMargin private val helloWorldWdlWithFileInput = - """ + s""" |task hello { | File inputFile | | command { - | echo ${inputFile} + | echo $${inputFile} | } | output { | String salutation = read_string(stdout()) @@ -79,12 +79,12 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc """.stripMargin private val helloWorldWdlWithFileArrayInput = - """ + s""" |task hello { | Array[File] inputFiles | | command { - | echo ${sep=' ' inputFiles} + | echo $${sep=' ' inputFiles} | } | output { | String salutation = read_string(stdout()) @@ -408,7 +408,10 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc cleanUpJob(jobPaths) } - private def cleanUpJob(jobPaths: JobPaths): Unit = File(jobPaths.workflowRoot).delete(true) + private def cleanUpJob(jobPaths: JobPaths): Unit = { + File(jobPaths.workflowRoot).delete(true) + () + } private def createCannedFile(prefix: String, contents: String, dir: Option[Path] = None): File = { val suffix = ".out" diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala index a411a7aef..db95a999e 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala @@ -17,11 +17,11 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { import BackendSpec._ val HelloWorld = - """ + s""" |task hello { | String addressee = "you" | command { - | echo "Hello ${addressee}!" + | echo "Hello $${addressee}!" | } | output { | String salutation = read_string(stdout()) @@ -61,7 +61,7 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { "return an instance of itself when tries to validate a valid Docker entry based on input" in { val expectedRuntimeAttributes = staticDefaults.copy(dockerImage = Option("you")) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { docker: "\${addressee}" }""").head + val runtimeAttributes = createRuntimeAttributes(HelloWorld, s"""runtime { docker: "\\$${addressee}" }""").head assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes) } @@ -84,7 +84,7 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { "return an instance of itself when tries to validate a valid docker working directory entry based on input" in { val expectedRuntimeAttributes = staticDefaults.copy(dockerWorkingDir = Option("you")) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { dockerWorkingDir: "\${addressee}" }""").head + val runtimeAttributes = createRuntimeAttributes(HelloWorld, s"""runtime { dockerWorkingDir: "\\$${addressee}" }""").head assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes) } @@ -107,7 +107,7 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { "return an instance of itself when tries to validate a valid docker output directory entry based on input" in { val expectedRuntimeAttributes = staticDefaults.copy(dockerOutputDir = Option("you")) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { dockerOutputDir: "\${addressee}" }""").head + val runtimeAttributes = createRuntimeAttributes(HelloWorld, s"""runtime { dockerOutputDir: "\\$${addressee}" }""").head assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes) } @@ -245,6 +245,7 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { } catch { case ex: RuntimeException => fail(s"Exception was not expected but received: ${ex.getMessage}") } + () } private def assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue], exMsg: String): Unit = { @@ -254,6 +255,7 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { } catch { case ex: RuntimeException => assert(ex.getMessage.contains(exMsg)) } + () } private def createRuntimeAttributes(wdlSource: WdlSource, runtimeAttributes: String): Seq[Map[String, WdlValue]] = { diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GenomicsFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GenomicsFactory.scala index 3427f3f09..7285171b5 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GenomicsFactory.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GenomicsFactory.scala @@ -6,7 +6,6 @@ import com.google.api.client.auth.oauth2.Credential import com.google.api.client.http.HttpTransport import com.google.api.client.json.JsonFactory import com.google.api.services.genomics.Genomics -import cromwell.filesystems.gcs.GoogleConfiguration object GenomicsFactory { diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala index 3b2a4bc11..b2e5974e0 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala @@ -6,6 +6,8 @@ import java.nio.file.{Path, Paths} import akka.actor.{Actor, ActorLogging, ActorRef, Props} import akka.event.LoggingReceive import better.files._ +import cats.instances.future._ +import cats.syntax.functor._ import com.google.api.client.googleapis.json.GoogleJsonResponseException import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse} import cromwell.backend.BackendLifecycleActor.AbortJobCommand @@ -277,7 +279,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes |echo $$? > $rcPath """.stripMargin.trim - def writeScript(): Future[Unit] = Future(File(jesCallPaths.gcsExecPath).write(fileContent)) + def writeScript(): Future[Unit] = Future { File(jesCallPaths.gcsExecPath).write(fileContent) } void implicit val system = context.system Retry.withRetry( @@ -291,7 +293,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes descriptor.workflowOptions.getOrElse(WorkflowOptionKeys.GoogleProject, jesAttributes.project) } - private def createJesRun(jesParameters: Seq[JesParameter], runIdForResumption: Option[String] = None): Future[Run] = { + private def createJesRun(jesParameters: Seq[JesParameter], runIdForResumption: Option[String]): Future[Run] = { def createRun() = Future(Run( runIdForResumption, @@ -423,7 +425,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes fileMetadata += JesMetadataKeys.MonitoringLog -> monitoringOutput.get.gcs } - val otherMetadata = Map( + val otherMetadata: Map[String, Any] = Map( JesMetadataKeys.GoogleProject -> jesAttributes.project, JesMetadataKeys.ExecutionBucket -> jesAttributes.executionBucket, JesMetadataKeys.EndpointUrl -> jesAttributes.endpointUrl, diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala index 8fd05c348..3c72ca7fc 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala @@ -5,7 +5,6 @@ import java.net.URL import cats.data._ import cats.data.Validated._ import cats.syntax.cartesian._ -import cats.syntax.validated._ import com.typesafe.config.Config import cromwell.backend.impl.jes.JesImplicits.GoogleAuthWorkflowOptions import cromwell.core.WorkflowOptions @@ -15,8 +14,6 @@ import lenthall.config.ValidatedConfig._ import cromwell.core.ErrorOr._ import wdl4s.ExceptionWithErrors -import scala.language.postfixOps - case class JesAttributes(project: String, genomicsAuth: GoogleAuthMode, gcsFilesystemAuth: GoogleAuthMode, diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala index 3f498ff71..1e87b4d1f 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala @@ -13,7 +13,6 @@ import cromwell.core.{ExecutionStore, OutputStore} import wdl4s.Call import wdl4s.expression.WdlStandardLibraryFunctions -import scala.language.postfixOps case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory { diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala index 9b50bb8e2..038c615dd 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala @@ -4,12 +4,15 @@ import java.nio.file.Path import akka.actor.Props import better.files._ +import cats.instances.future._ +import cats.syntax.functor._ import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor, BackendWorkflowFinalizationActor} import cromwell.core.Dispatcher.IoDispatcher import cromwell.core.{ExecutionStore, OutputStore, PathCopier} import wdl4s.Call import scala.concurrent.Future +import scala.language.postfixOps object JesFinalizationActor { def props(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], jesConfiguration: JesConfiguration, @@ -40,7 +43,7 @@ class JesFinalizationActor (override val workflowDescriptor: BackendWorkflowDesc private def deleteAuthenticationFile(): Future[Unit] = { (jesConfiguration.needAuthFileUpload, workflowPaths) match { - case (true, Some(paths)) => Future(File(paths.gcsAuthFilePath).delete(false)) map { _ => () } + case (true, Some(paths)) => Future { File(paths.gcsAuthFilePath).delete(false) } void case _ => Future.successful(()) } } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala index 04ac8f70d..e76a62e9d 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala @@ -3,6 +3,8 @@ package cromwell.backend.impl.jes import java.io.IOException import akka.actor.{ActorRef, Props} +import cats.instances.future._ +import cats.syntax.functor._ import com.google.api.services.genomics.Genomics import cromwell.backend.impl.jes.JesInitializationActor._ import cromwell.backend.impl.jes.authentication.{GcsLocalizing, JesAuthInformation, JesCredentials} @@ -15,9 +17,9 @@ import cromwell.core.WorkflowOptions import cromwell.core.retry.Retry import cromwell.filesystems.gcs.{ClientSecrets, GoogleAuthMode} import spray.json.JsObject +import wdl4s.Call import wdl4s.types.{WdlBooleanType, WdlFloatType, WdlIntegerType, WdlStringType} import wdl4s.values.WdlValue -import wdl4s.{Call, WdlExpression} import scala.concurrent.Future import scala.util.Try @@ -96,7 +98,7 @@ class JesInitializationActor(override val workflowDescriptor: BackendWorkflowDes val upload = () => Future(path.writeAsJson(content)) workflowLogger.info(s"Creating authentication file for workflow ${workflowDescriptor.id} at \n ${path.toString}") - Retry.withRetry(upload, isFatal = isFatalJesException, isTransient = isTransientJesException)(context.system) map { _ => () } recoverWith { + Retry.withRetry(upload, isFatal = isFatalJesException, isTransient = isTransientJesException)(context.system).void.recoverWith { case failure => Future.failed(new IOException("Failed to upload authentication file", failure)) } } getOrElse Future.successful(()) diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala index acfa8c6b2..4ded3e9d1 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala @@ -71,7 +71,7 @@ trait JesJobCachingActorHelper extends JobCachingActorHelper { fileMetadata += JesMetadataKeys.MonitoringLog -> monitoringOutput.get.gcs } - val otherMetadata = Map( + val otherMetadata: Map[String, Any] = Map( JesMetadataKeys.GoogleProject -> jesAttributes.project, JesMetadataKeys.ExecutionBucket -> jesAttributes.executionBucket, JesMetadataKeys.EndpointUrl -> jesAttributes.endpointUrl, diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala index 594b7f0c2..eed3268f5 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala @@ -12,7 +12,6 @@ import cromwell.services.keyvalue.KeyValueServiceActor._ import org.slf4j.LoggerFactory import scala.concurrent.{Future, Promise} -import scala.language.postfixOps object JesJobExecutionActor { val logger = LoggerFactory.getLogger("JesBackend") diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala index 19891ee61..593d1e6df 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala @@ -12,7 +12,6 @@ import cromwell.backend.validation._ import cromwell.core._ import lenthall.exception.MessageAggregation import cromwell.core.ErrorOr._ -import mouse.string._ import org.slf4j.Logger import wdl4s.types._ import wdl4s.values._ diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala index b24f7affe..0f948344c 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala @@ -6,7 +6,6 @@ import java.util.{ArrayList => JArrayList} import com.google.api.client.util.{ArrayMap => GArrayMap} import com.google.api.services.genomics.Genomics import com.google.api.services.genomics.model._ -import com.typesafe.config.ConfigFactory import cromwell.backend.BackendJobDescriptor import cromwell.backend.impl.jes.RunStatus.{Failed, Initializing, Running, Success} import cromwell.core.ExecutionEvent @@ -14,7 +13,6 @@ import cromwell.core.logging.JobLogger import org.slf4j.LoggerFactory import scala.collection.JavaConverters._ -import scala.concurrent.duration._ import scala.language.postfixOps object Run { @@ -156,13 +154,12 @@ case class Run(runId: String, genomicsInterface: Genomics) { } private def eventIfExists(name: String, metadata: Map[String, AnyRef], eventName: String): Option[ExecutionEvent] = { - metadata.get(name) map { - case time => ExecutionEvent(eventName, OffsetDateTime.parse(time.toString)) - } + metadata.get(name) map { time => ExecutionEvent(eventName, OffsetDateTime.parse(time.toString)) } } def abort(): Unit = { val cancellationRequest: CancelOperationRequest = new CancelOperationRequest() genomicsInterface.operations().cancel(runId, cancellationRequest).execute + () } } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala index 848b188c1..c1ce80b4f 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala @@ -7,7 +7,6 @@ import cats.syntax.cartesian._ import cats.syntax.validated._ import com.google.api.services.genomics.model.Disk import cromwell.core.ErrorOr._ -import mouse.string._ import wdl4s.ExceptionWithErrors import wdl4s.values._ diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala index a4a2b0ba8..24d417c99 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala @@ -3,8 +3,6 @@ package cromwell.backend.impl.jes import java.nio.file.{Files, Path} import com.google.api.client.http.HttpResponseException -import cromwell.backend.BackendWorkflowDescriptor -import cromwell.backend.impl.jes.JesImplicits.GoogleAuthWorkflowOptions import cromwell.filesystems.gcs._ package object io { diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala index d2527df6e..2a2ae3ec9 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala @@ -10,16 +10,15 @@ import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse import cromwell.backend.async.AsyncBackendJobExecutionActor.{Execute, ExecutionMode} import cromwell.backend.async.{AbortedExecutionHandle, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle} import cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.JesPendingExecutionHandle +import cromwell.backend.impl.jes.MockObjects._ import cromwell.backend.impl.jes.RunStatus.Failed import cromwell.backend.impl.jes.io.{DiskType, JesWorkingDisk} import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobDescriptorKey, BackendWorkflowDescriptor, PreemptedException, RuntimeAttributeDefinition} -import cromwell.core._ import cromwell.core.logging.LoggerWrapper +import cromwell.core.{WorkflowId, WorkflowOptions, _} import cromwell.filesystems.gcs._ import cromwell.util.SampleWdl import org.scalatest._ -import cromwell.core.{WorkflowId, WorkflowOptions} -import cromwell.filesystems.gcs.GoogleAuthMode.GoogleAuthOptions import org.scalatest.prop.Tables.Table import org.slf4j.Logger import org.specs2.mock.Mockito @@ -31,7 +30,6 @@ import wdl4s.{Call, LocallyQualifiedName, NamespaceWithWorkflow} import scala.concurrent.duration._ import scala.concurrent.{Await, ExecutionContext, Future, Promise} import scala.util.{Success, Try} -import cromwell.backend.impl.jes.MockObjects._ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackendJobExecutionActorSpec") with FlatSpecLike with Matchers with ImplicitSender with Mockito { @@ -41,11 +39,11 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend implicit val Timeout = 5.seconds.dilated val YoSup = - """ + s""" |task sup { | String addressee | command { - | echo "yo sup ${addressee}!" + | echo "yo sup $${addressee}!" | } | output { | String salutation = read_string(stdout()) @@ -72,15 +70,6 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend } private def buildInitializationData(jobDescriptor: BackendJobDescriptor, configuration: JesConfiguration) = { - def gcsFileSystem = { - val authOptions = new GoogleAuthOptions { - override def get(key: String): Try[String] = Try(throw new RuntimeException(s"key '$key' not found")) - } - - val storage = jesConfiguration.jesAttributes.gcsFilesystemAuth.buildStorage(authOptions, "appName") - GcsFileSystem(GcsFileSystemProvider(storage)(scala.concurrent.ExecutionContext.global)) - } - val workflowPaths = JesWorkflowPaths(jobDescriptor.workflowDescriptor, configuration, mockCredentials)(scala.concurrent.ExecutionContext.global) diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala index 2669552b9..4699402bc 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala @@ -25,11 +25,11 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe import BackendSpec._ val HelloWorld = - """ + s""" |task hello { | String addressee = "you" | command { - | echo "Hello ${addressee}!" + | echo "Hello $${addressee}!" | } | output { | String salutation = read_string(stdout()) diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala index efbcb53e6..204907110 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala @@ -2,7 +2,7 @@ package cromwell.backend.impl.jes import cromwell.backend.impl.jes.io.{DiskType, JesAttachedDisk, JesWorkingDisk} import cromwell.backend.validation.ContinueOnReturnCodeSet -import cromwell.backend.{BackendSpec, MemorySize, RuntimeAttributeDefinition} +import cromwell.backend.{MemorySize, RuntimeAttributeDefinition} import cromwell.core.WorkflowOptions import org.scalatest.{Matchers, WordSpecLike} import org.slf4j.helpers.NOPLogger @@ -190,11 +190,11 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito { private def assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes: Map[String, WdlValue], expectedRuntimeAttributes: JesRuntimeAttributes, workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = { val withDefaults = RuntimeAttributeDefinition.addDefaultsToAttributes(JesBackendLifecycleActorFactory.staticRuntimeAttributeDefinitions, workflowOptions) _ try { - assert(JesRuntimeAttributes(withDefaults(runtimeAttributes), NOPLogger.NOP_LOGGER) == expectedRuntimeAttributes) } catch { case ex: RuntimeException => fail(s"Exception was not expected but received: ${ex.getMessage}") } + () } private def assertJesRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue], exMsg: String, workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = { @@ -205,6 +205,7 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito { } catch { case ex: RuntimeException => assert(ex.getMessage.contains(exMsg)) } + () } private val emptyWorkflowOptions = WorkflowOptions.fromMap(Map.empty).get diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala index cf6fcea8c..ad9761bca 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala @@ -57,6 +57,7 @@ sealed trait ConfigAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecut s"""|#!/bin/bash |$command |""".stripMargin) + () } /** diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala index c17ebe269..8ac1e2c21 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala @@ -46,6 +46,7 @@ trait BackgroundAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecution | & |echo $$! |""".stripMargin) + () } override def getJob(exitValue: Int, stdout: Path, stderr: Path) = { @@ -80,5 +81,6 @@ trait BackgroundAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecution | |kill_children ${job.jobId} |""".stripMargin) + () } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/ProcessRunner.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/ProcessRunner.scala index 509ec01ba..bf6bebb28 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/ProcessRunner.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/ProcessRunner.scala @@ -17,7 +17,7 @@ class ProcessRunner(val argv: Seq[Any], val stdoutPath: Path, val stderrPath: Pa processBuilder.command(argv.map(_.toString): _*) processBuilder.redirectOutput(stdoutPath.toFile) processBuilder.redirectError(stderrPath.toFile) - val proccess = processBuilder.start() - proccess.waitFor() + val process = processBuilder.start() + process.waitFor() } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala index 3d81722c6..20c8c3d39 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala @@ -2,6 +2,8 @@ package cromwell.backend.sfs import java.nio.file.{FileSystem, Path, Paths} +import cats.instances.try_._ +import cats.syntax.functor._ import com.typesafe.config.Config import cromwell.backend.io.JobPaths import cromwell.core._ @@ -42,21 +44,26 @@ object SharedFileSystem { private def localizePathViaCopy(originalPath: File, executionPath: File): Try[Unit] = { executionPath.parent.createDirectories() val executionTmpPath = pathPlusSuffix(executionPath, ".tmp") - Try(originalPath.copyTo(executionTmpPath, overwrite = true).moveTo(executionPath, overwrite = true)) + Try(originalPath.copyTo(executionTmpPath, overwrite = true).moveTo(executionPath, overwrite = true)).void } private def localizePathViaHardLink(originalPath: File, executionPath: File): Try[Unit] = { executionPath.parent.createDirectories() // link.linkTo(target) returns target, // however we want to return the link, not the target, so map the result back to executionPath - Try(executionPath.linkTo(originalPath, symbolic = false)) map { _ => executionPath } + + // -Ywarn-value-discard + // Try(executionPath.linkTo(originalPath, symbolic = false)) map { _ => executionPath } + Try { executionPath.linkTo(originalPath, symbolic = false) } void } private def localizePathViaSymbolicLink(originalPath: File, executionPath: File): Try[Unit] = { if (originalPath.isDirectory) Failure(new UnsupportedOperationException("Cannot localize directory with symbolic links")) else { executionPath.parent.createDirectories() - Try(executionPath.linkTo(originalPath, symbolic = true)) map { _ => executionPath } + // -Ywarn-value-discard + // Try(executionPath.linkTo(originalPath, symbolic = true)) map { _ => executionPath } + Try { executionPath.linkTo(originalPath, symbolic = true) } void } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala index 6aa913658..6b5107b69 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala @@ -305,6 +305,7 @@ trait SharedFileSystemAsyncJobExecutionActor val stderr = pathPlusSuffix(jobPaths.stderr, "kill") val killer = new ProcessRunner(argv, stdout.path, stderr.path) killer.run() + () } def processReturnCode()(implicit ec: ExecutionContext): Future[ExecutionHandle] = { diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala index be4ab0ef9..62dc98b07 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala @@ -19,5 +19,9 @@ class SharedFileSystemCacheHitCopyingActor(override val jobDescriptor: BackendJo override protected def getPath(file: String) = Paths.get(file) - override protected def duplicate(source: Path, destination: Path) = sharedFileSystem.cacheCopy(source, destination) + override protected def duplicate(source: Path, destination: Path) = { + // -Ywarn-value-discard + sharedFileSystem.cacheCopy(source, destination) + () + } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala index 36b010547..54a370b39 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala @@ -7,8 +7,8 @@ import cromwell.backend.validation.RuntimeAttributesDefault import cromwell.backend.wfs.{DefaultWorkflowFileSystemProvider, WorkflowFileSystemProvider} import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor} import cromwell.core.{Dispatcher, WorkflowOptions} +import wdl4s.Call import wdl4s.values.WdlValue -import wdl4s.{Call, WdlExpression} import scala.concurrent.Future import scala.util.Try diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala index 17ea69fc7..81edb6f60 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala @@ -29,6 +29,7 @@ class ConfigHashingStrategySpec extends FlatSpec with Matchers with TableDrivenP override def beforeAll() = { file.write(steak) + () } private def randomName(): String = UUID.randomUUID().toString @@ -140,5 +141,6 @@ class ConfigHashingStrategySpec extends FlatSpec with Matchers with TableDrivenP override def afterAll() = { file.delete(true) md5File.delete(true) + () } } diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala index ac04a0bd9..64dbfa9d7 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala @@ -17,11 +17,11 @@ class SharedFileSystemInitializationActorSpec extends TestKitSuite("SharedFileSy val Timeout = 5.second.dilated val HelloWorld = - """ + s""" |task hello { | String addressee = "you" | command { - | echo "Hello ${addressee}!" + | echo "Hello $${addressee}!" | } | output { | String salutation = read_string(stdout()) diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemValidatedRuntimeAttributesBuilderSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemValidatedRuntimeAttributesBuilderSpec.scala index 2934eb040..c257ffa4d 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemValidatedRuntimeAttributesBuilderSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemValidatedRuntimeAttributesBuilderSpec.scala @@ -1,6 +1,5 @@ package cromwell.backend.sfs -import cromwell.backend.BackendSpec._ import cromwell.backend.RuntimeAttributeDefinition import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.backend.validation._ @@ -14,11 +13,11 @@ import wdl4s.values.{WdlBoolean, WdlInteger, WdlString, WdlValue} class SharedFileSystemValidatedRuntimeAttributesBuilderSpec extends WordSpecLike with Matchers with Mockito { val HelloWorld = - """ + s""" |task hello { | String addressee = "you" | command { - | echo "Hello ${addressee}!" + | echo "Hello $${addressee}!" | } | output { | String salutation = read_string(stdout()) @@ -33,7 +32,7 @@ class SharedFileSystemValidatedRuntimeAttributesBuilderSpec extends WordSpecLike """.stripMargin - val defaultRuntimeAttributes = Map( + val defaultRuntimeAttributes: Map[String, Any] = Map( DockerKey -> None, FailOnStderrKey -> false, ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(0))) @@ -170,6 +169,7 @@ class SharedFileSystemValidatedRuntimeAttributesBuilderSpec extends WordSpecLike failOnStderr should be(expectedRuntimeAttributes(FailOnStderrKey).asInstanceOf[Boolean]) continueOnReturnCode should be( expectedRuntimeAttributes(ContinueOnReturnCodeKey).asInstanceOf[ContinueOnReturnCode]) + () } private def assertRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue], exMsg: String, @@ -188,5 +188,6 @@ class SharedFileSystemValidatedRuntimeAttributesBuilderSpec extends WordSpecLike builder.build(addDefaultsToAttributes(runtimeAttributes), logger) } thrown.getMessage should include(exMsg) + () } } diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkClusterProcess.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkClusterProcess.scala index aa9575fd4..f731c990f 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkClusterProcess.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkClusterProcess.scala @@ -12,7 +12,6 @@ import scala.concurrent.{ExecutionContext, Future, Promise} import better.files._ import com.typesafe.scalalogging.Logger import org.slf4j.LoggerFactory -import spray.httpx.unmarshalling._ import scala.concurrent.duration._ import scala.util.{Failure, Success, Try} @@ -119,7 +118,9 @@ class SparkClusterProcess(implicit system: ActorSystem) extends SparkProcess override def completeMonitoringProcess(rcPath: Path, status: String, promise: Promise[Unit]) = { File(rcPath) write status - promise success Unit + val unitValue = () + promise success unitValue + () } def pollForJobStatus(subId: String): Future[SparkDriverStateQueryResponse] = { diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala index c03a2975d..2c4b5f94f 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala @@ -1,14 +1,14 @@ package cromwell.backend.impl.spark import akka.actor.{ActorRef, Props} +import cromwell.backend.impl.spark.SparkInitializationActor._ import cromwell.backend.validation.RuntimeAttributesDefault import cromwell.backend.validation.RuntimeAttributesKeys._ -import cromwell.backend.impl.spark.SparkInitializationActor._ import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor} import cromwell.core.WorkflowOptions +import wdl4s.Call import wdl4s.types.{WdlBooleanType, WdlIntegerType, WdlStringType} import wdl4s.values.WdlValue -import wdl4s.{Call, WdlExpression} import scala.concurrent.Future import scala.util.Try diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala index 3d34dda47..c793338eb 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala @@ -16,7 +16,6 @@ import wdl4s.util.TryUtil import scala.concurrent.{Future, Promise} import scala.sys.process.ProcessLogger import scala.util.{Failure, Success, Try} -import scala.language.postfixOps object SparkJobExecutionActor { val DefaultFileSystems = List(FileSystems.getDefault) @@ -145,7 +144,9 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, /** * Abort a running job. */ - override def abort(): Unit = Future.failed(new UnsupportedOperationException("SparkBackend currently doesn't support aborting jobs.")) + // -Ywarn-value-discard + // override def abort(): Unit = Future.failed(new UnsupportedOperationException("SparkBackend currently doesn't support aborting jobs.")) + override def abort(): Unit = throw new UnsupportedOperationException("SparkBackend currently doesn't support aborting jobs.") private def createExecutionFolderAndScript(): Unit = { @@ -178,11 +179,14 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, cmds.writeScript(sparkCommand, scriptPath, executionDir) File(scriptPath).addPermission(PosixFilePermission.OWNER_EXECUTE) + () } catch { case ex: Exception => log.error(ex, "Failed to prepare task: " + ex.getMessage) - executionResponse success FailedNonRetryableResponse(jobDescriptor.key, ex, None) + // -Ywarn-value-discard + // executionResponse success FailedNonRetryableResponse(jobDescriptor.key, ex, None) + () } } diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala index d04041f20..60f399218 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala @@ -8,7 +8,6 @@ import cromwell.core.PathFactory.EnhancedPath import scala.sys.process._ import better.files._ -import scala.language.postfixOps import scala.util.{Failure, Success, Try} object SparkCommands { diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkClusterProcessSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkClusterProcessSpec.scala index c2b003a53..f4dbbe1d4 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkClusterProcessSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkClusterProcessSpec.scala @@ -21,8 +21,6 @@ import cromwell.backend.impl.spark.SparkClusterProcess.{Failed, _} import org.scalatest.concurrent.ScalaFutures import spray.http._ import SparkClusterJsonProtocol._ -import spray.httpx.unmarshalling._ -import spray.httpx.SprayJsonSupport._ class SparkClusterProcessSpec extends TestKitSuite("SparkClusterProcess") with WordSpecLike diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala index fac5f389b..aa4400ac8 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala @@ -144,7 +144,10 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") Mockito.reset(sparkClusterProcess) } - override def afterAll(): Unit = system.terminate() + override def afterAll(): Unit = { + system.terminate() + () + } "executeTask method in cluster deploy mode " should { "return succeed response when the spark cluster process monitor method returns finished status" in { @@ -435,7 +438,10 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") } - private def cleanUpJob(jobPaths: JobPaths): Unit = File(jobPaths.workflowRoot).delete(true) + private def cleanUpJob(jobPaths: JobPaths): Unit = { + File(jobPaths.workflowRoot).delete(true) + () + } private def prepareJob(wdlSource: WdlSource = helloWorldWdl, runtimeString: String = passOnStderr, inputFiles: Option[Map[String, WdlValue]] = None, isCluster: Boolean = false): TestJobDescriptor = { val backendWorkflowDescriptor = buildWorkflowDescriptor(wdl = wdlSource, inputs = inputFiles.getOrElse(Map.empty), runtime = runtimeString) diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala index d166dca11..33724cb6d 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala @@ -1,15 +1,15 @@ package cromwell.backend.impl.spark -import cromwell.backend.{MemorySize, BackendWorkflowDescriptor} import cromwell.backend.validation.RuntimeAttributesKeys._ +import cromwell.backend.{BackendWorkflowDescriptor, MemorySize} import cromwell.core.{WorkflowId, WorkflowOptions} import org.scalatest.{Matchers, WordSpecLike} -import spray.json.{JsBoolean, JsNumber, JsObject, JsString, JsValue} +import spray.json.{JsBoolean, JsNumber, JsObject, JsValue} import wdl4s.WdlExpression._ import wdl4s.expression.NoFunctions import wdl4s.util.TryUtil -import wdl4s.{Call, WdlExpression, _} import wdl4s.values.WdlValue +import wdl4s.{Call, WdlExpression, _} class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { @@ -33,7 +33,7 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { val emptyWorkflowOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue])) - val staticDefaults = SparkRuntimeAttributes(1, MemorySize.parse("1 GB").get, None, "com.test.spark" , false) + val staticDefaults = SparkRuntimeAttributes(1, MemorySize.parse("1 GB").get, None, "com.test.spark" , failOnStderr = false) def workflowOptionsWithDefaultRA(defaults: Map[String, JsValue]) = { WorkflowOptions(JsObject(Map( @@ -88,7 +88,7 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { inputs: Map[String, WdlValue] = Map.empty, options: WorkflowOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue])), runtime: String) = { - new BackendWorkflowDescriptor( + BackendWorkflowDescriptor( WorkflowId.randomId(), NamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime.format("appMainClass", "com.test.spark"))), inputs, @@ -118,6 +118,7 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { } catch { case ex: RuntimeException => fail(s"Exception was not expected but received: ${ex.getMessage}") } + () } private def assertSparkRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue], exMsg: String): Unit = { @@ -127,5 +128,6 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { } catch { case ex: RuntimeException => assert(ex.getMessage.contains(exMsg)) } + () } } From 7b5274338f4bf238f28a6d48b82e0a2f451f0135 Mon Sep 17 00:00:00 2001 From: Ruchi Date: Fri, 7 Oct 2016 08:17:10 -0400 Subject: [PATCH 016/375] Print PID to stdout.background, SGE (#1520) * fix pid being added to stdout for sge + change file extension from background -> submit for SGE stdout/stderr --- .../cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala index 6b5107b69..3013b12af 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala @@ -231,7 +231,9 @@ trait SharedFileSystemAsyncJobExecutionActor * @return A process runner that will relatively quickly submit the script asynchronously. */ def makeProcessRunner(): ProcessRunner = { - new ProcessRunner(processArgs.argv, jobPaths.stdout, jobPaths.stderr) + val stdout = pathPlusSuffix(jobPaths.stdout, "submit") + val stderr = pathPlusSuffix(jobPaths.stderr, "submit") + new ProcessRunner(processArgs.argv, stdout.path, stderr.path) } /** From 89faa14ef2e9e94ad6e290a833c18b719014f92a Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Fri, 7 Oct 2016 16:08:29 -0400 Subject: [PATCH 017/375] Firstly, you must find... another shrubbery! Closes #1211 --- .../main/scala/cromwell/backend/io/WorkflowPaths.scala | 4 ++-- .../backend/wfs/WorkflowFileSystemProvider.scala | 6 +++--- .../scala/cromwell/backend/io/WorkflowPathsSpec.scala | 1 + .../main/scala/cromwell/core/DockerCredentials.scala | 14 +++++++------- .../scala/cromwell/core/logging/WorkflowLogger.scala | 10 +++++----- .../scala/cromwell/database/slick/SlickDatabase.scala | 10 +++++----- .../main/scala/cromwell/engine/EngineFilesystems.scala | 4 ++-- .../cromwell/engine/backend/BackendConfiguration.scala | 5 ++--- .../engine/workflow/WorkflowManagerActor.scala | 11 +++++------ .../lifecycle/MaterializeWorkflowDescriptorActor.scala | 8 ++++---- .../lifecycle/execution/WorkflowExecutionActor.scala | 4 ++-- .../main/scala/cromwell/server/CromwellRootActor.scala | 5 ++--- .../main/scala/cromwell/server/CromwellServer.scala | 4 ++-- .../filesystems/gcs/GcsFileSystemProvider.scala | 18 +++++++++++++----- project/Dependencies.scala | 3 ++- project/Settings.scala | 1 + .../scala/cromwell/services/ServiceRegistryActor.scala | 8 +++----- .../main/scala/cromwell/services/ServicesStore.scala | 4 ++-- .../services/metadata/impl/MetadataServiceActor.scala | 5 ++--- .../cromwell/backend/impl/jes/JesAttributes.scala | 4 ++-- .../config/ConfigBackendLifecycleActorFactory.scala | 9 +++++---- .../impl/sfs/config/ConfigHashingStrategy.scala | 6 +++--- .../backend/impl/sfs/config/ConfigWdlNamespace.scala | 12 ++++++------ .../backend/sfs/GcsWorkflowFileSystemProvider.scala | 4 ++-- .../sfs/SharedFileSystemJobCachingActorHelper.scala | 5 +++-- 25 files changed, 86 insertions(+), 79 deletions(-) diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala index d05797307..23bdae992 100644 --- a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala +++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala @@ -5,14 +5,14 @@ import java.nio.file.{FileSystem, FileSystems, Path, Paths} import com.typesafe.config.Config import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} import cromwell.core.PathFactory -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ object WorkflowPaths{ val DockerRoot = Paths.get("/root") } class WorkflowPaths(workflowDescriptor: BackendWorkflowDescriptor, config: Config, val fileSystems: List[FileSystem] = List(FileSystems.getDefault)) extends PathFactory { - val executionRoot = Paths.get(config.getStringOr("root", "cromwell-executions")).toAbsolutePath + val executionRoot = Paths.get(config.as[Option[String]]("root").getOrElse("cromwell-executions")).toAbsolutePath private def workflowPathBuilder(root: Path) = { root.resolve(workflowDescriptor.workflowNamespace.workflow.unqualifiedName) diff --git a/backend/src/main/scala/cromwell/backend/wfs/WorkflowFileSystemProvider.scala b/backend/src/main/scala/cromwell/backend/wfs/WorkflowFileSystemProvider.scala index 720605482..de8272473 100644 --- a/backend/src/main/scala/cromwell/backend/wfs/WorkflowFileSystemProvider.scala +++ b/backend/src/main/scala/cromwell/backend/wfs/WorkflowFileSystemProvider.scala @@ -2,11 +2,11 @@ package cromwell.backend.wfs import java.nio.file.FileSystem -import com.typesafe.config.Config +import com.typesafe.config.{Config, ConfigFactory} import cromwell.backend.io.WorkflowPaths import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor} import cromwell.core.WorkflowOptions -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ import scala.concurrent.ExecutionContext @@ -16,7 +16,7 @@ object WorkflowFileSystemProvider { providers: Traversable[WorkflowFileSystemProvider], fileSystemExecutionContext: ExecutionContext): WorkflowPaths = { val backendConfig = configurationDescriptor.backendConfig - val fileSystemConfig = backendConfig.getConfigOr("filesystems") + val fileSystemConfig = backendConfig.as[Option[Config]]("filesystems").getOrElse(ConfigFactory.empty()) val globalConfig = configurationDescriptor.globalConfig val params = WorkflowFileSystemProviderParams(fileSystemConfig, globalConfig, workflowDescriptor.workflowOptions, fileSystemExecutionContext) diff --git a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala index ff56ebaeb..bfae5930d 100644 --- a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala +++ b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala @@ -13,6 +13,7 @@ class WorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec with Moc val backendConfig = mock[Config] "WorkflowPaths" should "provide correct paths for a workflow" in { + when(backendConfig.hasPath(any[String])).thenReturn(true) when(backendConfig.getString(any[String])).thenReturn("local-cromwell-executions") // This is the folder defined in the config as the execution root dir val wd = buildWorkflowDescriptor(TestWorkflows.HelloWorld) val workflowPaths = new WorkflowPaths(wd, backendConfig) diff --git a/core/src/main/scala/cromwell/core/DockerCredentials.scala b/core/src/main/scala/cromwell/core/DockerCredentials.scala index a63076e2d..cfe7be01f 100644 --- a/core/src/main/scala/cromwell/core/DockerCredentials.scala +++ b/core/src/main/scala/cromwell/core/DockerCredentials.scala @@ -16,23 +16,23 @@ case class DockerConfiguration(dockerCredentials: Option[DockerCredentials], doc * Singleton encapsulating a DockerConf instance. */ object DockerConfiguration { - import lenthall.config.ScalaConfig._ private val dockerKeys = Set("account", "token") def build(config: Config) = { + import net.ceedubs.ficus.Ficus._ val dockerConf: Option[DockerCredentials] = for { - dockerConf <- config.getConfigOption("dockerhub") + dockerConf <- config.as[Option[Config]]("dockerhub") _ = dockerConf.warnNotRecognized(dockerKeys, "dockerhub") account <- dockerConf.validateString("account").toOption token <- dockerConf.validateString("token").toOption - } yield new DockerCredentials(account, token) + } yield DockerCredentials(account, token) val dockerHubConf = { - new DockerHubConfiguration( - namespace = config.getStringOr("docker.hub.namespace", "docker.io"), - v1Registry = config.getStringOr("docker.hub.v1Registry", "index.docker.io"), - v2Registry = config.getStringOr("docker.hub.v2Registry", "registry-1.docker.io") + DockerHubConfiguration( + namespace = config.as[Option[String]]("docker.hub.namespace").getOrElse("docker.io"), + v1Registry = config.as[Option[String]]("docker.hub.v1Registry").getOrElse("index.docker.io"), + v2Registry = config.as[Option[String]]("docker.hub.v2Registry").getOrElse("registry-1.docker.io") ) } new DockerConfiguration(dockerConf, dockerHubConf) diff --git a/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala b/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala index b36dd58a9..b79e9f7b6 100644 --- a/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala +++ b/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala @@ -8,9 +8,9 @@ import ch.qos.logback.classic.encoder.PatternLayoutEncoder import ch.qos.logback.classic.spi.ILoggingEvent import ch.qos.logback.classic.{Level, LoggerContext} import ch.qos.logback.core.FileAppender -import com.typesafe.config.ConfigFactory +import com.typesafe.config.{Config, ConfigFactory} import cromwell.core.WorkflowId -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ import org.slf4j.helpers.NOPLogger import org.slf4j.{Logger, LoggerFactory} @@ -71,9 +71,9 @@ object WorkflowLogger { val workflowLogConfiguration: Option[WorkflowLogConfiguration] = { for { - workflowConfig <- conf.getConfigOption("workflow-options") - dir <- workflowConfig.getStringOption("workflow-log-dir") if !dir.isEmpty - temporary <- workflowConfig.getBooleanOption("workflow-log-temporary") orElse Option(true) + workflowConfig <- conf.as[Option[Config]]("workflow-options") + dir <- workflowConfig.as[Option[String]]("workflow-log-dir") if !dir.isEmpty + temporary <- workflowConfig.as[Option[Boolean]]("workflow-log-temporary") orElse Option(true) } yield WorkflowLogConfiguration(Paths.get(dir).toAbsolutePath, temporary) } diff --git a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala index fbfbece8a..80a4413fd 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala @@ -6,7 +6,7 @@ import java.util.concurrent.{ExecutorService, Executors} import com.typesafe.config.Config import cromwell.database.slick.tables.DataAccessComponent import cromwell.database.sql.SqlDatabase -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ import org.slf4j.LoggerFactory import slick.backend.DatabaseConfig import slick.driver.JdbcProfile @@ -38,7 +38,7 @@ object SlickDatabase { // generate unique schema instances that don't conflict. // // Otherwise, create one DataAccess and hold on to the reference. - if (slickDatabase.databaseConfig.getBooleanOr("slick.createSchema", default = true)) { + if (slickDatabase.databaseConfig.as[Option[Boolean]]("slick.createSchema").getOrElse(true)) { import slickDatabase.dataAccess.driver.api._ Await.result(slickDatabase.database.run(slickDatabase.dataAccess.schema.create), Duration.Inf) } @@ -89,9 +89,9 @@ class SlickDatabase(override val originalDatabaseConfig: Config) extends SqlData * Reuses the error reporter from the database's executionContext. */ private val actionThreadPool: ExecutorService = { - val dbNumThreads = databaseConfig.getIntOr("db.numThreads", 20) - val dbMaximumPoolSize = databaseConfig.getIntOr("db.maxConnections", dbNumThreads * 5) - val actionThreadPoolSize = databaseConfig.getIntOr("actionThreadPoolSize", dbNumThreads) min dbMaximumPoolSize + val dbNumThreads = databaseConfig.as[Option[Int]]("db.numThreads").getOrElse(20) + val dbMaximumPoolSize = databaseConfig.as[Option[Int]]("db.maxConnections").getOrElse(dbNumThreads * 5) + val actionThreadPoolSize = databaseConfig.as[Option[Int]]("actionThreadPoolSize").getOrElse(dbNumThreads) min dbMaximumPoolSize Executors.newFixedThreadPool(actionThreadPoolSize) } diff --git a/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala b/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala index e18bd3dae..ab9cbceac 100644 --- a/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala +++ b/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala @@ -7,8 +7,8 @@ import com.typesafe.config.ConfigFactory import cromwell.core.WorkflowOptions import cromwell.engine.backend.EnhancedWorkflowOptions._ import cromwell.filesystems.gcs.{GcsFileSystem, GcsFileSystemProvider, GoogleConfiguration} -import lenthall.config.ScalaConfig._ import lenthall.exception.MessageAggregation +import net.ceedubs.ficus.Ficus._ import scala.concurrent.ExecutionContext @@ -16,7 +16,7 @@ object EngineFilesystems { private val config = ConfigFactory.load private val googleConf: GoogleConfiguration = GoogleConfiguration(config) - private val googleAuthMode = config.getStringOption("engine.filesystems.gcs.auth") map { confMode => + private val googleAuthMode = config.as[Option[String]]("engine.filesystems.gcs.auth") map { confMode => googleConf.auth(confMode) match { case Valid(mode) => mode case Invalid(errors) => throw new RuntimeException() with MessageAggregation { diff --git a/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala b/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala index fcd6da9ce..49248312e 100644 --- a/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala +++ b/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala @@ -2,8 +2,7 @@ package cromwell.engine.backend import com.typesafe.config.{Config, ConfigFactory} import cromwell.backend.{BackendConfigurationDescriptor, BackendLifecycleActorFactory} -import lenthall.config.ScalaConfig._ - +import net.ceedubs.ficus.Ficus._ import scala.collection.JavaConverters._ import scala.util.{Failure, Success, Try} @@ -29,7 +28,7 @@ object BackendConfiguration { BackendConfigurationEntry( backendName, entry.getString("actor-factory"), - entry.getConfigOr("config") + entry.as[Option[Config]]("config").getOrElse(ConfigFactory.empty("empty")) ) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala index 1fa7fdb0e..59b105dd3 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala @@ -14,8 +14,7 @@ import cromwell.engine.workflow.workflowstore.{WorkflowStoreActor, WorkflowStore import cromwell.jobstore.JobStoreActor.{JobStoreWriteFailure, JobStoreWriteSuccess, RegisterWorkflowCompleted} import cromwell.services.metadata.MetadataService._ import cromwell.webservice.EngineStatsActor -import lenthall.config.ScalaConfig.EnhancedScalaConfig - +import net.ceedubs.ficus.Ficus._ import scala.concurrent.duration._ import scala.concurrent.{Await, Promise} @@ -94,9 +93,9 @@ class WorkflowManagerActor(config: Config, jobTokenDispenserActor: ActorRef) = this( ConfigFactory.load, workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) - private val maxWorkflowsRunning = config.getConfig("system").getIntOr("max-concurrent-workflows", default=DefaultMaxWorkflowsToRun) - private val maxWorkflowsToLaunch = config.getConfig("system").getIntOr("max-workflow-launch-count", default=DefaultMaxWorkflowsToLaunch) - private val newWorkflowPollRate = config.getConfig("system").getIntOr("new-workflow-poll-rate", default=DefaultNewWorkflowPollRate).seconds + private val maxWorkflowsRunning = config.getConfig("system").as[Option[Int]]("max-concurrent-workflows").getOrElse(DefaultMaxWorkflowsToRun) + private val maxWorkflowsToLaunch = config.getConfig("system").as[Option[Int]]("max-workflow-launch-count").getOrElse(DefaultMaxWorkflowsToLaunch) + private val newWorkflowPollRate = config.getConfig("system").as[Option[Int]]("new-workflow-poll-rate").getOrElse(DefaultNewWorkflowPollRate).seconds private val logger = Logging(context.system, this) private val tag = self.path.name @@ -114,7 +113,7 @@ class WorkflowManagerActor(config: Config, private def addShutdownHook() = { // Only abort jobs on SIGINT if the config explicitly sets backend.abortJobsOnTerminate = true. val abortJobsOnTerminate = - config.getConfig("system").getBooleanOr("abort-jobs-on-terminate", default = false) + config.getConfig("system").as[Option[Boolean]]("abort-jobs-on-terminate").getOrElse(false) if (abortJobsOnTerminate) { sys.addShutdownHook { diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala index 67e93faf2..1a58b849a 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala @@ -21,8 +21,8 @@ import cromwell.engine.backend.CromwellBackends import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.{MaterializeWorkflowDescriptorActorData, MaterializeWorkflowDescriptorActorState} import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} -import lenthall.config.ScalaConfig.EnhancedScalaConfig import cromwell.core.ErrorOr._ +import net.ceedubs.ficus.Ficus._ import spray.json.{JsObject, _} import wdl4s._ import wdl4s.expression.NoFunctions @@ -89,7 +89,7 @@ object MaterializeWorkflowDescriptorActor { } } - val enabled = conf.getBooleanOption("call-caching.enabled").getOrElse(false) + val enabled = conf.as[Option[Boolean]]("call-caching.enabled").getOrElse(false) if (enabled) { val readFromCache = readOptionalOption(ReadFromCache) val writeToCache = readOptionalOption(WriteToCache) @@ -184,7 +184,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor workflowOptions: WorkflowOptions, conf: Config, engineFilesystems: List[FileSystem]): ErrorOr[EngineWorkflowDescriptor] = { - val defaultBackendName = conf.getStringOption("backend.default") + val defaultBackendName = conf.as[Option[String]]("backend.default") val rawInputsValidation = validateRawInputs(sourceFiles.inputsJson) val failureModeValidation = validateWorkflowFailureMode(workflowOptions, conf) val backendAssignmentsValidation = validateBackendAssignments(namespace.workflow.calls, workflowOptions, defaultBackendName) @@ -332,7 +332,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor private def validateWorkflowFailureMode(workflowOptions: WorkflowOptions, conf: Config): ErrorOr[WorkflowFailureMode] = { val modeString: Try[String] = workflowOptions.get(WorkflowOptions.WorkflowFailureMode) match { case Success(x) => Success(x) - case Failure(_: OptionNotFoundException) => Success(conf.getStringOption("workflow-options.workflow-failure-mode") getOrElse DefaultWorkflowFailureMode) + case Failure(_: OptionNotFoundException) => Success(conf.as[Option[String]]("workflow-options.workflow-failure-mode") getOrElse DefaultWorkflowFailureMode) case Failure(t) => Failure(t) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index a3ef978ec..c40815057 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -27,6 +27,7 @@ import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata._ import cromwell.webservice.EngineStatsActor import lenthall.exception.ThrowableAggregation +import net.ceedubs.ficus.Ficus._ import wdl4s.types.WdlArrayType import wdl4s.util.TryUtil import wdl4s.values.{WdlArray, WdlValue} @@ -241,7 +242,6 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, extends LoggingFSM[WorkflowExecutionActorState, WorkflowExecutionActorData] with WorkflowLogging { import WorkflowExecutionActor._ - import lenthall.config.ScalaConfig._ override def supervisorStrategy = AllForOneStrategy() { case ex: ActorInitializationException => @@ -256,7 +256,7 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, implicit val ec = context.dispatcher - val MaxRetries = ConfigFactory.load().getIntOption("system.max-retries") match { + val MaxRetries = ConfigFactory.load().as[Option[Int]]("system.max-retries") match { case Some(value) => value case None => workflowLogger.warn(s"Failed to load the max-retries value from the configuration. Defaulting back to a value of '$DefaultMaxRetriesFallbackValue'.") diff --git a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala index 9e8c04c17..b3c64b67f 100644 --- a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala +++ b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala @@ -12,8 +12,7 @@ import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor import cromwell.engine.workflow.workflowstore.{SqlWorkflowStore, WorkflowStore, WorkflowStoreActor} import cromwell.jobstore.{JobStore, JobStoreActor, SqlJobStore} import cromwell.services.{ServiceRegistryActor, SingletonServicesStore} -import lenthall.config.ScalaConfig.EnhancedScalaConfig - +import net.ceedubs.ficus.Ficus._ /** * An actor which serves as the lord protector for the rest of Cromwell, allowing us to have more fine grain * control on top level supervision, etc. @@ -31,7 +30,7 @@ import lenthall.config.ScalaConfig.EnhancedScalaConfig private val config = ConfigFactory.load() lazy val serviceRegistryActor: ActorRef = context.actorOf(ServiceRegistryActor.props(config), "ServiceRegistryActor") - lazy val numberOfWorkflowLogCopyWorkers = config.getConfig("system").getIntOr("number-of-workflow-log-copy-workers", default=DefaultNumberOfWorkflowLogCopyWorkers) + lazy val numberOfWorkflowLogCopyWorkers = config.getConfig("system").as[Option[Int]]("number-of-workflow-log-copy-workers").getOrElse(DefaultNumberOfWorkflowLogCopyWorkers) lazy val workflowLogCopyRouter: ActorRef = context.actorOf(RoundRobinPool(numberOfWorkflowLogCopyWorkers) .withSupervisorStrategy(CopyWorkflowLogsActor.strategy) diff --git a/engine/src/main/scala/cromwell/server/CromwellServer.scala b/engine/src/main/scala/cromwell/server/CromwellServer.scala index 0dd8ac59f..dca31a625 100644 --- a/engine/src/main/scala/cromwell/server/CromwellServer.scala +++ b/engine/src/main/scala/cromwell/server/CromwellServer.scala @@ -7,9 +7,9 @@ import akka.util.Timeout import com.typesafe.config.Config import cromwell.webservice.WorkflowJsonSupport._ import cromwell.webservice.{APIResponse, CromwellApiService, SwaggerService} -import lenthall.config.ScalaConfig._ import lenthall.spray.SprayCanHttpService._ import lenthall.spray.WrappedRoute._ +import net.ceedubs.ficus.Ficus._ import spray.http.{ContentType, MediaTypes, _} import spray.json._ @@ -56,7 +56,7 @@ class CromwellServerActor(config: Config) extends CromwellRootActor with Cromwel override def actorRefFactory = context override def receive = handleTimeouts orElse runRoute(possibleRoutes) - val possibleRoutes = workflowRoutes.wrapped("api", config.getBooleanOr("api.routeUnwrapped")) ~ swaggerUiResourceRoute + val possibleRoutes = workflowRoutes.wrapped("api", config.as[Option[Boolean]]("api.routeUnwrapped").getOrElse(false)) ~ swaggerUiResourceRoute val timeoutError = APIResponse.error(new TimeoutException("The server was not able to produce a timely response to your request.")).toJson.prettyPrint def handleTimeouts: Receive = { diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala index fd40a5403..845ec29ef 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala @@ -19,8 +19,9 @@ import com.google.api.services.storage.Storage import com.google.api.services.storage.model.StorageObject import com.google.cloud.hadoop.gcsio.{GoogleCloudStorageReadChannel, GoogleCloudStorageWriteChannel, ObjectWriteConditions} import com.google.cloud.hadoop.util.{ApiErrorExtractor, AsyncWriteChannelOptions, ClientRequestHelper} -import com.typesafe.config.ConfigFactory -import lenthall.config.ScalaConfig.EnhancedScalaConfig +import com.typesafe.config.{Config, ConfigFactory, ConfigMemorySize} +import net.ceedubs.ficus.Ficus._ +import net.ceedubs.ficus.readers.ValueReader import scala.annotation.tailrec import scala.collection.JavaConverters._ @@ -53,6 +54,11 @@ object GcsFileSystemProvider { withRetry(f, retries - 1) case Failure(ex) => throw ex } + + // TODO refactor as part of Ficus and submit a PR + implicit val configMemorySizeValueReader: ValueReader[ConfigMemorySize] = new ValueReader[ConfigMemorySize] { + override def read(config: Config, path: String): ConfigMemorySize = config.getMemorySize(path) + } } /** @@ -140,8 +146,10 @@ class GcsFileSystemProvider private[gcs](storageClient: Try[Storage], val execut - com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.setUploadBufferSize - com.google.api.client.googleapis.media.MediaHttpUploader.setContentAndHeadersOnCurrentRequest */ - private[this] lazy val uploadBufferBytes = config.getBytesOr("google.upload-buffer-bytes", - MediaHttpUploader.MINIMUM_CHUNK_SIZE).toInt + private[this] lazy val uploadBufferBytes = { + val configBytes = config.as[Option[ConfigMemorySize]]("google.upload-buffer-bytes").map(_.toBytes.toInt) + configBytes.getOrElse(MediaHttpUploader.MINIMUM_CHUNK_SIZE) + } /** * Overrides the default implementation to provide a writable channel (which newByteChannel doesn't). @@ -234,7 +242,7 @@ class GcsFileSystemProvider private[gcs](storageClient: Try[Storage], val execut override def isHidden(path: Path): Boolean = throw new NotImplementedError() - private[this] lazy val maxResults = config.getIntOr("google.list-max-results", 1000).toLong + private[this] lazy val maxResults = config.as[Option[Int]]("google.list-max-results").getOrElse(1000).toLong private def list(gcsDir: NioGcsPath) = { val listRequest = client.objects().list(gcsDir.bucket).setMaxResults(maxResults) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 5b632d157..ca5420d5e 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,7 +1,7 @@ import sbt._ object Dependencies { - lazy val lenthallV = "0.19-98b3f3a-SNAPSHOT" + lazy val lenthallV = "0.19-882a763-SNAPSHOT" lazy val wdl4sV = "0.6-2964173-SNAPSHOT" lazy val sprayV = "1.3.3" /* @@ -24,6 +24,7 @@ object Dependencies { "org.broadinstitute" %% "lenthall" % lenthallV, "org.typelevel" %% "cats" % catsV, "com.github.benhutchison" %% "mouse" % "0.5", + "com.iheart" %% "ficus" % "1.3.0", "org.scalatest" %% "scalatest" % "3.0.0" % Test, "org.specs2" %% "specs2" % "3.7" % Test ) diff --git a/project/Settings.scala b/project/Settings.scala index 627ac080c..4a092ce68 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -12,6 +12,7 @@ import sbtdocker.DockerPlugin.autoImport._ object Settings { val commonResolvers = List( + Resolver.jcenterRepo, "Broad Artifactory Releases" at "https://artifactory.broadinstitute.org/artifactory/libs-release/", "Broad Artifactory Snapshots" at "https://artifactory.broadinstitute.org/artifactory/libs-snapshot/" ) diff --git a/services/src/main/scala/cromwell/services/ServiceRegistryActor.scala b/services/src/main/scala/cromwell/services/ServiceRegistryActor.scala index 357f5d33f..e58cd5c05 100644 --- a/services/src/main/scala/cromwell/services/ServiceRegistryActor.scala +++ b/services/src/main/scala/cromwell/services/ServiceRegistryActor.scala @@ -3,8 +3,7 @@ package cromwell.services import akka.actor.SupervisorStrategy.Escalate import akka.actor.{Actor, ActorInitializationException, ActorLogging, ActorRef, OneForOneStrategy, Props} import com.typesafe.config.{Config, ConfigFactory, ConfigObject} -import lenthall.config.ScalaConfig._ - +import net.ceedubs.ficus.Ficus._ import scala.collection.JavaConverters._ object ServiceRegistryActor { @@ -31,9 +30,8 @@ object ServiceRegistryActor { } private def serviceProps(serviceName: String, globalConfig: Config, serviceStanza: Config): Props = { - val serviceConfigStanza = serviceStanza.getConfigOr("config", ConfigFactory.parseString("")) - val className = serviceStanza.getStringOr( - "class", + val serviceConfigStanza = serviceStanza.as[Option[Config]]("config").getOrElse(ConfigFactory.parseString("")) + val className = serviceStanza.as[Option[String]]("class").getOrElse( throw new IllegalArgumentException(s"Invalid configuration for service $serviceName: missing 'class' definition") ) diff --git a/services/src/main/scala/cromwell/services/ServicesStore.scala b/services/src/main/scala/cromwell/services/ServicesStore.scala index a73c3f3ec..c00aad09c 100644 --- a/services/src/main/scala/cromwell/services/ServicesStore.scala +++ b/services/src/main/scala/cromwell/services/ServicesStore.scala @@ -4,7 +4,7 @@ import com.typesafe.config.ConfigFactory import cromwell.database.migration.liquibase.LiquibaseUtils import cromwell.database.slick.SlickDatabase import cromwell.database.sql.SqlDatabase -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ import org.slf4j.LoggerFactory trait ServicesStore { @@ -15,7 +15,7 @@ object ServicesStore { implicit class EnhancedSqlDatabase[A <: SqlDatabase](val sqlDatabase: A) extends AnyVal { def initialized: A = { - if (sqlDatabase.databaseConfig.getBooleanOr("liquibase.updateSchema", default = true)) { + if (sqlDatabase.databaseConfig.as[Option[Boolean]]("liquibase.updateSchema").getOrElse(true)) { sqlDatabase withConnection LiquibaseUtils.updateSchema } sqlDatabase diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala index ab9e20f29..a92b92fbd 100644 --- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala +++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala @@ -9,15 +9,14 @@ import cromwell.services.SingletonServicesStore import cromwell.services.metadata.MetadataService.{PutMetadataAction, ReadAction, RefreshSummary, ValidateWorkflowIdAndExecute} import cromwell.services.metadata.impl.MetadataServiceActor._ import cromwell.services.metadata.impl.MetadataSummaryRefreshActor.{MetadataSummaryFailure, MetadataSummarySuccess, SummarizeMetadata} -import lenthall.config.ScalaConfig._ - +import net.ceedubs.ficus.Ficus._ import scala.concurrent.duration.{Duration, FiniteDuration} import scala.util.{Failure, Success, Try} object MetadataServiceActor { val MetadataSummaryRefreshInterval: Option[FiniteDuration] = { - val duration = Duration(ConfigFactory.load().getStringOr("services.MetadataService.metadata-summary-refresh-interval", "2 seconds")) + val duration = Duration(ConfigFactory.load().as[Option[String]]("services.MetadataService.metadata-summary-refresh-interval").getOrElse("2 seconds")) if (duration.isFinite()) Option(duration.asInstanceOf[FiniteDuration]) else None } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala index 3c72ca7fc..2e1d00b0e 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala @@ -9,8 +9,8 @@ import com.typesafe.config.Config import cromwell.backend.impl.jes.JesImplicits.GoogleAuthWorkflowOptions import cromwell.core.WorkflowOptions import cromwell.filesystems.gcs.{GoogleAuthMode, GoogleConfiguration} -import lenthall.config.ScalaConfig._ import lenthall.config.ValidatedConfig._ +import net.ceedubs.ficus.Ficus._ import cromwell.core.ErrorOr._ import wdl4s.ExceptionWithErrors @@ -46,7 +46,7 @@ object JesAttributes { val project: ValidatedNel[String, String] = backendConfig.validateString("project") val executionBucket: ValidatedNel[String, String] = backendConfig.validateString("root") val endpointUrl: ErrorOr[URL] = backendConfig.validateURL("genomics.endpoint-url") - val maxPollingInterval: Int = backendConfig.getIntOption("maximum-polling-interval").getOrElse(600) + val maxPollingInterval: Int = backendConfig.as[Option[Int]]("maximum-polling-interval").getOrElse(600) val genomicsAuthName: ErrorOr[String] = backendConfig.validateString("genomics.auth") val gcsFilesystemAuthName: ErrorOr[String] = backendConfig.validateString("filesystems.gcs.auth") diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala index 93ad1fe3a..0debe3aee 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala @@ -1,11 +1,12 @@ package cromwell.backend.impl.sfs.config +import com.typesafe.config.Config import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction import cromwell.backend.impl.sfs.config.ConfigConstants._ import cromwell.backend.sfs._ import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, RuntimeAttributeDefinition} import cromwell.core.JobExecutionToken.JobExecutionTokenType -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ import org.slf4j.LoggerFactory /** @@ -18,13 +19,13 @@ class ConfigBackendLifecycleActorFactory(name: String, val configurationDescript lazy val logger = LoggerFactory.getLogger(getClass) lazy val hashingStrategy = { - configurationDescriptor.backendConfig.getConfigOption("filesystems.local.caching") map ConfigHashingStrategy.apply getOrElse ConfigHashingStrategy.defaultStrategy + configurationDescriptor.backendConfig.as[Option[Config]]("filesystems.local.caching") map ConfigHashingStrategy.apply getOrElse ConfigHashingStrategy.defaultStrategy } override def initializationActorClass = classOf[ConfigInitializationActor] override def asyncJobExecutionActorClass: Class[_ <: ConfigAsyncJobExecutionActor] = { - val runInBackground = configurationDescriptor.backendConfig.getBooleanOr(RunInBackgroundConfig, default = false) + val runInBackground = configurationDescriptor.backendConfig.as[Option[Boolean]](RunInBackgroundConfig).getOrElse(false) if (runInBackground) classOf[BackgroundConfigAsyncJobExecutionActor] else @@ -47,7 +48,7 @@ class ConfigBackendLifecycleActorFactory(name: String, val configurationDescript override lazy val fileHashingActorCount: Int = 5 override val jobExecutionTokenType: JobExecutionTokenType = { - val concurrentJobLimit = configurationDescriptor.backendConfig.getIntOption("concurrent-job-limit") + val concurrentJobLimit = configurationDescriptor.backendConfig.as[Option[Int]]("concurrent-job-limit") JobExecutionTokenType(name, concurrentJobLimit) } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala index 21719550e..6261e65c9 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala @@ -6,7 +6,7 @@ import com.typesafe.config.Config import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest import cromwell.util.TryWithResource._ import cromwell.util.FileUtil._ -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ import org.apache.commons.codec.digest.DigestUtils import org.slf4j.LoggerFactory @@ -17,9 +17,9 @@ object ConfigHashingStrategy { val defaultStrategy = HashFileStrategy(false) def apply(hashingConfig: Config): ConfigHashingStrategy = { - val checkSiblingMd5 = hashingConfig.getBooleanOr("check-sibling-md5", default = false) + val checkSiblingMd5 = hashingConfig.as[Option[Boolean]]("check-sibling-md5").getOrElse(false) - hashingConfig.getStringOr("hashing-strategy", "file") match { + hashingConfig.as[Option[String]]("hashing-strategy").getOrElse("file") match { case "path" => HashPathStrategy(checkSiblingMd5) case "file" => HashFileStrategy(checkSiblingMd5) case what => diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala index 7b6909edb..cb56e35a0 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala @@ -2,7 +2,7 @@ package cromwell.backend.impl.sfs.config import com.typesafe.config.Config import cromwell.backend.impl.sfs.config.ConfigConstants._ -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ import wdl4s._ /** @@ -14,20 +14,20 @@ class ConfigWdlNamespace(backendConfig: Config) { import ConfigWdlNamespace._ - private val configRuntimeAttributes = backendConfig.getStringOr(RuntimeAttributesConfig) + private val configRuntimeAttributes = backendConfig.as[Option[String]](RuntimeAttributesConfig).getOrElse("") - private val submitCommandOption = backendConfig.getStringOption(SubmitConfig) + private val submitCommandOption = backendConfig.as[Option[String]](SubmitConfig) private val submitSourceOption = submitCommandOption.map(makeWdlSource( SubmitTask, _, submitRuntimeAttributes + configRuntimeAttributes)) - private val submitDockerCommandOption = backendConfig.getStringOption(SubmitDockerConfig) + private val submitDockerCommandOption = backendConfig.as[Option[String]](SubmitDockerConfig) private val submitDockerSourceOption = submitDockerCommandOption.map(makeWdlSource( SubmitDockerTask, _, submitRuntimeAttributes + submitDockerRuntimeAttributes + configRuntimeAttributes)) - private val killCommandOption = backendConfig.getStringOption(KillConfig) + private val killCommandOption = backendConfig.as[Option[String]](KillConfig) private val killSourceOption = killCommandOption.map(makeWdlSource(KillTask, _, jobIdRuntimeAttributes)) - private val checkAliveCommandOption = backendConfig.getStringOption(CheckAliveConfig) + private val checkAliveCommandOption = backendConfig.as[Option[String]](CheckAliveConfig) private val checkAliveSourceOption = checkAliveCommandOption.map(makeWdlSource( CheckAliveTask, _, jobIdRuntimeAttributes)) diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala index 4cde3b7c9..d96140014 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala @@ -4,14 +4,14 @@ import cats.data.Validated.{Invalid, Valid} import cromwell.backend.wfs.{WorkflowFileSystemProvider, WorkflowFileSystemProviderParams} import cromwell.filesystems.gcs.GoogleAuthMode.GoogleAuthOptions import cromwell.filesystems.gcs.{GcsFileSystem, GcsFileSystemProvider, GoogleConfiguration} -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ import wdl4s.ValidationException import scala.util.Try object GcsWorkflowFileSystemProvider extends WorkflowFileSystemProvider { override def fileSystemOption(params: WorkflowFileSystemProviderParams): Option[GcsFileSystem] = { - params.fileSystemConfig.getStringOption("gcs.auth") map gcsFileSystem(params) + params.fileSystemConfig.as[Option[String]]("gcs.auth") map gcsFileSystem(params) } private def gcsFileSystem(params: WorkflowFileSystemProviderParams)(gcsAuthName: String): GcsFileSystem = { diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala index a423d6cb7..d9d4f4213 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala @@ -1,12 +1,13 @@ package cromwell.backend.sfs import akka.actor.{Actor, ActorRef} +import com.typesafe.config.{Config, ConfigFactory} import cromwell.backend.BackendInitializationData import cromwell.backend.callcaching.JobCachingActorHelper import cromwell.backend.io.JobPaths import cromwell.backend.validation.{RuntimeAttributesValidation, ValidatedRuntimeAttributes} import cromwell.core.logging.JobLogging -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ trait SharedFileSystemJobCachingActorHelper extends JobCachingActorHelper { this: Actor with JobLogging => @@ -37,7 +38,7 @@ trait SharedFileSystemJobCachingActorHelper extends JobCachingActorHelper { lazy val sharedFileSystem = new SharedFileSystem { override lazy val sharedFileSystemConfig = { - configurationDescriptor.backendConfig.getConfigOr("filesystems.local") + configurationDescriptor.backendConfig.as[Option[Config]]("filesystems.local").getOrElse(ConfigFactory.empty()) } } } From 350772909e11ea4bbf977854fa84c55b4a757b37 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 11 Oct 2016 13:37:22 -0400 Subject: [PATCH 018/375] Fixed .pullapprove.yml --- .pullapprove.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pullapprove.yml b/.pullapprove.yml index a1b902b1c..8e5e37ffa 100644 --- a/.pullapprove.yml +++ b/.pullapprove.yml @@ -7,7 +7,6 @@ reviewers: members: - cjllanwarne - Horneth - - scottfrazer - mcovarr - geoffjentry - kshakir From 5eb428585a914915180b2d3717882dab714cc350 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 29 Sep 2016 10:01:05 -0400 Subject: [PATCH 019/375] Updated the JES status polling to be batched --- .../backend/BackendLifecycleActorFactory.scala | 5 +- .../test/scala/cromwell/util/AkkaTestUtil.scala | 26 +++- .../backend/BackendSingletonCollection.scala | 5 + .../cromwell/engine/workflow/WorkflowActor.scala | 10 +- .../engine/workflow/WorkflowManagerActor.scala | 18 ++- .../execution/EngineJobExecutionActor.scala | 5 +- .../lifecycle/execution/JobPreparationActor.scala | 16 ++- .../execution/WorkflowExecutionActor.scala | 9 +- .../scala/cromwell/server/CromwellRootActor.scala | 8 +- .../test/scala/cromwell/CromwellTestkitSpec.scala | 3 +- .../scala/cromwell/SimpleWorkflowActorSpec.scala | 4 +- .../mock/DefaultBackendJobExecutionActor.scala | 3 +- .../RetryableBackendLifecycleActorFactory.scala | 3 +- .../workflow/SingleWorkflowRunnerActorSpec.scala | 4 +- .../engine/workflow/WorkflowActorSpec.scala | 3 +- .../execution/WorkflowExecutionActorSpec.scala | 11 +- .../lifecycle/execution/ejea/PerTestHelper.scala | 6 +- .../JobExecutionTokenDispenserActorSpec.scala | 31 ++--- ...ingActor.scala => TestTokenGrabbingActor.scala} | 15 +- .../impl/htcondor/HtCondorBackendFactory.scala | 3 +- .../jes/JesAsyncBackendJobExecutionActor.scala | 107 +++++++------- .../impl/jes/JesBackendLifecycleActorFactory.scala | 7 +- .../impl/jes/JesBackendSingletonActor.scala | 20 +++ .../backend/impl/jes/JesJobExecutionActor.scala | 13 +- .../main/scala/cromwell/backend/impl/jes/Run.scala | 24 ++-- .../cromwell/backend/impl/jes/RunStatus.scala | 2 +- .../jes/statuspolling/JesApiQueryManager.scala | 108 +++++++++++++++ .../impl/jes/statuspolling/JesPollingActor.scala | 122 ++++++++++++++++ .../jes/statuspolling/JesPollingActorClient.scala | 51 +++++++ .../jes/JesAsyncBackendJobExecutionActorSpec.scala | 56 ++++---- .../scala/cromwell/backend/impl/jes/RunSpec.scala | 3 +- .../jes/statuspolling/JesApiQueryManagerSpec.scala | 153 +++++++++++++++++++++ .../jes/statuspolling/JesPollingActorSpec.scala | 131 ++++++++++++++++++ ...redFileSystemBackendLifecycleActorFactory.scala | 3 +- .../backend/impl/spark/SparkBackendFactory.scala | 6 +- 35 files changed, 832 insertions(+), 162 deletions(-) create mode 100644 engine/src/main/scala/cromwell/engine/backend/BackendSingletonCollection.scala rename engine/src/test/scala/cromwell/engine/workflow/tokens/{TokenGrabbingActor.scala => TestTokenGrabbingActor.scala} (64%) create mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala create mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala create mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala create mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorClient.scala create mode 100644 supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala create mode 100644 supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala index 0101da9f0..5a6c5d268 100644 --- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala +++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala @@ -20,7 +20,8 @@ trait BackendLifecycleActorFactory { def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef): Props + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props /** * Providing this method to generate Props for a cache hit copying actor is optional. @@ -33,6 +34,8 @@ trait BackendLifecycleActorFactory { */ def cacheHitCopyingActorProps: Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef) => Props] = None + def backendSingletonActorProps: Option[Props] = None + def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], executionStore: ExecutionStore, diff --git a/core/src/test/scala/cromwell/util/AkkaTestUtil.scala b/core/src/test/scala/cromwell/util/AkkaTestUtil.scala index 1a1ebd618..10b05dc2b 100644 --- a/core/src/test/scala/cromwell/util/AkkaTestUtil.scala +++ b/core/src/test/scala/cromwell/util/AkkaTestUtil.scala @@ -1,6 +1,6 @@ package cromwell.util -import akka.actor.{Actor, ActorLogging, Props} +import akka.actor.{Actor, ActorLogging, ActorRef, ActorSystem, Kill, PoisonPill, Props, SupervisorStrategy} import akka.testkit.TestProbe object AkkaTestUtil { @@ -16,4 +16,28 @@ object AkkaTestUtil { } }) } + + def actorDeathMethods(system: ActorSystem): List[(String, ActorRef => Unit)] = List( + ("external_stop", (a: ActorRef) => system.stop(a)), + ("internal_stop", (a: ActorRef) => a ! InternalStop), + ("poison_pill", (a: ActorRef) => a ! PoisonPill), + ("kill_message", (a: ActorRef) => a ! Kill), + ("throw_exception", (a: ActorRef) => a ! ThrowException) + ) + + case object InternalStop + case object ThrowException + + class StoppingSupervisor extends Actor { + override val supervisorStrategy = SupervisorStrategy.stoppingStrategy + def receive = Actor.emptyBehavior + } + + class DeathTestActor extends Actor { + private def stoppingReceive: Actor.Receive = { + case InternalStop => context.stop(self) + case ThrowException => throw new Exception("Don't panic, dear debugger! This was a deliberate exception for the test case.") + } + override def receive = stoppingReceive orElse Actor.ignoringBehavior + } } diff --git a/engine/src/main/scala/cromwell/engine/backend/BackendSingletonCollection.scala b/engine/src/main/scala/cromwell/engine/backend/BackendSingletonCollection.scala new file mode 100644 index 000000000..ecb1a6753 --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/backend/BackendSingletonCollection.scala @@ -0,0 +1,5 @@ +package cromwell.engine.backend + +import akka.actor.ActorRef + +final case class BackendSingletonCollection(backendSingletonActors: Map[String, Option[ActorRef]]) diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala index 684c81cce..1035872d3 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala @@ -11,6 +11,7 @@ import cromwell.core.WorkflowOptions.FinalWorkflowLogDir import cromwell.core._ import cromwell.core.logging.{WorkflowLogger, WorkflowLogging} import cromwell.engine._ +import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.WorkflowActor._ import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.{MaterializeWorkflowDescriptorCommand, MaterializeWorkflowDescriptorFailureResponse, MaterializeWorkflowDescriptorSuccessResponse} import cromwell.engine.workflow.lifecycle.WorkflowFinalizationActor.{StartFinalizationCommand, WorkflowFinalizationFailedResponse, WorkflowFinalizationSucceededResponse} @@ -140,9 +141,10 @@ object WorkflowActor { workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, - jobTokenDispenserActor: ActorRef): Props = { + jobTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection): Props = { Props(new WorkflowActor(workflowId, startMode, wdlSource, conf, serviceRegistryActor, workflowLogCopyRouter, - jobStoreActor, callCacheReadActor, jobTokenDispenserActor)).withDispatcher(EngineDispatcher) + jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection)).withDispatcher(EngineDispatcher) } } @@ -157,7 +159,8 @@ class WorkflowActor(val workflowId: WorkflowId, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, - jobTokenDispenserActor: ActorRef) + jobTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection) extends LoggingFSM[WorkflowActorState, WorkflowActorData] with WorkflowLogging with PathFactory { implicit val ec = context.dispatcher @@ -206,6 +209,7 @@ class WorkflowActor(val workflowId: WorkflowId, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, + backendSingletonCollection, initializationData, restarting = restarting), name = s"WorkflowExecutionActor-$workflowId") diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala index 59b105dd3..08bade654 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala @@ -8,6 +8,7 @@ import cats.data.NonEmptyList import com.typesafe.config.{Config, ConfigFactory} import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.{WorkflowAborted, WorkflowId} +import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.WorkflowActor._ import cromwell.engine.workflow.WorkflowManagerActor._ import cromwell.engine.workflow.workflowstore.{WorkflowStoreActor, WorkflowStoreState} @@ -42,9 +43,10 @@ object WorkflowManagerActor { workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, - jobTokenDispenserActor: ActorRef): Props = { + jobTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection): Props = { Props(new WorkflowManagerActor( - workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) + workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection) ).withDispatcher(EngineDispatcher) } @@ -82,7 +84,8 @@ class WorkflowManagerActor(config: Config, val workflowLogCopyRouter: ActorRef, val jobStoreActor: ActorRef, val callCacheReadActor: ActorRef, - val jobTokenDispenserActor: ActorRef) + val jobTokenDispenserActor: ActorRef, + val backendSingletonCollection: BackendSingletonCollection) extends LoggingFSM[WorkflowManagerState, WorkflowManagerData] { def this(workflowStore: ActorRef, @@ -90,8 +93,9 @@ class WorkflowManagerActor(config: Config, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, - jobTokenDispenserActor: ActorRef) = this( - ConfigFactory.load, workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) + jobTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection) = this( + ConfigFactory.load, workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection) private val maxWorkflowsRunning = config.getConfig("system").as[Option[Int]]("max-concurrent-workflows").getOrElse(DefaultMaxWorkflowsToRun) private val maxWorkflowsToLaunch = config.getConfig("system").as[Option[Int]]("max-workflow-launch-count").getOrElse(DefaultMaxWorkflowsToLaunch) @@ -111,7 +115,7 @@ class WorkflowManagerActor(config: Config, } private def addShutdownHook() = { - // Only abort jobs on SIGINT if the config explicitly sets backend.abortJobsOnTerminate = true. + // Only abort jobs on SIGINT if the config explicitly sets system.abortJobsOnTerminate = true. val abortJobsOnTerminate = config.getConfig("system").as[Option[Boolean]]("abort-jobs-on-terminate").getOrElse(false) @@ -261,7 +265,7 @@ class WorkflowManagerActor(config: Config, } val wfProps = WorkflowActor.props(workflowId, startMode, workflow.sources, config, serviceRegistryActor, - workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) + workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection) val wfActor = context.actorOf(wfProps, name = s"WorkflowActor-$workflowId") wfActor ! SubscribeTransitionCallBack(self) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala index 64f3eec7a..07114e7e8 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala @@ -39,6 +39,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, jobTokenDispenserActor: ActorRef, + backendSingletonActor: Option[ActorRef], backendName: String, callCachingMode: CallCachingMode) extends LoggingFSM[EngineJobExecutionActorState, EJEAData] with WorkflowLogging { @@ -282,7 +283,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, def createJobPreparationActor(jobPrepProps: Props, name: String): ActorRef = context.actorOf(jobPrepProps, name) def prepareJob() = { val jobPreparationActorName = s"BackendPreparationActor_for_$jobTag" - val jobPrepProps = JobPreparationActor.props(executionData, jobDescriptorKey, factory, initializationData, serviceRegistryActor) + val jobPrepProps = JobPreparationActor.props(executionData, jobDescriptorKey, factory, initializationData, serviceRegistryActor, backendSingletonActor) val jobPreparationActor = createJobPreparationActor(jobPrepProps, jobPreparationActorName) jobPreparationActor ! JobPreparationActor.Start goto(PreparingJob) @@ -455,6 +456,7 @@ object EngineJobExecutionActor { jobStoreActor: ActorRef, callCacheReadActor: ActorRef, jobTokenDispenserActor: ActorRef, + backendSingletonActor: Option[ActorRef], backendName: String, callCachingMode: CallCachingMode) = { Props(new EngineJobExecutionActor( @@ -468,6 +470,7 @@ object EngineJobExecutionActor { jobStoreActor = jobStoreActor, callCacheReadActor = callCacheReadActor, jobTokenDispenserActor = jobTokenDispenserActor, + backendSingletonActor = backendSingletonActor, backendName = backendName: String, callCachingMode = callCachingMode)).withDispatcher(EngineDispatcher) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala index cc7f76f92..12c719994 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala @@ -14,10 +14,11 @@ import wdl4s.values.WdlValue import scala.util.{Failure, Success, Try} final case class JobPreparationActor(executionData: WorkflowExecutionActorData, - jobKey: BackendJobDescriptorKey, - factory: BackendLifecycleActorFactory, - initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef) + jobKey: BackendJobDescriptorKey, + factory: BackendLifecycleActorFactory, + initializationData: Option[BackendInitializationData], + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]) extends Actor with WdlLookup with WorkflowLogging { override lazy val workflowDescriptor: EngineWorkflowDescriptor = executionData.workflowDescriptor @@ -92,7 +93,7 @@ final case class JobPreparationActor(executionData: WorkflowExecutionActorData, evaluatedRuntimeAttributes <- evaluateRuntimeAttributes(unevaluatedRuntimeAttributes, expressionLanguageFunctions, inputEvaluation) attributesWithDefault = curriedAddDefaultsToAttributes(evaluatedRuntimeAttributes) jobDescriptor = BackendJobDescriptor(workflowDescriptor.backendDescriptor, jobKey, attributesWithDefault, inputEvaluation) - } yield BackendJobPreparationSucceeded(jobDescriptor, factory.jobExecutionActorProps(jobDescriptor, initializationData, serviceRegistryActor))) match { + } yield BackendJobPreparationSucceeded(jobDescriptor, factory.jobExecutionActorProps(jobDescriptor, initializationData, serviceRegistryActor, backendSingletonActor))) match { case Success(s) => s case Failure(f) => BackendJobPreparationFailed(jobKey, f) } @@ -111,9 +112,10 @@ object JobPreparationActor { jobKey: BackendJobDescriptorKey, factory: BackendLifecycleActorFactory, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef) = { + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]) = { // Note that JobPreparationActor doesn't run on the engine dispatcher as it mostly executes backend-side code // (WDL expression evaluation using Backend's expressionLanguageFunctions) - Props(new JobPreparationActor(executionData, jobKey, factory, initializationData, serviceRegistryActor)) + Props(new JobPreparationActor(executionData, jobKey, factory, initializationData, serviceRegistryActor, backendSingletonActor)) } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index c40815057..b7358296b 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -17,7 +17,7 @@ import cromwell.core.OutputStore.OutputEntry import cromwell.core.WorkflowOptions.WorkflowFailureMode import cromwell.core._ import cromwell.core.logging.WorkflowLogging -import cromwell.engine.backend.CromwellBackends +import cromwell.engine.backend.{BackendSingletonCollection, CromwellBackends} import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.JobRunning import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor.BackendJobPreparationFailed import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.WorkflowExecutionActorState @@ -140,10 +140,11 @@ object WorkflowExecutionActor { jobStoreActor: ActorRef, callCacheReadActor: ActorRef, jobTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection, initializationData: AllBackendInitializationData, restarting: Boolean): Props = { Props(WorkflowExecutionActor(workflowId, workflowDescriptor, serviceRegistryActor, jobStoreActor, - callCacheReadActor, jobTokenDispenserActor, initializationData, restarting)).withDispatcher(EngineDispatcher) + callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection, initializationData, restarting)).withDispatcher(EngineDispatcher) } implicit class EnhancedExecutionStore(val executionStore: ExecutionStore) extends AnyVal { @@ -237,6 +238,7 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, jobTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection, initializationData: AllBackendInitializationData, restarting: Boolean) extends LoggingFSM[WorkflowExecutionActorState, WorkflowExecutionActorData] with WorkflowLogging { @@ -584,9 +586,10 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, factories.get(backendName) match { case Some(factory) => val ejeaName = s"${workflowDescriptor.id}-EngineJobExecutionActor-${jobKey.tag}" + val backendSingleton = backendSingletonCollection.backendSingletonActors(backendName) val ejeaProps = EngineJobExecutionActor.props( self, jobKey, data, factory, initializationData.get(backendName), restarting, serviceRegistryActor, - jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendName, workflowDescriptor.callCachingMode) + jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingleton, backendName, workflowDescriptor.callCachingMode) val ejeaRef = context.actorOf(ejeaProps, ejeaName) pushNewJobMetadata(jobKey, backendName) ejeaRef ! EngineJobExecutionActor.Execute diff --git a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala index b3c64b67f..cc0f5f4aa 100644 --- a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala +++ b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala @@ -5,6 +5,7 @@ import akka.actor.{Actor, ActorInitializationException, ActorRef, OneForOneStrat import akka.event.Logging import akka.routing.RoundRobinPool import com.typesafe.config.ConfigFactory +import cromwell.engine.backend.{BackendSingletonCollection, CromwellBackends} import cromwell.engine.workflow.WorkflowManagerActor import cromwell.engine.workflow.lifecycle.CopyWorkflowLogsActor import cromwell.engine.workflow.lifecycle.execution.callcaching.{CallCache, CallCacheReadActor} @@ -48,11 +49,16 @@ import net.ceedubs.ficus.Ficus._ .props(CallCacheReadActor.props(callCache)), "CallCacheReadActor") + lazy val backendSingletons = CromwellBackends.instance.get.backendLifecycleActorFactories map { + case (name, factory) => name -> (factory.backendSingletonActorProps map context.actorOf) + } + lazy val backendSingletonCollection = BackendSingletonCollection(backendSingletons) + lazy val jobExecutionTokenDispenserActor = context.actorOf(JobExecutionTokenDispenserActor.props) lazy val workflowManagerActor = context.actorOf( WorkflowManagerActor.props( - workflowStoreActor, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobExecutionTokenDispenserActor), + workflowStoreActor, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobExecutionTokenDispenserActor, backendSingletonCollection), "WorkflowManagerActor") override def receive = { diff --git a/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala b/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala index c4aec5879..090d94fd1 100644 --- a/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala +++ b/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala @@ -48,7 +48,8 @@ case class TestBackendLifecycleActorFactory(configurationDescriptor: BackendConf override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef): Props = { + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props = { throw new NotImplementedError("this is not implemented") } diff --git a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala index 34ec73895..f327c6da4 100644 --- a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala @@ -8,6 +8,7 @@ import com.typesafe.config.ConfigFactory import cromwell.MetadataWatchActor.{FailureMatcher, Matcher} import cromwell.SimpleWorkflowActorSpec._ import cromwell.core.{WorkflowId, WorkflowSourceFiles} +import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.WorkflowActor import cromwell.engine.workflow.WorkflowActor._ import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor @@ -43,7 +44,8 @@ class SimpleWorkflowActorSpec extends CromwellTestkitSpec with BeforeAndAfter { workflowLogCopyRouter = system.actorOf(Props.empty, s"workflow-copy-log-router-$workflowId-${UUID.randomUUID()}"), jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props), callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props), - jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props)), + jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props), + backendSingletonCollection = BackendSingletonCollection(Map("Local" -> None))), supervisor = supervisor.ref, name = s"workflow-actor-$workflowId" ) diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala b/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala index 6893f6baf..f98fa17bd 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala @@ -30,7 +30,8 @@ class DefaultBackendLifecycleActorFactory(name: String, configurationDescriptor: override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef): Props = { + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props = { DefaultBackendJobExecutionActor.props(jobDescriptor, configurationDescriptor) } diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala index 74e900c6d..46f28f447 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala @@ -13,7 +13,8 @@ class RetryableBackendLifecycleActorFactory(name: String, configurationDescripto override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef): Props = { + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props = { RetryableBackendJobExecutionActor.props(jobDescriptor, configurationDescriptor) } diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala index 73e8bfa7b..540da9863 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala @@ -10,6 +10,7 @@ import better.files._ import com.typesafe.config.ConfigFactory import cromwell.CromwellTestkitSpec._ import cromwell.core.WorkflowSourceFiles +import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.SingleWorkflowRunnerActor.RunWorkflow import cromwell.engine.workflow.SingleWorkflowRunnerActorSpec._ import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor @@ -65,7 +66,8 @@ abstract class SingleWorkflowRunnerActorSpec extends CromwellTestkitSpec { dummyLogCopyRouter, jobStore, callCacheReadActor, - jobTokenDispenserActor)), "WorkflowManagerActor") + jobTokenDispenserActor, + BackendSingletonCollection(Map.empty))), "WorkflowManagerActor") } def createRunnerActor(sampleWdl: SampleWdl = ThreeStep, managerActor: => ActorRef = workflowManagerActor(), diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala index 860ac6dab..b98c5657f 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala @@ -6,6 +6,7 @@ import com.typesafe.config.{Config, ConfigFactory} import cromwell.backend.AllBackendInitializationData import cromwell.core.{ExecutionStore, OutputStore, WorkflowId, WorkflowSourceFiles} import cromwell.engine.EngineWorkflowDescriptor +import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.WorkflowActor._ import cromwell.engine.workflow.lifecycle.EngineLifecycleActorAbortCommand import cromwell.engine.workflow.lifecycle.WorkflowFinalizationActor.{StartFinalizationCommand, WorkflowFinalizationSucceededResponse} @@ -156,7 +157,7 @@ class MockWorkflowActor(val finalizationProbe: TestProbe, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, - jobTokenDispenserActor: ActorRef) extends WorkflowActor(workflowId, startMode, workflowSources, conf, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) { + jobTokenDispenserActor: ActorRef) extends WorkflowActor(workflowId, startMode, workflowSources, conf, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, BackendSingletonCollection(Map.empty)) { override def makeFinalizationActor(workflowDescriptor: EngineWorkflowDescriptor, executionStore: ExecutionStore, outputStore: OutputStore) = finalizationProbe.ref } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala index 639d8f250..01af65d1a 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala @@ -5,7 +5,7 @@ import akka.testkit.{EventFilter, TestActorRef, TestDuration, TestProbe} import com.typesafe.config.ConfigFactory import cromwell.backend.AllBackendInitializationData import cromwell.core.WorkflowId -import cromwell.engine.backend.{BackendConfigurationEntry, CromwellBackends} +import cromwell.engine.backend.{BackendConfigurationEntry, BackendSingletonCollection, CromwellBackends} import cromwell.engine.workflow.WorkflowDescriptorBuilder import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.ExecuteWorkflowCommand import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor @@ -29,6 +29,9 @@ class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter } }) + val MockBackendName = "Mock" + val MockBackendSingletonCollection = BackendSingletonCollection(Map(MockBackendName -> None)) + val stubbedConfig = ConfigFactory.load().getConfig("backend.providers.Mock").getConfig("config") val runtimeSection = @@ -64,7 +67,7 @@ class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter val workflowExecutionActor = system.actorOf( WorkflowExecutionActor.props(workflowId, engineWorkflowDescriptor, serviceRegistryActor, jobStoreActor, - callCacheReadActor.ref, jobTokenDispenserActor, AllBackendInitializationData.empty, restarting = false), + callCacheReadActor.ref, jobTokenDispenserActor, MockBackendSingletonCollection, AllBackendInitializationData.empty, restarting = false), "WorkflowExecutionActor") EventFilter.info(pattern = ".*Final Outputs", occurrences = 1).intercept { @@ -87,7 +90,7 @@ class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props) val MockBackendConfigEntry = BackendConfigurationEntry( - name = "Mock", + name = MockBackendName, lifecycleActorFactoryClass = "cromwell.engine.backend.mock.DefaultBackendLifecycleActorFactory", stubbedConfig ) @@ -97,7 +100,7 @@ class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter val engineWorkflowDescriptor = createMaterializedEngineWorkflowDescriptor(workflowId, SampleWdl.SimpleScatterWdl.asWorkflowSources(runtime = runtimeSection)) val workflowExecutionActor = system.actorOf( WorkflowExecutionActor.props(workflowId, engineWorkflowDescriptor, serviceRegistry, jobStore, - callCacheReadActor, jobTokenDispenserActor, AllBackendInitializationData.empty, restarting = false), + callCacheReadActor, jobTokenDispenserActor, MockBackendSingletonCollection, AllBackendInitializationData.empty, restarting = false), "WorkflowExecutionActor") val scatterLog = "Starting calls: scatter0.inside_scatter:0:1, scatter0.inside_scatter:1:1, scatter0.inside_scatter:2:1, scatter0.inside_scatter:3:1, scatter0.inside_scatter:4:1" diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala index 80ae87fa9..aa3f5a28c 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala @@ -10,6 +10,7 @@ import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.core.callcaching.{CallCachingActivity, CallCachingMode, CallCachingOff} import cromwell.core.{ExecutionStore, JobExecutionToken, OutputStore, WorkflowId} import cromwell.engine.EngineWorkflowDescriptor +import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.lifecycle.execution.{EngineJobExecutionActor, WorkflowExecutionActorData} import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{EJEAData, EngineJobExecutionActorState} import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CallCacheHashes} @@ -81,7 +82,8 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef): Props = bjeaProps + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props = bjeaProps override def cacheHitCopyingActorProps: Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef) => Props] = Option((_, _, _) => callCacheHitCopyingProbe.props) @@ -142,7 +144,7 @@ private[ejea] class MockEjea(helper: PerTestHelper, callCacheReadActor: ActorRef, jobTokenDispenserActor: ActorRef, backendName: String, - callCachingMode: CallCachingMode) extends EngineJobExecutionActor(replyTo, jobDescriptorKey, executionData, factory, initializationData, restarting, serviceRegistryActor, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendName, callCachingMode) { + callCachingMode: CallCachingMode) extends EngineJobExecutionActor(replyTo, jobDescriptorKey, executionData, factory, initializationData, restarting, serviceRegistryActor, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, None, backendName, callCachingMode) { override def makeFetchCachedResultsActor(cacheHit: CacheHit, taskOutputs: Seq[TaskOutput]) = helper.fetchCachedResultsActorCreations = helper.fetchCachedResultsActorCreations.foundOne((cacheHit, taskOutputs)) override def initializeJobHashing(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity) = helper.jobHashingInitializations = helper.jobHashingInitializations.foundOne((jobDescriptor, activity)) diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala index b71c6a48f..0575b7b33 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala @@ -2,14 +2,15 @@ package cromwell.engine.workflow.tokens import java.util.UUID -import akka.actor.{ ActorRef, ActorSystem, Kill, PoisonPill} +import akka.actor.{ActorRef, ActorSystem, Kill, PoisonPill} import org.scalatest._ import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed, JobExecutionTokenRequest, JobExecutionTokenReturn} import JobExecutionTokenDispenserActorSpec._ import cromwell.core.JobExecutionToken -import cromwell.engine.workflow.tokens.TokenGrabbingActor.StoppingSupervisor +import cromwell.engine.workflow.tokens.TestTokenGrabbingActor.StoppingSupervisor +import cromwell.util.AkkaTestUtil import org.scalatest.concurrent.Eventually import scala.concurrent.duration._ @@ -201,23 +202,15 @@ class JobExecutionTokenDispenserActorSpec extends TestKit(ActorSystem("JETDASpec dummyActors(5).expectNoMsg(MaxWaitTime) } - val actorDeathMethods: List[(String, ActorRef => Unit)] = List( - ("external_stop", (a: ActorRef) => system.stop(a)), - ("internal_stop", (a: ActorRef) => a ! TokenGrabbingActor.InternalStop), - ("poison_pill", (a: ActorRef) => a ! PoisonPill), - ("kill_message", (a: ActorRef) => a ! Kill), - ("throw_exception", (a: ActorRef) => a ! TokenGrabbingActor.ThrowException) - ) - - actorDeathMethods foreach { case (name, stopMethod) => + AkkaTestUtil.actorDeathMethods(system) foreach { case (name, stopMethod) => it should s"recover tokens lost to actors which are $name before they hand back their token" in { - var currentTokens: Map[TestActorRef[TokenGrabbingActor], JobExecutionToken] = Map.empty - var tokenGrabbingActors: Map[Int, TestActorRef[TokenGrabbingActor]] = Map.empty + var currentTokens: Map[TestActorRef[TestTokenGrabbingActor], JobExecutionToken] = Map.empty + var tokenGrabbingActors: Map[Int, TestActorRef[TestTokenGrabbingActor]] = Map.empty val grabberSupervisor = TestActorRef(new StoppingSupervisor()) // Set up by taking all 5 tokens out, and then adding 2 to the queue: 5 indexedTimes { i => - val newGrabbingActor = TestActorRef[TokenGrabbingActor](TokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"grabber_${name}_" + i) + val newGrabbingActor = TestActorRef[TestTokenGrabbingActor](TestTokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"grabber_${name}_" + i) tokenGrabbingActors += i -> newGrabbingActor eventually { newGrabbingActor.underlyingActor.token.isDefined should be(true) @@ -226,7 +219,7 @@ class JobExecutionTokenDispenserActorSpec extends TestKit(ActorSystem("JETDASpec } val unassignedActorIndex = 5 - val newGrabbingActor = TestActorRef(new TokenGrabbingActor(actorRefUnderTest, LimitedTo5Tokens), s"grabber_${name}_" + unassignedActorIndex) + val newGrabbingActor = TestActorRef(new TestTokenGrabbingActor(actorRefUnderTest, LimitedTo5Tokens), s"grabber_${name}_" + unassignedActorIndex) tokenGrabbingActors += unassignedActorIndex -> newGrabbingActor eventually { newGrabbingActor.underlyingActor.rejections should be(1) @@ -245,13 +238,13 @@ class JobExecutionTokenDispenserActorSpec extends TestKit(ActorSystem("JETDASpec } it should "skip over dead actors when assigning tokens to the actor queue" in { - var currentTokens: Map[TestActorRef[TokenGrabbingActor], JobExecutionToken] = Map.empty - var tokenGrabbingActors: Map[Int, TestActorRef[TokenGrabbingActor]] = Map.empty + var currentTokens: Map[TestActorRef[TestTokenGrabbingActor], JobExecutionToken] = Map.empty + var tokenGrabbingActors: Map[Int, TestActorRef[TestTokenGrabbingActor]] = Map.empty val grabberSupervisor = TestActorRef(new StoppingSupervisor()) // Set up by taking all 5 tokens out, and then adding 2 to the queue: 5 indexedTimes { i => - val newGrabbingActor = TestActorRef[TokenGrabbingActor](TokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"skip_test_" + i) + val newGrabbingActor = TestActorRef[TestTokenGrabbingActor](TestTokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"skip_test_" + i) tokenGrabbingActors += i -> newGrabbingActor eventually { newGrabbingActor.underlyingActor.token.isDefined should be(true) @@ -260,7 +253,7 @@ class JobExecutionTokenDispenserActorSpec extends TestKit(ActorSystem("JETDASpec } 2 indexedTimes { i => val index = i + 5 - val newGrabbingActor = TestActorRef[TokenGrabbingActor](TokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"skip_test_" + index) + val newGrabbingActor = TestActorRef[TestTokenGrabbingActor](TestTokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"skip_test_" + index) tokenGrabbingActors += index -> newGrabbingActor eventually { newGrabbingActor.underlyingActor.rejections should be(1) diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/TokenGrabbingActor.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/TestTokenGrabbingActor.scala similarity index 64% rename from engine/src/test/scala/cromwell/engine/workflow/tokens/TokenGrabbingActor.scala rename to engine/src/test/scala/cromwell/engine/workflow/tokens/TestTokenGrabbingActor.scala index 06eff8583..669e6783f 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/tokens/TokenGrabbingActor.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/TestTokenGrabbingActor.scala @@ -4,12 +4,12 @@ import akka.actor.{Actor, ActorRef, Props, SupervisorStrategy} import cromwell.core.JobExecutionToken import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed, JobExecutionTokenRequest} -import cromwell.engine.workflow.tokens.TokenGrabbingActor.{InternalStop, ThrowException} +import cromwell.util.AkkaTestUtil /** * Grabs a token and doesn't let it go! */ -class TokenGrabbingActor(tokenDispenser: ActorRef, tokenType: JobExecutionTokenType) extends Actor { +class TestTokenGrabbingActor(tokenDispenser: ActorRef, tokenType: JobExecutionTokenType) extends Actor { var token: Option[JobExecutionToken] = None var rejections = 0 @@ -17,19 +17,16 @@ class TokenGrabbingActor(tokenDispenser: ActorRef, tokenType: JobExecutionTokenT def receive = { case JobExecutionTokenDispensed(dispensedToken) => token = Option(dispensedToken) case JobExecutionTokenDenied(positionInQueue) => rejections += 1 - case ThrowException => throw new RuntimeException("Test exception (don't be scared by the stack trace, it's deliberate!)") - case InternalStop => context.stop(self) + case AkkaTestUtil.ThrowException => throw new RuntimeException("Test exception (don't be scared by the stack trace, it's deliberate!)") + case AkkaTestUtil.InternalStop => context.stop(self) } tokenDispenser ! JobExecutionTokenRequest(tokenType) } -object TokenGrabbingActor { +object TestTokenGrabbingActor { - def props(tokenDispenserActor: ActorRef, tokenType: JobExecutionTokenType) = Props(new TokenGrabbingActor(tokenDispenserActor, tokenType)) - - case object ThrowException - case object InternalStop + def props(tokenDispenserActor: ActorRef, tokenType: JobExecutionTokenType) = Props(new TestTokenGrabbingActor(tokenDispenserActor, tokenType)) class StoppingSupervisor extends Actor { override val supervisorStrategy = SupervisorStrategy.stoppingStrategy diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala index 7faf6baed..75f2f779a 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala @@ -24,7 +24,8 @@ case class HtCondorBackendFactory(name: String, configurationDescriptor: Backend override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef): Props = { + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props = { HtCondorJobExecutionActor.props(jobDescriptor, configurationDescriptor, serviceRegistryActor, resolveCacheProviderProps(jobDescriptor.workflowDescriptor.workflowOptions)) } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala index b2e5974e0..a08be8df9 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala @@ -17,6 +17,7 @@ import cromwell.backend.impl.jes.JesImplicits.PathString import cromwell.backend.impl.jes.JesJobExecutionActor.JesOperationIdKey import cromwell.backend.impl.jes.RunStatus.TerminalRunStatus import cromwell.backend.impl.jes.io._ +import cromwell.backend.impl.jes.statuspolling.JesPollingActorClient import cromwell.backend.{AttemptedLookupResult, BackendJobDescriptor, BackendWorkflowDescriptor, PreemptedException} import cromwell.core.Dispatcher.BackendDispatcher import cromwell.core._ @@ -44,12 +45,14 @@ object JesAsyncBackendJobExecutionActor { completionPromise: Promise[BackendJobExecutionResponse], jesWorkflowInfo: JesConfiguration, initializationData: JesBackendInitializationData, - serviceRegistryActor: ActorRef): Props = { + serviceRegistryActor: ActorRef, + jesBackendSingletonActor: ActorRef): Props = { Props(new JesAsyncBackendJobExecutionActor(jobDescriptor, completionPromise, jesWorkflowInfo, initializationData, - serviceRegistryActor)).withDispatcher(BackendDispatcher) + serviceRegistryActor, + jesBackendSingletonActor)).withDispatcher(BackendDispatcher) } object WorkflowOptionKeys { @@ -80,11 +83,14 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes override val completionPromise: Promise[BackendJobExecutionResponse], override val jesConfiguration: JesConfiguration, override val initializationData: JesBackendInitializationData, - override val serviceRegistryActor: ActorRef) - extends Actor with ActorLogging with AsyncBackendJobExecutionActor with JesJobCachingActorHelper with JobLogging { + override val serviceRegistryActor: ActorRef, + val jesBackendSingletonActor: ActorRef) + extends Actor with ActorLogging with AsyncBackendJobExecutionActor with JesJobCachingActorHelper with JobLogging with JesPollingActorClient { import JesAsyncBackendJobExecutionActor._ + override val pollingActor = jesBackendSingletonActor + override lazy val pollBackOff = SimpleExponentialBackoff( initialInterval = 30 seconds, maxInterval = 10 minutes, multiplier = 1.1) @@ -121,7 +127,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes case KvPutSuccess(_) => // expected after the KvPut for the operation ID } - override def receive: Receive = jesReceiveBehavior orElse super.receive + override def receive: Receive = pollingActorClientReceive orElse jesReceiveBehavior orElse super.receive private def globOutputPath(glob: String) = callRootPath.resolve(s"glob-${glob.md5Sum}/") @@ -366,50 +372,55 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes /** * Update the ExecutionHandle */ - override def poll(previous: ExecutionHandle)(implicit ec: ExecutionContext): Future[ExecutionHandle] = Future { + override def poll(previous: ExecutionHandle)(implicit ec: ExecutionContext): Future[ExecutionHandle] = { previous match { case handle: JesPendingExecutionHandle => - val runId = handle.run.runId - jobLogger.debug(s"$tag Polling JES Job $runId") - val previousStatus = handle.previousStatus - val status = Try(handle.run.status()) - status foreach { currentStatus => - if (!(handle.previousStatus contains currentStatus)) { - // If this is the first time checking the status, we log the transition as '-' to 'currentStatus'. Otherwise - // just use the state names. - val prevStateName = previousStatus map { _.toString } getOrElse "-" - jobLogger.info(s"$tag Status change from $prevStateName to $currentStatus") - tellMetadata(Map("backendStatus" -> currentStatus)) - } - } - status match { - case Success(s: TerminalRunStatus) => - val metadata = Map( - JesMetadataKeys.MachineType -> s.machineType.getOrElse("unknown"), - JesMetadataKeys.InstanceName -> s.instanceName.getOrElse("unknown"), - JesMetadataKeys.Zone -> s.zone.getOrElse("unknown") - ) - - tellMetadata(metadata) - executionResult(s, handle) - case Success(s) => handle.copy(previousStatus = Option(s)).future // Copy the current handle with updated previous status. - case Failure(e: GoogleJsonResponseException) if e.getStatusCode == 404 => - jobLogger.error(s"$tag JES Job ID ${handle.run.runId} has not been found, failing call") - FailedNonRetryableExecutionHandle(e).future - case Failure(e: Exception) => - // Log exceptions and return the original handle to try again. - jobLogger.warn(s"Caught exception, retrying", e) - handle.future - case Failure(e: Error) => Future.failed(e) // JVM-ending calamity. - case Failure(throwable) => - // Someone has subclassed Throwable directly? - FailedNonRetryableExecutionHandle(throwable).future - } + jobLogger.debug(s"$tag Polling JES Job ${handle.run.runId}") + pollStatus(handle.run) map updateExecutionHandleSuccess(handle) recover updateExecutionHandleFailure(handle) flatten case f: FailedNonRetryableExecutionHandle => f.future case s: SuccessfulExecutionHandle => s.future case badHandle => Future.failed(new IllegalArgumentException(s"Unexpected execution handle: $badHandle")) } - } flatten + } + + private def updateExecutionHandleFailure(oldHandle: JesPendingExecutionHandle): PartialFunction[Throwable, Future[ExecutionHandle]] = { + case e: GoogleJsonResponseException if e.getStatusCode == 404 => + jobLogger.error(s"$tag JES Job ID ${oldHandle.run.runId} has not been found, failing call") + FailedNonRetryableExecutionHandle(e).future + case e: Exception => + // Log exceptions and return the original handle to try again. + jobLogger.warn(s"Caught exception, retrying", e) + oldHandle.future + case e: Error => Future.failed(e) // JVM-ending calamity. + case throwable => + // Someone has subclassed Throwable directly? + FailedNonRetryableExecutionHandle(throwable).future + } + + private def updateExecutionHandleSuccess(oldHandle: JesPendingExecutionHandle)(status: RunStatus): Future[ExecutionHandle] = { + val previousStatus = oldHandle.previousStatus + if (!(previousStatus contains status)) { + // If this is the first time checking the status, we log the transition as '-' to 'currentStatus'. Otherwise + // just use the state names. + val prevStateName = previousStatus map { _.toString } getOrElse "-" + jobLogger.info(s"$tag Status change from $prevStateName to $status") + tellMetadata(Map("backendStatus" -> status)) + } + + status match { + case s: TerminalRunStatus => + val metadata = Map( + JesMetadataKeys.MachineType -> s.machineType.getOrElse("unknown"), + JesMetadataKeys.InstanceName -> s.instanceName.getOrElse("unknown"), + JesMetadataKeys.Zone -> s.zone.getOrElse("unknown") + ) + + tellMetadata(metadata) + executionResult(s, oldHandle) + case s => oldHandle.copy(previousStatus = Option(s)).future // Copy the current handle with updated previous status. + + } + } /** * Fire and forget start info to the metadata service @@ -512,19 +523,19 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes errorMessage.substring(0, errorMessage.indexOf(':')).toInt } - private def preempted(errorCode: Int, errorMessage: Option[String]): Boolean = { + private def preempted(errorCode: Int, errorMessage: List[String]): Boolean = { def isPreemptionCode(code: Int) = code == 13 || code == 14 try { - errorCode == 10 && errorMessage.isDefined && isPreemptionCode(extractErrorCodeFromErrorMessage(errorMessage.get)) && preemptible + errorCode == 10 && errorMessage.exists(e => isPreemptionCode(extractErrorCodeFromErrorMessage(e))) && preemptible } catch { case _: NumberFormatException | _: StringIndexOutOfBoundsException => - jobLogger.warn(s"Unable to parse JES error code from error message: {}, assuming this was not a preempted VM.", errorMessage.get) + jobLogger.warn(s"Unable to parse JES error code from error messages: [{}], assuming this was not a preempted VM.", errorMessage.mkString(", ")) false } } - private def handleFailure(errorCode: Int, errorMessage: Option[String]) = { + private def handleFailure(errorCode: Int, errorMessage: List[String]) = { import lenthall.numeric.IntegerUtil._ val taskName = s"${workflowDescriptor.id}:${call.unqualifiedName}" @@ -550,7 +561,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes } else { val id = workflowDescriptor.id val name = jobDescriptor.call.unqualifiedName - val message = errorMessage.getOrElse("null") + val message = if (errorMessage.isEmpty) "null" else errorMessage.mkString(", ") val exception = new RuntimeException(s"Task $id:$name failed: error code $errorCode. Message: $message") FailedNonRetryableExecutionHandle(exception, None).future } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala index 1e87b4d1f..92f653c10 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala @@ -28,10 +28,11 @@ case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef): Props = { + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props = { // The `JesInitializationActor` will only return a non-`Empty` `JesBackendInitializationData` from a successful `beforeAll` // invocation, so the `get` here is safe. - JesJobExecutionActor.props(jobDescriptor, jesConfiguration, initializationData.toJes.get, serviceRegistryActor).withDispatcher(BackendDispatcher) + JesJobExecutionActor.props(jobDescriptor, jesConfiguration, initializationData.toJes.get, serviceRegistryActor, backendSingletonActor).withDispatcher(BackendDispatcher) } override def cacheHitCopyingActorProps = Option(cacheHitCopyingActorInner _) @@ -71,6 +72,8 @@ case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor initializationData.toJes.get.workflowPaths.rootPath } + override def backendSingletonActorProps = Option(JesBackendSingletonActor.props()) + override lazy val fileHashingFunction: Option[FileHashingFunction] = Option(FileHashingFunction(JesBackendFileHashing.getCrc32c)) } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala new file mode 100644 index 000000000..3b830107a --- /dev/null +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala @@ -0,0 +1,20 @@ +package cromwell.backend.impl.jes + +import akka.actor.{Actor, ActorLogging, Props} +import cromwell.backend.impl.jes.statuspolling.{JesApiQueryManager} +import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll + +class JesBackendSingletonActor extends Actor with ActorLogging { + + val pollingActor = context.actorOf(JesApiQueryManager.props) + + override def receive = { + case poll: DoPoll => + log.debug("Forwarding status poll to JES polling actor") + pollingActor.forward(poll) + } +} + +object JesBackendSingletonActor { + def props(): Props = Props(new JesBackendSingletonActor()) +} diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala index eed3268f5..b1a8ba8d6 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala @@ -19,8 +19,9 @@ object JesJobExecutionActor { def props(jobDescriptor: BackendJobDescriptor, jesWorkflowInfo: JesConfiguration, initializationData: JesBackendInitializationData, - serviceRegistryActor: ActorRef): Props = { - Props(new JesJobExecutionActor(jobDescriptor, jesWorkflowInfo, initializationData, serviceRegistryActor)) + serviceRegistryActor: ActorRef, + jesBackendSingletonActor: Option[ActorRef]): Props = { + Props(new JesJobExecutionActor(jobDescriptor, jesWorkflowInfo, initializationData, serviceRegistryActor, jesBackendSingletonActor)) } val JesOperationIdKey = "__jes_operation_id" @@ -29,9 +30,12 @@ object JesJobExecutionActor { case class JesJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, jesConfiguration: JesConfiguration, initializationData: JesBackendInitializationData, - serviceRegistryActor: ActorRef) + serviceRegistryActor: ActorRef, + jesBackendSingletonActorOption: Option[ActorRef]) extends BackendJobExecutionActor { + val jesBackendSingletonActor = jesBackendSingletonActorOption.getOrElse(throw new RuntimeException("JES Backend actor cannot exist without the JES backend singleton actor")) + private def jesReceiveBehavior: Receive = LoggingReceive { case AbortJobCommand => executor.foreach(_ ! AbortJobCommand) @@ -63,7 +67,8 @@ case class JesJobExecutionActor(override val jobDescriptor: BackendJobDescriptor completionPromise, jesConfiguration, initializationData, - serviceRegistryActor) + serviceRegistryActor, + jesBackendSingletonActor) val executorRef = context.actorOf(executionProps, "JesAsyncBackendJobExecutionActor") executor = Option(executorRef) () diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala index 0f948344c..b5a8b5f9f 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala @@ -90,24 +90,19 @@ object Run { implicit class RunOperationExtension(val operation: Operation) extends AnyVal { def hasStarted = operation.getMetadata.asScala.get("startTime") isDefined } -} - -case class Run(runId: String, genomicsInterface: Genomics) { - import Run._ - def status(): RunStatus = { - val op = genomicsInterface.operations().get(runId).execute + def interpretOperationStatus(op: Operation): RunStatus = { if (op.getDone) { - val eventList = getEventList(op) - val ceInfo = op.getMetadata.get ("runtimeMetadata").asInstanceOf[GArrayMap[String,Object]].get("computeEngine").asInstanceOf[GArrayMap[String, String]] - val machineType = Option(ceInfo.get("machineType")) - val instanceName = Option(ceInfo.get("instanceName")) - val zone = Option(ceInfo.get("zone")) + lazy val eventList = getEventList(op) + lazy val ceInfo = op.getMetadata.get ("runtimeMetadata").asInstanceOf[GArrayMap[String,Object]].get("computeEngine").asInstanceOf[GArrayMap[String, String]] + lazy val machineType = Option(ceInfo.get("machineType")) + lazy val instanceName = Option(ceInfo.get("instanceName")) + lazy val zone = Option(ceInfo.get("zone")) // If there's an error, generate a Failed status. Otherwise, we were successful! Option(op.getError) match { case None => Success(eventList, machineType, zone, instanceName) - case Some(error) => Failed(error.getCode, Option(error.getMessage), eventList, machineType, zone, instanceName) + case Some(error) => Failed(error.getCode, Option(error.getMessage).toList, eventList, machineType, zone, instanceName) } } else if (op.hasStarted) { Running @@ -156,6 +151,11 @@ case class Run(runId: String, genomicsInterface: Genomics) { private def eventIfExists(name: String, metadata: Map[String, AnyRef], eventName: String): Option[ExecutionEvent] = { metadata.get(name) map { time => ExecutionEvent(eventName, OffsetDateTime.parse(time.toString)) } } +} + +case class Run(runId: String, genomicsInterface: Genomics) { + + def getOperationCommand = genomicsInterface.operations().get(runId) def abort(): Unit = { val cancellationRequest: CancelOperationRequest = new CancelOperationRequest() diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala index 7a2e31b77..d0609573e 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala @@ -27,7 +27,7 @@ object RunStatus { override def toString = "Success" } - final case class Failed(errorCode: Int, errorMessage: Option[String], eventList: Seq[ExecutionEvent], machineType: Option[String], zone: Option[String], instanceName: Option[String]) + final case class Failed(errorCode: Int, errorMessage: List[String], eventList: Seq[ExecutionEvent], machineType: Option[String], zone: Option[String], instanceName: Option[String]) extends TerminalRunStatus { // Don't want to include errorMessage or code in the snappy status toString: override def toString = "Failed" diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala new file mode 100644 index 000000000..f3a7739ac --- /dev/null +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala @@ -0,0 +1,108 @@ +package cromwell.backend.impl.jes.statuspolling + +import akka.actor.{Actor, ActorLogging, ActorRef, Props, SupervisorStrategy, Terminated} +import cats.data.NonEmptyList +import cromwell.backend.impl.jes.Run +import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager._ + +import scala.collection.immutable.Queue + +/** + * Currently, just holds a set of JES status poll requests until a PollingActor pulls the work. + * TODO: Could eventually move all of the JES queries into a single work-pulling model. + */ +class JesApiQueryManager extends Actor with ActorLogging { + + private var workQueue: Queue[JesStatusPollQuery] = Queue.empty + private var workInProgress: Map[ActorRef, JesPollingWorkBatch] = Map.empty + + // If the statusPoller dies, we want to stop it and handle the termination ourselves. + override val supervisorStrategy = SupervisorStrategy.stoppingStrategy + private def statusPollerProps = JesPollingActor.props(self) + private var statusPoller: ActorRef = _ + resetStatusPoller() + + override def receive = { + case DoPoll(run) => workQueue :+= JesStatusPollQuery(sender, run) + case RequestJesPollingWork(maxBatchSize) => + log.debug(s"Request for JES Polling Work received (max batch: $maxBatchSize, current queue size is ${workQueue.size})") + handleJesPollingRequest(sender, maxBatchSize) + case Terminated(actorRef) => handleTerminated(actorRef) + case other => log.error(s"Unexpected message to JesPollingManager: $other") + } + + /** + * !! Function Impurity Warning: Modifies Actor Data !! + */ + private def handleJesPollingRequest(workPullingJesPollingActor: ActorRef, maxBatchSize: Int) = { + workInProgress -= workPullingJesPollingActor + val beheaded = beheadWorkQueue(maxBatchSize) + beheaded.workToDo match { + case Some(work) => + sendWork(workPullingJesPollingActor, JesPollingWorkBatch(work)) + case None => + log.debug(s"No work for JES poller. Sad.") + workPullingJesPollingActor ! NoWorkToDo + } + + workQueue = beheaded.newWorkQueue + } + + private def sendWork(workPullingJesPollingActor: ActorRef, work: JesPollingWorkBatch) = { + log.debug(s"Sending work to JES poller.") + workPullingJesPollingActor ! work + workInProgress += (workPullingJesPollingActor -> work) + } + + private final case class BeheadedWorkQueue(workToDo: Option[NonEmptyList[JesStatusPollQuery]], newWorkQueue: Queue[JesStatusPollQuery]) + private def beheadWorkQueue(maxBatchSize: Int): BeheadedWorkQueue = { + + val head = workQueue.take(maxBatchSize).toList + val tail = workQueue.drop(maxBatchSize) + + head match { + case h :: t => BeheadedWorkQueue(Option(NonEmptyList(h, t)), tail) + case Nil => BeheadedWorkQueue(None, Queue.empty) + } + } + + private def handleTerminated(terminee: ActorRef) = { + // Currently we can assume this is a polling actor. Might change in a future update: + workInProgress.get(terminee) match { + case Some(work) => + // Ouch. We should tell all of its clients that it fell over. And then start a new one. + log.error(s"The JES polling actor $terminee unexpectedly terminated while conducting ${work.workBatch.tail.size + 1} polls. Making a new one...") + work.workBatch.toList foreach { _.requester ! JesPollingActor.JesPollError } + case None => + // It managed to die while doing absolutely nothing...!? + // Maybe it deserves an entry in https://en.wikipedia.org/wiki/List_of_unusual_deaths + // Oh well, in the mean time don't do anything, just start a new one + log.error(s"The JES polling actor $terminee managed to unexpectedly terminate whilst doing absolutely nothing. This is probably a programming error. Making a new one...") + } + resetStatusPoller() + } + + private def resetStatusPoller() = { + statusPoller = makeStatusPoller() + context.watch(statusPoller) + log.info(s"watching $statusPoller") + } + + private[statuspolling] def makeStatusPoller(): ActorRef = context.actorOf(statusPollerProps) +} + +object JesApiQueryManager { + + def props: Props = Props(new JesApiQueryManager) + + /** + * Poll the job represented by the Run. + */ + final case class DoPoll(run: Run) + + private[statuspolling] final case class JesStatusPollQuery(requester: ActorRef, run: Run) + private[statuspolling] final case class JesPollingWorkBatch(workBatch: NonEmptyList[JesStatusPollQuery]) + private[statuspolling] case object NoWorkToDo + + private[statuspolling] final case class RequestJesPollingWork(maxBatchSize: Int) +} diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala new file mode 100644 index 000000000..4152d3933 --- /dev/null +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala @@ -0,0 +1,122 @@ +package cromwell.backend.impl.jes.statuspolling + +import akka.actor.{Actor, ActorLogging, ActorRef, Props} +import cats.data.NonEmptyList +import com.google.api.client.googleapis.batch.BatchRequest +import com.google.api.client.googleapis.batch.json.JsonBatchCallback +import com.google.api.client.googleapis.json.GoogleJsonError +import com.google.api.client.http.HttpHeaders +import com.google.api.services.genomics.model.Operation +import cromwell.backend.impl.jes.Run +import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.{JesPollingWorkBatch, JesStatusPollQuery, NoWorkToDo} +import cromwell.backend.impl.jes.statuspolling.JesPollingActor._ + +import scala.collection.JavaConversions._ +import scala.concurrent.{ExecutionContext, Future, Promise} +import scala.util.{Failure, Success, Try} +import scala.concurrent.duration._ + +/** + * Polls JES for status. Pipes the results back (so expect either a RunStatus or a akka.actor.Status.Failure). + */ +class JesPollingActor(pollingManager: ActorRef) extends Actor with ActorLogging { + + // We want to query at just under our fixed JES QPS limit of 20 per second. That should hopefully allow some room at the edges + // for things like new calls, etc. + val MaxBatchSize = 100 + val BatchInterval = 5.5.seconds + self ! NoWorkToDo // Starts the check-for-work cycle + + implicit val ec: ExecutionContext = context.dispatcher + + override def receive = { + + case JesPollingWorkBatch(workBatch) => + log.debug(s"Got a polling batch with ${workBatch.tail.size + 1} requests.") + val batchResultFutures = handleBatch(workBatch) + val overallFuture = Future.sequence(batchResultFutures.toList) + overallFuture.andThen(interstitialRecombobulation) + () + case NoWorkToDo => + scheduleCheckForWork() + } + + private def handleBatch(workBatch: NonEmptyList[JesStatusPollQuery]): NonEmptyList[Future[Try[Unit]]] = { + + // Assume that the auth for the first element is also able to query the remaining Runs + val batch: BatchRequest = createBatch(workBatch.head.run) + + // Create the batch: + val batchFutures = workBatch map { pollingRequest => + val completionPromise = Promise[Try[Unit]]() + val resultHandler = batchResultHandler(pollingRequest.requester, completionPromise) + enqueueStatusPollInBatch(pollingRequest.run, batch, resultHandler) + completionPromise.future + } + + // Execute the batch and return the map: + runBatch(batch) + batchFutures + } + + // These are separate functions so that the tests can hook in and replace the JES-side stuff + private[statuspolling] def createBatch(run: Run): BatchRequest = run.genomicsInterface.batch() + private[statuspolling] def enqueueStatusPollInBatch(run: Run, batch: BatchRequest, resultHandler: JsonBatchCallback[Operation]) = { + run.getOperationCommand.queue(batch, resultHandler) + } + private[statuspolling] def runBatch(batch: BatchRequest) = batch.execute() + + private def batchResultHandler(originalRequester: ActorRef, completionPromise: Promise[Try[Unit]]) = new JsonBatchCallback[Operation] { + override def onSuccess(operation: Operation, responseHeaders: HttpHeaders): Unit = { + log.debug(s"Batch result onSuccess callback triggered!") + originalRequester ! interpretOperationStatus(operation) + completionPromise.trySuccess(Success(())) + () + } + + override def onFailure(e: GoogleJsonError, responseHeaders: HttpHeaders): Unit = { + log.debug(s"Batch request onFailure callback triggered!") + originalRequester ! JesPollFailed(e, responseHeaders) + completionPromise.trySuccess(Failure(new Exception(mkErrorString(e)))) + () + } + } + + private[statuspolling] def mkErrorString(e: GoogleJsonError) = e.getErrors.toList.mkString(", ") + private[statuspolling] def interpretOperationStatus(operation: Operation) = Run.interpretOperationStatus(operation) + + // TODO: FSMify this actor? + private def interstitialRecombobulation: PartialFunction[Try[List[Try[Unit]]], Unit] = { + case Success(allSuccesses) if allSuccesses.forall(_.isSuccess) => + log.debug(s"All status polls completed successfully.") + scheduleCheckForWork() + case Success(someFailures) => + val errors = someFailures collect { case Failure(t) => t.getMessage } + if (log.isDebugEnabled) { + log.warning("{} failures fetching JES statuses", errors.size) + } else { + log.warning("{} failures fetching JES statuses: {}", errors.size, errors.mkString(", ")) + } + scheduleCheckForWork() + case Failure(t) => + // NB: Should be impossible since we only ever do completionPromise.trySuccess() + log.error("Completion promise unexpectedly set to Failure: {}", t.getMessage) + scheduleCheckForWork() + } + + /** + * Schedules a check for work. + * Warning: Only use this from inside a receive method. + */ + private def scheduleCheckForWork(): Unit = { + context.system.scheduler.scheduleOnce(BatchInterval) { pollingManager ! JesApiQueryManager.RequestJesPollingWork(MaxBatchSize) } + () + } +} + +object JesPollingActor { + def props(pollingManager: ActorRef) = Props(new JesPollingActor(pollingManager)) + + final case class JesPollFailed(e: GoogleJsonError, responseHeaders: HttpHeaders) + case object JesPollError +} diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorClient.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorClient.scala new file mode 100644 index 000000000..1bf7328f8 --- /dev/null +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorClient.scala @@ -0,0 +1,51 @@ +package cromwell.backend.impl.jes.statuspolling + +import java.io.IOException + +import akka.actor.{Actor, ActorLogging, ActorRef} +import cromwell.backend.impl.jes.statuspolling.JesPollingActor.{JesPollError, JesPollFailed} +import cromwell.backend.impl.jes.{Run, RunStatus} + +import scala.concurrent.{Future, Promise} +import scala.util.{Failure, Success, Try} + +/** + * I'm putting this stuff in a mixin to avoid polluting the main class. + * + * Be sure to make the main class's receive look like: + * override def receive = pollingActorClientReceive orElse { ... } + */ +trait JesPollingActorClient { this: Actor with ActorLogging => + + private var pollingActorClientPromise: Option[Promise[RunStatus]] = None + + val pollingActor: ActorRef + + def pollingActorClientReceive: Actor.Receive = { + case r: RunStatus => + log.debug(s"Polled status received: $r") + completePromise(Success(r)) + case JesPollFailed(e, responseHeaders) => + log.debug("JES poll failed! Sad.") + completePromise(Failure(new IOException(s"Google request failed: ${e.toPrettyString}"))) + case JesPollError => + log.debug("JES poll failed when polling actor died unexpectedly! Sad.") + completePromise(Failure(new RuntimeException("Unexpected actor death!"))) + } + + private def completePromise(runStatus: Try[RunStatus]) = { + pollingActorClientPromise foreach { _.complete(runStatus) } + pollingActorClientPromise = None + } + + def pollStatus(run: Run): Future[RunStatus] = { + pollingActorClientPromise match { + case Some(p) => p.future + case None => + pollingActor ! JesApiQueryManager.DoPoll(run) + val newPromise = Promise[RunStatus]() + pollingActorClientPromise = Option(newPromise) + newPromise.future + } + } +} diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala index 2a2ae3ec9..5601e1ebb 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala @@ -5,7 +5,7 @@ import java.util.UUID import akka.actor.{ActorRef, Props} import akka.event.LoggingAdapter -import akka.testkit.{ImplicitSender, TestActorRef, TestDuration} +import akka.testkit.{ImplicitSender, TestActorRef, TestDuration, TestProbe} import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse import cromwell.backend.async.AsyncBackendJobExecutionActor.{Execute, ExecutionMode} import cromwell.backend.async.{AbortedExecutionHandle, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle} @@ -30,6 +30,8 @@ import wdl4s.{Call, LocallyQualifiedName, NamespaceWithWorkflow} import scala.concurrent.duration._ import scala.concurrent.{Await, ExecutionContext, Future, Promise} import scala.util.{Success, Try} +import cromwell.backend.impl.jes.MockObjects._ +import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackendJobExecutionActorSpec") with FlatSpecLike with Matchers with ImplicitSender with Mockito { @@ -79,8 +81,9 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend class TestableJesJobExecutionActor(jobDescriptor: BackendJobDescriptor, promise: Promise[BackendJobExecutionResponse], jesConfiguration: JesConfiguration, - functions: JesExpressionFunctions = TestableJesExpressionFunctions) - extends JesAsyncBackendJobExecutionActor(jobDescriptor, promise, jesConfiguration, buildInitializationData(jobDescriptor, jesConfiguration), emptyActor) { + functions: JesExpressionFunctions = TestableJesExpressionFunctions, + jesSingletonActor: ActorRef = emptyActor) + extends JesAsyncBackendJobExecutionActor(jobDescriptor, promise, jesConfiguration, buildInitializationData(jobDescriptor, jesConfiguration), emptyActor, jesSingletonActor) { override lazy val jobLogger = new LoggerWrapper { override def akkaLogger: Option[LoggingAdapter] = Option(log) @@ -121,30 +124,33 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend private def executionActor(jobDescriptor: BackendJobDescriptor, configurationDescriptor: BackendConfigurationDescriptor, promise: Promise[BackendJobExecutionResponse], - errorCode: Int, - innerErrorCode: Int): ActorRef = { + jesSingletonActor: ActorRef): ActorRef = { // Mock/stub out the bits that would reach out to JES. val run = mock[Run] - run.status() returns Failed(errorCode, Option(s"$innerErrorCode: I seen some things man"), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")) - val handle = JesPendingExecutionHandle(jobDescriptor, Seq.empty, run, None) - class ExecuteOrRecoverActor extends TestableJesJobExecutionActor(jobDescriptor, promise, jesConfiguration) { + class ExecuteOrRecoverActor extends TestableJesJobExecutionActor(jobDescriptor, promise, jesConfiguration, jesSingletonActor = jesSingletonActor) { override def executeOrRecover(mode: ExecutionMode)(implicit ec: ExecutionContext): Future[ExecutionHandle] = Future.successful(handle) } system.actorOf(Props(new ExecuteOrRecoverActor), "ExecuteOrRecoverActor-" + UUID.randomUUID) } - private def run(attempt: Int, preemptible: Int, errorCode: Int, innerErrorCode: Int): BackendJobExecutionResponse = { - within(Timeout) { - val promise = Promise[BackendJobExecutionResponse]() - val jobDescriptor = buildPreemptibleJobDescriptor(attempt, preemptible) - val backend = executionActor(jobDescriptor, JesBackendConfigurationDescriptor, promise, errorCode, innerErrorCode) - backend ! Execute - Await.result(promise.future, Timeout) + private def runAndFail(attempt: Int, preemptible: Int, errorCode: Int, innerErrorCode: Int): BackendJobExecutionResponse = { + + val runStatus = Failed(errorCode, List(s"$innerErrorCode: I seen some things man"), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")) + val statusPoller = TestProbe() + + val promise = Promise[BackendJobExecutionResponse]() + val jobDescriptor = buildPreemptibleJobDescriptor(attempt, preemptible) + val backend = executionActor(jobDescriptor, JesBackendConfigurationDescriptor, promise, statusPoller.ref) + backend ! Execute + statusPoller.expectMsgPF(max = Timeout, hint = "awaiting status poll") { + case DoPoll(_) => backend ! runStatus } + + Await.result(promise.future, Timeout) } def buildPreemptibleTestActorRef(attempt: Int, preemptible: Int): TestActorRef[TestableJesJobExecutionActor] = { @@ -180,7 +186,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend expectations foreach { case (attempt, preemptible, errorCode, innerErrorCode, shouldRetry) => it should s"handle call failures appropriately with respect to preemption (attempt=$attempt, preemptible=$preemptible, errorCode=$errorCode, innerErrorCode=$innerErrorCode)" in { - run(attempt, preemptible, errorCode, innerErrorCode).getClass.getSimpleName match { + runAndFail(attempt, preemptible, errorCode, innerErrorCode).getClass.getSimpleName match { case "FailedNonRetryableResponse" => false shouldBe shouldRetry case "FailedRetryableResponse" => true shouldBe shouldRetry case huh => fail(s"Unexpected response class name: '$huh'") @@ -195,7 +201,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val handle = mock[JesPendingExecutionHandle] implicit val ec = system.dispatcher - val failedStatus = Failed(10, Some("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")) + val failedStatus = Failed(10, List("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")) val executionResult = Await.result(jesBackend.executionResult(failedStatus, handle), 2.seconds) executionResult.isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true val failedHandle = executionResult.asInstanceOf[FailedNonRetryableExecutionHandle] @@ -208,7 +214,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val handle = mock[JesPendingExecutionHandle] implicit val ec = system.dispatcher - val failedStatus = Failed(10, Some("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")) + val failedStatus = Failed(10, List("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")) val executionResult = Await.result(jesBackend.executionResult(failedStatus, handle), 2.seconds) executionResult.isInstanceOf[FailedRetryableExecutionHandle] shouldBe true val retryableHandle = executionResult.asInstanceOf[FailedRetryableExecutionHandle] @@ -224,7 +230,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val handle = mock[JesPendingExecutionHandle] implicit val ec = system.dispatcher - val failedStatus = Failed(10, Some("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")) + val failedStatus = Failed(10, List("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")) val executionResult = Await.result(jesBackend.executionResult(failedStatus, handle), 2.seconds) executionResult.isInstanceOf[FailedRetryableExecutionHandle] shouldBe true val retryableHandle = executionResult.asInstanceOf[FailedRetryableExecutionHandle] @@ -241,22 +247,22 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend implicit val ec = system.dispatcher Await.result(jesBackend.executionResult( - Failed(10, Some("15: Other type of error."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds + Failed(10, List("15: Other type of error."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true Await.result(jesBackend.executionResult( - Failed(11, Some("14: Wrong errorCode."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds + Failed(11, List("14: Wrong errorCode."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true Await.result(jesBackend.executionResult( - Failed(10, Some("Weird error message."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds + Failed(10, List("Weird error message."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true Await.result(jesBackend.executionResult( - Failed(10, Some("UnparsableInt: Even weirder error message."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds + Failed(10, List("UnparsableInt: Even weirder error message."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true Await.result(jesBackend.executionResult( - Failed(10, None, Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds + Failed(10, List.empty, Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true Await.result(jesBackend.executionResult( - Failed(10, Some("Operation canceled at"), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds + Failed(10, List("Operation canceled at"), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds ) shouldBe AbortedExecutionHandle actorRef.stop() diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala index 0486d9ec5..39430abb2 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala @@ -38,8 +38,7 @@ class RunSpec extends FlatSpec with Matchers with MockitoTrait { val mockedCredentials = new MockGoogleCredential.Builder().build() val genomics = new Genomics(mockedCredentials.getTransport, mockedCredentials.getJsonFactory, mockedCredentials) - val run = new Run("runId", genomics) - val list = run.getEventList(op) + val list = Run.getEventList(op) list should contain theSameElementsAs List( ExecutionEvent("waiting for quota", OffsetDateTime.parse("2015-12-05T00:00:00+00:00")), ExecutionEvent("initializing VM", OffsetDateTime.parse("2015-12-05T00:00:01+00:00")), diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala new file mode 100644 index 000000000..1eaa42297 --- /dev/null +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala @@ -0,0 +1,153 @@ +package cromwell.backend.impl.jes.statuspolling + +import akka.actor.{ActorRef, Props} +import akka.testkit.{TestActorRef, TestProbe} +import cromwell.backend.impl.jes.Run +import cromwell.core.TestKitSuite +import org.scalatest.{FlatSpecLike, Matchers} + +import scala.concurrent.duration._ +import akka.testkit._ +import JesApiQueryManagerSpec._ +import cromwell.util.AkkaTestUtil +import org.scalatest.concurrent.Eventually + +import scala.collection.immutable.Queue + +class JesApiQueryManagerSpec extends TestKitSuite("JesApiQueryManagerSpec") with FlatSpecLike with Matchers with Eventually { + + behavior of "JesApiQueryManagerSpec" + + implicit val TestExecutionTimeout = 10.seconds.dilated + implicit val DefaultPatienceConfig = PatienceConfig(TestExecutionTimeout) + val AwaitAlmostNothing = 30.milliseconds.dilated + val BatchSize = 5 + + it should "queue up and dispense status poll requests, in order" in { + val statusPoller = TestProbe(name = "StatusPoller") + val jaqmActor: TestActorRef[TestJesApiQueryManager] = TestActorRef(TestJesApiQueryManager.props(statusPoller.ref)) + + var statusRequesters = ((0 until BatchSize * 2) map { i => i -> TestProbe(name = s"StatusRequester_$i") }).toMap + + // Initially, we should have no work: + jaqmActor.tell(msg = JesApiQueryManager.RequestJesPollingWork(BatchSize), sender = statusPoller.ref) + statusPoller.expectMsg(max = TestExecutionTimeout, obj = JesApiQueryManager.NoWorkToDo) + + // Send a few status poll requests: + statusRequesters foreach { case (index, probe) => + jaqmActor.tell(msg = JesApiQueryManager.DoPoll(Run(index.toString, null)), sender = probe.ref) + } + + // Should have no messages to the actual statusPoller yet: + statusPoller.expectNoMsg(max = AwaitAlmostNothing) + + // Verify batches: + 2 times { + jaqmActor.tell(msg = JesApiQueryManager.RequestJesPollingWork(BatchSize), sender = statusPoller.ref) + statusPoller.expectMsgPF(max = TestExecutionTimeout) { + case JesApiQueryManager.JesPollingWorkBatch(workBatch) => + val requesters = statusRequesters.take(BatchSize) + statusRequesters = statusRequesters.drop(BatchSize) + + val zippedWithRequesters = workBatch.toList.zip(requesters) + zippedWithRequesters foreach { case (pollQuery, (index, testProbe)) => + pollQuery.requester should be(testProbe.ref) + pollQuery.run.runId should be(index.toString) + } + } + } + + // Finally, we should have no work: + jaqmActor.tell(msg = JesApiQueryManager.RequestJesPollingWork(BatchSize), sender = statusPoller.ref) + statusPoller.expectMsg(max = TestExecutionTimeout, obj = JesApiQueryManager.NoWorkToDo) + + jaqmActor.underlyingActor.testPollerCreations should be(1) + } + + AkkaTestUtil.actorDeathMethods(system) foreach { case (name, stopMethod) => + it should s"catch polling actors if they $name and then recreate them" in { + + val statusPoller1 = TestActorRef(Props(new AkkaTestUtil.DeathTestActor()), TestActorRef(new AkkaTestUtil.StoppingSupervisor())) + val statusPoller2 = TestActorRef(Props(new AkkaTestUtil.DeathTestActor())) + val jaqmActor: TestActorRef[TestJesApiQueryManager] = TestActorRef(TestJesApiQueryManager.props(statusPoller1, statusPoller2)) + + val statusRequesters = ((0 until BatchSize * 2) map { i => i -> TestProbe(name = s"StatusRequester_$i") }).toMap + + // Send a few status poll requests: + BatchSize indexedTimes { index => + val probe = statusRequesters(index) + jaqmActor.tell(msg = JesApiQueryManager.DoPoll(Run(index.toString, null)), sender = probe.ref) + } + BatchSize indexedTimes { i => + val index = i + BatchSize // For the second half of the statusRequester set + val probe = statusRequesters(index) + jaqmActor.tell(msg = JesApiQueryManager.DoPoll(Run(index.toString, null)), sender = probe.ref) + } + + // Request a set of work from the middle of the queue: + val batchOffset = 2 + jaqmActor.tell(msg = JesApiQueryManager.RequestJesPollingWork(batchOffset), sender = statusPoller1) + jaqmActor.tell(msg = JesApiQueryManager.RequestJesPollingWork(BatchSize), sender = statusPoller1) + + // Kill the original status poller: + stopMethod(statusPoller1) + + // Only the appropriate requesters get an error: + (0 until batchOffset) foreach { index => + val probe = statusRequesters(index) + probe.expectNoMsg(max = AwaitAlmostNothing) + } + (batchOffset until batchOffset + BatchSize) foreach { index => + val probe = statusRequesters(index) + probe.expectMsg(max = TestExecutionTimeout, hint = s"Polling error to requester #$index", obj = JesPollingActor.JesPollError) + } + (batchOffset + BatchSize until 2 * BatchSize) foreach { index => + val probe = statusRequesters(index) + probe.expectNoMsg(max = AwaitAlmostNothing) + } + + // Check the next status poller gets created: + eventually { jaqmActor.underlyingActor.testPollerCreations should be(2) } + } + } +} + +object JesApiQueryManagerSpec { + implicit class intWithTimes(n: Int) { + def times(f: => Unit) = 1 to n foreach { _ => f } + def indexedTimes(f: Int => Unit) = 0 until n foreach { i => f(i) } + } +} + +/** + * This test class allows us to hook into the JesApiQueryManager's makeStatusPoller and provide our own TestProbes instead + */ +class TestJesApiQueryManager(statusPollerProbes: ActorRef*) extends JesApiQueryManager { + + var testProbes: Queue[ActorRef] = _ + var testPollerCreations: Int = _ + + private def init() = { + testProbes = Queue(statusPollerProbes: _*) + testPollerCreations = 0 + } + + override private[statuspolling] def makeStatusPoller(): ActorRef = { + // Initialise the queue, if necessary: + if (testProbes == null) { + init() + } + + // Register that the creation was requested: + testPollerCreations += 1 + + // Pop the queue to get the next test probe: + val (probe, newQueue) = testProbes.dequeue + testProbes = newQueue + probe + } +} + +object TestJesApiQueryManager { + def props(statusPollers: ActorRef*): Props = Props(new TestJesApiQueryManager(statusPollers: _*)) +} diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala new file mode 100644 index 000000000..b861cbf0f --- /dev/null +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala @@ -0,0 +1,131 @@ +package cromwell.backend.impl.jes.statuspolling + +import akka.actor.{ActorRef, Props} +import akka.testkit.{TestActorRef, TestProbe} +import cromwell.core.{ExecutionEvent, TestKitSuite} +import org.scalatest.{BeforeAndAfter, FlatSpecLike, Matchers} +import org.scalatest.concurrent.Eventually + +import scala.concurrent.duration._ +import akka.testkit._ +import cats.data.NonEmptyList +import com.google.api.client.googleapis.batch.BatchRequest +import com.google.api.client.googleapis.batch.json.JsonBatchCallback +import com.google.api.client.googleapis.json.GoogleJsonError +import com.google.api.services.genomics.model.Operation +import cromwell.backend.impl.jes.{Run, RunStatus} +import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.JesStatusPollQuery +import cromwell.backend.impl.jes.statuspolling.JesPollingActor.JesPollFailed +import cromwell.backend.impl.jes.statuspolling.TestJesPollingActor.{CallbackFailure, CallbackSuccess, JesBatchCallbackResponse} +import org.specs2.mock.Mockito + +import scala.collection.immutable.Queue + +class JesPollingActorSpec extends TestKitSuite("JesPollingActor") with FlatSpecLike with Matchers with Eventually with BeforeAndAfter with Mockito { + + behavior of "JesPollingActor" + + implicit val TestExecutionTimeout = 10.seconds.dilated + implicit val DefaultPatienceConfig = PatienceConfig(TestExecutionTimeout) + val AwaitAlmostNothing = 30.milliseconds.dilated + + var managerProbe: TestProbe = _ + var jpActor: TestActorRef[TestJesPollingActor] = _ + + it should "query for work and wait for a reply" in { + managerProbe.expectMsgClass(max = TestExecutionTimeout, c = classOf[JesApiQueryManager.RequestJesPollingWork]) + managerProbe.expectNoMsg(max = AwaitAlmostNothing) + } + + it should "respond directly to requesters with various run statuses" in { + managerProbe.expectMsgClass(max = TestExecutionTimeout, c = classOf[JesApiQueryManager.RequestJesPollingWork]) + + val requester1 = TestProbe() + val query1 = JesStatusPollQuery(requester1.ref, mock[Run]) + val requester2 = TestProbe() + val query2 = JesStatusPollQuery(requester2.ref, mock[Run]) + val requester3 = TestProbe() + val query3 = JesStatusPollQuery(requester3.ref, mock[Run]) + + // For two requests the callback succeeds (first with RunStatus.Success, then RunStatus.Failed). The third callback fails: + jpActor.underlyingActor.callbackResponses :+= CallbackSuccess + jpActor.underlyingActor.callbackResponses :+= CallbackSuccess + jpActor.underlyingActor.callbackResponses :+= CallbackFailure + + val successStatus = RunStatus.Success(Seq.empty[ExecutionEvent], None, None, None) + val failureStatus = RunStatus.Failed(-1, List.empty[String], Seq.empty[ExecutionEvent], None, None, None) + jpActor.underlyingActor.operationStatusResponses :+= successStatus + jpActor.underlyingActor.operationStatusResponses :+= failureStatus + + jpActor.tell(msg = JesApiQueryManager.JesPollingWorkBatch(NonEmptyList(query1, List(query2, query3))), sender = managerProbe.ref) + eventually { jpActor.underlyingActor.resultHandlers.size should be(3) } + eventually { jpActor.underlyingActor.runBatchRequested should be(true) } + + // The manager shouldn't have been asked for more work yet: + managerProbe.expectNoMsg(max = AwaitAlmostNothing) + + // Ok, let's trigger the callbacks: + jpActor.underlyingActor.executeBatch() + + requester1.expectMsg(successStatus) + requester2.expectMsg(failureStatus) + requester3.expectMsgClass(classOf[JesPollFailed]) + + // And the poller is done! Now the manager should now have (only one) request for more work: + managerProbe.expectMsgClass(max = TestExecutionTimeout, c = classOf[JesApiQueryManager.RequestJesPollingWork]) + } + + before { + managerProbe = TestProbe() + jpActor = TestActorRef(TestJesPollingActor.props(managerProbe.ref), managerProbe.ref) + } +} + +object JesPollingActorSpec extends Mockito { + def mockRun(runId: String): Run = { + val run = mock[Run] + run.runId returns runId + run + } +} + +/** + * Testable JES polling actor. + * - Mocks out the methods which actually call out to JES, and allows the callbacks to be triggered in a testable way + * - Also waits a **lot** less time before polls! + */ +class TestJesPollingActor(manager: ActorRef) extends JesPollingActor(manager) with Mockito { + override val BatchInterval = 10.milliseconds + + var operationStatusResponses: Queue[RunStatus] = Queue.empty + var resultHandlers: Queue[JsonBatchCallback[Operation]] = Queue.empty + var callbackResponses: Queue[JesBatchCallbackResponse] = Queue.empty + var runBatchRequested: Boolean = false + + override private[statuspolling] def createBatch(run: Run): BatchRequest = null + override private[statuspolling] def runBatch(batch: BatchRequest): Unit = runBatchRequested = true + + def executeBatch(): Unit = { + resultHandlers.zip(callbackResponses) foreach { case (handler, response) => response match { + case CallbackSuccess => handler.onSuccess(null, null) + case CallbackFailure => + val error: GoogleJsonError = null + handler.onFailure(error, null) + }} + } + override private[statuspolling] def enqueueStatusPollInBatch(run: Run, batch: BatchRequest, resultHandler: JsonBatchCallback[Operation]): Unit = resultHandlers :+= resultHandler + override private[statuspolling] def interpretOperationStatus(operation: Operation): RunStatus = { + val (status, newQueue) = operationStatusResponses.dequeue + operationStatusResponses = newQueue + status + } + override private[statuspolling] def mkErrorString(e: GoogleJsonError) = "NA" +} + +object TestJesPollingActor { + def props(manager: ActorRef) = Props(new TestJesPollingActor(manager)) + + sealed trait JesBatchCallbackResponse + case object CallbackSuccess extends JesBatchCallbackResponse + case object CallbackFailure extends JesBatchCallbackResponse +} diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala index d812b1c8e..edacd5303 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala @@ -50,7 +50,8 @@ trait SharedFileSystemBackendLifecycleActorFactory extends BackendLifecycleActor override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationDataOption: Option[BackendInitializationData], - serviceRegistryActor: ActorRef) = { + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]) = { def propsCreator(completionPromise: Promise[BackendJobExecutionResponse]): Props = { val params = SharedFileSystemAsyncJobExecutionActorParams(serviceRegistryActor, jobDescriptor, configurationDescriptor, completionPromise, initializationDataOption) diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala index ba09b6cb9..ef01c9985 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala @@ -13,8 +13,10 @@ case class SparkBackendFactory(name: String, configurationDescriptor: BackendCon Option(SparkInitializationActor.props(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor)) } - override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef): Props = { + override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, + initializationData: Option[BackendInitializationData], + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props = { SparkJobExecutionActor.props(jobDescriptor, configurationDescriptor) } From 5ceeee5c63b9f85b44102a86fcd3d26ea4a95d5f Mon Sep 17 00:00:00 2001 From: Ruchi Date: Wed, 12 Oct 2016 11:14:44 -0400 Subject: [PATCH 020/375] fix gotc build (#1557) --- docker/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/install.sh b/docker/install.sh index 2afc67f9e..b9fb8e91a 100755 --- a/docker/install.sh +++ b/docker/install.sh @@ -4,7 +4,7 @@ set -e CROMWELL_DIR=$1 cd $CROMWELL_DIR -sbt 'set test in assembly := {}' notests:assembly +sbt assembly CROMWELL_JAR=$(find target | grep 'cromwell.*\.jar') mv $CROMWELL_JAR ./cromwell.jar sbt clean From d8b0bd90c7ba3c0a87eb99ca0f9d9b77937e88a5 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Wed, 12 Oct 2016 15:02:56 -0400 Subject: [PATCH 021/375] Modified the reference docker submit command to avoid bash redirection --- core/src/main/resources/reference.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 9bf2807ea..a9a7bb7bf 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -149,7 +149,7 @@ backend { run-in-background = true runtime-attributes = "String? docker" submit = "/bin/bash ${script}" - submit-docker = "docker run --rm -v ${cwd}:${docker_cwd} -i ${docker} /bin/bash < ${script}" + submit-docker = "docker run --rm -v ${cwd}:${docker_cwd} -i ${docker} /bin/bash ${docker_cwd}/execution/script" # Root directory where Cromwell writes job results. This directory must be # visible and writeable by the Cromwell process as well as the jobs that Cromwell From 86c80a890941e0cc7b2549ce0aa5f41316194ad4 Mon Sep 17 00:00:00 2001 From: Thib Date: Wed, 12 Oct 2016 17:28:58 -0400 Subject: [PATCH 022/375] Call caching output copy failures handling. Closes #1510 (#1559) * smarter handling of cache hit copying failures --- core/src/main/scala/cromwell/core/PathCopier.scala | 16 +++- .../database/slick/CallCachingSlickDatabase.scala | 8 ++ .../slick/tables/CallCachingEntryComponent.scala | 7 ++ .../database/sql/CallCachingSqlDatabase.scala | 5 +- .../execution/EngineJobExecutionActor.scala | 90 +++++++++++++++++----- .../execution/callcaching/CallCache.scala | 24 +++--- .../callcaching/CallCacheInvalidateActor.scala | 36 +++++++++ .../execution/callcaching/CallCacheReadActor.scala | 4 +- .../callcaching/EngineJobHashingActor.scala | 11 ++- .../callcaching/FetchCachedResultsActor.scala | 14 ++-- .../execution/callcaching/EJHADataSpec.scala | 18 ++--- .../callcaching/EngineJobHashingActorSpec.scala | 35 +++++---- .../PredictableCallCacheReadActor.scala | 6 +- .../EjeaBackendIsCopyingCachedOutputsSpec.scala | 34 ++++---- .../execution/ejea/EjeaCheckingCallCacheSpec.scala | 13 ++-- ...EjeaFetchingCachedOutputsFromDatabaseSpec.scala | 14 ++-- .../ejea/EjeaInvalidatingCacheEntrySpec.scala | 53 +++++++++++++ .../ejea/EngineJobExecutionActorSpec.scala | 1 + .../ejea/EngineJobExecutionActorSpecUtil.scala | 20 +++++ .../lifecycle/execution/ejea/PerTestHelper.scala | 13 ++-- .../JobExecutionTokenDispenserActorSpec.scala | 8 +- .../backend/impl/jes/JesCacheHitCopyingActor.scala | 2 +- 22 files changed, 312 insertions(+), 120 deletions(-) create mode 100644 engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala create mode 100644 engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaInvalidatingCacheEntrySpec.scala diff --git a/core/src/main/scala/cromwell/core/PathCopier.scala b/core/src/main/scala/cromwell/core/PathCopier.scala index c941ea890..f90dad604 100644 --- a/core/src/main/scala/cromwell/core/PathCopier.scala +++ b/core/src/main/scala/cromwell/core/PathCopier.scala @@ -1,9 +1,12 @@ package cromwell.core +import java.io.IOException import java.nio.file.Path import better.files._ +import scala.util.{Failure, Try} + object PathCopier { def getDestinationFilePath(sourceContextPath: Path, sourceFilePath: Path, destinationDirPath: Path): Path = { val relativeFileString = sourceContextPath.toAbsolutePath.relativize(sourceFilePath.toAbsolutePath).toString @@ -13,7 +16,7 @@ object PathCopier { /** * Copies from a relative source to destination dir. NOTE: Copies are not atomic, and may create a partial copy. */ - def copy(sourceContextPath: Path, sourceFilePath: Path, destinationDirPath: Path): Unit = { + def copy(sourceContextPath: Path, sourceFilePath: Path, destinationDirPath: Path): Try[Unit] = { val destinationFilePath = getDestinationFilePath(sourceContextPath, sourceFilePath, destinationDirPath) copy(sourceFilePath, destinationFilePath) } @@ -21,9 +24,14 @@ object PathCopier { /** * Copies from source to destination. NOTE: Copies are not atomic, and may create a partial copy. */ - def copy(sourceFilePath: Path, destinationFilePath: Path): Unit = { + def copy(sourceFilePath: Path, destinationFilePath: Path): Try[Unit] = { Option(File(destinationFilePath).parent).foreach(_.createDirectories()) - File(sourceFilePath).copyTo(destinationFilePath, overwrite = true) - () + Try { + File(sourceFilePath).copyTo(destinationFilePath, overwrite = true) + + () + } recoverWith { + case ex => Failure(new IOException(s"Failed to copy ${sourceFilePath.toUri} to ${destinationFilePath.toUri}", ex)) + } } } diff --git a/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala index 51ea7c3a4..ce40e4e6a 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala @@ -46,4 +46,12 @@ trait CallCachingSlickDatabase extends CallCachingSqlDatabase { runTransaction(action) } + + override def invalidateCall(callCachingEntryId: Int) + (implicit ec: ExecutionContext): Future[Unit] = { + import cats.syntax.functor._ + import cats.instances.future._ + val action = dataAccess.allowResultReuseForCallCachingEntryId(callCachingEntryId).update(false) + runTransaction(action) void + } } diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala index 8cf97e2a5..189402814 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala @@ -40,4 +40,11 @@ trait CallCachingEntryComponent { if callCachingEntry.callCachingEntryId === callCachingEntryId } yield callCachingEntry ) + + val allowResultReuseForCallCachingEntryId = Compiled( + (callCachingEntryId: Rep[Int]) => for { + callCachingEntry <- callCachingEntries + if callCachingEntry.callCachingEntryId === callCachingEntryId + } yield callCachingEntry.allowResultReuse + ) } diff --git a/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala index abf262ae3..b2b3d221a 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala @@ -11,6 +11,9 @@ trait CallCachingSqlDatabase { def queryCallCachingEntryIds(hashKeyHashValues: NonEmptyList[(String, String)]) (implicit ec: ExecutionContext): Future[Seq[Int]] - def queryCallCaching(callCachingResultMetainfoId: Int) + def queryCallCaching(callCachingEntryId: Int) (implicit ec: ExecutionContext): Future[Option[CallCachingJoin]] + + def invalidateCall(callCachingEntryId: Int) + (implicit ec: ExecutionContext): Future[Unit] } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala index 07114e7e8..25c1852dc 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala @@ -4,6 +4,7 @@ import java.time.OffsetDateTime import akka.actor.{ActorRef, ActorRefFactory, LoggingFSM, Props} import akka.routing.RoundRobinPool +import cats.data.NonEmptyList import cromwell.backend.BackendCacheHitCopyingActor.CopyOutputsCommand import cromwell.backend.BackendJobExecutionActor._ import cromwell.backend.{BackendInitializationData, BackendJobDescriptor, BackendJobDescriptorKey, BackendLifecycleActorFactory} @@ -129,8 +130,8 @@ class EngineJobExecutionActor(replyTo: ActorRef, writeToMetadata(Map(callCachingReadResultMetadataKey -> "Cache Miss")) log.debug("Cache miss for job {}", jobTag) runJob(data) - case Event(hit @ CacheHit(cacheResultId), data: ResponsePendingData) => - fetchCachedResults(data, jobDescriptorKey.call.task.outputs, hit) + case Event(hit: CacheHit, data: ResponsePendingData) => + fetchCachedResults(jobDescriptorKey.call.task.outputs, hit.cacheResultIds.head, data) case Event(HashError(t), data: ResponsePendingData) => writeToMetadata(Map(callCachingReadResultMetadataKey -> s"Hashing Error: ${t.getMessage}")) disableCallCaching(t) @@ -141,8 +142,8 @@ class EngineJobExecutionActor(replyTo: ActorRef, case Event(CachedOutputLookupSucceeded(wdlValueSimpletons, jobDetritus, returnCode, cacheResultId, cacheHitDetails), data: ResponsePendingData) => writeToMetadata(Map(callCachingReadResultMetadataKey -> s"Cache Hit: $cacheHitDetails")) log.debug("Cache hit for {}! Fetching cached result {}", jobTag, cacheResultId) - makeBackendCopyCacheHit(cacheResultId, wdlValueSimpletons, jobDetritus, returnCode, data) - case Event(CachedOutputLookupFailed(metaInfoId, error), data: ResponsePendingData) => + makeBackendCopyCacheHit(wdlValueSimpletons, jobDetritus, returnCode, data) + case Event(CachedOutputLookupFailed(callCachingEntryId, error), data: ResponsePendingData) => log.warning("Can't make a copy of the cached job outputs for {} due to {}. Running job.", jobTag, error) runJob(data) case Event(hashes: CallCacheHashes, data: ResponsePendingData) => @@ -155,20 +156,21 @@ class EngineJobExecutionActor(replyTo: ActorRef, when(BackendIsCopyingCachedOutputs) { // Backend copying response: - case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, Some(Success(hashes)))) => + case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, Some(Success(hashes)), _)) => saveCacheResults(hashes, data.withSuccessResponse(response)) - case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, None)) if effectiveCallCachingMode.writeToCache => + case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, None, _)) if effectiveCallCachingMode.writeToCache => // Wait for the CallCacheHashes stay using data.withSuccessResponse(response) case Event(response: SucceededResponse, data: ResponsePendingData) => // bad hashes or cache write off saveJobCompletionToJobStore(data.withSuccessResponse(response)) - case Event(response: BackendJobExecutionResponse, data: ResponsePendingData) => - // This matches all response types other than `SucceededResponse`. + case Event(response: BackendJobExecutionResponse, data @ ResponsePendingData(_, _, _, Some(cacheHit))) => response match { - case f: BackendJobFailedResponse =>log.error("{}: Failed copying cache results, falling back to running job: {}", jobDescriptorKey, f.throwable) - case _ => // + case f: BackendJobFailedResponse => + invalidateCacheHit(cacheHit.cacheResultIds.head) + log.error(f.throwable, "Failed copying cache results for job {}, invalidating cache entry.", jobDescriptorKey) + goto(InvalidatingCacheEntry) + case _ => runJob(data) } - runJob(data) // Hashes arrive: case Event(hashes: CallCacheHashes, data: SucceededResponseData) => @@ -186,6 +188,21 @@ class EngineJobExecutionActor(replyTo: ActorRef, stay using data.copy(hashes = Option(Failure(t))) } + when(InvalidatingCacheEntry) { + case Event(response: CallCacheInvalidatedResponse, data: ResponsePendingData) => + handleCacheInvalidatedResponse(response, data) + + // Hashes arrive: + case Event(hashes: CallCacheHashes, data: ResponsePendingData) => + addHashesAndStay(data, hashes) + + // Hash error occurs: + case Event(HashError(t), data: ResponsePendingData) => + disableCacheWrite(t) + // Can't write hashes for this job, but continue to wait for the copy response. + stay using data.copy(hashes = Option(Failure(t))) + } + when(RunningJob) { case Event(hashes: CallCacheHashes, data: SucceededResponseData) => saveCacheResults(hashes, data) @@ -199,10 +216,10 @@ class EngineJobExecutionActor(replyTo: ActorRef, disableCallCaching(t) stay using data.copy(hashes = Option(Failure(t))) - case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, Some(Success(hashes)))) if effectiveCallCachingMode.writeToCache => + case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, Some(Success(hashes)), _)) if effectiveCallCachingMode.writeToCache => eventList ++= response.executionEvents saveCacheResults(hashes, data.withSuccessResponse(response)) - case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, None)) if effectiveCallCachingMode.writeToCache => + case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, None, _)) if effectiveCallCachingMode.writeToCache => log.debug(s"Got job result for {}, awaiting hashes", jobTag) stay using data.withSuccessResponse(response) case Event(response: BackendJobExecutionResponse, data: ResponsePendingData) => @@ -302,24 +319,24 @@ class EngineJobExecutionActor(replyTo: ActorRef, () } - def makeFetchCachedResultsActor(cacheHit: CacheHit, taskOutputs: Seq[TaskOutput]): Unit = { - context.actorOf(FetchCachedResultsActor.props(cacheHit, self, new CallCache(SingletonServicesStore.databaseInterface))) + def makeFetchCachedResultsActor(callCachingEntryId: CallCachingEntryId, taskOutputs: Seq[TaskOutput]): Unit = { + context.actorOf(FetchCachedResultsActor.props(callCachingEntryId, self, new CallCache(SingletonServicesStore.databaseInterface))) () } - private def fetchCachedResults(data: ResponsePendingData, taskOutputs: Seq[TaskOutput], cacheHit: CacheHit) = { - makeFetchCachedResultsActor(cacheHit, taskOutputs) - goto(FetchingCachedOutputsFromDatabase) + private def fetchCachedResults(taskOutputs: Seq[TaskOutput], callCachingEntryId: CallCachingEntryId, data: ResponsePendingData) = { + makeFetchCachedResultsActor(callCachingEntryId, taskOutputs) + goto(FetchingCachedOutputsFromDatabase) using data } - private def makeBackendCopyCacheHit(cacheHit: CacheHit, wdlValueSimpletons: Seq[WdlValueSimpleton], jobDetritusFiles: Map[String,String], returnCode: Option[Int], data: ResponsePendingData) = { + private def makeBackendCopyCacheHit(wdlValueSimpletons: Seq[WdlValueSimpleton], jobDetritusFiles: Map[String,String], returnCode: Option[Int], data: ResponsePendingData) = { factory.cacheHitCopyingActorProps match { case Some(propsMaker) => val backendCacheHitCopyingActorProps = propsMaker(data.jobDescriptor, initializationData, serviceRegistryActor) val cacheHitCopyActor = context.actorOf(backendCacheHitCopyingActorProps, buildCacheHitCopyingActorName(data.jobDescriptor)) cacheHitCopyActor ! CopyOutputsCommand(wdlValueSimpletons, jobDetritusFiles, returnCode) replyTo ! JobRunning(data.jobDescriptor, None) - goto(BackendIsCopyingCachedOutputs) using data + goto(BackendIsCopyingCachedOutputs) case None => // This should be impossible with the FSM, but luckily, we CAN recover if some foolish future programmer makes this happen: val errorMessage = "Call caching copying should never have even been attempted with no copy actor props! (Programmer error!)" @@ -337,6 +354,22 @@ class EngineJobExecutionActor(replyTo: ActorRef, goto(RunningJob) using data } + private def handleCacheInvalidatedResponse(response: CallCacheInvalidatedResponse, data: ResponsePendingData) = { + response match { + case CallCacheInvalidatedFailure(failure) => log.error(failure, "Failed to invalidate cache entry for job: {}", jobDescriptorKey) + case _ => + } + + data.popCacheHitId match { + case newData @ ResponsePendingData(_, _, _, Some(cacheHit)) => + log.info("Trying to use another cache hit for job: {}", jobDescriptorKey) + fetchCachedResults(jobDescriptorKey.call.task.outputs, cacheHit.cacheResultIds.head, newData) + case newData => + log.info("Could not find another cache hit, falling back to running job: {}", jobDescriptorKey) + runJob(newData) + } + } + private def buildJobExecutionActorName(jobDescriptor: BackendJobDescriptor) = { s"$workflowId-BackendJobExecutionActor-$jobTag" } @@ -351,6 +384,12 @@ class EngineJobExecutionActor(replyTo: ActorRef, () } + protected def invalidateCacheHit(cacheId: CallCachingEntryId): Unit = { + val callCache = new CallCache(SingletonServicesStore.databaseInterface) + context.actorOf(CallCacheInvalidateActor.props(callCache, cacheId), s"CallCacheInvalidateActor${cacheId.id}-$tag") + () + } + private def saveCacheResults(hashes: CallCacheHashes, data: SucceededResponseData) = { createSaveCacheResultsActor(hashes, data.successResponse) val updatedData = data.copy(hashes = Option(Success(hashes))) @@ -439,6 +478,7 @@ object EngineJobExecutionActor { case object RunningJob extends EngineJobExecutionActorState case object UpdatingCallCache extends EngineJobExecutionActorState case object UpdatingJobStore extends EngineJobExecutionActorState + case object InvalidatingCacheEntry extends EngineJobExecutionActorState /** Commands */ sealed trait EngineJobExecutionActorCommand @@ -483,7 +523,8 @@ object EngineJobExecutionActor { private[execution] case class ResponsePendingData(jobDescriptor: BackendJobDescriptor, bjeaProps: Props, - hashes: Option[Try[CallCacheHashes]] = None) extends EJEAData { + hashes: Option[Try[CallCacheHashes]] = None, + cacheHit: Option[CacheHit] = None) extends EJEAData { def withSuccessResponse(success: SucceededResponse) = SucceededResponseData(success, hashes) @@ -491,6 +532,13 @@ object EngineJobExecutionActor { case success: SucceededResponse => SucceededResponseData(success, hashes) case failure => NotSucceededResponseData(failure, hashes) } + + def withCacheHit(cacheHit: Option[CacheHit]) = this.copy(cacheHit = cacheHit) + + def popCacheHitId: ResponsePendingData = cacheHit match { + case Some(hit) => this.copy(cacheHit = NonEmptyList.fromList(hit.cacheResultIds.tail) map CacheHit.apply) + case None => this + } } private[execution] trait ResponseData extends EJEAData { diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala index 16ecc89f0..8c5331c42 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala @@ -13,7 +13,7 @@ import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashing import scala.concurrent.{ExecutionContext, Future} -final case class MetaInfoId(id: Int) +final case class CallCachingEntryId(id: Int) /** * Given a database-layer CallCacheStore, this accessor can access the database with engine-friendly data types. @@ -34,7 +34,7 @@ class CallCache(database: CallCachingSqlDatabase) { addToCache(metaInfo, hashes, result, jobDetritus) } - private def addToCache(metaInfo: CallCachingEntry, hashes: Set[HashResult], + private def addToCache(callCachingEntry: CallCachingEntry, hashes: Set[HashResult], result: Iterable[WdlValueSimpleton], jobDetritus: Map[String, String]) (implicit ec: ExecutionContext): Future[Unit] = { @@ -56,25 +56,29 @@ class CallCache(database: CallCachingSqlDatabase) { } val callCachingJoin = - CallCachingJoin(metaInfo, hashesToInsert.toSeq, resultToInsert.toSeq, jobDetritusToInsert.toSeq) + CallCachingJoin(callCachingEntry, hashesToInsert.toSeq, resultToInsert.toSeq, jobDetritusToInsert.toSeq) database.addCallCaching(callCachingJoin) } - def fetchMetaInfoIdsMatchingHashes(callCacheHashes: CallCacheHashes)(implicit ec: ExecutionContext): Future[Set[MetaInfoId]] = { - metaInfoIdsMatchingHashes(NonEmptyList.fromListUnsafe(callCacheHashes.hashes.toList)) + def callCachingEntryIdsMatchingHashes(callCacheHashes: CallCacheHashes)(implicit ec: ExecutionContext): Future[Set[CallCachingEntryId]] = { + callCachingEntryIdsMatchingHashes(NonEmptyList.fromListUnsafe(callCacheHashes.hashes.toList)) } - private def metaInfoIdsMatchingHashes(hashKeyValuePairs: NonEmptyList[HashResult]) - (implicit ec: ExecutionContext): Future[Set[MetaInfoId]] = { + private def callCachingEntryIdsMatchingHashes(hashKeyValuePairs: NonEmptyList[HashResult]) + (implicit ec: ExecutionContext): Future[Set[CallCachingEntryId]] = { val result = database.queryCallCachingEntryIds(hashKeyValuePairs map { case HashResult(hashKey, hashValue) => (hashKey.key, hashValue.value) }) - result.map(_.toSet.map(MetaInfoId)) + result.map(_.toSet.map(CallCachingEntryId)) } - def fetchCachedResult(metaInfoId: MetaInfoId)(implicit ec: ExecutionContext): Future[Option[CallCachingJoin]] = { - database.queryCallCaching(metaInfoId.id) + def fetchCachedResult(callCachingEntryId: CallCachingEntryId)(implicit ec: ExecutionContext): Future[Option[CallCachingJoin]] = { + database.queryCallCaching(callCachingEntryId.id) + } + + def invalidate(callCachingEntryId: CallCachingEntryId)(implicit ec: ExecutionContext) = { + database.invalidateCall(callCachingEntryId.id) } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala new file mode 100644 index 000000000..ef09d32e9 --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala @@ -0,0 +1,36 @@ +package cromwell.engine.workflow.lifecycle.execution.callcaching + +import akka.actor.{Actor, ActorLogging, Props} + +import scala.concurrent.ExecutionContext +import scala.util.{Failure, Success} + +class CallCacheInvalidateActor(callCache: CallCache, cacheId: CallCachingEntryId) extends Actor with ActorLogging { + + implicit val ec: ExecutionContext = context.dispatcher + + def receiver = context.parent + + callCache.invalidate(cacheId) onComplete { + case Success(_) => + receiver ! CallCacheInvalidatedSuccess + context.stop(self) + case Failure(t) => + receiver ! CallCacheInvalidatedFailure(t) + context.stop(self) + } + + override def receive: Receive = { + case any => log.error("Unexpected message to InvalidateCallCacheActor: " + any) + } +} + +object CallCacheInvalidateActor { + def props(callCache: CallCache, cacheId: CallCachingEntryId) = { + Props(new CallCacheInvalidateActor(callCache: CallCache, cacheId: CallCachingEntryId)) + } +} + +sealed trait CallCacheInvalidatedResponse +case object CallCacheInvalidatedSuccess extends CallCacheInvalidatedResponse +case class CallCacheInvalidatedFailure(t: Throwable) extends CallCacheInvalidatedResponse \ No newline at end of file diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala index 229fc47a2..8aa1f6d6b 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala @@ -31,7 +31,7 @@ class CallCacheReadActor(cache: CallCache) extends Actor with ActorLogging { } private def runRequest(callCacheHashes: CallCacheHashes): Unit = { - val response = cache.fetchMetaInfoIdsMatchingHashes(callCacheHashes) map { + val response = cache.callCachingEntryIdsMatchingHashes(callCacheHashes) map { CacheResultMatchesForHashes(callCacheHashes.hashes, _) } recover { case t => CacheResultLookupFailure(t) @@ -66,6 +66,6 @@ object CallCacheReadActor { case class CacheLookupRequest(callCacheHashes: CallCacheHashes) sealed trait CallCacheReadActorResponse - case class CacheResultMatchesForHashes(hashResults: Set[HashResult], cacheResultIds: Set[MetaInfoId]) extends CallCacheReadActorResponse + case class CacheResultMatchesForHashes(hashResults: Set[HashResult], cacheResultIds: Set[CallCachingEntryId]) extends CallCacheReadActorResponse case class CacheResultLookupFailure(reason: Throwable) extends CallCacheReadActorResponse } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala index 0f03a5840..b4ad358f5 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala @@ -1,6 +1,7 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching import akka.actor.{ActorLogging, ActorRef, LoggingFSM, Props} +import cats.data.NonEmptyList import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest import cromwell.backend.{BackendInitializationData, BackendJobDescriptor, RuntimeAttributeDefinition} import cromwell.core.callcaching._ @@ -123,7 +124,8 @@ case class EngineJobHashingActor(receiver: ActorRef, } private def respondWithHitOrMissThenTransition(newData: EJHAData) = { - val hitOrMissResponse: EJHAResponse = newData.cacheHit map CacheHit getOrElse CacheMiss + import cats.data.NonEmptyList + val hitOrMissResponse: EJHAResponse = newData.cacheHits map { _.toList } flatMap NonEmptyList.fromList map CacheHit.apply getOrElse CacheMiss receiver ! hitOrMissResponse if (!activity.writeToCache) { @@ -199,7 +201,7 @@ object EngineJobHashingActor { case object GeneratingAllHashes extends EJHAState sealed trait EJHAResponse - case class CacheHit(cacheResultId: MetaInfoId) extends EJHAResponse + case class CacheHit(cacheResultIds: NonEmptyList[CallCachingEntryId]) extends EJHAResponse case object CacheMiss extends EJHAResponse case class HashError(t: Throwable) extends EJHAResponse { override def toString = s"HashError(${t.getMessage})" @@ -223,7 +225,7 @@ object EngineJobHashingActor { * @param hashesKnown The set of all hashes calculated so far (including initial hashes) * @param remainingHashesNeeded The set of hashes which are still needed for writing to the database */ -private[callcaching] case class EJHAData(possibleCacheResults: Option[Set[MetaInfoId]], +private[callcaching] case class EJHAData(possibleCacheResults: Option[Set[CallCachingEntryId]], remainingCacheChecks: Set[HashKey], hashesKnown: Set[HashResult], remainingHashesNeeded: Set[HashKey]) { @@ -249,7 +251,8 @@ private[callcaching] case class EJHAData(possibleCacheResults: Option[Set[MetaIn def allHashesKnown = remainingHashesNeeded.isEmpty def allCacheResultsIntersected = remainingCacheChecks.isEmpty def cacheHit = if (allCacheResultsIntersected) possibleCacheResults flatMap { _.headOption } else None - def isDefinitelyCacheHit = cacheHit.isDefined + def cacheHits = if (allCacheResultsIntersected) possibleCacheResults else None + def isDefinitelyCacheHit = cacheHits.isDefined def isDefinitelyCacheMiss = possibleCacheResults.exists(_.isEmpty) def isDefinitelyCacheHitOrMiss = isDefinitelyCacheHit || isDefinitelyCacheMiss } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala index c8f6be4a2..b0900cb16 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala @@ -3,29 +3,27 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching import akka.actor.{Actor, ActorLogging, ActorRef, Props} import cromwell.Simpletons._ import cromwell.core.simpleton.WdlValueSimpleton -import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CacheHit import cromwell.engine.workflow.lifecycle.execution.callcaching.FetchCachedResultsActor.{CachedOutputLookupFailed, CachedOutputLookupSucceeded} import scala.concurrent.ExecutionContext import scala.util.{Failure, Success} object FetchCachedResultsActor { - def props(cacheHit: CacheHit, replyTo: ActorRef, callCache: CallCache): Props = - Props(new FetchCachedResultsActor(cacheHit, replyTo, callCache)) + def props(callCachingEntryId: CallCachingEntryId, replyTo: ActorRef, callCache: CallCache): Props = + Props(new FetchCachedResultsActor(callCachingEntryId, replyTo, callCache)) sealed trait CachedResultResponse - case class CachedOutputLookupFailed(metaInfoId: MetaInfoId, failure: Throwable) extends CachedResultResponse + case class CachedOutputLookupFailed(callCachingEntryId: CallCachingEntryId, failure: Throwable) extends CachedResultResponse case class CachedOutputLookupSucceeded(simpletons: Seq[WdlValueSimpleton], callOutputFiles: Map[String,String], - returnCode: Option[Int], cacheHit: CacheHit, cacheHitDetails: String) extends CachedResultResponse + returnCode: Option[Int], cacheHit: CallCachingEntryId, cacheHitDetails: String) extends CachedResultResponse } -class FetchCachedResultsActor(cacheHit: CacheHit, replyTo: ActorRef, callCache: CallCache) +class FetchCachedResultsActor(cacheResultId: CallCachingEntryId, replyTo: ActorRef, callCache: CallCache) extends Actor with ActorLogging { { implicit val ec: ExecutionContext = context.dispatcher - val cacheResultId = cacheHit.cacheResultId callCache.fetchCachedResult(cacheResultId) onComplete { case Success(Some(result)) => @@ -39,7 +37,7 @@ class FetchCachedResultsActor(cacheHit: CacheHit, replyTo: ActorRef, callCache: replyTo ! CachedOutputLookupSucceeded(simpletons, jobDetritusFiles.toMap, result.callCachingEntry.returnCode, - cacheHit, sourceCacheDetails) + cacheResultId, sourceCacheDetails) case Success(None) => val reason = new RuntimeException(s"Cache hit vanished between discovery and retrieval: $cacheResultId") replyTo ! CachedOutputLookupFailed(cacheResultId, reason) diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EJHADataSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EJHADataSpec.scala index 45b996018..9c3d8d547 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EJHADataSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EJHADataSpec.scala @@ -59,14 +59,14 @@ class EJHADataSpec extends FlatSpec with Matchers { // To save you time I'll just tell you: the intersection of all these sets is Set(5) val cacheLookupResults: List[CacheResultMatchesForHashes] = List( - CacheResultMatchesForHashes(Set(makeHashResult(hashKey1)), Set(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) map MetaInfoId), - CacheResultMatchesForHashes(Set(makeHashResult(hashKey2)), Set(1, 2, 3, 4, 5, 6) map MetaInfoId), - CacheResultMatchesForHashes(Set(makeHashResult(hashKey3)), Set(1, 2, 3, 5, 7, 8, 9, 10) map MetaInfoId), - CacheResultMatchesForHashes(Set(makeHashResult(hashKey4)), Set(4, 5, 6, 7, 8, 9, 10) map MetaInfoId), - CacheResultMatchesForHashes(Set(makeHashResult(hashKey5)), Set(1, 2, 5, 6, 7, 10) map MetaInfoId)) + CacheResultMatchesForHashes(Set(makeHashResult(hashKey1)), Set(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) map CallCachingEntryId), + CacheResultMatchesForHashes(Set(makeHashResult(hashKey2)), Set(1, 2, 3, 4, 5, 6) map CallCachingEntryId), + CacheResultMatchesForHashes(Set(makeHashResult(hashKey3)), Set(1, 2, 3, 5, 7, 8, 9, 10) map CallCachingEntryId), + CacheResultMatchesForHashes(Set(makeHashResult(hashKey4)), Set(4, 5, 6, 7, 8, 9, 10) map CallCachingEntryId), + CacheResultMatchesForHashes(Set(makeHashResult(hashKey5)), Set(1, 2, 5, 6, 7, 10) map CallCachingEntryId)) val newData = cacheLookupResults.foldLeft(data)( (d, c) => d.intersectCacheResults(c) ) newData.possibleCacheResults match{ - case Some(set) => set should be(Set(MetaInfoId(5))) + case Some(set) => set should be(Set(CallCachingEntryId(5))) case None => fail("There should be a cache result set") } newData.allCacheResultsIntersected should be(true) @@ -79,9 +79,9 @@ class EJHADataSpec extends FlatSpec with Matchers { // To save you time I'll just tell you: the intersection of all these sets is empty Set() val cacheLookupResults: List[CacheResultMatchesForHashes] = List( - CacheResultMatchesForHashes(Set(makeHashResult(hashKey1)), Set(1, 2, 3, 4, 5, 6) map MetaInfoId), - CacheResultMatchesForHashes(Set(makeHashResult(hashKey2)), Set(1, 2, 3, 7, 8, 9) map MetaInfoId), - CacheResultMatchesForHashes(Set(makeHashResult(hashKey3)), Set(5, 7, 8, 9, 10) map MetaInfoId)) + CacheResultMatchesForHashes(Set(makeHashResult(hashKey1)), Set(1, 2, 3, 4, 5, 6) map CallCachingEntryId), + CacheResultMatchesForHashes(Set(makeHashResult(hashKey2)), Set(1, 2, 3, 7, 8, 9) map CallCachingEntryId), + CacheResultMatchesForHashes(Set(makeHashResult(hashKey3)), Set(5, 7, 8, 9, 10) map CallCachingEntryId)) val newData = cacheLookupResults.foldLeft(data)( (d, c) => d.intersectCacheResults(c) ) newData.possibleCacheResults match{ case Some(set) => set should be(Set.empty) diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala index 25ceffe3c..d79a383fb 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala @@ -2,6 +2,7 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching import akka.actor.{ActorRef, ActorSystem, Props} import akka.testkit.{ImplicitSender, TestKit, TestProbe} +import cats.data.NonEmptyList import cromwell.CromwellTestkitSpec import cromwell.backend.callcaching.FileHashingActor.{FileHashResponse, SingleFileHashRequest} import cromwell.backend.{BackendInitializationData, BackendJobDescriptor, BackendJobDescriptorKey, BackendWorkflowDescriptor, RuntimeAttributeDefinition} @@ -35,11 +36,11 @@ class EngineJobHashingActorSpec extends TestKit(new CromwellTestkitSpec.TestWork } s"Respect the CallCachingMode and report back $expectation for the ${activity.readWriteMode} activity" in { - val singleMetaInfoIdSet = Set(MetaInfoId(1)) + val singleCallCachingEntryIdSet = Set(CallCachingEntryId(1)) val replyTo = TestProbe() val deathWatch = TestProbe() - val cacheLookupResponses: Map[String, Set[MetaInfoId]] = if (activity.readFromCache) standardCacheLookupResponses(singleMetaInfoIdSet, singleMetaInfoIdSet, singleMetaInfoIdSet, singleMetaInfoIdSet) else Map.empty + val cacheLookupResponses: Map[String, Set[CallCachingEntryId]] = if (activity.readFromCache) standardCacheLookupResponses(singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet) else Map.empty val ejha = createEngineJobHashingActor( replyTo = replyTo.ref, activity = activity, @@ -47,7 +48,7 @@ class EngineJobHashingActorSpec extends TestKit(new CromwellTestkitSpec.TestWork deathWatch watch ejha - if (activity.readFromCache) replyTo.expectMsg(CacheHit(MetaInfoId(1))) + if (activity.readFromCache) replyTo.expectMsg(CacheHit(NonEmptyList.of(CallCachingEntryId(1)))) if (activity.writeToCache) replyTo.expectMsgPF(max = 5 seconds, hint = "awaiting cache hit message") { case CallCacheHashes(hashes) => hashes.size should be(4) case x => fail(s"Cache hit anticipated! Instead got a ${x.getClass.getSimpleName}") @@ -57,13 +58,13 @@ class EngineJobHashingActorSpec extends TestKit(new CromwellTestkitSpec.TestWork } s"Wait for requests to the FileHashingActor for the ${activity.readWriteMode} activity" in { - val singleMetaInfoIdSet = Set(MetaInfoId(1)) + val singleCallCachingEntryIdSet = Set(CallCachingEntryId(1)) val replyTo = TestProbe() val fileHashingActor = TestProbe() val deathWatch = TestProbe() - val initialCacheLookupResponses: Map[String, Set[MetaInfoId]] = if (activity.readFromCache) standardCacheLookupResponses(singleMetaInfoIdSet, singleMetaInfoIdSet, singleMetaInfoIdSet, singleMetaInfoIdSet) else Map.empty - val fileCacheLookupResponses = Map("input: File inputFile1" -> singleMetaInfoIdSet, "input: File inputFile2" -> singleMetaInfoIdSet) + val initialCacheLookupResponses: Map[String, Set[CallCachingEntryId]] = if (activity.readFromCache) standardCacheLookupResponses(singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet) else Map.empty + val fileCacheLookupResponses = Map("input: File inputFile1" -> singleCallCachingEntryIdSet, "input: File inputFile2" -> singleCallCachingEntryIdSet) val jobDescriptor = templateJobDescriptor(inputs = Map( "inputFile1" -> WdlFile("path"), @@ -86,7 +87,7 @@ class EngineJobHashingActorSpec extends TestKit(new CromwellTestkitSpec.TestWork } } - if (activity.readFromCache) replyTo.expectMsg(CacheHit(MetaInfoId(1))) + if (activity.readFromCache) replyTo.expectMsg(CacheHit(NonEmptyList.of(CallCachingEntryId(1)))) if (activity.writeToCache) replyTo.expectMsgPF(max = 5 seconds, hint = "awaiting cache hit message") { case CallCacheHashes(hashes) => hashes.size should be(6) case x => fail(s"Cache hit anticipated! Instead got a ${x.getClass.getSimpleName}") @@ -96,13 +97,13 @@ class EngineJobHashingActorSpec extends TestKit(new CromwellTestkitSpec.TestWork } s"Cache miss for bad FileHashingActor results but still return hashes in the ${activity.readWriteMode} activity" in { - val singleMetaInfoIdSet = Set(MetaInfoId(1)) + val singleCallCachingEntryIdSet = Set(CallCachingEntryId(1)) val replyTo = TestProbe() val fileHashingActor = TestProbe() val deathWatch = TestProbe() - val initialCacheLookupResponses: Map[String, Set[MetaInfoId]] = if (activity.readFromCache) standardCacheLookupResponses(singleMetaInfoIdSet, singleMetaInfoIdSet, singleMetaInfoIdSet, singleMetaInfoIdSet) else Map.empty - val fileCacheLookupResponses = Map("input: File inputFile1" -> Set(MetaInfoId(2)), "input: File inputFile2" -> singleMetaInfoIdSet) + val initialCacheLookupResponses: Map[String, Set[CallCachingEntryId]] = if (activity.readFromCache) standardCacheLookupResponses(singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet) else Map.empty + val fileCacheLookupResponses = Map("input: File inputFile1" -> Set(CallCachingEntryId(2)), "input: File inputFile2" -> singleCallCachingEntryIdSet) val jobDescriptor = templateJobDescriptor(inputs = Map( "inputFile1" -> WdlFile("path"), @@ -139,11 +140,11 @@ class EngineJobHashingActorSpec extends TestKit(new CromwellTestkitSpec.TestWork } s"Detect call cache misses for the ${activity.readWriteMode} activity" in { - val singleMetaInfoIdSet = Set(MetaInfoId(1)) + val singleCallCachingEntryIdSet = Set(CallCachingEntryId(1)) val replyTo = TestProbe() val deathWatch = TestProbe() - val cacheLookupResponses: Map[String, Set[MetaInfoId]] = if (activity.readFromCache) standardCacheLookupResponses(singleMetaInfoIdSet, singleMetaInfoIdSet, Set(MetaInfoId(2)), singleMetaInfoIdSet) else Map.empty + val cacheLookupResponses: Map[String, Set[CallCachingEntryId]] = if (activity.readFromCache) standardCacheLookupResponses(singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, Set(CallCachingEntryId(2)), singleCallCachingEntryIdSet) else Map.empty val ejha = createEngineJobHashingActor( replyTo = replyTo.ref, activity = activity, @@ -177,7 +178,7 @@ object EngineJobHashingActorSpec extends MockitoSugar { jobDescriptor: BackendJobDescriptor = templateJobDescriptor(), initializationData: Option[BackendInitializationData] = None, fileHashingActor: Option[ActorRef] = None, - cacheLookupResponses: Map[String, Set[MetaInfoId]] = Map.empty, + cacheLookupResponses: Map[String, Set[CallCachingEntryId]] = Map.empty, runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition] = Set.empty, backendName: String = "whatever" )(implicit system: ActorSystem) = { @@ -206,10 +207,10 @@ object EngineJobHashingActorSpec extends MockitoSugar { jobDescriptor } - def standardCacheLookupResponses(commandTemplate: Set[MetaInfoId], - inputCount: Set[MetaInfoId], - backendName: Set[MetaInfoId], - outputCount: Set[MetaInfoId]) = Map( + def standardCacheLookupResponses(commandTemplate: Set[CallCachingEntryId], + inputCount: Set[CallCachingEntryId], + backendName: Set[CallCachingEntryId], + outputCount: Set[CallCachingEntryId]) = Map( "command template" -> commandTemplate, "input count" -> inputCount, "backend name" -> backendName, diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/PredictableCallCacheReadActor.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/PredictableCallCacheReadActor.scala index 4182095c6..c98587840 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/PredictableCallCacheReadActor.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/PredictableCallCacheReadActor.scala @@ -10,7 +10,7 @@ import scala.util.{Failure, Success, Try} /** * Has a set of responses which it will respond with. If it gets a request for anything that it's not expecting to respond to will generate a failure. */ -class PredictableCallCacheReadActor(responses: Map[String, Set[MetaInfoId]]) extends Actor with ActorLogging { +class PredictableCallCacheReadActor(responses: Map[String, Set[CallCachingEntryId]]) extends Actor with ActorLogging { var responsesRemaining = responses @@ -25,7 +25,7 @@ class PredictableCallCacheReadActor(responses: Map[String, Set[MetaInfoId]]) ext } } - private def respond(sndr: ActorRef, hashes: Set[HashResult], result: Try[Set[MetaInfoId]]) = result match { + private def respond(sndr: ActorRef, hashes: Set[HashResult], result: Try[Set[CallCachingEntryId]]) = result match { case Success(cacheMatches) => sndr ! CacheResultMatchesForHashes(hashes, cacheMatches) case Failure(t) => sndr ! CacheResultLookupFailure(t) } @@ -35,7 +35,7 @@ class PredictableCallCacheReadActor(responses: Map[String, Set[MetaInfoId]]) ext case None => Failure(new Exception(s"Error looking up response $name!")) } - private def resultLookupFolder(current: Try[Set[MetaInfoId]], next: HashResult): Try[Set[MetaInfoId]] = current flatMap { c => + private def resultLookupFolder(current: Try[Set[CallCachingEntryId]], next: HashResult): Try[Set[CallCachingEntryId]] = current flatMap { c => val lookedUp = toTry(next.hashKey.key, responses.get(next.hashKey.key)) lookedUp map { l => c.intersect(l) } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaBackendIsCopyingCachedOutputsSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaBackendIsCopyingCachedOutputsSpec.scala index 51de21e6b..7aa4ccfda 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaBackendIsCopyingCachedOutputsSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaBackendIsCopyingCachedOutputsSpec.scala @@ -1,17 +1,19 @@ package cromwell.engine.workflow.lifecycle.execution.ejea +import cats.data.NonEmptyList import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ import EngineJobExecutionActorSpec._ import cromwell.core.callcaching.CallCachingMode -import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CallCacheHashes, EJHAResponse, HashError} +import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CallCacheHashes, EJHAResponse, HashError} +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId import scala.util.{Failure, Success, Try} import cromwell.engine.workflow.lifecycle.execution.ejea.HasJobSuccessResponse.SuccessfulCallCacheHashes -class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec with HasJobSuccessResponse with HasJobFailureResponses with CanExpectJobStoreWrites with CanExpectCacheWrites { +class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec with HasJobSuccessResponse with HasJobFailureResponses with CanExpectJobStoreWrites with CanExpectCacheWrites with CanExpectCacheInvalidation { override implicit val stateUnderTest = BackendIsCopyingCachedOutputs - "An EJEA in FetchingCachedOutputsFromDatabase state" should { + "An EJEA in BackendIsCopyingCachedOutputs state" should { val hashErrorCause = new Exception("blah") val hashResultsDataValue = Some(Success(SuccessfulCallCacheHashes)) @@ -95,19 +97,18 @@ class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec } } - RestartOrExecuteCommandTuples foreach { case RestartOrExecuteCommandTuple(operationName, restarting, expectedMessage) => - s"$operationName the job immediately when it gets a failure result, and it was going to receive $hashComboName, if call caching is $mode" in { - ejea = ejeaInBackendIsCopyingCachedOutputsState(initialHashData, mode, restarting = restarting) + s"invalidate a call for caching if backend coping failed when it was going to receive $hashComboName, if call caching is $mode" in { + ejea = ejeaInBackendIsCopyingCachedOutputsState(initialHashData, mode) // Send the response from the copying actor ejea ! failureNonRetryableResponse - helper.bjeaProbe.expectMsg(awaitTimeout, expectedMessage) - ejea.stateName should be(RunningJob) - ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper. bjeaProps, initialHashData)) + expectInvalidateCallCacheActor(cacheId) + eventually { ejea.stateName should be(InvalidatingCacheEntry) } + ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper. bjeaProps, initialHashData, cacheHit)) } - s"$operationName the job (preserving and received hashes) when call caching is $mode, the EJEA has $hashComboName and then gets a success result" in { - ejea = ejeaInBackendIsCopyingCachedOutputsState(initialHashData, mode, restarting = restarting) + s"invalidate a call for caching if backend coping failed (preserving and received hashes) when call caching is $mode, the EJEA has $hashComboName and then gets a success result" in { + ejea = ejeaInBackendIsCopyingCachedOutputsState(initialHashData, mode) // Send the response from the EJHA (if there was one!): ejhaResponse foreach { ejea ! _ } @@ -118,15 +119,16 @@ class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec // Send the response from the copying actor ejea ! failureNonRetryableResponse - helper.bjeaProbe.expectMsg(awaitTimeout, expectedMessage) - ejea.stateName should be(RunningJob) - ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper. bjeaProps, finalHashData)) + expectInvalidateCallCacheActor(cacheId) + eventually { ejea.stateName should be(InvalidatingCacheEntry) } + ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper. bjeaProps, finalHashData, cacheHit)) } - } } } } - def standardResponsePendingData(hashes: Option[Try[CallCacheHashes]]) = ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, hashes) + private val cacheId: CallCachingEntryId = CallCachingEntryId(74) + private val cacheHit = Option(CacheHit(NonEmptyList.of(cacheId))) + def standardResponsePendingData(hashes: Option[Try[CallCacheHashes]]) = ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, hashes, cacheHit) def ejeaInBackendIsCopyingCachedOutputsState(initialHashes: Option[Try[CallCacheHashes]], callCachingMode: CallCachingMode, restarting: Boolean = false) = helper.buildEJEA(restarting = restarting, callCachingMode = callCachingMode).setStateInline(data = standardResponsePendingData(initialHashes)) } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala index d9de4432b..930133d10 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala @@ -1,25 +1,22 @@ package cromwell.engine.workflow.lifecycle.execution.ejea +import cats.data.NonEmptyList import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{CheckingCallCache, FetchingCachedOutputsFromDatabase, ResponsePendingData, RunningJob} import EngineJobExecutionActorSpec.EnhancedTestEJEA import cromwell.core.callcaching.{CallCachingActivity, CallCachingOff, ReadCache} import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CacheMiss, HashError} -import cromwell.engine.workflow.lifecycle.execution.callcaching.MetaInfoId +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId import org.scalatest.concurrent.Eventually -class EjeaCheckingCallCacheSpec extends EngineJobExecutionActorSpec with Eventually { +class EjeaCheckingCallCacheSpec extends EngineJobExecutionActorSpec with Eventually with CanExpectFetchCachedResults { override implicit val stateUnderTest = CheckingCallCache "An EJEA in CheckingCallCache mode" should { "Try to fetch the call cache outputs if it gets a CacheHit" in { createCheckingCallCacheEjea() - ejea ! CacheHit(MetaInfoId(75)) - eventually { helper.fetchCachedResultsActorCreations.hasExactlyOne should be(true) } - helper.fetchCachedResultsActorCreations checkIt { - case (CacheHit(metainfoId), _) => metainfoId should be(MetaInfoId(75)) - case _ => fail("Incorrect creation of the fetchCachedResultsActor") - } + ejea ! CacheHit(NonEmptyList.of(CallCachingEntryId(75))) + expectFetchCachedResultsActor(CallCachingEntryId(75)) ejea.stateName should be(FetchingCachedOutputsFromDatabase) } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaFetchingCachedOutputsFromDatabaseSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaFetchingCachedOutputsFromDatabaseSpec.scala index cc6c83b41..4acd2efab 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaFetchingCachedOutputsFromDatabaseSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaFetchingCachedOutputsFromDatabaseSpec.scala @@ -1,14 +1,14 @@ package cromwell.engine.workflow.lifecycle.execution.ejea -import cromwell.core.WorkflowId -import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ -import EngineJobExecutionActorSpec._ import cromwell.backend.BackendCacheHitCopyingActor.CopyOutputsCommand +import cromwell.core.WorkflowId import cromwell.core.callcaching.{CallCachingActivity, ReadAndWriteCache} import cromwell.core.simpleton.WdlValueSimpleton -import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, HashError} +import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ +import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.HashError import cromwell.engine.workflow.lifecycle.execution.callcaching.FetchCachedResultsActor.{CachedOutputLookupFailed, CachedOutputLookupSucceeded} -import cromwell.engine.workflow.lifecycle.execution.callcaching.MetaInfoId +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId +import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec._ import cromwell.engine.workflow.lifecycle.execution.ejea.HasJobSuccessResponse.SuccessfulCallCacheHashes import wdl4s.values.WdlString @@ -36,7 +36,7 @@ class EjeaFetchingCachedOutputsFromDatabaseSpec extends EngineJobExecutionActorS val detritusMap = Map("stdout" -> "//somePath") val cachedReturnCode = Some(17) val sourceCacheDetails = s"${WorkflowId.randomId}:call-someTask:1" - ejea ! CachedOutputLookupSucceeded(cachedSimpletons, detritusMap, cachedReturnCode, CacheHit(MetaInfoId(75)), sourceCacheDetails) + ejea ! CachedOutputLookupSucceeded(cachedSimpletons, detritusMap, cachedReturnCode, CallCachingEntryId(75), sourceCacheDetails) helper.callCacheHitCopyingProbe.expectMsg(CopyOutputsCommand(cachedSimpletons, detritusMap, cachedReturnCode)) // Check we end up in the right state: @@ -61,7 +61,7 @@ class EjeaFetchingCachedOutputsFromDatabaseSpec extends EngineJobExecutionActorS // Send the response from the "Fetch" actor val failureReason = new Exception("You can't handle the truth!") - ejea ! CachedOutputLookupFailed(MetaInfoId(90210), failureReason) + ejea ! CachedOutputLookupFailed(CallCachingEntryId(90210), failureReason) helper.bjeaProbe.expectMsg(awaitTimeout, expectedMessage) // Check we end up in the right state: diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaInvalidatingCacheEntrySpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaInvalidatingCacheEntrySpec.scala new file mode 100644 index 000000000..40e43d77b --- /dev/null +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaInvalidatingCacheEntrySpec.scala @@ -0,0 +1,53 @@ +package cromwell.engine.workflow.lifecycle.execution.ejea + +import cats.data.NonEmptyList +import cromwell.core.callcaching.{CallCachingActivity, ReadCache} +import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ +import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CacheHit +import cromwell.engine.workflow.lifecycle.execution.callcaching.{CallCacheInvalidatedFailure, CallCacheInvalidatedSuccess, CallCachingEntryId} +import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec._ + +class EjeaInvalidatingCacheEntrySpec extends EngineJobExecutionActorSpec with CanExpectFetchCachedResults { + + override implicit val stateUnderTest = InvalidatingCacheEntry + + "An EJEA in InvalidatingCacheEntry state" should { + + val invalidationErrorCause = new Exception("blah") + val invalidateSuccess = CallCacheInvalidatedSuccess + val invalidateFailure = CallCacheInvalidatedFailure(invalidationErrorCause) + + val metaInfo31: CallCachingEntryId = CallCachingEntryId(31) + val metaInfo32: CallCachingEntryId = CallCachingEntryId(32) + val cacheHitWithTwoIds = CacheHit(NonEmptyList.of(metaInfo32, metaInfo31)) + val cacheHitWithSingleId = CacheHit(NonEmptyList.of(metaInfo31)) + + List(invalidateSuccess, invalidateFailure) foreach { invalidateActorResponse => + s"try the next available hit when response is $invalidateActorResponse" in { + ejea = ejeaInvalidatingCacheEntryState(Option(cacheHitWithTwoIds), restarting = false) + // Send the response from the invalidate actor + ejea ! invalidateActorResponse + + helper.bjeaProbe.expectNoMsg(awaitAlmostNothing) + expectFetchCachedResultsActor(cacheHitWithSingleId.cacheResultIds.head) + eventually { ejea.stateName should be(FetchingCachedOutputsFromDatabase) } + ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, None, Option(cacheHitWithSingleId))) + } + + RestartOrExecuteCommandTuples foreach { case RestartOrExecuteCommandTuple(operationName, restarting, expectedMessage) => + s"$operationName a job if cache invalidation succeeds and there are no other cache hits to try when invalidate response is $invalidateActorResponse" in { + ejea = ejeaInvalidatingCacheEntryState(Option(cacheHitWithSingleId), restarting = restarting) + // Send the response from the invalidate actor + ejea ! invalidateActorResponse + + helper.bjeaProbe.expectMsg(awaitTimeout, expectedMessage) + eventually { ejea.stateName should be(RunningJob) } + ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper. bjeaProps, None, None)) + } + } + } + } + + def standardResponsePendingData(hit: Option[CacheHit]) = ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, None, hit) + def ejeaInvalidatingCacheEntryState(hit: Option[CacheHit], restarting: Boolean = false) = helper.buildEJEA(restarting = restarting, callCachingMode = CallCachingActivity(ReadCache)).setStateInline(data = standardResponsePendingData(hit)) +} diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala index e8eaeec1a..cbba49811 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala @@ -40,6 +40,7 @@ trait EngineJobExecutionActorSpec extends CromwellTestkitSpec List( ("FetchCachedResultsActor", helper.fetchCachedResultsActorCreations), ("JobHashingActor", helper.jobHashingInitializations), + ("CallCacheInvalidateActor", helper.invalidateCacheActorCreations), ("CallCacheWriteActor", helper.callCacheWriteActorCreations)) foreach { case (name, GotTooMany(list)) => fail(s"Too many $name creations (${list.size})") case _ => // Fine. diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala index 60cc183a9..8ef607c5b 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala @@ -5,6 +5,7 @@ import cromwell.core.JobOutput import cromwell.core.callcaching._ import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{EJEAData, SucceededResponseData, UpdatingCallCache, UpdatingJobStore} import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId import cromwell.jobstore.JobStoreActor.RegisterJobCompleted import cromwell.jobstore.{JobResultSuccess, JobStoreKey} import org.scalatest.concurrent.Eventually @@ -60,6 +61,25 @@ private[ejea] trait CanExpectHashingInitialization extends Eventually { self: En } } +private[ejea] trait CanExpectFetchCachedResults extends Eventually { self: EngineJobExecutionActorSpec => + def expectFetchCachedResultsActor(expectedCallCachingEntryId: CallCachingEntryId): Unit = { + eventually { helper.fetchCachedResultsActorCreations.hasExactlyOne should be(true) } + helper.fetchCachedResultsActorCreations.checkIt { + case (callCachingEntryId, _) => callCachingEntryId should be(expectedCallCachingEntryId) + case _ => fail("Incorrect creation of the fetchCachedResultsActor") + } + } +} + +private[ejea] trait CanExpectCacheInvalidation extends Eventually { self: EngineJobExecutionActorSpec => + def expectInvalidateCallCacheActor(expectedCacheId: CallCachingEntryId): Unit = { + eventually { helper.invalidateCacheActorCreations.hasExactlyOne should be(true) } + helper.invalidateCacheActorCreations.checkIt { cacheId => + cacheId shouldBe expectedCacheId + } + } +} + private[ejea] trait HasJobSuccessResponse { self: EngineJobExecutionActorSpec => val successRc = Option(171) val successOutputs = Map("a" -> JobOutput(WdlInteger(3)), "b" -> JobOutput(WdlString("bee"))) diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala index aa3f5a28c..16274a275 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala @@ -10,10 +10,10 @@ import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.core.callcaching.{CallCachingActivity, CallCachingMode, CallCachingOff} import cromwell.core.{ExecutionStore, JobExecutionToken, OutputStore, WorkflowId} import cromwell.engine.EngineWorkflowDescriptor -import cromwell.engine.backend.BackendSingletonCollection +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId import cromwell.engine.workflow.lifecycle.execution.{EngineJobExecutionActor, WorkflowExecutionActorData} import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{EJEAData, EngineJobExecutionActorState} -import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CallCacheHashes} +import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes import org.specs2.mock.Mockito import wdl4s.WdlExpression.ScopedLookupFunction import wdl4s.expression.{NoFunctions, WdlFunctions, WdlStandardLibraryFunctions} @@ -62,9 +62,10 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock val backendWorkflowDescriptor = BackendWorkflowDescriptor(workflowId, null, null, null) val backendJobDescriptor = BackendJobDescriptor(backendWorkflowDescriptor, jobDescriptorKey, runtimeAttributes = Map.empty, inputs = Map.empty) - var fetchCachedResultsActorCreations: ExpectOne[(CacheHit, Seq[TaskOutput])] = NothingYet + var fetchCachedResultsActorCreations: ExpectOne[(CallCachingEntryId, Seq[TaskOutput])] = NothingYet var jobHashingInitializations: ExpectOne[(BackendJobDescriptor, CallCachingActivity)] = NothingYet var callCacheWriteActorCreations: ExpectOne[(CallCacheHashes, SucceededResponse)] = NothingYet + var invalidateCacheActorCreations: ExpectOne[CallCachingEntryId] = NothingYet val deathwatch = TestProbe() val bjeaProbe = TestProbe() @@ -146,9 +147,11 @@ private[ejea] class MockEjea(helper: PerTestHelper, backendName: String, callCachingMode: CallCachingMode) extends EngineJobExecutionActor(replyTo, jobDescriptorKey, executionData, factory, initializationData, restarting, serviceRegistryActor, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, None, backendName, callCachingMode) { - override def makeFetchCachedResultsActor(cacheHit: CacheHit, taskOutputs: Seq[TaskOutput]) = helper.fetchCachedResultsActorCreations = helper.fetchCachedResultsActorCreations.foundOne((cacheHit, taskOutputs)) + override def makeFetchCachedResultsActor(cacheId: CallCachingEntryId, taskOutputs: Seq[TaskOutput]) = helper.fetchCachedResultsActorCreations = helper.fetchCachedResultsActorCreations.foundOne((cacheId, taskOutputs)) override def initializeJobHashing(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity) = helper.jobHashingInitializations = helper.jobHashingInitializations.foundOne((jobDescriptor, activity)) override def createSaveCacheResultsActor(hashes: CallCacheHashes, success: SucceededResponse) = helper.callCacheWriteActorCreations = helper.callCacheWriteActorCreations.foundOne((hashes, success)) - + override def invalidateCacheHit(cacheId: CallCachingEntryId): Unit = { + helper.invalidateCacheActorCreations = helper.invalidateCacheActorCreations.foundOne(cacheId) + } override def createJobPreparationActor(jobPrepProps: Props, name: String) = jobPreparationProbe.ref } diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala index 0575b7b33..444948c0e 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala @@ -2,15 +2,15 @@ package cromwell.engine.workflow.tokens import java.util.UUID -import akka.actor.{ActorRef, ActorSystem, Kill, PoisonPill} -import org.scalatest._ +import akka.actor.{ActorSystem, PoisonPill} import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} +import cromwell.core.JobExecutionToken import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed, JobExecutionTokenRequest, JobExecutionTokenReturn} -import JobExecutionTokenDispenserActorSpec._ -import cromwell.core.JobExecutionToken +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActorSpec._ import cromwell.engine.workflow.tokens.TestTokenGrabbingActor.StoppingSupervisor import cromwell.util.AkkaTestUtil +import org.scalatest._ import org.scalatest.concurrent.Eventually import scala.concurrent.duration._ diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala index e24d8627d..078e2e081 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala @@ -13,7 +13,7 @@ case class JesCacheHitCopyingActor(override val jobDescriptor: BackendJobDescrip initializationData: JesBackendInitializationData, serviceRegistryActor: ActorRef) extends BackendCacheHitCopyingActor with CacheHitDuplicating with JesJobCachingActorHelper with JobLogging { - override protected def duplicate(source: Path, destination: Path) = PathCopier.copy(source, destination) + override protected def duplicate(source: Path, destination: Path) = PathCopier.copy(source, destination).get override protected def destinationCallRootPath = jesCallPaths.callRootPath From f86624aeb3375f43cad670935beaf0cc667b9895 Mon Sep 17 00:00:00 2001 From: Ruchi Date: Thu, 13 Oct 2016 14:15:48 -0400 Subject: [PATCH 023/375] release_notes for 0.22 (#1565) * release_notes * typos and reword --- CHANGELOG.md | 86 +++++++++++++++++++++++++++++++++++++++------- project/Dependencies.scala | 4 +-- 2 files changed, 75 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ff214c55..cf48b60be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,68 @@ # Cromwell Change Log -## 0.20 +## 0.22 -* The default per-upload bytes size for GCS is now the minumum 256K -instead of 64M. There is also an undocumented config key -`google.upload-buffer-bytes` that allows adjusting this internal value. +* Improved retries for Call Caching and general bug fixes. +* Users will experience better scalability of status polling for Google JES. +* Now there are configurable caching strategies for a SharedFileSystem backend (i.e. Local, SFS) in the backend's stanza: + See below for detailed descriptions of each configurable key. -* Updated Docker Hub hash retriever to parse json with [custom media -types](https://github.com/docker/distribution/blob/05b0ab0/docs/spec/manifest-v2-1.md). +``` +backend { + ... + providers { + SFS_BackendName { + actor-factory = ... + config { + ... + filesystems { + local { + localization: [ + ... + ] + caching { + duplication-strategy: [ + "hard-link", "soft-link", "copy" + ] + # Possible values: file, path + # "file" will compute an md5 hash of the file content. + # "path" will compute an md5 hash of the file path. This strategy will only be effective if the duplication-strategy (above) is set to "soft-link", + # in order to allow for the original file path to be hashed. + hashing-strategy: "file" + + # When true, will check if a sibling file with the same name and the .md5 extension exists, and if it does, use the content of this file as a hash. + # If false or the md5 does not exist, will proceed with the above-defined hashing strategy. + check-sibling-md5: false + } +``` +* Multiple Input JSON files can now be submitted in server mode through the existing submission endpoint: /api/workflows/:version. + This endpoint accepts a POST request with a multipart/form-data encoded body. You can now include multiple keys for workflow inputs. -* Added a `/batch` submit endpoint that accepts a single wdl with -multiple input files. + Each key below can contain an optional JSON file of the workflow inputs. A skeleton file can be generated from wdltool using the "inputs" subcommand. + NOTE: In case of key conflicts between multiple JSON files, higher values of x in workflowInputs_x override lower values. For example, an input + specified in workflowInputs_3 will override an input with the same name that was given in workflowInputs or workflowInputs_2. Similarly, an input + specified in workflowInputs_5 will override an input with the same name in any other input file. -* The `/query` endpoint now supports querying by `id`, and submitting -parameters as a HTTP POST. + workflowInputs + workflowInputs_2 + workflowInputs_3 + workflowInputs_4 + workflowInputs_5 -## 0.21 +* You can now limit the number of concurrent jobs for a backend by specifying the following option in the backend's config stanza: +``` +backend { + ... + providers { + BackendName { + actor-factory = ... + config { + concurrent-job-limit = 5 +``` +## 0.21 + * Warning: Significant database updates when you switch from version 0.19 to 0.21 of Cromwell. There may be a long wait period for the migration to finish for large databases. Please refer to MIGRATION.md for more details. @@ -71,7 +116,7 @@ task { command { echo "I'm private !" } - + runtime { docker: "ubuntu:latest" noAddress: true @@ -94,7 +139,7 @@ passed absolute paths for input `File`s. * Override the default database configuration by setting the keys `database.driver`, `database.db.driver`, `database.db.url`, etc. * Override the default database configuration by setting the keys -`database.driver`, `database.db.driver`, `database.db.url`, etc. +`database.driver`, `database.db.driver`, `database.db.url`, etc. For example: ``` @@ -111,3 +156,18 @@ database { } ``` +## 0.20 + +* The default per-upload bytes size for GCS is now the minimum 256K +instead of 64M. There is also an undocumented config key +`google.upload-buffer-bytes` that allows adjusting this internal value. + +* Updated Docker Hub hash retriever to parse json with [custom media +types](https://github.com/docker/distribution/blob/05b0ab0/docs/spec/manifest-v2-1.md). + +* Added a `/batch` submit endpoint that accepts a single wdl with +multiple input files. + +* The `/query` endpoint now supports querying by `id`, and submitting +parameters as a HTTP POST. + diff --git a/project/Dependencies.scala b/project/Dependencies.scala index ca5420d5e..a902e1b2a 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,8 +1,8 @@ import sbt._ object Dependencies { - lazy val lenthallV = "0.19-882a763-SNAPSHOT" - lazy val wdl4sV = "0.6-2964173-SNAPSHOT" + lazy val lenthallV = "0.19" + lazy val wdl4sV = "0.6" lazy val sprayV = "1.3.3" /* spray-json is an independent project from the "spray suite" From 78f63fcae7846a6aa9bf726f10dede715499daeb Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Wed, 12 Oct 2016 15:53:58 -0400 Subject: [PATCH 024/375] Beginnings of the ECS backend! --- MakingABackend.MD | 119 +++++++++++++++++++++ .../backend/BackendLifecycleActorFactory.scala | 5 +- .../scala/cromwell/backend/wdl/PureFunctions.scala | 18 ++-- build.sbt | 8 ++ core/src/main/resources/reference.conf | 7 ++ project/Dependencies.scala | 4 + project/Settings.scala | 5 + .../backend/impl/aws/AwsBackendActorFactory.scala | 12 +++ .../backend/impl/aws/AwsJobExecutionActor.scala | 18 ++++ 9 files changed, 185 insertions(+), 11 deletions(-) create mode 100644 MakingABackend.MD create mode 100644 supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBackendActorFactory.scala create mode 100644 supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsJobExecutionActor.scala diff --git a/MakingABackend.MD b/MakingABackend.MD new file mode 100644 index 000000000..aecb427c9 --- /dev/null +++ b/MakingABackend.MD @@ -0,0 +1,119 @@ +# Making a backend + +## Part 0: Introduction + +- These notes were added while making a new AWS backend for Amazon AWS. + +## Part 1: The skeleton: + +To start with, I just need to create a bunch of boilerplate which will eventually be filled in with all of the lovely AWS details! + +### Defining the awsBackend project: + +- Added entries to project/Settings.scala, project/Dependencies.scala and build.sbt +- This was mainly just a copy/paste from existing backend projects. I made a few typos renaming everything and linking the dependencies properly though! +- E.g. In my first commit I forgot to update the libraryDependencies name for my AWS backend project: +``` + val awsBackendSettings = List( + name := "cromwell-aws-backend", + libraryDependencies ++= awsBackendDependencies + ) ++ commonSettings +``` +- I guessed that I'd need the AWS SDK so I included that immediately in Dependencies.scala: +``` + val awsBackendDependencies = List( + "com.amazonaws" % "aws-java-sdk" % "1.11.41" + ) +``` +- In build.scala I had to also edit the `lazy val root` to include a new `.aggregate(awsBackend)` and a new `.dependsOn(awsBackend)` + +### Directory structure: + +- This is probably going to be autogenerated for you in the directories specified in the above files. I'd already added my own directory structure and sbt managed to pick it up correctly in `supportedBackends/aws`. + +### AWS Job Execution Actor: +- To run a job, Cromwell needs to instantiate a Job Execution actor. I'll fill in the details later but for now, I'll just add the constructor, props, and an unimplemented method definition for `execute`: +``` +class AwsJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, + override val configurationDescriptor: BackendConfigurationDescriptor) extends BackendJobExecutionActor { + + override def execute: Future[BackendJobExecutionResponse] = ??? +} + +object AwsJobExecutionActor { + def props(jobDescriptor: BackendJobDescriptor, + configurationDescriptor: BackendConfigurationDescriptor): Props = Props(new AwsJobExecutionActor(jobDescriptor, configurationDescriptor)) +} +``` + +### Actor factory: +- This is the class which tells Cromwell which classes represent job execution actors, initialization actors and so on. I'm just adding a skeleton for now, with a constructor of the form the Cromwell expects: +``` +case class AwsBackendActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory { + + override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, + initializationData: Option[BackendInitializationData], + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props = AwsJobExecutionActor.props(jobDescriptor, configurationDescriptor) +} +``` +- There are a few other actor definitions that can be added to this file over time. But the only one that Cromwell *requires* to work is the job execution actor. + +### Reference conf: + +- Reference.conf is a set of reference options which shows people how to enable the backends that they want. So I'll add the initial config which people would add if they wanted the AWS backend (commented out in the reference so it's not enabled by default). This goes below all the other backend references: +``` + #AWS { + # actor-factory = "cromwell.backend.impl.aws.AwsBackendActorFactory" + # config { + # + # } + #} +``` + +### Application.conf + +- OK so I've now told people how to add this backend... Now I actually add it to my own personal configuration file so I can try it out! +``` +backend { + default = "AWS" + providers { + AWS { + actor-factory = "cromwell.backend.impl.aws.AwsBackendActorFactory" + config { + + } + } + } +} +``` + +### Trying it out +So we now have a backend skeleton! What happens when we run it? Well hopefully Cromwell will instantiate the backend far enough to reach the unimplemented execute method and then fall over. Let's give it a go! +- I fire up cromwell in server mode with my modified application.conf. +- I create a sample WDL that would sleep for 20 seconds if it actually worked: +The input WDL: +``` +task sleep { + command { sleep 20 } +} +workflow main { + call sleep +} +``` +- I submit the WDL to the swagger endpoint (http://localhost:8000/swagger/index.html?url=/swagger/cromwell.yaml) and watch the server logs... +- And as expected: +``` +2016-10-13 13:14:29,017 cromwell-system-akka.dispatchers.engine-dispatcher-39 INFO - MaterializeWorkflowDescriptorActor [UUID(ddd827ba)]: Call-to-Backend assignments: main.sleep -> AWS +2016-10-13 13:14:30,167 cromwell-system-akka.dispatchers.engine-dispatcher-39 INFO - WorkflowExecutionActor-ddd827ba-091f-4c6f-b98f-cc9825717007 [UUID(ddd827ba)]: Starting calls: main.sleep:NA:1 +2016-10-13 13:14:30,983 cromwell-system-akka.actor.default-dispatcher-5 ERROR - guardian failed, shutting down system +scala.NotImplementedError: an implementation is missing + at scala.Predef$.$qmark$qmark$qmark(Predef.scala:230) + at cromwell.backend.impl.aws.AwsJobExecutionActor.execute(AwsJobExecutionActor.scala:12) +``` +- OK, so now I just need to implement `execute(): Future[JobExecutionResult]` and Cromwell can interface with AWS. How hard can it be! + +## Part 2: Using Amazon to sleep 20 seconds + +To be continued... + diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala index 5a6c5d268..cbc47c0e5 100644 --- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala +++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala @@ -7,6 +7,7 @@ import com.typesafe.config.Config import cromwell.backend.callcaching.FileHashingActor import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction import cromwell.backend.io.WorkflowPaths +import cromwell.backend.wdl.OnlyPureFunctions import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.core.{ExecutionStore, OutputStore} import wdl4s.Call @@ -16,7 +17,7 @@ import wdl4s.expression.WdlStandardLibraryFunctions trait BackendLifecycleActorFactory { def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], - serviceRegistryActor: ActorRef): Option[Props] + serviceRegistryActor: ActorRef): Option[Props] = None def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], @@ -44,7 +45,7 @@ trait BackendLifecycleActorFactory { def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor, jobKey: BackendJobDescriptorKey, - initializationData: Option[BackendInitializationData]): WdlStandardLibraryFunctions + initializationData: Option[BackendInitializationData]): WdlStandardLibraryFunctions = OnlyPureFunctions def getExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config, initializationData: Option[BackendInitializationData]): Path = { new WorkflowPaths(workflowDescriptor, backendConfig).executionRoot diff --git a/backend/src/main/scala/cromwell/backend/wdl/PureFunctions.scala b/backend/src/main/scala/cromwell/backend/wdl/PureFunctions.scala index 9f297d477..94d5a4d7e 100644 --- a/backend/src/main/scala/cromwell/backend/wdl/PureFunctions.scala +++ b/backend/src/main/scala/cromwell/backend/wdl/PureFunctions.scala @@ -7,15 +7,15 @@ import wdl4s.values.{WdlArray, WdlFile, WdlFloat, WdlInteger, WdlString, WdlValu import scala.util.{Failure, Success, Try} case object OnlyPureFunctions extends WdlStandardLibraryFunctions with PureFunctions { - override def readFile(path: String): String = throw new NotImplementedError("readFile not available in PureNoFunctions.") - override def read_json(params: Seq[Try[WdlValue]]): Try[WdlValue] = throw new NotImplementedError("read_json not available in PureNoFunctions.") - override def write_json(params: Seq[Try[WdlValue]]): Try[WdlFile] = throw new NotImplementedError("write_json not available in PureNoFunctions.") - override def size(params: Seq[Try[WdlValue]]): Try[WdlFloat] = throw new NotImplementedError("size not available in PureNoFunctions.") - override def write_tsv(params: Seq[Try[WdlValue]]): Try[WdlFile] = throw new NotImplementedError("write_tsv not available in PureNoFunctions.") - override def stdout(params: Seq[Try[WdlValue]]): Try[WdlFile] = throw new NotImplementedError("stdout not available in PureNoFunctions.") - override def glob(path: String, pattern: String): Seq[String] = throw new NotImplementedError("glob not available in PureNoFunctions.") - override def writeTempFile(path: String, prefix: String, suffix: String, content: String): String = throw new NotImplementedError("writeTempFile not available in PureNoFunctions.") - override def stderr(params: Seq[Try[WdlValue]]): Try[WdlFile] = throw new NotImplementedError("stderr not available in PureNoFunctions.") + override def readFile(path: String): String = throw new NotImplementedError("readFile not available in OnlyPureFunctions.") + override def read_json(params: Seq[Try[WdlValue]]): Try[WdlValue] = throw new NotImplementedError("read_json not available in OnlyPureFunctions.") + override def write_json(params: Seq[Try[WdlValue]]): Try[WdlFile] = throw new NotImplementedError("write_json not available in OnlyPureFunctions.") + override def size(params: Seq[Try[WdlValue]]): Try[WdlFloat] = throw new NotImplementedError("size not available in OnlyPureFunctions.") + override def write_tsv(params: Seq[Try[WdlValue]]): Try[WdlFile] = throw new NotImplementedError("write_tsv not available in OnlyPureFunctions.") + override def stdout(params: Seq[Try[WdlValue]]): Try[WdlFile] = throw new NotImplementedError("stdout not available in OnlyPureFunctions.") + override def glob(path: String, pattern: String): Seq[String] = throw new NotImplementedError("glob not available in OnlyPureFunctions.") + override def writeTempFile(path: String, prefix: String, suffix: String, content: String): String = throw new NotImplementedError("writeTempFile not available in OnlyPureFunctions.") + override def stderr(params: Seq[Try[WdlValue]]): Try[WdlFile] = throw new NotImplementedError("stderr not available in OnlyPureFunctions.") } trait PureFunctions { this: WdlStandardLibraryFunctions => diff --git a/build.sbt b/build.sbt index 8e666691f..7d764ef58 100644 --- a/build.sbt +++ b/build.sbt @@ -43,6 +43,12 @@ lazy val sfsBackend = (project in backendRoot / "sfs") .dependsOn(gcsFileSystem) .dependsOn(backend % "test->test") +lazy val awsBackend = (project in backendRoot / "aws") + .settings(awsBackendSettings:_*) + .withTestSettings + .dependsOn(backend) + .dependsOn(backend % "test->test") + lazy val htCondorBackend = (project in backendRoot / "htcondor") .settings(htCondorBackendSettings:_*) .withTestSettings @@ -92,10 +98,12 @@ lazy val root = (project in file(".")) .aggregate(htCondorBackend) .aggregate(sparkBackend) .aggregate(jesBackend) + .aggregate(awsBackend) .aggregate(engine) // Next level of projects to include in the fat jar (their dependsOn will be transitively included) .dependsOn(engine) .dependsOn(jesBackend) + .dependsOn(awsBackend) .dependsOn(htCondorBackend) .dependsOn(sparkBackend) // Dependencies for tests diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index a9a7bb7bf..45071fd51 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -338,6 +338,13 @@ backend { # } #} + #AWS { + # actor-factory = "cromwell.backend.impl.aws.AwsBackendActorFactory" + # config { + # + # } + #} + } } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index a902e1b2a..e8a3fb09e 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -113,6 +113,10 @@ object Dependencies { "org.mongodb" %% "casbah" % "3.0.0" ) + val awsBackendDependencies = List( + "com.amazonaws" % "aws-java-sdk" % "1.11.41" + ) + val sparkBackendDependencies = List( "io.spray" %% "spray-client" % sprayV ) ++ sprayServerDependencies diff --git a/project/Settings.scala b/project/Settings.scala index 4a092ce68..2848b2937 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -133,6 +133,11 @@ object Settings { name := "cromwell-sfs-backend" ) ++ commonSettings + val awsBackendSettings = List( + name := "cromwell-aws-backend", + libraryDependencies ++= awsBackendDependencies + ) ++ commonSettings + val htCondorBackendSettings = List( name := "cromwell-htcondor-backend", libraryDependencies ++= htCondorBackendDependencies diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBackendActorFactory.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBackendActorFactory.scala new file mode 100644 index 000000000..6b16379af --- /dev/null +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBackendActorFactory.scala @@ -0,0 +1,12 @@ +package cromwell.backend.impl.aws + +import akka.actor.{ActorRef, Props} +import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, BackendLifecycleActorFactory} + +case class AwsBackendActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory { + + override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, + initializationData: Option[BackendInitializationData], + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props = AwsJobExecutionActor.props(jobDescriptor, configurationDescriptor) +} diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsJobExecutionActor.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsJobExecutionActor.scala new file mode 100644 index 000000000..9852758b9 --- /dev/null +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsJobExecutionActor.scala @@ -0,0 +1,18 @@ +package cromwell.backend.impl.aws + +import akka.actor.Props +import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse +import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobExecutionActor} + +import scala.concurrent.Future + +class AwsJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, + override val configurationDescriptor: BackendConfigurationDescriptor) extends BackendJobExecutionActor { + + override def execute: Future[BackendJobExecutionResponse] = ??? +} + +object AwsJobExecutionActor { + def props(jobDescriptor: BackendJobDescriptor, + configurationDescriptor: BackendConfigurationDescriptor): Props = Props(new AwsJobExecutionActor(jobDescriptor, configurationDescriptor)) +} From 9e212397480b65aaa455fcdb1f18643ded5d23d3 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 13 Oct 2016 14:57:36 -0400 Subject: [PATCH 025/375] Amazon Sleeps As a Service (ASAAS) --- MakingABackend.MD | 86 +++++++++++++++++++++- core/src/main/resources/reference.conf | 4 +- .../backend/impl/aws/AwsJobExecutionActor.scala | 65 +++++++++++++++- .../backend/impl/aws/util/AwsSdkAsyncHandler.scala | 28 +++++++ 4 files changed, 175 insertions(+), 8 deletions(-) create mode 100644 supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/util/AwsSdkAsyncHandler.scala diff --git a/MakingABackend.MD b/MakingABackend.MD index aecb427c9..5106b3726 100644 --- a/MakingABackend.MD +++ b/MakingABackend.MD @@ -4,13 +4,13 @@ - These notes were added while making a new AWS backend for Amazon AWS. -## Part 1: The skeleton: +## Part 1 (October 13 2016): The skeleton: To start with, I just need to create a bunch of boilerplate which will eventually be filled in with all of the lovely AWS details! ### Defining the awsBackend project: -- Added entries to project/Settings.scala, project/Dependencies.scala and build.sbt +- Added entries to `project/Settings.scala`, `project/Dependencies.scala` and `build.sbt` - This was mainly just a copy/paste from existing backend projects. I made a few typos renaming everything and linking the dependencies properly though! - E.g. In my first commit I forgot to update the libraryDependencies name for my AWS backend project: ``` @@ -113,7 +113,85 @@ scala.NotImplementedError: an implementation is missing ``` - OK, so now I just need to implement `execute(): Future[JobExecutionResult]` and Cromwell can interface with AWS. How hard can it be! -## Part 2: Using Amazon to sleep 20 seconds +## Part 2 (October 13 2016): Using Amazon to sleep 20 seconds + +### Starting point +- This was a learning experience after using the Google pipelines service to submit jobs! +- To get myself started, I've manually created an ECS cluster which I've called `ecs-t2micro-cluster` via the ECS web console. + +### Trial and Error + +- I see in the aws sdk docs that there's an AmazonECSAsyncClient class. That sounds promising! Luckily I already added the dependency on AWS SDK in Part 1 so I guess I can just write something basic in my AwsJobExecutionActor class and see what happens: + +- I ended up having to add some credentials options to the configuration file. The new `reference.conf` now looks like: +``` + #AWS { + # actor-factory = "cromwell.backend.impl.aws.AwsBackendActorFactory" + # config { + # ## These two settings are required to authenticate with the ECS service: + # accessKeyId = "..." + # secretKey = "..." + # } + #} +``` + +- After a little bit of experimentation with the ECS API, I was able to come up with a backend that works but is very limited... It is entirely synchronous in the `execute` method. That's certainly not a final answer but it works OK for running a single task. And we can now run that single `sleep` command successfully on the Amazon EC2 Container Service! + - The synchronous `execute` method: +``` +class AwsJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, + override val configurationDescriptor: BackendConfigurationDescriptor) extends BackendJobExecutionActor { + + val awsAccessKeyId = configurationDescriptor.backendConfig.as[String]("accessKeyId") + val awsSecretKey = configurationDescriptor.backendConfig.as[String]("secretKey") + + val clusterName = "ecs-t2micro-cluster" + + val credentials = new AWSCredentials { + override def getAWSAccessKeyId: String = awsAccessKeyId + override def getAWSSecretKey: String = awsSecretKey + } + val ecsAsyncClient = new AmazonECSAsyncClient(credentials) + + override def execute: Future[BackendJobExecutionResponse] = { + + val commandOverride = new ContainerOverride().withName("simple-app").withCommand(jobDescriptor.call.instantiateCommandLine(Map.empty, OnlyPureFunctions, identity).get) + + val runRequest: RunTaskRequest = new RunTaskRequest() + .withCluster(clusterName) + .withCount(1) + .withTaskDefinition("ubuntuTask:1") + .withOverrides(new TaskOverride().withContainerOverrides(commandOverride)) + + val submitResultHandler = new AwsSdkAsyncHandler[RunTaskRequest, RunTaskResult]() + val _ = ecsAsyncClient.runTaskAsync(runRequest, submitResultHandler) + + submitResultHandler.future map { + case AwsSdkAsyncResult(_, result) => + log.info("AWS submission completed:\n{}", result.toString) + val taskArn= result.getTasks.asScala.head.getTaskArn + val taskDescription = waitUntilDone(taskArn) + + log.info("AWS task completed!\n{}", taskDescription.toString) + SucceededResponse(jobDescriptor.key, Option(0), Map.empty, None, Seq.empty) + } + } + + private def waitUntilDone(taskArn: String): Task = { + val describeTasksRequest = new DescribeTasksRequest().withCluster(clusterName).withTasks(List(taskArn).asJava) + + val resultHandler = new AwsSdkAsyncHandler[DescribeTasksRequest, DescribeTasksResult]() + val _ = ecsAsyncClient.describeTasksAsync(describeTasksRequest, resultHandler) + + val desribedTasks = Await.result(resultHandler.future, Duration.Inf) + val taskDescription = desribedTasks.result.getTasks.asScala.head + if (taskDescription.getLastStatus == DesiredStatus.STOPPED.toString) { + taskDescription + } else { + Thread.sleep(200) + waitUntilDone(taskArn) + } + } +} +``` -To be continued... diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 45071fd51..6305228d9 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -341,7 +341,9 @@ backend { #AWS { # actor-factory = "cromwell.backend.impl.aws.AwsBackendActorFactory" # config { - # + # ## These two settings are required to authenticate with the ECS service: + # accessKeyId = "..." + # secretKey = "..." # } #} diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsJobExecutionActor.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsJobExecutionActor.scala index 9852758b9..3b3015a36 100644 --- a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsJobExecutionActor.scala +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsJobExecutionActor.scala @@ -1,15 +1,74 @@ package cromwell.backend.impl.aws import akka.actor.Props -import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse +import com.amazonaws.auth.AWSCredentials +import com.amazonaws.services.ecs.AmazonECSAsyncClient +import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, SucceededResponse} import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobExecutionActor} +import com.amazonaws.services.ecs.model._ +import cromwell.backend.impl.aws.util.AwsSdkAsyncHandler +import cromwell.backend.impl.aws.util.AwsSdkAsyncHandler.AwsSdkAsyncResult +import cromwell.backend.wdl.OnlyPureFunctions +import net.ceedubs.ficus.Ficus._ -import scala.concurrent.Future +import scala.collection.JavaConverters._ +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, Future} class AwsJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, override val configurationDescriptor: BackendConfigurationDescriptor) extends BackendJobExecutionActor { - override def execute: Future[BackendJobExecutionResponse] = ??? + val awsAccessKeyId = configurationDescriptor.backendConfig.as[String]("accessKeyId") + val awsSecretKey = configurationDescriptor.backendConfig.as[String]("secretKey") + + val clusterName = "ecs-t2micro-cluster" + + val credentials = new AWSCredentials { + override def getAWSAccessKeyId: String = awsAccessKeyId + override def getAWSSecretKey: String = awsSecretKey + } + val ecsAsyncClient = new AmazonECSAsyncClient(credentials) + + override def execute: Future[BackendJobExecutionResponse] = { + + val commandOverride = new ContainerOverride().withName("simple-app").withCommand(jobDescriptor.call.instantiateCommandLine(Map.empty, OnlyPureFunctions, identity).get) + + val runRequest: RunTaskRequest = new RunTaskRequest() + .withCluster(clusterName) + .withCount(1) + .withTaskDefinition("ubuntuTask:1") + .withOverrides(new TaskOverride().withContainerOverrides(commandOverride)) + + val submitResultHandler = new AwsSdkAsyncHandler[RunTaskRequest, RunTaskResult]() + val _ = ecsAsyncClient.runTaskAsync(runRequest, submitResultHandler) + + submitResultHandler.future map { + case AwsSdkAsyncResult(_, result) => + log.info("AWS submission completed:\n{}", result.toString) + val taskArn= result.getTasks.asScala.head.getTaskArn + val taskDescription = waitUntilDone(taskArn) + + log.info("AWS task completed!\n{}", taskDescription.toString) + SucceededResponse(jobDescriptor.key, Option(0), Map.empty, None, Seq.empty) + } + } + + private def waitUntilDone(taskArn: String): Task = { + val describeTasksRequest = new DescribeTasksRequest().withCluster(clusterName).withTasks(List(taskArn).asJava) + + val resultHandler = new AwsSdkAsyncHandler[DescribeTasksRequest, DescribeTasksResult]() + val _ = ecsAsyncClient.describeTasksAsync(describeTasksRequest, resultHandler) + + val describedTasks = Await.result(resultHandler.future, Duration.Inf) + val taskDescription = describedTasks.result.getTasks.asScala.head + if (taskDescription.getLastStatus == DesiredStatus.STOPPED.toString) { + taskDescription + } else { + log.info(s"Still waiting for completion. Last known status: {}", taskDescription.getLastStatus) + Thread.sleep(2000) + waitUntilDone(taskArn) + } + } } object AwsJobExecutionActor { diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/util/AwsSdkAsyncHandler.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/util/AwsSdkAsyncHandler.scala new file mode 100644 index 000000000..7433dc4ca --- /dev/null +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/util/AwsSdkAsyncHandler.scala @@ -0,0 +1,28 @@ +package cromwell.backend.impl.aws.util + +import com.amazonaws.AmazonWebServiceRequest +import com.amazonaws.handlers.AsyncHandler +import cromwell.backend.impl.aws.util.AwsSdkAsyncHandler._ + +import scala.concurrent.Promise + +/** + * Scala wrapper around the AWS SDK's AsyncHandler callbacks + */ +final class AwsSdkAsyncHandler[REQUEST <: AmazonWebServiceRequest, RESULT] extends AsyncHandler[REQUEST, RESULT] { + private val promise = Promise[AwsSdkAsyncResult[REQUEST, RESULT]]() + override def onError(exception: Exception): Unit = { + promise.tryFailure(exception) + () + } + override def onSuccess(request: REQUEST, result: RESULT): Unit = { + promise.trySuccess(AwsSdkAsyncResult(request, result)) + () + } + + def future = promise.future +} + +object AwsSdkAsyncHandler { + case class AwsSdkAsyncResult[REQUEST <: AmazonWebServiceRequest, RESULT](request: REQUEST, result: RESULT) +} From f93df6bbd0e04dd99f071cf83a1e7d678e098527 Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Fri, 14 Oct 2016 11:48:51 -0400 Subject: [PATCH 026/375] Tyranny makes a comeback. --- .../scala/cromwell/database/slick/CallCachingSlickDatabase.scala | 1 + project/Settings.scala | 3 ++- .../jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala | 6 +----- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala index ce40e4e6a..deb7b6d5e 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala @@ -5,6 +5,7 @@ import cromwell.database.sql._ import cromwell.database.sql.joins.CallCachingJoin import scala.concurrent.{ExecutionContext, Future} +import scala.language.postfixOps trait CallCachingSlickDatabase extends CallCachingSqlDatabase { this: SlickDatabase => diff --git a/project/Settings.scala b/project/Settings.scala index 2848b2937..1f44bda42 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -48,7 +48,8 @@ object Settings { "-Ywarn-numeric-widen", "-Ywarn-value-discard", "-Ywarn-unused", - "-Ywarn-unused-import" + "-Ywarn-unused-import", + "-Xfatal-warnings" ) lazy val assemblySettings = Seq( diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala index 39430abb2..5398ed66c 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala @@ -3,13 +3,11 @@ package cromwell.backend.impl.jes import java.time.OffsetDateTime import java.util -import com.google.api.client.googleapis.testing.auth.oauth2.MockGoogleCredential import com.google.api.client.util.ArrayMap -import com.google.api.services.genomics.Genomics import com.google.api.services.genomics.model.Operation +import cromwell.core.ExecutionEvent import org.scalatest.{FlatSpec, Matchers} import org.specs2.mock.{Mockito => MockitoTrait} -import cromwell.core.ExecutionEvent import scala.collection.JavaConverters._ @@ -36,8 +34,6 @@ class RunSpec extends FlatSpec with Matchers with MockitoTrait { op.setMetadata(metadata.asJava) - val mockedCredentials = new MockGoogleCredential.Builder().build() - val genomics = new Genomics(mockedCredentials.getTransport, mockedCredentials.getJsonFactory, mockedCredentials) val list = Run.getEventList(op) list should contain theSameElementsAs List( ExecutionEvent("waiting for quota", OffsetDateTime.parse("2015-12-05T00:00:00+00:00")), From fc7d8f5fc38e757528615748b1290027c39a473a Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Fri, 14 Oct 2016 13:28:19 -0400 Subject: [PATCH 027/375] Added index and attempt number to failure messages --- engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala | 2 +- .../cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala | 2 +- .../cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala index f327c6da4..f676f170d 100644 --- a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala @@ -111,7 +111,7 @@ class SimpleWorkflowActorSpec extends CromwellTestkitSpec with BeforeAndAfter { } "fail when a call fails" in { - val expectedError = "Call goodbye.goodbye: return code was 1" + val expectedError = "Call goodbye.goodbye:-1:1: return code was 1" val failureMatcher = FailureMatcher(expectedError) val TestableWorkflowActorAndMetadataPromise(workflowActor, supervisor, promise) = buildWorkflowActor(SampleWdl.GoodbyeWorld, SampleWdl.GoodbyeWorld.wdlJson, workflowId, failureMatcher) val probe = TestProbe() diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala index a08be8df9..f3f1831de 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala @@ -588,7 +588,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes FailedNonRetryableExecutionHandle(new RuntimeException( s"execution failed: could not parse return code as integer: ${returnCodeContents.get}")).future case _: RunStatus.Success if !continueOnReturnCode.continueFor(returnCode.get) => - val badReturnCodeMessage = s"Call ${call.fullyQualifiedName}: return code was ${returnCode.getOrElse("(none)")}" + val badReturnCodeMessage = s"Call ${jobDescriptor.key}: return code was ${returnCode.getOrElse("(none)")}" FailedNonRetryableExecutionHandle(new RuntimeException(badReturnCodeMessage), returnCode.toOption).future case success: RunStatus.Success => handleSuccess(postProcess, returnCode.get, jesCallPaths.detritusPaths.mapValues(_.toString), handle, success.eventList).future diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala index 3013b12af..3c7f86d30 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala @@ -313,7 +313,7 @@ trait SharedFileSystemAsyncJobExecutionActor def processReturnCode()(implicit ec: ExecutionContext): Future[ExecutionHandle] = { val returnCodeTry = Try(File(jobPaths.returnCode).contentAsString.stripLineEnd.toInt) - lazy val badReturnCodeMessage = s"Call ${call.fullyQualifiedName}: return code was ${returnCodeTry.getOrElse("(none)")}" + lazy val badReturnCodeMessage = s"Call ${jobDescriptor.key}: return code was ${returnCodeTry.getOrElse("(none)")}" lazy val badReturnCodeResponse = Future.successful( FailedNonRetryableExecutionHandle(new Exception(badReturnCodeMessage), returnCodeTry.toOption)) From 33ce0957fd27ee933f978d08a0abc54b5b284004 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Fri, 14 Oct 2016 14:53:46 -0400 Subject: [PATCH 028/375] Function added --- .../scala/cromwell/backend/wdl/PureFunctions.scala | 33 ++++++++++++++++++++-- .../cromwell/backend/wdl/PureFunctionsSpec.scala | 29 +++++++++++++++++++ project/Dependencies.scala | 2 +- 3 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 backend/src/test/scala/cromwell/backend/wdl/PureFunctionsSpec.scala diff --git a/backend/src/main/scala/cromwell/backend/wdl/PureFunctions.scala b/backend/src/main/scala/cromwell/backend/wdl/PureFunctions.scala index 94d5a4d7e..dac9194d6 100644 --- a/backend/src/main/scala/cromwell/backend/wdl/PureFunctions.scala +++ b/backend/src/main/scala/cromwell/backend/wdl/PureFunctions.scala @@ -1,7 +1,7 @@ package cromwell.backend.wdl import wdl4s.expression.WdlStandardLibraryFunctions -import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType} +import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType, WdlType} import wdl4s.values.{WdlArray, WdlFile, WdlFloat, WdlInteger, WdlString, WdlValue} import scala.util.{Failure, Success, Try} @@ -20,10 +20,39 @@ case object OnlyPureFunctions extends WdlStandardLibraryFunctions with PureFunct trait PureFunctions { this: WdlStandardLibraryFunctions => + def transpose(params: Seq[Try[WdlValue]]): Try[WdlArray] = { + def extractExactlyOneArg: Try[WdlValue] = params.size match { + case 1 => params.head + case n => Failure(new IllegalArgumentException(s"Invalid number of parameters for engine function transpose: $n. Ensure transpose(x: Array[Array[X]]) takes exactly 1 parameters.")) + } + + case class ExpandedTwoDimensionalArray(innerType: WdlType, value: Seq[Seq[WdlValue]]) + def validateAndExpand(value: WdlValue): Try[ExpandedTwoDimensionalArray] = value match { + case WdlArray(WdlArrayType(WdlArrayType(innerType)), array: Seq[WdlValue]) => expandWdlArray(array) map { ExpandedTwoDimensionalArray(innerType, _) } + case array @ WdlArray(WdlArrayType(nonArrayType), _) => Failure(new IllegalArgumentException(s"Array must be two-dimensional to be transposed but given array of $nonArrayType")) + case otherValue => Failure(new IllegalArgumentException(s"Function 'transpose' must be given a two-dimensional array but instead got ${otherValue.typeName}")) + } + + def expandWdlArray(outerArray: Seq[WdlValue]): Try[Seq[Seq[WdlValue]]] = Try { + outerArray map { + case array: WdlArray => array.value + case otherValue => throw new IllegalArgumentException(s"Function 'transpose' must be given a two-dimensional array but instead got WdlArray[${otherValue.typeName}]") + } + } + + def transpose(expandedTwoDimensionalArray: ExpandedTwoDimensionalArray): Try[WdlArray] = Try { + val innerType = expandedTwoDimensionalArray.innerType + val array = expandedTwoDimensionalArray.value + WdlArray(WdlArrayType(WdlArrayType(innerType)), array.transpose map { WdlArray(WdlArrayType(innerType), _) }) + } + + extractExactlyOneArg.flatMap(validateAndExpand).flatMap(transpose) + } + def range(params: Seq[Try[WdlValue]]): Try[WdlArray] = { def extractAndValidateArguments = params.size match { case 1 => validateArguments(params.head) - case n => Failure(new IllegalArgumentException(s"Invalid number of parameters for engine function seq: $n. Ensure seq(x: WdlInteger) takes exactly 1 parameters.")) + case n => Failure(new IllegalArgumentException(s"Invalid number of parameters for engine function range: $n. Ensure range(x: WdlInteger) takes exactly 1 parameters.")) } def validateArguments(value: Try[WdlValue]) = value match { diff --git a/backend/src/test/scala/cromwell/backend/wdl/PureFunctionsSpec.scala b/backend/src/test/scala/cromwell/backend/wdl/PureFunctionsSpec.scala new file mode 100644 index 000000000..284bf7636 --- /dev/null +++ b/backend/src/test/scala/cromwell/backend/wdl/PureFunctionsSpec.scala @@ -0,0 +1,29 @@ +package cromwell.backend.wdl + +import org.scalatest.{FlatSpec, Matchers} +import wdl4s.types.{WdlArrayType, WdlIntegerType} +import wdl4s.values.{WdlArray, WdlInteger} + +import scala.util.Success + + +class PureFunctionsSpec extends FlatSpec with Matchers { + + behavior of "transpose" + + it should "transpose a 2x3 into a 3x2" in { + val inArray = WdlArray(WdlArrayType(WdlArrayType(WdlIntegerType)), List( + WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(1), WdlInteger(2), WdlInteger(3))), + WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(4), WdlInteger(5), WdlInteger(6))) + )) + + val expectedResult = WdlArray(WdlArrayType(WdlArrayType(WdlIntegerType)), List( + WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(1), WdlInteger(4))), + WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(2), WdlInteger(5))), + WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(3), WdlInteger(6))) + )) + + OnlyPureFunctions.transpose(Seq(Success(inArray))) should be(Success(expectedResult)) + } + +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index e8a3fb09e..97f848656 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -2,7 +2,7 @@ import sbt._ object Dependencies { lazy val lenthallV = "0.19" - lazy val wdl4sV = "0.6" + lazy val wdl4sV = "0.7-a990675-SNAPSHOT" lazy val sprayV = "1.3.3" /* spray-json is an independent project from the "spray suite" From 076ed1a25403ccec3e619db00ad7778af79a38a0 Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Fri, 14 Oct 2016 16:52:20 -0400 Subject: [PATCH 029/375] no pipe stripping closes #799 --- .../backend/impl/htcondor/HtCondorWrapper.scala | 15 +++++++++------ .../sfs/config/ConfigAsyncJobExecutionActor.scala | 12 ++++++++---- .../SharedFileSystemAsyncJobExecutionActor.scala | 21 +++++++++++++-------- .../cromwell/backend/impl/spark/SparkProcess.scala | 17 +++++++++++------ 4 files changed, 41 insertions(+), 24 deletions(-) diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala index 76a81c08a..2c59716a7 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala @@ -55,12 +55,15 @@ class HtCondorCommands extends StrictLogging { */ def writeScript(instantiatedCommand: String, filePath: Path, containerRoot: Path): Unit = { logger.debug(s"Writing bash script for execution. Command: $instantiatedCommand.") - File(filePath).write( - s"""#!/bin/sh - |cd $containerRoot - |$instantiatedCommand - |echo $$? > rc - |""".stripMargin) + val scriptBody = s""" + +#!/bin/sh +cd $containerRoot +$instantiatedCommand +echo $$? > rc + +""".trim + "\n" + File(filePath).write(scriptBody) () } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala index ad9761bca..a92c66ffc 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala @@ -53,10 +53,14 @@ sealed trait ConfigAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecut getOrElse(throw new RuntimeException(s"Unable to find task $taskName")) val command = task.instantiateCommand(inputs, NoFunctions).get jobLogger.info(s"executing: $command") - script.write( - s"""|#!/bin/bash - |$command - |""".stripMargin) + val scriptBody = + s""" + +#!/bin/bash +$command + +""".trim + "\n" + script.write(scriptBody) () } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala index 3c7f86d30..2ba4edd48 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala @@ -244,14 +244,19 @@ trait SharedFileSystemAsyncJobExecutionActor val rcPath = if (isDockerRun) jobPaths.toDockerPath(jobPaths.returnCode) else jobPaths.returnCode val rcTmpPath = s"$rcPath.tmp" - File(jobPaths.script).write( - s"""#!/bin/sh - |( - | cd $cwd - | $instantiatedCommand - |) - |echo $$? > $rcTmpPath - |mv $rcTmpPath $rcPath""".stripMargin) + val scriptBody = s""" + +#!/bin/sh +( + cd $cwd + $instantiatedCommand +) +echo $$? > $rcTmpPath +mv $rcTmpPath $rcPath + +""".trim + "\n" + + File(jobPaths.script).write(scriptBody) } /** diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala index 60f399218..883645ecd 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala @@ -29,12 +29,17 @@ class SparkCommands extends StrictLogging { * as some extra shell code for monitoring jobs */ def writeScript(instantiatedCommand: String, filePath: Path, containerRoot: Path) = { - File(filePath).write( - s"""#!/bin/sh - |cd $containerRoot - |$instantiatedCommand - |echo $$? > rc - |""".stripMargin) + + val scriptBody = + s""" + +#!/bin/sh +cd $containerRoot +$instantiatedCommand +echo $$? > rc + + """.trim + "\n" + File(filePath).write(scriptBody) } def sparkSubmitCommand(attributes: Map[String, Any]): String = { From b468e99560208eb8cb910d4a4134024f51a6a943 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Fri, 14 Oct 2016 18:33:59 -0400 Subject: [PATCH 030/375] Maximum polling interval option reinstated --- .../cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala index f3f1831de..6e697032c 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala @@ -92,7 +92,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes override val pollingActor = jesBackendSingletonActor override lazy val pollBackOff = SimpleExponentialBackoff( - initialInterval = 30 seconds, maxInterval = 10 minutes, multiplier = 1.1) + initialInterval = 30 seconds, maxInterval = jesAttributes.maxPollingInterval seconds, multiplier = 1.1) override lazy val executeOrRecoverBackOff = SimpleExponentialBackoff( initialInterval = 3 seconds, maxInterval = 20 seconds, multiplier = 1.1) From 2218cdcfd244649990d0d05b09d774732ee301f9 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Mon, 17 Oct 2016 12:11:43 -0400 Subject: [PATCH 031/375] Bumped upcoming version. Added changelog entry for 23 --- CHANGELOG.md | 4 ++++ project/Version.scala | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf48b60be..c8b44b140 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Cromwell Change Log +## 23 + +* Included the WDL matrix `transpose` function for `Array[Array[X]]` types. + ## 0.22 * Improved retries for Call Caching and general bug fixes. diff --git a/project/Version.scala b/project/Version.scala index e6de0a5cd..d72b63ac4 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -4,7 +4,7 @@ import sbt._ object Version { // Upcoming release, or current if we're on the master branch - val cromwellVersion = "0.22" + val cromwellVersion = "23" // Adapted from SbtGit.versionWithGit def cromwellVersionWithGit: Seq[Setting[_]] = From c9ea9dd0614c05f4c3ae142c17bbba1783072361 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Mon, 17 Oct 2016 23:31:23 -0400 Subject: [PATCH 032/375] Stop invoking scalacheck during the sbt build by replacing a) specs2 with specs2-mock plus pegdown, and b) excluding cats dependencies (also in wdl4s). Removed cromwell dependency duplications (see the verboseness in excising cats' duplicated dependencies). Just in case, pass scalatest arguments only to scalatest. --- project/Dependencies.scala | 26 ++++++++++++++++---------- project/Testing.scala | 20 +++++++++++--------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 97f848656..55b7046b0 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -2,7 +2,7 @@ import sbt._ object Dependencies { lazy val lenthallV = "0.19" - lazy val wdl4sV = "0.7-a990675-SNAPSHOT" + lazy val wdl4sV = "0.7-90945ea-SNAPSHOT" lazy val sprayV = "1.3.3" /* spray-json is an independent project from the "spray suite" @@ -22,11 +22,20 @@ object Dependencies { private val baseDependencies = List( "org.broadinstitute" %% "lenthall" % lenthallV, - "org.typelevel" %% "cats" % catsV, + /* + Exclude test framework cats-laws and its transitive dependency scalacheck. + If sbt detects scalacheck, it tries to run it. + Explicitly excluding the two problematic artifacts instead of including the three (or four?). + https://github.com/typelevel/cats/tree/v0.7.2#getting-started + */ + "org.typelevel" %% "cats" % "0.7.2" + exclude("org.typelevel", "cats-laws_2.11") + exclude("org.typelevel", "cats-kernel-laws_2.11"), "com.github.benhutchison" %% "mouse" % "0.5", "com.iheart" %% "ficus" % "1.3.0", "org.scalatest" %% "scalatest" % "3.0.0" % Test, - "org.specs2" %% "specs2" % "3.7" % Test + "org.pegdown" % "pegdown" % "1.6.0" % Test, + "org.specs2" %% "specs2-mock" % "3.8.5" % Test ) private val slf4jBindingDependencies = List( @@ -90,11 +99,6 @@ object Dependencies { val databaseSqlDependencies = baseDependencies ++ slickDependencies ++ dbmsDependencies - val databaseMigrationDependencies = List( - "org.broadinstitute" %% "wdl4s" % wdl4sV, // Used in migration scripts - "com.github.pathikrit" %% "better-files" % betterFilesV % Test - ) ++ baseDependencies ++ liquibaseDependencies ++ dbmsDependencies - val coreDependencies = List( "com.typesafe.scala-logging" %% "scala-logging" % "3.4.0", "org.broadinstitute" %% "wdl4s" % wdl4sV, @@ -108,6 +112,10 @@ object Dependencies { // TODO: We're not using the "F" in slf4j. Core only supports logback, specifically the WorkflowLogger. slf4jBindingDependencies + val databaseMigrationDependencies = List( + "com.github.pathikrit" %% "better-files" % betterFilesV % Test + ) ++ liquibaseDependencies ++ dbmsDependencies + val htCondorBackendDependencies = List( "com.twitter" %% "chill" % "0.8.0", "org.mongodb" %% "casbah" % "3.0.0" @@ -122,11 +130,9 @@ object Dependencies { ) ++ sprayServerDependencies val engineDependencies = List( - "com.typesafe.scala-logging" %% "scala-logging" % "3.4.0", "org.webjars" % "swagger-ui" % "2.1.1", "commons-codec" % "commons-codec" % "1.10", "commons-io" % "commons-io" % "2.5", - "org.typelevel" %% "cats" % catsV, "com.github.pathikrit" %% "better-files" % betterFilesV, "io.swagger" % "swagger-parser" % "1.0.22" % Test, "org.yaml" % "snakeyaml" % "1.17" % Test diff --git a/project/Testing.scala b/project/Testing.scala index 82df707d5..46ca8f155 100644 --- a/project/Testing.scala +++ b/project/Testing.scala @@ -10,23 +10,25 @@ object Testing { lazy val DbmsTest = config("dbms") extend Test lazy val DockerTestTag = "DockerTest" - lazy val UseDockerTaggedTests = Tests.Argument("-n", DockerTestTag) - lazy val DontUseDockerTaggedTests = Tests.Argument("-l", DockerTestTag) + lazy val UseDockerTaggedTests = Tests.Argument(TestFrameworks.ScalaTest, "-n", DockerTestTag) + lazy val DontUseDockerTaggedTests = Tests.Argument(TestFrameworks.ScalaTest, "-l", DockerTestTag) lazy val CromwellIntegrationTestTag = "CromwellIntegrationTest" - lazy val UseCromwellIntegrationTaggedTests = Tests.Argument("-n", CromwellIntegrationTestTag) - lazy val DontUseCromwellIntegrationTaggedTests = Tests.Argument("-l", CromwellIntegrationTestTag) + lazy val UseCromwellIntegrationTaggedTests = + Tests.Argument(TestFrameworks.ScalaTest, "-n", CromwellIntegrationTestTag) + lazy val DontUseCromwellIntegrationTaggedTests = + Tests.Argument(TestFrameworks.ScalaTest, "-l", CromwellIntegrationTestTag) lazy val GcsIntegrationTestTag = "GcsIntegrationTest" - lazy val UseGcsIntegrationTaggedTests = Tests.Argument("-n", GcsIntegrationTestTag) - lazy val DontUseGcsIntegrationTaggedTests = Tests.Argument("-l", GcsIntegrationTestTag) + lazy val UseGcsIntegrationTaggedTests = Tests.Argument(TestFrameworks.ScalaTest, "-n", GcsIntegrationTestTag) + lazy val DontUseGcsIntegrationTaggedTests = Tests.Argument(TestFrameworks.ScalaTest, "-l", GcsIntegrationTestTag) lazy val DbmsTestTag = "DbmsTest" - lazy val UseDbmsTaggedTests = Tests.Argument("-n", DbmsTestTag) - lazy val DontUseDbmsTaggedTests = Tests.Argument("-l", DbmsTestTag) + lazy val UseDbmsTaggedTests = Tests.Argument(TestFrameworks.ScalaTest, "-n", DbmsTestTag) + lazy val DontUseDbmsTaggedTests = Tests.Argument(TestFrameworks.ScalaTest, "-l", DbmsTestTag) lazy val PostMVPTag = "PostMVP" - lazy val DontUsePostMVPTaggedTests = Tests.Argument("-l", PostMVPTag) + lazy val DontUsePostMVPTaggedTests = Tests.Argument(TestFrameworks.ScalaTest, "-l", PostMVPTag) lazy val TestReportArgs = Tests.Argument(TestFrameworks.ScalaTest, "-oDSI", "-h", "target/test-reports") From 01b127bfa3986078c259babe18cec92fb5b2e410 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Mon, 17 Oct 2016 23:33:38 -0400 Subject: [PATCH 033/375] 3 seconds timeout (instead of the 1 second default) for each of the slick and liquibase databases being compared. Removed dead docker case class. Formatting updates for sbt-docker. --- .../cromwell/core/callcaching/CallCachingMode.scala | 4 ---- project/Settings.scala | 19 +++++++++---------- .../scala/cromwell/services/ServicesStoreSpec.scala | 1 + 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/core/src/main/scala/cromwell/core/callcaching/CallCachingMode.scala b/core/src/main/scala/cromwell/core/callcaching/CallCachingMode.scala index df9fb679a..94717ac1c 100644 --- a/core/src/main/scala/cromwell/core/callcaching/CallCachingMode.scala +++ b/core/src/main/scala/cromwell/core/callcaching/CallCachingMode.scala @@ -34,7 +34,3 @@ sealed trait ReadWriteMode { case object ReadCache extends ReadWriteMode { override val w = false } case object WriteCache extends ReadWriteMode { override val r = false } case object ReadAndWriteCache extends ReadWriteMode - -sealed trait DockerHashingType -case object HashDockerName extends DockerHashingType -case object HashDockerNameAndLookupDockerHash extends DockerHashingType diff --git a/project/Settings.scala b/project/Settings.scala index 1f44bda42..bfe24077c 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -6,8 +6,8 @@ import Version._ import sbt.Keys._ import sbt._ import sbtassembly.AssemblyPlugin.autoImport._ -import sbtrelease.ReleasePlugin import sbtdocker.DockerPlugin.autoImport._ +import sbtrelease.ReleasePlugin object Settings { @@ -59,14 +59,14 @@ object Settings { logLevel in assembly := Level.Info, assemblyMergeStrategy in assembly := customMergeStrategy ) - + lazy val dockerSettings = Seq( imageNames in docker := Seq( - ImageName( - namespace = Option("broadinstitute"), - repository = name.value, - tag = Some(s"${version.value}") - ) + ImageName( + namespace = Option("broadinstitute"), + repository = name.value, + tag = Some(s"${version.value}") + ) ), dockerfile in docker := { // The assembly task generates a fat JAR file @@ -78,7 +78,7 @@ object Settings { expose(8000) add(artifact, artifactTargetPath) runRaw(s"ln -s $artifactTargetPath /app/cromwell.jar") - + // If you use the 'exec' form for an entry point, shell processing is not performed and // environment variable substitution does not occur. Thus we have to /bin/bash here // and pass along any subsequent command line arguments @@ -90,8 +90,7 @@ object Settings { cache = false, removeIntermediateContainers = BuildOptions.Remove.Always ) - ) - + ) val commonSettings = ReleasePlugin.projectSettings ++ testSettings ++ assemblySettings ++ dockerSettings ++ cromwellVersionWithGit ++ publishingSettings ++ List( diff --git a/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala b/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala index 291b41988..7558a3988 100644 --- a/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala +++ b/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala @@ -160,6 +160,7 @@ object ServicesStoreSpec { s""" |db.url = "jdbc:hsqldb:mem:$${uniqueSchema};shutdown=false;hsqldb.tx=mvcc" |db.driver = "org.hsqldb.jdbcDriver" + |db.connectionTimeout = 3000 |driver = "slick.driver.HsqldbDriver$$" |liquibase.updateSchema = false |""".stripMargin) From 98b7f2fdbeecf8c5e282c68228993bef8cbe9289 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Mon, 17 Oct 2016 17:05:15 -0400 Subject: [PATCH 034/375] JABJEA failure catcher --- .../backend/impl/jes/JesJobExecutionActor.scala | 25 +++-- .../impl/jes/JesJobExecutionActorSpec.scala | 111 +++++++++++++++++++++ 2 files changed, 129 insertions(+), 7 deletions(-) create mode 100644 supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala index b1a8ba8d6..476ee04aa 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala @@ -1,6 +1,7 @@ package cromwell.backend.impl.jes -import akka.actor.{ActorRef, Props} +import akka.actor.SupervisorStrategy.{Decider, Stop} +import akka.actor.{ActorRef, OneForOneStrategy, Props} import akka.event.LoggingReceive import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse} import cromwell.backend.BackendLifecycleActor.AbortJobCommand @@ -62,13 +63,15 @@ case class JesJobExecutionActor(override val jobDescriptor: BackendJobDescriptor private var executor: Option[ActorRef] = None + private[jes] def jabjeaProps = JesAsyncBackendJobExecutionActor.props(jobDescriptor, + completionPromise, + jesConfiguration, + initializationData, + serviceRegistryActor, + jesBackendSingletonActor) + private def launchExecutor: Future[Unit] = Future { - val executionProps = JesAsyncBackendJobExecutionActor.props(jobDescriptor, - completionPromise, - jesConfiguration, - initializationData, - serviceRegistryActor, - jesBackendSingletonActor) + val executionProps = jabjeaProps val executorRef = context.actorOf(executionProps, "JesAsyncBackendJobExecutionActor") executor = Option(executorRef) () @@ -95,4 +98,12 @@ case class JesJobExecutionActor(override val jobDescriptor: BackendJobDescriptor } override def abort(): Unit = {} + + // Supervision strategy: if the JABJEA throws an exception, stop the actor and fail the job. + def jobFailingDecider: Decider = { + case e: Exception => + completionPromise.tryFailure(new RuntimeException("JesAsyncBackendJobExecutionActor failed and didn't catch its exception.", e)) + Stop + } + override val supervisorStrategy = OneForOneStrategy()(jobFailingDecider) } diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala new file mode 100644 index 000000000..20e1c5d85 --- /dev/null +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala @@ -0,0 +1,111 @@ +package cromwell.backend.impl.jes + +import akka.actor.{Actor, ActorRef, Props} +import akka.testkit.{TestActorRef, TestProbe} +import cromwell.backend.BackendJobDescriptor +import cromwell.core.TestKitSuite +import org.scalatest.{FlatSpecLike, Matchers} +import org.specs2.mock.Mockito + +import scala.concurrent.duration._ +import akka.testkit._ +import cromwell.backend.BackendJobExecutionActor.{ExecuteJobCommand, FailedNonRetryableResponse} +import cromwell.backend.impl.jes.ControllableFailingJabjea.JabjeaExplode + +import scala.concurrent.{ExecutionContext, Promise} + +class JesJobExecutionActorSpec extends TestKitSuite("JesJobExecutionActorSpec") with FlatSpecLike with Matchers with Mockito { + + behavior of "JesJobExecutionActor" + + private val AwaitAlmostNothing = 100.milliseconds.dilated + private val TimeoutDuration = 10.seconds.dilated + implicit val ec: ExecutionContext = system.dispatcher + + it should "catch failures in JABJEA initialization and fail the job accordingly" in { + val jobDescriptor = mock[BackendJobDescriptor] + val jesWorkflowInfo = mock[JesConfiguration] + val initializationData = mock[JesBackendInitializationData] + val serviceRegistryActor = system.actorOf(Props.empty) + val jesBackendSingletonActor = Option(system.actorOf(Props.empty)) + + val parent = TestProbe() + val deathwatch = TestProbe() + val testJJEA = TestActorRef[TestJesJobExecutionActor]( + props = Props(new TestJesJobExecutionActor(jobDescriptor, jesWorkflowInfo, initializationData, serviceRegistryActor, jesBackendSingletonActor, Props(new ConstructorFailingJABJEA))), + supervisor = parent.ref) + deathwatch watch testJJEA + + // Nothing happens: + parent.expectNoMsg(max = AwaitAlmostNothing) + deathwatch.expectNoMsg(max = AwaitAlmostNothing) + + testJJEA.tell(msg = ExecuteJobCommand, sender = parent.ref) + + parent.expectMsgPF(max = TimeoutDuration) { + case FailedNonRetryableResponse(jobKey, e, errorCode) => + e.getMessage should be("JesAsyncBackendJobExecutionActor failed and didn't catch its exception.") + } + } + + it should "catch failures at a random point during JABJEA processing and fail the job accordingly" in { + val jobDescriptor = mock[BackendJobDescriptor] + val jesWorkflowInfo = mock[JesConfiguration] + val initializationData = mock[JesBackendInitializationData] + val serviceRegistryActor = system.actorOf(Props.empty) + val jesBackendSingletonActor = Option(system.actorOf(Props.empty)) + + val parent = TestProbe() + val deathwatch = TestProbe() + val jabjeaConstructionPromise = Promise[ActorRef]() + val testJJEA = TestActorRef[TestJesJobExecutionActor]( + props = Props(new TestJesJobExecutionActor(jobDescriptor, jesWorkflowInfo, initializationData, serviceRegistryActor, jesBackendSingletonActor, Props(new ControllableFailingJabjea(jabjeaConstructionPromise)))), + supervisor = parent.ref) + deathwatch watch testJJEA + + // Nothing happens: + parent.expectNoMsg(max = AwaitAlmostNothing) + deathwatch.expectNoMsg(max = AwaitAlmostNothing) + + testJJEA.tell(msg = ExecuteJobCommand, sender = parent.ref) + + // Wait for the JABJEA to be spawned. Then kill it: + parent.expectNoMsg(max = AwaitAlmostNothing) + deathwatch.expectNoMsg(max = AwaitAlmostNothing) + jabjeaConstructionPromise.future foreach { _ ! JabjeaExplode } + + parent.expectMsgPF(max = TimeoutDuration) { + case FailedNonRetryableResponse(jobKey, e, errorCode) => + e.getMessage should be("JesAsyncBackendJobExecutionActor failed and didn't catch its exception.") + } + } +} + +class TestJesJobExecutionActor(jobDescriptor: BackendJobDescriptor, + jesWorkflowInfo: JesConfiguration, + initializationData: JesBackendInitializationData, + serviceRegistryActor: ActorRef, + jesBackendSingletonActor: Option[ActorRef], + fakeJabjeaProps: Props) extends JesJobExecutionActor(jobDescriptor, jesWorkflowInfo, initializationData, serviceRegistryActor, jesBackendSingletonActor) { + override def jabjeaProps: Props = fakeJabjeaProps +} + +class ConstructorFailingJABJEA extends ControllableFailingJabjea(Promise[ActorRef]()) { + // Explode immediately in the constructor: + explode() +} + +class ControllableFailingJabjea(constructionPromise: Promise[ActorRef]) extends Actor { + def explode() = { + val boom = 1 == 1 + if (boom) throw new RuntimeException("Test Exception! Don't panic if this appears during a test run!") + } + constructionPromise.trySuccess(self) + override def receive = { + case JabjeaExplode => explode() + } +} + +object ControllableFailingJabjea { + case object JabjeaExplode +} From df68deb1300d0a4f1cc82eb060d54e1db4c1bb85 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 18 Oct 2016 14:48:58 -0400 Subject: [PATCH 035/375] Added invalidate-bad-cache to call caching options --- CHANGELOG.md | 1 + README.md | 2 ++ core/src/main/resources/reference.conf | 7 ++++ .../core/callcaching/CallCachingMode.scala | 4 ++- .../MaterializeWorkflowDescriptorActor.scala | 8 +++-- .../execution/EngineJobExecutionActor.scala | 19 ++++++++--- .../callcaching/CallCacheInvalidateActor.scala | 1 + .../EjeaBackendIsCopyingCachedOutputsSpec.scala | 39 ++++++++++++++++++---- .../lifecycle/execution/ejea/PerTestHelper.scala | 4 +-- 9 files changed, 68 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8b44b140..e067d616b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 23 * Included the WDL matrix `transpose` function for `Array[Array[X]]` types. +* Added an option `call-caching.invalidate-bad-cache-results` (default: `true`). If true, Cromwell will invalidate cached results which have failed to copy as part of a cache hit. ## 0.22 diff --git a/README.md b/README.md index 1800278e9..026300bb8 100644 --- a/README.md +++ b/README.md @@ -1533,10 +1533,12 @@ To enable Call Caching, add the following to your Cromwell [configuration](#conf ``` call-caching { enabled = true + invalidate-bad-cache-results = true } ``` When `call-caching.enabled=true` (default: `false`), Cromwell will be able to to copy results from previously run jobs (when appropriate). +When `invalidate-bad-cache-results=true` (default: `true`), Cromwell will invalidate any cache results which fail to copy during a cache-hit. This is usually desired but might be unwanted if a cache might fail to copy for external reasons, such as a difference in user authentication. Cromwell also accepts two [workflow option](#workflow-options) related to call caching: diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 6305228d9..bae9b445e 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -97,7 +97,14 @@ workflow-options { // Optional call-caching configuration. call-caching { + # Allows re-use of existing results for jobs you've already run + # (default: false) enabled = false + + # Whether to invalidate a cache result forever if we cannot reuse them. Disable this if you expect some cache copies + # to fail for external reasons which should not invalidate the cache (e.g. auth differences between users): + # (default: true) + invalidate-bad-cache-results = true } google { diff --git a/core/src/main/scala/cromwell/core/callcaching/CallCachingMode.scala b/core/src/main/scala/cromwell/core/callcaching/CallCachingMode.scala index 94717ac1c..ed939d6aa 100644 --- a/core/src/main/scala/cromwell/core/callcaching/CallCachingMode.scala +++ b/core/src/main/scala/cromwell/core/callcaching/CallCachingMode.scala @@ -19,7 +19,7 @@ case object CallCachingOff extends CallCachingMode { override val withoutWrite = this } -case class CallCachingActivity(readWriteMode: ReadWriteMode) extends CallCachingMode { +case class CallCachingActivity(readWriteMode: ReadWriteMode, options: CallCachingOptions = CallCachingOptions(invalidateBadCacheResults = true)) extends CallCachingMode { override val readFromCache = readWriteMode.r override val writeToCache = readWriteMode.w override lazy val withoutRead: CallCachingMode = if (!writeToCache) CallCachingOff else this.copy(readWriteMode = WriteCache) @@ -34,3 +34,5 @@ sealed trait ReadWriteMode { case object ReadCache extends ReadWriteMode { override val w = false } case object WriteCache extends ReadWriteMode { override val r = false } case object ReadAndWriteCache extends ReadWriteMode + +final case class CallCachingOptions(invalidateBadCacheResults: Boolean = true) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala index 1a58b849a..2f8a1e57c 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala @@ -90,15 +90,17 @@ object MaterializeWorkflowDescriptorActor { } val enabled = conf.as[Option[Boolean]]("call-caching.enabled").getOrElse(false) + val invalidateBadCacheResults = conf.as[Option[Boolean]]("call-caching.invalidate-bad-cache-results").getOrElse(true) + val callCachingOptions = CallCachingOptions(invalidateBadCacheResults) if (enabled) { val readFromCache = readOptionalOption(ReadFromCache) val writeToCache = readOptionalOption(WriteToCache) (readFromCache |@| writeToCache) map { case (false, false) => CallCachingOff - case (true, false) => CallCachingActivity(ReadCache) - case (false, true) => CallCachingActivity(WriteCache) - case (true, true) => CallCachingActivity(ReadAndWriteCache) + case (true, false) => CallCachingActivity(ReadCache, callCachingOptions) + case (false, true) => CallCachingActivity(WriteCache, callCachingOptions) + case (true, true) => CallCachingActivity(ReadAndWriteCache, callCachingOptions) } } else { diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala index 25c1852dc..daae34ed2 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala @@ -165,10 +165,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, saveJobCompletionToJobStore(data.withSuccessResponse(response)) case Event(response: BackendJobExecutionResponse, data @ ResponsePendingData(_, _, _, Some(cacheHit))) => response match { - case f: BackendJobFailedResponse => - invalidateCacheHit(cacheHit.cacheResultIds.head) - log.error(f.throwable, "Failed copying cache results for job {}, invalidating cache entry.", jobDescriptorKey) - goto(InvalidatingCacheEntry) + case f: BackendJobFailedResponse => invalidateCacheHitAndTransition(cacheHit.cacheResultIds.head, data, f.throwable) case _ => runJob(data) } @@ -384,6 +381,20 @@ class EngineJobExecutionActor(replyTo: ActorRef, () } + private def invalidateCacheHitAndTransition(cacheId: CallCachingEntryId, data: ResponsePendingData, reason: Throwable) = { + val invalidationRequired = effectiveCallCachingMode match { + case CallCachingOff => throw new RuntimeException("Should not be calling invalidateCacheHit if call caching is off!") // Very unexpected. Fail out of this bad-state EJEA. + case activity: CallCachingActivity => activity.options.invalidateBadCacheResults + } + if (invalidationRequired) { + log.error(reason, "Failed copying cache results for job {}, invalidating cache entry.", jobDescriptorKey) + invalidateCacheHit(cacheId) + goto(InvalidatingCacheEntry) + } else { + handleCacheInvalidatedResponse(CallCacheInvalidationUnnecessary, data) + } + } + protected def invalidateCacheHit(cacheId: CallCachingEntryId): Unit = { val callCache = new CallCache(SingletonServicesStore.databaseInterface) context.actorOf(CallCacheInvalidateActor.props(callCache, cacheId), s"CallCacheInvalidateActor${cacheId.id}-$tag") diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala index ef09d32e9..a0ec75fc8 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala @@ -33,4 +33,5 @@ object CallCacheInvalidateActor { sealed trait CallCacheInvalidatedResponse case object CallCacheInvalidatedSuccess extends CallCacheInvalidatedResponse +case object CallCacheInvalidationUnnecessary extends CallCacheInvalidatedResponse case class CallCacheInvalidatedFailure(t: Throwable) extends CallCacheInvalidatedResponse \ No newline at end of file diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaBackendIsCopyingCachedOutputsSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaBackendIsCopyingCachedOutputsSpec.scala index 7aa4ccfda..b39711759 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaBackendIsCopyingCachedOutputsSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaBackendIsCopyingCachedOutputsSpec.scala @@ -3,9 +3,10 @@ package cromwell.engine.workflow.lifecycle.execution.ejea import cats.data.NonEmptyList import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ import EngineJobExecutionActorSpec._ -import cromwell.core.callcaching.CallCachingMode +import cromwell.core.callcaching._ import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CallCacheHashes, EJHAResponse, HashError} import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId + import scala.util.{Failure, Success, Try} import cromwell.engine.workflow.lifecycle.execution.ejea.HasJobSuccessResponse.SuccessfulCallCacheHashes @@ -97,20 +98,43 @@ class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec } } + if (mode.readFromCache) { s"invalidate a call for caching if backend coping failed when it was going to receive $hashComboName, if call caching is $mode" in { ejea = ejeaInBackendIsCopyingCachedOutputsState(initialHashData, mode) // Send the response from the copying actor ejea ! failureNonRetryableResponse expectInvalidateCallCacheActor(cacheId) - eventually { ejea.stateName should be(InvalidatingCacheEntry) } - ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper. bjeaProps, initialHashData, cacheHit)) + eventually { + ejea.stateName should be(InvalidatingCacheEntry) + } + ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, initialHashData, cacheHit)) + } + + s"not invalidate a call for caching if backend coping failed when invalidation is disabled, when it was going to receive $hashComboName, if call caching is $mode" in { + val invalidationDisabledOptions = CallCachingOptions(invalidateBadCacheResults = false) + val cacheInvalidationDisabledMode = mode match { + case CallCachingActivity(rw, options) => CallCachingActivity(rw, invalidationDisabledOptions) + case _ => fail(s"Mode $mode not appropriate for cache invalidation tests") + } + ejea = ejeaInBackendIsCopyingCachedOutputsState(initialHashData, cacheInvalidationDisabledMode) + // Send the response from the copying actor + ejea ! failureNonRetryableResponse + + eventually { + ejea.stateName should be(RunningJob) + } + // Make sure we didn't start invalidating anything: + helper.invalidateCacheActorCreations.hasExactlyOne should be(false) + ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, initialHashData, None)) } s"invalidate a call for caching if backend coping failed (preserving and received hashes) when call caching is $mode, the EJEA has $hashComboName and then gets a success result" in { ejea = ejeaInBackendIsCopyingCachedOutputsState(initialHashData, mode) // Send the response from the EJHA (if there was one!): - ejhaResponse foreach { ejea ! _ } + ejhaResponse foreach { + ejea ! _ + } // Nothing should happen here: helper.jobStoreProbe.expectNoMsg(awaitAlmostNothing) @@ -120,9 +144,12 @@ class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec ejea ! failureNonRetryableResponse expectInvalidateCallCacheActor(cacheId) - eventually { ejea.stateName should be(InvalidatingCacheEntry) } - ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper. bjeaProps, finalHashData, cacheHit)) + eventually { + ejea.stateName should be(InvalidatingCacheEntry) + } + ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, finalHashData, cacheHit)) } + } } } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala index 16274a275..dadcd35b4 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala @@ -150,8 +150,6 @@ private[ejea] class MockEjea(helper: PerTestHelper, override def makeFetchCachedResultsActor(cacheId: CallCachingEntryId, taskOutputs: Seq[TaskOutput]) = helper.fetchCachedResultsActorCreations = helper.fetchCachedResultsActorCreations.foundOne((cacheId, taskOutputs)) override def initializeJobHashing(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity) = helper.jobHashingInitializations = helper.jobHashingInitializations.foundOne((jobDescriptor, activity)) override def createSaveCacheResultsActor(hashes: CallCacheHashes, success: SucceededResponse) = helper.callCacheWriteActorCreations = helper.callCacheWriteActorCreations.foundOne((hashes, success)) - override def invalidateCacheHit(cacheId: CallCachingEntryId): Unit = { - helper.invalidateCacheActorCreations = helper.invalidateCacheActorCreations.foundOne(cacheId) - } + override def invalidateCacheHit(cacheId: CallCachingEntryId): Unit = { helper.invalidateCacheActorCreations = helper.invalidateCacheActorCreations.foundOne(cacheId) } override def createJobPreparationActor(jobPrepProps: Props, name: String) = jobPreparationProbe.ref } From 4ef57673e1e2bf114010b052cf54c5e1dd69abe3 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Wed, 19 Oct 2016 15:05:37 -0400 Subject: [PATCH 036/375] More documentation for SFS caching options --- README.md | 33 +++++++++++++++++++++++++++++++++ core/src/main/resources/reference.conf | 1 + 2 files changed, 34 insertions(+) diff --git a/README.md b/README.md index 1800278e9..046f6e104 100644 --- a/README.md +++ b/README.md @@ -1528,6 +1528,7 @@ Cromwell's call cache is maintained in its database. For best mileage with call > **Note:** If call caching is enabled, be careful not to change the contents of the output directory for any previously run job. Doing so might cause cache hits in Cromwell to copy over modified data and Cromwell currently does not check that the contents of the output directory changed. +## Configuring Call Caching To enable Call Caching, add the following to your Cromwell [configuration](#configuring-cromwell): ``` @@ -1538,6 +1539,7 @@ call-caching { When `call-caching.enabled=true` (default: `false`), Cromwell will be able to to copy results from previously run jobs (when appropriate). +## Call Caching Workflow Options Cromwell also accepts two [workflow option](#workflow-options) related to call caching: * If call caching is enabled, but one wishes to run a workflow but not add any of the calls into the call cache when they finish, the `write_to_cache` option can be set to `false`. This value defaults to `true`. @@ -1545,6 +1547,37 @@ Cromwell also accepts two [workflow option](#workflow-options) related to call c > **Note:** If call caching is disabled, the to workflow options `read_from_cache` and `write_to_cache` will be ignored and the options will be treated as though they were 'false'. +## Local Filesystem Options +When running a job on the Config (Shared Filesystem) backend, Cromwell provides some additional options in the backend's config section: + +``` + config { + ... + filesystems { + ... + local { + ... + caching { + # When copying a cached result, what type of file duplication should occur. Attempted in the order listed below: + duplication-strategy: [ + "hard-link", "soft-link", "copy" + ] + + # Possible values: file, path + # "file" will compute an md5 hash of the file content. + # "path" will compute an md5 hash of the file path. This strategy will only be effective if the duplication-strategy (above) is set to "soft-link", + # in order to allow for the original file path to be hashed. + hashing-strategy: "file" + + # When true, will check if a sibling file with the same name and the .md5 extension exists, and if it does, use the content of this file as a hash. + # If false or the md5 does not exist, will proceed with the above-defined hashing strategy. + check-sibling-md5: false + } + } + } + } +``` + # REST API The `server` subcommand on the executable JAR will start an HTTP server which can accept WDL files to run as well as check status and output of existing workflows. diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 6305228d9..4de694af9 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -163,6 +163,7 @@ backend { ] caching { + # When copying a cached result, what type of file duplication should occur. Attempted in the order listed below: duplication-strategy: [ "hard-link", "soft-link", "copy" ] From e3d136bdf3b2c34f7a01f1ddf170fc3c801cd454 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Wed, 19 Oct 2016 16:15:25 -0400 Subject: [PATCH 037/375] defaults --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 046f6e104..72bcadb91 100644 --- a/README.md +++ b/README.md @@ -1567,10 +1567,12 @@ When running a job on the Config (Shared Filesystem) backend, Cromwell provides # "file" will compute an md5 hash of the file content. # "path" will compute an md5 hash of the file path. This strategy will only be effective if the duplication-strategy (above) is set to "soft-link", # in order to allow for the original file path to be hashed. + # Default: file hashing-strategy: "file" # When true, will check if a sibling file with the same name and the .md5 extension exists, and if it does, use the content of this file as a hash. # If false or the md5 does not exist, will proceed with the above-defined hashing strategy. + # Default: false check-sibling-md5: false } } From 8c24348f81598c2c3f0d3b7179f872bc016d4691 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Wed, 19 Oct 2016 14:06:41 -0400 Subject: [PATCH 038/375] Added queued and starting jobs to the timing diagram --- CHANGELOG.md | 1 + .../main/scala/cromwell/core/ExecutionStatus.scala | 2 +- .../resources/workflowTimings/workflowTimings.html | 9 +++---- .../execution/EngineJobExecutionActor.scala | 2 ++ .../execution/WorkflowExecutionActor.scala | 28 ++++++++++++++++++---- .../execution/WorkflowExecutionActorData.scala | 2 +- 6 files changed, 34 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e067d616b..60e78f43a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * Included the WDL matrix `transpose` function for `Array[Array[X]]` types. * Added an option `call-caching.invalidate-bad-cache-results` (default: `true`). If true, Cromwell will invalidate cached results which have failed to copy as part of a cache hit. +* Timing diagrams and metadata now receive more fine grained workflow states between submission and Running. ## 0.22 diff --git a/core/src/main/scala/cromwell/core/ExecutionStatus.scala b/core/src/main/scala/cromwell/core/ExecutionStatus.scala index 76acb29a4..353b44d65 100644 --- a/core/src/main/scala/cromwell/core/ExecutionStatus.scala +++ b/core/src/main/scala/cromwell/core/ExecutionStatus.scala @@ -2,7 +2,7 @@ package cromwell.core object ExecutionStatus extends Enumeration { type ExecutionStatus = Value - val NotStarted, Starting, Running, Failed, Preempted, Done, Aborted = Value + val NotStarted, QueuedInCromwell, Starting, Running, Failed, Preempted, Done, Aborted = Value val TerminalStatuses = Set(Failed, Done, Aborted, Preempted) implicit class EnhancedExecutionStatus(val status: ExecutionStatus) extends AnyVal { diff --git a/engine/src/main/resources/workflowTimings/workflowTimings.html b/engine/src/main/resources/workflowTimings/workflowTimings.html index bd9de21d6..626f4cf14 100644 --- a/engine/src/main/resources/workflowTimings/workflowTimings.html +++ b/engine/src/main/resources/workflowTimings/workflowTimings.html @@ -107,14 +107,15 @@ if (callStart < firstEventStart) addDataTableRow(dataTable, thisCallName, "cromwell starting overhead", callStart, firstEventStart); if (callEnd > finalEventEnd) addDataTableRow(dataTable, thisCallName, "cromwell final overhead", finalEventEnd, callEnd); } - } else if (callList[callIndex].executionStatus == "Running") { + } else if (callList[callIndex].executionStatus == "Running" || callList[callIndex].executionStatus == "QueuedInCromwell" || callList[callIndex].executionStatus == "Starting") { + var status = callList[callIndex].executionStatus executionCallsCount++; var endDate = workflowEnd; if(endDate == null) { - addDataTableRow(dataTable, thisCallName, "Running", new Date(callList[callIndex].start), new Date(Date.now())); + addDataTableRow(dataTable, thisCallName, status, new Date(callList[callIndex].start), new Date(Date.now())); } else { - addDataTableRow(dataTable, thisCallName, "Still running when workflow ended", new Date(callList[callIndex].start), endDate); + addDataTableRow(dataTable, thisCallName, "Still ".concat(status).concat(" when workflow ended"), new Date(callList[callIndex].start), endDate); } } } @@ -135,9 +136,9 @@ }); } + -
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala index daae34ed2..15da08fe7 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala @@ -78,6 +78,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, when(RequestingExecutionToken) { case Event(JobExecutionTokenDispensed(jobExecutionToken), NoData) => executionToken = Option(jobExecutionToken) + replyTo ! JobStarting(jobDescriptorKey) if (restarting) { val jobStoreKey = jobDescriptorKey.toJobStoreKey(workflowId) jobStoreActor ! QueryJobCompletion(jobStoreKey, jobDescriptorKey.call.task.outputs) @@ -496,6 +497,7 @@ object EngineJobExecutionActor { case object Execute extends EngineJobExecutionActorCommand final case class JobRunning(jobDescriptor: BackendJobDescriptor, backendJobExecutionActor: Option[ActorRef]) + final case class JobStarting(jobKey: JobKey) def props(replyTo: ActorRef, jobDescriptorKey: BackendJobDescriptorKey, diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index b7358296b..c1ffaa527 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -18,7 +18,7 @@ import cromwell.core.WorkflowOptions.WorkflowFailureMode import cromwell.core._ import cromwell.core.logging.WorkflowLogging import cromwell.engine.backend.{BackendSingletonCollection, CromwellBackends} -import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.JobRunning +import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{JobRunning, JobStarting} import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor.BackendJobPreparationFailed import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.WorkflowExecutionActorState import cromwell.engine.workflow.lifecycle.{EngineLifecycleActorAbortCommand, EngineLifecycleActorAbortedResponse} @@ -318,7 +318,13 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, } when(WorkflowExecutionInProgressState) { + case Event(JobStarting(jobKey), stateData) => + // The EJEA is telling us that the job is now Starting. Update the metadata and our execution store. + val statusChange = MetadataEvent(metadataKey(jobKey, CallMetadataKeys.ExecutionStatus), MetadataValue(ExecutionStatus.Starting)) + serviceRegistryActor ! PutMetadataAction(statusChange) + stay() using stateData.mergeExecutionDiff(WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Starting))) case Event(JobRunning(jobDescriptor, backendJobExecutionActor), stateData) => + // The EJEA is telling us that the job is now Running. Update the metadata and our execution store. pushRunningJobMetadata(jobDescriptor) stay() using stateData .addBackendJobExecutionActor(jobDescriptor.key, backendJobExecutionActor) @@ -544,8 +550,14 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, } TryUtil.sequence(executionDiffs) match { - case Success(diffs) if diffs.exists(_.containsNewEntry) => startRunnableScopes(data.mergeExecutionDiffs(diffs)) - case Success(diffs) => data.mergeExecutionDiffs(diffs) + case Success(diffs) => + // Update the metadata for the jobs we just sent to EJEAs (they'll start off queued up waiting for tokens): + pushQueuedJobMetadata(diffs) + if (diffs.exists(_.containsNewEntry)) { + startRunnableScopes(data.mergeExecutionDiffs(diffs)) + } else { + data.mergeExecutionDiffs(diffs) + } case Failure(e) => data } } @@ -559,6 +571,14 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, serviceRegistryActor ! PutMetadataAction(startEvents) } + private def pushQueuedJobMetadata(diffs: Seq[WorkflowExecutionDiff]) = { + val startingEvents = for { + diff <- diffs + (jobKey, executionState) <- diff.executionStore if jobKey.isInstanceOf[BackendJobDescriptorKey] && executionState == ExecutionStatus.QueuedInCromwell + } yield MetadataEvent(metadataKey(jobKey, CallMetadataKeys.ExecutionStatus), MetadataValue(ExecutionStatus.QueuedInCromwell)) + serviceRegistryActor ! PutMetadataAction(startingEvents) + } + private def pushRunningJobMetadata(jobDescriptor: BackendJobDescriptor) = { val inputEvents = jobDescriptor.inputs match { case empty if empty.isEmpty => @@ -593,7 +613,7 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, val ejeaRef = context.actorOf(ejeaProps, ejeaName) pushNewJobMetadata(jobKey, backendName) ejeaRef ! EngineJobExecutionActor.Execute - Success(WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Starting))) + Success(WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.QueuedInCromwell))) case None => throw WorkflowExecutionException(NonEmptyList.of(new Exception(s"Could not get BackendLifecycleActor for backend $backendName"))) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala index 599c8f1b4..8e6f61c4f 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala @@ -49,7 +49,7 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript } // activeJobs is the subset of the executionStore that are either running or will run in the future. val activeJobs = executionStore.store.toList filter { - case (jobKey, jobStatus) => (jobStatus == NotStarted && upstreamFailed(jobKey.scope).isEmpty) || jobStatus == Starting || jobStatus == Running + case (jobKey, jobStatus) => (jobStatus == NotStarted && upstreamFailed(jobKey.scope).isEmpty) || jobStatus == QueuedInCromwell || jobStatus == Starting || jobStatus == Running } activeJobs match { From 3486aa194a033d4f8da30cfce575333d6382038d Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 20 Oct 2016 15:27:22 -0400 Subject: [PATCH 039/375] Robustifications and removals of the persistent WorkflowStore and JobStore in Single Workflow mode --- .../main/scala/cromwell/util/PromiseActor.scala | 38 +++-- .../workflow/SingleWorkflowRunnerActor.scala | 161 ++++++++++----------- .../engine/workflow/WorkflowManagerActor.scala | 17 +-- .../workflowstore/InMemoryWorkflowStore.scala | 0 .../cromwell/jobstore/EmptyJobStoreActor.scala | 15 ++ src/main/scala/cromwell/Main.scala | 4 +- 6 files changed, 126 insertions(+), 109 deletions(-) rename engine/src/{test => main}/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala (100%) create mode 100644 engine/src/main/scala/cromwell/jobstore/EmptyJobStoreActor.scala diff --git a/core/src/main/scala/cromwell/util/PromiseActor.scala b/core/src/main/scala/cromwell/util/PromiseActor.scala index 6813a53c7..58aea267a 100644 --- a/core/src/main/scala/cromwell/util/PromiseActor.scala +++ b/core/src/main/scala/cromwell/util/PromiseActor.scala @@ -4,6 +4,28 @@ import akka.actor._ import scala.concurrent.{Future, Promise} +private class PromiseActor(promise: Promise[Any], sendTo: ActorRef, msg: Any) extends Actor with ActorLogging { + + context.watch(sendTo) + sendTo ! msg + + override def receive = { + case Status.Failure(f) => + promise.tryFailure(f) + context.stop(self) + case Terminated(actorRef) => + if (actorRef == sendTo) { + promise.tryFailure(new RuntimeException("Promise-watched actor completed before sending back a message")) + } else { + log.error("Spooky happenstances! A Terminated({}) message was sent to a private Promise actor which wasn't watching it!?", actorRef) + } + context.stop(self) + case success => + promise.trySuccess(success) + context.stop(self) + } +} + object PromiseActor { /** * Sends a message to an actor and returns the future associated with the fullfilment of the reply @@ -16,12 +38,11 @@ object PromiseActor { */ private def askNoTimeout(message: Any, sendTo: ActorRef)(implicit actorRefFactory: ActorRefFactory): Future[Any] = { val promise = Promise[Any]() - val promiseActor = actorRefFactory.actorOf(props(promise)) - sendTo.tell(message, promiseActor) + val _ = actorRefFactory.actorOf(props(promise, sendTo, message)) promise.future } - def props(promise: Promise[Any]): Props = Props(new PromiseActor(promise)) + def props(promise: Promise[Any], sendTo: ActorRef, msg: Any): Props = Props(new PromiseActor(promise, sendTo, msg)) implicit class EnhancedActorRef(val actorRef: ActorRef) extends AnyVal { def askNoTimeout(message: Any)(implicit actorRefFactory: ActorRefFactory): Future[Any] = { @@ -29,14 +50,3 @@ object PromiseActor { } } } - -private class PromiseActor(promise: Promise[Any]) extends Actor { - override def receive = { - case Status.Failure(f) => - promise.tryFailure(f) - context.stop(self) - case success => - promise.trySuccess(success) - context.stop(self) - } -} diff --git a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala index 8abb0874c..125d8a255 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala @@ -12,8 +12,9 @@ import cromwell.core.retry.SimpleExponentialBackoff import cromwell.core.{ExecutionStore => _, _} import cromwell.engine.workflow.SingleWorkflowRunnerActor._ import cromwell.engine.workflow.WorkflowManagerActor.RetrieveNewWorkflows -import cromwell.engine.workflow.workflowstore.WorkflowStoreActor +import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreActor} import cromwell.engine.workflow.workflowstore.WorkflowStoreActor.SubmitWorkflow +import cromwell.jobstore.EmptyJobStoreActor import cromwell.server.CromwellRootActor import cromwell.services.metadata.MetadataService.{GetSingleWorkflowMetadataAction, GetStatus, WorkflowOutputs} import cromwell.webservice.PerRequest.RequestComplete @@ -26,46 +27,9 @@ import scala.concurrent.duration._ import scala.language.postfixOps import scala.util.{Failure, Try} -object SingleWorkflowRunnerActor { - def props(source: WorkflowSourceFiles, metadataOutputFile: Option[Path]): Props = { - Props(new SingleWorkflowRunnerActor(source, metadataOutputFile)) - } - - sealed trait RunnerMessage - // The message to actually run the workflow is made explicit so the non-actor Main can `ask` this actor to do the - // running and collect a result. - case object RunWorkflow extends RunnerMessage - private case object IssuePollRequest extends RunnerMessage - private case object IssueReply extends RunnerMessage - - sealed trait RunnerState - case object NotStarted extends RunnerState - case object RunningWorkflow extends RunnerState - case object RequestingOutputs extends RunnerState - case object RequestingMetadata extends RunnerState - case object Done extends RunnerState - - final case class RunnerData(replyTo: Option[ActorRef] = None, - terminalState: Option[WorkflowState] = None, - id: Option[WorkflowId] = None, - failures: Seq[Throwable] = Seq.empty) { - - def addFailure(message: String): RunnerData = addFailure(new RuntimeException(message)) - - def addFailure(e: Throwable): RunnerData = this.copy(failures = e +: failures) - } - - implicit class EnhancedJsObject(val jsObject: JsObject) extends AnyVal { - def state: WorkflowState = WorkflowState.fromString(jsObject.fields("status").asInstanceOf[JsString].value) - } - - private val Tag = "SingleWorkflowRunnerActor" -} - /** * Designed explicitly for the use case of the 'run' functionality in Main. This Actor will start a workflow, - * print out the outputs when complete and then shut down the actor system. Note that multiple aspects of this - * are sub-optimal for future use cases where one might want a single workflow being run. + * print out the outputs when complete and reply with a result. */ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: Option[Path]) extends CromwellRootActor with LoggingFSM[RunnerState, RunnerData] { @@ -73,32 +37,10 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: import SingleWorkflowRunnerActor._ private val backoff = SimpleExponentialBackoff(1 second, 1 minute, 1.2) - startWith(NotStarted, RunnerData()) - - private def requestMetadata: State = { - val metadataBuilder = context.actorOf(MetadataBuilderActor.props(serviceRegistryActor), s"MetadataRequest-Workflow-${stateData.id.get}") - metadataBuilder ! GetSingleWorkflowMetadataAction(stateData.id.get, None, None) - goto (RequestingMetadata) - } - - private def schedulePollRequest(): Unit = { - // -Ywarn-value-discard should stash Cancellable to cancel - context.system.scheduler.scheduleOnce(backoff.backoffMillis.millis, self, IssuePollRequest) - () - } + override lazy val workflowStore = new InMemoryWorkflowStore() + override lazy val jobStoreActor = context.actorOf(EmptyJobStoreActor.props) - private def requestStatus(): Unit = { - // This requests status via the metadata service rather than instituting an FSM watch on the underlying workflow actor. - // Cromwell's eventual consistency means it isn't safe to use an FSM transition to a terminal state as the signal for - // when outputs or metadata have stabilized. - val metadataBuilder = context.actorOf(MetadataBuilderActor.props(serviceRegistryActor), s"StatusRequest-Workflow-${stateData.id.get}-request-${UUID.randomUUID()}") - metadataBuilder ! GetStatus(stateData.id.get) - } - - private def issueReply: State = { - self ! IssueReply - goto (Done) - } + startWith(NotStarted, RunnerData()) when (NotStarted) { case Event(RunWorkflow, data) => @@ -131,41 +73,25 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: case Event(RequestComplete((StatusCodes.OK, jsObject: JsObject)), data) if jsObject.state == WorkflowFailed => val updatedData = data.copy(terminalState = Option(WorkflowFailed)).addFailure(s"Workflow ${data.id.get} transitioned to state Failed") // If there's an output path specified then request metadata, otherwise issue a reply to the original sender. - val nextState = if (metadataOutputPath.isDefined) requestMetadata else issueReply - nextState using updatedData + if (metadataOutputPath.isDefined) requestMetadata else issueReply(updatedData) } when (RequestingOutputs) { case Event(RequestComplete((StatusCodes.OK, outputs: JsObject)), _) => outputOutputs(outputs) - if (metadataOutputPath.isDefined) requestMetadata else issueReply + if (metadataOutputPath.isDefined) requestMetadata else issueReply(stateData) } when (RequestingMetadata) { case Event(RequestComplete((StatusCodes.OK, metadata: JsObject)), _) => outputMetadata(metadata) - issueReply - } - - when (Done) { - case Event(IssueReply, data) => - data.terminalState foreach { state => log.info(s"$Tag workflow finished with status '$state'.") } - data.failures foreach { e => log.error(e, e.getMessage) } - - val message: Any = data.terminalState collect { case WorkflowSucceeded => () } getOrElse Status.Failure(data.failures.head) - data.replyTo foreach { _ ! message } - stay() + issueReply(stateData) } onTransition { case NotStarted -> RunningWorkflow => schedulePollRequest() } - private def failAndFinish(e: Throwable): State = { - log.error(e, s"$Tag received Failure message: ${e.getMessage}") - issueReply using stateData.addFailure(e) - } - whenUnhandled { // Handle failures for all failure responses generically. case Event(r: WorkflowStoreActor.WorkflowAbortFailed, data) => failAndFinish(r.reason) @@ -180,6 +106,41 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: stay() } + private def requestMetadata: State = { + val metadataBuilder = context.actorOf(MetadataBuilderActor.props(serviceRegistryActor), s"MetadataRequest-Workflow-${stateData.id.get}") + metadataBuilder ! GetSingleWorkflowMetadataAction(stateData.id.get, None, None) + goto (RequestingMetadata) + } + + private def schedulePollRequest(): Unit = { + // -Ywarn-value-discard should stash Cancellable to cancel + context.system.scheduler.scheduleOnce(backoff.backoffMillis.millis, self, IssuePollRequest) + () + } + + private def requestStatus(): Unit = { + // This requests status via the metadata service rather than instituting an FSM watch on the underlying workflow actor. + // Cromwell's eventual consistency means it isn't safe to use an FSM transition to a terminal state as the signal for + // when outputs or metadata have stabilized. + val metadataBuilder = context.actorOf(MetadataBuilderActor.props(serviceRegistryActor), s"StatusRequest-Workflow-${stateData.id.get}-request-${UUID.randomUUID()}") + metadataBuilder ! GetStatus(stateData.id.get) + } + + private def issueReply(data: RunnerData): State = { + data.terminalState foreach { state => log.info(s"$Tag workflow finished with status '$state'.") } + data.failures foreach { e => log.error(e, e.getMessage) } + + val message: Any = data.terminalState collect { case WorkflowSucceeded => () } getOrElse Status.Failure(data.failures.head) + data.replyTo foreach { _ ! message } + context.stop(self) + stay + } + + private def failAndFinish(e: Throwable): State = { + log.error(e, s"$Tag received Failure message: ${e.getMessage}") + issueReply(stateData.addFailure(e)) + } + /** * Outputs the outputs to stdout, and then requests the metadata. */ @@ -199,3 +160,37 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: } void } } + +object SingleWorkflowRunnerActor { + def props(source: WorkflowSourceFiles, metadataOutputFile: Option[Path]): Props = { + Props(new SingleWorkflowRunnerActor(source, metadataOutputFile)) + } + + sealed trait RunnerMessage + // The message to actually run the workflow is made explicit so the non-actor Main can `ask` this actor to do the + // running and collect a result. + case object RunWorkflow extends RunnerMessage + private case object IssuePollRequest extends RunnerMessage + + sealed trait RunnerState + case object NotStarted extends RunnerState + case object RunningWorkflow extends RunnerState + case object RequestingOutputs extends RunnerState + case object RequestingMetadata extends RunnerState + + final case class RunnerData(replyTo: Option[ActorRef] = None, + terminalState: Option[WorkflowState] = None, + id: Option[WorkflowId] = None, + failures: Seq[Throwable] = Seq.empty) { + + def addFailure(message: String): RunnerData = addFailure(new RuntimeException(message)) + + def addFailure(e: Throwable): RunnerData = this.copy(failures = e +: failures) + } + + implicit class EnhancedJsObject(val jsObject: JsObject) extends AnyVal { + def state: WorkflowState = WorkflowState.fromString(jsObject.fields("status").asInstanceOf[JsString].value) + } + + private val Tag = "SingleWorkflowRunnerActor" +} diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala index 08bade654..cb773e7e1 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala @@ -1,6 +1,5 @@ package cromwell.engine.workflow - import akka.actor.FSM.{CurrentState, SubscribeTransitionCallBack, Transition} import akka.actor._ import akka.event.Logging @@ -16,8 +15,8 @@ import cromwell.jobstore.JobStoreActor.{JobStoreWriteFailure, JobStoreWriteSucce import cromwell.services.metadata.MetadataService._ import cromwell.webservice.EngineStatsActor import net.ceedubs.ficus.Ficus._ + import scala.concurrent.duration._ -import scala.concurrent.{Await, Promise} object WorkflowManagerActor { val DefaultMaxWorkflowsToRun = 5000 @@ -104,8 +103,6 @@ class WorkflowManagerActor(config: Config, private val logger = Logging(context.system, this) private val tag = self.path.name - private val donePromise = Promise[Unit]() - private var abortingWorkflowToReplyTo = Map.empty[WorkflowId, ActorRef] override def preStart(): Unit = { @@ -115,15 +112,18 @@ class WorkflowManagerActor(config: Config, } private def addShutdownHook() = { - // Only abort jobs on SIGINT if the config explicitly sets system.abortJobsOnTerminate = true. + // Only abort jobs on SIGINT if the config explicitly sets system.abort-jobs-on-terminate = true. val abortJobsOnTerminate = config.getConfig("system").as[Option[Boolean]]("abort-jobs-on-terminate").getOrElse(false) if (abortJobsOnTerminate) { sys.addShutdownHook { - logger.info(s"$tag: Received shutdown signal. Aborting all running workflows...") + logger.info(s"$tag: Received shutdown signal.") self ! AbortAllWorkflowsCommand - Await.result(donePromise.future, Duration.Inf) + while (stateData != null && stateData.workflows.nonEmpty) { + log.info(s"Waiting for ${stateData.workflows.size} workflows to abort...") + Thread.sleep(1000) + } } } } @@ -242,8 +242,7 @@ class WorkflowManagerActor(config: Config, onTransition { case _ -> Done => - logger.info(s"$tag All workflows finished. Stopping self.") - donePromise.trySuccess(()) + logger.info(s"$tag All workflows finished") () case fromState -> toState => logger.debug(s"$tag transitioning from $fromState to $toState") diff --git a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala similarity index 100% rename from engine/src/test/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala rename to engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala diff --git a/engine/src/main/scala/cromwell/jobstore/EmptyJobStoreActor.scala b/engine/src/main/scala/cromwell/jobstore/EmptyJobStoreActor.scala new file mode 100644 index 000000000..a2eb585a3 --- /dev/null +++ b/engine/src/main/scala/cromwell/jobstore/EmptyJobStoreActor.scala @@ -0,0 +1,15 @@ +package cromwell.jobstore + +import akka.actor.{Actor, Props} +import cromwell.jobstore.JobStoreActor._ + +class EmptyJobStoreActor extends Actor { + override def receive: Receive = { + case w: JobStoreWriterCommand => sender ! JobStoreWriteSuccess(w) + case _: QueryJobCompletion => sender ! JobNotComplete + } +} + +object EmptyJobStoreActor { + def props: Props = Props(new EmptyJobStoreActor()) +} diff --git a/src/main/scala/cromwell/Main.scala b/src/main/scala/cromwell/Main.scala index 74a5405c1..45bd9e838 100644 --- a/src/main/scala/cromwell/Main.scala +++ b/src/main/scala/cromwell/Main.scala @@ -74,11 +74,9 @@ object Main extends App { import PromiseActor.EnhancedActorRef - val promise = runner.askNoTimeout(RunWorkflow) - waitAndExit(promise, CromwellSystem) + waitAndExit(runner.askNoTimeout(RunWorkflow), CromwellSystem) } - private def waitAndExit(futureResult: Future[Any], workflowManagerSystem: CromwellSystem): Unit = { Await.ready(futureResult, Duration.Inf) From 881f16cff2161be33ad82d3d2e9c627f1418c34e Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Fri, 21 Oct 2016 17:28:38 -0400 Subject: [PATCH 040/375] Robust METADATA_VALUE embiggening. Closes #1607. --- .../migration/src/main/resources/changelog.xml | 1 + .../changesets/embiggen_metadata_value.xml | 50 ++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 database/migration/src/main/resources/changesets/embiggen_metadata_value.xml diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml index 251e15d79..162b9cbcf 100644 --- a/database/migration/src/main/resources/changelog.xml +++ b/database/migration/src/main/resources/changelog.xml @@ -45,6 +45,7 @@ + diff --git a/database/migration/src/main/resources/changesets/embiggen_metadata_value.xml b/database/migration/src/main/resources/changesets/embiggen_metadata_value.xml new file mode 100644 index 000000000..e0b1ed38d --- /dev/null +++ b/database/migration/src/main/resources/changesets/embiggen_metadata_value.xml @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + Either METADATA_JOURNAL or METADATA_ENTRY must exist, but not both (Liquibase doesn't have an xor). + + + + + + + + + The METADATA_ENTRY version of the embiggener. This should gracefully handle the absence of the table + in the event the table is still called METADATA_JOURNAL when this runs. + + + + + + + + + + The METADATA_JOURNAL version of the embiggener. This should gracefully handle the absence of the table + in the event the table has been renamed to METADATA_ENTRY when this runs. + + + + + From 79f6e124866c9d65a2b9727d02a9c299cd255afa Mon Sep 17 00:00:00 2001 From: Ruchi Date: Mon, 24 Oct 2016 11:36:10 -0400 Subject: [PATCH 041/375] Reduce hashing closes #1603 (#1613) * MC's improvements to upstreamFailed query --- .../workflow/lifecycle/execution/WorkflowExecutionActorData.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala index 8e6f61c4f..ded1bcfcc 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala @@ -45,7 +45,7 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript // `List`ify the `prerequisiteScopes` to avoid expensive hashing of `Scope`s when assembling the result. def upstream(scope: Scope): List[Scope] = scope.prerequisiteScopes.toList ++ scope.prerequisiteScopes.toList.flatMap(upstream) def upstreamFailed(scope: Scope) = upstream(scope) filter { s => - executionStore.store.map({ case (a, b) => a.scope -> b }).get(s).contains(Failed) + executionStore.store.exists({ case (key, status) => status == Failed && key.scope == s }) } // activeJobs is the subset of the executionStore that are either running or will run in the future. val activeJobs = executionStore.store.toList filter { From 55ffcb4bafdabc0ccad4626afa2000a8b35f8dc7 Mon Sep 17 00:00:00 2001 From: Thib Date: Tue, 25 Oct 2016 15:24:25 -0400 Subject: [PATCH 042/375] Gcs Nio (#1519) GCS NIO refactoring --- .../backend/BackendJobExecutionActor.scala | 4 +- .../scala/cromwell/backend/OutputEvaluator.scala | 4 +- .../cromwell/backend/async/ExecutionHandle.scala | 4 +- .../cromwell/backend/async/ExecutionResult.scala | 4 +- .../backend/callcaching/CacheHitDuplicating.scala | 26 +- .../scala/cromwell/backend/io/WorkflowPaths.scala | 7 +- .../WorkflowPathsBackendInitializationData.scala | 7 +- .../scala/cromwell/backend/wdl/FileSystems.scala | 30 --- .../cromwell/backend/wdl/ReadLikeFunctions.scala | 7 +- .../cromwell/backend/wdl/WriteFunctions.scala | 12 +- .../wfs/DefaultWorkflowFileSystemProvider.scala | 9 - .../backend/wfs/DefaultWorkflowPathBuilder.scala | 8 + .../backend/wfs/WorkflowFileSystemProvider.scala | 34 --- .../cromwell/backend/wfs/WorkflowPathBuilder.scala | 25 ++ build.sbt | 1 + .../src/main/scala/cromwell/core/PathFactory.scala | 135 ---------- .../cromwell/core/path/CustomRetryParams.scala | 25 ++ .../cromwell/core/path/DefaultPathBuilder.scala | 21 ++ .../core/path/DefaultPathBuilderFactory.scala | 8 + .../cromwell/core/path/JavaWriterImplicits.scala | 13 + .../scala/cromwell/core/path/PathBuilder.scala | 10 + .../cromwell/core/path/PathBuilderFactory.scala | 11 + .../cromwell/core/{ => path}/PathCopier.scala | 23 +- .../scala/cromwell/core/path/PathFactory.scala | 57 ++++ .../scala/cromwell/core/path/PathImplicits.scala | 15 ++ .../cromwell/core/path/PathParsingException.scala | 5 + .../main/scala/cromwell/core/path/PathWriter.scala | 76 ++++++ .../cromwell/core/path/proxy/FileSystemProxy.scala | 25 ++ .../scala/cromwell/core/path/proxy/PathProxy.scala | 44 +++ .../proxy/RetryableFileSystemProviderProxy.scala | 57 ++++ .../main/scala/cromwell/core/retry/Backoff.scala | 2 +- .../src/main/scala/cromwell/core/retry/Retry.scala | 4 +- core/src/main/scala/cromwell/util/TryUtil.scala | 45 ++++ .../core/path/RetryableFileSystemProxySpec.scala | 278 +++++++++++++++++++ .../scala/cromwell/engine/EngineFilesystems.scala | 47 +++- .../cromwell/engine/EngineWorkflowDescriptor.scala | 5 +- .../main/scala/cromwell/engine/WdlFunctions.scala | 8 +- .../engine/backend/EnhancedWorkflowOptions.scala | 16 -- .../workflow/SingleWorkflowRunnerActor.scala | 8 +- .../cromwell/engine/workflow/WorkflowActor.scala | 5 +- .../engine/workflow/WorkflowManagerActor.scala | 9 +- .../workflow/lifecycle/CopyWorkflowLogsActor.scala | 3 +- .../lifecycle/CopyWorkflowOutputsActor.scala | 7 +- .../MaterializeWorkflowDescriptorActor.scala | 22 +- .../execution/EngineJobExecutionActor.scala | 2 +- .../execution/WorkflowExecutionActorData.scala | 2 +- .../execution/callcaching/CallCache.scala | 6 +- .../cromwell/engine/EngineFunctionsSpec.scala | 5 +- .../workflow/SingleWorkflowRunnerActorSpec.scala | 6 +- .../MaterializeWorkflowDescriptorActorSpec.scala | 2 +- .../lifecycle/execution/ejea/PerTestHelper.scala | 10 +- .../filesystems/gcs/ContentTypeOption.scala | 15 -- .../filesystems/gcs/GcsFileAttributes.scala | 23 -- .../cromwell/filesystems/gcs/GcsFileSystem.scala | 68 ----- .../filesystems/gcs/GcsFileSystemProvider.scala | 295 --------------------- .../cromwell/filesystems/gcs/GcsPathBuilder.scala | 100 +++++++ .../filesystems/gcs/GcsPathBuilderFactory.scala | 48 ++++ .../cromwell/filesystems/gcs/GoogleAuthMode.scala | 186 ------------- .../filesystems/gcs/GoogleConfiguration.scala | 14 +- .../cromwell/filesystems/gcs/NioGcsPath.scala | 191 ------------- .../filesystems/gcs/auth/GoogleAuthMode.scala | 187 +++++++++++++ .../gcs/auth/RefreshableOAuth2Credentials.scala | 31 +++ .../scala/cromwell/filesystems/gcs/package.scala | 6 - .../filesystems/gcs/GcsIntegrationTest.scala | 5 - .../filesystems/gcs/GcsPathBuilderSpec.scala | 31 +++ .../filesystems/gcs/GoogleConfigurationSpec.scala | 16 +- .../gcs/GoogleCredentialFactorySpec.scala | 158 ----------- .../filesystems/gcs/MockGcsFileSystemBuilder.scala | 9 - .../cromwell/filesystems/gcs/NioGcsPathSpec.scala | 291 -------------------- .../filesystems/gcs/RefreshTokenModeSpec.scala | 26 -- project/Dependencies.scala | 16 +- project/Merging.scala | 5 + src/bin/travis/testCentaurJes.sh | 2 +- .../scala/cromwell/CromwellCommandLineSpec.scala | 2 +- .../impl/htcondor/HtCondorBackendFactory.scala | 2 +- .../impl/htcondor/HtCondorJobExecutionActor.scala | 11 +- .../backend/impl/htcondor/HtCondorWrapper.scala | 5 +- .../htcondor/HtCondorJobExecutionActorSpec.scala | 1 + .../backend/impl/jes/GenomicsFactory.scala | 24 +- .../jes/JesAsyncBackendJobExecutionActor.scala | 114 +++----- .../cromwell/backend/impl/jes/JesAttributes.scala | 17 +- .../impl/jes/JesBackendLifecycleActorFactory.scala | 2 +- .../backend/impl/jes/JesCacheHitCopyingActor.scala | 2 +- .../cromwell/backend/impl/jes/JesCallPaths.scala | 14 +- .../backend/impl/jes/JesConfiguration.scala | 26 +- .../backend/impl/jes/JesExpressionFunctions.scala | 36 +-- .../backend/impl/jes/JesFinalizationActor.scala | 3 +- .../cromwell/backend/impl/jes/JesImplicits.scala | 41 --- .../backend/impl/jes/JesInitializationActor.scala | 27 +- .../impl/jes/JesJobCachingActorHelper.scala | 5 +- .../backend/impl/jes/JesWorkflowPaths.scala | 48 ++-- .../backend/impl/jes/authentication/JesAuths.scala | 5 + .../impl/jes/authentication/JesCredentials.scala | 5 - .../jes/authentication/JesVMAuthentication.scala | 2 +- .../jes/callcaching/JesBackendFileHashing.scala | 4 +- .../cromwell/backend/impl/jes/io/package.scala | 9 +- .../jes/JesAsyncBackendJobExecutionActorSpec.scala | 46 ++-- .../backend/impl/jes/JesCallPathsSpec.scala | 25 +- .../backend/impl/jes/JesConfigurationSpec.scala | 19 +- .../impl/jes/JesInitializationActorSpec.scala | 5 +- .../backend/impl/jes/JesWorkflowPathsSpec.scala | 16 +- .../cromwell/backend/impl/jes/MockObjects.scala | 9 - .../impl/sfs/config/ConfigBackendFileHashing.scala | 12 +- .../impl/sfs/config/ConfigHashingStrategy.scala | 26 +- .../sfs/GcsWorkflowFileSystemProvider.scala | 36 --- .../cromwell/backend/sfs/SharedFileSystem.scala | 18 +- .../SharedFileSystemAsyncJobExecutionActor.scala | 23 +- ...redFileSystemBackendLifecycleActorFactory.scala | 24 +- .../sfs/SharedFileSystemCacheHitCopyingActor.scala | 7 +- .../sfs/SharedFileSystemExpressionFunctions.scala | 17 +- .../sfs/SharedFileSystemInitializationActor.scala | 14 +- .../SharedFileSystemJobCachingActorHelper.scala | 3 +- .../sfs/config/ConfigHashingStrategySpec.scala | 10 + .../SharedFileSystemInitializationActorSpec.scala | 2 +- .../backend/sfs/SharedFileSystemSpec.scala | 12 +- .../backend/impl/spark/SparkBackendFactory.scala | 2 +- .../impl/spark/SparkJobExecutionActor.scala | 12 +- .../cromwell/backend/impl/spark/SparkProcess.scala | 6 +- .../impl/spark/SparkJobExecutionActorSpec.scala | 3 +- 119 files changed, 1665 insertions(+), 2009 deletions(-) delete mode 100644 backend/src/main/scala/cromwell/backend/wdl/FileSystems.scala delete mode 100644 backend/src/main/scala/cromwell/backend/wfs/DefaultWorkflowFileSystemProvider.scala create mode 100644 backend/src/main/scala/cromwell/backend/wfs/DefaultWorkflowPathBuilder.scala delete mode 100644 backend/src/main/scala/cromwell/backend/wfs/WorkflowFileSystemProvider.scala create mode 100644 backend/src/main/scala/cromwell/backend/wfs/WorkflowPathBuilder.scala delete mode 100644 core/src/main/scala/cromwell/core/PathFactory.scala create mode 100644 core/src/main/scala/cromwell/core/path/CustomRetryParams.scala create mode 100644 core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala create mode 100644 core/src/main/scala/cromwell/core/path/DefaultPathBuilderFactory.scala create mode 100644 core/src/main/scala/cromwell/core/path/JavaWriterImplicits.scala create mode 100644 core/src/main/scala/cromwell/core/path/PathBuilder.scala create mode 100644 core/src/main/scala/cromwell/core/path/PathBuilderFactory.scala rename core/src/main/scala/cromwell/core/{ => path}/PathCopier.scala (65%) create mode 100644 core/src/main/scala/cromwell/core/path/PathFactory.scala create mode 100644 core/src/main/scala/cromwell/core/path/PathImplicits.scala create mode 100644 core/src/main/scala/cromwell/core/path/PathParsingException.scala create mode 100644 core/src/main/scala/cromwell/core/path/PathWriter.scala create mode 100644 core/src/main/scala/cromwell/core/path/proxy/FileSystemProxy.scala create mode 100644 core/src/main/scala/cromwell/core/path/proxy/PathProxy.scala create mode 100644 core/src/main/scala/cromwell/core/path/proxy/RetryableFileSystemProviderProxy.scala create mode 100644 core/src/main/scala/cromwell/util/TryUtil.scala create mode 100644 core/src/test/scala/cromwell/core/path/RetryableFileSystemProxySpec.scala delete mode 100644 engine/src/main/scala/cromwell/engine/backend/EnhancedWorkflowOptions.scala delete mode 100644 filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/ContentTypeOption.scala delete mode 100644 filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileAttributes.scala delete mode 100644 filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystem.scala delete mode 100644 filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala create mode 100644 filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala create mode 100644 filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilderFactory.scala delete mode 100644 filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleAuthMode.scala delete mode 100644 filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/NioGcsPath.scala create mode 100644 filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleAuthMode.scala create mode 100644 filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/RefreshableOAuth2Credentials.scala delete mode 100644 filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/package.scala delete mode 100644 filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsIntegrationTest.scala create mode 100644 filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala delete mode 100644 filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleCredentialFactorySpec.scala delete mode 100644 filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/MockGcsFileSystemBuilder.scala delete mode 100644 filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/NioGcsPathSpec.scala delete mode 100644 filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/RefreshTokenModeSpec.scala delete mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesImplicits.scala create mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/authentication/JesAuths.scala delete mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/authentication/JesCredentials.scala delete mode 100644 supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/MockObjects.scala delete mode 100644 supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala diff --git a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala index 60cdc1e02..93f1605fe 100644 --- a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala @@ -1,5 +1,7 @@ package cromwell.backend +import java.nio.file.Path + import akka.actor.ActorLogging import akka.event.LoggingReceive import cromwell.backend.BackendJobExecutionActor._ @@ -22,7 +24,7 @@ object BackendJobExecutionActor { sealed trait BackendJobExecutionActorResponse extends BackendWorkflowLifecycleActorResponse sealed trait BackendJobExecutionResponse extends BackendJobExecutionActorResponse { def jobKey: BackendJobDescriptorKey } - case class SucceededResponse(jobKey: BackendJobDescriptorKey, returnCode: Option[Int], jobOutputs: JobOutputs, jobDetritusFiles: Option[Map[String, String]], executionEvents: Seq[ExecutionEvent]) extends BackendJobExecutionResponse + case class SucceededResponse(jobKey: BackendJobDescriptorKey, returnCode: Option[Int], jobOutputs: JobOutputs, jobDetritusFiles: Option[Map[String, Path]], executionEvents: Seq[ExecutionEvent]) extends BackendJobExecutionResponse case class AbortedResponse(jobKey: BackendJobDescriptorKey) extends BackendJobExecutionResponse sealed trait BackendJobFailedResponse extends BackendJobExecutionResponse { def throwable: Throwable; def returnCode: Option[Int] } case class FailedNonRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse diff --git a/backend/src/main/scala/cromwell/backend/OutputEvaluator.scala b/backend/src/main/scala/cromwell/backend/OutputEvaluator.scala index 0c6365265..ec7cf0c38 100644 --- a/backend/src/main/scala/cromwell/backend/OutputEvaluator.scala +++ b/backend/src/main/scala/cromwell/backend/OutputEvaluator.scala @@ -1,6 +1,6 @@ package cromwell.backend -import cromwell.core.JobOutput +import cromwell.core.{JobOutputs, JobOutput} import wdl4s._ import wdl4s.expression.WdlStandardLibraryFunctions import wdl4s.util.TryUtil @@ -11,7 +11,7 @@ import scala.util.{Success, Try} object OutputEvaluator { def evaluateOutputs(jobDescriptor: BackendJobDescriptor, wdlFunctions: WdlStandardLibraryFunctions, - postMapper: WdlValue => Try[WdlValue] = v => Success(v)) = { + postMapper: WdlValue => Try[WdlValue] = v => Success(v)): Try[JobOutputs] = { val inputs = jobDescriptor.inputs val evaluatedOutputs = jobDescriptor.call.task.outputs. foldLeft(Map.empty[LocallyQualifiedName, Try[JobOutput]])((outputMap, output) => { diff --git a/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala b/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala index 88232f3b2..7c626b153 100644 --- a/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala +++ b/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala @@ -1,5 +1,7 @@ package cromwell.backend.async +import java.nio.file.Path + import cromwell.backend.BackendJobDescriptor import cromwell.core.{ExecutionEvent, JobOutputs} @@ -12,7 +14,7 @@ trait ExecutionHandle { def result: ExecutionResult } -final case class SuccessfulExecutionHandle(outputs: JobOutputs, returnCode: Int, jobDetritusFiles: Map[String, String], executionEvents: Seq[ExecutionEvent], resultsClonedFrom: Option[BackendJobDescriptor] = None) extends ExecutionHandle { +final case class SuccessfulExecutionHandle(outputs: JobOutputs, returnCode: Int, jobDetritusFiles: Map[String, Path], executionEvents: Seq[ExecutionEvent], resultsClonedFrom: Option[BackendJobDescriptor] = None) extends ExecutionHandle { override val isDone = true override val result = SuccessfulExecution(outputs, returnCode, jobDetritusFiles, executionEvents, resultsClonedFrom) } diff --git a/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala b/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala index 267bea877..fb4614129 100644 --- a/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala +++ b/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala @@ -1,5 +1,7 @@ package cromwell.backend.async +import java.nio.file.Path + import cromwell.backend.BackendJobDescriptor import cromwell.core.{ExecutionEvent, JobOutputs} @@ -13,7 +15,7 @@ sealed trait ExecutionResult */ final case class SuccessfulExecution(outputs: JobOutputs, returnCode: Int, - jobDetritusFiles: Map[String, String], + jobDetritusFiles: Map[String, Path], executionEvents: Seq[ExecutionEvent], resultsClonedFrom: Option[BackendJobDescriptor] = None) extends ExecutionResult diff --git a/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala b/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala index c199de6d4..140881094 100644 --- a/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala +++ b/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala @@ -6,10 +6,13 @@ import akka.actor.ActorRef import cromwell.backend.BackendCacheHitCopyingActor import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, SucceededResponse} import cromwell.backend.io.JobPaths -import cromwell.core.PathCopier +import cromwell.core.path.PathCopier import cromwell.core.simpleton.{WdlValueBuilder, WdlValueSimpleton} import wdl4s.values.WdlFile +import scala.language.postfixOps +import scala.util.Try + /** * Mixin implementing common functionality for a BackendCacheHitCopyingActor. * @@ -35,7 +38,7 @@ trait CacheHitDuplicating { * @param file the string version of the path * @return an absolute path to the file with potential credentials embedded within. */ - protected def getPath(file: String): Path + protected def getPath(file: String): Try[Path] protected def destinationCallRootPath: Path @@ -47,9 +50,10 @@ trait CacheHitDuplicating { protected def metadataKeyValues: Map[String, Any] private def lookupSourceCallRootPath(sourceJobDetritusFiles: Map[String, String]): Path = { - sourceJobDetritusFiles.get(JobPaths.CallRootPathKey).map(getPath).getOrElse(throw new RuntimeException( - s"${JobPaths.CallRootPathKey} wasn't found for call ${jobDescriptor.call.fullyQualifiedName}") - ) + sourceJobDetritusFiles.get(JobPaths.CallRootPathKey).map(getPath).get recover { + case failure => + throw new RuntimeException(s"${JobPaths.CallRootPathKey} wasn't found for call ${jobDescriptor.call.fullyQualifiedName}", failure) + } get } /** @@ -59,27 +63,27 @@ trait CacheHitDuplicating { sourceCallRootPath: Path): Seq[WdlValueSimpleton] = { wdlValueSimpletons map { case WdlValueSimpleton(key, wdlFile: WdlFile) => - val sourcePath = getPath(wdlFile.value) + val sourcePath = getPath(wdlFile.value).get val destinationPath = PathCopier.getDestinationFilePath(sourceCallRootPath, sourcePath, destinationCallRootPath) duplicate(sourcePath, destinationPath) - WdlValueSimpleton(key, WdlFile(destinationPath.toString)) + WdlValueSimpleton(key, WdlFile(destinationPath.toUri.toString)) case wdlValueSimpleton => wdlValueSimpleton } } - private def copyDetritus(sourceJobDetritusFiles: Map[String, String]): Map[String, String] = { + private def copyDetritus(sourceJobDetritusFiles: Map[String, String]): Map[String, Path] = { val sourceKeys = sourceJobDetritusFiles.keySet val destinationKeys = destinationJobDetritusPaths.keySet val fileKeys = sourceKeys.intersect(destinationKeys).filterNot(_ == JobPaths.CallRootPathKey) val destinationJobDetritusFiles = fileKeys map { fileKey => - val sourcePath = getPath(sourceJobDetritusFiles(fileKey)) + val sourcePath = getPath(sourceJobDetritusFiles(fileKey)).get val destinationPath = destinationJobDetritusPaths(fileKey) duplicate(sourcePath, destinationPath) - (fileKey, destinationPath.toString) + (fileKey, destinationPath) } - destinationJobDetritusFiles.toMap + (JobPaths.CallRootPathKey -> destinationCallRootPath.toString) + destinationJobDetritusFiles.toMap + (JobPaths.CallRootPathKey -> destinationCallRootPath) } override def copyCachedOutputs(wdlValueSimpletons: Seq[WdlValueSimpleton], diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala index 23bdae992..c61ea9108 100644 --- a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala +++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala @@ -1,17 +1,18 @@ package cromwell.backend.io -import java.nio.file.{FileSystem, FileSystems, Path, Paths} +import java.nio.file.{Path, Paths} import com.typesafe.config.Config import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} -import cromwell.core.PathFactory import net.ceedubs.ficus.Ficus._ +import cromwell.core.path.{DefaultPathBuilder, PathBuilder, PathFactory} object WorkflowPaths{ val DockerRoot = Paths.get("/root") + val DefaultPathBuilders = List(DefaultPathBuilder) } -class WorkflowPaths(workflowDescriptor: BackendWorkflowDescriptor, config: Config, val fileSystems: List[FileSystem] = List(FileSystems.getDefault)) extends PathFactory { +class WorkflowPaths(workflowDescriptor: BackendWorkflowDescriptor, config: Config, val pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends PathFactory { val executionRoot = Paths.get(config.as[Option[String]]("root").getOrElse("cromwell-executions")).toAbsolutePath private def workflowPathBuilder(root: Path) = { diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsBackendInitializationData.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsBackendInitializationData.scala index 6ac8ba960..b0861d6bb 100644 --- a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsBackendInitializationData.scala +++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsBackendInitializationData.scala @@ -1,8 +1,7 @@ package cromwell.backend.io -import java.nio.file.FileSystem - import cromwell.backend.BackendInitializationData +import cromwell.core.path.PathBuilder /** * Extension of backend initialization data that also provides a `WorkflowPaths`, and by proxy its `List[FileSystem]`. @@ -39,7 +38,7 @@ object WorkflowPathsBackendInitializationData { BackendInitializationData.as[WorkflowPathsBackendInitializationData](initializationDataOption).workflowPaths } - def fileSystems(initializationDataOption: Option[BackendInitializationData]): List[FileSystem] = { - workflowPaths(initializationDataOption).fileSystems + def pathBuilders(initializationDataOption: Option[BackendInitializationData]): List[PathBuilder] = { + workflowPaths(initializationDataOption).pathBuilders } } diff --git a/backend/src/main/scala/cromwell/backend/wdl/FileSystems.scala b/backend/src/main/scala/cromwell/backend/wdl/FileSystems.scala deleted file mode 100644 index 8919b0058..000000000 --- a/backend/src/main/scala/cromwell/backend/wdl/FileSystems.scala +++ /dev/null @@ -1,30 +0,0 @@ -package cromwell.backend.wdl - -import java.nio.file.{FileSystem, Path} - -import cromwell.core.PathFactory - -trait FileSystems extends PathFactory { - - /** - * Ordered list of filesystems to be used to execute wdl functions needing IO. - */ - def fileSystems: List[FileSystem] - - /** - * Function applied after a string is successfully resolved to a java.nio.Path - */ - def postMapping(path: Path): Path = path - - /** - * Function applied before a string is attempted to be resolved to a java.nio.Path - */ - def preMapping(string: String): String = string - - /** - * Use fileSystems in order to try to create a java.nio.Path from path that will be used to perform IO. - * If no filesystem is able to construct a Path from the String, an exception will be raised. - */ - protected final def toPath(path: String) = postMapping(buildPath(preMapping(path), fileSystems)) - -} diff --git a/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala b/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala index 1f06e10d8..0f3e4679e 100644 --- a/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala +++ b/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala @@ -1,6 +1,7 @@ package cromwell.backend.wdl import cromwell.backend.MemorySize +import cromwell.core.path.PathFactory import wdl4s.expression.WdlStandardLibraryFunctions import wdl4s.parser.MemoryUnit import wdl4s.types.{WdlArrayType, WdlFileType, WdlObjectType, WdlStringType} @@ -8,7 +9,7 @@ import wdl4s.values._ import scala.util.{Failure, Success, Try} -trait ReadLikeFunctions extends FileSystems { this: WdlStandardLibraryFunctions => +trait ReadLikeFunctions extends PathFactory { this: WdlStandardLibraryFunctions => import better.files._ /** @@ -27,7 +28,7 @@ trait ReadLikeFunctions extends FileSystems { this: WdlStandardLibraryFunctions wdlObjects <- WdlObject.fromTsv(contents) } yield wdlObjects - override def readFile(path: String): String = File(toPath(path)).contentAsString + override def readFile(path: String): String = File(buildPath(path)).contentAsString /** * Read all lines from the file referenced by the first parameter and return an Array[String] @@ -93,7 +94,7 @@ trait ReadLikeFunctions extends FileSystems { this: WdlStandardLibraryFunctions for { value <- wdlValue unit <- convertTo - } yield MemorySize(File(toPath(value.valueString)).size.toDouble, MemoryUnit.Bytes).to(unit).amount + } yield MemorySize(File(buildPath(value.valueString)).size.toDouble, MemoryUnit.Bytes).to(unit).amount } params match { diff --git a/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala b/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala index cb8eea297..0f602c76e 100644 --- a/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala +++ b/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala @@ -16,21 +16,17 @@ trait WriteFunctions { this: WdlStandardLibraryFunctions => */ def writeDirectory: Path - private lazy val absoluteDirectory = { - File(writeDirectory).createDirectories().path - } - - override def tempFilePath = absoluteDirectory.toString + private lazy val _writeDirectory = File(writeDirectory).createDirectories() def writeTempFile(path: String,prefix: String,suffix: String,content: String): String = throw new NotImplementedError("This method is not used anywhere and should be removed") private def writeContent(baseName: String, content: String): Try[WdlFile] = { - val fullPath = File(absoluteDirectory)./(s"$baseName${content.md5Sum}.tmp") + val tmpFile = _writeDirectory / s"$baseName-${content.md5Sum}.tmp" Try { - if (!fullPath.exists) fullPath.write(content) + if (tmpFile.notExists) tmpFile.write(content) } map { _ => - WdlFile(fullPath.pathAsString) + WdlFile(tmpFile.uri.toString) } } diff --git a/backend/src/main/scala/cromwell/backend/wfs/DefaultWorkflowFileSystemProvider.scala b/backend/src/main/scala/cromwell/backend/wfs/DefaultWorkflowFileSystemProvider.scala deleted file mode 100644 index a10b46a0d..000000000 --- a/backend/src/main/scala/cromwell/backend/wfs/DefaultWorkflowFileSystemProvider.scala +++ /dev/null @@ -1,9 +0,0 @@ -package cromwell.backend.wfs - -import java.nio.file.FileSystems - -object DefaultWorkflowFileSystemProvider extends WorkflowFileSystemProvider { - override def fileSystemOption(params: WorkflowFileSystemProviderParams) = { - Option(FileSystems.getDefault) - } -} diff --git a/backend/src/main/scala/cromwell/backend/wfs/DefaultWorkflowPathBuilder.scala b/backend/src/main/scala/cromwell/backend/wfs/DefaultWorkflowPathBuilder.scala new file mode 100644 index 000000000..43af11d87 --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/wfs/DefaultWorkflowPathBuilder.scala @@ -0,0 +1,8 @@ +package cromwell.backend.wfs + +import cromwell.core.path.DefaultPathBuilder + + +object DefaultWorkflowPathBuilder extends WorkflowPathBuilder { + override def pathBuilderOption(params: WorkflowFileSystemProviderParams) = Option(DefaultPathBuilder) +} diff --git a/backend/src/main/scala/cromwell/backend/wfs/WorkflowFileSystemProvider.scala b/backend/src/main/scala/cromwell/backend/wfs/WorkflowFileSystemProvider.scala deleted file mode 100644 index de8272473..000000000 --- a/backend/src/main/scala/cromwell/backend/wfs/WorkflowFileSystemProvider.scala +++ /dev/null @@ -1,34 +0,0 @@ -package cromwell.backend.wfs - -import java.nio.file.FileSystem - -import com.typesafe.config.{Config, ConfigFactory} -import cromwell.backend.io.WorkflowPaths -import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor} -import cromwell.core.WorkflowOptions -import net.ceedubs.ficus.Ficus._ - -import scala.concurrent.ExecutionContext - -object WorkflowFileSystemProvider { - def workflowPaths(configurationDescriptor: BackendConfigurationDescriptor, - workflowDescriptor: BackendWorkflowDescriptor, - providers: Traversable[WorkflowFileSystemProvider], - fileSystemExecutionContext: ExecutionContext): WorkflowPaths = { - val backendConfig = configurationDescriptor.backendConfig - val fileSystemConfig = backendConfig.as[Option[Config]]("filesystems").getOrElse(ConfigFactory.empty()) - val globalConfig = configurationDescriptor.globalConfig - val params = WorkflowFileSystemProviderParams(fileSystemConfig, globalConfig, workflowDescriptor.workflowOptions, - fileSystemExecutionContext) - val fileSystems = providers.flatMap(_.fileSystemOption(params)).toList - new WorkflowPaths(workflowDescriptor, configurationDescriptor.backendConfig, fileSystems) - } -} - -final case class WorkflowFileSystemProviderParams(fileSystemConfig: Config, globalConfig: Config, - workflowOptions: WorkflowOptions, - fileSystemExecutionContext: ExecutionContext) - -trait WorkflowFileSystemProvider { - def fileSystemOption(params: WorkflowFileSystemProviderParams): Option[FileSystem] -} diff --git a/backend/src/main/scala/cromwell/backend/wfs/WorkflowPathBuilder.scala b/backend/src/main/scala/cromwell/backend/wfs/WorkflowPathBuilder.scala new file mode 100644 index 000000000..a593ed6fe --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/wfs/WorkflowPathBuilder.scala @@ -0,0 +1,25 @@ +package cromwell.backend.wfs + +import com.typesafe.config.Config +import cromwell.backend.io.WorkflowPaths +import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor} +import cromwell.core.WorkflowOptions +import cromwell.core.path.PathBuilder + +import scala.concurrent.ExecutionContext + +object WorkflowPathBuilder { + def workflowPaths(configurationDescriptor: BackendConfigurationDescriptor, + workflowDescriptor: BackendWorkflowDescriptor, + pathBuilders: List[PathBuilder]): WorkflowPaths = { + new WorkflowPaths(workflowDescriptor, configurationDescriptor.backendConfig, pathBuilders) + } +} + +final case class WorkflowFileSystemProviderParams(fileSystemConfig: Config, globalConfig: Config, + workflowOptions: WorkflowOptions, + fileSystemExecutionContext: ExecutionContext) + +trait WorkflowPathBuilder { + def pathBuilderOption(params: WorkflowFileSystemProviderParams): Option[PathBuilder] +} diff --git a/build.sbt b/build.sbt index 7d764ef58..86089b464 100644 --- a/build.sbt +++ b/build.sbt @@ -9,6 +9,7 @@ lazy val gcsFileSystem = (project in file("filesystems/gcs")) .settings(gcsFileSystemSettings:_*) .withTestSettings .dependsOn(core) + .dependsOn(core % "test->test") lazy val databaseSql = (project in file("database/sql")) .settings(databaseSqlSettings:_*) diff --git a/core/src/main/scala/cromwell/core/PathFactory.scala b/core/src/main/scala/cromwell/core/PathFactory.scala deleted file mode 100644 index f85a05c95..000000000 --- a/core/src/main/scala/cromwell/core/PathFactory.scala +++ /dev/null @@ -1,135 +0,0 @@ -package cromwell.core - -import java.io.Writer -import java.nio.file.{FileSystem, Path} - -import better.files.File - -import scala.collection.immutable.Queue -import scala.util.{Success, Failure, Try} - -class FileSystemNotFound(str: String) extends CromwellFatalException( - new IllegalArgumentException(s"Could not find suitable filesystem to parse $str") -) - -trait PathFactory { - private val schemeMatcher = """([a-z]+://).*""".r - - def findFileSystem(rawString: String, fss: List[FileSystem], mapping: PartialFunction[FileSystem, Try[Path]]) = { - fss.toStream collect mapping collectFirst { case Success(p) => p } getOrElse { - throw new FileSystemNotFound(rawString) - } - } - - def buildPath(rawString: String, fileSystems: List[FileSystem]): Path = { - findFileSystem(rawString, fileSystems, { - case fs: FileSystem => - if (hasWrongScheme(rawString, fs)) { - Failure(new IllegalArgumentException(s"$rawString scheme doesn't match ${fs.provider.getScheme}")) - } else { - Try(fs.getPath(rawString)) - } - }) - } - - def buildFile(rawString: String, fileSystems: List[FileSystem]): File = File(buildPath(rawString, fileSystems)) - - private def hasWrongScheme(rawString: String, fileSystem: FileSystem): Boolean = { - schemeMatcher.findFirstMatchIn(rawString) match { - case Some(m) => m.group(1) != fileSystem.provider().getScheme - case _ => false - } - } -} - -object PathFactory { - def swapExt(filePath: String, oldExt: String, newExt: String): String = { - filePath.stripSuffix(oldExt) + newExt - } - - implicit class EnhancedPath(val path: Path) extends AnyVal { - def swapExt(oldExt: String, newExt: String): Path = { - path.getFileSystem.getPath(s"${path.toString.stripSuffix(oldExt)}$newExt") - } - - def untailed = UntailedWriter(path) - - def tailed(tailedSize: Int) = TailedWriter(path, tailedSize) - } - - implicit class FlushingAndClosingWriter(writer: Writer) { - /** Convenience method to flush and close in one shot. */ - def flushAndClose() = { - writer.flush() - writer.close() - } - } -} - -/** - * Used with a `ProcessLogger`, writes lines with a newline. - */ -trait PathWriter { - import better.files._ - - val path: Path - lazy val writer: Writer = File(path).newBufferedWriter - - /** - * Passed to `ProcessLogger` to add a new line. - * - * @param string Line to add to the logs. - */ - def writeWithNewline(string: String): Unit = { - writer.write(string) - writer.write("\n") - } -} - -/** - * Used with a `ProcessLogger`, writes lines with a newline. - * - * @param path Path to the log file. - */ -case class UntailedWriter(path: Path) extends PathWriter - -/** - * Used with a `ProcessLogger`, queues up the `tailedSize` number of lines. - * - * @param path Path to the log file. - * @param tailedSize Maximum number of lines to save in the internal FIFO queue. - */ -case class TailedWriter(path: Path, tailedSize: Int) extends PathWriter { - private var isTailed = false - private var tailedLines: Queue[String] = Queue.empty - - /** - * Passed to `ProcessLogger` to add a new line, and adds the line to the tailed queue. - * - * @param string Line to add to the logs. - */ - override def writeWithNewline(string: String): Unit = { - tailedLines :+= string - while (tailedLines.size > tailedSize) { - tailedLines = tailedLines.takeRight(tailedSize) - isTailed = true - } - super.writeWithNewline(string) - } - - /** - * Returns a descriptive tail of the `path` and the last `tailedLines` written. - * - * @return a descriptive tail of the `path` and the last `tailedLines` written. - */ - def tailString: String = { - if (tailedLines.isEmpty) { - s"Contents of $path were empty." - } else if (isTailed) { - s"Last ${tailedLines.size} of $path:\n${tailedLines.mkString("\n")}" - } else { - s"Contents of $path:\n${tailedLines.mkString("\n")}" - } - } -} - diff --git a/core/src/main/scala/cromwell/core/path/CustomRetryParams.scala b/core/src/main/scala/cromwell/core/path/CustomRetryParams.scala new file mode 100644 index 000000000..b27b9135e --- /dev/null +++ b/core/src/main/scala/cromwell/core/path/CustomRetryParams.scala @@ -0,0 +1,25 @@ +package cromwell.core.path + +import cromwell.core.retry.{Backoff, SimpleExponentialBackoff} + +import scala.concurrent.duration.Duration +import scala.concurrent.duration._ +import scala.language.postfixOps + +object CustomRetryParams { + val Default = CustomRetryParams( + timeout = Duration.Inf, + maxRetries = Option(3), + backoff = SimpleExponentialBackoff(1 seconds, 3 seconds, 1.5D), + isTransient = throwableToFalse, + isFatal = throwableToFalse + ) + + def throwableToFalse(t: Throwable) = false +} + +case class CustomRetryParams(timeout: Duration, + maxRetries: Option[Int], + backoff: Backoff, + isTransient: Throwable => Boolean, + isFatal: Throwable => Boolean) diff --git a/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala b/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala new file mode 100644 index 000000000..7dc60c1e8 --- /dev/null +++ b/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala @@ -0,0 +1,21 @@ +package cromwell.core.path + +import java.net.URI +import java.nio.file.{FileSystems, Path} + +import scala.util.Try + +/** + * PathBuilder using the default FileSystem to attempt to build a Path. + */ +case object DefaultPathBuilder extends PathBuilder { + override def name = "Default" + + override def build(pathAsString: String): Try[Path] = Try { + val uri = URI.create(pathAsString) + val host = Option(uri.getHost) getOrElse "" + val path = Option(uri.getPath) getOrElse "" + + FileSystems.getDefault.getPath(host, path) + } +} diff --git a/core/src/main/scala/cromwell/core/path/DefaultPathBuilderFactory.scala b/core/src/main/scala/cromwell/core/path/DefaultPathBuilderFactory.scala new file mode 100644 index 000000000..5339fae3c --- /dev/null +++ b/core/src/main/scala/cromwell/core/path/DefaultPathBuilderFactory.scala @@ -0,0 +1,8 @@ +package cromwell.core.path + +import akka.actor.ActorSystem +import cromwell.core.WorkflowOptions + +case object DefaultPathBuilderFactory extends PathBuilderFactory { + override def withOptions(options: WorkflowOptions)(implicit actorSystem: ActorSystem) = DefaultPathBuilder +} diff --git a/core/src/main/scala/cromwell/core/path/JavaWriterImplicits.scala b/core/src/main/scala/cromwell/core/path/JavaWriterImplicits.scala new file mode 100644 index 000000000..cc1b7f40d --- /dev/null +++ b/core/src/main/scala/cromwell/core/path/JavaWriterImplicits.scala @@ -0,0 +1,13 @@ +package cromwell.core.path + +import java.io.Writer + +object JavaWriterImplicits { + implicit class FlushingAndClosingWriter(writer: Writer) { + /** Convenience method to flush and close in one shot. */ + def flushAndClose() = { + writer.flush() + writer.close() + } + } +} diff --git a/core/src/main/scala/cromwell/core/path/PathBuilder.scala b/core/src/main/scala/cromwell/core/path/PathBuilder.scala new file mode 100644 index 000000000..c21310192 --- /dev/null +++ b/core/src/main/scala/cromwell/core/path/PathBuilder.scala @@ -0,0 +1,10 @@ +package cromwell.core.path + +import java.nio.file.Path + +import scala.util.Try + +trait PathBuilder { + def name: String + def build(pathAsString: String): Try[Path] +} diff --git a/core/src/main/scala/cromwell/core/path/PathBuilderFactory.scala b/core/src/main/scala/cromwell/core/path/PathBuilderFactory.scala new file mode 100644 index 000000000..7ee20eb2d --- /dev/null +++ b/core/src/main/scala/cromwell/core/path/PathBuilderFactory.scala @@ -0,0 +1,11 @@ +package cromwell.core.path + +import akka.actor.ActorSystem +import cromwell.core.WorkflowOptions + +/** + * Provide a method that can instantiate a path builder with the specified workflow options. + */ +trait PathBuilderFactory { + def withOptions(options: WorkflowOptions)(implicit actorSystem: ActorSystem): PathBuilder +} diff --git a/core/src/main/scala/cromwell/core/PathCopier.scala b/core/src/main/scala/cromwell/core/path/PathCopier.scala similarity index 65% rename from core/src/main/scala/cromwell/core/PathCopier.scala rename to core/src/main/scala/cromwell/core/path/PathCopier.scala index f90dad604..58eb29068 100644 --- a/core/src/main/scala/cromwell/core/PathCopier.scala +++ b/core/src/main/scala/cromwell/core/path/PathCopier.scala @@ -1,4 +1,4 @@ -package cromwell.core +package cromwell.core.path import java.io.IOException import java.nio.file.Path @@ -8,8 +8,27 @@ import better.files._ import scala.util.{Failure, Try} object PathCopier { + + /* + * Remove p1 from p2 as long as they match. + */ + private def truncateCommonRoot(p1: Path, p2: Path): String = { + def names(p: Path) = 0 until p.getNameCount map p.getName + + val names1 = names(p1) + + val truncated = names(p2).zipWithIndex.dropWhile { + case (n1, n2) => n2 < names1.size && n1.equals(names1(n2)) + } map { _._1 } + + truncated match { + case empty if empty.isEmpty => "" + case truncs => truncs.reduceLeft(_.resolve(_)).toString + } + } + def getDestinationFilePath(sourceContextPath: Path, sourceFilePath: Path, destinationDirPath: Path): Path = { - val relativeFileString = sourceContextPath.toAbsolutePath.relativize(sourceFilePath.toAbsolutePath).toString + val relativeFileString = truncateCommonRoot(sourceContextPath.toAbsolutePath, sourceFilePath.toAbsolutePath) destinationDirPath.resolve(relativeFileString) } diff --git a/core/src/main/scala/cromwell/core/path/PathFactory.scala b/core/src/main/scala/cromwell/core/path/PathFactory.scala new file mode 100644 index 000000000..ff050b559 --- /dev/null +++ b/core/src/main/scala/cromwell/core/path/PathFactory.scala @@ -0,0 +1,57 @@ +package cromwell.core.path + +import java.nio.file.Path + +import better.files.File + +import scala.util.Success + +/** + * Convenience trait delegating to the PathFactory singleton + */ +trait PathFactory { + /** + * Path builders to be applied (in order) to attempt to build a java.nio.Path from a string. + */ + def pathBuilders: List[PathBuilder] + + /** + * Function applied after a string is successfully resolved to a java.nio.Path + */ + def postMapping(path: Path): Path = path + + /** + * Function applied before a string is attempted to be resolved to a java.nio.Path + */ + def preMapping(string: String): String = string + + /** + * Attempts to build a java.nio.Path from a String + */ + def buildPath(string: String): Path = PathFactory.buildPath(string, pathBuilders, preMapping, postMapping) + + /** + * Attempts to build a better.files.File from a String + */ + def buildFile(string: String): File = PathFactory.buildFile(string, pathBuilders, preMapping, postMapping) +} + +object PathFactory { + /** + * Attempts to build a java.nio.Path from a String + */ + def buildPath(string: String, + pathBuilders: List[PathBuilder], + preMapping: String => String = identity[String], + postMapping: Path => Path = identity[Path]): Path = { + pathBuilders.toStream map { _.build(preMapping(string)) } collectFirst { case Success(p) => postMapping(p) } getOrElse { + val pathBuilderNames: String = pathBuilders map { _.name } mkString ", " + throw new PathParsingException(s"Could not find suitable filesystem among $pathBuilderNames to parse $string.") + } + } + + def buildFile(string: String, + pathBuilders: List[PathBuilder], + preMapping: String => String = identity[String], + postMapping: Path => Path = identity[Path]): File = File(buildPath(string, pathBuilders, preMapping, postMapping)) +} diff --git a/core/src/main/scala/cromwell/core/path/PathImplicits.scala b/core/src/main/scala/cromwell/core/path/PathImplicits.scala new file mode 100644 index 000000000..83d4bff7c --- /dev/null +++ b/core/src/main/scala/cromwell/core/path/PathImplicits.scala @@ -0,0 +1,15 @@ +package cromwell.core.path + +import java.nio.file.Path + +object PathImplicits { + implicit class EnhancedPath(val path: Path) extends AnyVal { + def swapExt(oldExt: String, newExt: String): Path = { + path.getFileSystem.getPath(s"${path.toString.stripSuffix(oldExt)}$newExt") + } + + def untailed = UntailedWriter(path) + + def tailed(tailedSize: Int) = TailedWriter(path, tailedSize) + } +} diff --git a/core/src/main/scala/cromwell/core/path/PathParsingException.scala b/core/src/main/scala/cromwell/core/path/PathParsingException.scala new file mode 100644 index 000000000..9bf6b5a7c --- /dev/null +++ b/core/src/main/scala/cromwell/core/path/PathParsingException.scala @@ -0,0 +1,5 @@ +package cromwell.core.path + +import cromwell.core.CromwellFatalException + +case class PathParsingException(message: String) extends CromwellFatalException(new IllegalArgumentException(message)) diff --git a/core/src/main/scala/cromwell/core/path/PathWriter.scala b/core/src/main/scala/cromwell/core/path/PathWriter.scala new file mode 100644 index 000000000..e24fb74fe --- /dev/null +++ b/core/src/main/scala/cromwell/core/path/PathWriter.scala @@ -0,0 +1,76 @@ +package cromwell.core.path + +import java.io.Writer +import java.nio.file.Path + +import scala.collection.immutable.Queue + + + +/** + * Used with a `ProcessLogger`, writes lines with a newline. + */ +trait PathWriter { + import better.files._ + + val path: Path + lazy val writer: Writer = File(path).newBufferedWriter + + /** + * Passed to `ProcessLogger` to add a new line. + * + * @param string Line to add to the logs. + */ + def writeWithNewline(string: String): Unit = { + writer.write(string) + writer.write("\n") + } +} + +/** + * Used with a `ProcessLogger`, writes lines with a newline. + * + * @param path Path to the log file. + */ +case class UntailedWriter(path: Path) extends PathWriter + +/** + * Used with a `ProcessLogger`, queues up the `tailedSize` number of lines. + * + * @param path Path to the log file. + * @param tailedSize Maximum number of lines to save in the internal FIFO queue. + */ +case class TailedWriter(path: Path, tailedSize: Int) extends PathWriter { + private var isTailed = false + private var tailedLines: Queue[String] = Queue.empty + + /** + * Passed to `ProcessLogger` to add a new line, and adds the line to the tailed queue. + * + * @param string Line to add to the logs. + */ + override def writeWithNewline(string: String): Unit = { + tailedLines :+= string + while (tailedLines.size > tailedSize) { + tailedLines = tailedLines.takeRight(tailedSize) + isTailed = true + } + super.writeWithNewline(string) + } + + /** + * Returns a descriptive tail of the `path` and the last `tailedLines` written. + * + * @return a descriptive tail of the `path` and the last `tailedLines` written. + */ + def tailString: String = { + if (tailedLines.isEmpty) { + s"Contents of $path were empty." + } else if (isTailed) { + s"Last ${tailedLines.size} of $path:\n${tailedLines.mkString("\n")}" + } else { + s"Contents of $path:\n${tailedLines.mkString("\n")}" + } + } +} + diff --git a/core/src/main/scala/cromwell/core/path/proxy/FileSystemProxy.scala b/core/src/main/scala/cromwell/core/path/proxy/FileSystemProxy.scala new file mode 100644 index 000000000..f9e9b5817 --- /dev/null +++ b/core/src/main/scala/cromwell/core/path/proxy/FileSystemProxy.scala @@ -0,0 +1,25 @@ +package cromwell.core.path.proxy + +import java.lang.Iterable +import java.nio.file._ +import java.nio.file.attribute.UserPrincipalLookupService +import java.nio.file.spi.FileSystemProvider +import java.util + +class FileSystemProxy(delegate: FileSystem, injectedProvider: FileSystemProvider) extends FileSystem { + + override def provider(): FileSystemProvider = injectedProvider + + /* delegated */ + override def supportedFileAttributeViews(): util.Set[String] = delegate.supportedFileAttributeViews() + override def getSeparator: String = delegate.getSeparator + override def getRootDirectories: Iterable[Path] = delegate.getRootDirectories + override def newWatchService(): WatchService = delegate.newWatchService() + override def getFileStores: Iterable[FileStore] = delegate.getFileStores + override def isReadOnly: Boolean = delegate.isReadOnly + override def getPath(first: String, more: String*): Path = new PathProxy(delegate.getPath(first, more: _*), this) + override def isOpen: Boolean = delegate.isOpen + override def close(): Unit = delegate.close() + override def getPathMatcher(syntaxAndPattern: String): PathMatcher = delegate.getPathMatcher(syntaxAndPattern) + override def getUserPrincipalLookupService: UserPrincipalLookupService = delegate.getUserPrincipalLookupService +} diff --git a/core/src/main/scala/cromwell/core/path/proxy/PathProxy.scala b/core/src/main/scala/cromwell/core/path/proxy/PathProxy.scala new file mode 100644 index 000000000..28428e0a3 --- /dev/null +++ b/core/src/main/scala/cromwell/core/path/proxy/PathProxy.scala @@ -0,0 +1,44 @@ +package cromwell.core.path.proxy + +import java.io.File +import java.net.URI +import java.nio.file.WatchEvent.{Kind, Modifier} +import java.nio.file._ +import java.util + +import scala.util.Try + +class PathProxy(delegate: Path, injectedFileSystem: FileSystem) extends Path { + def unbox[T](clazz: Class[T]): Try[T] = Try { + clazz.cast(delegate) + } + + override def getFileSystem: FileSystem = injectedFileSystem + + /* delegated */ + override def subpath(beginIndex: Int, endIndex: Int): Path = delegate.subpath(beginIndex, endIndex) + override def toFile: File = delegate.toFile + override def resolveSibling(other: Path): Path = delegate.resolveSibling(other) + override def resolveSibling(other: String): Path = delegate.resolveSibling(other) + override def isAbsolute: Boolean = delegate.isAbsolute + override def getName(index: Int): Path = delegate.getName(index) + override def getParent: Path = delegate.getParent + override def toAbsolutePath: Path = delegate.toAbsolutePath + override def relativize(other: Path): Path = delegate.relativize(other) + override def getNameCount: Int = delegate.getNameCount + override def toUri: URI = delegate.toUri + override def compareTo(other: Path): Int = delegate.compareTo(other) + override def register(watcher: WatchService, events: Array[Kind[_]], modifiers: Modifier*): WatchKey = delegate.register(watcher, events, modifiers: _*) + override def register(watcher: WatchService, events: Kind[_]*): WatchKey = delegate.register(watcher, events: _*) + override def getFileName: Path = delegate.getFileName + override def getRoot: Path = delegate.getRoot + override def iterator(): util.Iterator[Path] = delegate.iterator() + override def normalize(): Path = delegate.normalize() + override def endsWith(other: Path): Boolean = delegate.endsWith(other) + override def endsWith(other: String): Boolean = delegate.endsWith(other) + override def resolve(other: Path): Path = delegate.resolve(other) + override def resolve(other: String): Path = delegate.resolve(other) + override def startsWith(other: Path): Boolean = delegate.startsWith(other) + override def startsWith(other: String): Boolean = delegate.startsWith(other) + override def toRealPath(options: LinkOption*): Path = delegate.toRealPath(options: _*) +} diff --git a/core/src/main/scala/cromwell/core/path/proxy/RetryableFileSystemProviderProxy.scala b/core/src/main/scala/cromwell/core/path/proxy/RetryableFileSystemProviderProxy.scala new file mode 100644 index 000000000..db3975292 --- /dev/null +++ b/core/src/main/scala/cromwell/core/path/proxy/RetryableFileSystemProviderProxy.scala @@ -0,0 +1,57 @@ +package cromwell.core.path.proxy + +import java.net.URI +import java.nio.channels.SeekableByteChannel +import java.nio.file.DirectoryStream.Filter +import java.nio.file._ +import java.nio.file.attribute.{BasicFileAttributes, FileAttribute, FileAttributeView} +import java.nio.file.spi.FileSystemProvider +import java.util + +import akka.actor.ActorSystem +import cromwell.core.path.CustomRetryParams +import cromwell.core.retry.Retry + +import scala.concurrent.{Await, Future} + +class RetryableFileSystemProviderProxy[T <: FileSystemProvider](delegate: T, retryParams: CustomRetryParams = CustomRetryParams.Default)(implicit actorSystem: ActorSystem) extends FileSystemProvider { + private val iOExecutionContext = actorSystem.dispatchers.lookup("akka.dispatchers.io-dispatcher") + + // the nio interface is synchronous so we need to wait for the result + def withRetry[U](f: () => U): U = Await.result( + Retry.withRetry( + () => Future(f())(iOExecutionContext), + retryParams.maxRetries, + retryParams.backoff, + retryParams.isTransient, + retryParams.isFatal + ), + retryParams.timeout + ) + + override def getPath(uri: URI): Path = { + val path = delegate.getPath(uri) + new PathProxy(path, new FileSystemProxy(path.getFileSystem, this)) + } + override def newFileSystem(uri: URI, env: util.Map[String, _]): FileSystem = { + new FileSystemProxy(delegate.newFileSystem(uri, env), this) + } + override def getScheme: String = delegate.getScheme + override def getFileSystem(uri: URI): FileSystem = delegate.getFileSystem(uri) + override def getFileStore(path: Path): FileStore = delegate.getFileStore(path) + + /* retried operations */ + override def move(source: Path, target: Path, options: CopyOption*): Unit = withRetry { () => delegate.move(source, target, options: _*) } + override def checkAccess(path: Path, modes: AccessMode*): Unit = withRetry { () => delegate.checkAccess(path, modes: _*) } + override def createDirectory(dir: Path, attrs: FileAttribute[_]*): Unit = withRetry { () => delegate.createDirectory(dir, attrs: _*) } + override def newByteChannel(path: Path, options: util.Set[_ <: OpenOption], attrs: FileAttribute[_]*): SeekableByteChannel = withRetry { () => delegate.newByteChannel(path, options, attrs: _*) } + override def isHidden(path: Path): Boolean = withRetry { () => delegate.isHidden(path) } + override def copy(source: Path, target: Path, options: CopyOption*): Unit = withRetry { () => delegate.copy(source, target, options: _*) } + override def delete(path: Path): Unit = withRetry { () => delegate.delete(path) } + override def newDirectoryStream(dir: Path, filter: Filter[_ >: Path]): DirectoryStream[Path] = withRetry { () => delegate.newDirectoryStream(dir, filter) } + override def setAttribute(path: Path, attribute: String, value: scala.Any, options: LinkOption*): Unit = withRetry { () => delegate.setAttribute(path, attribute, value, options: _*) } + override def readAttributes[A <: BasicFileAttributes](path: Path, `type`: Class[A], options: LinkOption*): A = withRetry { () => delegate.readAttributes(path, `type`, options: _*) } + override def readAttributes(path: Path, attributes: String, options: LinkOption*): util.Map[String, AnyRef] = withRetry { () => delegate.readAttributes(path, attributes, options: _*) } + override def isSameFile(path: Path, path2: Path): Boolean = withRetry { () => delegate.isSameFile(path, path2) } + override def getFileAttributeView[V <: FileAttributeView](path: Path, `type`: Class[V], options: LinkOption*): V = withRetry { () => delegate.getFileAttributeView(path, `type`, options: _*) } +} diff --git a/core/src/main/scala/cromwell/core/retry/Backoff.scala b/core/src/main/scala/cromwell/core/retry/Backoff.scala index af07de595..70b3f82f4 100644 --- a/core/src/main/scala/cromwell/core/retry/Backoff.scala +++ b/core/src/main/scala/cromwell/core/retry/Backoff.scala @@ -4,7 +4,7 @@ import com.google.api.client.util.ExponentialBackOff import scala.concurrent.duration.{Duration, FiniteDuration} -sealed trait Backoff { +trait Backoff { /** Next interval in millis */ def backoffMillis: Long /** Get the next instance of backoff. This should be called after every call to backoffMillis */ diff --git a/core/src/main/scala/cromwell/core/retry/Retry.scala b/core/src/main/scala/cromwell/core/retry/Retry.scala index 7ac181129..002a8d6e5 100644 --- a/core/src/main/scala/cromwell/core/retry/Retry.scala +++ b/core/src/main/scala/cromwell/core/retry/Retry.scala @@ -25,7 +25,7 @@ object Retry { */ def withRetry[A](f: () => Future[A], maxRetries: Option[Int] = Option(10), - backoff: SimpleExponentialBackoff = SimpleExponentialBackoff(5 seconds, 10 seconds, 1.1D), + backoff: Backoff = SimpleExponentialBackoff(5 seconds, 10 seconds, 1.1D), isTransient: Throwable => Boolean = throwableToFalse, isFatal: Throwable => Boolean = throwableToFalse) (implicit actorSystem: ActorSystem): Future[A] = { @@ -38,7 +38,7 @@ object Retry { case throwable if isFatal(throwable) => Future.failed(new CromwellFatalException(throwable)) case throwable if !isFatal(throwable) => val retriesLeft = if (isTransient(throwable)) maxRetries else maxRetries map { _ - 1 } - after(delay, actorSystem.scheduler)(withRetry(f, backoff = backoff, maxRetries = retriesLeft)) + after(delay, actorSystem.scheduler)(withRetry(f, backoff = backoff, maxRetries = retriesLeft, isTransient = isTransient, isFatal = isFatal)) } } else f() recoverWith { case e: Exception => Future.failed(new CromwellFatalException(e)) diff --git a/core/src/main/scala/cromwell/util/TryUtil.scala b/core/src/main/scala/cromwell/util/TryUtil.scala new file mode 100644 index 000000000..18f7ea58a --- /dev/null +++ b/core/src/main/scala/cromwell/util/TryUtil.scala @@ -0,0 +1,45 @@ +package cromwell.util + +import java.io.{PrintWriter, StringWriter} + +import lenthall.exception.ThrowableAggregation + +import scala.util.{Success, Failure, Try} + +case class AggregatedException(exceptions: Seq[Throwable], prefixError: String = "") extends ThrowableAggregation { + override def throwables: Traversable[Throwable] = exceptions + override def exceptionContext: String = prefixError +} + +object TryUtil { + private def stringifyFailure(failure: Try[Any]): String = { + val stringWriter = new StringWriter() + val writer = new PrintWriter(stringWriter) + failure recover { case e => e.printStackTrace(writer) } + writer.flush() + writer.close() + stringWriter.toString + } + + def stringifyFailures[T](possibleFailures: Traversable[Try[T]]): Traversable[String] = + possibleFailures.collect { case failure: Failure[T] => stringifyFailure(failure) } + + private def sequenceIterable[T](tries: Iterable[Try[_]], unbox: () => T, prefixErrorMessage: String) = { + tries collect { case f: Failure[_] => f } match { + case failures if failures.nonEmpty => + val exceptions = failures.toSeq.map(_.exception) + Failure(AggregatedException(exceptions, prefixErrorMessage)) + case _ => Success(unbox()) + } + } + + def sequence[T](tries: Seq[Try[T]], prefixErrorMessage: String = ""): Try[Seq[T]] = { + def unbox = tries map { _.get } + sequenceIterable(tries, unbox _, prefixErrorMessage) + } + + def sequenceMap[T, U](tries: Map[T, Try[U]], prefixErrorMessage: String = ""): Try[Map[T, U]] = { + def unbox = tries mapValues { _.get } + sequenceIterable(tries.values, unbox _, prefixErrorMessage) + } +} diff --git a/core/src/test/scala/cromwell/core/path/RetryableFileSystemProxySpec.scala b/core/src/test/scala/cromwell/core/path/RetryableFileSystemProxySpec.scala new file mode 100644 index 000000000..b6de83ee7 --- /dev/null +++ b/core/src/test/scala/cromwell/core/path/RetryableFileSystemProxySpec.scala @@ -0,0 +1,278 @@ +package cromwell.core.path + +import java.io.FileNotFoundException +import java.nio.channels.SeekableByteChannel +import java.nio.file.DirectoryStream.Filter +import java.nio.file.attribute.{BasicFileAttributes, FileAttributeView} +import java.nio.file.spi.FileSystemProvider +import java.nio.file.{DirectoryStream, OpenOption, Path, StandardOpenOption} +import java.util.concurrent.TimeoutException + +import cromwell.core.path.proxy.RetryableFileSystemProviderProxy +import cromwell.core.retry.Backoff +import cromwell.core.{CromwellFatalException, TestKitSuite} +import org.mockito.Matchers._ +import org.mockito.Mockito._ +import org.mockito.invocation.InvocationOnMock +import org.mockito.stubbing.Answer +import org.scalatest.{FlatSpecLike, Matchers} + +import scala.concurrent.duration._ +import scala.language.postfixOps + +class RetryableFileSystemProxySpec extends TestKitSuite with FlatSpecLike with Matchers { + + behavior of "RetryableFileSystemProxySpec" + + case class ThrowParams(exception: Exception, nbTimes: Int) + + abstract class FileSystemAnswer[T](delay: Option[Duration] = None, + throws: Option[ThrowParams] = None) extends Answer[T] { + + var nbThrows = 0 + + def delayAndOrThrow() = { + delay foreach { d => Thread.sleep(d.toMillis) } + throws foreach { e => + if (nbThrows < e.nbTimes) { + nbThrows = nbThrows + 1 + throw e.exception + } + } + } + } + + def mockFileSystem(delay: Option[Duration] = None, + throws: Option[ThrowParams] = None): FileSystemProvider = { + + val provider = mock(classOf[FileSystemProvider]) + + def answerUnit: Answer[Unit] = new FileSystemAnswer[Unit](delay, throws) { + override def answer(invocation: InvocationOnMock): Unit = delayAndOrThrow() + } + + def answerBoolean: Answer[Boolean] = new FileSystemAnswer[Boolean](delay, throws) { + override def answer(invocation: InvocationOnMock): Boolean = { + delayAndOrThrow() + true + } + } + + def answerSeekableByteChannel: Answer[SeekableByteChannel] = new FileSystemAnswer[SeekableByteChannel](delay, throws) { + override def answer(invocation: InvocationOnMock): SeekableByteChannel = { + delayAndOrThrow() + mock(classOf[SeekableByteChannel]) + } + } + + def answerDirectoryStream: Answer[DirectoryStream[Path]] = new FileSystemAnswer[DirectoryStream[Path]](delay, throws) { + override def answer(invocation: InvocationOnMock): DirectoryStream[Path] = { + delayAndOrThrow() + mock(classOf[DirectoryStream[Path]]) + } + } + + def answerBasicFileAttributes: Answer[BasicFileAttributes] = new FileSystemAnswer[BasicFileAttributes](delay, throws) { + override def answer(invocation: InvocationOnMock): BasicFileAttributes = { + delayAndOrThrow() + mock(classOf[BasicFileAttributes]) + } + } + + def answerMap: Answer[java.util.Map[String, AnyRef]] = new FileSystemAnswer[java.util.Map[String, AnyRef]](delay, throws) { + override def answer(invocation: InvocationOnMock): java.util.Map[String, AnyRef] = { + delayAndOrThrow() + new java.util.HashMap[String, AnyRef]() + } + } + + def answerFileAttributeView: Answer[FileAttributeView] = new FileSystemAnswer[FileAttributeView](delay, throws) { + override def answer(invocation: InvocationOnMock): FileAttributeView = { + delayAndOrThrow() + mock(classOf[FileAttributeView]) + } + } + + when(provider.move(any[Path], any[Path])).thenAnswer(answerUnit) + when(provider.checkAccess(any[Path])).thenAnswer(answerUnit) + when(provider.createDirectory(any[Path])).thenAnswer(answerUnit) + when(provider.newByteChannel(any[Path], any[java.util.Set[OpenOption]])).thenAnswer(answerSeekableByteChannel) + when(provider.isHidden(any[Path])).thenAnswer(answerBoolean) + when(provider.copy(any[Path], any[Path])).thenAnswer(answerUnit) + when(provider.delete(any[Path])).thenAnswer(answerUnit) + when(provider.newDirectoryStream(any[Path], any[Filter[Path]]())).thenAnswer(answerDirectoryStream) + when(provider.setAttribute(any[Path], any[String], any[Object])).thenAnswer(answerUnit) + when(provider.readAttributes(any[Path], any[String])).thenAnswer(answerMap) + when(provider.readAttributes(any[Path], any[Class[BasicFileAttributes]])).thenAnswer(answerBasicFileAttributes) + when(provider.isSameFile(any[Path], any[Path])).thenAnswer(answerBoolean) + when(provider.getFileAttributeView(any[Path], any[Class[FileAttributeView]])).thenAnswer(answerFileAttributeView) + + provider + } + + val testRetryParams = CustomRetryParams.Default.copy(backoff = new Backoff { + override def next: Backoff = this + override def backoffMillis: Long = 0 + }) + + val pathMock = mock(classOf[Path]) + + it should "timeout if the operation takes too long" in { + val retryParams = testRetryParams.copy(timeout = 100 millis) + val mockFs = mockFileSystem(delay = Option(200 millis)) + val retryableFs = new RetryableFileSystemProviderProxy(mockFs, retryParams)(system) + + a[TimeoutException] shouldBe thrownBy(retryableFs.move(pathMock, pathMock)) + a[TimeoutException] shouldBe thrownBy(retryableFs.checkAccess(pathMock)) + a[TimeoutException] shouldBe thrownBy(retryableFs.createDirectory(pathMock)) + a[TimeoutException] shouldBe thrownBy(retryableFs.newByteChannel(pathMock, mock(classOf[java.util.Set[StandardOpenOption]]))) + a[TimeoutException] shouldBe thrownBy(retryableFs.isHidden(pathMock)) + a[TimeoutException] shouldBe thrownBy(retryableFs.copy(pathMock, pathMock)) + a[TimeoutException] shouldBe thrownBy(retryableFs.delete(pathMock)) + a[TimeoutException] shouldBe thrownBy(retryableFs.newDirectoryStream(pathMock, mock(classOf[Filter[Path]]))) + a[TimeoutException] shouldBe thrownBy(retryableFs.setAttribute(pathMock, "", "")) + a[TimeoutException] shouldBe thrownBy(retryableFs.readAttributes(pathMock, classOf[BasicFileAttributes])) + a[TimeoutException] shouldBe thrownBy(retryableFs.readAttributes(pathMock, "")) + a[TimeoutException] shouldBe thrownBy(retryableFs.isSameFile(pathMock, pathMock)) + a[TimeoutException] shouldBe thrownBy(retryableFs.getFileAttributeView(pathMock, classOf[FileAttributeView])) + } + + it should "retry on failure and finally succeed if under retry max" in { + val retryParams = testRetryParams.copy(maxRetries = Option(4)) + val mockFs = mockFileSystem(throws = Option(ThrowParams(new Exception(), nbTimes = 2))) + val retryableFs = new RetryableFileSystemProviderProxy(mockFs, retryParams)(system) + + retryableFs.move(pathMock, pathMock) + retryableFs.checkAccess(pathMock) + retryableFs.createDirectory(pathMock) + retryableFs.newByteChannel(pathMock, mock(classOf[java.util.Set[StandardOpenOption]])) + retryableFs.isHidden(pathMock) + retryableFs.copy(pathMock, pathMock) + retryableFs.delete(pathMock) + retryableFs.newDirectoryStream(pathMock, mock(classOf[Filter[Path]])) + retryableFs.setAttribute(pathMock, "", "") + retryableFs.readAttributes(pathMock, classOf[BasicFileAttributes]) + retryableFs.readAttributes(pathMock, "") + retryableFs.isSameFile(pathMock, pathMock) + retryableFs.getFileAttributeView(pathMock, classOf[FileAttributeView]) + + verify(mockFs, times(3)).move(any[Path], any[Path]) + verify(mockFs, times(3)).checkAccess(any[Path]) + verify(mockFs, times(3)).createDirectory(any[Path]) + verify(mockFs, times(3)).newByteChannel(any[Path], any[java.util.Set[OpenOption]]) + verify(mockFs, times(3)).isHidden(any[Path]) + verify(mockFs, times(3)).copy(any[Path], any[Path]) + verify(mockFs, times(3)).delete(any[Path]) + verify(mockFs, times(3)).newDirectoryStream(any[Path], any[Filter[Path]]()) + verify(mockFs, times(3)).setAttribute(any[Path], any[String], any[Object]) + verify(mockFs, times(3)).readAttributes(any[Path], any[String]) + verify(mockFs, times(3)).readAttributes(any[Path], any[Class[BasicFileAttributes]]) + verify(mockFs, times(3)).isSameFile(any[Path], any[Path]) + verify(mockFs, times(3)).getFileAttributeView(any[Path], any[Class[FileAttributeView]]) + } + + it should "retry on failure and fail if over retry max" in { + val retryParams = testRetryParams.copy(maxRetries = Option(2)) + val mockFs = mockFileSystem(throws = Option(ThrowParams(new IllegalArgumentException(), nbTimes = 3))) + val retryableFs = new RetryableFileSystemProviderProxy(mockFs, retryParams)(system) + + (the [CromwellFatalException] thrownBy retryableFs.move(pathMock, pathMock)).getCause shouldBe a[IllegalArgumentException] + (the [CromwellFatalException] thrownBy retryableFs.checkAccess(pathMock)).getCause shouldBe a[IllegalArgumentException] + (the [CromwellFatalException] thrownBy retryableFs.createDirectory(pathMock)).getCause shouldBe a[IllegalArgumentException] + (the [CromwellFatalException] thrownBy retryableFs.newByteChannel(pathMock, mock(classOf[java.util.Set[StandardOpenOption]]))).getCause shouldBe a[IllegalArgumentException] + (the [CromwellFatalException] thrownBy retryableFs.isHidden(pathMock)).getCause shouldBe a[IllegalArgumentException] + (the [CromwellFatalException] thrownBy retryableFs.copy(pathMock, pathMock)).getCause shouldBe a[IllegalArgumentException] + (the [CromwellFatalException] thrownBy retryableFs.delete(pathMock)).getCause shouldBe a[IllegalArgumentException] + (the [CromwellFatalException] thrownBy retryableFs.newDirectoryStream(pathMock, mock(classOf[Filter[Path]]))).getCause shouldBe a[IllegalArgumentException] + (the [CromwellFatalException] thrownBy retryableFs.setAttribute(pathMock, "", "")).getCause shouldBe a[IllegalArgumentException] + (the [CromwellFatalException] thrownBy retryableFs.readAttributes(pathMock, classOf[BasicFileAttributes])).getCause shouldBe a[IllegalArgumentException] + (the [CromwellFatalException] thrownBy retryableFs.readAttributes(pathMock, "")).getCause shouldBe a[IllegalArgumentException] + (the [CromwellFatalException] thrownBy retryableFs.isSameFile(pathMock, pathMock)).getCause shouldBe a[IllegalArgumentException] + (the [CromwellFatalException] thrownBy retryableFs.getFileAttributeView(pathMock, classOf[FileAttributeView])).getCause shouldBe a[IllegalArgumentException] + + verify(mockFs, times(3)).move(any[Path], any[Path]) + verify(mockFs, times(3)).checkAccess(any[Path]) + verify(mockFs, times(3)).createDirectory(any[Path]) + verify(mockFs, times(3)).newByteChannel(any[Path], any[java.util.Set[OpenOption]]) + verify(mockFs, times(3)).isHidden(any[Path]) + verify(mockFs, times(3)).copy(any[Path], any[Path]) + verify(mockFs, times(3)).delete(any[Path]) + verify(mockFs, times(3)).newDirectoryStream(any[Path], any[Filter[Path]]()) + verify(mockFs, times(3)).setAttribute(any[Path], any[String], any[Object]) + verify(mockFs, times(3)).readAttributes(any[Path], any[String]) + verify(mockFs, times(3)).readAttributes(any[Path], any[Class[BasicFileAttributes]]) + verify(mockFs, times(3)).isSameFile(any[Path], any[Path]) + verify(mockFs, times(3)).getFileAttributeView(any[Path], any[Class[FileAttributeView]]) + } + + it should "ignore transient exceptions" in { + def isTransient(t: Throwable) = t.isInstanceOf[FileNotFoundException] + val retryParams = testRetryParams.copy(maxRetries = Option(1), isTransient = isTransient) + val mockFs = mockFileSystem(throws = Option(ThrowParams(new FileNotFoundException(), nbTimes = 2))) + val retryableFs = new RetryableFileSystemProviderProxy(mockFs, retryParams)(system) + + retryableFs.move(pathMock, pathMock) + retryableFs.checkAccess(pathMock) + retryableFs.createDirectory(pathMock) + retryableFs.newByteChannel(pathMock, mock(classOf[java.util.Set[StandardOpenOption]])) + retryableFs.isHidden(pathMock) + retryableFs.copy(pathMock, pathMock) + retryableFs.delete(pathMock) + retryableFs.newDirectoryStream(pathMock, mock(classOf[Filter[Path]])) + retryableFs.setAttribute(pathMock, "", "") + retryableFs.readAttributes(pathMock, classOf[BasicFileAttributes]) + retryableFs.readAttributes(pathMock, "") + retryableFs.isSameFile(pathMock, pathMock) + retryableFs.getFileAttributeView(pathMock, classOf[FileAttributeView]) + + verify(mockFs, times(3)).move(any[Path], any[Path]) + verify(mockFs, times(3)).checkAccess(any[Path]) + verify(mockFs, times(3)).createDirectory(any[Path]) + verify(mockFs, times(3)).newByteChannel(any[Path], any[java.util.Set[OpenOption]]) + verify(mockFs, times(3)).isHidden(any[Path]) + verify(mockFs, times(3)).copy(any[Path], any[Path]) + verify(mockFs, times(3)).delete(any[Path]) + verify(mockFs, times(3)).newDirectoryStream(any[Path], any[Filter[Path]]()) + verify(mockFs, times(3)).setAttribute(any[Path], any[String], any[Object]) + verify(mockFs, times(3)).readAttributes(any[Path], any[String]) + verify(mockFs, times(3)).readAttributes(any[Path], any[Class[BasicFileAttributes]]) + verify(mockFs, times(3)).isSameFile(any[Path], any[Path]) + verify(mockFs, times(3)).getFileAttributeView(any[Path], any[Class[FileAttributeView]]) + } + + it should "fail immediately on fatal exceptions" in { + def isFatal(t: Throwable) = t.isInstanceOf[FileNotFoundException] + val retryParams = testRetryParams.copy(maxRetries = Option(5), isFatal = isFatal) + val mockFs = mockFileSystem(throws = Option(ThrowParams(new FileNotFoundException(), nbTimes = 3))) + val retryableFs = new RetryableFileSystemProviderProxy(mockFs, retryParams)(system) + + (the [CromwellFatalException] thrownBy retryableFs.move(pathMock, pathMock)).getCause shouldBe a[FileNotFoundException] + (the [CromwellFatalException] thrownBy retryableFs.checkAccess(pathMock)).getCause shouldBe a[FileNotFoundException] + (the [CromwellFatalException] thrownBy retryableFs.createDirectory(pathMock)).getCause shouldBe a[FileNotFoundException] + (the [CromwellFatalException] thrownBy retryableFs.newByteChannel(pathMock, mock(classOf[java.util.Set[StandardOpenOption]]))).getCause shouldBe a[FileNotFoundException] + (the [CromwellFatalException] thrownBy retryableFs.isHidden(pathMock)).getCause shouldBe a[FileNotFoundException] + (the [CromwellFatalException] thrownBy retryableFs.copy(pathMock, pathMock)).getCause shouldBe a[FileNotFoundException] + (the [CromwellFatalException] thrownBy retryableFs.delete(pathMock)).getCause shouldBe a[FileNotFoundException] + (the [CromwellFatalException] thrownBy retryableFs.newDirectoryStream(pathMock, mock(classOf[Filter[Path]]))).getCause shouldBe a[FileNotFoundException] + (the [CromwellFatalException] thrownBy retryableFs.setAttribute(pathMock, "", "")).getCause shouldBe a[FileNotFoundException] + (the [CromwellFatalException] thrownBy retryableFs.readAttributes(pathMock, classOf[BasicFileAttributes])).getCause shouldBe a[FileNotFoundException] + (the [CromwellFatalException] thrownBy retryableFs.readAttributes(pathMock, "")).getCause shouldBe a[FileNotFoundException] + (the [CromwellFatalException] thrownBy retryableFs.isSameFile(pathMock, pathMock)).getCause shouldBe a[FileNotFoundException] + (the [CromwellFatalException] thrownBy retryableFs.getFileAttributeView(pathMock, classOf[FileAttributeView])).getCause shouldBe a[FileNotFoundException] + + verify(mockFs, times(1)).move(any[Path], any[Path]) + verify(mockFs, times(1)).checkAccess(any[Path]) + verify(mockFs, times(1)).createDirectory(any[Path]) + verify(mockFs, times(1)).newByteChannel(any[Path], any[java.util.Set[OpenOption]]) + verify(mockFs, times(1)).isHidden(any[Path]) + verify(mockFs, times(1)).copy(any[Path], any[Path]) + verify(mockFs, times(1)).delete(any[Path]) + verify(mockFs, times(1)).newDirectoryStream(any[Path], any[Filter[Path]]()) + verify(mockFs, times(1)).setAttribute(any[Path], any[String], any[Object]) + verify(mockFs, times(1)).readAttributes(any[Path], any[String]) + verify(mockFs, times(1)).readAttributes(any[Path], any[Class[BasicFileAttributes]]) + verify(mockFs, times(1)).isSameFile(any[Path], any[Path]) + verify(mockFs, times(1)).getFileAttributeView(any[Path], any[Class[FileAttributeView]]) + } + +} diff --git a/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala b/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala index ab9cbceac..a738984dd 100644 --- a/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala +++ b/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala @@ -1,18 +1,40 @@ package cromwell.engine -import java.nio.file.{FileSystem, FileSystems} - +import akka.actor.ActorSystem import cats.data.Validated.{Invalid, Valid} +import com.google.api.client.http.HttpResponseException import com.typesafe.config.ConfigFactory import cromwell.core.WorkflowOptions -import cromwell.engine.backend.EnhancedWorkflowOptions._ -import cromwell.filesystems.gcs.{GcsFileSystem, GcsFileSystemProvider, GoogleConfiguration} +import cromwell.core.path.{CustomRetryParams, DefaultPathBuilder, PathBuilder} +import cromwell.core.retry.SimpleExponentialBackoff +import cromwell.filesystems.gcs.{GoogleConfiguration, RetryableGcsPathBuilderFactory} import lenthall.exception.MessageAggregation import net.ceedubs.ficus.Ficus._ -import scala.concurrent.ExecutionContext +import scala.concurrent.duration._ +import scala.language.postfixOps + +case class EngineFilesystems(actorSystem: ActorSystem) { -object EngineFilesystems { + private def isFatalGcsException(t: Throwable): Boolean = t match { + case e: HttpResponseException if e.getStatusCode == 403 => true + case e: HttpResponseException if e.getStatusCode == 400 && e.getContent.contains("INVALID_ARGUMENT") => true + case _ => false + } + + private def isTransientGcsException(t: Throwable): Boolean = t match { + // Quota exceeded + case e: HttpResponseException if e.getStatusCode == 429 => true + case _ => false + } + + private val GcsRetryParams = CustomRetryParams( + timeout = Duration.Inf, + maxRetries = Option(3), + backoff = SimpleExponentialBackoff(1 seconds, 3 seconds, 1.5D), + isTransient = isTransientGcsException, + isFatal = isFatalGcsException + ) private val config = ConfigFactory.load private val googleConf: GoogleConfiguration = GoogleConfiguration(config) @@ -26,14 +48,11 @@ object EngineFilesystems { } } - def filesystemsForWorkflow(workflowOptions: WorkflowOptions)(implicit ec: ExecutionContext): List[FileSystem] = { - def gcsFileSystem: Option[GcsFileSystem] = { - googleAuthMode map { mode => - val storage = mode.buildStorage(workflowOptions.toGoogleAuthOptions, googleConf.applicationName) - GcsFileSystem(GcsFileSystemProvider(storage)) - } - } + private val gcsPathBuilderFactory = googleAuthMode map { mode => + RetryableGcsPathBuilderFactory(mode, customRetryParams = GcsRetryParams) + } - List(gcsFileSystem, Option(FileSystems.getDefault)).flatten + def pathBuildersForWorkflow(workflowOptions: WorkflowOptions): List[PathBuilder] = { + List(gcsPathBuilderFactory map { _.withOptions(workflowOptions)(actorSystem) }, Option(DefaultPathBuilder)).flatten } } diff --git a/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala b/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala index c493b41a6..accda4c8a 100644 --- a/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala +++ b/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala @@ -1,17 +1,16 @@ package cromwell.engine -import java.nio.file.FileSystem - import cromwell.backend.BackendWorkflowDescriptor import cromwell.core.WorkflowOptions.WorkflowOption import cromwell.core.callcaching.CallCachingMode +import cromwell.core.path.PathBuilder import wdl4s._ final case class EngineWorkflowDescriptor(backendDescriptor: BackendWorkflowDescriptor, workflowInputs: WorkflowCoercedInputs, backendAssignments: Map[Call, String], failureMode: WorkflowFailureMode, - engineFilesystems: List[FileSystem], + pathBuilders: List[PathBuilder], callCachingMode: CallCachingMode) { def id = backendDescriptor.id def namespace = backendDescriptor.workflowNamespace diff --git a/engine/src/main/scala/cromwell/engine/WdlFunctions.scala b/engine/src/main/scala/cromwell/engine/WdlFunctions.scala index 3cc8ee1ca..b715f5a49 100644 --- a/engine/src/main/scala/cromwell/engine/WdlFunctions.scala +++ b/engine/src/main/scala/cromwell/engine/WdlFunctions.scala @@ -1,17 +1,13 @@ package cromwell.engine -import java.nio.file.FileSystem - import cromwell.backend.wdl.{PureFunctions, ReadLikeFunctions} +import cromwell.core.path.PathBuilder import wdl4s.expression.WdlStandardLibraryFunctions import wdl4s.values.{WdlFile, WdlValue} import scala.util.{Failure, Try} -class WdlFunctions(val fileSystems: List[FileSystem]) extends WdlStandardLibraryFunctions with ReadLikeFunctions with PureFunctions { - /** - * Ordered list of filesystems to be used to execute WDL functions needing IO. - */ +class WdlFunctions(val pathBuilders: List[PathBuilder]) extends WdlStandardLibraryFunctions with ReadLikeFunctions with PureFunctions { private def fail(name: String) = Failure(new NotImplementedError(s"$name() not supported at the workflow level yet")) override def write_json(params: Seq[Try[WdlValue]]): Try[WdlFile] = fail("write_json") diff --git a/engine/src/main/scala/cromwell/engine/backend/EnhancedWorkflowOptions.scala b/engine/src/main/scala/cromwell/engine/backend/EnhancedWorkflowOptions.scala deleted file mode 100644 index e2043cb65..000000000 --- a/engine/src/main/scala/cromwell/engine/backend/EnhancedWorkflowOptions.scala +++ /dev/null @@ -1,16 +0,0 @@ -package cromwell.engine.backend - -import cromwell.core.WorkflowOptions -import cromwell.filesystems.gcs.GoogleAuthMode -import cromwell.filesystems.gcs.GoogleAuthMode.GoogleAuthOptions - -import scala.util.Try - -object EnhancedWorkflowOptions { - - implicit class GoogleAuthWorkflowOptions(val workflowOptions: WorkflowOptions) extends AnyVal { - def toGoogleAuthOptions: GoogleAuthMode.GoogleAuthOptions = new GoogleAuthOptions { - override def get(key: String): Try[String] = workflowOptions.get(key) - } - } -} diff --git a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala index 125d8a255..3e0636ce1 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala @@ -73,13 +73,13 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: case Event(RequestComplete((StatusCodes.OK, jsObject: JsObject)), data) if jsObject.state == WorkflowFailed => val updatedData = data.copy(terminalState = Option(WorkflowFailed)).addFailure(s"Workflow ${data.id.get} transitioned to state Failed") // If there's an output path specified then request metadata, otherwise issue a reply to the original sender. - if (metadataOutputPath.isDefined) requestMetadata else issueReply(updatedData) + if (metadataOutputPath.isDefined) requestMetadata(updatedData) else issueReply(updatedData) } when (RequestingOutputs) { case Event(RequestComplete((StatusCodes.OK, outputs: JsObject)), _) => outputOutputs(outputs) - if (metadataOutputPath.isDefined) requestMetadata else issueReply(stateData) + if (metadataOutputPath.isDefined) requestMetadata(stateData) else issueReply(stateData) } when (RequestingMetadata) { @@ -106,10 +106,10 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: stay() } - private def requestMetadata: State = { + private def requestMetadata(data: RunnerData): State = { val metadataBuilder = context.actorOf(MetadataBuilderActor.props(serviceRegistryActor), s"MetadataRequest-Workflow-${stateData.id.get}") metadataBuilder ! GetSingleWorkflowMetadataAction(stateData.id.get, None, None) - goto (RequestingMetadata) + goto (RequestingMetadata) using data } private def schedulePollRequest(): Unit = { diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala index 1035872d3..5802c7697 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala @@ -10,6 +10,7 @@ import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.WorkflowOptions.FinalWorkflowLogDir import cromwell.core._ import cromwell.core.logging.{WorkflowLogger, WorkflowLogging} +import cromwell.core.path.PathFactory import cromwell.engine._ import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.WorkflowActor._ @@ -161,7 +162,7 @@ class WorkflowActor(val workflowId: WorkflowId, callCacheReadActor: ActorRef, jobTokenDispenserActor: ActorRef, backendSingletonCollection: BackendSingletonCollection) - extends LoggingFSM[WorkflowActorState, WorkflowActorData] with WorkflowLogging with PathFactory { + extends LoggingFSM[WorkflowActorState, WorkflowActorData] with WorkflowLogging { implicit val ec = context.dispatcher @@ -303,7 +304,7 @@ class WorkflowActor(val workflowId: WorkflowId, stateData.workflowDescriptor foreach { wd => wd.getWorkflowOption(FinalWorkflowLogDir) match { case Some(destinationDir) => - workflowLogCopyRouter ! CopyWorkflowLogsActor.Copy(wd.id, buildPath(destinationDir, wd.engineFilesystems)) + workflowLogCopyRouter ! CopyWorkflowLogsActor.Copy(wd.id, PathFactory.buildPath(destinationDir, wd.pathBuilders)) case None if WorkflowLogger.isTemporary => workflowLogger.deleteLogFile() case _ => } diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala index cb773e7e1..f8c4eefb5 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala @@ -15,6 +15,7 @@ import cromwell.jobstore.JobStoreActor.{JobStoreWriteFailure, JobStoreWriteSucce import cromwell.services.metadata.MetadataService._ import cromwell.webservice.EngineStatsActor import net.ceedubs.ficus.Ficus._ +import org.apache.commons.lang3.exception.ExceptionUtils import scala.concurrent.duration._ @@ -178,7 +179,7 @@ class WorkflowManagerActor(config: Config, Responses from services */ case Event(WorkflowFailedResponse(workflowId, inState, reasons), data) => - log.error(s"$tag Workflow $workflowId failed (during $inState): ${reasons.mkString("\n")}") + log.error(s"$tag Workflow $workflowId failed (during $inState): ${expandFailureReasons(reasons)}") stay() /* Watched transitions @@ -276,4 +277,10 @@ class WorkflowManagerActor(config: Config, private def scheduleNextNewWorkflowPoll() = { context.system.scheduler.scheduleOnce(newWorkflowPollRate, self, RetrieveNewWorkflows)(context.dispatcher) } + + private def expandFailureReasons(reasons: Seq[Throwable]) = { + reasons map { reason => + reason.getMessage + "\n" + ExceptionUtils.getStackTrace(reason) + } mkString "\n" + } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala index aa25fdfb8..55ef429be 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala @@ -27,8 +27,7 @@ object CopyWorkflowLogsActor { // Which could be used for other copying work (outputs, call logs..) class CopyWorkflowLogsActor(serviceRegistryActor: ActorRef) extends Actor - with ActorLogging - with PathFactory { + with ActorLogging { def copyAndClean(src: Path, dest: Path) = { File(dest).parent.createDirectories() diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala index a5028e5af..d9056200e 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala @@ -5,9 +5,10 @@ import java.nio.file.Path import akka.actor.Props import cromwell.backend.BackendWorkflowFinalizationActor.{FinalizationResponse, FinalizationSuccess} import cromwell.backend.{AllBackendInitializationData, BackendConfigurationDescriptor, BackendInitializationData, BackendLifecycleActorFactory} -import cromwell.core._ import cromwell.core.Dispatcher.IoDispatcher import cromwell.core.WorkflowOptions._ +import cromwell.core._ +import cromwell.core.path.{PathCopier, PathFactory} import cromwell.engine.EngineWorkflowDescriptor import cromwell.engine.backend.{BackendConfiguration, CromwellBackends} import wdl4s.ReportableSymbol @@ -26,8 +27,10 @@ class CopyWorkflowOutputsActor(workflowId: WorkflowId, val workflowDescriptor: E initializationData: AllBackendInitializationData) extends EngineWorkflowFinalizationActor with PathFactory { + override val pathBuilders = workflowDescriptor.pathBuilders + private def copyWorkflowOutputs(workflowOutputsFilePath: String): Unit = { - val workflowOutputsPath = buildPath(workflowOutputsFilePath, workflowDescriptor.engineFilesystems) + val workflowOutputsPath = buildPath(workflowOutputsFilePath) val reportableOutputs = workflowDescriptor.backendDescriptor.workflowNamespace.workflow.outputs diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala index 2f8a1e57c..15e7edbd5 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala @@ -1,7 +1,5 @@ package cromwell.engine.workflow.lifecycle -import java.nio.file.FileSystem - import akka.actor.{ActorRef, FSM, LoggingFSM, Props} import cats.data.Validated._ import cats.instances.list._ @@ -14,8 +12,10 @@ import com.typesafe.scalalogging.LazyLogging import cromwell.backend.BackendWorkflowDescriptor import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.WorkflowOptions.{ReadFromCache, WorkflowOption, WriteToCache} +import cromwell.core._ import cromwell.core.callcaching._ import cromwell.core.logging.WorkflowLogging +import cromwell.core.path.PathBuilder import cromwell.engine._ import cromwell.engine.backend.CromwellBackends import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.{MaterializeWorkflowDescriptorActorData, MaterializeWorkflowDescriptorActorState} @@ -167,8 +167,8 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor (_, _) } flatMap { case (namespace, workflowOptions) => pushWfNameMetadataService(namespace.workflow.unqualifiedName) - val engineFileSystems = EngineFilesystems.filesystemsForWorkflow(workflowOptions)(iOExecutionContext) - buildWorkflowDescriptor(id, sourceFiles, namespace, workflowOptions, conf, engineFileSystems) + val pathBuilders = EngineFilesystems(context.system).pathBuildersForWorkflow(workflowOptions) + buildWorkflowDescriptor(id, sourceFiles, namespace, workflowOptions, conf, pathBuilders) } } @@ -185,7 +185,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor namespace: NamespaceWithWorkflow, workflowOptions: WorkflowOptions, conf: Config, - engineFilesystems: List[FileSystem]): ErrorOr[EngineWorkflowDescriptor] = { + pathBuilders: List[PathBuilder]): ErrorOr[EngineWorkflowDescriptor] = { val defaultBackendName = conf.as[Option[String]]("backend.default") val rawInputsValidation = validateRawInputs(sourceFiles.inputsJson) val failureModeValidation = validateWorkflowFailureMode(workflowOptions, conf) @@ -195,7 +195,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor (rawInputsValidation |@| failureModeValidation |@| backendAssignmentsValidation |@| callCachingModeValidation ) map { (_, _, _, _) } flatMap { case (rawInputs, failureMode, backendAssignments, callCachingMode) => - buildWorkflowDescriptor(id, namespace, rawInputs, backendAssignments, workflowOptions, failureMode, engineFilesystems, callCachingMode) + buildWorkflowDescriptor(id, namespace, rawInputs, backendAssignments, workflowOptions, failureMode, pathBuilders, callCachingMode) } } @@ -205,7 +205,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor backendAssignments: Map[Call, String], workflowOptions: WorkflowOptions, failureMode: WorkflowFailureMode, - engineFileSystems: List[FileSystem], + pathBuilders: List[PathBuilder], callCachingMode: CallCachingMode): ErrorOr[EngineWorkflowDescriptor] = { def checkTypes(inputs: Map[FullyQualifiedName, WdlValue]): ErrorOr[Map[FullyQualifiedName, WdlValue]] = { @@ -225,10 +225,10 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor for { coercedInputs <- validateCoercedInputs(rawInputs, namespace) _ = pushWfInputsToMetadataService(coercedInputs) - declarations <- validateDeclarations(namespace, workflowOptions, coercedInputs, engineFileSystems) + declarations <- validateDeclarations(namespace, workflowOptions, coercedInputs, pathBuilders) declarationsAndInputs <- checkTypes(declarations ++ coercedInputs) backendDescriptor = BackendWorkflowDescriptor(id, namespace, declarationsAndInputs, workflowOptions) - } yield EngineWorkflowDescriptor(backendDescriptor, coercedInputs, backendAssignments, failureMode, engineFileSystems, callCachingMode) + } yield EngineWorkflowDescriptor(backendDescriptor, coercedInputs, backendAssignments, failureMode, pathBuilders, callCachingMode) } private def pushWfInputsToMetadataService(workflowInputs: WorkflowCoercedInputs): Unit = { @@ -292,8 +292,8 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor private def validateDeclarations(namespace: NamespaceWithWorkflow, options: WorkflowOptions, coercedInputs: WorkflowCoercedInputs, - engineFileSystems: List[FileSystem]): ErrorOr[WorkflowCoercedInputs] = { - namespace.staticWorkflowDeclarationsRecursive(coercedInputs, new WdlFunctions(engineFileSystems)) match { + pathBuilders: List[PathBuilder]): ErrorOr[WorkflowCoercedInputs] = { + namespace.staticWorkflowDeclarationsRecursive(coercedInputs, new WdlFunctions(pathBuilders)) match { case Success(d) => d.validNel case Failure(e) => s"Workflow has invalid declarations: ${e.getMessage}".invalidNel } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala index 15da08fe7..b6c991724 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala @@ -284,7 +284,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, } private def disableCacheWrite(reason: Throwable) = { - log.error("{}: Disabling cache writing for this job.", jobTag) + log.error(reason, "{}: Disabling cache writing for this job.", jobTag) if (effectiveCallCachingMode.writeToCache) { effectiveCallCachingMode = effectiveCallCachingMode.withoutWrite writeCallCachingModeToMetadata() diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala index ded1bcfcc..3a844e6d2 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala @@ -22,7 +22,7 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript backendJobExecutionActors: Map[JobKey, ActorRef], outputStore: OutputStore) extends WdlLookup { - override val expressionLanguageFunctions = new WdlFunctions(workflowDescriptor.engineFilesystems) + override val expressionLanguageFunctions = new WdlFunctions(workflowDescriptor.pathBuilders) def jobExecutionSuccess(jobKey: JobKey, outputs: JobOutputs) = this.copy( executionStore = executionStore.add(Map(jobKey -> Done)), diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala index 8c5331c42..7683eba92 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala @@ -1,5 +1,7 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching +import java.nio.file.Path + import cats.data.NonEmptyList import cromwell.backend.BackendJobExecutionActor.SucceededResponse import cromwell.core.ExecutionIndex.IndexEnhancedIndex @@ -35,7 +37,7 @@ class CallCache(database: CallCachingSqlDatabase) { } private def addToCache(callCachingEntry: CallCachingEntry, hashes: Set[HashResult], - result: Iterable[WdlValueSimpleton], jobDetritus: Map[String, String]) + result: Iterable[WdlValueSimpleton], jobDetritus: Map[String, Path]) (implicit ec: ExecutionContext): Future[Unit] = { val hashesToInsert: Iterable[CallCachingHashEntry] = { @@ -51,7 +53,7 @@ class CallCache(database: CallCachingSqlDatabase) { val jobDetritusToInsert: Iterable[CallCachingDetritusEntry] = { jobDetritus map { - case (fileName, filePath) => CallCachingDetritusEntry(fileName, filePath) + case (fileName, filePath) => CallCachingDetritusEntry(fileName, filePath.toUri.toString) } } diff --git a/engine/src/test/scala/cromwell/engine/EngineFunctionsSpec.scala b/engine/src/test/scala/cromwell/engine/EngineFunctionsSpec.scala index ec512107e..0160bc6f1 100644 --- a/engine/src/test/scala/cromwell/engine/EngineFunctionsSpec.scala +++ b/engine/src/test/scala/cromwell/engine/EngineFunctionsSpec.scala @@ -1,8 +1,9 @@ package cromwell.engine -import java.nio.file.{FileSystem, FileSystems, Path} +import java.nio.file.Path import cromwell.backend.wdl.{PureFunctions, ReadLikeFunctions, WriteFunctions} +import cromwell.core.path.{DefaultPathBuilder, PathBuilder} import org.scalatest.prop.TableDrivenPropertyChecks._ import org.scalatest.prop.Tables.Table import org.scalatest.{FlatSpec, Matchers} @@ -38,7 +39,7 @@ class EngineFunctionsSpec extends FlatSpec with Matchers { "sub" should "replace a string according to a pattern" in { class TestEngineFn extends WdlStandardLibraryImpl { override def glob(path: String, pattern: String): Seq[String] = ??? - override def fileSystems: List[FileSystem] = List(FileSystems.getDefault) + override def pathBuilders: List[PathBuilder] = List(DefaultPathBuilder) override def writeDirectory: Path = ??? } diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala index 540da9863..ede3d57ff 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala @@ -6,6 +6,7 @@ import java.time.OffsetDateTime import akka.actor._ import akka.pattern.ask import akka.testkit.TestKit +import akka.util.Timeout import better.files._ import com.typesafe.config.ConfigFactory import cromwell.CromwellTestkitSpec._ @@ -77,6 +78,8 @@ abstract class SingleWorkflowRunnerActorSpec extends CromwellTestkitSpec { def singleWorkflowActor(sampleWdl: SampleWdl = ThreeStep, managerActor: => ActorRef = workflowManagerActor(), outputFile: => Option[Path] = None): Unit = { + implicit val timeout = Timeout(TimeoutDuration) + val actorRef = createRunnerActor(sampleWdl, managerActor, outputFile) val futureResult = actorRef ? RunWorkflow Await.ready(futureResult, Duration.Inf) @@ -111,9 +114,8 @@ class SingleWorkflowRunnerActorWithMetadataSpec extends SingleWorkflowRunnerActo singleWorkflowActor( sampleWdl = wdlFile, outputFile = Option(metadataFile.path)) + TestKit.shutdownActorSystem(system, TimeoutDuration) } - TestKit.shutdownActorSystem(system, TimeoutDuration) - val metadataFileContent = metadataFile.contentAsString val metadata = metadataFileContent.parseJson.asJsObject.fields metadata.get("id") shouldNot be(empty) diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala index 970bc7156..97e72e99d 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala @@ -73,7 +73,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestkitSpec with Be case (call, assignment) if call.task.name.equals("hello") => assignment shouldBe "Local" case (call, assignment) => fail(s"Unexpected call: ${call.task.name}") } - wfDesc.engineFilesystems.size shouldBe 1 + wfDesc.pathBuilders.size shouldBe 1 case MaterializeWorkflowDescriptorFailureResponse(reason) => fail(s"Materialization failed with $reason") case unknown => fail(s"Unexpected materialization response: $unknown") diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala index dadcd35b4..f53bd605b 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala @@ -10,18 +10,18 @@ import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.core.callcaching.{CallCachingActivity, CallCachingMode, CallCachingOff} import cromwell.core.{ExecutionStore, JobExecutionToken, OutputStore, WorkflowId} import cromwell.engine.EngineWorkflowDescriptor -import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId -import cromwell.engine.workflow.lifecycle.execution.{EngineJobExecutionActor, WorkflowExecutionActorData} import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{EJEAData, EngineJobExecutionActorState} +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes +import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec._ +import cromwell.engine.workflow.lifecycle.execution.{EngineJobExecutionActor, WorkflowExecutionActorData} +import cromwell.util.AkkaTestUtil._ import org.specs2.mock.Mockito import wdl4s.WdlExpression.ScopedLookupFunction +import wdl4s._ import wdl4s.expression.{NoFunctions, WdlFunctions, WdlStandardLibraryFunctions} import wdl4s.types.{WdlIntegerType, WdlStringType} import wdl4s.values.{WdlInteger, WdlString, WdlValue} -import wdl4s._ -import cromwell.util.AkkaTestUtil._ -import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec._ import scala.util.Success diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/ContentTypeOption.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/ContentTypeOption.scala deleted file mode 100644 index e6f83b0e4..000000000 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/ContentTypeOption.scala +++ /dev/null @@ -1,15 +0,0 @@ -package cromwell.filesystems.gcs - -import java.nio.file.OpenOption - -object ContentTypeOption { - sealed trait ContentType - case object PlainText extends ContentType with OpenOption { - override def toString = "plain/text" - } - case object Json extends ContentType with OpenOption { - override def toString = "application/json" - } -} - - diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileAttributes.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileAttributes.scala deleted file mode 100644 index 5d45641de..000000000 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileAttributes.scala +++ /dev/null @@ -1,23 +0,0 @@ -package cromwell.filesystems.gcs - -import java.nio.file.attribute.{BasicFileAttributes, FileTime} - -import com.google.api.services.storage.Storage -import com.google.api.services.storage.model.StorageObject -import org.apache.commons.codec.digest.DigestUtils - -class GcsFileAttributes(path: NioGcsPath, storageClient: Storage) extends BasicFileAttributes { - override def fileKey(): AnyRef = DigestUtils.md5Hex(path.toString) - override def isRegularFile: Boolean = throw new NotImplementedError("To be implemented when/if needed") - override def isOther: Boolean = throw new NotImplementedError("To be implemented when/if needed") - override def lastModifiedTime(): FileTime = throw new NotImplementedError("To be implemented when/if needed") - override def size(): Long = { - val getObject = storageClient.objects.get(path.bucket, path.objectName) - val storageObject: StorageObject = getObject.execute() - storageObject.getSize.longValue() - } - override def isDirectory: Boolean = path.isDirectory - override def isSymbolicLink: Boolean = false - override def creationTime(): FileTime = throw new NotImplementedError("To be implemented when/if needed") - override def lastAccessTime(): FileTime = throw new NotImplementedError("To be implemented when/if needed") -} diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystem.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystem.scala deleted file mode 100644 index 215b18935..000000000 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystem.scala +++ /dev/null @@ -1,68 +0,0 @@ -package cromwell.filesystems.gcs - -import java.lang.Iterable -import java.nio.file._ -import java.nio.file.attribute.UserPrincipalLookupService -import java.nio.file.spi.FileSystemProvider -import java.util.{Collections, Set => JSet} - - -case class NotAGcsPathException(path: String) extends IllegalArgumentException(s"$path is not a valid GCS path.") - -object GcsFileSystem { - val Separator = "/" - private[gcs] val Scheme = "gs" - private[gcs] val Protocol = s"$Scheme://" - private val GsUriRegex = s"""$Protocol(.*)""".r - private val AttributeViews = Collections.singleton("basic") - - def isAbsoluteGcsPath(str: String) = str match { - case GsUriRegex(chunks) => true - case _ => false - } - - def apply(provider: GcsFileSystemProvider) = new GcsFileSystem(provider) -} - -/** - * Implements the java.nio.FileSystem interface for GoogleCloudStorage. - */ -class GcsFileSystem private(val gcsFileSystemProvider: GcsFileSystemProvider) extends FileSystem { - - import GcsFileSystem._ - - override def supportedFileAttributeViews(): JSet[String] = AttributeViews - - override def getSeparator: String = Separator - - override def getRootDirectories: Iterable[Path] = Collections.emptyList[Path] - - override def newWatchService(): WatchService = throw new NotImplementedError("GCS FS does not support Watch Service at this time") - - override def getFileStores: Iterable[FileStore] = Collections.emptyList() - - override def isReadOnly: Boolean = false - - override def provider(): FileSystemProvider = gcsFileSystemProvider - - override def isOpen: Boolean = true - - override def close(): Unit = throw new UnsupportedOperationException("GCS FS cannot be closed") - - override def getPathMatcher(syntaxAndPattern: String): PathMatcher = FileSystems.getDefault.getPathMatcher(syntaxAndPattern) - - override def getUserPrincipalLookupService: UserPrincipalLookupService = throw new UnsupportedOperationException() - - private def buildPath(first: String, more: Seq[String], forceDirectory: Boolean) = { - val directory = forceDirectory || (more.isEmpty && first.endsWith(Separator)) || more.lastOption.exists(_.endsWith(Separator)) - first match { - case GsUriRegex(chunks) => new NioGcsPath(chunks.split(Separator) ++ more.toArray[String], true, directory)(this) - case empty if empty.isEmpty => new NioGcsPath(Array.empty[String] ++ more.toArray[String], false, false)(this) - case _ => throw NotAGcsPathException(s"$first is not a gcs path") - } - } - - override def getPath(first: String, more: String*): Path = buildPath(first, more, forceDirectory = false) - - def getPathAsDirectory(first: String, more: String*): Path = buildPath(first, more, forceDirectory = true) -} diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala deleted file mode 100644 index 845ec29ef..000000000 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala +++ /dev/null @@ -1,295 +0,0 @@ -package cromwell.filesystems.gcs - -import java.io.{FileNotFoundException, OutputStream} -import java.net.URI -import java.nio.channels.{Channels, SeekableByteChannel} -import java.nio.file.DirectoryStream.Filter -import java.nio.file._ -import java.nio.file.attribute.{BasicFileAttributes, FileAttribute, FileAttributeView} -import java.nio.file.spi.FileSystemProvider -import java.util -import java.util.Collections -import java.util.concurrent.{AbstractExecutorService, TimeUnit} - -import cats.instances.try_._ -import cats.syntax.functor._ -import com.google.api.client.googleapis.json.GoogleJsonResponseException -import com.google.api.client.googleapis.media.MediaHttpUploader -import com.google.api.services.storage.Storage -import com.google.api.services.storage.model.StorageObject -import com.google.cloud.hadoop.gcsio.{GoogleCloudStorageReadChannel, GoogleCloudStorageWriteChannel, ObjectWriteConditions} -import com.google.cloud.hadoop.util.{ApiErrorExtractor, AsyncWriteChannelOptions, ClientRequestHelper} -import com.typesafe.config.{Config, ConfigFactory, ConfigMemorySize} -import net.ceedubs.ficus.Ficus._ -import net.ceedubs.ficus.readers.ValueReader - -import scala.annotation.tailrec -import scala.collection.JavaConverters._ -import scala.concurrent.duration._ -import scala.concurrent.{ExecutionContext, ExecutionContextExecutorService} -import scala.language.postfixOps -import scala.util.{Failure, Success, Try} - -object GcsFileSystemProvider { - def apply(storageClient: Storage)(implicit ec: ExecutionContext) = { - new GcsFileSystemProvider(Success(storageClient), ec) - } - - object AcceptAllFilter extends DirectoryStream.Filter[Path] { - override def accept(entry: Path): Boolean = true - } - - // To choose these numbers I first entered a prolonged period of personal consideration and deep thought. - // Then, at the end of this time, I decided to just pick some numbers arbitrarily. - private val retryInterval = 500 milliseconds - private val retryCount = 3 - - def withRetry[A](f: => A, retries: Int = retryCount): A = Try(f) match { - case Success(res) => res - case Failure(ex: GoogleJsonResponseException) - if retries > 0 && - (ex.getStatusCode == 404 || ex.getStatusCode == 500) => - // FIXME remove this sleep - Thread.sleep(retryInterval.toMillis) - withRetry(f, retries - 1) - case Failure(ex) => throw ex - } - - // TODO refactor as part of Ficus and submit a PR - implicit val configMemorySizeValueReader: ValueReader[ConfigMemorySize] = new ValueReader[ConfigMemorySize] { - override def read(config: Config, path: String): ConfigMemorySize = config.getMemorySize(path) - } -} - -/** - * Converts a Scala ExecutionContext to a Java ExecutorService. - * https://groups.google.com/forum/#!topic/scala-user/ZyHrfzD7eX8 - */ -object ExecutionContextExecutorServiceBridge { - def apply(ec: ExecutionContext): ExecutionContextExecutorService = ec match { - case null => throw new RuntimeException("Execution context cannot be null") - case eces: ExecutionContextExecutorService => eces - case executionContext => new AbstractExecutorService with ExecutionContextExecutorService { - override def prepare(): ExecutionContext = executionContext - override def isShutdown = false - override def isTerminated = false - override def shutdown() = () - override def shutdownNow() = Collections.emptyList[Runnable] - override def execute(runnable: Runnable): Unit = executionContext execute runnable - override def reportFailure(t: Throwable): Unit = executionContext reportFailure t - override def awaitTermination(length: Long,unit: TimeUnit): Boolean = false - } - } -} - -/** - * Implements java.nio.FileSystemProvider for GoogleCloudStorage - * This implementation is not complete and mostly a proof of concept that it's possible to *copy* around files from/to local/gcs. - * Copying is the only functionality that has been successfully tested (same and cross filesystems). - * - * If/when switching to Google's GCS NIO implementation, callers may need to implement various utilities built into - * this implementation, including: - * - * - Minimizing the upload buffer size, assuming the default is also on the order of megabytes of memory per upload - * - Automatically retrying transient errors - * - etc. - * - * @param storageClient Google API Storage object - * @param executionContext executionContext, will be used to perform async writes to GCS after being converted to a Java execution service - */ -class GcsFileSystemProvider private[gcs](storageClient: Try[Storage], val executionContext: ExecutionContext) extends FileSystemProvider { - import GcsFileSystemProvider._ - - private[this] lazy val config = ConfigFactory.load() - - // We want to throw an exception here if we try to use this class with a failed gcs interface - lazy val client = storageClient.get - private val executionService = ExecutionContextExecutorServiceBridge(executionContext) - private val errorExtractor = new ApiErrorExtractor() - def notAGcsPath(path: Path) = throw new IllegalArgumentException(s"$path is not a GCS path.") - - lazy val defaultFileSystem: GcsFileSystem = GcsFileSystem(this) - - private def exists(path: Path): Unit = path match { - case gcsPath: NioGcsPath => - val attempt: Try[Any] = Try(withRetry(client.objects.get(gcsPath.bucket, gcsPath.objectName).execute)) recover { - case ex: GoogleJsonResponseException - if ex.getStatusCode == 404 => if (!gcsPath.isDirectory) throw new FileNotFoundException(path.toString) - } - attempt.void.get - case _ => throw new FileNotFoundException(path.toString) - } - - /** - * Note: options and attributes are not honored. - */ - override def newByteChannel(path: Path, options: util.Set[_ <: OpenOption], attrs: FileAttribute[_]*): SeekableByteChannel = { - def createReadChannel(gcsPath: NioGcsPath) = new GoogleCloudStorageReadChannel(client, - gcsPath.bucket, - gcsPath.objectName, - errorExtractor, - new ClientRequestHelper[StorageObject]() - ) - - path match { - case gcsPath: NioGcsPath => withRetry(createReadChannel(gcsPath)) - case _ => notAGcsPath(path) - } - } - - /* - For now, default all upload buffers as small as possible, 256K per upload. Without this default the buffers are 64M. - In the future, we may possibly be able to pass information to the NioGcsPath with the expected... or Google's GCS NIO - implementation will be finished we'll need to revisit this issue again. - - See also: - - com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.setUploadBufferSize - - com.google.api.client.googleapis.media.MediaHttpUploader.setContentAndHeadersOnCurrentRequest - */ - private[this] lazy val uploadBufferBytes = { - val configBytes = config.as[Option[ConfigMemorySize]]("google.upload-buffer-bytes").map(_.toBytes.toInt) - configBytes.getOrElse(MediaHttpUploader.MINIMUM_CHUNK_SIZE) - } - - /** - * Overrides the default implementation to provide a writable channel (which newByteChannel doesn't). - * NOTE: options are not honored. - */ - override def newOutputStream(path: Path, options: OpenOption*): OutputStream = { - val contentType = options collectFirst { - case e: ContentTypeOption.ContentType => e.toString - } getOrElse ContentTypeOption.PlainText.toString - - def initializeOutputStream(gcsPath: NioGcsPath) = { - val channel = new GoogleCloudStorageWriteChannel( - executionService, - client, - new ClientRequestHelper[StorageObject](), - gcsPath.bucket, - gcsPath.objectName, - AsyncWriteChannelOptions.newBuilder().setUploadBufferSize(uploadBufferBytes).build(), - new ObjectWriteConditions(), - Map.empty[String, String].asJava, - contentType) - channel.initialize() - Channels.newOutputStream(channel) - } - - path match { - case gcsPath: NioGcsPath => withRetry(initializeOutputStream(gcsPath)) - case _ => notAGcsPath(path) - } - } - - override def copy(source: Path, target: Path, options: CopyOption*): Unit = { - (source, target) match { - case (s: NioGcsPath, d: NioGcsPath) => - def innerCopy(): Unit = { - val storageObject = client.objects.get(s.bucket, s.objectName).execute - client.objects.copy(s.bucket, s.objectName, d.bucket, d.objectName, storageObject).execute - () - } - - withRetry(innerCopy()) - case _ => throw new UnsupportedOperationException(s"Can only copy from GCS to GCS: $source or $target is not a GCS path") - } - } - - override def delete(path: Path): Unit = { - path match { - case gcs: NioGcsPath => try { - withRetry { - client.objects.delete(gcs.bucket, gcs.objectName).execute() - () - } - } catch { - case ex: GoogleJsonResponseException if ex.getStatusCode == 404 => throw new NoSuchFileException(path.toString) - } - case _ => notAGcsPath(path) - } - } - - override def readAttributes[A <: BasicFileAttributes](path: Path, `type`: Class[A], options: LinkOption*): A = path match { - case gcsPath: NioGcsPath => - exists(path) - new GcsFileAttributes(gcsPath, client).asInstanceOf[A] - case _ => notAGcsPath(path) - } - - override def move(source: Path, target: Path, options: CopyOption*): Unit = { - (source, target) match { - case (s: NioGcsPath, d: NioGcsPath) => - def moveInner(): Unit = { - val storageObject = client.objects.get(s.bucket, s.objectName).execute - client.objects.rewrite(s.bucket, s.objectName, d.bucket, d.objectName, storageObject).execute - () - } - - withRetry(moveInner()) - case _ => throw new UnsupportedOperationException(s"Can only move from GCS to GCS: $source or $target is not a GCS path") - } - } - - def crc32cHash(path: Path) = path match { - case gcsDir: NioGcsPath => withRetry(client.objects().get(gcsDir.bucket, gcsDir.objectName).execute().getCrc32c) - case _ => notAGcsPath(path) - } - - override def checkAccess(path: Path, modes: AccessMode*): Unit = { exists(path); () } - override def createDirectory(dir: Path, attrs: FileAttribute[_]*): Unit = {} - - override def getFileSystem(uri: URI): FileSystem = defaultFileSystem - - override def isHidden(path: Path): Boolean = throw new NotImplementedError() - - private[this] lazy val maxResults = config.as[Option[Int]]("google.list-max-results").getOrElse(1000).toLong - - private def list(gcsDir: NioGcsPath) = { - val listRequest = client.objects().list(gcsDir.bucket).setMaxResults(maxResults) - listRequest.setPrefix(gcsDir.objectName) - - def objectToPath(storageObject: StorageObject): Path = { - NioGcsPath(s"$getScheme${storageObject.getBucket}${GcsFileSystem.Separator}${storageObject.getName}")(gcsDir.getFileSystem.asInstanceOf[GcsFileSystem]) - } - - // Contains a Seq corresponding to the current page of objects, plus a token for the next page of objects, if any. - case class ListPageResult(objects: Seq[StorageObject], nextPageToken: Option[String]) - - def requestListPage(pageToken: Option[String]): ListPageResult = { - val objects = withRetry(listRequest.setPageToken(pageToken.orNull).execute()) - ListPageResult(objects.getItems.asScala, Option(objects.getNextPageToken)) - } - - @tailrec - def remainingObjects(pageToken: Option[String], acc: Seq[StorageObject]): Seq[StorageObject] = { - if (pageToken.isEmpty) acc - else { - val page = requestListPage(pageToken) - remainingObjects(page.nextPageToken, acc ++ page.objects) - } - } - - val firstPage = requestListPage(pageToken = None) - val allObjects = remainingObjects(firstPage.nextPageToken, firstPage.objects) - - new DirectoryStream[Path] { - override def iterator(): util.Iterator[Path] = (allObjects map objectToPath).toIterator.asJava - override def close(): Unit = {} - } - } - - override def newDirectoryStream(dir: Path, filter: Filter[_ >: Path]): DirectoryStream[Path] = dir match { - case gcsDir: NioGcsPath => list(gcsDir) - case _ => notAGcsPath(dir) - } - override def setAttribute(path: Path, attribute: String, value: scala.Any, options: LinkOption*): Unit = throw new NotImplementedError() - override def getPath(uri: URI): Path = throw new NotImplementedError() - override def newFileSystem(uri: URI, env: util.Map[String, _]): FileSystem = { - throw new UnsupportedOperationException("GcsFileSystem provider doesn't support creation of new FileSystems at this time. Use getFileSystem instead.") - } - override def readAttributes(path: Path, attributes: String, options: LinkOption*): util.Map[String, AnyRef] = throw new NotImplementedError() - override def isSameFile(path: Path, path2: Path): Boolean = throw new NotImplementedError() - override def getFileAttributeView[V <: FileAttributeView](path: Path, `type`: Class[V], options: LinkOption*): V = throw new NotImplementedError() - override def getFileStore(path: Path): FileStore = throw new NotImplementedError() - override def getScheme: String = GcsFileSystem.Protocol -} diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala new file mode 100644 index 000000000..0649da9b7 --- /dev/null +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala @@ -0,0 +1,100 @@ +package cromwell.filesystems.gcs + +import java.net.URI +import java.nio.file.Path +import java.nio.file.spi.FileSystemProvider + +import akka.actor.ActorSystem +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport +import com.google.api.client.json.jackson2.JacksonFactory +import com.google.cloud.RetryParams +import com.google.cloud.storage.StorageOptions +import com.google.cloud.storage.contrib.nio.{CloudStorageConfiguration, CloudStorageFileSystem, CloudStoragePath} +import com.google.common.base.Preconditions._ +import cromwell.core.WorkflowOptions +import cromwell.core.path.proxy.{PathProxy, RetryableFileSystemProviderProxy} +import cromwell.core.path.{CustomRetryParams, PathBuilder} +import cromwell.filesystems.gcs.auth.GoogleAuthMode + +import scala.util.{Failure, Try} + +object GcsPathBuilder { + + val JsonFactory = JacksonFactory.getDefaultInstance + val HttpTransport = GoogleNetHttpTransport.newTrustedTransport + + def checkValid(uri: URI) = { + checkNotNull(uri.getScheme, s"%s does not have a gcs scheme", uri) + checkArgument( + uri.getScheme.equalsIgnoreCase(CloudStorageFileSystem.URI_SCHEME), + "Cloud Storage URIs must have '%s' scheme: %s", + CloudStorageFileSystem.URI_SCHEME, + uri + ) + checkNotNull(uri.getHost, s"%s does not have a host", uri) + } + + def isValidGcsUrl(str: String): Boolean = { + Try(checkValid(URI.create(str))).isSuccess + } + + def isGcsPath(path: Path): Boolean = { + path.getFileSystem.provider().getScheme == CloudStorageFileSystem.URI_SCHEME + } +} + +class GcsPathBuilder(authMode: GoogleAuthMode, + retryParams: RetryParams, + cloudStorageConfiguration: CloudStorageConfiguration, + options: WorkflowOptions) extends PathBuilder { + authMode.validate(options) + + protected val storageOptions = StorageOptions.builder() + .authCredentials(authMode.authCredentials(options)) + .retryParams(retryParams) + .build() + + // The CloudStorageFileSystemProvider constructor is not public. Currently the only way to obtain one is through a CloudStorageFileSystem + // Moreover at this point we can use the same provider for all operations as we have usable credentials + // In order to avoid recreating a provider with every getPath call, create a dummy FileSystem just to get its provider + protected val _provider = CloudStorageFileSystem.forBucket("dummy", cloudStorageConfiguration, storageOptions).provider() + + protected def provider: FileSystemProvider = _provider + /* + * The StorageService already contains a StorageRpc object that contains a com.google.api.services.storage.Storage object + * However it is not accessible from StorageService. + * com.google.cloud.storage.Storage has some batching capabilities but not for copying. + * In order to support batch copy, we need a com.google.api.services.storage.Storage. + */ + def getHash(path: Path): Try[String] = { + path match { + case gcsPath: CloudStoragePath => Try(storageOptions.service().get(gcsPath.bucket(), gcsPath.toRealPath().toString).crc32c()) + case proxy: PathProxy => + val gcsPath = proxy.unbox(classOf[CloudStoragePath]).get + Try(storageOptions.service().get(gcsPath.bucket(), gcsPath.toRealPath().toString).crc32c()) + case other => Failure(new IllegalArgumentException(s"$other is not a CloudStoragePath")) + } + } + + def build(string: String): Try[Path] = { + Try { + val uri = URI.create(string) + GcsPathBuilder.checkValid(uri) + provider.getPath(uri) + } + } + + override def name: String = "Gcs" +} + +class RetryableGcsPathBuilder(authMode: GoogleAuthMode, + googleRetryParams: RetryParams, + customRetryParams: CustomRetryParams, + cloudStorageConfiguration: CloudStorageConfiguration, + options: WorkflowOptions)(implicit actorSystem: ActorSystem) + extends GcsPathBuilder(authMode, googleRetryParams, cloudStorageConfiguration, options) { + + override protected def provider = new RetryableFileSystemProviderProxy(_provider, customRetryParams) + + override def getHash(path: Path) = provider.withRetry(() => super.getHash(path)) +} diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilderFactory.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilderFactory.scala new file mode 100644 index 000000000..83aad3ce8 --- /dev/null +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilderFactory.scala @@ -0,0 +1,48 @@ +package cromwell.filesystems.gcs + +import akka.actor.ActorSystem +import com.google.api.client.googleapis.media.MediaHttpUploader +import com.google.cloud.RetryParams +import com.google.cloud.storage.contrib.nio.CloudStorageConfiguration +import com.typesafe.config.ConfigFactory +import cromwell.core.WorkflowOptions +import cromwell.core.path.{CustomRetryParams, PathBuilderFactory} +import cromwell.filesystems.gcs.auth.GoogleAuthMode +import net.ceedubs.ficus.Ficus._ + +object GcsPathBuilderFactory { + + private[this] lazy val UploadBufferBytes = { + ConfigFactory.load().as[Option[Int]]("google.upload-buffer-bytes").getOrElse(MediaHttpUploader.MINIMUM_CHUNK_SIZE) + } + + val DefaultRetryParams = RetryParams.defaultInstance() + val DefaultCloudStorageConfiguration = { + CloudStorageConfiguration.builder() + .blockSize(UploadBufferBytes) + .permitEmptyPathComponents(true) + .stripPrefixSlash(true) + .usePseudoDirectories(true) + .build() + } +} + +case class GcsPathBuilderFactory(authMode: GoogleAuthMode, + retryParams: RetryParams = GcsPathBuilderFactory.DefaultRetryParams, + cloudStorageConfiguration: CloudStorageConfiguration = GcsPathBuilderFactory.DefaultCloudStorageConfiguration) + + extends PathBuilderFactory { + + def withOptions(options: WorkflowOptions)(implicit actorSystem: ActorSystem) = new GcsPathBuilder(authMode, retryParams, cloudStorageConfiguration, options) +} + +case class RetryableGcsPathBuilderFactory(authMode: GoogleAuthMode, + googleRetryParams: RetryParams = GcsPathBuilderFactory.DefaultRetryParams, + customRetryParams: CustomRetryParams = CustomRetryParams.Default, + cloudStorageConfiguration: CloudStorageConfiguration = GcsPathBuilderFactory.DefaultCloudStorageConfiguration) + + + extends PathBuilderFactory { + + def withOptions(options: WorkflowOptions)(implicit actorSystem: ActorSystem) = new RetryableGcsPathBuilder(authMode, googleRetryParams, customRetryParams, cloudStorageConfiguration, options) +} diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleAuthMode.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleAuthMode.scala deleted file mode 100644 index 2930cc911..000000000 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleAuthMode.scala +++ /dev/null @@ -1,186 +0,0 @@ -package cromwell.filesystems.gcs - -import java.io.{FileNotFoundException, IOException, InputStreamReader} -import java.nio.file.{Files, Paths} - -import com.google.api.client.auth.oauth2.Credential -import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp -import com.google.api.client.googleapis.auth.oauth2.{GoogleAuthorizationCodeFlow, GoogleClientSecrets, GoogleCredential} -import com.google.api.client.googleapis.extensions.java6.auth.oauth2.GooglePromptReceiver -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport -import com.google.api.client.json.JsonFactory -import com.google.api.client.json.jackson2.JacksonFactory -import com.google.api.client.util.store.FileDataStoreFactory -import com.google.api.services.storage.{Storage, StorageScopes} -import cromwell.filesystems.gcs.GoogleAuthMode.{GcsScopes, GoogleAuthOptions} -import org.slf4j.LoggerFactory - -import scala.collection.JavaConverters._ -import scala.util.{Failure, Success, Try} - -object GoogleAuthMode { - - lazy val jsonFactory = JacksonFactory.getDefaultInstance - lazy val httpTransport = GoogleNetHttpTransport.newTrustedTransport - val RefreshTokenOptionKey = "refresh_token" - - /** - * Before it returns the raw credential, checks if the token will expire within 60 seconds. - * - * TODO: Needs more design / testing around thread safety. - * For example, the credential returned is mutable, and may be modified by another thread. - * - * Most Google clients have the ability to refresh tokens automatically, as they use the standard Google - * HttpTransport that automatically triggers credential refreshing via Credential.handleResponse. Since Cromwell - * contacts https://gcr.io directly via HTTP requests using spray-client, we need to keep the token fresh ourselves. - * - * @see Credential#handleResponse(HttpRequest, HttpResponse, boolean) - */ - implicit class EnhancedCredentials(val credential: Credential) extends AnyVal { - def freshCredential: Try[Credential] = { - val stillValid = Option(credential.getExpiresInSeconds).exists(_ > 60) - if (stillValid || credential.refreshToken()) { - Success(credential) - } else { - Failure(new Exception("Unable to refresh token")) - } - } - } - - def buildStorage(credential: Credential, applicationName: String) = { - new Storage.Builder( - httpTransport, - jsonFactory, - credential).setApplicationName(applicationName).build() - } - - trait GoogleAuthOptions { - def get(key: String): Try[String] - } - - val GcsScopes = List( - StorageScopes.DEVSTORAGE_FULL_CONTROL, - StorageScopes.DEVSTORAGE_READ_WRITE - ) -} - - -sealed trait GoogleAuthMode { - def credential(options: GoogleAuthOptions): Credential - - def assertWorkflowOptions(options: GoogleAuthOptions): Unit = () - - def name: String - - def requiresAuthFile: Boolean = false - - protected lazy val log = LoggerFactory.getLogger(getClass.getSimpleName) - - protected def validateCredentials(credential: Credential) = { - Try(credential.refreshToken()) match { - case Failure(ex) => throw new RuntimeException(s"Google credentials are invalid: ${ex.getMessage}") - case Success(_) => credential - } - } - - def buildStorage(options: GoogleAuthOptions, applicationName: String): Storage = { - GoogleAuthMode.buildStorage(credential(options), applicationName) - } -} - -final case class ServiceAccountMode(override val name: String, accountId: String, pemPath: String, scopes: List[String] = GcsScopes) extends GoogleAuthMode { - import GoogleAuthMode._ - - private lazy val credentials: Credential = { - val pemFile = Paths.get(pemPath).toAbsolutePath - if (!Files.exists(pemFile)) { - throw new FileNotFoundException(s"PEM file $pemFile does not exist") - } - validateCredentials( - new GoogleCredential.Builder().setTransport(httpTransport) - .setJsonFactory(jsonFactory) - .setServiceAccountId(accountId) - .setServiceAccountScopes(scopes.asJava) - .setServiceAccountPrivateKeyFromPemFile(pemFile.toFile) - .build() - ) - } - - override def credential(options: GoogleAuthOptions) = credentials -} - -final case class UserMode(override val name: String, user: String, secretsFile: String, datastoreDir: String, scopes: List[String] = GcsScopes) extends GoogleAuthMode { - import GoogleAuthMode._ - - private def filePathToSecrets(secrets: String, jsonFactory: JsonFactory) = { - val secretsPath = Paths.get(secrets).toAbsolutePath - if(!Files.isReadable(secretsPath)) { - log.warn("Secrets file does not exist or is not readable.") - } - val secretStream = new InputStreamReader(Files.newInputStream(secretsPath)) - - GoogleClientSecrets.load(jsonFactory, secretStream) - } - - private lazy val credentials: Credential = { - val clientSecrets = filePathToSecrets(secretsFile, jsonFactory) - val dataStore = Paths.get(datastoreDir).toAbsolutePath - val dataStoreFactory = new FileDataStoreFactory(dataStore.toFile) - val flow = new GoogleAuthorizationCodeFlow.Builder(httpTransport, - jsonFactory, - clientSecrets, - scopes.asJava).setDataStoreFactory(dataStoreFactory).build - validateCredentials(new AuthorizationCodeInstalledApp(flow, new GooglePromptReceiver).authorize(user)) - } - - override def credential(options: GoogleAuthOptions) = credentials -} - -// It would be goofy to have multiple auths that are application_default, but Cromwell won't prevent it. -final case class ApplicationDefaultMode(override val name: String, scopes: List[String] = GcsScopes) extends GoogleAuthMode { - import GoogleAuthMode._ - - private lazy val credentials: Credential = { - try { - validateCredentials(GoogleCredential.getApplicationDefault().createScoped(scopes.asJava)) - } catch { - case e: IOException => - log.warn("Failed to get application default credentials", e) - throw e - } - } - - override def credential(options: GoogleAuthOptions) = credentials -} - -final case class RefreshTokenMode(name: String, clientId: String, clientSecret: String) extends GoogleAuthMode with ClientSecrets { - import GoogleAuthMode._ - - override def requiresAuthFile = true - - /** - * Throws if the refresh token is not specified. - */ - override def assertWorkflowOptions(options: GoogleAuthOptions): Unit = { getToken(options); () } - - private def getToken(options: GoogleAuthOptions): String = { - options.get(RefreshTokenOptionKey).getOrElse(throw new IllegalArgumentException(s"Missing parameters in workflow options: $RefreshTokenOptionKey")) - } - - override def credential(options: GoogleAuthOptions): Credential = { - validateCredentials( - new GoogleCredential.Builder().setTransport(httpTransport) - .setJsonFactory(jsonFactory) - .setClientSecrets(clientId, clientSecret) - .build() - .setRefreshToken(getToken(options)) - ) - } -} - -trait ClientSecrets { - val clientId: String - val clientSecret: String -} - -final case class SimpleClientSecrets(clientId: String, clientSecret: String) extends ClientSecrets diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala index 9c5579839..8fa93b61d 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala @@ -7,14 +7,12 @@ import cats.syntax.traverse._ import cats.syntax.validated._ import com.google.api.services.storage.StorageScopes import com.typesafe.config.Config +import cromwell.filesystems.gcs.auth._ import lenthall.config.ConfigValidationException import lenthall.config.ValidatedConfig._ import cromwell.core.ErrorOr._ import org.slf4j.LoggerFactory -import scala.collection.JavaConverters._ - - final case class GoogleConfiguration private (applicationName: String, authsByName: Map[String, GoogleAuthMode]) { def auth(name: String): ErrorOr[GoogleAuthMode] = { @@ -28,15 +26,15 @@ final case class GoogleConfiguration private (applicationName: String, authsByNa } object GoogleConfiguration { - + import scala.collection.JavaConverters._ private val log = LoggerFactory.getLogger("GoogleConfiguration") - private val GoogleScopes = List( + val GoogleScopes = List( StorageScopes.DEVSTORAGE_FULL_CONTROL, StorageScopes.DEVSTORAGE_READ_WRITE, "https://www.googleapis.com/auth/genomics", "https://www.googleapis.com/auth/compute" - ) + ).asJava def apply(config: Config): GoogleConfiguration = { @@ -55,10 +53,10 @@ object GoogleConfiguration { } def refreshTokenAuth(authConfig: Config, name: String) = authConfig validateAny { - cfg => RefreshTokenMode(name, cfg.getString("client-id"), cfg.getString("client-secret")) + cfg => RefreshTokenMode(name, cfg.getString("client-id"), cfg.getString("client-secret"), GoogleScopes) } - def applicationDefaultAuth(name: String): ErrorOr[GoogleAuthMode] = ApplicationDefaultMode(name, GoogleScopes).validNel + def applicationDefaultAuth(name: String): ErrorOr[GoogleAuthMode] = ApplicationDefaultMode(name).validNel val name = authConfig.getString("name") val scheme = authConfig.getString("scheme") diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/NioGcsPath.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/NioGcsPath.scala deleted file mode 100644 index 65e148f77..000000000 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/NioGcsPath.scala +++ /dev/null @@ -1,191 +0,0 @@ -package cromwell.filesystems.gcs - -import java.io.File -import java.net.URI -import java.nio.file.WatchEvent.{Kind, Modifier} -import java.nio.file._ -import java.util - -import scala.collection.JavaConverters._ -import scala.language.postfixOps -import scala.util.Try - -object NioGcsPath { - def apply(path: String)(implicit gcsFileSystem: GcsFileSystem) = gcsFileSystem.getPath(path) - - implicit class PathEnhanced(val path: Path) extends AnyVal { - def asGcsPath(implicit gcsFileSystem: GcsFileSystem) = path match { - case gcsPath: NioGcsPath => gcsPath - case otherPath: Path => getSoftPath(otherPath.toString).asInstanceOf[NioGcsPath] - case _ => throw new IllegalArgumentException("Only GcsPaths are supported.") - } - } - - /** Allow instantiation of a relative gcs path. - * Relative GCS paths can only be created via NioGcsPath methods (eg: subpath, getName...) but not through the GcsFileSystem.getPath method - * in order to avoid floating paths without root. It also ensures that a relative local path cannot mistakenly be parsed as a GCS path. - * */ - private def getSoftPath(first: String, more: String*)(implicit gcsFs: GcsFileSystem): Path = Try(gcsFs.getPath(first, more: _*)) recover { - case e: NotAGcsPathException => new NioGcsPath(first.split(GcsFileSystem.Separator) ++ more.toArray[String], false, first.endsWith(GcsFileSystem.Separator))(gcsFs) - } get - - val Protocol = GcsFileSystem.Protocol -} - -/** - * NOTE: Currently called NioGcsPath so it can exist alongside the current GcsPath class. - * If this approach was to be validated the current GcsPath class would be replaced by this one. - * This class proposes an implementation of the java.nio.Path interface for GoogleCloudStorage. - * The following methods are yet to be implemented: - * relativize - * compareTo - * @param chunks array containing all parts of the path in between separators - except the protocol (gs://) - * eg: gs://path/to/resource.txt -> chunks = [path, to, resource.txt] - * @param absolute true if this path is to be considered absolute. - * Only absolute GCS paths can be used to actually locate resources. - * Calling methods on an absolute path can return a relative paths (eg subpath). - * @param gcsFileSystem the gcsFileSystem to be used when performing operations on this path - */ -class NioGcsPath(private val chunks: Array[String], absolute: Boolean, val isDirectory: Boolean)(implicit gcsFileSystem: GcsFileSystem) extends Path { - import NioGcsPath._ - - private val separator = GcsFileSystem.Separator - - private val objectChunks = chunks match { - case values if isAbsolute && values.nonEmpty => values.tail - case _ => chunks - } - - private val fullPath = chunksToString(chunks) - - lazy val bucket: String = chunks match { - case values if values.isEmpty && isAbsolute => throw new IllegalStateException("An absolute gcs path cannot be empty") - case _ => if(isAbsolute) chunks.head else { - throw new UnsupportedOperationException("Attached gcs filesystem has no root and is not Absolute. The corresponding bucket is unknown.") - } - } - - val objectName = chunksToString(objectChunks) - - private def chunksToString(chunksArray: Array[String]): String = chunksArray.mkString(separator) - - override def subpath(beginIndex: Int, endIndex: Int): Path = { - val directory = if (endIndex == chunks.length - 1) isDirectory else true - new NioGcsPath(chunks.slice(beginIndex, endIndex), isAbsolute && beginIndex == 0, directory) - } - - override def toFile: File = throw new UnsupportedOperationException("A GCS path cannot be converted to a File.") - - override def resolveSibling(other: Path): Path = { - val otherPath = other.asGcsPath - new NioGcsPath(getParent.asGcsPath.chunks ++ otherPath.chunks, isAbsolute, otherPath.isDirectory) - } - - override def resolveSibling(other: String): Path = { - val otherPath = getSoftPath(other).asGcsPath - new NioGcsPath(getParent.asGcsPath.chunks ++ getSoftPath(other).asGcsPath.chunks, isAbsolute, otherPath.isDirectory) - } - - override def getFileSystem: FileSystem = gcsFileSystem - - override def getName(index: Int): Path = { - val directory = if (index == chunks.length - 1) isDirectory else true - new NioGcsPath(Array(chunks(index)), isAbsolute && index == 0, directory) - } - - override def getParent: Path = chunks match { - case values if values.isEmpty || values.length == 1 => null - case values => new NioGcsPath(values.init, isAbsolute, true) - } - - override def toAbsolutePath: Path = if (isAbsolute) this else { - throw new UnsupportedOperationException(s"Attached gcs filesystem has no root. path $toString can't be converted to an absolute path.") - } - - override def relativize(other: Path): Path = other match { - case gcs: NioGcsPath => new NioGcsPath(gcs.chunks.diff(this.chunks), false, gcs.isDirectory) - case _ => throw new IllegalArgumentException(s"$other is not a GCS path.") - } - - override def getNameCount: Int = chunks.length - - override def toUri: URI = new URI(GcsFileSystem.Scheme, bucket, s"/$objectName", null) - - override def compareTo(other: Path): Int = throw new NotImplementedError() - - override def register(watcher: WatchService, events: Array[Kind[_]], modifiers: Modifier*): WatchKey = throw new UnsupportedOperationException() - - override def register(watcher: WatchService, events: Kind[_]*): WatchKey = throw new UnsupportedOperationException() - - override def getFileName: Path = chunks match { - case values if values.isEmpty => null - case _ => new NioGcsPath(Array(chunks.last), isAbsolute && chunks.length == 1, isDirectory) - } - - override def getRoot: Path = new NioGcsPath(Array(bucket), true, true) - - override def iterator(): util.Iterator[Path] = { - if (chunks.isEmpty) chunks.map(_.asInstanceOf[Path]).iterator.asJava else { - val init = chunks.init map { elt => new NioGcsPath(Array(elt), false, true).asInstanceOf[Path] } - val fullIterator = init :+ new NioGcsPath(Array(chunks.last), false, isDirectory).asInstanceOf[Path] - fullIterator.iterator.asJava - } - } - - override def normalize(): Path = if (isAbsolute) this else throw new UnsupportedOperationException("Cannot normalize a relative GCS path.") - - override def endsWith(other: Path): Boolean = { - other match { - case rel: NioGcsPath if !isAbsolute && rel.isAbsolute => false - case _: NioGcsPath => chunks.endsWith(other.asGcsPath.chunks) - case _ => false - } - } - - override def endsWith(other: String): Boolean = { - Try(getSoftPath(other)) map { - case rel: NioGcsPath if !isAbsolute && rel.isAbsolute => false - case path@(_: NioGcsPath) => chunks.endsWith(path.asGcsPath.chunks) - case _ => false - } getOrElse false - } - - override def resolve(other: Path): Path = { - if (other.isAbsolute) other - else { - val otherGcs = other.asGcsPath - new NioGcsPath(chunks ++ otherGcs.chunks, isAbsolute, otherGcs.isDirectory) - } - } - - override def resolve(other: String): Path = { - val otherPath = getSoftPath(other).asGcsPath - if (otherPath.isAbsolute) otherPath - else new NioGcsPath(chunks ++ otherPath.asGcsPath.chunks, isAbsolute, otherPath.isDirectory) - } - - override def toRealPath(options: LinkOption*): Path = this - - override def startsWith(other: Path): Boolean = { - other match { - case rel: NioGcsPath if !isAbsolute && rel.isAbsolute => false - case _: NioGcsPath => chunks.startsWith(other.asGcsPath.chunks) - case _ => false - } - } - - override def startsWith(other: String): Boolean = { - Try(getSoftPath(other)) map { - case rel: NioGcsPath if !isAbsolute && rel.isAbsolute => false - case path@(_: NioGcsPath) => chunks.startsWith(path.asGcsPath.chunks) - case _ => false - } getOrElse false - } - - override def toString: String = { - if (absolute) s"$Protocol$fullPath" - else fullPath - } - - override def isAbsolute: Boolean = absolute -} diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleAuthMode.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleAuthMode.scala new file mode 100644 index 000000000..26d18833d --- /dev/null +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleAuthMode.scala @@ -0,0 +1,187 @@ +package cromwell.filesystems.gcs.auth + +import java.io.{FileNotFoundException, InputStreamReader} +import java.nio.file.Paths + +import better.files._ +import com.google.api.client.auth.oauth2.Credential +import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp +import com.google.api.client.googleapis.auth.oauth2.{GoogleAuthorizationCodeFlow, GoogleClientSecrets, GoogleCredential} +import com.google.api.client.googleapis.extensions.java6.auth.oauth2.GooglePromptReceiver +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport +import com.google.api.client.googleapis.testing.auth.oauth2.MockGoogleCredential +import com.google.api.client.json.jackson2.JacksonFactory +import com.google.api.client.util.store.FileDataStoreFactory +import com.google.api.services.storage.StorageScopes +import com.google.auth.oauth2.{ClientId, ServiceAccountCredentials} +import com.google.cloud.AuthCredentials +import cromwell.core.WorkflowOptions +import cromwell.filesystems.gcs.auth.GoogleAuthMode._ +import org.slf4j.LoggerFactory + +import scala.collection.JavaConverters._ +import scala.util.{Failure, Success, Try} + +object GoogleAuthMode { + + lazy val jsonFactory = JacksonFactory.getDefaultInstance + lazy val httpTransport = GoogleNetHttpTransport.newTrustedTransport + + val RefreshTokenOptionKey = "refresh_token" + val GcsScopes = List( + StorageScopes.DEVSTORAGE_FULL_CONTROL, + StorageScopes.DEVSTORAGE_READ_WRITE + ).asJava + + def checkReadable(file: File) = { + if (!file.isReadable) throw new FileNotFoundException(s"File $file does not exist or is not readable") + } + + case object NoAuthMode extends GoogleAuthMode { + override def name = "no_auth" + + override def authCredentials(options: WorkflowOptions): AuthCredentials = AuthCredentials.noAuth() + override def credential(options: WorkflowOptions): Credential = new MockGoogleCredential.Builder().build() + } +} + + +sealed trait GoogleAuthMode { + protected lazy val log = LoggerFactory.getLogger(getClass.getSimpleName) + + /** + * Validate the auth mode against provided options + */ + def validate(options: WorkflowOptions): Unit = {()} + + def name: String + // Create an AuthCredentials object from the google-cloud library (https://github.com/GoogleCloudPlatform/google-cloud-java using https://github.com/google/google-auth-library-java under the hood) + def authCredentials(options: WorkflowOptions): AuthCredentials + // Create a Credential object from the google.api.client.auth library (https://github.com/google/google-api-java-client) + def credential(options: WorkflowOptions): Credential + + def requiresAuthFile: Boolean = false + + protected def validateAuthCredentials(authCredentials: AuthCredentials, scopes: java.util.Collection[String]): AuthCredentials = validate(authCredentials, authCredentials.credentials().createScoped(scopes).refresh) + + protected def validateCredential(credential: Credential) = validate(credential, credential.refreshToken) + + private def validate[T](credential: T, validation: () => Any): T = { + Try(validation()) match { + case Failure(ex) => throw new RuntimeException(s"Google credentials are invalid: ${ex.getMessage}") + case Success(_) => credential + } + } +} + +final case class ServiceAccountMode(override val name: String, + accountId: String, + pemPath: String, + scopes: java.util.List[String]) extends GoogleAuthMode { + private val pemFile = File(pemPath) + checkReadable(pemFile) + + private lazy val _authCredentials: AuthCredentials = { + val saCredentials = ServiceAccountCredentials.fromPkcs8(accountId, accountId, pemFile.contentAsString, null, scopes) + validateAuthCredentials(AuthCredentials.createFor(saCredentials.getClientId, saCredentials.getPrivateKey), scopes) + } + + private lazy val _credential: Credential = { + validateCredential( + new GoogleCredential.Builder().setTransport(httpTransport) + .setJsonFactory(jsonFactory) + .setServiceAccountId(accountId) + .setServiceAccountScopes(scopes) + .setServiceAccountPrivateKeyFromPemFile(pemFile.toJava) + .build() + ) + } + + override def authCredentials(options: WorkflowOptions) = _authCredentials + + override def credential(options: WorkflowOptions): Credential = _credential +} + +final case class UserMode(override val name: String, + user: String, + val secretsPath: String, + datastoreDir: String, + scopes: java.util.List[String]) extends GoogleAuthMode { + + private lazy val secrets = { + val secretsFile = File(secretsPath) + checkReadable(secretsFile) + + val secretStream = new InputStreamReader(secretsFile.newInputStream) + + GoogleClientSecrets.load(jsonFactory, secretStream) + } + + private lazy val _credential: Credential = { + val dataStore = Paths.get(datastoreDir).toAbsolutePath + val dataStoreFactory = new FileDataStoreFactory(dataStore.toFile) + val flow = new GoogleAuthorizationCodeFlow.Builder(httpTransport, jsonFactory, secrets, scopes).setDataStoreFactory(dataStoreFactory).build + validateCredential(new AuthorizationCodeInstalledApp(flow, new GooglePromptReceiver).authorize(user)) + } + + private lazy val _authCredentials: AuthCredentials = { + new RefreshableOAuth2Credentials(_credential.getRefreshToken, new ClientId(secrets.getDetails.getClientId, secrets.getDetails.getClientSecret)) + } + + override def credential(options: WorkflowOptions) = _credential + + override def authCredentials(options: WorkflowOptions) = _authCredentials +} + +private object ApplicationDefault { + private [auth] lazy val _AuthCredentials = AuthCredentials.createApplicationDefaults() + private [auth] lazy val _Credential: Credential = GoogleCredential.getApplicationDefault() +} + +final case class ApplicationDefaultMode(name: String) extends GoogleAuthMode { + override def authCredentials(options: WorkflowOptions) = ApplicationDefault._AuthCredentials + override def credential(options: WorkflowOptions) = ApplicationDefault._Credential +} + +final case class RefreshTokenMode(name: String, + clientId: String, + clientSecret: String, + scopes: java.util.List[String]) extends GoogleAuthMode with ClientSecrets { + import GoogleAuthMode._ + override def requiresAuthFile = true + + private def extractRefreshToken(options: WorkflowOptions): String = { + options.get(RefreshTokenOptionKey) getOrElse { + throw new IllegalArgumentException(s"Missing parameters in workflow options: $RefreshTokenOptionKey") + } + } + + override def validate(options: WorkflowOptions) = { + extractRefreshToken(options) + + () + } + + override def authCredentials(options: WorkflowOptions): AuthCredentials = { + val refreshToken = extractRefreshToken(options) + validateAuthCredentials(new RefreshableOAuth2Credentials(refreshToken, new ClientId(clientId, clientSecret)), scopes) + } + + override def credential(options: WorkflowOptions): Credential = { + val refreshToken = extractRefreshToken(options) + validateCredential( + new GoogleCredential.Builder().setTransport(httpTransport) + .setJsonFactory(jsonFactory) + .setClientSecrets(clientId, clientSecret) + .build() + .setRefreshToken(refreshToken) + ) + } +} + +trait ClientSecrets { + val clientId: String + val clientSecret: String +} + +final case class SimpleClientSecrets(clientId: String, clientSecret: String) extends ClientSecrets diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/RefreshableOAuth2Credentials.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/RefreshableOAuth2Credentials.scala new file mode 100644 index 000000000..ae1e32ef5 --- /dev/null +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/RefreshableOAuth2Credentials.scala @@ -0,0 +1,31 @@ +package cromwell.filesystems.gcs.auth + +import java.io.Serializable +import java.util.Objects + +import com.google.auth.oauth2.{ClientId, GoogleCredentials, UserCredentials} +import com.google.cloud.{AuthCredentials, RestorableState} + +class RefreshableOAuth2Credentials(refreshToken: String, clientId: ClientId) extends AuthCredentials { + private val _credentials: GoogleCredentials = new UserCredentials(clientId.getClientId, clientId.getClientSecret, refreshToken) + + private class RefreshableOAuth2CredentialsState(val refreshToken: String, val clientId: ClientId) extends RestorableState[AuthCredentials] with Serializable { + + override def restore: AuthCredentials = new RefreshableOAuth2Credentials(refreshToken, clientId) + + override def hashCode: Int = Objects.hash(refreshToken, clientId.getClientId, clientId.getClientSecret) + + override def equals(obj: Any): Boolean = { + obj.isInstanceOf[RefreshableOAuth2CredentialsState] && { + val other = obj.asInstanceOf[RefreshableOAuth2CredentialsState] + Objects.equals(refreshToken, other.refreshToken) && + Objects.equals(clientId.getClientId, other.clientId.getClientId) && + Objects.equals(clientId.getClientSecret, other.clientId.getClientSecret) + } + } + } + + override def credentials: GoogleCredentials = _credentials + + def capture: RestorableState[AuthCredentials] = new RefreshableOAuth2CredentialsState(refreshToken, clientId) +} diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/package.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/package.scala deleted file mode 100644 index 0ec2c0316..000000000 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/package.scala +++ /dev/null @@ -1,6 +0,0 @@ -package cromwell.filesystems - - -package object gcs { - type RefreshToken = String -} diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsIntegrationTest.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsIntegrationTest.scala deleted file mode 100644 index 3c6a28734..000000000 --- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsIntegrationTest.scala +++ /dev/null @@ -1,5 +0,0 @@ -package cromwell.filesystems.gcs - -import org.scalatest.Tag - -object GcsIntegrationTest extends Tag("GcsIntegrationTest") diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala new file mode 100644 index 000000000..598cb5461 --- /dev/null +++ b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala @@ -0,0 +1,31 @@ +package cromwell.filesystems.gcs + +import com.google.cloud.RetryParams +import com.google.cloud.storage.contrib.nio.CloudStorageConfiguration +import cromwell.core.path.CustomRetryParams +import cromwell.core.path.proxy.RetryableFileSystemProviderProxy +import cromwell.core.{TestKitSuite, WorkflowOptions} +import cromwell.filesystems.gcs.auth.GoogleAuthMode +import org.scalatest.{FlatSpecLike, Matchers} + +class GcsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers { + + implicit val as = system + + behavior of "GcsPathBuilderSpec" + + it should "create a path with a retryable provider" in { + val retryablePathBuilder = new RetryableGcsPathBuilder( + GoogleAuthMode.NoAuthMode, + RetryParams.defaultInstance(), + CustomRetryParams.Default, + CloudStorageConfiguration.DEFAULT, + WorkflowOptions.empty + ) + + val path = retryablePathBuilder.build("gs://bucket/object") + path.isSuccess shouldBe true + path.get.getFileSystem.provider() shouldBe a[RetryableFileSystemProviderProxy[_]] + } + +} diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala index 3eeeaf568..cc1c9fd6c 100644 --- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala +++ b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala @@ -1,6 +1,8 @@ package cromwell.filesystems.gcs +import better.files.File import com.typesafe.config.{ConfigException, ConfigFactory} +import cromwell.filesystems.gcs.auth.{ApplicationDefaultMode, RefreshTokenMode, ServiceAccountMode, UserMode} import lenthall.config.ConfigValidationException import org.scalatest.{FlatSpec, Matchers} @@ -10,8 +12,10 @@ class GoogleConfigurationSpec extends FlatSpec with Matchers { behavior of "GoogleConfiguration" it should "parse all manner of well-formed auths" in { + val mockFile = File.newTemporaryFile() + val righteousGoogleConfig = - """ + s""" |google { | application-name = "cromwell" | @@ -30,14 +34,14 @@ class GoogleConfigurationSpec extends FlatSpec with Matchers { | name = "name-user" | scheme = "user_account" | user = "me" - | secrets-file = "/very/secret/file.txt" + | secrets-file = "${mockFile.pathAsString}" | data-store-dir = "/where/the/data/at" | }, | { | name = "name-service" | scheme = "service_account" | service-account-id = "my-google-account" - | pem-file = "/yonder/file.pem" + | pem-file = "${mockFile.pathAsString}" | } | ] |} @@ -61,13 +65,15 @@ class GoogleConfigurationSpec extends FlatSpec with Matchers { val user = (auths collectFirst { case a: UserMode => a }).get user.name shouldBe "name-user" - user.secretsFile shouldBe "/very/secret/file.txt" + user.secretsPath shouldBe mockFile.pathAsString user.datastoreDir shouldBe "/where/the/data/at" val service = (auths collectFirst { case a: ServiceAccountMode => a }).get service.name shouldBe "name-service" service.accountId shouldBe "my-google-account" - service.pemPath shouldBe "/yonder/file.pem" + service.pemPath shouldBe mockFile.pathAsString + + mockFile.delete(true) } diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleCredentialFactorySpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleCredentialFactorySpec.scala deleted file mode 100644 index 541dc75eb..000000000 --- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleCredentialFactorySpec.scala +++ /dev/null @@ -1,158 +0,0 @@ -package cromwell.filesystems.gcs - -import java.nio.file.Paths - -import com.google.api.client.auth.oauth2.Credential -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport -import com.google.api.client.json.jackson2.JacksonFactory -import com.typesafe.config.ConfigFactory -import cromwell.filesystems.gcs.GoogleAuthMode.EnhancedCredentials -import org.scalatest.{FlatSpec, Matchers} - -import scala.util.Try - -class GoogleCredentialFactorySpec extends FlatSpec with Matchers { - import GoogleCredentialFactorySpec._ - - behavior of "GoogleCredentialFactory" - - it should "refresh a token using user credentials" taggedAs GcsIntegrationTest in { - val credential = UserMode( - name = "user", - user = secretConf("user"), - secretsFile = secretConf("secrets-file"), - datastoreDir = secretConf("data-store-dir")).credential(emptyOptions) - - val firstCredentialTry: Try[Credential] = credential.freshCredential - assert(firstCredentialTry.isSuccess) - val firstCredential = firstCredentialTry.get - firstCredential.getAccessToken shouldNot be(empty) - - firstCredential.setExpiresInSeconds(59L) - - val secondCredentialTry: Try[Credential] = firstCredential.freshCredential - assert(secondCredentialTry.isSuccess) - - val secondCredential = secondCredentialTry.get - secondCredential.getAccessToken shouldNot be(empty) - secondCredential.getExpiresInSeconds shouldNot be(null) - secondCredential.getExpiresInSeconds.longValue should be > 60L - } - - it should "refresh a token using a service account" taggedAs GcsIntegrationTest in { - val credential = ServiceAccountMode( - name = "service", - accountId = secretConf("service-account-id"), - pemPath = secretConf("pem-file")).credential(emptyOptions) - - val firstCredentialTry: Try[Credential] = credential.freshCredential - assert(firstCredentialTry.isSuccess) - val firstCredential = firstCredentialTry.get - firstCredential.getAccessToken shouldNot be(empty) - - firstCredential.setExpiresInSeconds(59L) - - val secondCredentialTry: Try[Credential] = firstCredential.freshCredential - assert(secondCredentialTry.isSuccess) - - val secondCredential = secondCredentialTry.get - secondCredential.getAccessToken shouldNot be(empty) - secondCredential.getExpiresInSeconds shouldNot be(null) - secondCredential.getExpiresInSeconds.longValue should be > 60L - } - - it should "refresh a token using a refresh token" taggedAs GcsIntegrationTest in { - val opts = GoogleOptionsMap(Map("refresh_token" -> secretConf("refresh_token"))) - - val credential = RefreshTokenMode(name = "refresh", - clientId = secretConf("client-id"), - clientSecret = secretConf("client-secret")).credential(opts) - - val firstUserCredentialsTry = credential.freshCredential - - assert(firstUserCredentialsTry.isSuccess) - val firstUserCredentials = firstUserCredentialsTry.get - - val firstRefreshedUserCredentialsTry: Try[Credential] = firstUserCredentials.freshCredential - assert(firstRefreshedUserCredentialsTry.isSuccess) - val firstRefreshedUserCredentials = firstRefreshedUserCredentialsTry.get - firstRefreshedUserCredentials.getAccessToken shouldNot be(empty) - - firstRefreshedUserCredentials.setExpiresInSeconds(59L) - - val secondRefreshedUserCredentialsTry: Try[Credential] = firstRefreshedUserCredentials.freshCredential - assert(secondRefreshedUserCredentialsTry.isSuccess) - - val secondRefreshedUserCredentials = secondRefreshedUserCredentialsTry.get - secondRefreshedUserCredentials.getAccessToken shouldNot be(empty) - secondRefreshedUserCredentials.getExpiresInSeconds shouldNot be(null) - secondRefreshedUserCredentials.getExpiresInSeconds.longValue should be > 60L - } - - it should "not refresh an empty token" in { - - val wrongCredentials = new GoogleCredential.Builder() - .setTransport(GoogleNetHttpTransport.newTrustedTransport) - .setJsonFactory(JacksonFactory.getDefaultInstance) - .setClientSecrets("fakeId", "fakeSecret") - .build() - - val exception = wrongCredentials.freshCredential.failed.get - - exception.getMessage should be("Unable to refresh token") - } - - it should "refresh a token using application default credentials" taggedAs GcsIntegrationTest in { - val credential = applicationDefaultCredential - - val firstCredentialTry: Try[Credential] = credential.freshCredential - assert(firstCredentialTry.isSuccess) - val firstCredential = firstCredentialTry.get - firstCredential.getAccessToken shouldNot be(empty) - - firstCredential.setExpiresInSeconds(59L) - - val secondCredentialTry: Try[Credential] = firstCredential.freshCredential - assert(secondCredentialTry.isSuccess) - - val secondCredential = secondCredentialTry.get - secondCredential.getAccessToken shouldNot be(empty) - secondCredential.getExpiresInSeconds shouldNot be(null) - secondCredential.getExpiresInSeconds.longValue should be > 60L - } -} - -object GoogleCredentialFactorySpec { - /* - - To run this integration spec, your cromwell-credentials.conf file should have the following keys for the listed tests: - - // For testing UserMode - user = "" - secrets-file = "" - data-store-dir = "" - - // For testing ServiceAccountMode - service-account-id = "" - pem-file = "" - - // For testing RefreshTokenMode - client-id = "" - client-secret = "" - refresh_token = "" - - */ - - private lazy val credentialsConfig = ConfigFactory.parseFile(Paths.get("cromwell-credentials.conf").toFile) - - private def secretConf(path: String) = credentialsConfig.getString(path) - - private val emptyOptions = GoogleOptionsMap(Map.empty) - - def applicationDefaultCredential = ApplicationDefaultMode(name = "default").credential(emptyOptions) -} - -case class GoogleOptionsMap(map: Map[String, String]) extends GoogleAuthMode.GoogleAuthOptions { - override def get(key: String): Try[String] = Try { map(key) } -} diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/MockGcsFileSystemBuilder.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/MockGcsFileSystemBuilder.scala deleted file mode 100644 index af569d7f0..000000000 --- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/MockGcsFileSystemBuilder.scala +++ /dev/null @@ -1,9 +0,0 @@ -package cromwell.filesystems.gcs - -import scala.util.Failure - -object MockGcsFileSystemBuilder { - val mockGcsFileSystem = new GcsFileSystemProvider( - Failure(new Exception("No Storage object available")), - scala.concurrent.ExecutionContext.global).defaultFileSystem -} diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/NioGcsPathSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/NioGcsPathSpec.scala deleted file mode 100644 index ccb35efa6..000000000 --- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/NioGcsPathSpec.scala +++ /dev/null @@ -1,291 +0,0 @@ -package cromwell.filesystems.gcs - -import java.nio.file.Path - -import org.scalatest.mockito.MockitoSugar -import org.scalatest.prop.TableDrivenPropertyChecks._ -import org.scalatest.prop.Tables.Table -import org.scalatest.{FlatSpec, Matchers} - -class NioGcsPathSpec extends FlatSpec with Matchers with MockitoSugar { - - behavior of "NioGcsPath" - - implicit val GCSFs = MockGcsFileSystemBuilder.mockGcsFileSystem - - it should "implement toString" in { - val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true, true) - val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false, true) - - absPath1.toString shouldBe "gs://absolute/path/to/somewhere" - relPath1.toString shouldBe "some/relative/path" - } - - it should "implement subpath" in { - val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true, true) - val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false, true) - - val absSub1 = absPath1.subpath(0, 2) - absSub1.isAbsolute shouldBe true - absSub1.toString shouldBe "gs://absolute/path" - - val absSub2 = absPath1.subpath(1, 2) - absSub2.isAbsolute shouldBe false - absSub2.toString shouldBe "path" - - val relSub1 = relPath1.subpath(0, 2) - relSub1.isAbsolute shouldBe false - relSub1.toString shouldBe "some/relative" - } - - it should "implement resolveSibling" in { - val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true, true) - val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false, true) - val relPath2 = new NioGcsPath(Array("another", "relative", "resource", "path"), false, true) - - val absSibling = absPath1.resolveSibling("somewhere else") - absSibling.isAbsolute shouldBe true - absSibling.toString shouldBe "gs://absolute/path/to/somewhere else" - - val absSiblingPath = absPath1.resolveSibling(relPath1) - absSiblingPath.isAbsolute shouldBe true - absSiblingPath.toString shouldBe "gs://absolute/path/to/some/relative/path" - - val absRel = relPath1.resolveSibling("other path") - absRel.isAbsolute shouldBe false - absRel.toString shouldBe "some/relative/other path" - - val absRelPath = relPath1.resolveSibling(relPath2) - absRelPath.isAbsolute shouldBe false - absRelPath.toString shouldBe "some/relative/another/relative/resource/path" - } - - it should "implement resolve" in { - val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true, true) - val absPath2 = new NioGcsPath(Array("absolute", "location"), true, true) - val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false, true) - val relPath2 = new NioGcsPath(Array("another", "relative", "resource", "path"), false, true) - - val absToRel = absPath1.resolve(relPath1) - absToRel.isAbsolute shouldBe true - absToRel.toString shouldBe "gs://absolute/path/to/somewhere/some/relative/path" - - val absToAbs = absPath1.resolve(absPath2) - absToAbs.isAbsolute shouldBe true - absToAbs.toString shouldBe "gs://absolute/location" - - val relToAbs = relPath1.resolve(absPath1) - relToAbs.isAbsolute shouldBe true - relToAbs.toString shouldBe "gs://absolute/path/to/somewhere" - - val relToRel = relPath1.resolve(relPath2) - relToRel.isAbsolute shouldBe false - relToRel.toString shouldBe "some/relative/path/another/relative/resource/path" - } - - it should "implement getName" in { - val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true, true) - val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false, true) - - val nameAbs1 = absPath1.getName(0) - nameAbs1.isAbsolute shouldBe true - nameAbs1.toString shouldBe "gs://absolute" - - val nameAbs2 = absPath1.getName(1) - nameAbs2.isAbsolute shouldBe false - nameAbs2.toString shouldBe "path" - - val nameRel1 = relPath1.getName(0) - nameRel1.isAbsolute shouldBe false - nameRel1.toString shouldBe "some" - - val nameRel2 = relPath1.getName(1) - nameRel2.isAbsolute shouldBe false - nameRel2.toString shouldBe "relative" - } - - it should "implement getParent" in { - val empty = new NioGcsPath(Array.empty[String], true, true) - val singleton = new NioGcsPath(Array("singleton"), true, true) - val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true, true) - val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false, true) - - val parentAbs1 = absPath1.getParent - parentAbs1.isAbsolute shouldBe true - parentAbs1.toString shouldBe "gs://absolute/path/to" - - empty.getParent shouldBe null - singleton.getParent shouldBe null - - val nameRel1 = relPath1.getParent - nameRel1.isAbsolute shouldBe false - nameRel1.toString shouldBe "some/relative" - } - - it should "implement toAbsolutePath" in { - val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true, true) - val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false, true) - - val abs = absPath1.toAbsolutePath - abs.isAbsolute shouldBe true - abs.toString shouldBe "gs://absolute/path/to/somewhere" - - an[Exception] shouldBe thrownBy(relPath1.toAbsolutePath) - } - - it should "implement getNameCount" in { - val empty = new NioGcsPath(Array.empty[String], true, true) - val singleton = new NioGcsPath(Array("singleton"), true, true) - val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true, true) - val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false, true) - - absPath1.getNameCount shouldBe 4 - relPath1.getNameCount shouldBe 3 - empty.getNameCount shouldBe 0 - singleton.getNameCount shouldBe 1 - } - - it should "implement getFileName" in { - val empty = new NioGcsPath(Array.empty[String], true, true) - val singletonAbs = new NioGcsPath(Array("singleton"), true, true) - val singletonRel = new NioGcsPath(Array("singleton"), false, true) - val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true, true) - val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false, true) - - val emptyFileName = empty.getFileName - emptyFileName shouldBe null - - val singletonAbsFileName = singletonAbs.getFileName - singletonAbsFileName.isAbsolute shouldBe true - singletonAbsFileName.toString shouldBe "gs://singleton" - - val singletonRelFileName = singletonRel.getFileName - singletonRelFileName.isAbsolute shouldBe false - singletonRelFileName.toString shouldBe "singleton" - - val relFileName = relPath1.getFileName - relFileName.isAbsolute shouldBe false - relFileName.toString shouldBe "path" - - val absFileName = absPath1.getFileName - absFileName.isAbsolute shouldBe false - absFileName.toString shouldBe "somewhere" - } - - it should "implement getIterator" in { - val empty = new NioGcsPath(Array.empty[String], true, true) - val singletonAbs = new NioGcsPath(Array("singleton"), true, true) - val singletonRel = new NioGcsPath(Array("singleton"), false, true) - val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true, true) - val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false, true) - - empty.iterator().hasNext shouldBe false - - val singletonAbsIterator = singletonAbs.iterator() - val nextAbsSingleton: Path = singletonAbsIterator.next() - nextAbsSingleton.isAbsolute shouldBe false - nextAbsSingleton.toString shouldBe "singleton" - singletonAbsIterator.hasNext shouldBe false - - val singletonRelIterator = singletonRel.iterator() - val nextRelSingleton: Path = singletonRelIterator.next() - nextRelSingleton.isAbsolute shouldBe false - nextRelSingleton.toString shouldBe "singleton" - singletonRelIterator.hasNext shouldBe false - - val relIterator = relPath1.iterator() - val nextRel: Path = relIterator.next() - nextRel.isAbsolute shouldBe false - nextRel.toString shouldBe "some" - relIterator.next().toString shouldBe "relative" - relIterator.next().toString shouldBe "path" - relIterator.hasNext shouldBe false - - val absIterator = absPath1.iterator() - val absRel: Path = absIterator.next() - absRel.isAbsolute shouldBe false - absRel.toString shouldBe "absolute" - absIterator.next().toString shouldBe "path" - absIterator.next().toString shouldBe "to" - absIterator.next().toString shouldBe "somewhere" - absIterator.hasNext shouldBe false - } - - it should "implement startsWith" in { - val empty = new NioGcsPath(Array.empty[String], false, true) - val singletonAbs = new NioGcsPath(Array("absolute"), true, true) - - val absPath = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true, true) - val startsWithAbsPath = new NioGcsPath(Array("absolute", "path", "to"), true, true) - val doesntStartsWithAbsPath = new NioGcsPath(Array("absolute", "path", "to", "another", "place"), true, true) - val absPathStartingLikeRel = new NioGcsPath(Array("some", "relative", "path"), true, true) - - val relPath = new NioGcsPath(Array("some", "relative", "path"), false, true) - val startsWithRelPath = new NioGcsPath(Array("some", "relative"), false, true) - val doesntStartsWithRelPath = new NioGcsPath(Array("some", "relative", "other", "path"), false, true) - val relPathStartingLikeAbs = new NioGcsPath(Array("absolute", "path", "to"), false, true) - - val paths = Table( - ("path1", "path2", "result"), - (empty, empty, true), - (empty, absPath, false), - (singletonAbs, singletonAbs, true), - (absPath, startsWithAbsPath, true), - (absPath, doesntStartsWithAbsPath, false), - (absPath, relPathStartingLikeAbs, true), - (absPath, relPath, false), - (relPath, startsWithRelPath, true), - (relPath, doesntStartsWithRelPath, false), - (relPath, absPathStartingLikeRel, false), - (relPath, absPath, false) - ) - - forAll(paths) { (p1, p2, res) => - val startsWith: Boolean = p1.startsWith(p2) - startsWith shouldBe res - val startsWith1: Boolean = p1.startsWith(p2.toString) - startsWith1 shouldBe res - } - } - - it should "implement endsWith" in { - val empty = new NioGcsPath(Array.empty[String], false, true) - val singletonAbs = new NioGcsPath(Array("absolute"), true, true) - - val absPath = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true, true) - val doesntEndWithAbsPath = new NioGcsPath(Array("absolute", "path", "to", "another", "place"), true, true) - val absPathEndingLikeRel = new NioGcsPath(Array("relative", "path"), true, true) - - val relPath = new NioGcsPath(Array("some", "relative", "path"), false, true) - val endsWithRelPath = new NioGcsPath(Array("relative", "path"), false, true) - val doesntStartsWithRelPath = new NioGcsPath(Array("relative", "other", "path"), false, true) - val relPathEndingLikeAbs = new NioGcsPath(Array("path", "to", "somewhere"), false, true) - - val paths = Table( - ("path1", "path2", "result"), - (empty, empty, true), - (empty, absPath, false), - (singletonAbs, singletonAbs, true), - (absPath, absPath, true), - (absPath, doesntEndWithAbsPath, false), - (absPath, relPathEndingLikeAbs, true), - (absPath, relPath, false), - (relPath, endsWithRelPath, true), - (relPath, doesntStartsWithRelPath, false), - (relPath, absPathEndingLikeRel, false), - (relPath, absPath, false) - ) - - forAll(paths) { (p1, p2, res) => - p1.endsWith(p2) shouldBe res - p1.endsWith(p2.toString) shouldBe res - } - } - - it should "implement toUri" in { - val file = new NioGcsPath(Array("some", "file"), true, false) - val uri = file.toUri - uri.toString shouldBe "gs://some/file" - } - -} diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/RefreshTokenModeSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/RefreshTokenModeSpec.scala deleted file mode 100644 index f959dcad5..000000000 --- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/RefreshTokenModeSpec.scala +++ /dev/null @@ -1,26 +0,0 @@ -package cromwell.filesystems.gcs - -import org.scalatest.{FlatSpec, Matchers} - -class RefreshTokenModeSpec extends FlatSpec with Matchers { - - val refreshToken = RefreshTokenMode(name = "bar", clientId = "secret-id", clientSecret = "secret-secret") - - behavior of "RefreshTokenMode" - - it should "assert good workflow options" in { - val goodOptions = GoogleOptionsMap(Map("refresh_token" -> "token")) - refreshToken.assertWorkflowOptions(goodOptions) - } - - it should "fail to assert bad workflow options" in { - val badOptions = GoogleOptionsMap(Map("fresh_tokin" -> "broken")) - val noOptions = GoogleOptionsMap(Map.empty[String, String]) - - List(badOptions, noOptions).foreach { option => - the[IllegalArgumentException] thrownBy { - refreshToken.assertWorkflowOptions(option) - } should have message s"Missing parameters in workflow options: refresh_token" - } - } -} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 55b7046b0..db7fd5ceb 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -14,7 +14,8 @@ object Dependencies { lazy val sprayJsonV = "1.3.2" lazy val akkaV = "2.4.9" lazy val slickV = "3.1.1" - lazy val googleClientApiV = "1.20.0" + lazy val googleClientApiV = "1.22.0" + lazy val googleGenomicsServicesApiV = "1.20.0" lazy val betterFilesV = "2.16.0" lazy val catsV = "0.7.2" @@ -74,10 +75,10 @@ object Dependencies { ) private val googleCloudDependencies = List( - "com.google.gcloud" % "gcloud-java" % "0.0.9", - "com.google.oauth-client" % "google-oauth-client" % googleClientApiV, - "com.google.cloud.bigdataoss" % "gcsio" % "1.4.4", - "com.google.apis" % "google-api-services-genomics" % ("v1alpha2-rev14-" + googleClientApiV) + "com.google.apis" % "google-api-services-genomics" % ("v1alpha2-rev14-" + googleGenomicsServicesApiV), + "com.google.cloud" % "google-cloud-nio" % "0.3.0" + exclude("com.google.api.grpc", "grpc-google-common-protos") + exclude("com.google.cloud.datastore", "datastore-v1-protos") ) private val dbmsDependencies = List( @@ -95,7 +96,9 @@ object Dependencies { // Sub-project dependencies, added in addition to any dependencies inherited from .dependsOn(). - val gcsFileSystemDependencies = baseDependencies ++ googleApiClientDependencies ++ googleCloudDependencies + val gcsFileSystemDependencies = baseDependencies ++ googleApiClientDependencies ++ googleCloudDependencies ++ List ( + "com.github.pathikrit" %% "better-files" % betterFilesV + ) val databaseSqlDependencies = baseDependencies ++ slickDependencies ++ dbmsDependencies @@ -133,7 +136,6 @@ object Dependencies { "org.webjars" % "swagger-ui" % "2.1.1", "commons-codec" % "commons-codec" % "1.10", "commons-io" % "commons-io" % "2.5", - "com.github.pathikrit" %% "better-files" % betterFilesV, "io.swagger" % "swagger-parser" % "1.0.22" % Test, "org.yaml" % "snakeyaml" % "1.17" % Test ) ++ sprayServerDependencies diff --git a/project/Merging.scala b/project/Merging.scala index 0591a09f9..2c941a76e 100644 --- a/project/Merging.scala +++ b/project/Merging.scala @@ -25,10 +25,15 @@ object Merging { MergeStrategy.filterDistinctLines case ("spring.schemas" :: Nil) | ("spring.handlers" :: Nil) => MergeStrategy.filterDistinctLines + case "io.netty.versions.properties" :: Nil => + MergeStrategy.first + case "maven" :: "com.google.guava" :: xs => + MergeStrategy.first case _ => MergeStrategy.deduplicate } case "asm-license.txt" | "overview.html" | "cobertura.properties" => MergeStrategy.discard + case _ => MergeStrategy.deduplicate } } \ No newline at end of file diff --git a/src/bin/travis/testCentaurJes.sh b/src/bin/travis/testCentaurJes.sh index 04813698d..0c92423c3 100755 --- a/src/bin/travis/testCentaurJes.sh +++ b/src/bin/travis/testCentaurJes.sh @@ -69,7 +69,7 @@ EXIT_CODE="${PIPESTATUS[0]}" export WORKFLOW_ID=`grep "SingleWorkflowRunnerActor: Workflow submitted " log.txt | perl -pe 's/\e\[?.*?[\@-~]//g' | cut -f7 -d" "` # Grab the Centaur log from GCS and cat it so we see it in the main travis log. -export CENTAUR_LOG_PATH="gs://cloud-cromwell-dev/cromwell_execution/travis/centaur/${WORKFLOW_ID}/call-centaur//cromwell_root/logs/centaur.log" +export CENTAUR_LOG_PATH="gs://cloud-cromwell-dev/cromwell_execution/travis/centaur/${WORKFLOW_ID}/call-centaur/cromwell_root/logs/centaur.log" gsutil cp ${CENTAUR_LOG_PATH} centaur.log cat centaur.log echo "More logs for this run are available at https://console.cloud.google.com/storage/browser/cloud-cromwell-dev/cromwell_execution/travis/centaur/${WORKFLOW_ID}/call-centaur/" diff --git a/src/test/scala/cromwell/CromwellCommandLineSpec.scala b/src/test/scala/cromwell/CromwellCommandLineSpec.scala index 42f03fdfc..0c0593db7 100644 --- a/src/test/scala/cromwell/CromwellCommandLineSpec.scala +++ b/src/test/scala/cromwell/CromwellCommandLineSpec.scala @@ -1,7 +1,7 @@ package cromwell import better.files._ -import cromwell.core.PathFactory._ +import cromwell.core.path.PathImplicits._ import cromwell.util.SampleWdl import cromwell.util.SampleWdl.ThreeStep import org.scalatest.{FlatSpec, Matchers} diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala index 75f2f779a..f353931d5 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala @@ -39,7 +39,7 @@ case class HtCondorBackendFactory(name: String, configurationDescriptor: Backend jobPaths.stderr.toAbsolutePath.toString ) - new SharedFileSystemExpressionFunctions(HtCondorJobExecutionActor.fileSystems, callContext) + new SharedFileSystemExpressionFunctions(HtCondorJobExecutionActor.pathBuilders, callContext) } private def resolveCacheProviderProps(workflowOptions: WorkflowOptions) = { diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala index 400c6a55f..72794ba11 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala @@ -1,6 +1,5 @@ package cromwell.backend.impl.htcondor -import java.nio.file.FileSystems import java.nio.file.attribute.PosixFilePermission import java.util.UUID @@ -11,6 +10,7 @@ import cromwell.backend.impl.htcondor.caching.CacheActor._ import cromwell.backend.impl.htcondor.caching.localization.CachedResultLocalization import cromwell.backend.io.JobPaths import cromwell.backend.sfs.{SharedFileSystem, SharedFileSystemExpressionFunctions} +import cromwell.core.path.{DefaultPathBuilder, PathBuilder} import cromwell.services.keyvalue.KeyValueServiceActor._ import cromwell.services.metadata.CallMetadataKeys import org.apache.commons.codec.digest.DigestUtils @@ -20,6 +20,7 @@ import wdl4s.types.{WdlArrayType, WdlFileType} import wdl4s.util.TryUtil import wdl4s.values.WdlArray +import cromwell.core.path.JavaWriterImplicits._ import scala.concurrent.{Future, Promise} import scala.sys.process.ProcessLogger import scala.util.{Failure, Success, Try} @@ -27,7 +28,7 @@ import scala.util.{Failure, Success, Try} object HtCondorJobExecutionActor { val HtCondorJobIdKey = "htCondor_job_id" - val fileSystems = List(FileSystems.getDefault) + val pathBuilders = List(DefaultPathBuilder) def props(jobDescriptor: BackendJobDescriptor, configurationDescriptor: BackendConfigurationDescriptor, serviceRegistryActor: ActorRef, cacheActorProps: Option[Props]): Props = Props(new HtCondorJobExecutionActor(jobDescriptor, configurationDescriptor, serviceRegistryActor, cacheActorProps)) @@ -41,9 +42,9 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor import HtCondorJobExecutionActor._ import better.files._ - import cromwell.core.PathFactory._ private val tag = s"CondorJobExecutionActor-${jobDescriptor.call.fullyQualifiedName}:" + override val pathBuilders: List[PathBuilder] = HtCondorJobExecutionActor.pathBuilders implicit val executionContext = context.dispatcher @@ -72,7 +73,7 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor private lazy val stderrWriter = extProcess.tailedWriter(100, submitFileStderr) private val call = jobDescriptor.key.call - private val callEngineFunction = SharedFileSystemExpressionFunctions(jobPaths, fileSystems) + private val callEngineFunction = SharedFileSystemExpressionFunctions(jobPaths, pathBuilders) private val lookup = jobDescriptor.inputs.apply _ @@ -276,7 +277,7 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor executionDir.toString.toFile.createIfNotExists(asDirectory = true, createParents = true) log.debug("{} Resolving job command", tag) - val command = localizeInputs(jobPaths.callInputsRoot, runtimeAttributes.dockerImage.isDefined, fileSystems, jobDescriptor.inputs) flatMap { + val command = localizeInputs(jobPaths.callInputsRoot, runtimeAttributes.dockerImage.isDefined, jobDescriptor.inputs) flatMap { localizedInputs => resolveJobCommand(localizedInputs) } diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala index 2c59716a7..5b5d014cf 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala @@ -5,8 +5,9 @@ import java.nio.file.{Files, Path} import better.files._ import com.typesafe.scalalogging.StrictLogging import cromwell.backend.impl.htcondor -import cromwell.core.PathFactory.{EnhancedPath, FlushingAndClosingWriter} -import cromwell.core.{TailedWriter, UntailedWriter} +import cromwell.core.path.{TailedWriter, UntailedWriter} +import cromwell.core.path.PathImplicits._ +import cromwell.core.path.JavaWriterImplicits._ import scala.sys.process._ diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala index 589554e94..eafb3a338 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala @@ -14,6 +14,7 @@ import cromwell.backend.impl.htcondor.caching.model.CachedExecutionResult import cromwell.backend.io.JobPaths import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendSpec} import cromwell.core._ +import cromwell.core.path.{PathWriter, TailedWriter, UntailedWriter} import cromwell.services.keyvalue.KeyValueServiceActor.{KvGet, KvPair, KvPut} import org.mockito.Matchers._ import org.mockito.Mockito diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GenomicsFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GenomicsFactory.scala index 7285171b5..574e58153 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GenomicsFactory.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GenomicsFactory.scala @@ -2,22 +2,22 @@ package cromwell.backend.impl.jes import java.net.URL -import com.google.api.client.auth.oauth2.Credential -import com.google.api.client.http.HttpTransport -import com.google.api.client.json.JsonFactory import com.google.api.services.genomics.Genomics +import cromwell.core.WorkflowOptions +import cromwell.filesystems.gcs.auth.GoogleAuthMode -object GenomicsFactory { +case class GenomicsFactory(applicationName: String, authMode: GoogleAuthMode, endpointUrl: URL) { - def apply(applicationName: String, credential: Credential, endpointUrl: URL): Genomics = { - GoogleGenomics.from(applicationName, endpointUrl, credential, credential.getJsonFactory, credential.getTransport) - } + def withOptions(options: WorkflowOptions) = { + val credential = authMode.credential(options) - // Wrapper object around Google's Genomics class providing a convenience 'from' "method" - object GoogleGenomics { - def from(applicationName: String, endpointUrl: URL, credential: Credential, jsonFactory: JsonFactory, httpTransport: HttpTransport): Genomics = { - new Genomics.Builder(httpTransport, jsonFactory, credential).setApplicationName(applicationName).setRootUrl(endpointUrl.toString).build - } + new Genomics.Builder( + credential.getTransport, + credential.getJsonFactory, + credential) + .setApplicationName(applicationName) + .setRootUrl(endpointUrl.toString) + .build } } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala index 6e697032c..e0e93e839 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala @@ -9,21 +9,22 @@ import better.files._ import cats.instances.future._ import cats.syntax.functor._ import com.google.api.client.googleapis.json.GoogleJsonResponseException +import com.google.cloud.storage.contrib.nio.CloudStoragePath import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse} import cromwell.backend.BackendLifecycleActor.AbortJobCommand +import cromwell.backend._ import cromwell.backend.async.AsyncBackendJobExecutionActor.{ExecutionMode, JobId} import cromwell.backend.async.{AbortedExecutionHandle, AsyncBackendJobExecutionActor, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle, NonRetryableExecution, SuccessfulExecutionHandle} -import cromwell.backend.impl.jes.JesImplicits.PathString import cromwell.backend.impl.jes.JesJobExecutionActor.JesOperationIdKey import cromwell.backend.impl.jes.RunStatus.TerminalRunStatus import cromwell.backend.impl.jes.io._ import cromwell.backend.impl.jes.statuspolling.JesPollingActorClient -import cromwell.backend.{AttemptedLookupResult, BackendJobDescriptor, BackendWorkflowDescriptor, PreemptedException} +import cromwell.backend.{BackendJobDescriptor, BackendWorkflowDescriptor, PreemptedException} import cromwell.core.Dispatcher.BackendDispatcher import cromwell.core._ import cromwell.core.logging.JobLogging +import cromwell.core.path.proxy.PathProxy import cromwell.core.retry.{Retry, SimpleExponentialBackoff} -import cromwell.filesystems.gcs.NioGcsPath import cromwell.services.keyvalue.KeyValueServiceActor._ import cromwell.services.metadata._ import wdl4s.AstTools._ @@ -31,7 +32,6 @@ import wdl4s.WdlExpression.ScopedLookupFunction import wdl4s._ import wdl4s.command.ParameterCommandPart import wdl4s.expression.NoFunctions -import wdl4s.util.TryUtil import wdl4s.values._ import scala.concurrent.duration._ @@ -103,9 +103,9 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes override lazy val retryable = jobDescriptor.key.attempt <= runtimeAttributes.preemptible private lazy val cmdInput = - JesFileInput(ExecParamName, jesCallPaths.gcsExecPath.toString, Paths.get(jesCallPaths.gcsExecFilename), workingDisk) + JesFileInput(ExecParamName, jesCallPaths.gcsExecPath.toUri.toString, Paths.get(jesCallPaths.gcsExecFilename), workingDisk) private lazy val jesCommandLine = s"/bin/bash ${cmdInput.containerPath}" - private lazy val rcJesOutput = JesFileOutput(returnCodeFilename, returnCodeGcsPath.toString, Paths.get(returnCodeFilename), workingDisk) + private lazy val rcJesOutput = JesFileOutput(returnCodeFilename, returnCodeGcsPath.toUri.toString, Paths.get(returnCodeFilename), workingDisk) private lazy val standardParameters = Seq(rcJesOutput) private lazy val returnCodeContents = Try(File(returnCodeGcsPath).contentAsString) @@ -132,18 +132,18 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes private def globOutputPath(glob: String) = callRootPath.resolve(s"glob-${glob.md5Sum}/") private def gcsAuthParameter: Option[JesInput] = { - if (jesAttributes.gcsFilesystemAuth.requiresAuthFile || dockerConfiguration.isDefined) - Option(JesLiteralInput(ExtraConfigParamName, jesCallPaths.gcsAuthFilePath.toString)) + if (jesAttributes.auths.gcs.requiresAuthFile || dockerConfiguration.isDefined) + Option(JesLiteralInput(ExtraConfigParamName, jesCallPaths.gcsAuthFilePath.toUri.toString)) else None } private lazy val callContext = CallContext( callRootPath, - jesStdoutFile.toString, - jesStderrFile.toString + jesStdoutFile.toUri.toString, + jesStderrFile.toUri.toString ) - private[jes] lazy val callEngineFunctions = new JesExpressionFunctions(List(jesCallPaths.gcsFileSystem), callContext) + private[jes] lazy val callEngineFunctions = new JesExpressionFunctions(List(jesCallPaths.gcsPathBuilder), callContext) private val lookup: ScopedLookupFunction = { val declarations = workflowDescriptor.workflowNamespace.workflow.declarations ++ call.task.declarations @@ -170,10 +170,12 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes * relativeLocalizationPath("gs://some/bucket/foo.txt") -> "some/bucket/foo.txt" */ private def relativeLocalizationPath(file: WdlFile): WdlFile = { - Try(getPath(file.value)) match { - case Success(gcsPath: NioGcsPath) => WdlFile(gcsPath.bucket + "/" + gcsPath.objectName, file.isGlob) - case Success(gcsPath) => file - case Failure(e) => file + getPath(file.value) match { + case Success(path) => { + val value: WdlSource = path.toUri.getHost + path.toUri.getPath + WdlFile(value, file.isGlob) + } + case _ => file } } @@ -249,8 +251,8 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes // Create the mappings. GLOB mappings require special treatment (i.e. stick everything matching the glob in a folder) wdlFileOutputs.distinct map { wdlFile => val destination = wdlFile match { - case WdlSingleFile(filePath) => callRootPath.resolve(filePath).toString - case WdlGlobFile(filePath) => globOutputPath(filePath).toString + case WdlSingleFile(filePath) => callRootPath.resolve(filePath.stripPrefix("/")).toUri.toString + case WdlGlobFile(filePath) => globOutputPath(filePath).toUri.toString } val (relpath, disk) = relativePathAndAttachedDisk(wdlFile.value, runtimeAttributes.disks) JesFileOutput(makeSafeJesReferenceName(wdlFile.value), destination, relpath, disk) @@ -285,14 +287,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes |echo $$? > $rcPath """.stripMargin.trim - def writeScript(): Future[Unit] = Future { File(jesCallPaths.gcsExecPath).write(fileContent) } void - - implicit val system = context.system - Retry.withRetry( - writeScript, - isTransient = isTransientJesException, - isFatal = isFatalJesException - ) + Future(File(jesCallPaths.gcsExecPath).write(fileContent)) void } private def googleProject(descriptor: BackendWorkflowDescriptor): String = { @@ -305,7 +300,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes runIdForResumption, jobDescriptor = jobDescriptor, runtimeAttributes = runtimeAttributes, - callRootPath = callRootPath.toString, + callRootPath = callRootPath.toUri.toString, commandLine = jesCommandLine, logFileName = jesLogFilename, jesParameters, @@ -457,59 +452,35 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), metadataKeyValues) } - private def customLookupFunction(alreadyGeneratedOutputs: Map[String, WdlValue])(toBeLookedUp: String): WdlValue = alreadyGeneratedOutputs.getOrElse(toBeLookedUp, lookup(toBeLookedUp)) - + /** + * Attempts to find the JES file output corresponding to the WdlValue + */ private[jes] def wdlValueToGcsPath(jesOutputs: Seq[JesFileOutput])(value: WdlValue): WdlValue = { def toGcsPath(wdlFile: WdlFile) = jesOutputs collectFirst { case o if o.name == makeSafeJesReferenceName(wdlFile.valueString) => WdlFile(o.gcs) } getOrElse value + value match { case wdlArray: WdlArray => wdlArray map wdlValueToGcsPath(jesOutputs) case wdlMap: WdlMap => wdlMap map { case (k, v) => wdlValueToGcsPath(jesOutputs)(k) -> wdlValueToGcsPath(jesOutputs)(v) } - case file: WdlFile => if (file.value.isGcsUrl) file else toGcsPath(file) + case file: WdlFile => toGcsPath(file) case other => other } } - private def outputLookup(taskOutput: TaskOutput, currentList: Seq[AttemptedLookupResult]) = for { - /** - * This will evaluate the task output expression and coerces it to the task output's type. - * If the result is a WdlFile, then attempt to find the JesOutput with the same path and - * return a WdlFile that represents the GCS path and not the local path. For example, - * - * 
-    * output {
-    *   File x = "out" + ".txt"
-    * }
-    *
- * - * "out" + ".txt" is evaluated to WdlString("out.txt") and then coerced into a WdlFile("out.txt") - * Then, via wdlFileToGcsPath(), we attempt to find the JesOutput with .name == "out.txt". - * If it is found, then WdlFile("gs://some_bucket/out.txt") will be returned. - */ - wdlValue <- taskOutput.requiredExpression.evaluate(customLookupFunction(currentList.toLookupMap), callEngineFunctions) - coercedValue <- taskOutput.wdlType.coerceRawValue(wdlValue) - value = wdlValueToGcsPath(generateJesOutputs(jobDescriptor))(coercedValue) - } yield value - - - private def outputFoldingFunction: (Seq[AttemptedLookupResult], TaskOutput) => Seq[AttemptedLookupResult] = { - (currentList: Seq[AttemptedLookupResult], taskOutput: TaskOutput) => { - currentList ++ Seq(AttemptedLookupResult(taskOutput.name, outputLookup(taskOutput, currentList))) - } - } - private def postProcess: Try[JobOutputs] = { - val outputs = call.task.outputs - val outputMappings = outputs.foldLeft(Seq.empty[AttemptedLookupResult])(outputFoldingFunction).map(_.toPair).toMap - TryUtil.sequenceMap(outputMappings) map { outputMap => - outputMap mapValues { v => JobOutput(v) } - } + def wdlValueToSuccess(value: WdlValue): Try[WdlValue] = Success(value) + + OutputEvaluator.evaluateOutputs( + jobDescriptor, + callEngineFunctions, + (wdlValueToSuccess _).compose(wdlValueToGcsPath(generateJesOutputs(jobDescriptor))) + ) } - private def handleSuccess(outputMappings: Try[JobOutputs], returnCode: Int, jobDetritusFiles: Map[String, String], executionHandle: ExecutionHandle, events: Seq[ExecutionEvent]): ExecutionHandle = { + private def handleSuccess(outputMappings: Try[JobOutputs], returnCode: Int, jobDetritusFiles: Map[String, Path], executionHandle: ExecutionHandle, events: Seq[ExecutionEvent]): ExecutionHandle = { outputMappings match { case Success(outputs) => SuccessfulExecutionHandle(outputs, returnCode, jobDetritusFiles, events) case Failure(ex: CromwellAggregatedException) if ex.throwables collectFirst { case s: SocketTimeoutException => s } isDefined => @@ -591,7 +562,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes val badReturnCodeMessage = s"Call ${jobDescriptor.key}: return code was ${returnCode.getOrElse("(none)")}" FailedNonRetryableExecutionHandle(new RuntimeException(badReturnCodeMessage), returnCode.toOption).future case success: RunStatus.Success => - handleSuccess(postProcess, returnCode.get, jesCallPaths.detritusPaths.mapValues(_.toString), handle, success.eventList).future + handleSuccess(postProcess, returnCode.get, jesCallPaths.detritusPaths, handle, success.eventList).future case RunStatus.Failed(errorCode, errorMessage, _, _, _, _) => handleFailure(errorCode, errorMessage) } } catch { @@ -611,8 +582,8 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes * @param gcsPath The input path * @return A path which is unique per input path */ - private def localFilePathFromCloudStoragePath(mountPoint: Path, gcsPath: NioGcsPath): Path = { - mountPoint.resolve(gcsPath.bucket).resolve(gcsPath.objectName) + private def localFilePathFromCloudStoragePath(mountPoint: Path, gcsPath: CloudStoragePath): Path = { + mountPoint.resolve(gcsPath.bucket()).resolve(gcsPath.toUri.getPath.stripPrefix("/")) } /** @@ -625,11 +596,14 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes private[jes] def gcsPathToLocal(wdlValue: WdlValue): WdlValue = { wdlValue match { case wdlFile: WdlFile => - Try(getPath(wdlFile.valueString)) match { - case Success(gcsPath: NioGcsPath) => + getPath(wdlFile.valueString) match { + case Success(gcsPath: CloudStoragePath) => WdlFile(localFilePathFromCloudStoragePath(workingDisk.mountPoint, gcsPath).toString, wdlFile.isGlob) - case Success(otherPath) => wdlValue - case Failure(e) => wdlValue + case Success(proxy: PathProxy) => + proxy.unbox(classOf[CloudStoragePath]) map { gcsPath => + WdlFile(localFilePathFromCloudStoragePath(workingDisk.mountPoint, gcsPath).toString, wdlFile.isGlob) + } getOrElse wdlValue + case _ => wdlValue } case wdlArray: WdlArray => wdlArray map gcsPathToLocal case wdlMap: WdlMap => wdlMap map { case (k, v) => gcsPathToLocal(k) -> gcsPathToLocal(v) } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala index 2e1d00b0e..066a4cb52 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala @@ -6,23 +6,18 @@ import cats.data._ import cats.data.Validated._ import cats.syntax.cartesian._ import com.typesafe.config.Config -import cromwell.backend.impl.jes.JesImplicits.GoogleAuthWorkflowOptions -import cromwell.core.WorkflowOptions -import cromwell.filesystems.gcs.{GoogleAuthMode, GoogleConfiguration} +import cromwell.backend.impl.jes.authentication.JesAuths +import cromwell.core.ErrorOr._ +import cromwell.filesystems.gcs.GoogleConfiguration import lenthall.config.ValidatedConfig._ import net.ceedubs.ficus.Ficus._ -import cromwell.core.ErrorOr._ import wdl4s.ExceptionWithErrors case class JesAttributes(project: String, - genomicsAuth: GoogleAuthMode, - gcsFilesystemAuth: GoogleAuthMode, + auths: JesAuths, executionBucket: String, endpointUrl: URL, - maxPollingInterval: Int) { - def genomicsCredential(options: WorkflowOptions) = genomicsAuth.credential(options.toGoogleAuthOptions) - def gcsCredential(options: WorkflowOptions) = gcsFilesystemAuth.credential(options.toGoogleAuthOptions) -} + maxPollingInterval: Int) object JesAttributes { @@ -54,7 +49,7 @@ object JesAttributes { (_, _, _, _, _) } flatMap { case (p, b, u, genomicsName, gcsName) => (googleConfig.auth(genomicsName) |@| googleConfig.auth(gcsName)) map { case (genomicsAuth, gcsAuth) => - JesAttributes(p, genomicsAuth, gcsAuth, b, u, maxPollingInterval) + JesAttributes(p, JesAuths(genomicsAuth, gcsAuth), b, u, maxPollingInterval) } } match { case Valid(r) => r diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala index 92f653c10..6d192f0a4 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala @@ -64,7 +64,7 @@ case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor initializationData: Option[BackendInitializationData]): WdlStandardLibraryFunctions = { val jesCallPaths = initializationData.toJes.get.workflowPaths.toJesCallPaths(jobKey) - new JesExpressionFunctions(List(jesCallPaths.gcsFileSystem), jesCallPaths.callContext) + new JesExpressionFunctions(List(jesCallPaths.gcsPathBuilder), jesCallPaths.callContext) } override def getExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config, diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala index 078e2e081..e84e4d3ec 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala @@ -5,7 +5,7 @@ import java.nio.file.Path import akka.actor.{ActorRef, Props} import cromwell.backend.callcaching.CacheHitDuplicating import cromwell.backend.{BackendCacheHitCopyingActor, BackendJobDescriptor} -import cromwell.core.PathCopier +import cromwell.core.path.PathCopier import cromwell.core.logging.JobLogging case class JesCacheHitCopyingActor(override val jobDescriptor: BackendJobDescriptor, diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCallPaths.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCallPaths.scala index f18daecaa..97dc3be39 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCallPaths.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCallPaths.scala @@ -2,20 +2,17 @@ package cromwell.backend.impl.jes import java.nio.file.Path -import cromwell.backend.impl.jes.authentication.JesCredentials +import akka.actor.ActorSystem import cromwell.backend.io.JobPaths import cromwell.backend.io.JobPaths._ import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} import cromwell.core.CallContext import cromwell.services.metadata.CallMetadataKeys -import scala.concurrent.ExecutionContext - object JesCallPaths { def apply(jobKey: BackendJobDescriptorKey, workflowDescriptor: BackendWorkflowDescriptor, - jesConfiguration: JesConfiguration, - credentials: JesCredentials)(implicit ec: ExecutionContext): JesCallPaths = { - new JesCallPaths(jobKey, workflowDescriptor, jesConfiguration, credentials) + jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem): JesCallPaths = { + new JesCallPaths(jobKey, workflowDescriptor, jesConfiguration) } val JesLogPathKey = "jesLog" @@ -23,9 +20,8 @@ object JesCallPaths { } class JesCallPaths(jobKey: BackendJobDescriptorKey, workflowDescriptor: BackendWorkflowDescriptor, - jesConfiguration: JesConfiguration, - credentials: JesCredentials)(implicit ec: ExecutionContext) extends - JesWorkflowPaths(workflowDescriptor, jesConfiguration, credentials)(ec) { + jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem) extends + JesWorkflowPaths(workflowDescriptor, jesConfiguration)(actorSystem) { val jesLogBasename = { val index = jobKey.index.map(s => s"-$s").getOrElse("") diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala index 6657250b0..88ece5bec 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala @@ -2,14 +2,34 @@ package cromwell.backend.impl.jes import cromwell.backend.BackendConfigurationDescriptor import cromwell.backend.impl.jes.authentication.JesDockerCredentials +import cromwell.backend.impl.jes.io._ import cromwell.core.DockerConfiguration -import cromwell.filesystems.gcs.GoogleConfiguration +import cromwell.core.path.CustomRetryParams +import cromwell.core.retry.SimpleExponentialBackoff +import cromwell.filesystems.gcs.{GoogleConfiguration, RetryableGcsPathBuilderFactory} + +import scala.concurrent.duration._ +import scala.language.postfixOps + +object JesConfiguration { + val GcsRetryParams = CustomRetryParams( + timeout = Duration.Inf, + maxRetries = Option(3), + backoff = SimpleExponentialBackoff(1 seconds, 3 seconds, 1.5D), + isTransient = isTransientJesException, + isFatal = isFatalJesException + ) +} class JesConfiguration(val configurationDescriptor: BackendConfigurationDescriptor) { + private val googleConfig = GoogleConfiguration(configurationDescriptor.globalConfig) + val root = configurationDescriptor.backendConfig.getString("root") - val googleConfig = GoogleConfiguration(configurationDescriptor.globalConfig) val jesAttributes = JesAttributes(googleConfig, configurationDescriptor.backendConfig) + val jesAuths = jesAttributes.auths + val gcsPathBuilderFactory = RetryableGcsPathBuilderFactory(jesAuths.gcs, customRetryParams = JesConfiguration.GcsRetryParams) + val genomicsFactory = GenomicsFactory(googleConfig.applicationName, jesAuths.genomics, jesAttributes.endpointUrl) val dockerCredentials = DockerConfiguration.build(configurationDescriptor.backendConfig).dockerCredentials map JesDockerCredentials.apply - val needAuthFileUpload = jesAttributes.gcsFilesystemAuth.requiresAuthFile || dockerCredentials.isDefined + val needAuthFileUpload = jesAuths.gcs.requiresAuthFile || dockerCredentials.isDefined } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala index 823108ca8..b2260f5c8 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala @@ -1,42 +1,44 @@ package cromwell.backend.impl.jes -import java.nio.file.{FileSystem, Path} +import java.nio.file.{Files, Path} -import better.files._ import cromwell.backend.wdl.{PureFunctions, ReadLikeFunctions, WriteFunctions} -import cromwell.backend.impl.jes.JesImplicits.PathString import cromwell.core.CallContext -import cromwell.filesystems.gcs.GcsFileSystem +import cromwell.core.path.PathBuilder +import cromwell.filesystems.gcs.GcsPathBuilder import wdl4s.expression.WdlStandardLibraryFunctions import wdl4s.values._ +import scala.collection.JavaConverters._ import scala.language.postfixOps import scala.util.{Success, Try} -class JesExpressionFunctions(override val fileSystems: List[FileSystem], - context: CallContext - ) extends WdlStandardLibraryFunctions with PureFunctions with ReadLikeFunctions with WriteFunctions { - import JesExpressionFunctions.EnhancedPath +class JesExpressionFunctions(override val pathBuilders: List[PathBuilder], context: CallContext) + extends WdlStandardLibraryFunctions with PureFunctions with ReadLikeFunctions with WriteFunctions { private def globDirectory(glob: String): String = s"glob-${glob.md5Sum}/" override def globPath(glob: String): String = context.root.resolve(globDirectory(glob)).toString override def glob(path: String, pattern: String): Seq[String] = { - File(path.toAbsolutePath(fileSystems).asDirectory). - glob("**/*") map { _.pathAsString } filterNot { _.toString == path } toSeq + /* Globbing is not implemented in the gcs-nio implementation yet (specifically PathMatcher) at this time (09/2016) + * which is fine here since all files in the globing directory have already be matched against the glob pattern by JES. + * + * Also, better.file.File can't be used here because it calls toAbsolutePath on the path in File.apply, which adds a leading "/" in the path. + * This makes the newDirectoryStream implementation of CloudStorageFileSystemProvider return nothing + * because it doesn't call toRealPath for this method before doing I/O and hence does not remove the "/" prefix (bug ?) + * See com.google.cloud.storage.contrib.nio.CloudStorageConfiguration#stripPrefixSlash() + */ + val directory = context.root.resolve(s"glob-${pattern.md5Sum}/").toRealPath() + Files.list(directory).iterator().asScala filterNot { Files.isDirectory(_) } map { _.toUri.toString } toSeq } - override def preMapping(str: String): String = if (!GcsFileSystem.isAbsoluteGcsPath(str)) context.root.resolve(str).toString else str + override def preMapping(str: String): String = if (!GcsPathBuilder.isValidGcsUrl(str)) { + context.root.resolve(str.stripPrefix("/")).toUri.toString + } else str override def stdout(params: Seq[Try[WdlValue]]) = Success(WdlFile(context.stdout)) override def stderr(params: Seq[Try[WdlValue]]) = Success(WdlFile(context.stderr)) override val writeDirectory: Path = context.root } - -object JesExpressionFunctions { - implicit class EnhancedPath(val path: Path) extends AnyVal { - def asDirectory = path.toString.toDirectory(path.getFileSystem) - } -} diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala index 038c615dd..e6c3e2d7f 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala @@ -8,7 +8,8 @@ import cats.instances.future._ import cats.syntax.functor._ import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor, BackendWorkflowFinalizationActor} import cromwell.core.Dispatcher.IoDispatcher -import cromwell.core.{ExecutionStore, OutputStore, PathCopier} +import cromwell.core.path.PathCopier +import cromwell.core.{ExecutionStore, OutputStore} import wdl4s.Call import scala.concurrent.Future diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesImplicits.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesImplicits.scala deleted file mode 100644 index 6c722c756..000000000 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesImplicits.scala +++ /dev/null @@ -1,41 +0,0 @@ -package cromwell.backend.impl.jes - -import java.nio.file.{FileSystem, Path} - -import cromwell.core.{PathFactory, WorkflowOptions} -import cromwell.filesystems.gcs.GoogleAuthMode.GoogleAuthOptions -import cromwell.filesystems.gcs.{GcsFileSystem, GoogleAuthMode} - -import scala.util.Try - -object JesImplicits { - implicit class GoogleAuthWorkflowOptions(val workflowOptions: WorkflowOptions) extends AnyVal { - def toGoogleAuthOptions: GoogleAuthMode.GoogleAuthOptions = new GoogleAuthOptions { - override def get(key: String): Try[String] = workflowOptions.get(key) - } - } - - object PathBuilder extends PathFactory - - implicit class PathString(val str: String) extends AnyVal { - def isGcsUrl: Boolean = str.startsWith("gs://") - def isUriWithProtocol: Boolean = "^[a-z]+://".r.findFirstIn(str).nonEmpty - - def toPath(fss: List[FileSystem]): Path = PathBuilder.buildPath(str, fss) - def toPath(fs: FileSystem): Path = str.toPath(List(fs)) - - def toAbsolutePath(fss: List[FileSystem]): Path = str.toPath(fss).toAbsolutePath - def toAbsolutePath(fs: FileSystem): Path = str.toAbsolutePath(List(fs)) - - def toDirectory(fss: List[FileSystem]): Path = buildPathAsDirectory(str, fss) - def toDirectory(fs: FileSystem): Path = str.toDirectory(List(fs)) - - // TODO this needs to go away because it's gcs specific. Replacing gcs FS with google implementation (when available) will take care of it - private def buildPathAsDirectory(rawString: String, fileSystems: List[FileSystem]): Path = { - PathBuilder.findFileSystem(rawString, fileSystems, { - case fs: GcsFileSystem => Try(fs.getPathAsDirectory(rawString)) - case fs => Try(fs.getPath(rawString)) - }) - } - } -} diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala index e76a62e9d..58eacb45b 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala @@ -7,21 +7,20 @@ import cats.instances.future._ import cats.syntax.functor._ import com.google.api.services.genomics.Genomics import cromwell.backend.impl.jes.JesInitializationActor._ -import cromwell.backend.impl.jes.authentication.{GcsLocalizing, JesAuthInformation, JesCredentials} +import cromwell.backend.impl.jes.authentication.{GcsLocalizing, JesAuthInformation} import cromwell.backend.impl.jes.io._ import cromwell.backend.validation.RuntimeAttributesDefault import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.backend.{BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor} -import cromwell.core.Dispatcher.IoDispatcher import cromwell.core.WorkflowOptions -import cromwell.core.retry.Retry -import cromwell.filesystems.gcs.{ClientSecrets, GoogleAuthMode} +import cromwell.filesystems.gcs.auth.{ClientSecrets, GoogleAuthMode} import spray.json.JsObject import wdl4s.Call import wdl4s.types.{WdlBooleanType, WdlFloatType, WdlIntegerType, WdlStringType} import wdl4s.values.WdlValue import scala.concurrent.Future +import scala.language.postfixOps import scala.util.Try object JesInitializationActor { @@ -58,14 +57,11 @@ class JesInitializationActor(override val workflowDescriptor: BackendWorkflowDes private[jes] lazy val refreshTokenAuth: Option[JesAuthInformation] = { for { - clientSecrets <- List(jesConfiguration.jesAttributes.gcsFilesystemAuth) collectFirst { case s: ClientSecrets => s } + clientSecrets <- List(jesConfiguration.jesAttributes.auths.gcs) collectFirst { case s: ClientSecrets => s } token <- workflowDescriptor.workflowOptions.get(GoogleAuthMode.RefreshTokenOptionKey).toOption } yield GcsLocalizing(clientSecrets, token) } - private val iOExecutionContext = context.system.dispatchers.lookup(IoDispatcher) - - override protected def coerceDefaultRuntimeAttributes(options: WorkflowOptions): Try[Map[String, WdlValue]] = { RuntimeAttributesDefault.workflowOptionsDefault(options, JesRuntimeAttributes.coercionMap) } @@ -75,18 +71,13 @@ class JesInitializationActor(override val workflowDescriptor: BackendWorkflowDes */ override def beforeAll(): Future[Option[BackendInitializationData]] = { - val genomicsCredential = jesConfiguration.jesAttributes.genomicsCredential(workflowDescriptor.workflowOptions) - val gcsCredential = jesConfiguration.jesAttributes.gcsCredential(workflowDescriptor.workflowOptions) - - val jesCredentials = JesCredentials(genomicsCredential = genomicsCredential, gcsCredential = gcsCredential) def buildGenomics: Future[Genomics] = Future { - GenomicsFactory(jesConfiguration.googleConfig.applicationName, genomicsCredential, jesConfiguration.jesAttributes.endpointUrl) + jesConfiguration.genomicsFactory.withOptions(workflowDescriptor.workflowOptions) } for { - // generate single filesystem and genomics instances genomics <- buildGenomics - workflowPaths = new JesWorkflowPaths(workflowDescriptor, jesConfiguration, jesCredentials)(iOExecutionContext) + workflowPaths = new JesWorkflowPaths(workflowDescriptor, jesConfiguration)(context.system) _ <- if (jesConfiguration.needAuthFileUpload) writeAuthenticationFile(workflowPaths) else Future.successful(()) _ = publishWorkflowRoot(workflowPaths.workflowRootPath.toString) } yield Option(JesBackendInitializationData(workflowPaths, genomics)) @@ -95,12 +86,10 @@ class JesInitializationActor(override val workflowDescriptor: BackendWorkflowDes private def writeAuthenticationFile(workflowPath: JesWorkflowPaths): Future[Unit] = { generateAuthJson(jesConfiguration.dockerCredentials, refreshTokenAuth) map { content => val path = workflowPath.gcsAuthFilePath - val upload = () => Future(path.writeAsJson(content)) - workflowLogger.info(s"Creating authentication file for workflow ${workflowDescriptor.id} at \n ${path.toString}") - Retry.withRetry(upload, isFatal = isFatalJesException, isTransient = isTransientJesException)(context.system).void.recoverWith { + Future(path.writeAsJson(content)).void.recoverWith { case failure => Future.failed(new IOException("Failed to upload authentication file", failure)) - } + } void } getOrElse Future.successful(()) } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala index 4ded3e9d1..e30aa4e6f 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala @@ -10,6 +10,7 @@ import cromwell.backend.impl.jes.io.{JesAttachedDisk, JesWorkingDisk} import cromwell.core.logging.JobLogging import scala.language.postfixOps +import scala.util.Try trait JesJobCachingActorHelper extends JobCachingActorHelper { this: Actor with JobLogging => @@ -26,7 +27,7 @@ trait JesJobCachingActorHelper extends JobCachingActorHelper { def serviceRegistryActor: ActorRef - def getPath(str: String) = jesCallPaths.gcsFileSystem.getPath(str) + def getPath(str: String): Try[Path] = jesCallPaths.getPath(str) override lazy val configurationDescriptor = jesConfiguration.configurationDescriptor @@ -52,7 +53,7 @@ trait JesJobCachingActorHelper extends JobCachingActorHelper { // TODO: Move monitoring paths to JesCallPaths lazy val monitoringScript: Option[JesInput] = { jobDescriptor.workflowDescriptor.workflowOptions.get(WorkflowOptionKeys.MonitoringScript) map { path => - JesFileInput(s"$MonitoringParamName-in", getPath(path).toString, + JesFileInput(s"$MonitoringParamName-in", getPath(path).get.toUri.toString, JesWorkingDisk.MountPoint.resolve(JesMonitoringScript), workingDisk) } toOption } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala index 9b39c869a..6c3d2b27c 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala @@ -2,51 +2,57 @@ package cromwell.backend.impl.jes import java.nio.file.Path -import cromwell.backend.impl.jes.authentication.JesCredentials +import akka.actor.ActorSystem import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} +import cromwell.core.WorkflowOptions import cromwell.core.WorkflowOptions.FinalCallLogsDir -import cromwell.filesystems.gcs.{GcsFileSystem, GcsFileSystemProvider, GoogleAuthMode} +import cromwell.filesystems.gcs.{RetryableGcsPathBuilder, GcsPathBuilderFactory} -import scala.concurrent.ExecutionContext +import scala.language.postfixOps +import scala.util.Try object JesWorkflowPaths { private val GcsRootOptionKey = "jes_gcs_root" private val AuthFilePathOptionKey = "auth_bucket" def apply(workflowDescriptor: BackendWorkflowDescriptor, - jesConfiguration: JesConfiguration, - credentials: JesCredentials)(implicit ec: ExecutionContext) = { - new JesWorkflowPaths(workflowDescriptor, jesConfiguration, credentials) + jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem) = { + new JesWorkflowPaths(workflowDescriptor, jesConfiguration) } } class JesWorkflowPaths(workflowDescriptor: BackendWorkflowDescriptor, - jesConfiguration: JesConfiguration, - credentials: JesCredentials)(implicit ec: ExecutionContext) { + jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem) { - private val gcsStorage = GoogleAuthMode.buildStorage(credentials.gcsCredential, jesConfiguration.googleConfig.applicationName) - val gcsFileSystemProvider: GcsFileSystemProvider = GcsFileSystemProvider(gcsStorage)(ec) - val gcsFileSystem = GcsFileSystem(gcsFileSystemProvider) + private val rootString = workflowDescriptor.workflowOptions.getOrElse(JesWorkflowPaths.GcsRootOptionKey, jesConfiguration.root) + private val workflowOptions: WorkflowOptions = workflowDescriptor.workflowOptions - val rootPath: Path = - gcsFileSystem.getPath(workflowDescriptor.workflowOptions.getOrElse(JesWorkflowPaths.GcsRootOptionKey, jesConfiguration.root)) + val gcsPathBuilder: RetryableGcsPathBuilder = jesConfiguration.gcsPathBuilderFactory.withOptions(workflowOptions) - val workflowRootPath: Path = rootPath.resolve(workflowDescriptor.workflowNamespace.workflow.unqualifiedName) - .resolve(workflowDescriptor.id.toString) + def getPath(gcsUrl: String): Try[Path] = gcsPathBuilder.build(gcsUrl) + def getHash(gcsUrl: Path) = gcsPathBuilder.getHash(gcsUrl) - val finalCallLogsPath = workflowDescriptor.getWorkflowOption(FinalCallLogsDir) map { gcsFileSystem.getPath(_) } + val rootPath: Path = getPath(rootString) recover { + case ex => throw new Exception(s"Failed to : $rootString", ex) + } get + + val workflowRootPath: Path = rootPath.resolve(workflowDescriptor.workflowNamespace.workflow.unqualifiedName).resolve(s"${workflowDescriptor.id.toString}/") + + val finalCallLogsPath = workflowDescriptor.getWorkflowOption(FinalCallLogsDir) map getPath map { _.get } val gcsAuthFilePath: Path = { /* * This is an "exception". The filesystem used here is built from genomicsAuth * unlike everywhere else where the filesystem used is built from gcsFileSystemAuth */ - val genomicsStorage = GoogleAuthMode.buildStorage(credentials.genomicsCredential, jesConfiguration.googleConfig.applicationName) - val fileSystemWithGenomicsAuth = GcsFileSystem(GcsFileSystemProvider(genomicsStorage)(ec)) - val bucket = workflowDescriptor.workflowOptions.get(JesWorkflowPaths.AuthFilePathOptionKey) getOrElse workflowRootPath.toString + val genomicsCredentials = jesConfiguration.jesAuths.genomics + val bucket = workflowDescriptor.workflowOptions.get(JesWorkflowPaths.AuthFilePathOptionKey) getOrElse workflowRootPath.toUri.toString + val authBucket = GcsPathBuilderFactory(genomicsCredentials).withOptions(workflowOptions).build(bucket) recover { + case ex => throw new Exception(s"Invalid gcs auth_bucket path $bucket", ex) + } get - fileSystemWithGenomicsAuth.getPath(bucket).resolve(s"${workflowDescriptor.id}_auth.json") + authBucket.resolve(s"${workflowDescriptor.id}_auth.json") } - def toJesCallPaths(jobKey: BackendJobDescriptorKey) = JesCallPaths(jobKey, workflowDescriptor, jesConfiguration, credentials)(ec) + def toJesCallPaths(jobKey: BackendJobDescriptorKey) = JesCallPaths(jobKey, workflowDescriptor, jesConfiguration) } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/authentication/JesAuths.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/authentication/JesAuths.scala new file mode 100644 index 000000000..bb6b048c4 --- /dev/null +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/authentication/JesAuths.scala @@ -0,0 +1,5 @@ +package cromwell.backend.impl.jes.authentication + +import cromwell.filesystems.gcs.auth.GoogleAuthMode + +case class JesAuths(genomics: GoogleAuthMode, gcs: GoogleAuthMode) diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/authentication/JesCredentials.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/authentication/JesCredentials.scala deleted file mode 100644 index b4316fde3..000000000 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/authentication/JesCredentials.scala +++ /dev/null @@ -1,5 +0,0 @@ -package cromwell.backend.impl.jes.authentication - -import com.google.api.client.auth.oauth2.Credential - -case class JesCredentials(genomicsCredential: Credential, gcsCredential: Credential) diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/authentication/JesVMAuthentication.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/authentication/JesVMAuthentication.scala index 67ddd1df3..9c92b380a 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/authentication/JesVMAuthentication.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/authentication/JesVMAuthentication.scala @@ -1,7 +1,7 @@ package cromwell.backend.impl.jes.authentication import cromwell.core.DockerCredentials -import cromwell.filesystems.gcs.ClientSecrets +import cromwell.filesystems.gcs.auth.ClientSecrets import spray.json.{JsString, JsValue} /** diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/callcaching/JesBackendFileHashing.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/callcaching/JesBackendFileHashing.scala index 5c7cf4a8a..42cf8f7be 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/callcaching/JesBackendFileHashing.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/callcaching/JesBackendFileHashing.scala @@ -9,8 +9,8 @@ import scala.util.{Failure, Try} private[jes] object JesBackendFileHashing { def getCrc32c(singleFileHashRequest: SingleFileHashRequest, log: LoggingAdapter): Try[String] = { def usingJesInitData(jesInitData: JesBackendInitializationData) = for { - path <- Try(jesInitData.workflowPaths.gcsFileSystem.getPath(singleFileHashRequest.file.valueString)) - crc32c <- Try(jesInitData.workflowPaths.gcsFileSystemProvider.crc32cHash(path)) + path <- jesInitData.workflowPaths.getPath(singleFileHashRequest.file.valueString) + crc32c <- jesInitData.workflowPaths.getHash(path) } yield crc32c singleFileHashRequest.initializationData match { diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala index 24d417c99..2a2ae8ac2 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala @@ -3,19 +3,14 @@ package cromwell.backend.impl.jes import java.nio.file.{Files, Path} import com.google.api.client.http.HttpResponseException -import cromwell.filesystems.gcs._ +import com.google.cloud.storage.contrib.nio.CloudStorageOptions package object io { implicit class PathEnhanced(val path: Path) extends AnyVal { import better.files._ - def hash = path match { - case gcs: NioGcsPath => gcs.getFileSystem.provider().asInstanceOf[GcsFileSystemProvider].crc32cHash(gcs) - case _ => File(path).md5 - } - def writeAsJson(content: String): File = { - Files.write(path, content.getBytes, ContentTypeOption.Json) + Files.write(path, content.getBytes, CloudStorageOptions.withMimeType("application/json")) } } diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala index 5601e1ebb..6072e1d83 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala @@ -6,17 +6,18 @@ import java.util.UUID import akka.actor.{ActorRef, Props} import akka.event.LoggingAdapter import akka.testkit.{ImplicitSender, TestActorRef, TestDuration, TestProbe} +import com.google.cloud.storage.contrib.nio.CloudStoragePath import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse import cromwell.backend.async.AsyncBackendJobExecutionActor.{Execute, ExecutionMode} import cromwell.backend.async.{AbortedExecutionHandle, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle} import cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.JesPendingExecutionHandle -import cromwell.backend.impl.jes.MockObjects._ import cromwell.backend.impl.jes.RunStatus.Failed import cromwell.backend.impl.jes.io.{DiskType, JesWorkingDisk} import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobDescriptorKey, BackendWorkflowDescriptor, PreemptedException, RuntimeAttributeDefinition} import cromwell.core.logging.LoggerWrapper import cromwell.core.{WorkflowId, WorkflowOptions, _} -import cromwell.filesystems.gcs._ +import cromwell.filesystems.gcs.GcsPathBuilderFactory +import cromwell.filesystems.gcs.auth.GoogleAuthMode.NoAuthMode import cromwell.util.SampleWdl import org.scalatest._ import org.scalatest.prop.Tables.Table @@ -30,12 +31,13 @@ import wdl4s.{Call, LocallyQualifiedName, NamespaceWithWorkflow} import scala.concurrent.duration._ import scala.concurrent.{Await, ExecutionContext, Future, Promise} import scala.util.{Success, Try} -import cromwell.backend.impl.jes.MockObjects._ import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackendJobExecutionActorSpec") with FlatSpecLike with Matchers with ImplicitSender with Mockito { + val mockPathBuilder = GcsPathBuilderFactory(NoAuthMode).withOptions(mock[WorkflowOptions]) + import JesTestConfig._ implicit val Timeout = 5.seconds.dilated @@ -65,16 +67,14 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val NoOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue])) - val TestableCallContext = CallContext(MockGcsFileSystemBuilder.mockGcsFileSystem.getPath("gs://root"), "out", "err") + val TestableCallContext = CallContext(mockPathBuilder.build("gs://root").get, "out", "err") val TestableJesExpressionFunctions = { - new JesExpressionFunctions(List(MockGcsFileSystemBuilder.mockGcsFileSystem), TestableCallContext) + new JesExpressionFunctions(List(mockPathBuilder), TestableCallContext) } private def buildInitializationData(jobDescriptor: BackendJobDescriptor, configuration: JesConfiguration) = { - val workflowPaths = JesWorkflowPaths(jobDescriptor.workflowDescriptor, - configuration, - mockCredentials)(scala.concurrent.ExecutionContext.global) + val workflowPaths = JesWorkflowPaths(jobDescriptor.workflowDescriptor, configuration)(system) JesBackendInitializationData(workflowPaths, null) } @@ -393,14 +393,14 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend it should "generate correct JesOutputs" in { val inputs = Map( - "in" -> WdlFile("gs://a/b/c.txt") + "in" -> WdlFile("gs://blah/b/c.txt") ) val jesBackend = makeJesActorRef(SampleWdl.FilePassingWorkflow, "a", inputs).underlyingActor val jobDescriptor = jesBackend.jobDescriptor val workflowId = jesBackend.workflowId val jesInputs = jesBackend.generateJesInputs(jobDescriptor) jesInputs should have size 1 - jesInputs should contain(JesFileInput("in-0", "gs://a/b/c.txt", Paths.get("a/b/c.txt"), workingDisk)) + jesInputs should contain(JesFileInput("in-0", "gs://blah/b/c.txt", Paths.get("blah/b/c.txt"), workingDisk)) val jesOutputs = jesBackend.generateJesOutputs(jobDescriptor) jesOutputs should have size 1 jesOutputs should contain(JesFileOutput("out", @@ -413,7 +413,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend ) class TestJesExpressionFunctions extends JesExpressionFunctions( - List(MockGcsFileSystemBuilder.mockGcsFileSystem), TestableCallContext) { + List(mockPathBuilder), TestableCallContext) { override def write_lines(params: Seq[Try[WdlValue]]): Try[WdlFile] = { Success(WdlFile(s"gs://some/path/file.txt")) } @@ -597,14 +597,14 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val jesBackend = testActorRef.underlyingActor // TODO: NioGcsPath.equals not implemented, so use toString instead - jesBackend.jesCallPaths.stdoutPath should be(a[NioGcsPath]) - jesBackend.jesCallPaths.stdoutPath.toString shouldBe + jesBackend.jesCallPaths.stdoutPath should be(a[CloudStoragePath]) + jesBackend.jesCallPaths.stdoutPath.toUri.toString shouldBe "gs://path/to/gcs_root/hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello-stdout.log" - jesBackend.jesCallPaths.stderrPath should be(a[NioGcsPath]) - jesBackend.jesCallPaths.stderrPath.toString shouldBe + jesBackend.jesCallPaths.stderrPath should be(a[CloudStoragePath]) + jesBackend.jesCallPaths.stderrPath.toUri.toString shouldBe "gs://path/to/gcs_root/hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello-stderr.log" - jesBackend.jesCallPaths.jesLogPath should be(a[NioGcsPath]) - jesBackend.jesCallPaths.jesLogPath.toString shouldBe + jesBackend.jesCallPaths.jesLogPath should be(a[CloudStoragePath]) + jesBackend.jesCallPaths.jesLogPath.toUri.toString shouldBe "gs://path/to/gcs_root/hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello.log" } @@ -628,14 +628,14 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val jesBackend = testActorRef.underlyingActor - jesBackend.jesCallPaths.stdoutPath should be(a[NioGcsPath]) - jesBackend.jesCallPaths.stdoutPath.toString shouldBe + jesBackend.jesCallPaths.stdoutPath should be(a[CloudStoragePath]) + jesBackend.jesCallPaths.stdoutPath.toUri.toString shouldBe "gs://path/to/gcs_root/w/e6236763-c518-41d0-9688-432549a8bf7d/call-B/shard-2/B-2-stdout.log" - jesBackend.jesCallPaths.stderrPath should be(a[NioGcsPath]) - jesBackend.jesCallPaths.stderrPath.toString shouldBe + jesBackend.jesCallPaths.stderrPath should be(a[CloudStoragePath]) + jesBackend.jesCallPaths.stderrPath.toUri.toString shouldBe "gs://path/to/gcs_root/w/e6236763-c518-41d0-9688-432549a8bf7d/call-B/shard-2/B-2-stderr.log" - jesBackend.jesCallPaths.jesLogPath should be(a[NioGcsPath]) - jesBackend.jesCallPaths.jesLogPath.toString shouldBe + jesBackend.jesCallPaths.jesLogPath should be(a[CloudStoragePath]) + jesBackend.jesCallPaths.jesLogPath.toUri.toString shouldBe "gs://path/to/gcs_root/w/e6236763-c518-41d0-9688-432549a8bf7d/call-B/shard-2/B-2.log" } diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala index d27328706..c0c571526 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala @@ -1,13 +1,12 @@ package cromwell.backend.impl.jes import cromwell.backend.BackendSpec +import cromwell.core.TestKitSuite import cromwell.util.SampleWdl -import org.scalatest.{FlatSpec, Matchers} +import org.scalatest.{FlatSpecLike, Matchers} import org.specs2.mock.Mockito -import scala.concurrent.ExecutionContext.Implicits.global -import cromwell.backend.impl.jes.MockObjects._ -class JesCallPathsSpec extends FlatSpec with Matchers with Mockito { +class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with Mockito { import BackendSpec._ import JesTestConfig._ @@ -20,7 +19,7 @@ class JesCallPathsSpec extends FlatSpec with Matchers with Mockito { val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) val callPaths = JesCallPaths(jobDescriptorKey, workflowDescriptor, - jesConfiguration, mockCredentials) + jesConfiguration) callPaths.returnCodeFilename should be("hello-rc.txt") callPaths.stderrFilename should be("hello-stderr.log") callPaths.stdoutFilename should be("hello-stdout.log") @@ -32,15 +31,14 @@ class JesCallPathsSpec extends FlatSpec with Matchers with Mockito { val jobDescriptorKey = firstJobDescriptorKey(workflowDescriptor) val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) - val callPaths = JesCallPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration, - mockCredentials) - callPaths.returnCodePath.toString should + val callPaths = JesCallPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration) + callPaths.returnCodePath.toUri.toString should be(s"gs://my-cromwell-workflows-bucket/hello/${workflowDescriptor.id}/call-hello/hello-rc.txt") - callPaths.stdoutPath.toString should + callPaths.stdoutPath.toUri.toString should be(s"gs://my-cromwell-workflows-bucket/hello/${workflowDescriptor.id}/call-hello/hello-stdout.log") - callPaths.stderrPath.toString should + callPaths.stderrPath.toUri.toString should be(s"gs://my-cromwell-workflows-bucket/hello/${workflowDescriptor.id}/call-hello/hello-stderr.log") - callPaths.jesLogPath.toString should + callPaths.jesLogPath.toUri.toString should be(s"gs://my-cromwell-workflows-bucket/hello/${workflowDescriptor.id}/call-hello/hello.log") } @@ -49,9 +47,8 @@ class JesCallPathsSpec extends FlatSpec with Matchers with Mockito { val jobDescriptorKey = firstJobDescriptorKey(workflowDescriptor) val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) - val callPaths = JesCallPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration, - mockCredentials) - callPaths.callContext.root.toString should + val callPaths = JesCallPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration) + callPaths.callContext.root.toUri.toString should be(s"gs://my-cromwell-workflows-bucket/hello/${workflowDescriptor.id}/call-hello") callPaths.callContext.stdout should be("hello-stdout.log") callPaths.callContext.stderr should be("hello-stderr.log") diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala index c2b77d38c..46068343c 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala @@ -1,16 +1,25 @@ package cromwell.backend.impl.jes +import better.files.File import com.typesafe.config.{ConfigValueFactory, ConfigFactory} import cromwell.backend.BackendConfigurationDescriptor import org.scalatest.prop.TableDrivenPropertyChecks -import org.scalatest.{FlatSpec, Matchers} +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} -class JesConfigurationSpec extends FlatSpec with Matchers with TableDrivenPropertyChecks { +class JesConfigurationSpec extends FlatSpec with Matchers with TableDrivenPropertyChecks with BeforeAndAfterAll { behavior of "JesConfigurationSpec" + val mockFile = File.newTemporaryFile() + + override def afterAll(): Unit = { + mockFile.delete(true) + + () + } + val globalConfig = ConfigFactory.parseString( - """ + s""" |google { | | application-name = "cromwell" @@ -24,13 +33,13 @@ class JesConfigurationSpec extends FlatSpec with Matchers with TableDrivenProper | name = "user-via-refresh" | scheme = "refresh_token" | client-id = "secret_id" - | client-secret = "secret_secret" + | client-secret = "${mockFile.pathAsString}" | }, | { | name = "service-account" | scheme = "service_account" | service-account-id = "my-service-account" - | pem-file = "/path/to/file.pem" + | pem-file = "${mockFile.pathAsString}" | } | ] |} diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala index 4699402bc..20fe205d1 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala @@ -9,7 +9,8 @@ import cromwell.backend.impl.jes.authentication.GcsLocalizing import cromwell.backend.{BackendConfigurationDescriptor, BackendSpec, BackendWorkflowDescriptor} import cromwell.core.logging.LoggingTest._ import cromwell.core.{TestKitSuite, WorkflowOptions} -import cromwell.filesystems.gcs.{RefreshTokenMode, SimpleClientSecrets} +import cromwell.filesystems.gcs.GoogleConfiguration +import cromwell.filesystems.gcs.auth.{SimpleClientSecrets, RefreshTokenMode} import cromwell.util.{EncryptionSpec, SampleWdl} import org.scalatest.{FlatSpecLike, Matchers} import org.specs2.mock.Mockito @@ -197,7 +198,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe val TestingBits(actorRef, _) = buildJesInitializationTestingBits(refreshTokenConfig) val actor = actorRef.underlyingActor - actor.refreshTokenAuth should be(Some(GcsLocalizing(RefreshTokenMode("user-via-refresh", "secret_id", "secret_secret"), "mytoken"))) + actor.refreshTokenAuth should be(Some(GcsLocalizing(RefreshTokenMode("user-via-refresh", "secret_id", "secret_secret", GoogleConfiguration.GoogleScopes), "mytoken"))) } it should "generate the correct json content for no docker token and no refresh token" in { diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala index 3f1dea365..3ad37ee6d 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala @@ -1,12 +1,12 @@ package cromwell.backend.impl.jes import cromwell.backend.BackendSpec +import cromwell.core.TestKitSuite import cromwell.util.SampleWdl -import org.scalatest.{FlatSpec, Matchers} +import org.scalatest.{FlatSpecLike, Matchers} import org.specs2.mock.Mockito -import cromwell.backend.impl.jes.MockObjects._ -class JesWorkflowPathsSpec extends FlatSpec with Matchers with Mockito { +class JesWorkflowPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with Mockito { import BackendSpec._ import JesTestConfig._ @@ -16,11 +16,11 @@ class JesWorkflowPathsSpec extends FlatSpec with Matchers with Mockito { val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.wdlSource()) val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) - val workflowPaths = JesWorkflowPaths(workflowDescriptor, jesConfiguration, mockCredentials)(scala.concurrent.ExecutionContext.global) - workflowPaths.rootPath.toString should be("gs://my-cromwell-workflows-bucket") - workflowPaths.workflowRootPath.toString should - be(s"gs://my-cromwell-workflows-bucket/hello/${workflowDescriptor.id}") - workflowPaths.gcsAuthFilePath.toString should + val workflowPaths = JesWorkflowPaths(workflowDescriptor, jesConfiguration)(system) + workflowPaths.rootPath.toUri.toString should be("gs://my-cromwell-workflows-bucket/") + workflowPaths.workflowRootPath.toUri.toString should + be(s"gs://my-cromwell-workflows-bucket/hello/${workflowDescriptor.id}/") + workflowPaths.gcsAuthFilePath.toUri.toString should be(s"gs://my-cromwell-workflows-bucket/hello/${workflowDescriptor.id}/${workflowDescriptor.id}_auth.json") } } diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/MockObjects.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/MockObjects.scala deleted file mode 100644 index 1cde38c47..000000000 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/MockObjects.scala +++ /dev/null @@ -1,9 +0,0 @@ -package cromwell.backend.impl.jes - -import com.google.api.client.googleapis.testing.auth.oauth2.MockGoogleCredential -import cromwell.backend.impl.jes.authentication.JesCredentials - -object MockObjects { - val mockCredential = new MockGoogleCredential.Builder().build() - val mockCredentials = JesCredentials(mockCredential, mockCredential) -} diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendFileHashing.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendFileHashing.scala index 0fc377aac..f8ff92cfd 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendFileHashing.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendFileHashing.scala @@ -3,13 +3,21 @@ package cromwell.backend.impl.sfs.config import akka.event.LoggingAdapter import better.files._ import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest +import cromwell.core.path.DefaultPathBuilder import cromwell.util.TryWithResource._ +import scala.language.postfixOps import scala.util.Try private[config] object ConfigBackendFileHashing { - def getMd5Result(request: SingleFileHashRequest, log: LoggingAdapter): Try[String] = - tryWithResource(() => File(request.file.valueString).newInputStream) { inputStream => + def getMd5Result(request: SingleFileHashRequest, log: LoggingAdapter): Try[String] ={ + val path = DefaultPathBuilder.build(request.file.valueString) recover { + case failure => throw new RuntimeException("Failed to construct path to hash", failure) + } get + + tryWithResource(() => File(path).newInputStream) { inputStream => org.apache.commons.codec.digest.DigestUtils.md5Hex(inputStream) } + } + } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala index 6261e65c9..f453cf315 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala @@ -4,13 +4,15 @@ import akka.event.LoggingAdapter import better.files.File import com.typesafe.config.Config import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest +import cromwell.backend.sfs.SharedFileSystemBackendInitializationData +import cromwell.core.path.PathFactory import cromwell.util.TryWithResource._ import cromwell.util.FileUtil._ import net.ceedubs.ficus.Ficus._ import org.apache.commons.codec.digest.DigestUtils import org.slf4j.LoggerFactory -import scala.util.Try +import scala.util.{Failure, Try} object ConfigHashingStrategy { val logger = LoggerFactory.getLogger(getClass) @@ -37,14 +39,22 @@ abstract class ConfigHashingStrategy { protected lazy val checkSiblingMessage = if (checkSiblingMd5) "Check first for sibling md5 and if not found " else "" def getHash(request: SingleFileHashRequest, log: LoggingAdapter): Try[String] = { - val file = File(request.file.valueString).followSymlinks + def usingSFSInitData(initData: SharedFileSystemBackendInitializationData) = { + val pathBuilders = initData.workflowPaths.pathBuilders + val file = PathFactory.buildFile(request.file.valueString, pathBuilders).followSymlinks - if (checkSiblingMd5) { - precomputedMd5(file) match { - case Some(md5) => Try(md5.contentAsString) - case None => hash(file) - } - } else hash(file) + if (checkSiblingMd5) { + precomputedMd5(file) match { + case Some(md5) => Try(md5.contentAsString) + case None => hash(file) + } + } else hash(file) + } + + request.initializationData match { + case Some(initData: SharedFileSystemBackendInitializationData) => usingSFSInitData(initData) + case _ => Failure(new IllegalArgumentException("Need SharedFileSystemBackendInitializationData to calculate hash.")) + } } private def precomputedMd5(file: File): Option[File] = { diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala deleted file mode 100644 index d96140014..000000000 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala +++ /dev/null @@ -1,36 +0,0 @@ -package cromwell.backend.sfs - -import cats.data.Validated.{Invalid, Valid} -import cromwell.backend.wfs.{WorkflowFileSystemProvider, WorkflowFileSystemProviderParams} -import cromwell.filesystems.gcs.GoogleAuthMode.GoogleAuthOptions -import cromwell.filesystems.gcs.{GcsFileSystem, GcsFileSystemProvider, GoogleConfiguration} -import net.ceedubs.ficus.Ficus._ -import wdl4s.ValidationException - -import scala.util.Try - -object GcsWorkflowFileSystemProvider extends WorkflowFileSystemProvider { - override def fileSystemOption(params: WorkflowFileSystemProviderParams): Option[GcsFileSystem] = { - params.fileSystemConfig.as[Option[String]]("gcs.auth") map gcsFileSystem(params) - } - - private def gcsFileSystem(params: WorkflowFileSystemProviderParams)(gcsAuthName: String): GcsFileSystem = { - val workflowOptions = params.workflowOptions - val globalConfig = params.globalConfig - val googleConfig = GoogleConfiguration(globalConfig) - val googleAuthModeValidation = googleConfig.auth(gcsAuthName) - - val gcsAuthMode = googleAuthModeValidation match { - case Valid(googleAuthMode) => googleAuthMode - case Invalid(errors) => - throw new ValidationException("Could not create gcs filesystem from configuration", errors) - } - - val authOptions = new GoogleAuthOptions { - override def get(key: String): Try[String] = workflowOptions.get(key) - } - - val storage = gcsAuthMode.buildStorage(authOptions, googleConfig.applicationName) - GcsFileSystem(GcsFileSystemProvider(storage)(params.fileSystemExecutionContext)) - } -} diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala index 20c8c3d39..d67c8846c 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala @@ -1,15 +1,16 @@ package cromwell.backend.sfs -import java.nio.file.{FileSystem, Path, Paths} +import java.nio.file.{Path, Paths} import cats.instances.try_._ import cats.syntax.functor._ import com.typesafe.config.Config import cromwell.backend.io.JobPaths import cromwell.core._ +import cromwell.core.path.PathFactory +import cromwell.util.TryUtil import wdl4s.CallInputs import wdl4s.types.{WdlArrayType, WdlMapType} -import wdl4s.util.TryUtil import wdl4s.values._ import scala.collection.JavaConverters._ @@ -49,9 +50,6 @@ object SharedFileSystem { private def localizePathViaHardLink(originalPath: File, executionPath: File): Try[Unit] = { executionPath.parent.createDirectories() - // link.linkTo(target) returns target, - // however we want to return the link, not the target, so map the result back to executionPath - // -Ywarn-value-discard // Try(executionPath.linkTo(originalPath, symbolic = false)) map { _ => executionPath } Try { executionPath.linkTo(originalPath, symbolic = false) } void @@ -61,8 +59,6 @@ object SharedFileSystem { if (originalPath.isDirectory) Failure(new UnsupportedOperationException("Cannot localize directory with symbolic links")) else { executionPath.parent.createDirectories() - // -Ywarn-value-discard - // Try(executionPath.linkTo(originalPath, symbolic = true)) map { _ => executionPath } Try { executionPath.linkTo(originalPath, symbolic = true) } void } } @@ -134,6 +130,9 @@ trait SharedFileSystem extends PathFactory { case array: WdlArray => val mappedArray = array.value map outputMapper(job) TryUtil.sequence(mappedArray) map { WdlArray(array.wdlType, _) } + case map: WdlMap => + val mappedMap = map.value mapValues outputMapper(job) + TryUtil.sequenceMap(mappedMap) map { WdlMap(map.wdlType, _) } case other => Success(other) } } @@ -149,7 +148,7 @@ trait SharedFileSystem extends PathFactory { * end up with this implementation and thus use it to satisfy their contract with Backend. * This is yuck-tastic and I consider this a FIXME, but not for this refactor */ - def localizeInputs(inputsRoot: Path, docker: Boolean, filesystems: List[FileSystem], inputs: CallInputs): Try[CallInputs] = { + def localizeInputs(inputsRoot: Path, docker: Boolean, inputs: CallInputs): Try[CallInputs] = { val strategies = if (docker) DockerLocalizers else Localizers // Use URI to identify protocol scheme and strip it out @@ -165,8 +164,9 @@ trait SharedFileSystem extends PathFactory { * Transform an original input path to a path in the call directory. * The new path matches the original path, it only "moves" the root to be the call directory. */ + def toCallPath(path: String): Try[PairOfFiles] = Try { - val src = buildFile(path, filesystems) + val src = buildFile(path) // Strip out potential prefix protocol val localInputPath = stripProtocolScheme(src.path) val dest = if (File(inputsRoot).isParentOf(localInputPath)) File(localInputPath) diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala index 2ba4edd48..98f4dbc11 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala @@ -1,6 +1,6 @@ package cromwell.backend.sfs -import java.nio.file.{FileAlreadyExistsException, Path, Paths} +import java.nio.file.{FileAlreadyExistsException, Path} import akka.actor.{Actor, ActorLogging, ActorRef} import akka.event.LoggingReceive @@ -15,6 +15,7 @@ import cromwell.backend.validation._ import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, OutputEvaluator} import cromwell.core.JobOutputs import cromwell.core.logging.JobLogging +import cromwell.core.path.DefaultPathBuilder import cromwell.core.retry.SimpleExponentialBackoff import cromwell.services.keyvalue.KeyValueServiceActor._ import wdl4s.values.{WdlArray, WdlFile, WdlMap, WdlValue} @@ -135,11 +136,13 @@ trait SharedFileSystemAsyncJobExecutionActor override lazy val backendInitializationDataOption = params.backendInitializationDataOption - def toDockerPath(path: WdlValue): WdlValue = { + def toUnixPath(docker: Boolean)(path: WdlValue): WdlValue = { path match { - case file: WdlFile => WdlFile(jobPaths.toDockerPath(Paths.get(path.valueString)).toString) - case array: WdlArray => WdlArray(array.wdlType, array.value map toDockerPath) - case map: WdlMap => WdlMap(map.wdlType, map.value mapValues toDockerPath) + case file: WdlFile => + val cleanPath = DefaultPathBuilder.build(path.valueString).get + WdlFile(if (docker) jobPaths.toDockerPath(cleanPath).toString else cleanPath.toString) + case array: WdlArray => WdlArray(array.wdlType, array.value map toUnixPath(docker)) + case map: WdlMap => WdlMap(map.wdlType, map.value mapValues toUnixPath(docker)) case wdlValue => wdlValue } } @@ -150,8 +153,8 @@ trait SharedFileSystemAsyncJobExecutionActor lazy val workflowDescriptor = jobDescriptor.workflowDescriptor lazy val call = jobDescriptor.key.call - lazy val fileSystems = WorkflowPathsBackendInitializationData.fileSystems(backendInitializationDataOption) - lazy val callEngineFunction = SharedFileSystemExpressionFunctions(jobPaths, fileSystems) + lazy val pathBuilders = WorkflowPathsBackendInitializationData.pathBuilders(backendInitializationDataOption) + lazy val callEngineFunction = SharedFileSystemExpressionFunctions(jobPaths, pathBuilders) override lazy val workflowId = jobDescriptor.workflowDescriptor.id override lazy val jobTag = jobDescriptor.key.tag @@ -165,9 +168,9 @@ trait SharedFileSystemAsyncJobExecutionActor } def instantiatedScript: String = { - val pathTransformFunction: WdlValue => WdlValue = if (isDockerRun) toDockerPath else identity + val pathTransformFunction: WdlValue => WdlValue = toUnixPath(isDockerRun) val tryCommand = sharedFileSystem.localizeInputs(jobPaths.callInputsRoot, - isDockerRun, fileSystems, jobDescriptor.inputs) flatMap { localizedInputs => + isDockerRun, jobDescriptor.inputs) flatMap { localizedInputs => call.task.instantiateCommand(localizedInputs, callEngineFunction, pathTransformFunction) } tryCommand.get @@ -328,7 +331,7 @@ mv $rcTmpPath $rcPath def processSuccess(returnCode: Int) = { val successfulFuture = for { outputs <- Future.fromTry(processOutputs()) - } yield SuccessfulExecutionHandle(outputs, returnCode, jobPaths.detritusPaths.mapValues(_.toString), Seq.empty) + } yield SuccessfulExecutionHandle(outputs, returnCode, jobPaths.detritusPaths, Seq.empty) successfulFuture recover { case failed: Throwable => diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala index edacd5303..65152eb83 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala @@ -1,10 +1,15 @@ package cromwell.backend.sfs import akka.actor.{ActorRef, Props} +import cats.data.Validated.{Invalid, Valid} import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, BackendJobDescriptorKey, BackendLifecycleActorFactory, BackendWorkflowDescriptor} import cromwell.core.Dispatcher import cromwell.core.Dispatcher._ +import cromwell.core.path.{PathBuilderFactory, DefaultPathBuilderFactory} +import cromwell.filesystems.gcs.{GcsPathBuilderFactory, GoogleConfiguration} +import lenthall.exception.MessageAggregation +import net.ceedubs.ficus.Ficus._ import wdl4s.Call import wdl4s.expression.WdlStandardLibraryFunctions @@ -18,6 +23,23 @@ import scala.concurrent.Promise trait SharedFileSystemBackendLifecycleActorFactory extends BackendLifecycleActorFactory { /** + * If the backend sets a gcs authentication mode, try to create a PathBuilderFactory with it. + */ + lazy val gcsPathBuilderFactory: Option[GcsPathBuilderFactory] = { + configurationDescriptor.backendConfig.as[Option[String]]("filesystems.gcs.auth") map { configAuth => + GoogleConfiguration(configurationDescriptor.globalConfig).auth(configAuth) match { + case Valid(auth) => GcsPathBuilderFactory(auth) + case Invalid(error) => throw new MessageAggregation { + override def exceptionContext: String = "Failed to parse gcs auth configuration" + override def errorMessages: Traversable[String] = error.toList + } + } + } + } + + lazy val pathBuilderFactories: List[PathBuilderFactory] = List(gcsPathBuilderFactory, Option(DefaultPathBuilderFactory)).flatten + + /** * Config values for the backend, and a pointer to the global config. * * This is the single parameter passed into each factory during creation. @@ -44,7 +66,7 @@ trait SharedFileSystemBackendLifecycleActorFactory extends BackendLifecycleActor override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], serviceRegistryActor: ActorRef) = { val params = SharedFileSystemInitializationActorParams(serviceRegistryActor, workflowDescriptor, - configurationDescriptor, calls) + configurationDescriptor, calls, pathBuilderFactories) Option(Props(initializationActorClass, params).withDispatcher(Dispatcher.BackendDispatcher)) } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala index 62dc98b07..6ba44914d 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala @@ -1,10 +1,13 @@ package cromwell.backend.sfs -import java.nio.file.{Path, Paths} +import java.nio.file.Path import akka.actor.ActorRef import cromwell.backend.callcaching.CacheHitDuplicating import cromwell.backend.{BackendCacheHitCopyingActor, BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor} +import cromwell.core.path.PathFactory + +import scala.util.Try class SharedFileSystemCacheHitCopyingActor(override val jobDescriptor: BackendJobDescriptor, override val configurationDescriptor: BackendConfigurationDescriptor, @@ -17,7 +20,7 @@ class SharedFileSystemCacheHitCopyingActor(override val jobDescriptor: BackendJo override lazy val destinationJobDetritusPaths = jobPaths.detritusPaths - override protected def getPath(file: String) = Paths.get(file) + override protected def getPath(file: String) = Try(PathFactory.buildPath(file, jobPaths.pathBuilders)) override protected def duplicate(source: Path, destination: Path) = { // -Ywarn-value-discard diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala index 1f73cac38..417760a84 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala @@ -1,11 +1,12 @@ package cromwell.backend.sfs -import java.nio.file.{FileSystem, Path} +import java.nio.file.Path import cromwell.backend.io.{JobPaths, WorkflowPathsBackendInitializationData} import cromwell.backend.wdl._ import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptorKey, BackendWorkflowDescriptor} import cromwell.core.CallContext +import cromwell.core.path.PathBuilder import wdl4s.expression.WdlStandardLibraryFunctions import wdl4s.values.{WdlFile, WdlValue} @@ -20,23 +21,23 @@ object SharedFileSystemExpressionFunctions { def apply(workflowDescriptor: BackendWorkflowDescriptor, jobKey: BackendJobDescriptorKey, configurationDescriptor: BackendConfigurationDescriptor, - fileSystems: List[FileSystem]): SharedFileSystemExpressionFunctions = { + pathBuilders: List[PathBuilder]): SharedFileSystemExpressionFunctions = { val jobPaths = new JobPaths(workflowDescriptor, configurationDescriptor.backendConfig, jobKey) val callContext = CallContext( jobPaths.callExecutionRoot, jobPaths.stdout.toString, jobPaths.stderr.toString ) - new SharedFileSystemExpressionFunctions(fileSystems, callContext) + new SharedFileSystemExpressionFunctions(pathBuilders, callContext) } - def apply(jobPaths: JobPaths, fileSystems: List[FileSystem]): SharedFileSystemExpressionFunctions = { + def apply(jobPaths: JobPaths, pathBuilders: List[PathBuilder]): SharedFileSystemExpressionFunctions = { val callContext = CallContext( jobPaths.callExecutionRoot, jobPaths.stdout.toString, jobPaths.stderr.toString ) - new SharedFileSystemExpressionFunctions(fileSystems, callContext) + new SharedFileSystemExpressionFunctions(pathBuilders, callContext) } def apply(workflowDescriptor: BackendWorkflowDescriptor, @@ -50,11 +51,11 @@ object SharedFileSystemExpressionFunctions { jobPaths.stderr.toString ) - new SharedFileSystemExpressionFunctions(WorkflowPathsBackendInitializationData.fileSystems(initializationData), callContext) + new SharedFileSystemExpressionFunctions(WorkflowPathsBackendInitializationData.pathBuilders(initializationData), callContext) } } -class SharedFileSystemExpressionFunctions(override val fileSystems: List[FileSystem], +class SharedFileSystemExpressionFunctions(override val pathBuilders: List[PathBuilder], context: CallContext ) extends WdlStandardLibraryFunctions with PureFunctions with ReadLikeFunctions with WriteFunctions { import SharedFileSystemExpressionFunctions._ @@ -62,7 +63,7 @@ class SharedFileSystemExpressionFunctions(override val fileSystems: List[FileSys override def globPath(glob: String) = context.root.toString override def glob(path: String, pattern: String): Seq[String] = { - File(toPath(path)).glob(s"**/$pattern") map { _.pathAsString } toSeq + File(context.root).glob(s"**/$pattern") map { _.pathAsString } toSeq } override val writeDirectory = context.root diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala index 54a370b39..03bb1024d 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala @@ -4,9 +4,10 @@ import akka.actor.ActorRef import better.files._ import cromwell.backend.io.{WorkflowPaths, WorkflowPathsBackendInitializationData} import cromwell.backend.validation.RuntimeAttributesDefault -import cromwell.backend.wfs.{DefaultWorkflowFileSystemProvider, WorkflowFileSystemProvider} +import cromwell.backend.wfs.WorkflowPathBuilder import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor} -import cromwell.core.{Dispatcher, WorkflowOptions} +import cromwell.core.path.PathBuilderFactory +import cromwell.core.WorkflowOptions import wdl4s.Call import wdl4s.values.WdlValue @@ -18,7 +19,8 @@ case class SharedFileSystemInitializationActorParams serviceRegistryActor: ActorRef, workflowDescriptor: BackendWorkflowDescriptor, configurationDescriptor: BackendConfigurationDescriptor, - calls: Seq[Call] + calls: Seq[Call], + pathBuilderFactories: List[PathBuilderFactory] ) class SharedFileSystemBackendInitializationData @@ -49,11 +51,9 @@ class SharedFileSystemInitializationActor(params: SharedFileSystemInitialization ).toMap } - val providers = Seq(GcsWorkflowFileSystemProvider, DefaultWorkflowFileSystemProvider) - val ioDispatcher = context.system.dispatchers.lookup(Dispatcher.IoDispatcher) + val pathBuilders = params.pathBuilderFactories map { _.withOptions(workflowDescriptor.workflowOptions)(context.system) } - val workflowPaths = WorkflowFileSystemProvider.workflowPaths(configurationDescriptor, workflowDescriptor, - providers, ioDispatcher) + val workflowPaths = WorkflowPathBuilder.workflowPaths(configurationDescriptor, workflowDescriptor, pathBuilders) override def beforeAll(): Future[Option[BackendInitializationData]] = { Future.fromTry(Try { diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala index d9d4f4213..b958f3c6f 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala @@ -4,7 +4,7 @@ import akka.actor.{Actor, ActorRef} import com.typesafe.config.{Config, ConfigFactory} import cromwell.backend.BackendInitializationData import cromwell.backend.callcaching.JobCachingActorHelper -import cromwell.backend.io.JobPaths +import cromwell.backend.io.{WorkflowPathsBackendInitializationData, JobPaths} import cromwell.backend.validation.{RuntimeAttributesValidation, ValidatedRuntimeAttributes} import cromwell.core.logging.JobLogging import net.ceedubs.ficus.Ficus._ @@ -37,6 +37,7 @@ trait SharedFileSystemJobCachingActorHelper extends JobCachingActorHelper { } lazy val sharedFileSystem = new SharedFileSystem { + override val pathBuilders = WorkflowPathsBackendInitializationData.pathBuilders(backendInitializationDataOption) override lazy val sharedFileSystemConfig = { configurationDescriptor.backendConfig.as[Option[Config]]("filesystems.local").getOrElse(ConfigFactory.empty()) } diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala index 81edb6f60..bd25dfc51 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala @@ -6,6 +6,9 @@ import akka.event.LoggingAdapter import better.files._ import com.typesafe.config.{ConfigFactory, ConfigValueFactory} import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest +import cromwell.backend.io.WorkflowPaths +import cromwell.backend.sfs.SharedFileSystemBackendInitializationData +import cromwell.core.path.DefaultPathBuilder import org.apache.commons.codec.digest.DigestUtils import org.scalatest.prop.TableDrivenPropertyChecks import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} @@ -43,7 +46,14 @@ class ConfigHashingStrategySpec extends FlatSpec with Matchers with TableDrivenP symLink } else file + val workflowPaths = mock[WorkflowPaths] + workflowPaths.pathBuilders returns List(DefaultPathBuilder) + + val initData = mock[SharedFileSystemBackendInitializationData] + initData.workflowPaths returns workflowPaths + request.file returns WdlFile(requestFile.pathAsString) + request.initializationData returns Option(initData) request } diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala index 64dbfa9d7..1dc2811fa 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala @@ -37,7 +37,7 @@ class SharedFileSystemInitializationActorSpec extends TestKitSuite("SharedFileSy private def getActorRef(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], conf: BackendConfigurationDescriptor) = { - val params = SharedFileSystemInitializationActorParams(emptyActor, workflowDescriptor, conf, calls) + val params = SharedFileSystemInitializationActorParams(emptyActor, workflowDescriptor, conf, calls, List.empty) val props = Props(new SharedFileSystemInitializationActor(params)) system.actorOf(props, "SharedFileSystemInitializationActor") } diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala index c3874a14a..9b66911a6 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala @@ -1,9 +1,10 @@ package cromwell.backend.sfs -import java.nio.file.{FileSystems, Files} +import java.nio.file.Files import better.files._ import com.typesafe.config.{Config, ConfigFactory} +import cromwell.core.path.DefaultPathBuilder import org.scalatest.prop.TableDrivenPropertyChecks import org.scalatest.{FlatSpec, Matchers} import org.specs2.mock.Mockito @@ -16,7 +17,7 @@ class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with Tabl val defaultLocalization = ConfigFactory.parseString(""" localization: [copy, hard-link, soft-link] """) val hardLinkLocalization = ConfigFactory.parseString(""" localization: [hard-link] """) val softLinkLocalization = ConfigFactory.parseString(""" localization: [soft-link] """) - val localFS = List(FileSystems.getDefault) + val localPathBuilder = List(DefaultPathBuilder) def localizationTest(config: Config, @@ -35,8 +36,11 @@ class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with Tabl } val inputs = Map("input" -> WdlFile(orig.pathAsString)) - val sharedFS = new SharedFileSystem { override val sharedFileSystemConfig = config } - val result = sharedFS.localizeInputs(callDir.path, docker = docker, localFS, inputs) + val sharedFS = new SharedFileSystem { + override val pathBuilders = localPathBuilder + override val sharedFileSystemConfig = config + } + val result = sharedFS.localizeInputs(callDir.path, docker = docker, inputs) result.isSuccess shouldBe true result.get should contain theSameElementsAs Map("input" -> WdlFile(dest.pathAsString)) diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala index ef01c9985..b31cbb32d 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala @@ -29,6 +29,6 @@ case class SparkBackendFactory(name: String, configurationDescriptor: BackendCon jobPaths.stderr.toAbsolutePath.toString ) - new SharedFileSystemExpressionFunctions(SparkJobExecutionActor.DefaultFileSystems, callContext) + new SharedFileSystemExpressionFunctions(SparkJobExecutionActor.DefaultPathBuilders, callContext) } } diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala index c793338eb..c52cb1e4c 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala @@ -1,6 +1,5 @@ package cromwell.backend.impl.spark -import java.nio.file.FileSystems import java.nio.file.attribute.PosixFilePermission import akka.actor.Props @@ -9,7 +8,8 @@ import cromwell.backend.impl.spark.SparkClusterProcess._ import cromwell.backend.io.JobPaths import cromwell.backend.sfs.{SharedFileSystem, SharedFileSystemExpressionFunctions} import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobExecutionActor} -import cromwell.core.{TailedWriter, UntailedWriter} +import cromwell.core.path.{DefaultPathBuilder, TailedWriter, UntailedWriter} +import cromwell.core.path.JavaWriterImplicits._ import wdl4s.parser.MemoryUnit import wdl4s.util.TryUtil @@ -18,7 +18,7 @@ import scala.sys.process.ProcessLogger import scala.util.{Failure, Success, Try} object SparkJobExecutionActor { - val DefaultFileSystems = List(FileSystems.getDefault) + val DefaultPathBuilders = List(DefaultPathBuilder) def props(jobDescriptor: BackendJobDescriptor, configurationDescriptor: BackendConfigurationDescriptor): Props = Props(new SparkJobExecutionActor(jobDescriptor, configurationDescriptor)) @@ -29,8 +29,8 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, import SparkJobExecutionActor._ import better.files._ - import cromwell.core.PathFactory._ + override val pathBuilders = DefaultPathBuilders private val tag = s"SparkJobExecutionActor-${jobDescriptor.key.tag}:" lazy val cmds = new SparkCommands @@ -58,7 +58,7 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, private lazy val isClusterMode = isSparkClusterMode(sparkDeployMode, sparkMaster) private val call = jobDescriptor.key.call - private val callEngineFunction = SharedFileSystemExpressionFunctions(jobPaths, DefaultFileSystems) + private val callEngineFunction = SharedFileSystemExpressionFunctions(jobPaths, DefaultPathBuilders) private val lookup = jobDescriptor.inputs.apply _ @@ -155,7 +155,7 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, executionDir.toString.toFile.createIfNotExists(asDirectory = true, createParents = true) log.debug("{} Resolving job command", tag) - val command = localizeInputs(jobPaths.callInputsRoot, docker = false, DefaultFileSystems, jobDescriptor.inputs) flatMap { + val command = localizeInputs(jobPaths.callInputsRoot, docker = false, jobDescriptor.inputs) flatMap { localizedInputs => call.task.instantiateCommand(localizedInputs, callEngineFunction, identity) } diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala index 883645ecd..01d221161 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala @@ -2,12 +2,12 @@ package cromwell.backend.impl.spark import java.nio.file.Path +import better.files._ import com.typesafe.scalalogging.StrictLogging -import cromwell.core.{TailedWriter, UntailedWriter} -import cromwell.core.PathFactory.EnhancedPath +import cromwell.core.path.PathImplicits._ +import cromwell.core.path.{TailedWriter, UntailedWriter} import scala.sys.process._ -import better.files._ import scala.util.{Failure, Success, Try} object SparkCommands { diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala index aa4400ac8..fcf21b948 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala @@ -10,7 +10,8 @@ import cromwell.backend.BackendJobExecutionActor.{FailedNonRetryableResponse, Su import cromwell.backend.impl.spark.SparkClusterProcess._ import cromwell.backend.io._ import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendSpec} -import cromwell.core.{PathWriter, TailedWriter, TestKitSuite, UntailedWriter, _} +import cromwell.core.{WorkflowOptions, TestKitSuite} +import cromwell.core.path.{PathWriter, TailedWriter, UntailedWriter} import org.mockito.Matchers._ import org.mockito.Mockito import org.mockito.Mockito._ From ff9213f403940e23f7b2c3741ddb7ddf431b2559 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Mon, 24 Oct 2016 12:58:54 -0400 Subject: [PATCH 043/375] No more unsafe list.head in SWRA --- .../workflow/SingleWorkflowRunnerActor.scala | 142 +++++++++++++-------- 1 file changed, 88 insertions(+), 54 deletions(-) diff --git a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala index 3e0636ce1..c1561777b 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala @@ -32,60 +32,63 @@ import scala.util.{Failure, Try} * print out the outputs when complete and reply with a result. */ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: Option[Path]) - extends CromwellRootActor with LoggingFSM[RunnerState, RunnerData] { + extends CromwellRootActor with LoggingFSM[RunnerState, SwraData] { import SingleWorkflowRunnerActor._ - private val backoff = SimpleExponentialBackoff(1 second, 1 minute, 1.2) + private val backoff = SimpleExponentialBackoff(1 second, 1 minute, 1.05) override lazy val workflowStore = new InMemoryWorkflowStore() override lazy val jobStoreActor = context.actorOf(EmptyJobStoreActor.props) - startWith(NotStarted, RunnerData()) + startWith(NotStarted, EmptySwraData) when (NotStarted) { - case Event(RunWorkflow, data) => + case Event(RunWorkflow, EmptySwraData) => log.info(s"$Tag: Submitting workflow") workflowStoreActor ! SubmitWorkflow(source) - goto (RunningWorkflow) using data.copy(replyTo = Option(sender())) + goto(SubmittedWorkflow) using SubmittedSwraData(sender()) } - when (RunningWorkflow) { - case Event(WorkflowStoreActor.WorkflowSubmittedToStore(id), data) => + when (SubmittedWorkflow) { + case Event(WorkflowStoreActor.WorkflowSubmittedToStore(id), SubmittedSwraData(replyTo)) => log.info(s"$Tag: Workflow submitted UUID($id)") // Since we only have a single workflow, force the WorkflowManagerActor's hand in case the polling rate is long workflowManagerActor ! RetrieveNewWorkflows schedulePollRequest() - stay() using data.copy(id = Option(id)) - case Event(IssuePollRequest, data) => - data.id match { - case None => schedulePollRequest() - case _ => requestStatus() - } + goto(RunningWorkflow) using RunningSwraData(replyTo, id) + } + + when (RunningWorkflow) { + case Event(IssuePollRequest, RunningSwraData(_, id)) => + requestStatus(id) stay() - case Event(RequestComplete((StatusCodes.OK, jsObject: JsObject)), data) if !jsObject.state.isTerminal => + case Event(RequestComplete((StatusCodes.OK, jsObject: JsObject)), RunningSwraData(_, _)) if !jsObject.state.isTerminal => schedulePollRequest() stay() - case Event(RequestComplete((StatusCodes.OK, jsObject: JsObject)), data) if jsObject.state == WorkflowSucceeded => + case Event(RequestComplete((StatusCodes.OK, jsObject: JsObject)), RunningSwraData(replyTo, id)) if jsObject.state == WorkflowSucceeded => val metadataBuilder = context.actorOf(MetadataBuilderActor.props(serviceRegistryActor), - s"CompleteRequest-Workflow-${stateData.id.get}-request-${UUID.randomUUID()}") - metadataBuilder ! WorkflowOutputs(data.id.get) - goto(RequestingOutputs) using data.copy(terminalState = Option(WorkflowSucceeded)) - case Event(RequestComplete((StatusCodes.OK, jsObject: JsObject)), data) if jsObject.state == WorkflowFailed => - val updatedData = data.copy(terminalState = Option(WorkflowFailed)).addFailure(s"Workflow ${data.id.get} transitioned to state Failed") - // If there's an output path specified then request metadata, otherwise issue a reply to the original sender. - if (metadataOutputPath.isDefined) requestMetadata(updatedData) else issueReply(updatedData) + s"CompleteRequest-Workflow-$id-request-${UUID.randomUUID()}") + metadataBuilder ! WorkflowOutputs(id) + log.info(s"$Tag workflow finished with status '$WorkflowSucceeded'.") + goto(RequestingOutputs) using SucceededSwraData(replyTo, id) + case Event(RequestComplete((StatusCodes.OK, jsObject: JsObject)), RunningSwraData(replyTo, id)) if jsObject.state == WorkflowFailed => + log.info(s"$Tag workflow finished with status '$WorkflowFailed'.") + requestMetadataOrIssueReply(FailedSwraData(replyTo, id, new RuntimeException(s"Workflow $id transitioned to state Failed"))) + case Event(RequestComplete((StatusCodes.OK, jsObject: JsObject)), RunningSwraData(replyTo, id)) if jsObject.state == WorkflowAborted => + log.info(s"$Tag workflow finished with status '$WorkflowAborted'.") + requestMetadataOrIssueReply(AbortedSwraData(replyTo, id)) } when (RequestingOutputs) { - case Event(RequestComplete((StatusCodes.OK, outputs: JsObject)), _) => + case Event(RequestComplete((StatusCodes.OK, outputs: JsObject)), data: CompletedSwraData) => outputOutputs(outputs) - if (metadataOutputPath.isDefined) requestMetadata(stateData) else issueReply(stateData) + requestMetadataOrIssueReply(data) } when (RequestingMetadata) { - case Event(RequestComplete((StatusCodes.OK, metadata: JsObject)), _) => + case Event(RequestComplete((StatusCodes.OK, metadata: JsObject)), data: CompletedSwraData) => outputMetadata(metadata) - issueReply(stateData) + issueReply(data) } onTransition { @@ -94,22 +97,24 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: whenUnhandled { // Handle failures for all failure responses generically. - case Event(r: WorkflowStoreActor.WorkflowAbortFailed, data) => failAndFinish(r.reason) - case Event(Failure(e), data) => failAndFinish(e) - case Event(Status.Failure(e), data) => failAndFinish(e) - case Event(RequestComplete((_, snap)), _) => failAndFinish(new RuntimeException(s"Unexpected API completion message: $snap")) + case Event(r: WorkflowStoreActor.WorkflowAbortFailed, data) => failAndFinish(r.reason, data) + case Event(Failure(e), data) => failAndFinish(e, data) + case Event(Status.Failure(e), data) => failAndFinish(e, data) + case Event(RequestComplete((_, snap)), data) => failAndFinish(new RuntimeException(s"Unexpected API completion message: $snap"), data) case Event((CurrentState(_, _) | Transition(_, _, _)), _) => // ignore uninteresting current state and transition messages stay() - case Event(m, _) => - log.warning(s"$Tag: received unexpected message: $m") + case Event(m, d) => + log.warning(s"$Tag: received unexpected message: $m in state ${d.getClass.getSimpleName}") stay() } - private def requestMetadata(data: RunnerData): State = { - val metadataBuilder = context.actorOf(MetadataBuilderActor.props(serviceRegistryActor), s"MetadataRequest-Workflow-${stateData.id.get}") - metadataBuilder ! GetSingleWorkflowMetadataAction(stateData.id.get, None, None) - goto (RequestingMetadata) using data + private def requestMetadataOrIssueReply(newData: CompletedSwraData) = if (metadataOutputPath.isDefined) requestMetadata(newData) else issueReply(newData) + + private def requestMetadata(newData: CompletedSwraData) = { + val metadataBuilder = context.actorOf(MetadataBuilderActor.props(serviceRegistryActor), s"MetadataRequest-Workflow-${newData.id}") + metadataBuilder ! GetSingleWorkflowMetadataAction(newData.id, None, None) + goto(RequestingMetadata) using newData } private def schedulePollRequest(): Unit = { @@ -118,27 +123,49 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: () } - private def requestStatus(): Unit = { + private def requestStatus(id: WorkflowId): Unit = { // This requests status via the metadata service rather than instituting an FSM watch on the underlying workflow actor. // Cromwell's eventual consistency means it isn't safe to use an FSM transition to a terminal state as the signal for // when outputs or metadata have stabilized. - val metadataBuilder = context.actorOf(MetadataBuilderActor.props(serviceRegistryActor), s"StatusRequest-Workflow-${stateData.id.get}-request-${UUID.randomUUID()}") - metadataBuilder ! GetStatus(stateData.id.get) + val metadataBuilder = context.actorOf(MetadataBuilderActor.props(serviceRegistryActor), s"StatusRequest-Workflow-$id-request-${UUID.randomUUID()}") + metadataBuilder ! GetStatus(id) } - private def issueReply(data: RunnerData): State = { - data.terminalState foreach { state => log.info(s"$Tag workflow finished with status '$state'.") } - data.failures foreach { e => log.error(e, e.getMessage) } + private def issueSuccessReply(replyTo: ActorRef): State = { + replyTo.tell(msg = (), sender = self) // Because replyTo ! () is the parameterless call replyTo.!() + context.stop(self) + stay() + } - val message: Any = data.terminalState collect { case WorkflowSucceeded => () } getOrElse Status.Failure(data.failures.head) - data.replyTo foreach { _ ! message } + private def issueFailureReply(replyTo: ActorRef, e: Throwable): State = { + replyTo ! e context.stop(self) - stay + stay() + } + + private def issueReply(data: CompletedSwraData) = { + data match { + case s: SucceededSwraData => issueSuccessReply(s.replyTo) + case f: FailedSwraData => issueFailureReply(f.replyTo, f.failure) + case a: AbortedSwraData => issueSuccessReply(a.replyTo) + + } } - private def failAndFinish(e: Throwable): State = { + private def failAndFinish(e: Throwable, data: SwraData): State = { log.error(e, s"$Tag received Failure message: ${e.getMessage}") - issueReply(stateData.addFailure(e)) + data match { + case EmptySwraData => + log.error(e, s"Cannot issue response for {}. Need a 'replyTo' address to issue the response!", e.getMessage) + context.stop(self) + stay() + case SubmittedSwraData(replyTo) => + issueFailureReply(replyTo, e) + case RunningSwraData(replyTo, _) => + issueFailureReply(replyTo, e) + case c: CompletedSwraData => + issueFailureReply(c.replyTo, e) + } } /** @@ -174,19 +201,26 @@ object SingleWorkflowRunnerActor { sealed trait RunnerState case object NotStarted extends RunnerState + case object SubmittedWorkflow extends RunnerState case object RunningWorkflow extends RunnerState case object RequestingOutputs extends RunnerState case object RequestingMetadata extends RunnerState - final case class RunnerData(replyTo: Option[ActorRef] = None, - terminalState: Option[WorkflowState] = None, - id: Option[WorkflowId] = None, - failures: Seq[Throwable] = Seq.empty) { + sealed trait SwraData + case object EmptySwraData extends SwraData + final case class SubmittedSwraData(replyTo: ActorRef) extends SwraData + final case class RunningSwraData(replyTo: ActorRef, id: WorkflowId) extends SwraData - def addFailure(message: String): RunnerData = addFailure(new RuntimeException(message)) + sealed trait CompletedSwraData extends SwraData { def replyTo: ActorRef; def terminalState: WorkflowState; def id: WorkflowId } + final case class SucceededSwraData(replyTo: ActorRef, + id: WorkflowId) extends CompletedSwraData { override val terminalState = WorkflowSucceeded } - def addFailure(e: Throwable): RunnerData = this.copy(failures = e +: failures) - } + final case class FailedSwraData(replyTo: ActorRef, + id: WorkflowId, + failure: Throwable) extends CompletedSwraData { override val terminalState = WorkflowFailed } + + final case class AbortedSwraData(replyTo: ActorRef, + id: WorkflowId) extends CompletedSwraData { override val terminalState = WorkflowAborted } implicit class EnhancedJsObject(val jsObject: JsObject) extends AnyVal { def state: WorkflowState = WorkflowState.fromString(jsObject.fields("status").asInstanceOf[JsString].value) From eecbd7ccea7a61a05f0a4d9ac8c8ddff44ab6e5b Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Mon, 24 Oct 2016 13:54:40 -0400 Subject: [PATCH 044/375] Better failure responding --- .../main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala index c1561777b..8e5608b6e 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala @@ -138,7 +138,7 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: } private def issueFailureReply(replyTo: ActorRef, e: Throwable): State = { - replyTo ! e + replyTo ! Status.Failure(e) context.stop(self) stay() } From 321b73eec7f3d900796cfb6ee79fbdf2b8d6e168 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 25 Oct 2016 17:46:20 -0400 Subject: [PATCH 045/375] PR fixup --- .../workflow/SingleWorkflowRunnerActor.scala | 26 +++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala index 8e5608b6e..5300c1eb5 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala @@ -35,7 +35,7 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: extends CromwellRootActor with LoggingFSM[RunnerState, SwraData] { import SingleWorkflowRunnerActor._ - private val backoff = SimpleExponentialBackoff(1 second, 1 minute, 1.05) + private val backoff = SimpleExponentialBackoff(1 second, 1 minute, 1.2) override lazy val workflowStore = new InMemoryWorkflowStore() override lazy val jobStoreActor = context.actorOf(EmptyJobStoreActor.props) @@ -73,20 +73,20 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: goto(RequestingOutputs) using SucceededSwraData(replyTo, id) case Event(RequestComplete((StatusCodes.OK, jsObject: JsObject)), RunningSwraData(replyTo, id)) if jsObject.state == WorkflowFailed => log.info(s"$Tag workflow finished with status '$WorkflowFailed'.") - requestMetadataOrIssueReply(FailedSwraData(replyTo, id, new RuntimeException(s"Workflow $id transitioned to state Failed"))) + requestMetadataOrIssueReply(FailedSwraData(replyTo, id, new RuntimeException(s"Workflow $id transitioned to state $WorkflowFailed"))) case Event(RequestComplete((StatusCodes.OK, jsObject: JsObject)), RunningSwraData(replyTo, id)) if jsObject.state == WorkflowAborted => log.info(s"$Tag workflow finished with status '$WorkflowAborted'.") requestMetadataOrIssueReply(AbortedSwraData(replyTo, id)) } when (RequestingOutputs) { - case Event(RequestComplete((StatusCodes.OK, outputs: JsObject)), data: CompletedSwraData) => + case Event(RequestComplete((StatusCodes.OK, outputs: JsObject)), data: TerminalSwraData) => outputOutputs(outputs) requestMetadataOrIssueReply(data) } when (RequestingMetadata) { - case Event(RequestComplete((StatusCodes.OK, metadata: JsObject)), data: CompletedSwraData) => + case Event(RequestComplete((StatusCodes.OK, metadata: JsObject)), data: TerminalSwraData) => outputMetadata(metadata) issueReply(data) } @@ -109,9 +109,9 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: stay() } - private def requestMetadataOrIssueReply(newData: CompletedSwraData) = if (metadataOutputPath.isDefined) requestMetadata(newData) else issueReply(newData) + private def requestMetadataOrIssueReply(newData: TerminalSwraData) = if (metadataOutputPath.isDefined) requestMetadata(newData) else issueReply(newData) - private def requestMetadata(newData: CompletedSwraData) = { + private def requestMetadata(newData: TerminalSwraData) = { val metadataBuilder = context.actorOf(MetadataBuilderActor.props(serviceRegistryActor), s"MetadataRequest-Workflow-${newData.id}") metadataBuilder ! GetSingleWorkflowMetadataAction(newData.id, None, None) goto(RequestingMetadata) using newData @@ -143,7 +143,7 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: stay() } - private def issueReply(data: CompletedSwraData) = { + private def issueReply(data: TerminalSwraData) = { data match { case s: SucceededSwraData => issueSuccessReply(s.replyTo) case f: FailedSwraData => issueFailureReply(f.replyTo, f.failure) @@ -156,14 +156,14 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: log.error(e, s"$Tag received Failure message: ${e.getMessage}") data match { case EmptySwraData => - log.error(e, s"Cannot issue response for {}. Need a 'replyTo' address to issue the response!", e.getMessage) + log.error(e, "Cannot issue response. Need a 'replyTo' address to issue the exception response") context.stop(self) stay() case SubmittedSwraData(replyTo) => issueFailureReply(replyTo, e) case RunningSwraData(replyTo, _) => issueFailureReply(replyTo, e) - case c: CompletedSwraData => + case c: TerminalSwraData => issueFailureReply(c.replyTo, e) } } @@ -211,16 +211,16 @@ object SingleWorkflowRunnerActor { final case class SubmittedSwraData(replyTo: ActorRef) extends SwraData final case class RunningSwraData(replyTo: ActorRef, id: WorkflowId) extends SwraData - sealed trait CompletedSwraData extends SwraData { def replyTo: ActorRef; def terminalState: WorkflowState; def id: WorkflowId } + sealed trait TerminalSwraData extends SwraData { def replyTo: ActorRef; def terminalState: WorkflowState; def id: WorkflowId } final case class SucceededSwraData(replyTo: ActorRef, - id: WorkflowId) extends CompletedSwraData { override val terminalState = WorkflowSucceeded } + id: WorkflowId) extends TerminalSwraData { override val terminalState = WorkflowSucceeded } final case class FailedSwraData(replyTo: ActorRef, id: WorkflowId, - failure: Throwable) extends CompletedSwraData { override val terminalState = WorkflowFailed } + failure: Throwable) extends TerminalSwraData { override val terminalState = WorkflowFailed } final case class AbortedSwraData(replyTo: ActorRef, - id: WorkflowId) extends CompletedSwraData { override val terminalState = WorkflowAborted } + id: WorkflowId) extends TerminalSwraData { override val terminalState = WorkflowAborted } implicit class EnhancedJsObject(val jsObject: JsObject) extends AnyVal { def state: WorkflowState = WorkflowState.fromString(jsObject.fields("status").asInstanceOf[JsString].value) From d50275e5b07dda8db817a795f89d2c8ba50952b0 Mon Sep 17 00:00:00 2001 From: Thib Date: Thu, 27 Oct 2016 10:39:16 -0400 Subject: [PATCH 046/375] Update cromwell to latest wdl4s (#1610) * Update WDL4s --- .../backend/BackendJobExecutionActor.scala | 1 + .../cromwell/backend/BackendLifecycleActor.scala | 2 +- .../backend/BackendLifecycleActorFactory.scala | 4 +- .../BackendWorkflowInitializationActor.scala | 2 +- .../scala/cromwell/backend/OutputEvaluator.scala | 31 ------ .../backend/RuntimeAttributeDefinition.scala | 4 +- .../src/main/scala/cromwell/backend/backend.scala | 7 +- .../main/scala/cromwell/backend/wdl/Command.scala | 31 ++++++ .../cromwell/backend/wdl/OutputEvaluator.scala | 19 ++++ .../scala/cromwell/backend/wdl/PureFunctions.scala | 4 +- .../test/scala/cromwell/backend/BackendSpec.scala | 32 +++++-- .../scala/cromwell/backend/io/JobPathsSpec.scala | 28 +++--- .../scala/cromwell/backend/io/TestWorkflows.scala | 14 +-- .../cromwell/backend/io/WorkflowPathsSpec.scala | 8 +- .../cromwell/core/simpleton/WdlValueBuilder.scala | 2 +- .../core/simpleton/WdlValueBuilderSpec.scala | 6 +- core/src/test/scala/cromwell/util/SampleWdl.scala | 12 +-- .../MaterializeWorkflowDescriptorActor.scala | 18 ++-- .../lifecycle/WorkflowFinalizationActor.scala | 2 +- .../lifecycle/WorkflowInitializationActor.scala | 2 +- .../lifecycle/execution/JobPreparationActor.scala | 70 ++++---------- .../workflow/lifecycle/execution/WdlLookup.scala | 106 --------------------- .../execution/WorkflowExecutionActor.scala | 45 ++++++--- .../execution/WorkflowExecutionActorData.scala | 17 ++-- .../callcaching/EngineJobHashingActor.scala | 4 +- .../test/scala/cromwell/ArrayWorkflowSpec.scala | 8 +- .../test/scala/cromwell/CromwellTestkitSpec.scala | 4 +- .../scala/cromwell/DeclarationWorkflowSpec.scala | 4 +- .../src/test/scala/cromwell/MapWorkflowSpec.scala | 8 +- .../scala/cromwell/OptionalParamWorkflowSpec.scala | 10 +- .../scala/cromwell/SimpleWorkflowActorSpec.scala | 12 +-- .../cromwell/engine/WorkflowManagerActorSpec.scala | 2 +- .../mock/DefaultBackendJobExecutionActor.scala | 2 +- .../RetryableBackendLifecycleActorFactory.scala | 2 +- .../cromwell/engine/backend/mock/package.scala | 2 +- .../workflow/SingleWorkflowRunnerActorSpec.scala | 6 +- .../MaterializeWorkflowDescriptorActorSpec.scala | 10 +- .../execution/WorkflowExecutionActorSpec.scala | 2 +- .../callcaching/EngineJobHashingActorSpec.scala | 6 +- .../lifecycle/execution/ejea/PerTestHelper.scala | 18 ++-- .../cromwell/jobstore/JobStoreServiceSpec.scala | 3 +- .../webservice/MetadataBuilderActorSpec.scala | 5 +- project/Dependencies.scala | 2 +- src/bin/travis/resources/centaur.inputs | 10 +- src/bin/travis/resources/centaur.wdl | 2 +- src/bin/travis/testCentaurJes.sh | 4 +- .../backend/impl/aws/AwsJobExecutionActor.scala | 5 +- .../impl/htcondor/HtCondorBackendFactory.scala | 2 +- .../htcondor/HtCondorInitializationActor.scala | 4 +- .../impl/htcondor/HtCondorJobExecutionActor.scala | 15 +-- .../htcondor/HtCondorInitializationActorSpec.scala | 4 +- .../htcondor/HtCondorJobExecutionActorSpec.scala | 11 +-- .../htcondor/HtCondorRuntimeAttributesSpec.scala | 7 +- .../provider/mongodb/MongoCacheActorSpec.scala | 2 +- .../jes/JesAsyncBackendJobExecutionActor.scala | 56 ++--------- .../impl/jes/JesBackendLifecycleActorFactory.scala | 4 +- .../backend/impl/jes/JesFinalizationActor.scala | 4 +- .../backend/impl/jes/JesInitializationActor.scala | 4 +- .../jes/JesAsyncBackendJobExecutionActorSpec.scala | 52 +++++----- .../backend/impl/jes/JesCallPathsSpec.scala | 10 +- .../impl/jes/JesInitializationActorSpec.scala | 4 +- .../backend/impl/jes/JesWorkflowPathsSpec.scala | 4 +- .../sfs/config/ConfigAsyncJobExecutionActor.scala | 11 ++- .../impl/sfs/config/DeclarationValidation.scala | 16 ++-- .../cromwell/backend/sfs/SharedFileSystem.scala | 11 +-- .../SharedFileSystemAsyncJobExecutionActor.scala | 21 ++-- ...redFileSystemBackendLifecycleActorFactory.scala | 2 +- .../sfs/SharedFileSystemInitializationActor.scala | 4 +- .../SharedFileSystemInitializationActorSpec.scala | 4 +- .../SharedFileSystemJobExecutionActorSpec.scala | 15 ++- .../backend/sfs/SharedFileSystemSpec.scala | 10 +- .../backend/impl/spark/SparkBackendFactory.scala | 2 +- .../impl/spark/SparkInitializationActor.scala | 4 +- .../impl/spark/SparkJobExecutionActor.scala | 12 ++- .../impl/spark/SparkInitializationActorSpec.scala | 4 +- .../impl/spark/SparkJobExecutionActorSpec.scala | 7 +- .../impl/spark/SparkRuntimeAttributesSpec.scala | 9 +- 77 files changed, 402 insertions(+), 511 deletions(-) delete mode 100644 backend/src/main/scala/cromwell/backend/OutputEvaluator.scala create mode 100644 backend/src/main/scala/cromwell/backend/wdl/Command.scala create mode 100644 backend/src/main/scala/cromwell/backend/wdl/OutputEvaluator.scala delete mode 100644 engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WdlLookup.scala diff --git a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala index 93f1605fe..7bbdc27b2 100644 --- a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala @@ -6,6 +6,7 @@ import akka.actor.ActorLogging import akka.event.LoggingReceive import cromwell.backend.BackendJobExecutionActor._ import cromwell.backend.BackendLifecycleActor._ +import cromwell.backend.wdl.OutputEvaluator import cromwell.core.{ExecutionEvent, JobOutputs} import wdl4s.expression.WdlStandardLibraryFunctions import wdl4s.values.WdlValue diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala index 58ef72643..1856f1d85 100644 --- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala @@ -65,7 +65,7 @@ trait BackendWorkflowLifecycleActor extends BackendLifecycleActor with WorkflowL /** * The subset of calls which this backend will be expected to run */ - protected def calls: Seq[Call] + protected def calls: Set[Call] } trait BackendJobLifecycleActor extends BackendLifecycleActor with JobLogging { diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala index cbc47c0e5..c4b0efe14 100644 --- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala +++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala @@ -16,7 +16,7 @@ import wdl4s.expression.WdlStandardLibraryFunctions trait BackendLifecycleActorFactory { def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Seq[Call], + calls: Set[Call], serviceRegistryActor: ActorRef): Option[Props] = None def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, @@ -38,7 +38,7 @@ trait BackendLifecycleActorFactory { def backendSingletonActorProps: Option[Props] = None def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Seq[Call], + calls: Set[Call], executionStore: ExecutionStore, outputStore: OutputStore, initializationData: Option[BackendInitializationData]): Option[Props] = None diff --git a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala index f98234ce5..b8fc011df 100644 --- a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala @@ -36,7 +36,7 @@ object BackendWorkflowInitializationActor { trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor with ActorLogging { val serviceRegistryActor: ActorRef - def calls: Seq[Call] + def calls: Set[Call] /** * This method is meant only as a "pre-flight check" validation of runtime attribute expressions during workflow diff --git a/backend/src/main/scala/cromwell/backend/OutputEvaluator.scala b/backend/src/main/scala/cromwell/backend/OutputEvaluator.scala deleted file mode 100644 index ec7cf0c38..000000000 --- a/backend/src/main/scala/cromwell/backend/OutputEvaluator.scala +++ /dev/null @@ -1,31 +0,0 @@ -package cromwell.backend - -import cromwell.core.{JobOutputs, JobOutput} -import wdl4s._ -import wdl4s.expression.WdlStandardLibraryFunctions -import wdl4s.util.TryUtil -import wdl4s.values.WdlValue - -import scala.util.{Success, Try} - -object OutputEvaluator { - def evaluateOutputs(jobDescriptor: BackendJobDescriptor, - wdlFunctions: WdlStandardLibraryFunctions, - postMapper: WdlValue => Try[WdlValue] = v => Success(v)): Try[JobOutputs] = { - val inputs = jobDescriptor.inputs - val evaluatedOutputs = jobDescriptor.call.task.outputs. - foldLeft(Map.empty[LocallyQualifiedName, Try[JobOutput]])((outputMap, output) => { - val currentOutputs = outputMap collect { - case (name, value) if value.isSuccess => name -> value.get.wdlValue - } - def lookup = (currentOutputs ++ inputs).apply _ - val coerced = output.requiredExpression.evaluate(lookup, wdlFunctions) flatMap output.wdlType.coerceRawValue - val jobOutput = output.name -> (coerced flatMap postMapper map JobOutput) - - outputMap + jobOutput - - }) - - TryUtil.sequenceMap(evaluatedOutputs, s"Workflow ${jobDescriptor.workflowDescriptor.id} post processing failed.") - } -} diff --git a/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala b/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala index 238309143..ae9d4b9bf 100644 --- a/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala +++ b/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala @@ -20,8 +20,8 @@ object RuntimeAttributeDefinition { def evaluateRuntimeAttributes(unevaluated: RuntimeAttributes, wdlFunctions: WdlStandardLibraryFunctions, - evaluatedInputs: Map[LocallyQualifiedName, WdlValue]): Try[Map[String, WdlValue]] = { - val tryInputs = evaluatedInputs map { case (x, y) => x -> Success(y) } + evaluatedInputs: Map[Declaration, WdlValue]): Try[Map[String, WdlValue]] = { + val tryInputs = evaluatedInputs map { case (x, y) => x.unqualifiedName -> Success(y) } val mapBasedLookup = buildMapBasedLookup(tryInputs) _ val mapOfTries = unevaluated.attrs mapValues { expr => expr.evaluate(mapBasedLookup, wdlFunctions) diff --git a/backend/src/main/scala/cromwell/backend/backend.scala b/backend/src/main/scala/cromwell/backend/backend.scala index 8ac55a347..d950eb8ff 100644 --- a/backend/src/main/scala/cromwell/backend/backend.scala +++ b/backend/src/main/scala/cromwell/backend/backend.scala @@ -4,7 +4,7 @@ import com.typesafe.config.Config import cromwell.core.WorkflowOptions.WorkflowOption import cromwell.core.{JobKey, WorkflowId, WorkflowOptions} import wdl4s.values.WdlValue -import wdl4s.{Call, NamespaceWithWorkflow, _} +import wdl4s.{Call, WdlNamespaceWithWorkflow, _} import scala.util.Try @@ -25,7 +25,8 @@ case class BackendJobDescriptorKey(call: Call, index: Option[Int], attempt: Int) case class BackendJobDescriptor(workflowDescriptor: BackendWorkflowDescriptor, key: BackendJobDescriptorKey, runtimeAttributes: Map[LocallyQualifiedName, WdlValue], - inputs: Map[LocallyQualifiedName, WdlValue]) { + inputDeclarations: EvaluatedTaskInputs) { + val fullyQualifiedInputs = inputDeclarations map { case (declaration, value) => declaration.fullyQualifiedName -> value } val call = key.call override val toString = s"${key.mkTag(workflowDescriptor.id)}" } @@ -34,7 +35,7 @@ case class BackendJobDescriptor(workflowDescriptor: BackendWorkflowDescriptor, * For passing to a BackendActor construction time */ case class BackendWorkflowDescriptor(id: WorkflowId, - workflowNamespace: NamespaceWithWorkflow, + workflowNamespace: WdlNamespaceWithWorkflow, inputs: Map[FullyQualifiedName, WdlValue], workflowOptions: WorkflowOptions) { override def toString: String = s"[BackendWorkflowDescriptor id=${id.shortString} workflowName=${workflowNamespace.workflow.unqualifiedName}]" diff --git a/backend/src/main/scala/cromwell/backend/wdl/Command.scala b/backend/src/main/scala/cromwell/backend/wdl/Command.scala new file mode 100644 index 000000000..b0e3c9294 --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/wdl/Command.scala @@ -0,0 +1,31 @@ +package cromwell.backend.wdl + +import cromwell.backend.BackendJobDescriptor +import wdl4s.EvaluatedTaskInputs +import wdl4s.expression.WdlFunctions +import wdl4s.values.WdlValue + +import scala.util.{Success, Try} + +object Command { + + /** + * Instantiate the command for this job descriptor. + * + * @param jobDescriptor jobDescriptor to instantiate the command for + * @param callEngineFunction engine functions to use to evaluate expressions inside the command + * @param inputsPreProcessor function to be applied to the task inputs before they are used to instantiate the command + * Typically this is where localization and/or file path transformation work would be done. + * The return value of the function is the inputs map that will be used to resolve variables in the command line. + * @param valueMapper function to apply, during instantiation of the command line, after a variable is resolved + * @return + */ + def instantiate(jobDescriptor: BackendJobDescriptor, + callEngineFunction: WdlFunctions[WdlValue], + inputsPreProcessor: EvaluatedTaskInputs => Try[EvaluatedTaskInputs] = (i: EvaluatedTaskInputs) => Success(i), + valueMapper: WdlValue => WdlValue = identity): Try[String] = { + inputsPreProcessor(jobDescriptor.inputDeclarations) flatMap { mappedInputs => + jobDescriptor.call.task.instantiateCommand(mappedInputs, callEngineFunction, valueMapper) + } + } +} diff --git a/backend/src/main/scala/cromwell/backend/wdl/OutputEvaluator.scala b/backend/src/main/scala/cromwell/backend/wdl/OutputEvaluator.scala new file mode 100644 index 000000000..ab6c66b0d --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/wdl/OutputEvaluator.scala @@ -0,0 +1,19 @@ +package cromwell.backend.wdl + +import cromwell.backend.BackendJobDescriptor +import cromwell.core.JobOutput +import wdl4s.LocallyQualifiedName +import wdl4s.expression.WdlStandardLibraryFunctions +import wdl4s.values.WdlValue + +import scala.util.{Success, Try} + +object OutputEvaluator { + def evaluateOutputs(jobDescriptor: BackendJobDescriptor, + wdlFunctions: WdlStandardLibraryFunctions, + postMapper: WdlValue => Try[WdlValue] = v => Success(v)): Try[Map[LocallyQualifiedName, JobOutput]] = { + jobDescriptor.call.task.evaluateOutputs(jobDescriptor.inputDeclarations, wdlFunctions, postMapper) map { outputs => + outputs mapValues JobOutput + } + } +} diff --git a/backend/src/main/scala/cromwell/backend/wdl/PureFunctions.scala b/backend/src/main/scala/cromwell/backend/wdl/PureFunctions.scala index dac9194d6..583b6e2f3 100644 --- a/backend/src/main/scala/cromwell/backend/wdl/PureFunctions.scala +++ b/backend/src/main/scala/cromwell/backend/wdl/PureFunctions.scala @@ -20,7 +20,7 @@ case object OnlyPureFunctions extends WdlStandardLibraryFunctions with PureFunct trait PureFunctions { this: WdlStandardLibraryFunctions => - def transpose(params: Seq[Try[WdlValue]]): Try[WdlArray] = { + override def transpose(params: Seq[Try[WdlValue]]): Try[WdlArray] = { def extractExactlyOneArg: Try[WdlValue] = params.size match { case 1 => params.head case n => Failure(new IllegalArgumentException(s"Invalid number of parameters for engine function transpose: $n. Ensure transpose(x: Array[Array[X]]) takes exactly 1 parameters.")) @@ -49,7 +49,7 @@ trait PureFunctions { this: WdlStandardLibraryFunctions => extractExactlyOneArg.flatMap(validateAndExpand).flatMap(transpose) } - def range(params: Seq[Try[WdlValue]]): Try[WdlArray] = { + override def range(params: Seq[Try[WdlValue]]): Try[WdlArray] = { def extractAndValidateArguments = params.size match { case 1 => validateArguments(params.head) case n => Failure(new IllegalArgumentException(s"Invalid number of parameters for engine function range: $n. Ensure range(x: WdlInteger) takes exactly 1 parameters.")) diff --git a/backend/src/test/scala/cromwell/backend/BackendSpec.scala b/backend/src/test/scala/cromwell/backend/BackendSpec.scala index c5617ab43..522fa54a1 100644 --- a/backend/src/test/scala/cromwell/backend/BackendSpec.scala +++ b/backend/src/test/scala/cromwell/backend/BackendSpec.scala @@ -7,12 +7,13 @@ import cromwell.core.{WorkflowId, WorkflowOptions} import org.scalatest.Matchers import org.scalatest.concurrent.ScalaFutures import org.scalatest.time.{Millis, Seconds, Span} +import org.specs2.mock.Mockito import spray.json.{JsObject, JsValue} import wdl4s._ import wdl4s.expression.NoFunctions import wdl4s.values.WdlValue -trait BackendSpec extends ScalaFutures with Matchers { +trait BackendSpec extends ScalaFutures with Matchers with Mockito { implicit val defaultPatience = PatienceConfig(timeout = Span(5, Seconds), interval = Span(500, Millis)) @@ -26,21 +27,32 @@ trait BackendSpec extends ScalaFutures with Matchers { runtime: String = "") = { BackendWorkflowDescriptor( WorkflowId.randomId(), - NamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime)), + WdlNamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime)), inputs, options ) } + def fqnMapToDeclarationMap(m: Map[String, WdlValue]): Map[Declaration, WdlValue] = { + m map { + case (fqn, v) => + val mockDeclaration = mock[Declaration] + mockDeclaration.fullyQualifiedName returns fqn + mockDeclaration.unqualifiedName returns fqn.split('.').lastOption.getOrElse(fqn) + mockDeclaration -> v + } + } + def jobDescriptorFromSingleCallWorkflow(workflowDescriptor: BackendWorkflowDescriptor, inputs: Map[String, WdlValue], options: WorkflowOptions, runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition]): BackendJobDescriptor = { val call = workflowDescriptor.workflowNamespace.workflow.calls.head val jobKey = BackendJobDescriptorKey(call, None, 1) - val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, inputs).get // .get is OK here because this is a test + val inputDeclarations = call.evaluateTaskInputs(inputs, NoFunctions) + val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, inputDeclarations).get // .get is OK here because this is a test val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes) - BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputs) + BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations) } def jobDescriptorFromSingleCallWorkflow(wdl: WdlSource, @@ -49,9 +61,10 @@ trait BackendSpec extends ScalaFutures with Matchers { val workflowDescriptor = buildWorkflowDescriptor(wdl) val call = workflowDescriptor.workflowNamespace.workflow.calls.head val jobKey = BackendJobDescriptorKey(call, None, 1) - val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, workflowDescriptor.inputs).get // .get is OK here because this is a test + val inputDeclarations = fqnMapToDeclarationMap(workflowDescriptor.inputs) + val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, inputDeclarations).get // .get is OK here because this is a test val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes) - BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, workflowDescriptor.inputs) + BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations) } def jobDescriptorFromSingleCallWorkflow(wdl: WdlSource, @@ -62,9 +75,10 @@ trait BackendSpec extends ScalaFutures with Matchers { val workflowDescriptor = buildWorkflowDescriptor(wdl, runtime = runtime) val call = workflowDescriptor.workflowNamespace.workflow.calls.head val jobKey = BackendJobDescriptorKey(call, None, attempt) - val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, workflowDescriptor.inputs).get // .get is OK here because this is a test + val inputDeclarations = fqnMapToDeclarationMap(workflowDescriptor.inputs) + val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, inputDeclarations).get // .get is OK here because this is a test val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes) - BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, workflowDescriptor.inputs) + BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations) } def assertResponse(executionResponse: BackendJobExecutionResponse, expectedResponse: BackendJobExecutionResponse) = { @@ -103,7 +117,7 @@ trait BackendSpec extends ScalaFutures with Matchers { def firstJobDescriptor(workflowDescriptor: BackendWorkflowDescriptor, inputs: Map[String, WdlValue] = Map.empty) = { - BackendJobDescriptor(workflowDescriptor, firstJobDescriptorKey(workflowDescriptor), Map.empty, inputs) + BackendJobDescriptor(workflowDescriptor, firstJobDescriptorKey(workflowDescriptor), Map.empty, fqnMapToDeclarationMap(inputs)) } } diff --git a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala index f03ca2cf9..6de5450f7 100644 --- a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala +++ b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala @@ -37,41 +37,41 @@ class JobPathsSpec extends FlatSpec with Matchers with BackendSpec { val jobPaths = new JobPaths(wd, backendConfig, jobKey) val id = wd.id jobPaths.callRoot.toString shouldBe - File(s"local-cromwell-executions/hello/$id/call-hello").pathAsString + File(s"local-cromwell-executions/wf_hello/$id/call-hello").pathAsString jobPaths.callExecutionRoot.toString shouldBe - File(s"local-cromwell-executions/hello/$id/call-hello/execution").pathAsString + File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution").pathAsString jobPaths.returnCode.toString shouldBe - File(s"local-cromwell-executions/hello/$id/call-hello/execution/rc").pathAsString + File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/rc").pathAsString jobPaths.script.toString shouldBe - File(s"local-cromwell-executions/hello/$id/call-hello/execution/script").pathAsString + File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/script").pathAsString jobPaths.stderr.toString shouldBe - File(s"local-cromwell-executions/hello/$id/call-hello/execution/stderr").pathAsString + File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/stderr").pathAsString jobPaths.stdout.toString shouldBe - File(s"local-cromwell-executions/hello/$id/call-hello/execution/stdout").pathAsString + File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/stdout").pathAsString jobPaths.callExecutionRoot.toString shouldBe - File(s"local-cromwell-executions/hello/$id/call-hello/execution").pathAsString + File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution").pathAsString jobPaths.callDockerRoot.toString shouldBe - File(s"/root/hello/$id/call-hello").pathAsString + File(s"/root/wf_hello/$id/call-hello").pathAsString jobPaths.callExecutionDockerRoot.toString shouldBe - File(s"/root/hello/$id/call-hello/execution").pathAsString - jobPaths.toDockerPath(Paths.get(s"local-cromwell-executions/hello/$id/call-hello/execution/stdout")).toString shouldBe - File(s"/root/hello/$id/call-hello/execution/stdout").pathAsString + File(s"/root/wf_hello/$id/call-hello/execution").pathAsString + jobPaths.toDockerPath(Paths.get(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/stdout")).toString shouldBe + File(s"/root/wf_hello/$id/call-hello/execution/stdout").pathAsString jobPaths.toDockerPath(Paths.get("/root/dock/path")).toString shouldBe File("/root/dock/path").pathAsString val jobKeySharded = BackendJobDescriptorKey(call, Option(0), 1) val jobPathsSharded = new JobPaths(wd, backendConfig, jobKeySharded) jobPathsSharded.callExecutionRoot.toString shouldBe - File(s"local-cromwell-executions/hello/$id/call-hello/shard-0/execution").pathAsString + File(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/execution").pathAsString val jobKeyAttempt = BackendJobDescriptorKey(call, None, 2) val jobPathsAttempt = new JobPaths(wd, backendConfig, jobKeyAttempt) jobPathsAttempt.callExecutionRoot.toString shouldBe - File(s"local-cromwell-executions/hello/$id/call-hello/attempt-2/execution").pathAsString + File(s"local-cromwell-executions/wf_hello/$id/call-hello/attempt-2/execution").pathAsString val jobKeyShardedAttempt = BackendJobDescriptorKey(call, Option(0), 2) val jobPathsShardedAttempt = new JobPaths(wd, backendConfig, jobKeyShardedAttempt) jobPathsShardedAttempt.callExecutionRoot.toString shouldBe - File(s"local-cromwell-executions/hello/$id/call-hello/shard-0/attempt-2/execution").pathAsString + File(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/attempt-2/execution").pathAsString } } diff --git a/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala b/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala index 34497e2ef..28d35977d 100644 --- a/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala +++ b/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala @@ -23,7 +23,7 @@ object TestWorkflows { | RUNTIME |} | - |workflow hello { + |workflow wf_hello { | call hello |} """.stripMargin @@ -39,7 +39,7 @@ object TestWorkflows { | } |} | - |workflow goodbye { + |workflow wf_goodbye { | call goodbye |} """.stripMargin @@ -61,7 +61,7 @@ object TestWorkflows { | RUNTIME |} | - |workflow localize { + |workflow wf_localize { | File workflowFile | call localize { input: inputFileFromCallInputs = workflowFile } |} @@ -76,7 +76,7 @@ object TestWorkflows { | } |} | - |workflow abort { + |workflow wf_abort { | call abort |} """.stripMargin @@ -93,7 +93,7 @@ object TestWorkflows { | } |} | - |workflow scattering { + |workflow wf_scattering { | Array[Int] numbers = [1, 2, 3] | scatter (i in numbers) { | call scattering { input: intNumber = i } @@ -117,7 +117,7 @@ object TestWorkflows { | } |} | - |workflow localize { + |workflow wf_localize { | call localize |} """.stripMargin @@ -133,7 +133,7 @@ object TestWorkflows { | } |} | - |workflow localize { + |workflow wf_localize { | call localize |} """.stripMargin diff --git a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala index bfae5930d..d2dfd1b20 100644 --- a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala +++ b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala @@ -3,12 +3,10 @@ package cromwell.backend.io import better.files._ import com.typesafe.config.Config import cromwell.backend.BackendSpec -import org.mockito.Matchers._ import org.mockito.Mockito._ -import org.scalatest.mockito.MockitoSugar import org.scalatest.{FlatSpec, Matchers} -class WorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec with MockitoSugar { +class WorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec { val backendConfig = mock[Config] @@ -19,8 +17,8 @@ class WorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec with Moc val workflowPaths = new WorkflowPaths(wd, backendConfig) val id = wd.id workflowPaths.workflowRoot.toString shouldBe - File(s"local-cromwell-executions/hello/$id").pathAsString + File(s"local-cromwell-executions/wf_hello/$id").pathAsString workflowPaths.dockerWorkflowRoot.toString shouldBe - s"/root/hello/$id" + s"/root/wf_hello/$id" } } diff --git a/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala b/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala index d4e04dd0b..f9d2464e7 100644 --- a/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala +++ b/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala @@ -119,7 +119,7 @@ object WdlValueBuilder { // This is meant to "rehydrate" simpletonized WdlValues back to WdlValues. It is assumed that these WdlValues were // "dehydrated" to WdlValueSimpletons correctly. This code is not robust to corrupt input whatsoever. - val types = taskOutputs map { o => o.name -> o.wdlType } toMap + val types = taskOutputs map { o => o.unqualifiedName -> o.wdlType } toMap val simpletonsByOutputName = simpletons groupBy { _.simpletonKey match { case IdentifierAndPathPattern(i, _) => i } } val simpletonComponentsByOutputName = simpletonsByOutputName map { case (name, ss) => name -> (ss map simpletonToComponent(name)) } types map { case (name, outputType) => name -> toWdlValue(outputType, simpletonComponentsByOutputName(name))} diff --git a/core/src/test/scala/cromwell/core/simpleton/WdlValueBuilderSpec.scala b/core/src/test/scala/cromwell/core/simpleton/WdlValueBuilderSpec.scala index eab604c96..0d126e1bd 100644 --- a/core/src/test/scala/cromwell/core/simpleton/WdlValueBuilderSpec.scala +++ b/core/src/test/scala/cromwell/core/simpleton/WdlValueBuilderSpec.scala @@ -2,6 +2,8 @@ package cromwell.core.simpleton import cromwell.core.simpleton.WdlValueBuilderSpec._ import org.scalatest.{FlatSpec, Matchers} +import org.specs2.mock.Mockito +import wdl4s.parser.WdlParser.Ast import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlMapType, WdlStringType} import wdl4s.values.{WdlArray, WdlInteger, WdlMap, WdlString} import wdl4s.{TaskOutput, WdlExpression} @@ -11,7 +13,7 @@ object WdlValueBuilderSpec { val IgnoredExpression = WdlExpression.fromString(""" "" """) } -class WdlValueBuilderSpec extends FlatSpec with Matchers { +class WdlValueBuilderSpec extends FlatSpec with Matchers with Mockito { "Builder" should "build" in { @@ -42,7 +44,7 @@ class WdlValueBuilderSpec extends FlatSpec with Matchers { )) ) - val taskOutputs = wdlValues map { case (k, wv) => TaskOutput(k, wv.wdlType, IgnoredExpression) } + val taskOutputs = wdlValues map { case (k, wv) => TaskOutput(k, wv.wdlType, IgnoredExpression, mock[Ast], None) } import WdlValueSimpleton._ val actual = WdlValueBuilder.toWdlValues(taskOutputs, wdlValues.simplify) diff --git a/core/src/test/scala/cromwell/util/SampleWdl.scala b/core/src/test/scala/cromwell/util/SampleWdl.scala index dc2598cf1..28718d1e9 100644 --- a/core/src/test/scala/cromwell/util/SampleWdl.scala +++ b/core/src/test/scala/cromwell/util/SampleWdl.scala @@ -74,14 +74,14 @@ object SampleWdl { | RUNTIME |} | - |workflow hello { + |workflow wf_hello { | call hello |} """.stripMargin.replaceAll("RUNTIME", runtime) - val Addressee = "hello.hello.addressee" + val Addressee = "wf_hello.hello.addressee" val rawInputs = Map(Addressee -> "world") - val OutputKey = "hello.hello.salutation" + val OutputKey = "wf_hello.hello.salutation" val OutputValue = "Hello world!" } @@ -117,7 +117,7 @@ object SampleWdl { | } |} | - |workflow goodbye { + |workflow wf_goodbye { | call goodbye |} """.stripMargin @@ -149,7 +149,7 @@ object SampleWdl { | } |} | - |workflow hello { + |workflow wf_hello { | call hello | call goodbye {input: emptyInputString=hello.empty } | output { @@ -509,7 +509,7 @@ object SampleWdl { | RUNTIME |} | - |workflow whereami { + |workflow wf_whereami { | call whereami |} """.stripMargin.replaceAll("RUNTIME", runtime) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala index 15e7edbd5..fcd187133 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala @@ -182,7 +182,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor private def buildWorkflowDescriptor(id: WorkflowId, sourceFiles: WorkflowSourceFiles, - namespace: NamespaceWithWorkflow, + namespace: WdlNamespaceWithWorkflow, workflowOptions: WorkflowOptions, conf: Config, pathBuilders: List[PathBuilder]): ErrorOr[EngineWorkflowDescriptor] = { @@ -200,7 +200,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor } private def buildWorkflowDescriptor(id: WorkflowId, - namespace: NamespaceWithWorkflow, + namespace: WdlNamespaceWithWorkflow, rawInputs: Map[String, JsValue], backendAssignments: Map[Call, String], workflowOptions: WorkflowOptions, @@ -209,7 +209,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor callCachingMode: CallCachingMode): ErrorOr[EngineWorkflowDescriptor] = { def checkTypes(inputs: Map[FullyQualifiedName, WdlValue]): ErrorOr[Map[FullyQualifiedName, WdlValue]] = { - val allDeclarations = namespace.workflow.scopedDeclarations ++ namespace.workflow.calls.flatMap(_.scopedDeclarations) + val allDeclarations = namespace.workflow.declarations ++ namespace.workflow.calls.flatMap(_.declarations) val list: List[ErrorOr[(FullyQualifiedName, WdlValue)]] = inputs.map({ case (k, v) => allDeclarations.find(_.fullyQualifiedName == k) match { case Some(decl) if decl.wdlType.coerceRawValue(v).isFailure => @@ -245,7 +245,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor serviceRegistryActor ! PutMetadataAction(inputEvents) } - private def validateBackendAssignments(calls: Seq[Call], workflowOptions: WorkflowOptions, defaultBackendName: Option[String]): ErrorOr[Map[Call, String]] = { + private def validateBackendAssignments(calls: Set[Call], workflowOptions: WorkflowOptions, defaultBackendName: Option[String]): ErrorOr[Map[Call, String]] = { val callToBackendMap = Try { calls map { call => val backendPriorities = Seq( @@ -289,19 +289,19 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor } } - private def validateDeclarations(namespace: NamespaceWithWorkflow, + private def validateDeclarations(namespace: WdlNamespaceWithWorkflow, options: WorkflowOptions, coercedInputs: WorkflowCoercedInputs, pathBuilders: List[PathBuilder]): ErrorOr[WorkflowCoercedInputs] = { - namespace.staticWorkflowDeclarationsRecursive(coercedInputs, new WdlFunctions(pathBuilders)) match { + namespace.staticDeclarationsRecursive(coercedInputs, new WdlFunctions(pathBuilders)) match { case Success(d) => d.validNel case Failure(e) => s"Workflow has invalid declarations: ${e.getMessage}".invalidNel } } - private def validateNamespace(source: WdlSource): ErrorOr[NamespaceWithWorkflow] = { + private def validateNamespace(source: WdlSource): ErrorOr[WdlNamespaceWithWorkflow] = { try { - NamespaceWithWorkflow.load(source).validNel + WdlNamespaceWithWorkflow.load(source).validNel } catch { case e: Exception => s"Unable to load namespace from workflow: ${e.getMessage}".invalidNel } @@ -316,7 +316,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor } private def validateCoercedInputs(rawInputs: Map[String, JsValue], - namespace: NamespaceWithWorkflow): ErrorOr[WorkflowCoercedInputs] = { + namespace: WdlNamespaceWithWorkflow): ErrorOr[WorkflowCoercedInputs] = { namespace.coerceRawInputs(rawInputs) match { case Success(r) => r.validNel case Failure(e: ExceptionWithErrors) => Invalid(e.errors) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala index 9614696e2..d237bbc7b 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala @@ -62,7 +62,7 @@ case class WorkflowFinalizationActor(workflowId: WorkflowId, workflowDescriptor: case Event(StartFinalizationCommand, _) => val backendFinalizationActors = Try { for { - (backend, calls) <- workflowDescriptor.backendAssignments.groupBy(_._2).mapValues(_.keys.toSeq) + (backend, calls) <- workflowDescriptor.backendAssignments.groupBy(_._2).mapValues(_.keySet) props <- CromwellBackends.backendLifecycleFactoryActorByName(backend).map( _.workflowFinalizationActorProps(workflowDescriptor.backendDescriptor, calls, executionStore, outputStore, initializationData.get(backend)) ).get diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala index 2fd5d75aa..98ebf2564 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala @@ -78,7 +78,7 @@ case class WorkflowInitializationActor(workflowId: WorkflowId, case Event(StartInitializationCommand, _) => val backendInitializationActors = Try { for { - (backend, calls) <- workflowDescriptor.backendAssignments.groupBy(_._2).mapValues(_.keys.toSeq) + (backend, calls) <- workflowDescriptor.backendAssignments.groupBy(_._2).mapValues(_.keySet) props <- CromwellBackends.backendLifecycleFactoryActorByName(backend).map(factory => factory.workflowInitializationActorProps(workflowDescriptor.backendDescriptor, calls, serviceRegistryActor) ).get diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala index 12c719994..4cdc50a78 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala @@ -8,7 +8,6 @@ import cromwell.engine.EngineWorkflowDescriptor import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor._ import wdl4s._ import wdl4s.expression.WdlStandardLibraryFunctions -import wdl4s.util.TryUtil import wdl4s.values.WdlValue import scala.util.{Failure, Success, Try} @@ -19,13 +18,13 @@ final case class JobPreparationActor(executionData: WorkflowExecutionActorData, initializationData: Option[BackendInitializationData], serviceRegistryActor: ActorRef, backendSingletonActor: Option[ActorRef]) - extends Actor with WdlLookup with WorkflowLogging { + extends Actor with WorkflowLogging { - override lazy val workflowDescriptor: EngineWorkflowDescriptor = executionData.workflowDescriptor - override lazy val workflowId = workflowDescriptor.id - override lazy val executionStore: ExecutionStore = executionData.executionStore - override lazy val outputStore: OutputStore = executionData.outputStore - override lazy val expressionLanguageFunctions = factory.expressionLanguageFunctions( + lazy val workflowDescriptor: EngineWorkflowDescriptor = executionData.workflowDescriptor + lazy val workflowId = workflowDescriptor.id + lazy val executionStore: ExecutionStore = executionData.executionStore + lazy val outputStore: OutputStore = executionData.outputStore + lazy val expressionLanguageFunctions = factory.expressionLanguageFunctions( workflowDescriptor.backendDescriptor, jobKey, initializationData) override def receive = { @@ -37,54 +36,25 @@ final case class JobPreparationActor(executionData: WorkflowExecutionActorData, case unhandled => workflowLogger.warn(self.path.name + " received an unhandled message: " + unhandled) } - // Split inputs map (= evaluated workflow declarations + coerced json inputs) into [init\.*].last - private lazy val splitInputs = workflowDescriptor.backendDescriptor.inputs map { case (fqn, v) => splitFqn(fqn) -> v } - def resolveAndEvaluateInputs(jobKey: BackendJobDescriptorKey, - wdlFunctions: WdlStandardLibraryFunctions): Try[Map[LocallyQualifiedName, WdlValue]] = { - import RuntimeAttributeDefinition.buildMapBasedLookup + wdlFunctions: WdlStandardLibraryFunctions): Try[Map[Declaration, WdlValue]] = { Try { val call = jobKey.call - lazy val callInputsFromFile = unqualifiedInputsFromInputFile(call) - lazy val workflowScopedLookup = hierarchicalLookup(jobKey.call, jobKey.index) _ - - // Try to resolve, evaluate and coerce declarations in order - val inputEvaluationAttempt = call.task.declarations.foldLeft(Map.empty[LocallyQualifiedName, Try[WdlValue]])((inputs, declaration) => { - val name = declaration.name - - // Try to resolve the declaration, and upon success evaluate the expression - // If the declaration is resolved but can't be evaluated this will throw an evaluation exception - // If it can't be resolved it's ignored and won't appear in the final input map - val evaluated: Option[Try[WdlValue]] = declaration.expression match { - // Static expression in the declaration - case Some(expr) => Option(expr.evaluate(buildMapBasedLookup(inputs), wdlFunctions)) - // Expression found in the input mappings - case None if call.inputMappings.contains(name) => Option(call.inputMappings(name).evaluate(workflowScopedLookup, wdlFunctions)) - // Expression found in the input file - case None if callInputsFromFile.contains(name) => Option(Success(callInputsFromFile(name))) - // Expression can't be found - case _ => None - } - - // Leave out unresolved declarations - evaluated match { - case Some(value) => - val coercedValue = value flatMap declaration.wdlType.coerceRawValue - inputs + ((name, coercedValue)) - case None => inputs - } - }) - - TryUtil.sequenceMap(inputEvaluationAttempt, s"Input evaluation for Call ${call.fullyQualifiedName} failed") - }.flatten - } - - // Unqualified call inputs for a specific call, from the input json - private def unqualifiedInputsFromInputFile(call: Call): Map[LocallyQualifiedName, WdlValue] = splitInputs collect { - case((root, inputName), v) if root == call.fullyQualifiedName => inputName -> v + val scatterMap = jobKey.index flatMap { i => + // Will need update for nested scatters + call.upstream collectFirst { case s: Scatter => Map(s -> i) } + } getOrElse Map.empty[Scatter, Int] + + call.evaluateTaskInputs( + workflowDescriptor.backendDescriptor.inputs, + expressionLanguageFunctions, + outputStore.fetchCallOutputEntries, + scatterMap + ) + } } - private def prepareJobExecutionActor(inputEvaluation: Map[LocallyQualifiedName, WdlValue]): JobPreparationActorResponse = { + private def prepareJobExecutionActor(inputEvaluation: Map[Declaration, WdlValue]): JobPreparationActorResponse = { import RuntimeAttributeDefinition.{addDefaultsToAttributes, evaluateRuntimeAttributes} val curriedAddDefaultsToAttributes = addDefaultsToAttributes(factory.runtimeAttributeDefinitions(initializationData), workflowDescriptor.backendDescriptor.workflowOptions) _ diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WdlLookup.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WdlLookup.scala deleted file mode 100644 index 8b2af57ea..000000000 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WdlLookup.scala +++ /dev/null @@ -1,106 +0,0 @@ -package cromwell.engine.workflow.lifecycle.execution - -import cromwell.core.{ExecutionIndex, ExecutionStore, OutputStore} -import cromwell.engine.EngineWorkflowDescriptor -import ExecutionIndex._ -import wdl4s._ -import wdl4s.expression.WdlStandardLibraryFunctions -import wdl4s.values.{WdlArray, WdlCallOutputsObject, WdlValue} - -import scala.language.postfixOps -import scala.util.{Failure, Success, Try} - -trait WdlLookup { - - def workflowDescriptor: EngineWorkflowDescriptor - def executionStore: ExecutionStore - def outputStore: OutputStore - def expressionLanguageFunctions: WdlStandardLibraryFunctions - - private lazy val splitInputs = workflowDescriptor.backendDescriptor.inputs map { - case (fqn, v) => splitFqn(fqn) -> v - } - - // Unqualified workflow level inputs - private lazy val unqualifiedWorkflowInputs: Map[LocallyQualifiedName, WdlValue] = splitInputs collect { - case((root, inputName), v) if root == workflowDescriptor.namespace.workflow.unqualifiedName => inputName -> v - } - - /** - * Lookup an identifier by - * first looking at the completed calls map - * and if not found traversing up the scope hierarchy from the scope from which the lookup originated. - */ - def hierarchicalLookup(scope: Scope, index: ExecutionIndex)(identifier: String): WdlValue = { - // First lookup calls - lookupCall(scope, index, identifier) recoverWith { - // Lookup in the same scope (currently no scope support this but say we have scatter declarations, or multiple scatter variables, or nested workflows..) - case _: VariableNotFoundException | _: WdlExpressionException => scopedLookup(scope, index, identifier) - } recover { - // Lookup parent if present - case _: VariableNotFoundException | _: WdlExpressionException => scope.parent match { - case Some(parent) => hierarchicalLookup(parent, index)(identifier) - case None => throw new VariableNotFoundException(s"Can't find $identifier") - } - } get - } - - private def scopedLookup(scope: Scope, index: ExecutionIndex, identifier: String): Try[WdlValue] = { - def scopedLookupFunction = scope match { - case scatter: Scatter if index.isDefined => lookupScatter(scatter, index.get) _ - case workflow: Workflow => lookupWorkflowDeclaration _ - case _ => (_: String) => Failure(new VariableNotFoundException(s"Can't find $identifier in scope $scope")) - } - - scopedLookupFunction(identifier) - } - - // In this case, the scopedLookup function is effectively equivalent to looking into unqualifiedWorkflowInputs for the value - // because the resolution / evaluation / coercion has already happened in the MaterializeWorkflowDescriptorActor - private def lookupWorkflowDeclaration(identifier: String) = { - unqualifiedWorkflowInputs.get(identifier) match { - case Some(value) => Success(value) - case None => Failure(new WdlExpressionException(s"Could not resolve variable $identifier as a workflow input")) - } - } - - private def lookupScatter(scatter: Scatter, index: Int)(identifier: String): Try[WdlValue] = { - if (identifier == scatter.item) { - // Scatters are not indexed yet (they can't be nested) - val scatterLookup = hierarchicalLookup(scatter, None) _ - scatter.collection.evaluate(scatterLookup, expressionLanguageFunctions) map { - case collection: WdlArray if collection.value.isDefinedAt(index) => collection.value(index) - case collection: WdlArray => throw new RuntimeException(s"Index $index out of bound in $collection for scatter ${scatter.fullyQualifiedName}") - case other => throw new RuntimeException(s"Scatter ${scatter.fullyQualifiedName} collection is not an array: $other") - } recover { - case e => throw new RuntimeException(s"Failed to evaluate collection for scatter ${scatter.fullyQualifiedName}", e) - } - } else { - Failure(new VariableNotFoundException(identifier)) - } - } - - private def lookupCall(scope: Scope, scopeIndex: ExecutionIndex, identifier: String): Try[WdlCallOutputsObject] = { - val calls = executionStore.store.keys.view map { _.scope } collect { case c: Call => c } - - calls find { _.unqualifiedName == identifier } match { - case Some(matchedCall) => - /** - * After matching the Call, this determines if the `key` depends on a single shard - * of a scatter'd job or if it depends on the whole thing. Right now, the heuristic - * is "If we're both in a scatter block together, then I depend on a shard. If not, - * I depend on the collected value" - * - * TODO: nested-scatter - this will likely not be sufficient for nested scatters - */ - val index: ExecutionIndex = matchedCall.closestCommonAncestor(scope) flatMap { - case s: Scatter => scopeIndex - case _ => None - } - - outputStore.fetchCallOutputEntries(matchedCall, index) - case None => Failure(new WdlExpressionException(s"Could not find a call with identifier '$identifier'")) - } - } - -} diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index c1ffaa527..1b357efae 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -164,8 +164,18 @@ object WorkflowExecutionActor { } private def arePrerequisitesDone(key: JobKey): Boolean = { - val upstream = key.scope.prerequisiteScopes.toList.map(s => upstreamEntries(key, s)) - val downstream = key match { + val upstream = key.scope match { + case node: GraphNode => node.upstream collect { + // Only scatters and calls are in the execution store for now (not declarations) + // However declarations are nodes so they can be an upstream dependency + // We don't want to look for those in the execution store (yet ?) since upstreamEntry would return None + case n: Call => upstreamEntry(key, n) + case n: Scatter => upstreamEntry(key, n) + } + case _ => Set.empty + } + + val downstream: List[(JobKey, ExecutionStatus)] = key match { case collector: CollectorKey => findShardEntries(collector) case _ => Nil } @@ -182,7 +192,7 @@ object WorkflowExecutionActor { (upstream forall { _.nonEmpty }) && dependenciesResolved } - private def upstreamEntries(entry: JobKey, prerequisiteScope: Scope): Seq[ExecutionStoreEntry] = { + private def upstreamEntry(entry: JobKey, prerequisiteScope: Scope): Option[ExecutionStoreEntry] = { prerequisiteScope.closestCommonAncestor(entry.scope) match { /** * If this entry refers to a Scope which has a common ancestor with prerequisiteScope @@ -194,18 +204,18 @@ object WorkflowExecutionActor { * work as-is for nested scatter blocks */ case Some(ancestor: Scatter) => - executionStore.store filter { + executionStore.store find { case (k, _) => k.scope == prerequisiteScope && k.index == entry.index - } toSeq + } /** * Otherwise, simply refer to the entry the collector entry. This means that 'entry' depends * on every shard of the pre-requisite scope to finish. */ case _ => - executionStore.store filter { + executionStore.store find { case (k, _) => k.scope == prerequisiteScope && k.index.isEmpty - } toSeq + } } } } @@ -224,9 +234,9 @@ object WorkflowExecutionActor { } collector.scope.task.outputs map { taskOutput => val wdlValues = shardsOutputs.map( - _.getOrElse(taskOutput.name, throw new RuntimeException(s"Could not retrieve output ${taskOutput.name}"))) + _.getOrElse(taskOutput.unqualifiedName, throw new RuntimeException(s"Could not retrieve output ${taskOutput.unqualifiedName}"))) val arrayOfValues = new WdlArray(WdlArrayType(taskOutput.wdlType), wdlValues) - taskOutput.name -> JobOutput(arrayOfValues) + taskOutput.unqualifiedName -> JobOutput(arrayOfValues) } toMap } } @@ -286,11 +296,12 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, val workflow = workflowDescriptor.backendDescriptor.workflowNamespace.workflow // Only add direct children to the store, the rest is dynamically created when necessary val keys = workflow.children map { - case call: Call => BackendJobDescriptorKey(call, None, 1) - case scatter: Scatter => ScatterKey(scatter) + case call: Call => Option(BackendJobDescriptorKey(call, None, 1)) + case scatter: Scatter => Option(ScatterKey(scatter)) + case _ => None // FIXME there are other types of scopes now (Declarations, Ifs) Figure out what to do with those } - ExecutionStore(keys.map(_ -> NotStarted).toMap) + ExecutionStore(keys.flatten.map(_ -> NotStarted).toMap) } private def handleNonRetryableFailure(stateData: WorkflowExecutionActorData, failedJobKey: JobKey, reason: Throwable) = { @@ -580,13 +591,13 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, } private def pushRunningJobMetadata(jobDescriptor: BackendJobDescriptor) = { - val inputEvents = jobDescriptor.inputs match { + val inputEvents = jobDescriptor.inputDeclarations match { case empty if empty.isEmpty => List(MetadataEvent.empty(metadataKey(jobDescriptor.key, s"${CallMetadataKeys.Inputs}"))) case inputs => inputs flatMap { case (inputName, inputValue) => - wdlValueToMetadataEvents(metadataKey(jobDescriptor.key, s"${CallMetadataKeys.Inputs}:$inputName"), inputValue) + wdlValueToMetadataEvents(metadataKey(jobDescriptor.key, s"${CallMetadataKeys.Inputs}:${inputName.unqualifiedName}"), inputValue) } } @@ -621,7 +632,11 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, } private def processRunnableScatter(scatterKey: ScatterKey, data: WorkflowExecutionActorData): Try[WorkflowExecutionDiff] = { - val lookup = data.hierarchicalLookup(scatterKey.scope, None) _ + val lookup = scatterKey.scope.lookupFunction( + workflowDescriptor.workflowInputs, + data.expressionLanguageFunctions, + data.outputStore.fetchCallOutputEntries + ) scatterKey.scope.collection.evaluate(lookup, data.expressionLanguageFunctions) map { case a: WdlArray => WorkflowExecutionDiff(scatterKey.populate(a.value.size) + (scatterKey -> ExecutionStatus.Done)) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala index 3a844e6d2..96e4f1658 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala @@ -6,7 +6,7 @@ import cromwell.core.OutputStore.{OutputCallKey, OutputEntry} import cromwell.core._ import cromwell.engine.{EngineWorkflowDescriptor, WdlFunctions} import cromwell.util.JsonFormatting.WdlValueJsonFormatter -import wdl4s.Scope +import wdl4s.{GraphNode, Scope} object WorkflowExecutionDiff { @@ -20,9 +20,9 @@ final case class WorkflowExecutionDiff(executionStore: Map[JobKey, ExecutionStat case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescriptor, executionStore: ExecutionStore, backendJobExecutionActors: Map[JobKey, ActorRef], - outputStore: OutputStore) extends WdlLookup { + outputStore: OutputStore) { - override val expressionLanguageFunctions = new WdlFunctions(workflowDescriptor.pathBuilders) + val expressionLanguageFunctions = new WdlFunctions(workflowDescriptor.pathBuilders) def jobExecutionSuccess(jobKey: JobKey, outputs: JobOutputs) = this.copy( executionStore = executionStore.add(Map(jobKey -> Done)), @@ -43,9 +43,14 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript * If complete, this will return Some(finalStatus). Otherwise, returns None */ def workflowCompletionStatus: Option[ExecutionStatus] = { // `List`ify the `prerequisiteScopes` to avoid expensive hashing of `Scope`s when assembling the result. - def upstream(scope: Scope): List[Scope] = scope.prerequisiteScopes.toList ++ scope.prerequisiteScopes.toList.flatMap(upstream) - def upstreamFailed(scope: Scope) = upstream(scope) filter { s => - executionStore.store.exists({ case (key, status) => status == Failed && key.scope == s }) + def upstream(scope: GraphNode): List[Scope] = { + val directUpstream: List[Scope with GraphNode] = scope.upstream.toList + directUpstream ++ directUpstream.flatMap(upstream) + } + def upstreamFailed(scope: Scope) = scope match { + case node: GraphNode => upstream(node) filter { s => + executionStore.store.exists({ case (key, status) => status == Failed && key.scope == s }) + } } // activeJobs is the subset of the executionStore that are either running or will run in the future. val activeJobs = executionStore.store.toList filter { diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala index b4ad358f5..053e8a14e 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala @@ -65,7 +65,7 @@ case class EngineJobHashingActor(receiver: ActorRef, import cromwell.core.simpleton.WdlValueSimpleton._ - val inputSimpletons = jobDescriptor.inputs.simplify + val inputSimpletons = jobDescriptor.fullyQualifiedInputs.simplify val (fileInputSimpletons, nonFileInputSimpletons) = inputSimpletons partition { case WdlValueSimpleton(_, f: WdlFile) => true case _ => false @@ -107,7 +107,7 @@ case class EngineJobHashingActor(receiver: ActorRef, } val outputExpressionHashResults = jobDescriptor.call.task.outputs map { output => - HashResult(HashKey(s"output expression: ${output.wdlType.toWdlString} ${output.name}"), output.requiredExpression.valueString.md5HashValue) + HashResult(HashKey(s"output expression: ${output.wdlType.toWdlString} ${output.unqualifiedName}"), output.requiredExpression.valueString.md5HashValue) } // Build these all together for the final set of initial hashes: diff --git a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala index 843796c1a..44393924f 100644 --- a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala @@ -5,7 +5,7 @@ import java.nio.file.Files import akka.testkit._ import better.files._ import cromwell.util.SampleWdl -import wdl4s.NamespaceWithWorkflow +import wdl4s.WdlNamespaceWithWorkflow import wdl4s.expression.NoFunctions import wdl4s.types.{WdlArrayType, WdlFileType, WdlStringType} import wdl4s.values.{WdlArray, WdlFile, WdlInteger, WdlString} @@ -13,7 +13,7 @@ import wdl4s.values.{WdlArray, WdlFile, WdlInteger, WdlString} class ArrayWorkflowSpec extends CromwellTestkitSpec { val tmpDir = Files.createTempDirectory("ArrayWorkflowSpec") - val ns = NamespaceWithWorkflow.load(SampleWdl.ArrayLiteral(tmpDir).wdlSource("")) + val ns = WdlNamespaceWithWorkflow.load(SampleWdl.ArrayLiteral(tmpDir).wdlSource("")) val expectedArray = WdlArray(WdlArrayType(WdlFileType), Seq(WdlFile("f1"), WdlFile("f2"), WdlFile("f3"))) "A task which contains a parameter " should { @@ -32,7 +32,7 @@ class ArrayWorkflowSpec extends CromwellTestkitSpec { "A static Array[File] declaration" should { "be a valid declaration" in { - val declaration = ns.workflow.declarations.find {_.name == "arr"}.getOrElse { + val declaration = ns.workflow.declarations.find {_.unqualifiedName == "arr"}.getOrElse { fail("Expected declaration 'arr' to be found") } val expression = declaration.expression.getOrElse { @@ -47,7 +47,7 @@ class ArrayWorkflowSpec extends CromwellTestkitSpec { val catTask = ns.findTask("cat").getOrElse { fail("Expected to find task 'cat'") } - val command = catTask.instantiateCommand(Map("files" -> expectedArray), NoFunctions).getOrElse { + val command = catTask.instantiateCommand(catTask.inputsFromMap(Map("cat.files" -> expectedArray)), NoFunctions).getOrElse { fail("Expected instantiation to work") } command shouldEqual "cat -s f1 f2 f3" diff --git a/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala b/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala index 090d94fd1..8dba0882a 100644 --- a/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala +++ b/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala @@ -43,7 +43,7 @@ import scala.util.matching.Regex case class TestBackendLifecycleActorFactory(configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Seq[Call], + calls: Set[Call], serviceRegistryActor: ActorRef): Option[Props] = None override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, @@ -280,7 +280,7 @@ abstract class CromwellTestkitSpec(val twms: TestWorkflowManagerSystem = new Cro override protected def afterAll() = { twms.shutdownTestActorSystem(); () } - implicit val defaultPatience = PatienceConfig(timeout = Span(30, Seconds), interval = Span(100, Millis)) + implicit val defaultPatience = PatienceConfig(timeout = Span(200, Seconds), interval = Span(1000, Millis)) implicit val ec = system.dispatcher val dummyServiceRegistryActor = system.actorOf(Props.empty) diff --git a/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala b/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala index d829b84eb..1786733be 100644 --- a/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala @@ -1,7 +1,7 @@ package cromwell import wdl4s.types.{WdlFileType, WdlStringType} -import wdl4s.{NamespaceWithWorkflow, WorkflowInput} +import wdl4s.{WdlNamespaceWithWorkflow, WorkflowInput} import cromwell.util.SampleWdl import org.scalatest.{Matchers, WordSpecLike} @@ -9,7 +9,7 @@ import org.scalatest.{Matchers, WordSpecLike} class DeclarationWorkflowSpec extends Matchers with WordSpecLike { "A workflow with declarations in it" should { "compute inputs properly" in { - NamespaceWithWorkflow.load(SampleWdl.DeclarationsWorkflow.wdlSource(runtime="")).workflow.inputs shouldEqual Map( + WdlNamespaceWithWorkflow.load(SampleWdl.DeclarationsWorkflow.wdlSource(runtime="")).workflow.inputs shouldEqual Map( "two_step.cat.file" -> WorkflowInput("two_step.cat.file", WdlFileType, postfixQuantifier = None), "two_step.cgrep.str_decl" -> WorkflowInput("two_step.cgrep.str_decl", WdlStringType, postfixQuantifier = None), "two_step.cgrep.pattern" -> WorkflowInput("two_step.cgrep.pattern", WdlStringType, postfixQuantifier = None), diff --git a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala index 9a00c115e..03ec39426 100644 --- a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala @@ -3,7 +3,7 @@ package cromwell import akka.testkit._ import better.files._ import cromwell.util.SampleWdl -import wdl4s.NamespaceWithWorkflow +import wdl4s.WdlNamespaceWithWorkflow import wdl4s.expression.{NoFunctions, WdlFunctions} import wdl4s.types.{WdlFileType, WdlIntegerType, WdlMapType, WdlStringType} import wdl4s.values._ @@ -13,7 +13,7 @@ import scala.util.{Success, Try} class MapWorkflowSpec extends CromwellTestkitSpec { private val pwd = File(".") private val sampleWdl = SampleWdl.MapLiteral(pwd.path) - val ns = NamespaceWithWorkflow.load(sampleWdl.wdlSource("")) + val ns = WdlNamespaceWithWorkflow.load(sampleWdl.wdlSource("")) val expectedMap = WdlMap(WdlMapType(WdlFileType, WdlStringType), Map( WdlFile("f1") -> WdlString("alice"), WdlFile("f2") -> WdlString("bob"), @@ -42,7 +42,7 @@ class MapWorkflowSpec extends CromwellTestkitSpec { "A static Map[File, String] declaration" should { "be a valid declaration" in { - val declaration = ns.workflow.declarations.find {_.name == "map"}.getOrElse { + val declaration = ns.workflow.declarations.find {_.unqualifiedName == "map"}.getOrElse { fail("Expected declaration 'map' to be found") } val expression = declaration.expression.getOrElse { @@ -64,7 +64,7 @@ class MapWorkflowSpec extends CromwellTestkitSpec { case _ => throw new UnsupportedOperationException("Only write_map should be called") } } - val command = writeMapTask.instantiateCommand(Map("file_to_name" -> expectedMap), new CannedFunctions).getOrElse { + val command = writeMapTask.instantiateCommand(writeMapTask.inputsFromMap(Map("file_to_name" -> expectedMap)), new CannedFunctions).getOrElse { fail("Expected instantiation to work") } command shouldEqual "cat /test/map/path" diff --git a/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala b/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala index 919008315..cded78cc4 100644 --- a/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala @@ -27,15 +27,15 @@ class OptionalParamWorkflowSpec extends Matchers with WordSpecLike { fail("Expected to find task 'find'") } - val instantiateWithoutValue = findTask.instantiateCommand(Map("root" -> WdlFile("src")), NoFunctions) getOrElse { + val instantiateWithoutValue = findTask.instantiateCommand(findTask.inputsFromMap(Map("find.root" -> WdlFile("src"))), NoFunctions) getOrElse { fail("Expected instantiation to work") } instantiateWithoutValue shouldEqual "find src" - val instantiateWithValue = findTask.instantiateCommand(Map( - "root" -> WdlFile("src"), - "pattern" -> WdlString("*.java") - ), NoFunctions).getOrElse {fail("Expected instantiation to work")} + val instantiateWithValue = findTask.instantiateCommand(findTask.inputsFromMap(Map( + "find.root" -> WdlFile("src"), + "find.pattern" -> WdlString("*.java") + )), NoFunctions).getOrElse {fail("Expected instantiation to work")} instantiateWithValue shouldEqual "find src -name *.java" } } diff --git a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala index f676f170d..d2d9929f7 100644 --- a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala @@ -64,7 +64,7 @@ class SimpleWorkflowActorSpec extends CromwellTestkitSpec with BeforeAndAfter { val TestableWorkflowActorAndMetadataPromise(workflowActor, supervisor, _) = buildWorkflowActor(SampleWdl.HelloWorld, SampleWdl.HelloWorld.wdlJson, workflowId) val probe = TestProbe() probe watch workflowActor - startingCallsFilter("hello.hello") { + startingCallsFilter("wf_hello.hello") { workflowActor ! StartWorkflowCommand } @@ -75,7 +75,7 @@ class SimpleWorkflowActorSpec extends CromwellTestkitSpec with BeforeAndAfter { } "fail to construct with missing inputs" in { - val expectedError = "Required workflow input 'hello.hello.addressee' not specified." + val expectedError = "Required workflow input 'wf_hello.hello.addressee' not specified." val failureMatcher = FailureMatcher(expectedError) val TestableWorkflowActorAndMetadataPromise(workflowActor, supervisor, promise) = buildWorkflowActor(SampleWdl.HelloWorld, "{}", workflowId, failureMatcher) val probe = TestProbe() @@ -92,7 +92,7 @@ class SimpleWorkflowActorSpec extends CromwellTestkitSpec with BeforeAndAfter { } "fail to construct with inputs of the wrong type" in { - val expectedError = "Could not coerce value for 'hello.hello.addressee' into: WdlStringType" + val expectedError = "Could not coerce value for 'wf_hello.hello.addressee' into: WdlStringType" val failureMatcher = FailureMatcher(expectedError) val TestableWorkflowActorAndMetadataPromise(workflowActor, supervisor, promise) = buildWorkflowActor(SampleWdl.HelloWorld, s""" { "$Addressee" : 3} """, workflowId, failureMatcher) @@ -111,12 +111,12 @@ class SimpleWorkflowActorSpec extends CromwellTestkitSpec with BeforeAndAfter { } "fail when a call fails" in { - val expectedError = "Call goodbye.goodbye:-1:1: return code was 1" + val expectedError = "Call wf_goodbye.goodbye:NA:1: return code was 1" val failureMatcher = FailureMatcher(expectedError) val TestableWorkflowActorAndMetadataPromise(workflowActor, supervisor, promise) = buildWorkflowActor(SampleWdl.GoodbyeWorld, SampleWdl.GoodbyeWorld.wdlJson, workflowId, failureMatcher) val probe = TestProbe() probe watch workflowActor - startingCallsFilter("goodbye.goodbye") { + startingCallsFilter("wf_goodbye.goodbye") { workflowActor ! StartWorkflowCommand } Await.result(promise.future, TestExecutionTimeout) @@ -130,7 +130,7 @@ class SimpleWorkflowActorSpec extends CromwellTestkitSpec with BeforeAndAfter { } "gracefully handle malformed WDL" in { - val expectedError = "Input evaluation for Call test1.summary failedVariable 'Can't find bfile' not found" + val expectedError = "Input evaluation for Call test1.summary failed.\nVariable 'bfile' not found" val failureMatcher = FailureMatcher(expectedError) val TestableWorkflowActorAndMetadataPromise(workflowActor, supervisor, promise) = buildWorkflowActor(SampleWdl.CoercionNotDefined, SampleWdl.CoercionNotDefined.wdlJson, workflowId, failureMatcher) val probe = TestProbe() diff --git a/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala b/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala index 55faea29f..1d8fbb2db 100644 --- a/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala @@ -13,7 +13,7 @@ class WorkflowManagerActorSpec extends CromwellTestkitSpec with WorkflowDescript "run workflows in the correct directory" in { val outputs = runWdl(sampleWdl = SampleWdl.CurrentDirectory) - val outputName = "whereami.whereami.pwd" + val outputName = "wf_whereami.whereami.pwd" val salutation = outputs(outputName) val actualOutput = salutation.valueString.trim actualOutput should endWith("/call-whereami/execution") diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala b/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala index f98fa17bd..223b715bf 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala @@ -25,7 +25,7 @@ case class DefaultBackendJobExecutionActor(override val jobDescriptor: BackendJo class DefaultBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Seq[Call], + calls: Set[Call], serviceRegistryActor: ActorRef): Option[Props] = None override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala index 46f28f447..48636f014 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala @@ -8,7 +8,7 @@ import wdl4s.expression.{NoFunctions, WdlStandardLibraryFunctions} class RetryableBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Seq[Call], + calls: Set[Call], serviceRegistryActor: ActorRef): Option[Props] = None override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/package.scala b/engine/src/test/scala/cromwell/engine/backend/mock/package.scala index 4baeb9c33..a2f914121 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/package.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/package.scala @@ -9,7 +9,7 @@ package object mock { // This is used by stubbed backends that are to be used in tests to prepare dummy outputs for job def taskOutputToJobOutput(taskOutput: TaskOutput) = - taskOutput.name -> JobOutput(sampleValue(taskOutput.wdlType)) + taskOutput.unqualifiedName -> JobOutput(sampleValue(taskOutput.wdlType)) private def sampleValue(wdlType: WdlType): WdlValue = wdlType match { case WdlIntegerType => WdlInteger(3) diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala index ede3d57ff..983c188df 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala @@ -78,10 +78,8 @@ abstract class SingleWorkflowRunnerActorSpec extends CromwellTestkitSpec { def singleWorkflowActor(sampleWdl: SampleWdl = ThreeStep, managerActor: => ActorRef = workflowManagerActor(), outputFile: => Option[Path] = None): Unit = { - implicit val timeout = Timeout(TimeoutDuration) - val actorRef = createRunnerActor(sampleWdl, managerActor, outputFile) - val futureResult = actorRef ? RunWorkflow + val futureResult = actorRef.ask(RunWorkflow)(timeout = new Timeout(TimeoutDuration)) Await.ready(futureResult, Duration.Inf) () } @@ -202,7 +200,7 @@ class SingleWorkflowRunnerActorWithMetadataOnFailureSpec extends SingleWorkflowR val calls = metadata.get("calls").toFields calls should not be empty - val callSeq = calls("goodbye.goodbye").asInstanceOf[JsArray].elements + val callSeq = calls("wf_goodbye.goodbye").asInstanceOf[JsArray].elements callSeq should have size 1 val call = callSeq.head.asJsObject.fields val inputs = call.get("inputs").toFields diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala index 97e72e99d..057fa5456 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala @@ -62,10 +62,10 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestkitSpec with Be expectMsgPF() { case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => wfDesc.id shouldBe workflowId - wfDesc.name shouldBe "hello" + wfDesc.name shouldBe "wf_hello" wfDesc.namespace.tasks.size shouldBe 1 - wfDesc.workflowInputs.head shouldBe (("hello.hello.addressee", WdlString("world"))) - wfDesc.backendDescriptor.inputs.head shouldBe (("hello.hello.addressee", WdlString("world"))) + wfDesc.workflowInputs.head shouldBe (("wf_hello.hello.addressee", WdlString("world"))) + wfDesc.backendDescriptor.inputs.head shouldBe (("wf_hello.hello.addressee", WdlString("world"))) wfDesc.getWorkflowOption(WorkflowOptions.WriteToCache) shouldBe Option("true") wfDesc.getWorkflowOption(WorkflowOptions.ReadFromCache) shouldBe None // Default backend assignment is "Local": @@ -362,7 +362,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestkitSpec with Be within(Timeout) { expectMsgPF() { case MaterializeWorkflowDescriptorFailureResponse(reason) => - reason.getMessage should startWith("Workflow input processing failed.\nRequired workflow input 'hello.hello.addressee' not specified") + reason.getMessage should startWith("Workflow input processing failed.\nRequired workflow input 'wf_hello.hello.addressee' not specified") case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!") case unknown => fail(s"Unexpected materialization response: $unknown") @@ -388,7 +388,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestkitSpec with Be within(Timeout) { expectMsgPF() { case MaterializeWorkflowDescriptorFailureResponse(reason) => - reason.getMessage should startWith("Workflow input processing failed.\nInvalid right-side type of 'foo.j'. Expecting Int, got String") + reason.getMessage should startWith("Workflow input processing failed.\nUnable to load namespace from workflow: ERROR: Value for j is not coerceable into a Int") case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!") case unknown => fail(s"Unexpected materialization response: $unknown") } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala index 01af65d1a..f2136a69f 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala @@ -71,7 +71,7 @@ class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter "WorkflowExecutionActor") EventFilter.info(pattern = ".*Final Outputs", occurrences = 1).intercept { - EventFilter.info(pattern = "Starting calls: hello.hello", occurrences = 3).intercept { + EventFilter.info(pattern = "Starting calls: wf_hello.hello", occurrences = 3).intercept { workflowExecutionActor ! ExecuteWorkflowCommand } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala index d79a383fb..a4feebb01 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala @@ -5,7 +5,7 @@ import akka.testkit.{ImplicitSender, TestKit, TestProbe} import cats.data.NonEmptyList import cromwell.CromwellTestkitSpec import cromwell.backend.callcaching.FileHashingActor.{FileHashResponse, SingleFileHashRequest} -import cromwell.backend.{BackendInitializationData, BackendJobDescriptor, BackendJobDescriptorKey, BackendWorkflowDescriptor, RuntimeAttributeDefinition} +import cromwell.backend._ import cromwell.core.callcaching._ import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CacheMiss, CallCacheHashes} import org.scalatest.mockito.MockitoSugar @@ -168,7 +168,7 @@ class EngineJobHashingActorSpec extends TestKit(new CromwellTestkitSpec.TestWork } } -object EngineJobHashingActorSpec extends MockitoSugar { +object EngineJobHashingActorSpec extends BackendSpec { import org.mockito.Mockito._ def createEngineJobHashingActor @@ -203,7 +203,7 @@ object EngineJobHashingActorSpec extends MockitoSugar { when(task.outputs).thenReturn(List.empty) when(call.task).thenReturn(task) val workflowDescriptor = mock[BackendWorkflowDescriptor] - val jobDescriptor = BackendJobDescriptor(workflowDescriptor, BackendJobDescriptorKey(call, None, 1), Map.empty, inputs) + val jobDescriptor = BackendJobDescriptor(workflowDescriptor, BackendJobDescriptorKey(call, None, 1), Map.empty, fqnMapToDeclarationMap(inputs)) jobDescriptor } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala index f53bd605b..c3178e27f 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala @@ -20,6 +20,7 @@ import org.specs2.mock.Mockito import wdl4s.WdlExpression.ScopedLookupFunction import wdl4s._ import wdl4s.expression.{NoFunctions, WdlFunctions, WdlStandardLibraryFunctions} +import wdl4s.parser.WdlParser.Ast import wdl4s.types.{WdlIntegerType, WdlStringType} import wdl4s.values.{WdlInteger, WdlString, WdlValue} @@ -39,28 +40,31 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock val task = mock[Task] task.declarations returns Seq.empty - task.runtimeAttributes returns RuntimeAttributes(Map.empty) + task.runtimeAttributes returns new RuntimeAttributes(Map.empty) task.commandTemplateString returns "!!shazam!!" + task.name returns taskName val stringOutputExpression = mock[WdlExpression] stringOutputExpression.valueString returns "hello" stringOutputExpression.evaluate(any[ScopedLookupFunction], any[ WdlFunctions[WdlValue]]) returns Success(WdlString("hello")) - task.outputs returns Seq(TaskOutput("outString", WdlStringType, stringOutputExpression)) + task.outputs returns Seq(TaskOutput("outString", WdlStringType, stringOutputExpression, mock[Ast], Option(task))) val intInputExpression = mock[WdlExpression] intInputExpression.valueString returns "543" intInputExpression.evaluate(any[ScopedLookupFunction], any[WdlFunctions[WdlValue]]) returns Success(WdlInteger(543)) val intInputDeclaration = mock[Declaration] - intInputDeclaration.name returns "inInt" + intInputDeclaration.unqualifiedName returns "inInt" intInputDeclaration.expression returns Option(intInputExpression) intInputDeclaration.wdlType returns WdlIntegerType task.declarations returns Seq(intInputDeclaration) - val call: Call = Call(None, jobFqn, task, Set.empty, Map.empty, None) + val workflow = new Workflow(workflowName, Seq.empty, mock[Ast]) + val call: Call = new Call(None, task, Map.empty, mock[Ast]) + call.parent_=(workflow) val jobDescriptorKey = BackendJobDescriptorKey(call, jobIndex, jobAttempt) val backendWorkflowDescriptor = BackendWorkflowDescriptor(workflowId, null, null, null) - val backendJobDescriptor = BackendJobDescriptor(backendWorkflowDescriptor, jobDescriptorKey, runtimeAttributes = Map.empty, inputs = Map.empty) + val backendJobDescriptor = BackendJobDescriptor(backendWorkflowDescriptor, jobDescriptorKey, runtimeAttributes = Map.empty, inputDeclarations = Map.empty) var fetchCachedResultsActorCreations: ExpectOne[(CallCachingEntryId, Seq[TaskOutput])] = NothingYet var jobHashingInitializations: ExpectOne[(BackendJobDescriptor, CallCachingActivity)] = NothingYet @@ -94,12 +98,12 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock // These two factory methods should never be called from EJEA or any of its descendants: override def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Seq[Call], + calls: Set[Call], executionStore: ExecutionStore, outputStore: OutputStore, initializationData: Option[BackendInitializationData]): Option[Props] = throw new UnsupportedOperationException("Unexpected finalization actor creation!") override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Seq[Call], + calls: Set[Call], serviceRegistryActor: ActorRef): Option[Props] = throw new UnsupportedOperationException("Unexpected finalization actor creation!") } diff --git a/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala b/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala index fa1cc5067..abab45a69 100644 --- a/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala +++ b/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala @@ -8,6 +8,7 @@ import cromwell.jobstore.JobStoreServiceSpec._ import cromwell.services.SingletonServicesStore import org.scalatest.Matchers import org.specs2.mock.Mockito +import wdl4s.parser.WdlParser.Ast import wdl4s.types.WdlStringType import wdl4s.values.WdlString import wdl4s.{Call, Task, TaskOutput, WdlExpression} @@ -31,7 +32,7 @@ class JobStoreServiceSpec extends CromwellTestkitSpec with Matchers with Mockito val successCall = mock[Call] successCall.fullyQualifiedName returns "foo.bar" val mockTask = mock[Task] - mockTask.outputs returns Seq(TaskOutput("baz", WdlStringType, EmptyExpression)) + mockTask.outputs returns Seq(TaskOutput("baz", WdlStringType, EmptyExpression, mock[Ast], Option(mockTask))) successCall.task returns mockTask val successKey = BackendJobDescriptorKey(successCall, None, 1).toJobStoreKey(workflowId) diff --git a/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala b/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala index 5ae82221b..49b631876 100644 --- a/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala @@ -22,15 +22,14 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik behavior of "MetadataParser" - val defaultTimeout = 100 millis + val defaultTimeout = 200 millis val mockServiceRegistry = TestProbe() - val parentProbe = TestProbe() - def assertMetadataResponse(action: MetadataServiceAction, queryReply: MetadataQuery, events: Seq[MetadataEvent], expectedRes: String) = { + val parentProbe = TestProbe() val metadataBuilder = TestActorRef(MetadataBuilderActor.props(mockServiceRegistry.ref), parentProbe.ref, s"MetadataActor-${UUID.randomUUID()}") metadataBuilder ! action // Ask for everything mockServiceRegistry.expectMsg(defaultTimeout, action) // TestActor runs on CallingThreadDispatcher diff --git a/project/Dependencies.scala b/project/Dependencies.scala index db7fd5ceb..0d737e1d4 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -2,7 +2,7 @@ import sbt._ object Dependencies { lazy val lenthallV = "0.19" - lazy val wdl4sV = "0.7-90945ea-SNAPSHOT" + lazy val wdl4sV = "0.7-5302cbf-SNAPSHOT" lazy val sprayV = "1.3.3" /* spray-json is an independent project from the "spray suite" diff --git a/src/bin/travis/resources/centaur.inputs b/src/bin/travis/resources/centaur.inputs index c378c0f71..6a778986a 100644 --- a/src/bin/travis/resources/centaur.inputs +++ b/src/bin/travis/resources/centaur.inputs @@ -1,7 +1,7 @@ { - "centaur.centaur.cromwell_jar":"gs://cloud-cromwell-dev/travis-centaur/CROMWELL_JAR", - "centaur.centaur.cromwell_branch":"BRANCH", - "centaur.centaur.conf":"gs://cloud-cromwell-dev/travis-centaur/multiBackend.conf", - "centaur.centaur.pem":"gs://cloud-cromwell-dev/travis-centaur/cromwell-account.pem", - "centaur.centaur.token": "gs://cloud-cromwell-dev/travis-centaur/token.txt" + "centaur_workflow.centaur.cromwell_jar":"gs://cloud-cromwell-dev/travis-centaur/CROMWELL_JAR", + "centaur_workflow.centaur.cromwell_branch":"BRANCH", + "centaur_workflow.centaur.conf":"gs://cloud-cromwell-dev/travis-centaur/multiBackend.conf", + "centaur_workflow.centaur.pem":"gs://cloud-cromwell-dev/travis-centaur/cromwell-account.pem", + "centaur_workflow.centaur.token": "gs://cloud-cromwell-dev/travis-centaur/token.txt" } diff --git a/src/bin/travis/resources/centaur.wdl b/src/bin/travis/resources/centaur.wdl index 62e13c98c..7f7876407 100644 --- a/src/bin/travis/resources/centaur.wdl +++ b/src/bin/travis/resources/centaur.wdl @@ -26,6 +26,6 @@ task centaur { failOnStderr: false } } -workflow centaur { +workflow centaur_workflow { call centaur } diff --git a/src/bin/travis/testCentaurJes.sh b/src/bin/travis/testCentaurJes.sh index 0c92423c3..109ad032a 100755 --- a/src/bin/travis/testCentaurJes.sh +++ b/src/bin/travis/testCentaurJes.sh @@ -69,9 +69,9 @@ EXIT_CODE="${PIPESTATUS[0]}" export WORKFLOW_ID=`grep "SingleWorkflowRunnerActor: Workflow submitted " log.txt | perl -pe 's/\e\[?.*?[\@-~]//g' | cut -f7 -d" "` # Grab the Centaur log from GCS and cat it so we see it in the main travis log. -export CENTAUR_LOG_PATH="gs://cloud-cromwell-dev/cromwell_execution/travis/centaur/${WORKFLOW_ID}/call-centaur/cromwell_root/logs/centaur.log" +export CENTAUR_LOG_PATH="gs://cloud-cromwell-dev/cromwell_execution/travis/centaur_workflow/${WORKFLOW_ID}/call-centaur/cromwell_root/logs/centaur.log" gsutil cp ${CENTAUR_LOG_PATH} centaur.log cat centaur.log -echo "More logs for this run are available at https://console.cloud.google.com/storage/browser/cloud-cromwell-dev/cromwell_execution/travis/centaur/${WORKFLOW_ID}/call-centaur/" +echo "More logs for this run are available at https://console.cloud.google.com/storage/browser/cloud-cromwell-dev/cromwell_execution/travis/centaur_workflow/${WORKFLOW_ID}/call-centaur/" exit "${EXIT_CODE}" diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsJobExecutionActor.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsJobExecutionActor.scala index 3b3015a36..a81a6cf7e 100644 --- a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsJobExecutionActor.scala +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsJobExecutionActor.scala @@ -8,7 +8,7 @@ import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, B import com.amazonaws.services.ecs.model._ import cromwell.backend.impl.aws.util.AwsSdkAsyncHandler import cromwell.backend.impl.aws.util.AwsSdkAsyncHandler.AwsSdkAsyncResult -import cromwell.backend.wdl.OnlyPureFunctions +import cromwell.backend.wdl.{Command, OnlyPureFunctions} import net.ceedubs.ficus.Ficus._ import scala.collection.JavaConverters._ @@ -31,7 +31,8 @@ class AwsJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, override def execute: Future[BackendJobExecutionResponse] = { - val commandOverride = new ContainerOverride().withName("simple-app").withCommand(jobDescriptor.call.instantiateCommandLine(Map.empty, OnlyPureFunctions, identity).get) + val instantiatedCommand = Command.instantiate(jobDescriptor, OnlyPureFunctions).get + val commandOverride = new ContainerOverride().withName("simple-app").withCommand(instantiatedCommand) val runRequest: RunTaskRequest = new RunTaskRequest() .withCluster(clusterName) diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala index f353931d5..4c83e9dd3 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala @@ -17,7 +17,7 @@ case class HtCondorBackendFactory(name: String, configurationDescriptor: Backend extends BackendLifecycleActorFactory with StrictLogging { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Seq[Call], + calls: Set[Call], serviceRegistryActor: ActorRef): Option[Props] = { Option(HtCondorInitializationActor.props(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor)) } diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala index b0fdc75b2..c661acf19 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala @@ -19,14 +19,14 @@ object HtCondorInitializationActor { ContinueOnReturnCodeKey, CpuKey, MemoryKey, DiskKey) def props(workflowDescriptor: BackendWorkflowDescriptor, - calls: Seq[Call], + calls: Set[Call], configurationDescriptor: BackendConfigurationDescriptor, serviceRegistryActor: ActorRef): Props = Props(new HtCondorInitializationActor(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor)) } class HtCondorInitializationActor(override val workflowDescriptor: BackendWorkflowDescriptor, - override val calls: Seq[Call], + override val calls: Set[Call], override val configurationDescriptor: BackendConfigurationDescriptor, override val serviceRegistryActor: ActorRef) extends BackendWorkflowInitializationActor { diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala index 72794ba11..291165d03 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala @@ -10,17 +10,18 @@ import cromwell.backend.impl.htcondor.caching.CacheActor._ import cromwell.backend.impl.htcondor.caching.localization.CachedResultLocalization import cromwell.backend.io.JobPaths import cromwell.backend.sfs.{SharedFileSystem, SharedFileSystemExpressionFunctions} +import cromwell.backend.wdl.Command +import cromwell.core.path.JavaWriterImplicits._ import cromwell.core.path.{DefaultPathBuilder, PathBuilder} import cromwell.services.keyvalue.KeyValueServiceActor._ import cromwell.services.metadata.CallMetadataKeys import org.apache.commons.codec.digest.DigestUtils -import wdl4s._ +import wdl4s.EvaluatedTaskInputs import wdl4s.parser.MemoryUnit import wdl4s.types.{WdlArrayType, WdlFileType} import wdl4s.util.TryUtil import wdl4s.values.WdlArray -import cromwell.core.path.JavaWriterImplicits._ import scala.concurrent.{Future, Promise} import scala.sys.process.ProcessLogger import scala.util.{Failure, Success, Try} @@ -75,7 +76,7 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor private val call = jobDescriptor.key.call private val callEngineFunction = SharedFileSystemExpressionFunctions(jobPaths, pathBuilders) - private val lookup = jobDescriptor.inputs.apply _ + private val lookup = jobDescriptor.fullyQualifiedInputs.apply _ private val runtimeAttributes = { val evaluateAttrs = call.task.runtimeAttributes.attrs mapValues (_.evaluate(lookup, callEngineFunction)) @@ -253,7 +254,7 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor } private def calculateHash: String = { - val cmd = call.task.instantiateCommand(jobDescriptor.inputs, callEngineFunction, identity) match { + val cmd = Command.instantiate(jobDescriptor, callEngineFunction) match { case Success(command) => command case Failure(ex) => val errMsg = s"$tag Cannot instantiate job command for caching purposes due to ${ex.getMessage}." @@ -277,7 +278,7 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor executionDir.toString.toFile.createIfNotExists(asDirectory = true, createParents = true) log.debug("{} Resolving job command", tag) - val command = localizeInputs(jobPaths.callInputsRoot, runtimeAttributes.dockerImage.isDefined, jobDescriptor.inputs) flatMap { + val command = localizeInputs(jobPaths.callInputsRoot, runtimeAttributes.dockerImage.isDefined)(jobDescriptor.inputDeclarations) flatMap { localizedInputs => resolveJobCommand(localizedInputs) } @@ -307,7 +308,7 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor } } - private def resolveJobCommand(localizedInputs: CallInputs): Try[String] = { + private def resolveJobCommand(localizedInputs: EvaluatedTaskInputs): Try[String] = { val command = if (runtimeAttributes.dockerImage.isDefined) { modifyCommandForDocker(call.task.instantiateCommand(localizedInputs, callEngineFunction, identity), localizedInputs) } else { @@ -330,7 +331,7 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), metadataKeyValues) } - private def modifyCommandForDocker(jobCmd: Try[String], localizedInputs: CallInputs): Try[String] = { + private def modifyCommandForDocker(jobCmd: Try[String], localizedInputs: EvaluatedTaskInputs): Try[String] = { Try { val dockerInputDataVol = localizedInputs.values.collect { case file if file.wdlType == WdlFileType => diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala index 9e71d1a14..e564cf002 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala @@ -29,12 +29,12 @@ class HtCondorInitializationActorSpec extends TestKitSuite("HtCondorInitializati | RUNTIME |} | - |workflow hello { + |workflow wf_hello { | call hello |} """.stripMargin - private def getHtCondorBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], conf: BackendConfigurationDescriptor) = { + private def getHtCondorBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[Call], conf: BackendConfigurationDescriptor) = { system.actorOf(HtCondorInitializationActor.props(workflowDescriptor, calls, conf, emptyActor)) } diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala index eafb3a338..8109908f5 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala @@ -16,7 +16,6 @@ import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, B import cromwell.core._ import cromwell.core.path.{PathWriter, TailedWriter, UntailedWriter} import cromwell.services.keyvalue.KeyValueServiceActor.{KvGet, KvPair, KvPut} -import org.mockito.Matchers._ import org.mockito.Mockito import org.mockito.Mockito._ import org.scalatest.concurrent.PatienceConfiguration.Timeout @@ -55,7 +54,7 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc | RUNTIME |} | - |workflow hello { + |workflow wf_hello { | call hello |} """.stripMargin @@ -74,7 +73,7 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc | RUNTIME |} | - |workflow hello { + |workflow wf_hello { | call hello |} """.stripMargin @@ -93,7 +92,7 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc | RUNTIME |} | - |workflow hello { + |workflow wf_hello { | call hello |} """.stripMargin @@ -289,7 +288,7 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc """.stripMargin val jsonInputFile = createCannedFile("testFile", "some content").pathAsString val inputs = Map( - "inputFile" -> WdlFile(jsonInputFile) + "wf_hello.hello.inputFile" -> WdlFile(jsonInputFile) ) val jobDescriptor = prepareJob(helloWorldWdlWithFileInput, runtime, Option(inputs)) val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor) @@ -373,7 +372,7 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc createCannedFile(prefix = "testFile2", contents = "some other content", dir = Some(tempDir2)).pathAsString val inputs = Map( - "inputFiles" -> WdlArray(WdlArrayType(WdlFileType), Seq(WdlFile(jsonInputFile), WdlFile(jsonInputFile2))) + "wf_hello.hello.inputFiles" -> WdlArray(WdlArrayType(WdlFileType), Seq(WdlFile(jsonInputFile), WdlFile(jsonInputFile2))) ) val jobDescriptor = prepareJob(helloWorldWdlWithFileArrayInput, runtime, Option(inputs)) val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor) diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala index db95a999e..8e794634d 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala @@ -30,7 +30,7 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { | RUNTIME |} | - |workflow hello { + |workflow wf_hello { | call hello |} """.stripMargin @@ -262,12 +262,11 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { val workflowDescriptor = buildWorkflowDescriptor(wdlSource, runtime = runtimeAttributes) def createLookup(call: Call): ScopedLookupFunction = { - val declarations = workflowDescriptor.workflowNamespace.workflow.declarations ++ call.task.declarations val knownInputs = workflowDescriptor.inputs - WdlExpression.standardLookupFunction(knownInputs, declarations, NoFunctions) + call.lookupFunction(knownInputs, NoFunctions) } - workflowDescriptor.workflowNamespace.workflow.calls map { + workflowDescriptor.workflowNamespace.workflow.calls.toSeq map { call => val ra = call.task.runtimeAttributes.attrs mapValues { _.evaluate(createLookup(call), NoFunctions) } TryUtil.sequenceMap(ra, "Runtime attributes evaluation").get diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala index fa675e758..8100799fd 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala @@ -36,7 +36,7 @@ class MongoCacheActorSpec extends TestKit(ActorSystem("MongoCacheProviderActorSp val runtimeConfig = HtCondorRuntimeAttributes(ContinueOnReturnCodeSet(Set(0)), Some("tool-name"), Some("/workingDir"), Some("/outputDir"), true, 1, memorySize, diskSize) val jobHash = "88dde49db10f1551299fb9937f313c10" val taskStatus = "done" - val succeededResponseMock = SucceededResponse(BackendJobDescriptorKey(Call(None, "TestJob", null, null, null, None), None, 0), None, Map("test" -> JobOutput(WdlString("Test"))), None, Seq.empty) + val succeededResponseMock = SucceededResponse(BackendJobDescriptorKey(Call(Option("taskName"), null, null, null), None, 0), None, Map("test" -> JobOutput(WdlString("Test"))), None, Seq.empty) val serSucceededRespMock = KryoSerializedObject(serialize(succeededResponseMock)) val cachedExecutionResult = MongoCachedExecutionResult(jobHash, serSucceededRespMock) val cachedExecutionResultDbObject = JSON.parse(cachedExecutionResult.toJson.toString).asInstanceOf[DBObject] diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala index e0e93e839..3888fc972 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala @@ -12,13 +12,13 @@ import com.google.api.client.googleapis.json.GoogleJsonResponseException import com.google.cloud.storage.contrib.nio.CloudStoragePath import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse} import cromwell.backend.BackendLifecycleActor.AbortJobCommand -import cromwell.backend._ import cromwell.backend.async.AsyncBackendJobExecutionActor.{ExecutionMode, JobId} import cromwell.backend.async.{AbortedExecutionHandle, AsyncBackendJobExecutionActor, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle, NonRetryableExecution, SuccessfulExecutionHandle} import cromwell.backend.impl.jes.JesJobExecutionActor.JesOperationIdKey import cromwell.backend.impl.jes.RunStatus.TerminalRunStatus import cromwell.backend.impl.jes.io._ import cromwell.backend.impl.jes.statuspolling.JesPollingActorClient +import cromwell.backend.wdl.OutputEvaluator import cromwell.backend.{BackendJobDescriptor, BackendWorkflowDescriptor, PreemptedException} import cromwell.core.Dispatcher.BackendDispatcher import cromwell.core._ @@ -27,10 +27,7 @@ import cromwell.core.path.proxy.PathProxy import cromwell.core.retry.{Retry, SimpleExponentialBackoff} import cromwell.services.keyvalue.KeyValueServiceActor._ import cromwell.services.metadata._ -import wdl4s.AstTools._ -import wdl4s.WdlExpression.ScopedLookupFunction import wdl4s._ -import wdl4s.command.ParameterCommandPart import wdl4s.expression.NoFunctions import wdl4s.values._ @@ -145,11 +142,6 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes private[jes] lazy val callEngineFunctions = new JesExpressionFunctions(List(jesCallPaths.gcsPathBuilder), callContext) - private val lookup: ScopedLookupFunction = { - val declarations = workflowDescriptor.workflowNamespace.workflow.declarations ++ call.task.declarations - WdlExpression.standardLookupFunction(jobDescriptor.inputs, declarations, callEngineFunctions) - } - /** * Takes two arrays of remote and local WDL File paths and generates the necessary JesInputs. */ @@ -180,31 +172,13 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes } private[jes] def generateJesInputs(jobDescriptor: BackendJobDescriptor): Iterable[JesInput] = { - /** - * Commands in WDL tasks can also generate input files. For example: ./my_exec --file=${write_lines(arr)} - * - * write_lines(arr) would produce a string-ified version of the array stored as a GCS path. The next block of code - * will go through each ${...} expression within the task's command section and find all write_*() ASTs and - * evaluate them so the files are written to GCS and the they can be included as inputs to Google's Pipeline object - */ - val commandExpressions = jobDescriptor.key.scope.task.commandTemplate.collect({ - case x: ParameterCommandPart => x.expression - }) - - val writeFunctionAsts = commandExpressions.map(_.ast).flatMap(x => AstTools.findAsts(x, "FunctionCall")).collect({ - case y if y.getAttribute("name").sourceString.startsWith("write_") => y - }) - - val evaluatedExpressionMap = writeFunctionAsts map { ast => - val expression = WdlExpression(ast) - val value = expression.evaluate(lookup, callEngineFunctions) - expression.toWdlString.md5SumShort -> value - } toMap - - val writeFunctionFiles = evaluatedExpressionMap collect { case (k, v: Success[_]) => k -> v.get } collect { case (k, v: WdlFile) => k -> Seq(v)} + + val writeFunctionFiles = call.task.evaluateFilesFromCommand(jobDescriptor.fullyQualifiedInputs, callEngineFunctions) map { + case (expression, file) => expression.toWdlString.md5SumShort -> Seq(file) + } /** Collect all WdlFiles from inputs to the call */ - val callInputFiles: Map[FullyQualifiedName, Seq[WdlFile]] = jobDescriptor.inputs mapValues { _.collectAsSeq { case w: WdlFile => w } } + val callInputFiles: Map[FullyQualifiedName, Seq[WdlFile]] = jobDescriptor.fullyQualifiedInputs mapValues { _.collectAsSeq { case w: WdlFile => w } } (callInputFiles ++ writeFunctionFiles) flatMap { case (name, files) => jesInputsFromWdlFiles(name, files, files.map(relativeLocalizationPath), jobDescriptor) @@ -239,14 +213,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes } private[jes] def generateJesOutputs(jobDescriptor: BackendJobDescriptor): Seq[JesFileOutput] = { - val wdlFileOutputs = jobDescriptor.key.scope.task.outputs flatMap { taskOutput => - taskOutput.requiredExpression.evaluateFiles(lookup, NoFunctions, taskOutput.wdlType) match { - case Success(wdlFiles) => wdlFiles map relativeLocalizationPath - case Failure(ex) => - jobLogger.warn(s"Could not evaluate $taskOutput: ${ex.getMessage}", ex) - Seq.empty[WdlFile] - } - } + val wdlFileOutputs = call.task.findOutputFiles(jobDescriptor.fullyQualifiedInputs, NoFunctions) map relativeLocalizationPath // Create the mappings. GLOB mappings require special treatment (i.e. stick everything matching the glob in a folder) wdlFileOutputs.distinct map { wdlFile => @@ -260,8 +227,8 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes } private def instantiateCommand: Try[String] = { - val backendInputs = jobDescriptor.inputs mapValues gcsPathToLocal - jobDescriptor.call.instantiateCommandLine(backendInputs, callEngineFunctions, gcsPathToLocal) + val backendInputs = jobDescriptor.inputDeclarations mapValues gcsPathToLocal + jobDescriptor.call.task.instantiateCommand(backendInputs, callEngineFunctions, valueMapper = gcsPathToLocal) } private def uploadCommandScript(command: String, withMonitoring: Boolean): Future[Unit] = { @@ -452,9 +419,6 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), metadataKeyValues) } - /** - * Attempts to find the JES file output corresponding to the WdlValue - */ private[jes] def wdlValueToGcsPath(jesOutputs: Seq[JesFileOutput])(value: WdlValue): WdlValue = { def toGcsPath(wdlFile: WdlFile) = jesOutputs collectFirst { case o if o.name == makeSafeJesReferenceName(wdlFile.valueString) => WdlFile(o.gcs) @@ -559,7 +523,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes FailedNonRetryableExecutionHandle(new RuntimeException( s"execution failed: could not parse return code as integer: ${returnCodeContents.get}")).future case _: RunStatus.Success if !continueOnReturnCode.continueFor(returnCode.get) => - val badReturnCodeMessage = s"Call ${jobDescriptor.key}: return code was ${returnCode.getOrElse("(none)")}" + val badReturnCodeMessage = s"Call ${jobDescriptor.key.tag}: return code was ${returnCode.getOrElse("(none)")}" FailedNonRetryableExecutionHandle(new RuntimeException(badReturnCodeMessage), returnCode.toOption).future case success: RunStatus.Success => handleSuccess(postProcess, returnCode.get, jesCallPaths.detritusPaths, handle, success.eventList).future diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala index 6d192f0a4..8e9a261a4 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala @@ -21,7 +21,7 @@ case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor val jesConfiguration = new JesConfiguration(configurationDescriptor) override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Seq[Call], + calls: Set[Call], serviceRegistryActor: ActorRef): Option[Props] = { Option(JesInitializationActor.props(workflowDescriptor, calls, jesConfiguration, serviceRegistryActor).withDispatcher(BackendDispatcher)) } @@ -46,7 +46,7 @@ case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor } override def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Seq[Call], + calls: Set[Call], executionStore: ExecutionStore, outputStore: OutputStore, initializationData: Option[BackendInitializationData]) = { diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala index e6c3e2d7f..195e34e8b 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala @@ -16,14 +16,14 @@ import scala.concurrent.Future import scala.language.postfixOps object JesFinalizationActor { - def props(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], jesConfiguration: JesConfiguration, + def props(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[Call], jesConfiguration: JesConfiguration, executionStore: ExecutionStore, outputStore: OutputStore, initializationData: Option[JesBackendInitializationData]) = { Props(new JesFinalizationActor(workflowDescriptor, calls, jesConfiguration, executionStore, outputStore, initializationData)) } } class JesFinalizationActor (override val workflowDescriptor: BackendWorkflowDescriptor, - override val calls: Seq[Call], + override val calls: Set[Call], jesConfiguration: JesConfiguration, executionStore: ExecutionStore, outputStore: OutputStore, initializationData: Option[JesBackendInitializationData]) extends BackendWorkflowFinalizationActor { diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala index 58eacb45b..607c8c8c4 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala @@ -28,14 +28,14 @@ object JesInitializationActor { JesRuntimeAttributes.PreemptibleKey, JesRuntimeAttributes.BootDiskSizeKey, JesRuntimeAttributes.DisksKey) def props(workflowDescriptor: BackendWorkflowDescriptor, - calls: Seq[Call], + calls: Set[Call], jesConfiguration: JesConfiguration, serviceRegistryActor: ActorRef): Props = Props(new JesInitializationActor(workflowDescriptor, calls, jesConfiguration, serviceRegistryActor: ActorRef)) } class JesInitializationActor(override val workflowDescriptor: BackendWorkflowDescriptor, - override val calls: Seq[Call], + override val calls: Set[Call], private[jes] val jesConfiguration: JesConfiguration, override val serviceRegistryActor: ActorRef) extends BackendWorkflowInitializationActor { diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala index 6072e1d83..6da2c7fa3 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala @@ -13,7 +13,8 @@ import cromwell.backend.async.{AbortedExecutionHandle, ExecutionHandle, FailedNo import cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.JesPendingExecutionHandle import cromwell.backend.impl.jes.RunStatus.Failed import cromwell.backend.impl.jes.io.{DiskType, JesWorkingDisk} -import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobDescriptorKey, BackendWorkflowDescriptor, PreemptedException, RuntimeAttributeDefinition} +import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll +import cromwell.backend._ import cromwell.core.logging.LoggerWrapper import cromwell.core.{WorkflowId, WorkflowOptions, _} import cromwell.filesystems.gcs.GcsPathBuilderFactory @@ -26,7 +27,7 @@ import org.specs2.mock.Mockito import spray.json.{JsObject, JsValue} import wdl4s.types.{WdlArrayType, WdlFileType, WdlMapType, WdlStringType} import wdl4s.values.{WdlArray, WdlFile, WdlMap, WdlString, WdlValue} -import wdl4s.{Call, LocallyQualifiedName, NamespaceWithWorkflow} +import wdl4s.{Call, LocallyQualifiedName, WdlNamespaceWithWorkflow} import scala.concurrent.duration._ import scala.concurrent.{Await, ExecutionContext, Future, Promise} @@ -34,7 +35,7 @@ import scala.util.{Success, Try} import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackendJobExecutionActorSpec") - with FlatSpecLike with Matchers with ImplicitSender with Mockito { + with FlatSpecLike with Matchers with ImplicitSender with Mockito with BackendSpec { val mockPathBuilder = GcsPathBuilderFactory(NoAuthMode).withOptions(mock[WorkflowOptions]) @@ -58,7 +59,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend | } |} | - |workflow sup { + |workflow wf_sup { | call sup |} """.stripMargin @@ -110,7 +111,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend private def buildPreemptibleJobDescriptor(attempt: Int, preemptible: Int): BackendJobDescriptor = { val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - NamespaceWithWorkflow.load(YoSup.replace("[PREEMPTIBLE]", s"preemptible: $preemptible")), + WdlNamespaceWithWorkflow.load(YoSup.replace("[PREEMPTIBLE]", s"preemptible: $preemptible")), Inputs, NoOptions ) @@ -118,7 +119,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val job = workflowDescriptor.workflowNamespace.workflow.calls.head val key = BackendJobDescriptorKey(job, None, attempt) val runtimeAttributes = makeRuntimeAttributes(job) - BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Inputs) + BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(Inputs)) } private def executionActor(jobDescriptor: BackendJobDescriptor, @@ -284,7 +285,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - NamespaceWithWorkflow.load(YoSup.replace("[PREEMPTIBLE]", "")), + WdlNamespaceWithWorkflow.load(YoSup.replace("[PREEMPTIBLE]", "")), inputs, NoOptions ) @@ -292,13 +293,13 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val call = workflowDescriptor.workflowNamespace.workflow.calls.head val key = BackendJobDescriptorKey(call, None, 1) val runtimeAttributes = makeRuntimeAttributes(call) - val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, inputs) + val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs)) val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration)) val testActorRef = TestActorRef[TestableJesJobExecutionActor]( props, s"TestableJesJobExecutionActor-${jobDescriptor.workflowDescriptor.id}") - val mappedInputs = jobDescriptor.inputs mapValues testActorRef.underlyingActor.gcsPathToLocal + val mappedInputs = jobDescriptor.fullyQualifiedInputs mapValues testActorRef.underlyingActor.gcsPathToLocal mappedInputs(stringKey) match { case WdlString(v) => assert(v.equalsIgnoreCase(stringVal.value)) @@ -338,7 +339,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - NamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource), + WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource), inputs, NoOptions ) @@ -346,7 +347,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val job = workflowDescriptor.workflowNamespace.workflow.calls.head val runtimeAttributes = makeRuntimeAttributes(job) val key = BackendJobDescriptorKey(job, None, 1) - val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, inputs) + val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs)) val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration)) val testActorRef = TestActorRef[TestableJesJobExecutionActor]( @@ -377,7 +378,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend TestActorRef[TestableJesJobExecutionActor] = { val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - NamespaceWithWorkflow.load(sampleWdl.asWorkflowSources(DockerAndDiskRuntime).wdlSource), + WdlNamespaceWithWorkflow.load(sampleWdl.asWorkflowSources(DockerAndDiskRuntime).wdlSource), inputs, NoOptions ) @@ -385,7 +386,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val call = workflowDescriptor.workflowNamespace.workflow.findCallByName(callName).get val key = BackendJobDescriptorKey(call, None, 1) val runtimeAttributes = makeRuntimeAttributes(call) - val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, inputs) + val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs)) val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration, functions)) TestActorRef[TestableJesJobExecutionActor](props, s"TestableJesJobExecutionActor-${jobDescriptor.workflowDescriptor.id}") @@ -438,7 +439,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - NamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource), + WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource), inputs, NoOptions ) @@ -446,7 +447,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val job = workflowDescriptor.workflowNamespace.workflow.calls.head val runtimeAttributes = makeRuntimeAttributes(job) val key = BackendJobDescriptorKey(job, None, 1) - val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, inputs) + val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs)) val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration)) val testActorRef = TestActorRef[TestableJesJobExecutionActor]( @@ -466,7 +467,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - NamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource), + WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource), inputs, NoOptions ) @@ -474,7 +475,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val job = workflowDescriptor.workflowNamespace.workflow.calls.head val runtimeAttributes = makeRuntimeAttributes(job) val key = BackendJobDescriptorKey(job, None, 1) - val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, inputs) + val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs)) val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration)) val testActorRef = TestActorRef[TestableJesJobExecutionActor]( @@ -510,7 +511,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - NamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource), + WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource), Map.empty, NoOptions ) @@ -538,7 +539,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend it should "create a JesFileInput for the monitoring script, when specified" in { val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - NamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource), + WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource), Map.empty, WorkflowOptions.fromJsonString("""{"monitoring_script": "gs://path/to/script"}""").get ) @@ -559,7 +560,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend it should "not create a JesFileInput for the monitoring script, when not specified" in { val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - NamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource), + WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource), Map.empty, NoOptions ) @@ -579,7 +580,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend it should "return JES log paths for non-scattered call" in { val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId(UUID.fromString("e6236763-c518-41d0-9688-432549a8bf7c")), - NamespaceWithWorkflow.load( + WdlNamespaceWithWorkflow.load( SampleWdl.HelloWorld.asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).wdlSource), Map.empty, WorkflowOptions.fromJsonString(""" {"jes_gcs_root": "gs://path/to/gcs_root"} """).get @@ -596,22 +597,21 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val jesBackend = testActorRef.underlyingActor - // TODO: NioGcsPath.equals not implemented, so use toString instead jesBackend.jesCallPaths.stdoutPath should be(a[CloudStoragePath]) jesBackend.jesCallPaths.stdoutPath.toUri.toString shouldBe - "gs://path/to/gcs_root/hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello-stdout.log" + "gs://path/to/gcs_root/wf_hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello-stdout.log" jesBackend.jesCallPaths.stderrPath should be(a[CloudStoragePath]) jesBackend.jesCallPaths.stderrPath.toUri.toString shouldBe - "gs://path/to/gcs_root/hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello-stderr.log" + "gs://path/to/gcs_root/wf_hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello-stderr.log" jesBackend.jesCallPaths.jesLogPath should be(a[CloudStoragePath]) jesBackend.jesCallPaths.jesLogPath.toUri.toString shouldBe - "gs://path/to/gcs_root/hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello.log" + "gs://path/to/gcs_root/wf_hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello.log" } it should "return JES log paths for scattered call" in { val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId(UUID.fromString("e6236763-c518-41d0-9688-432549a8bf7d")), - NamespaceWithWorkflow.load( + WdlNamespaceWithWorkflow.load( new SampleWdl.ScatterWdl().asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).wdlSource), Map.empty, WorkflowOptions.fromJsonString(""" {"jes_gcs_root": "gs://path/to/gcs_root"} """).get diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala index c0c571526..60b3f2e5e 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala @@ -33,13 +33,13 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with val callPaths = JesCallPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration) callPaths.returnCodePath.toUri.toString should - be(s"gs://my-cromwell-workflows-bucket/hello/${workflowDescriptor.id}/call-hello/hello-rc.txt") + be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello-rc.txt") callPaths.stdoutPath.toUri.toString should - be(s"gs://my-cromwell-workflows-bucket/hello/${workflowDescriptor.id}/call-hello/hello-stdout.log") + be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello-stdout.log") callPaths.stderrPath.toUri.toString should - be(s"gs://my-cromwell-workflows-bucket/hello/${workflowDescriptor.id}/call-hello/hello-stderr.log") + be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello-stderr.log") callPaths.jesLogPath.toUri.toString should - be(s"gs://my-cromwell-workflows-bucket/hello/${workflowDescriptor.id}/call-hello/hello.log") + be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello.log") } it should "map the correct call context" in { @@ -49,7 +49,7 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with val callPaths = JesCallPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration) callPaths.callContext.root.toUri.toString should - be(s"gs://my-cromwell-workflows-bucket/hello/${workflowDescriptor.id}/call-hello") + be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello") callPaths.callContext.stdout should be("hello-stdout.log") callPaths.callContext.stderr should be("hello-stderr.log") } diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala index 20fe205d1..9b83c3d4a 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala @@ -39,7 +39,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe | RUNTIME |} | - |workflow hello { + |workflow wf_hello { | call hello |} """.stripMargin @@ -137,7 +137,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe val refreshTokenConfig = ConfigFactory.parseString(refreshTokenConfigTemplate) - private def getJesBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], conf: BackendConfigurationDescriptor) = { + private def getJesBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[Call], conf: BackendConfigurationDescriptor) = { system.actorOf(JesInitializationActor.props(workflowDescriptor, calls, new JesConfiguration(conf), emptyActor)) } diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala index 3ad37ee6d..b2fe12336 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala @@ -19,8 +19,8 @@ class JesWorkflowPathsSpec extends TestKitSuite with FlatSpecLike with Matchers val workflowPaths = JesWorkflowPaths(workflowDescriptor, jesConfiguration)(system) workflowPaths.rootPath.toUri.toString should be("gs://my-cromwell-workflows-bucket/") workflowPaths.workflowRootPath.toUri.toString should - be(s"gs://my-cromwell-workflows-bucket/hello/${workflowDescriptor.id}/") + be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/") workflowPaths.gcsAuthFilePath.toUri.toString should - be(s"gs://my-cromwell-workflows-bucket/hello/${workflowDescriptor.id}/${workflowDescriptor.id}_auth.json") + be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/${workflowDescriptor.id}_auth.json") } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala index a92c66ffc..a47209c20 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala @@ -48,10 +48,11 @@ sealed trait ConfigAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecut * @param taskName The name of the task to retrieve from the precomputed wdl namespace. * @param inputs The customized inputs to this task. */ - def writeTaskScript(script: File, taskName: String, inputs: CallInputs): Unit = { + def writeTaskScript(script: File, taskName: String, inputs: WorkflowCoercedInputs): Unit = { val task = configInitializationData.wdlNamespace.findTask(taskName). getOrElse(throw new RuntimeException(s"Unable to find task $taskName")) - val command = task.instantiateCommand(inputs, NoFunctions).get + val inputsWithFqns = inputs map { case (k, v) => s"$taskName.$k" -> v } + val command = task.instantiateCommand(task.inputsFromMap(inputsWithFqns), NoFunctions).get jobLogger.info(s"executing: $command") val scriptBody = s""" @@ -68,7 +69,7 @@ $command * The inputs that are not specified by the config, that will be passed into a command for both submit and * submit-docker. */ - private lazy val standardInputs: CallInputs = { + private lazy val standardInputs: WorkflowCoercedInputs = { Map( JobNameInput -> WdlString(jobName), CwdInput -> WdlString(jobPaths.callRoot.toString), @@ -81,7 +82,7 @@ $command /** * Extra arguments if this is a submit-docker command, or Map.empty. */ - private lazy val dockerInputs: CallInputs = { + private lazy val dockerInputs: WorkflowCoercedInputs = { if (isDockerRun) { Map( DockerCwdInput -> WdlString(jobPaths.callDockerRoot.toString) @@ -95,7 +96,7 @@ $command * The arguments generated from the backend config's list of attributes. These will include things like CPU, memory, * and other custom arguments like "backend_queue_name", "backend_billing_project", etc. */ - private lazy val runtimeAttributeInputs: CallInputs = { + private lazy val runtimeAttributeInputs: WorkflowCoercedInputs = { val declarationValidations = configInitializationData.declarationValidations val inputOptions = declarationValidations map { declarationValidation => declarationValidation.extractWdlValueOption(validatedRuntimeAttributes) map { wdlValue => diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala index ec3aecbf7..11b69147e 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala @@ -24,7 +24,7 @@ object DeclarationValidation { * @return The DeclarationValidation object for the declaration. */ def fromDeclaration(declaration: Declaration): DeclarationValidation = { - declaration.name match { + declaration.unqualifiedName match { // Docker and CPU are special keys understood by cromwell. case DockerValidation.key => new DeclarationValidation(declaration, DockerValidation.instance) case CpuValidation.key => new DeclarationValidation(declaration, CpuValidation.default) @@ -34,11 +34,11 @@ object DeclarationValidation { // All other declarations must be a Boolean, Float, Integer, or String. case _ => val validator: PrimitiveRuntimeAttributesValidation[_] = declaration.wdlType match { - case WdlBooleanType => new BooleanRuntimeAttributesValidation(declaration.name) - case WdlFloatType => new FloatRuntimeAttributesValidation(declaration.name) - case WdlIntegerType => new IntRuntimeAttributesValidation(declaration.name) - case WdlStringType => new StringRuntimeAttributesValidation(declaration.name) - case other => throw new RuntimeException(s"Unsupported config runtime attribute $other ${declaration.name}") + case WdlBooleanType => new BooleanRuntimeAttributesValidation(declaration.unqualifiedName) + case WdlFloatType => new FloatRuntimeAttributesValidation(declaration.unqualifiedName) + case WdlIntegerType => new IntRuntimeAttributesValidation(declaration.unqualifiedName) + case WdlStringType => new StringRuntimeAttributesValidation(declaration.unqualifiedName) + case other => throw new RuntimeException(s"Unsupported config runtime attribute $other ${declaration.unqualifiedName}") } new DeclarationValidation(declaration, validator) } @@ -52,7 +52,7 @@ object DeclarationValidation { * @param instanceValidation A basic instance validation for the declaration. */ class DeclarationValidation(declaration: Declaration, instanceValidation: RuntimeAttributesValidation[_]) { - val key = declaration.name + val key = declaration.unqualifiedName /** * Creates a validation, by adding on defaults if they're specified in the declaration, and then making the @@ -151,7 +151,7 @@ class MemoryDeclarationValidation(declaration: Declaration) } private lazy val declarationMemoryUnit: MemoryUnit = { - val suffix = memoryUnitSuffix(declaration.name) + val suffix = memoryUnitSuffix(declaration.unqualifiedName) val memoryUnitOption = MemoryUnit.values.find(_.suffixes.map(_.toLowerCase).contains(suffix.toLowerCase)) memoryUnitOption match { case Some(memoryUnit) => memoryUnit diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala index d67c8846c..b514ad8cc 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala @@ -9,7 +9,7 @@ import cromwell.backend.io.JobPaths import cromwell.core._ import cromwell.core.path.PathFactory import cromwell.util.TryUtil -import wdl4s.CallInputs +import wdl4s.EvaluatedTaskInputs import wdl4s.types.{WdlArrayType, WdlMapType} import wdl4s.values._ @@ -50,8 +50,6 @@ object SharedFileSystem { private def localizePathViaHardLink(originalPath: File, executionPath: File): Try[Unit] = { executionPath.parent.createDirectories() - // -Ywarn-value-discard - // Try(executionPath.linkTo(originalPath, symbolic = false)) map { _ => executionPath } Try { executionPath.linkTo(originalPath, symbolic = false) } void } @@ -144,11 +142,8 @@ trait SharedFileSystem extends PathFactory { /** * Return a possibly altered copy of inputs reflecting any localization of input file paths that might have * been performed for this `Backend` implementation. - * NOTE: This ends up being a backdoor implementation of Backend.adjustInputPaths as both LocalBackend and SgeBackend - * end up with this implementation and thus use it to satisfy their contract with Backend. - * This is yuck-tastic and I consider this a FIXME, but not for this refactor */ - def localizeInputs(inputsRoot: Path, docker: Boolean, inputs: CallInputs): Try[CallInputs] = { + def localizeInputs(inputsRoot: Path, docker: Boolean)(inputs: EvaluatedTaskInputs): Try[EvaluatedTaskInputs] = { val strategies = if (docker) DockerLocalizers else Localizers // Use URI to identify protocol scheme and strip it out @@ -181,7 +176,7 @@ trait SharedFileSystem extends PathFactory { // Optional function to adjust the path to "docker path" if the call runs in docker val localizeFunction = localizeWdlValue(toCallPath, strategies.toStream) _ val localizedValues = inputs.toSeq map { - case (name, value) => localizeFunction(value) map { name -> _ } + case (declaration, value) => localizeFunction(value) map { declaration -> _ } } TryUtil.sequence(localizedValues, "Failures during localization").map(_.toMap) recover { diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala index 98f4dbc11..5b8fc0a8c 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala @@ -12,7 +12,8 @@ import cromwell.backend.async.{AbortedExecutionHandle, AsyncBackendJobExecutionA import cromwell.backend.io.WorkflowPathsBackendInitializationData import cromwell.backend.sfs.SharedFileSystem._ import cromwell.backend.validation._ -import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, OutputEvaluator} +import cromwell.backend.wdl.{OutputEvaluator, Command} +import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor} import cromwell.core.JobOutputs import cromwell.core.logging.JobLogging import cromwell.core.path.DefaultPathBuilder @@ -168,12 +169,18 @@ trait SharedFileSystemAsyncJobExecutionActor } def instantiatedScript: String = { - val pathTransformFunction: WdlValue => WdlValue = toUnixPath(isDockerRun) - val tryCommand = sharedFileSystem.localizeInputs(jobPaths.callInputsRoot, - isDockerRun, jobDescriptor.inputs) flatMap { localizedInputs => - call.task.instantiateCommand(localizedInputs, callEngineFunction, pathTransformFunction) + val pathTransformFunction = toUnixPath(isDockerRun) _ + val localizer = sharedFileSystem.localizeInputs(jobPaths.callInputsRoot, isDockerRun) _ + + Command.instantiate( + jobDescriptor, + callEngineFunction, + localizer, + pathTransformFunction + ) match { + case Success(command) => command + case Failure(ex) => throw new RuntimeException("Failed to instantiate command line", ex) } - tryCommand.get } override def executeOrRecover(mode: ExecutionMode)(implicit ec: ExecutionContext) = { @@ -321,7 +328,7 @@ mv $rcTmpPath $rcPath def processReturnCode()(implicit ec: ExecutionContext): Future[ExecutionHandle] = { val returnCodeTry = Try(File(jobPaths.returnCode).contentAsString.stripLineEnd.toInt) - lazy val badReturnCodeMessage = s"Call ${jobDescriptor.key}: return code was ${returnCodeTry.getOrElse("(none)")}" + lazy val badReturnCodeMessage = s"Call ${jobDescriptor.key.tag}: return code was ${returnCodeTry.getOrElse("(none)")}" lazy val badReturnCodeResponse = Future.successful( FailedNonRetryableExecutionHandle(new Exception(badReturnCodeMessage), returnCodeTry.toOption)) diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala index 65152eb83..ad03963b6 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala @@ -63,7 +63,7 @@ trait SharedFileSystemBackendLifecycleActorFactory extends BackendLifecycleActor */ def asyncJobExecutionActorClass: Class[_ <: SharedFileSystemAsyncJobExecutionActor] - override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], + override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[Call], serviceRegistryActor: ActorRef) = { val params = SharedFileSystemInitializationActorParams(serviceRegistryActor, workflowDescriptor, configurationDescriptor, calls, pathBuilderFactories) diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala index 03bb1024d..b73797d88 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala @@ -19,7 +19,7 @@ case class SharedFileSystemInitializationActorParams serviceRegistryActor: ActorRef, workflowDescriptor: BackendWorkflowDescriptor, configurationDescriptor: BackendConfigurationDescriptor, - calls: Seq[Call], + calls: Set[Call], pathBuilderFactories: List[PathBuilderFactory] ) @@ -39,7 +39,7 @@ class SharedFileSystemInitializationActor(params: SharedFileSystemInitialization override lazy val workflowDescriptor: BackendWorkflowDescriptor = params.workflowDescriptor override lazy val configurationDescriptor: BackendConfigurationDescriptor = params.configurationDescriptor - override lazy val calls: Seq[Call] = params.calls + override lazy val calls: Set[Call] = params.calls override lazy val serviceRegistryActor: ActorRef = params.serviceRegistryActor def runtimeAttributesBuilder: SharedFileSystemValidatedRuntimeAttributesBuilder = diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala index 1dc2811fa..b39e3609c 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala @@ -30,12 +30,12 @@ class SharedFileSystemInitializationActorSpec extends TestKitSuite("SharedFileSy | RUNTIME |} | - |workflow hello { + |workflow wf_hello { | call hello |} """.stripMargin - private def getActorRef(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], + private def getActorRef(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[Call], conf: BackendConfigurationDescriptor) = { val params = SharedFileSystemInitializationActorParams(emptyActor, workflowDescriptor, conf, calls, List.empty) val props = Props(new SharedFileSystemInitializationActor(params)) diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala index f697a7f4b..3144fef17 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala @@ -15,7 +15,6 @@ import cromwell.core.Tags._ import cromwell.core._ import cromwell.services.keyvalue.KeyValueServiceActor.{KvJobKey, KvPair, ScopedKey} import org.scalatest.concurrent.PatienceConfiguration.Timeout -import org.scalatest.mockito.MockitoSugar import org.scalatest.prop.TableDrivenPropertyChecks import org.scalatest.{FlatSpecLike, OptionValues} import wdl4s.types._ @@ -25,7 +24,7 @@ import wdl4s.values._ import scala.concurrent.duration._ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSystemJobExecutionActorSpec") - with FlatSpecLike with BackendSpec with MockitoSugar with TableDrivenPropertyChecks with OptionValues { + with FlatSpecLike with BackendSpec with TableDrivenPropertyChecks with OptionValues { behavior of "SharedFileSystemJobExecutionActor" @@ -82,8 +81,8 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst val jsonInputFile = createCannedFile("localize", "content from json inputs").pathAsString val callInputFile = createCannedFile("localize", "content from call inputs").pathAsString val inputs = Map( - "inputFileFromCallInputs" -> WdlFile(callInputFile), - "inputFileFromJson" -> WdlFile(jsonInputFile) + "wf_localize.localize.inputFileFromCallInputs" -> WdlFile(callInputFile), + "wf_localize.localize.inputFileFromJson" -> WdlFile(jsonInputFile) ) val expectedOutputs: JobOutputs = Map( @@ -106,7 +105,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst val runtime = if (docker) """runtime { docker: "ubuntu:latest" } """ else "" val workflowDescriptor = buildWorkflowDescriptor(InputFiles, inputs, runtime = runtime) val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflowDescriptor, inputs, WorkflowOptions.empty, runtimeAttributeDefinitions), conf) - val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions) + val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, inputs, WorkflowOptions.empty, runtimeAttributeDefinitions) val expectedResponse = SucceededResponse(jobDescriptor.key, Some(0), expectedOutputs, None, Seq.empty) val jobPaths = new JobPaths(workflowDescriptor, conf.backendConfig, jobDescriptor.key) @@ -224,12 +223,12 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst 0 to 2 foreach { shard => // This assumes that engine will give us the evaluated value of the scatter item at the correct index // If this is not the case, more context/logic will need to be moved to the backend so it can figure it out by itself - val symbolMaps: Map[LocallyQualifiedName, WdlInteger] = Map("intNumber" -> WdlInteger(shard)) + val symbolMaps: Map[LocallyQualifiedName, WdlInteger] = Map("scattering.intNumber" -> WdlInteger(shard)) val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, WorkflowOptions.empty)(call.task.runtimeAttributes.attrs) val jobDescriptor: BackendJobDescriptor = - BackendJobDescriptor(workflowDescriptor, BackendJobDescriptorKey(call, Option(shard), 1), runtimeAttributes, symbolMaps) + BackendJobDescriptor(workflowDescriptor, BackendJobDescriptorKey(call, Option(shard), 1), runtimeAttributes, fqnMapToDeclarationMap(symbolMaps)) val backend = createBackend(jobDescriptor, emptyBackendConfig) val response = SucceededResponse(mock[BackendJobDescriptorKey], Some(0), Map("out" -> JobOutput(WdlInteger(shard))), None, Seq.empty) @@ -240,7 +239,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst it should "post process outputs" in { val inputFile = createCannedFile("localize", "content from json inputs").pathAsString val inputs = Map { - "inputFile" -> WdlFile(inputFile) + "wf_localize.localize.inputFile" -> WdlFile(inputFile) } val workflowDescriptor = buildWorkflowDescriptor(OutputProcess, inputs) val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, inputs, WorkflowOptions.empty, runtimeAttributeDefinitions) diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala index 9b66911a6..d7c39924c 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala @@ -5,12 +5,13 @@ import java.nio.file.Files import better.files._ import com.typesafe.config.{Config, ConfigFactory} import cromwell.core.path.DefaultPathBuilder +import cromwell.backend.BackendSpec import org.scalatest.prop.TableDrivenPropertyChecks import org.scalatest.{FlatSpec, Matchers} import org.specs2.mock.Mockito import wdl4s.values.WdlFile -class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with TableDrivenPropertyChecks { +class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with TableDrivenPropertyChecks with BackendSpec { behavior of "SharedFileSystem" @@ -35,15 +36,16 @@ class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with Tabl dest.touch() } - val inputs = Map("input" -> WdlFile(orig.pathAsString)) + val inputs = fqnMapToDeclarationMap(Map("input" -> WdlFile(orig.pathAsString))) val sharedFS = new SharedFileSystem { override val pathBuilders = localPathBuilder override val sharedFileSystemConfig = config } - val result = sharedFS.localizeInputs(callDir.path, docker = docker, inputs) + val localizedinputs = Map(inputs.head._1 -> WdlFile(dest.pathAsString)) + val result = sharedFS.localizeInputs(callDir.path, docker = docker)(inputs) result.isSuccess shouldBe true - result.get should contain theSameElementsAs Map("input" -> WdlFile(dest.pathAsString)) + result.get should contain theSameElementsAs localizedinputs dest.exists shouldBe true countLinks(dest) should be(linkNb) diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala index b31cbb32d..fd6e424b0 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala @@ -9,7 +9,7 @@ import wdl4s.Call import wdl4s.expression.WdlStandardLibraryFunctions case class SparkBackendFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor, actorSystem: ActorSystem) extends BackendLifecycleActorFactory { - override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], serviceRegistryActor: ActorRef): Option[Props] = { + override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[Call], serviceRegistryActor: ActorRef): Option[Props] = { Option(SparkInitializationActor.props(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor)) } diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala index 2c4b5f94f..4e00ee954 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala @@ -18,14 +18,14 @@ object SparkInitializationActor { SparkRuntimeAttributes.NumberOfExecutorsKey, SparkRuntimeAttributes.AppMainClassKey) def props(workflowDescriptor: BackendWorkflowDescriptor, - calls: Seq[Call], + calls: Set[Call], configurationDescriptor: BackendConfigurationDescriptor, serviceRegistryActor: ActorRef): Props = Props(new SparkInitializationActor(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor)) } class SparkInitializationActor(override val workflowDescriptor: BackendWorkflowDescriptor, - override val calls: Seq[Call], + override val calls: Set[Call], override val configurationDescriptor: BackendConfigurationDescriptor, override val serviceRegistryActor: ActorRef) extends BackendWorkflowInitializationActor { diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala index c52cb1e4c..9b2656ca5 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala @@ -7,6 +7,7 @@ import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, F import cromwell.backend.impl.spark.SparkClusterProcess._ import cromwell.backend.io.JobPaths import cromwell.backend.sfs.{SharedFileSystem, SharedFileSystemExpressionFunctions} +import cromwell.backend.wdl.Command import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobExecutionActor} import cromwell.core.path.{DefaultPathBuilder, TailedWriter, UntailedWriter} import cromwell.core.path.JavaWriterImplicits._ @@ -60,7 +61,7 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, private val call = jobDescriptor.key.call private val callEngineFunction = SharedFileSystemExpressionFunctions(jobPaths, DefaultPathBuilders) - private val lookup = jobDescriptor.inputs.apply _ + private val lookup = jobDescriptor.fullyQualifiedInputs.apply _ private val executionResponse = Promise[BackendJobExecutionResponse]() @@ -155,9 +156,12 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, executionDir.toString.toFile.createIfNotExists(asDirectory = true, createParents = true) log.debug("{} Resolving job command", tag) - val command = localizeInputs(jobPaths.callInputsRoot, docker = false, jobDescriptor.inputs) flatMap { - localizedInputs => call.task.instantiateCommand(localizedInputs, callEngineFunction, identity) - } + + val command = Command.instantiate( + jobDescriptor, + callEngineFunction, + localizeInputs(jobPaths.callInputsRoot, docker = false) + ) log.debug("{} Creating bash script for executing command: {}", tag, command) // TODO: we should use shapeless Heterogeneous list here not good to have generic map diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkInitializationActorSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkInitializationActorSpec.scala index f8aba1a4b..aafd665ab 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkInitializationActorSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkInitializationActorSpec.scala @@ -27,12 +27,12 @@ class SparkInitializationActorSpec extends TestKitSuite("SparkInitializationAc | RUNTIME |} | - |workflow hello { + |workflow wf_hello { | call hello |} """.stripMargin - private def getSparkBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], conf: BackendConfigurationDescriptor) = { + private def getSparkBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[Call], conf: BackendConfigurationDescriptor) = { system.actorOf(SparkInitializationActor.props(workflowDescriptor, calls, conf, emptyActor)) } diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala index fcf21b948..02e65ab4d 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala @@ -10,9 +10,8 @@ import cromwell.backend.BackendJobExecutionActor.{FailedNonRetryableResponse, Su import cromwell.backend.impl.spark.SparkClusterProcess._ import cromwell.backend.io._ import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendSpec} -import cromwell.core.{WorkflowOptions, TestKitSuite} +import cromwell.core.{TestKitSuite, WorkflowOptions} import cromwell.core.path.{PathWriter, TailedWriter, UntailedWriter} -import org.mockito.Matchers._ import org.mockito.Mockito import org.mockito.Mockito._ import org.scalatest.concurrent.PatienceConfiguration.Timeout @@ -51,7 +50,7 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") | RUNTIME |} | - |workflow hello { + |workflow wf_hello { | call hello |} """.stripMargin @@ -69,7 +68,7 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") | RUNTIME |} | - |workflow helloClusterMode { + |workflow wf_helloClusterMode { | call helloClusterMode |} """.stripMargin diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala index 33724cb6d..8d7888adb 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala @@ -9,7 +9,7 @@ import wdl4s.WdlExpression._ import wdl4s.expression.NoFunctions import wdl4s.util.TryUtil import wdl4s.values.WdlValue -import wdl4s.{Call, WdlExpression, _} +import wdl4s.{Call, _} class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { @@ -26,7 +26,7 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { | RUNTIME |} | - |workflow hello { + |workflow wf_hello { | call hello |} """.stripMargin @@ -90,7 +90,7 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { runtime: String) = { BackendWorkflowDescriptor( WorkflowId.randomId(), - NamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime.format("appMainClass", "com.test.spark"))), + WdlNamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime.format("appMainClass", "com.test.spark"))), inputs, options ) @@ -100,9 +100,8 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { val workflowDescriptor = buildWorkflowDescriptor(wdlSource, runtime = runtimeAttributes) def createLookup(call: Call): ScopedLookupFunction = { - val declarations = workflowDescriptor.workflowNamespace.workflow.declarations ++ call.task.declarations val knownInputs = workflowDescriptor.inputs - WdlExpression.standardLookupFunction(knownInputs, declarations, NoFunctions) + call.lookupFunction(knownInputs, NoFunctions) } workflowDescriptor.workflowNamespace.workflow.calls map { From 86215ce6bfdbba5aba2af84771d81065f31f7bc1 Mon Sep 17 00:00:00 2001 From: Jeff Gentry Date: Mon, 31 Oct 2016 11:15:14 -0400 Subject: [PATCH 047/375] update to latest akka (#1636) --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 0d737e1d4..016b70483 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -12,7 +12,7 @@ object Dependencies { - http://doc.akka.io/docs/akka/2.4/scala/http/common/json-support.html#akka-http-spray-json */ lazy val sprayJsonV = "1.3.2" - lazy val akkaV = "2.4.9" + lazy val akkaV = "2.4.12" lazy val slickV = "3.1.1" lazy val googleClientApiV = "1.22.0" lazy val googleGenomicsServicesApiV = "1.20.0" From 17b185d3fe1a3a6cf525f0a50cc6e2f45a83ec4c Mon Sep 17 00:00:00 2001 From: "Francisco M. Casares" Date: Wed, 2 Nov 2016 09:08:11 -0700 Subject: [PATCH 048/375] [HtCondor] Add support for native specifications in HtCondor backend. (#1639) * Added support for native specifications in HtCondor backend. * Changes after PR review. * Added missing info in README. --- README.md | 13 +++++++ .../impl/htcondor/HtCondorJobExecutionActor.scala | 2 +- .../impl/htcondor/HtCondorRuntimeAttributes.scala | 27 +++++++++++--- .../backend/impl/htcondor/HtCondorWrapper.scala | 7 +++- .../impl/htcondor/HtCondorCommandSpec.scala | 16 ++++---- .../htcondor/HtCondorRuntimeAttributesSpec.scala | 43 ++++++++++++++++++---- .../provider/mongodb/MongoCacheActorSpec.scala | 2 +- 7 files changed, 85 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 8c6d759da..9d2b37e6d 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management * [Caching configuration](#caching-configuration) * [Docker](#docker) * [CPU, Memory and Disk](#cpu-memory-and-disk) + * [Native Specifications](#native-specifications) * [Spark Backend](#spark-backend) * [Configuring Spark Project](#configuring-spark-project) * [Configuring Spark Master and Deploy Mode](#configuring-spark-master-and-deploy-mode) @@ -889,6 +890,18 @@ This backend supports CPU, memory and disk size configuration through the use of It they are not set, HtCondor backend will use default values. +### Native Specifications +The use of runtime attribute 'nativeSpecs' allows to the user to attach custom HtCondor configuration to tasks. +An example of this is when there is a need to work with 'requirements' or 'rank' configuration. + +``` +"runtimeAttributes": { + "nativeSpecs": ["requirements = Arch == \"INTEL\"", "rank = Memory >= 64"] +} +``` + +nativeSpecs attribute needs to be specified as an array of strings to work. + ## Spark Backend This backend adds support for execution of spark jobs in a workflow using the existing wdl format. diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala index 291165d03..dbecacdde 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala @@ -298,7 +298,7 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor HtCondorRuntimeKeys.Disk -> runtimeAttributes.disk.to(MemoryUnit.KB).amount.toLong ) - cmds.generateSubmitFile(submitFilePath, attributes) // This writes the condor submit file + cmds.generateSubmitFile(submitFilePath, attributes, runtimeAttributes.nativeSpecs) // This writes the condor submit file () } catch { diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala index f8dd9a595..758d2dbb4 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala @@ -12,8 +12,8 @@ import cromwell.backend.validation.RuntimeAttributesValidation._ import cromwell.core._ import cromwell.core.ErrorOr._ import lenthall.exception.MessageAggregation -import wdl4s.types.{WdlIntegerType, WdlStringType, WdlBooleanType, WdlType} -import wdl4s.values.{WdlString, WdlBoolean, WdlInteger, WdlValue} +import wdl4s.types._ +import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString, WdlValue} object HtCondorRuntimeAttributes { @@ -26,6 +26,7 @@ object HtCondorRuntimeAttributes { val DockerWorkingDirKey = "dockerWorkingDir" val DockerOutputDirKey = "dockerOutputDir" val DiskKey = "disk" + val NativeSpecsKey = "nativeSpecs" val staticDefaults = Map( FailOnStderrKey -> WdlBoolean(FailOnStderrDefaultValue), @@ -43,7 +44,8 @@ object HtCondorRuntimeAttributes { DockerOutputDirKey -> Set(WdlStringType), CpuKey -> Set(WdlIntegerType), MemoryKey -> Set(WdlStringType), - DiskKey -> Set(WdlStringType) + DiskKey -> Set(WdlStringType), + NativeSpecsKey -> Set(WdlArrayType(WdlStringType)) ) def apply(attrs: Map[String, WdlValue], options: WorkflowOptions): HtCondorRuntimeAttributes = { @@ -59,9 +61,10 @@ object HtCondorRuntimeAttributes { val cpu = validateCpu(withDefaultValues.get(CpuKey), noValueFoundFor(CpuKey)) val memory = validateMemory(withDefaultValues.get(MemoryKey), noValueFoundFor(MemoryKey)) val disk = validateDisk(withDefaultValues.get(DiskKey), noValueFoundFor(DiskKey)) + val nativeSpecs = validateNativeSpecs(withDefaultValues.get(NativeSpecsKey), None.validNel) - (continueOnReturnCode |@| docker |@| dockerWorkingDir |@| dockerOutputDir |@| failOnStderr |@| cpu |@| memory |@| disk) map { - new HtCondorRuntimeAttributes(_, _, _, _, _, _, _, _) + (continueOnReturnCode |@| docker |@| dockerWorkingDir |@| dockerOutputDir |@| failOnStderr |@| cpu |@| memory |@| disk |@| nativeSpecs) map { + new HtCondorRuntimeAttributes(_, _, _, _, _, _, _, _, _) } match { case Valid(x) => x case Invalid(nel) => throw new RuntimeException with MessageAggregation { @@ -97,6 +100,17 @@ object HtCondorRuntimeAttributes { case None => onMissingKey } } + + private def validateNativeSpecs(value: Option[WdlValue], onMissingKey: => ErrorOr[Option[Array[String]]]): ErrorOr[Option[Array[String]]] = { + val nativeSpecsWrongFormatMsg = s"Expecting $NativeSpecsKey runtime attribute to be an Array of Strings. Exception: %s" + value match { + case Some(ns: WdlArray) if ns.wdlType.memberType.equals(WdlStringType) => + val nsa = ns.value.map { value => value.valueString }.toArray + Option(nsa).validNel + case Some(_) => String.format(nativeSpecsWrongFormatMsg, "Not supported WDL type value").invalidNel + case None => onMissingKey + } + } } case class HtCondorRuntimeAttributes(continueOnReturnCode: ContinueOnReturnCode, @@ -106,4 +120,5 @@ case class HtCondorRuntimeAttributes(continueOnReturnCode: ContinueOnReturnCode, failOnStderr: Boolean, cpu: Int, memory: MemorySize, - disk: MemorySize) + disk: MemorySize, + nativeSpecs: Option[Array[String]]) diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala index 5b5d014cf..35af6dd1d 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala @@ -68,13 +68,16 @@ echo $$? > rc () } - def generateSubmitFile(path: Path, attributes: Map[String, Any]): String = { + def generateSubmitFile(path: Path, attributes: Map[String, Any], nativeSpecs: Option[Array[String]]): String = { def htCondorSubmitCommand(filePath: Path) = { s"${HtCondorCommands.Submit} ${filePath.toString}" } val submitFileWriter = path.untailed - attributes.foreach(attribute => submitFileWriter.writeWithNewline(s"${attribute._1}=${attribute._2}")) + attributes.foreach { attribute => submitFileWriter.writeWithNewline(s"${attribute._1}=${attribute._2}") } + //Native specs is intended for attaching HtCondor native configuration such as 'requirements' and 'rank' definition + //directly to the submit file. + nativeSpecs foreach { _.foreach { submitFileWriter.writeWithNewline } } submitFileWriter.writeWithNewline(HtCondorRuntimeKeys.Queue) submitFileWriter.writer.flushAndClose() logger.debug(s"submit file name is : $path") diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorCommandSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorCommandSpec.scala index ce1097f4e..9a03fecca 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorCommandSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorCommandSpec.scala @@ -5,17 +5,17 @@ import better.files._ import org.scalatest.{Matchers, WordSpecLike} class HtCondorCommandSpec extends WordSpecLike with Matchers { - val attributes = Map("executable" -> "test.sh", "input" -> "/temp/test", "error"->"stderr") - val resultAttributes = List("executable=test.sh","input=/temp/test","error=stderr", "queue") - val htCondorCommands = new HtCondorCommands + private val attributes = Map("executable" -> "test.sh", "input" -> "/temp/test", "error"->"stderr") + private val resultAttributes = List("executable=test.sh","input=/temp/test","error=stderr", "spec1", "spec2", "queue") + private val htCondorCommands = new HtCondorCommands + private val nativeSpecs = Option(Array("spec1", "spec2")) "submitCommand method" should { "return submit file with content passed to it" in { - val dir = File.newTemporaryFile() - val command = htCondorCommands.generateSubmitFile(dir.path,attributes) - val file = dir - resultAttributes shouldEqual dir.lines.toList - dir.delete() + val file = File.newTemporaryFile() + val command = htCondorCommands.generateSubmitFile(file.path, attributes, nativeSpecs) + resultAttributes shouldEqual file.lines.toList + file.delete() command shouldEqual s"condor_submit ${file.path}" } } diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala index 8e794634d..85e9185a0 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala @@ -39,7 +39,8 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { val memorySize = MemorySize.parse("0.512 GB").get val diskSize = MemorySize.parse("1.024 GB").get - val staticDefaults = new HtCondorRuntimeAttributes(ContinueOnReturnCodeSet(Set(0)), None, None, None, false, 1, memorySize, diskSize) + val staticDefaults = new HtCondorRuntimeAttributes(ContinueOnReturnCodeSet(Set(0)), None, None, None, false, 1, + memorySize, diskSize, None) def workflowOptionsWithDefaultRA(defaults: Map[String, JsValue]) = { WorkflowOptions(JsObject(Map( @@ -219,11 +220,16 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes) } - "throw an exception when tries to validate an invalid disk entry" in { + "throw an exception when tries to validate an invalid String disk entry" in { val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { docker: "ubuntu:latest" disk: "value" }""").head assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting memory runtime attribute to be an Integer or String with format '8 GB'") } + "throw an exception when tries to validate an invalid Integer array disk entry" in { + val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { docker: "ubuntu:latest" disk: [1] }""").head + assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting disk runtime attribute to be an Integer or String with format '8 GB'") + } + "use workflow options as default if disk key is missing" in { val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head val expectedRuntimeAttributes = staticDefaults.copy(disk = MemorySize.parse("65 GB").get) @@ -237,25 +243,48 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { val shouldBeIgnored = workflowOptionsWithDefaultRA(Map()) assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes) } + + "return an instance of itself when tries to validate a valid native specs entry" in { + val expectedRuntimeAttributes = staticDefaults.copy(nativeSpecs = Option(Array("spec1", "spec2"))) + val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { nativeSpecs: ["spec1", "spec2"] }""").head + assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes) + } + + "throw an exception when tries to validate an invalid native specs entry" in { + val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { nativeSpecs: [1, 2] }""").head + assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting nativeSpecs runtime attribute to be an Array of Strings.") + } } - private def assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes: Map[String, WdlValue], workflowOptions: WorkflowOptions, expectedRuntimeAttributes: HtCondorRuntimeAttributes): Unit = { + private def assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes: Map[String, WdlValue], + workflowOptions: WorkflowOptions, + expectedRuntimeAttributes: HtCondorRuntimeAttributes) = { try { - assert(HtCondorRuntimeAttributes(runtimeAttributes, workflowOptions) == expectedRuntimeAttributes) + val actualRuntimeAttr = HtCondorRuntimeAttributes(runtimeAttributes, workflowOptions) + assert(actualRuntimeAttr.cpu == expectedRuntimeAttributes.cpu) + assert(actualRuntimeAttr.disk == expectedRuntimeAttributes.disk) + assert(actualRuntimeAttr.memory == expectedRuntimeAttributes.memory) + assert(actualRuntimeAttr.continueOnReturnCode == expectedRuntimeAttributes.continueOnReturnCode) + assert(actualRuntimeAttr.failOnStderr == expectedRuntimeAttributes.failOnStderr) + assert(actualRuntimeAttr.dockerWorkingDir == expectedRuntimeAttributes.dockerWorkingDir) + assert(actualRuntimeAttr.dockerImage == expectedRuntimeAttributes.dockerImage) + assert(actualRuntimeAttr.dockerOutputDir == expectedRuntimeAttributes.dockerOutputDir) + expectedRuntimeAttributes.nativeSpecs match { + case Some(ns) => assert(ns.deep == expectedRuntimeAttributes.nativeSpecs.get.deep) + case None => assert(expectedRuntimeAttributes.nativeSpecs.isEmpty) + } } catch { case ex: RuntimeException => fail(s"Exception was not expected but received: ${ex.getMessage}") } - () } - private def assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue], exMsg: String): Unit = { + private def assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue], exMsg: String) = { try { HtCondorRuntimeAttributes(runtimeAttributes, emptyWorkflowOptions) fail("A RuntimeException was expected.") } catch { case ex: RuntimeException => assert(ex.getMessage.contains(exMsg)) } - () } private def createRuntimeAttributes(wdlSource: WdlSource, runtimeAttributes: String): Seq[Map[String, WdlValue]] = { diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala index 8100799fd..6790b5bd4 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala @@ -33,7 +33,7 @@ class MongoCacheActorSpec extends TestKit(ActorSystem("MongoCacheProviderActorSp val mongoDbCollectionMock = mock[MongoCollection] val memorySize = MemorySize.parse("0.512 GB").get val diskSize = MemorySize.parse("1.024 GB").get - val runtimeConfig = HtCondorRuntimeAttributes(ContinueOnReturnCodeSet(Set(0)), Some("tool-name"), Some("/workingDir"), Some("/outputDir"), true, 1, memorySize, diskSize) + val runtimeConfig = HtCondorRuntimeAttributes(ContinueOnReturnCodeSet(Set(0)), Some("tool-name"), Some("/workingDir"), Some("/outputDir"), true, 1, memorySize, diskSize, None) val jobHash = "88dde49db10f1551299fb9937f313c10" val taskStatus = "done" val succeededResponseMock = SucceededResponse(BackendJobDescriptorKey(Call(Option("taskName"), null, null, null), None, 0), None, Map("test" -> JobOutput(WdlString("Test"))), None, Seq.empty) From 7fa6f90ec71ab72f82da93bfb873390ada2b99be Mon Sep 17 00:00:00 2001 From: "Francisco M. Casares" Date: Thu, 3 Nov 2016 10:39:27 -0700 Subject: [PATCH 049/375] Add logging to SFS localization functions. (#1643) * Added logging to SFS localization functions. * Revert "Added logging to SFS localization functions." This reverts commit cd9655bc6a04340e3e209ddcc304007e0cda369b. * Reducing the amount of changes to the minimal due to reviewers request. * Change based on PR comment. * Removed by-name parameter. * Minor refactoring. --- .../cromwell/backend/sfs/SharedFileSystem.scala | 31 ++++++++++++++++------ 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala index b514ad8cc..55f01b9ae 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala @@ -5,6 +5,7 @@ import java.nio.file.{Path, Paths} import cats.instances.try_._ import cats.syntax.functor._ import com.typesafe.config.Config +import com.typesafe.scalalogging.StrictLogging import cromwell.backend.io.JobPaths import cromwell.core._ import cromwell.core.path.PathFactory @@ -17,7 +18,7 @@ import scala.collection.JavaConverters._ import scala.language.postfixOps import scala.util.{Failure, Success, Try} -object SharedFileSystem { +object SharedFileSystem extends StrictLogging { import better.files._ final case class AttemptedLookupResult(name: String, value: Try[WdlValue]) { @@ -43,24 +44,38 @@ object SharedFileSystem { } private def localizePathViaCopy(originalPath: File, executionPath: File): Try[Unit] = { - executionPath.parent.createDirectories() - val executionTmpPath = pathPlusSuffix(executionPath, ".tmp") - Try(originalPath.copyTo(executionTmpPath, overwrite = true).moveTo(executionPath, overwrite = true)).void + val action = Try { + executionPath.parent.createDirectories() + val executionTmpPath = pathPlusSuffix(executionPath, ".tmp") + originalPath.copyTo(executionTmpPath, overwrite = true).moveTo(executionPath, overwrite = true) + }.void + logOnFailure(action, "copy") } private def localizePathViaHardLink(originalPath: File, executionPath: File): Try[Unit] = { - executionPath.parent.createDirectories() - Try { executionPath.linkTo(originalPath, symbolic = false) } void + val action = Try { + executionPath.parent.createDirectories() + executionPath.linkTo(originalPath, symbolic = false) + }.void + logOnFailure(action, "hard link") } private def localizePathViaSymbolicLink(originalPath: File, executionPath: File): Try[Unit] = { if (originalPath.isDirectory) Failure(new UnsupportedOperationException("Cannot localize directory with symbolic links")) else { - executionPath.parent.createDirectories() - Try { executionPath.linkTo(originalPath, symbolic = true) } void + val action = Try { + executionPath.parent.createDirectories() + executionPath.linkTo(originalPath, symbolic = true) + }.void + logOnFailure(action, "symbolic link") } } + private def logOnFailure(action: Try[Unit], actionLabel: String): Try[Unit] = { + if (action.isFailure) logger.warn(s"Localization via $actionLabel has failed: ${action.failed.get.getMessage}", action.failed.get) + action + } + private def duplicate(description: String, source: File, dest: File, strategies: Stream[DuplicationStrategy]) = { import cromwell.util.FileUtil._ From 9a3b8b61a4f89f649f97bca999a870a631e00890 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Mon, 24 Oct 2016 23:52:14 -0400 Subject: [PATCH 050/375] Adding pair support in Cromwell --- CHANGELOG.md | 6 +- .../backend/BackendLifecycleActorFactory.scala | 4 +- .../BackendWorkflowInitializationActor.scala | 6 +- .../validation/RuntimeAttributesValidation.scala | 5 +- .../scala/cromwell/backend/wdl/PureFunctions.scala | 89 --------------- .../validation/RuntimeAttributesDefaultSpec.scala | 2 +- ...cala => PureStandardLibraryFunctionsSpec.scala} | 5 +- .../cromwell/core/simpleton/WdlValueBuilder.scala | 20 +++- .../core/simpleton/WdlValueSimpleton.scala | 1 + .../core/simpleton/WdlValueBuilderSpec.scala | 123 +++++++++++++++++---- .../main/scala/cromwell/engine/WdlFunctions.scala | 6 +- .../cromwell/engine/EngineFunctionsSpec.scala | 7 +- project/Dependencies.scala | 2 +- .../backend/impl/aws/AwsJobExecutionActor.scala | 5 +- .../backend/impl/jes/JesExpressionFunctions.scala | 7 +- .../sfs/SharedFileSystemExpressionFunctions.scala | 5 +- 16 files changed, 153 insertions(+), 140 deletions(-) delete mode 100644 backend/src/main/scala/cromwell/backend/wdl/PureFunctions.scala rename backend/src/test/scala/cromwell/backend/wdl/{PureFunctionsSpec.scala => PureStandardLibraryFunctionsSpec.scala} (79%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60e78f43a..2a63c8d97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,13 @@ ## 23 -* Included the WDL matrix `transpose` function for `Array[Array[X]]` types. * Added an option `call-caching.invalidate-bad-cache-results` (default: `true`). If true, Cromwell will invalidate cached results which have failed to copy as part of a cache hit. * Timing diagrams and metadata now receive more fine grained workflow states between submission and Running. +* Support for the Pair WDL type (e.g. `Pair[Int, File] floo = (3, "gs://blar/blaz/qlux.txt")`) +* Added support for new WDL functions: + * `zip: (Array[X], Array[Y]) => Array[Pair[X, Y]]` - align items in the two arrays by index and return them as WDL pairs + * `cross: (Array[X], Array[Y]) => Array[Pair[X, Y]]` - create every possible pair from the two input arrays and return them all as WDL pairs + * `transpose: (Array[Array[X]]) => Array[Array[X]]` compute the matrix transpose for a 2D array. Assumes each inner array has the same length. ## 0.22 diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala index c4b0efe14..f8adf04dd 100644 --- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala +++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala @@ -7,7 +7,7 @@ import com.typesafe.config.Config import cromwell.backend.callcaching.FileHashingActor import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction import cromwell.backend.io.WorkflowPaths -import cromwell.backend.wdl.OnlyPureFunctions +import wdl4s.expression.PureStandardLibraryFunctions import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.core.{ExecutionStore, OutputStore} import wdl4s.Call @@ -45,7 +45,7 @@ trait BackendLifecycleActorFactory { def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor, jobKey: BackendJobDescriptorKey, - initializationData: Option[BackendInitializationData]): WdlStandardLibraryFunctions = OnlyPureFunctions + initializationData: Option[BackendInitializationData]): WdlStandardLibraryFunctions = PureStandardLibraryFunctions def getExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config, initializationData: Option[BackendInitializationData]): Path = { new WorkflowPaths(workflowDescriptor, backendConfig).executionRoot diff --git a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala index b8fc011df..8895064be 100644 --- a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala @@ -4,7 +4,7 @@ import akka.actor.{ActorLogging, ActorRef} import akka.event.LoggingReceive import cromwell.backend.BackendLifecycleActor._ import cromwell.backend.BackendWorkflowInitializationActor._ -import cromwell.backend.wdl.OnlyPureFunctions +import wdl4s.expression.PureStandardLibraryFunctions import cromwell.core.{WorkflowMetadataKeys, WorkflowOptions} import cromwell.services.metadata.MetadataService.PutMetadataAction import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} @@ -53,7 +53,7 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w wdlExpressionMaybe match { case None => !valueRequired case Some(wdlExpression: WdlExpression) => - wdlExpression.evaluate(NoLookup, OnlyPureFunctions) map (_.wdlType) match { + wdlExpression.evaluate(NoLookup, PureStandardLibraryFunctions) map (_.wdlType) match { case Success(wdlType) => predicate(wdlType) case Failure(_) => true // If we can't evaluate it, we'll let it pass for now... } @@ -81,7 +81,7 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w wdlExpressionMaybe match { case None => !valueRequired case Some(wdlExpression: WdlExpression) => - wdlExpression.evaluate(NoLookup, OnlyPureFunctions) match { + wdlExpression.evaluate(NoLookup, PureStandardLibraryFunctions) match { case Success(wdlValue) => validateValue(wdlValue) case Failure(throwable) => true // If we can't evaluate it, we'll let it pass for now... } diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala index 9e56c71be..1ba92527e 100644 --- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala @@ -1,9 +1,8 @@ package cromwell.backend.validation import cats.syntax.validated._ -import cromwell.backend.wdl.OnlyPureFunctions +import wdl4s.expression.PureStandardLibraryFunctions import cromwell.backend.{MemorySize, RuntimeAttributeDefinition} -import cromwell.core._ import cromwell.core.ErrorOr._ import org.slf4j.Logger import wdl4s.WdlExpression @@ -336,7 +335,7 @@ trait RuntimeAttributesValidation[ValidatedType] { For now, if something tries to "lookup" a value, convert it to a WdlString. */ val wdlStringLookup: ScopedLookupFunction = (value: String) => WdlString(value) - wdlExpression.evaluate(wdlStringLookup, OnlyPureFunctions) match { + wdlExpression.evaluate(wdlStringLookup, PureStandardLibraryFunctions) match { case Success(wdlValue) => validateExpression.applyOrElse(wdlValue, (_: Any) => false) case Failure(throwable) => throw new RuntimeException(s"Expression evaluation failed due to $throwable: $wdlExpression", throwable) diff --git a/backend/src/main/scala/cromwell/backend/wdl/PureFunctions.scala b/backend/src/main/scala/cromwell/backend/wdl/PureFunctions.scala deleted file mode 100644 index 583b6e2f3..000000000 --- a/backend/src/main/scala/cromwell/backend/wdl/PureFunctions.scala +++ /dev/null @@ -1,89 +0,0 @@ -package cromwell.backend.wdl - -import wdl4s.expression.WdlStandardLibraryFunctions -import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType, WdlType} -import wdl4s.values.{WdlArray, WdlFile, WdlFloat, WdlInteger, WdlString, WdlValue} - -import scala.util.{Failure, Success, Try} - -case object OnlyPureFunctions extends WdlStandardLibraryFunctions with PureFunctions { - override def readFile(path: String): String = throw new NotImplementedError("readFile not available in OnlyPureFunctions.") - override def read_json(params: Seq[Try[WdlValue]]): Try[WdlValue] = throw new NotImplementedError("read_json not available in OnlyPureFunctions.") - override def write_json(params: Seq[Try[WdlValue]]): Try[WdlFile] = throw new NotImplementedError("write_json not available in OnlyPureFunctions.") - override def size(params: Seq[Try[WdlValue]]): Try[WdlFloat] = throw new NotImplementedError("size not available in OnlyPureFunctions.") - override def write_tsv(params: Seq[Try[WdlValue]]): Try[WdlFile] = throw new NotImplementedError("write_tsv not available in OnlyPureFunctions.") - override def stdout(params: Seq[Try[WdlValue]]): Try[WdlFile] = throw new NotImplementedError("stdout not available in OnlyPureFunctions.") - override def glob(path: String, pattern: String): Seq[String] = throw new NotImplementedError("glob not available in OnlyPureFunctions.") - override def writeTempFile(path: String, prefix: String, suffix: String, content: String): String = throw new NotImplementedError("writeTempFile not available in OnlyPureFunctions.") - override def stderr(params: Seq[Try[WdlValue]]): Try[WdlFile] = throw new NotImplementedError("stderr not available in OnlyPureFunctions.") -} - -trait PureFunctions { this: WdlStandardLibraryFunctions => - - override def transpose(params: Seq[Try[WdlValue]]): Try[WdlArray] = { - def extractExactlyOneArg: Try[WdlValue] = params.size match { - case 1 => params.head - case n => Failure(new IllegalArgumentException(s"Invalid number of parameters for engine function transpose: $n. Ensure transpose(x: Array[Array[X]]) takes exactly 1 parameters.")) - } - - case class ExpandedTwoDimensionalArray(innerType: WdlType, value: Seq[Seq[WdlValue]]) - def validateAndExpand(value: WdlValue): Try[ExpandedTwoDimensionalArray] = value match { - case WdlArray(WdlArrayType(WdlArrayType(innerType)), array: Seq[WdlValue]) => expandWdlArray(array) map { ExpandedTwoDimensionalArray(innerType, _) } - case array @ WdlArray(WdlArrayType(nonArrayType), _) => Failure(new IllegalArgumentException(s"Array must be two-dimensional to be transposed but given array of $nonArrayType")) - case otherValue => Failure(new IllegalArgumentException(s"Function 'transpose' must be given a two-dimensional array but instead got ${otherValue.typeName}")) - } - - def expandWdlArray(outerArray: Seq[WdlValue]): Try[Seq[Seq[WdlValue]]] = Try { - outerArray map { - case array: WdlArray => array.value - case otherValue => throw new IllegalArgumentException(s"Function 'transpose' must be given a two-dimensional array but instead got WdlArray[${otherValue.typeName}]") - } - } - - def transpose(expandedTwoDimensionalArray: ExpandedTwoDimensionalArray): Try[WdlArray] = Try { - val innerType = expandedTwoDimensionalArray.innerType - val array = expandedTwoDimensionalArray.value - WdlArray(WdlArrayType(WdlArrayType(innerType)), array.transpose map { WdlArray(WdlArrayType(innerType), _) }) - } - - extractExactlyOneArg.flatMap(validateAndExpand).flatMap(transpose) - } - - override def range(params: Seq[Try[WdlValue]]): Try[WdlArray] = { - def extractAndValidateArguments = params.size match { - case 1 => validateArguments(params.head) - case n => Failure(new IllegalArgumentException(s"Invalid number of parameters for engine function range: $n. Ensure range(x: WdlInteger) takes exactly 1 parameters.")) - } - - def validateArguments(value: Try[WdlValue]) = value match { - case Success(intValue: WdlValue) if WdlIntegerType.isCoerceableFrom(intValue.wdlType) => - Integer.valueOf(intValue.valueString) match { - case i if i >= 0 => Success(i) - case n => Failure(new IllegalArgumentException(s"Parameter to seq must be greater than or equal to 0 (but got $n)")) - } - case _ => Failure(new IllegalArgumentException(s"Invalid parameter for engine function seq: $value.")) - } - - extractAndValidateArguments map { intValue => WdlArray(WdlArrayType(WdlIntegerType), (0 until intValue).map(WdlInteger(_))) } - } - - override def sub(params: Seq[Try[WdlValue]]): Try[WdlString] = { - def extractArguments = params.size match { - case 3 => Success((params.head, params(1), params(2))) - case n => Failure(new IllegalArgumentException(s"Invalid number of parameters for engine function sub: $n. sub takes exactly 3 parameters.")) - } - - def validateArguments(values: (Try[WdlValue], Try[WdlValue], Try[WdlValue])) = values match { - case (Success(strValue), Success(WdlString(pattern)), Success(replaceValue)) - if WdlStringType.isCoerceableFrom(strValue.wdlType) && - WdlStringType.isCoerceableFrom(replaceValue.wdlType) => - Success((strValue.valueString, pattern, replaceValue.valueString)) - case _ => Failure(new IllegalArgumentException(s"Invalid parameters for engine function sub: $values.")) - } - - for { - args <- extractArguments - (str, pattern, replace) <- validateArguments(args) - } yield WdlString(pattern.r.replaceAllIn(str, replace)) - } -} diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala index d1df9d790..325f874b8 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala @@ -80,7 +80,7 @@ class RuntimeAttributesDefaultSpec extends FlatSpec with Matchers { val defaults = workflowOptionsDefault(workflowOptions, coercionMap) defaults.isFailure shouldBe true - defaults.failed.get.getMessage shouldBe s"Could not parse JsonValue ${map("str")} to valid WdlValue for runtime attribute str" + defaults.failed.get.getMessage shouldBe s": RuntimeException: Could not parse JsonValue ${map("str")} to valid WdlValue for runtime attribute str" } it should "fold default values" in { diff --git a/backend/src/test/scala/cromwell/backend/wdl/PureFunctionsSpec.scala b/backend/src/test/scala/cromwell/backend/wdl/PureStandardLibraryFunctionsSpec.scala similarity index 79% rename from backend/src/test/scala/cromwell/backend/wdl/PureFunctionsSpec.scala rename to backend/src/test/scala/cromwell/backend/wdl/PureStandardLibraryFunctionsSpec.scala index 284bf7636..fb6507377 100644 --- a/backend/src/test/scala/cromwell/backend/wdl/PureFunctionsSpec.scala +++ b/backend/src/test/scala/cromwell/backend/wdl/PureStandardLibraryFunctionsSpec.scala @@ -1,13 +1,14 @@ package cromwell.backend.wdl import org.scalatest.{FlatSpec, Matchers} +import wdl4s.expression.PureStandardLibraryFunctions import wdl4s.types.{WdlArrayType, WdlIntegerType} import wdl4s.values.{WdlArray, WdlInteger} import scala.util.Success -class PureFunctionsSpec extends FlatSpec with Matchers { +class PureStandardLibraryFunctionsSpec extends FlatSpec with Matchers { behavior of "transpose" @@ -23,7 +24,7 @@ class PureFunctionsSpec extends FlatSpec with Matchers { WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(3), WdlInteger(6))) )) - OnlyPureFunctions.transpose(Seq(Success(inArray))) should be(Success(expectedResult)) + PureStandardLibraryFunctions.transpose(Seq(Success(inArray))) should be(Success(expectedResult)) } } diff --git a/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala b/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala index f9d2464e7..c192f9123 100644 --- a/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala +++ b/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala @@ -1,8 +1,8 @@ package cromwell.core.simpleton import wdl4s.TaskOutput -import wdl4s.types.{WdlArrayType, WdlMapType, WdlPrimitiveType, WdlType} -import wdl4s.values.{WdlArray, WdlMap, WdlValue} +import wdl4s.types._ +import wdl4s.values.{WdlArray, WdlMap, WdlPair, WdlValue} import scala.language.postfixOps import cromwell.core.{JobOutput, JobOutputs} @@ -73,6 +73,19 @@ object WdlValueBuilder { component.path match { case MapElementPattern(key, more) => key.unescapeMeta -> component.copy(path = more)} } + // Returns a tuple of the key into the pair (i.e. left or right) and a `SimpletonComponent` whose path reflects the "descent" + // into the pair. e.g. for a component + // SimpletonComponent(":left:foo", someValue) this would return (PairLeft -> SimpletonComponent(":baz", someValue)). + sealed trait PairLeftOrRight + case object PairLeft extends PairLeftOrRight + case object PairRight extends PairLeftOrRight + def descendIntoPair(component: SimpletonComponent): (PairLeftOrRight, SimpletonComponent) = { + component.path match { + case MapElementPattern("left", more) => PairLeft -> component.copy(path = more) + case MapElementPattern("right", more) => PairRight -> component.copy(path = more) + } + } + // Group tuples by key using a Map with key type `K`. def group[K](tuples: Traversable[(K, SimpletonComponent)]): Map[K, Traversable[SimpletonComponent]] = { tuples groupBy { case (i, _) => i } mapValues { _ map { case (i, s) => s} } @@ -87,6 +100,9 @@ object WdlValueBuilder { val groupedByMapKey: Map[String, Traversable[SimpletonComponent]] = group(components map descendIntoMap) // map keys are guaranteed by the WDL spec to be primitives, so the "coerceRawValue(..).get" is safe. WdlMap(mapType, groupedByMapKey map { case (k, ss) => mapType.keyType.coerceRawValue(k).get -> toWdlValue(mapType.valueType, ss) }) + case pairType: WdlPairType => + val groupedByLeftOrRight: Map[PairLeftOrRight, Traversable[SimpletonComponent]] = group(components map descendIntoPair) + WdlPair(toWdlValue(pairType.leftType, groupedByLeftOrRight(PairLeft)), toWdlValue(pairType.rightType, groupedByLeftOrRight(PairRight))) } } diff --git a/core/src/main/scala/cromwell/core/simpleton/WdlValueSimpleton.scala b/core/src/main/scala/cromwell/core/simpleton/WdlValueSimpleton.scala index c417ad3ee..edcf4c2c9 100644 --- a/core/src/main/scala/cromwell/core/simpleton/WdlValueSimpleton.scala +++ b/core/src/main/scala/cromwell/core/simpleton/WdlValueSimpleton.scala @@ -28,6 +28,7 @@ object WdlValueSimpleton { case prim: WdlPrimitive => List(WdlValueSimpleton(name, prim)) case WdlArray(_, arrayValue) => arrayValue.zipWithIndex flatMap { case (arrayItem, index) => arrayItem.simplify(s"$name[$index]") } case WdlMap(_, mapValue) => mapValue flatMap { case (key, value) => value.simplify(s"$name:${key.valueString.escapeMeta}") } + case WdlPair(left, right) => left.simplify(s"$name:left") ++ right.simplify(s"$name:right") case wdlObject: WdlObjectLike => wdlObject.value flatMap { case (key, value) => value.simplify(s"$name:${key.escapeMeta}") } case other => throw new Exception(s"Cannot simplify wdl value $other of type ${other.wdlType}") } diff --git a/core/src/test/scala/cromwell/core/simpleton/WdlValueBuilderSpec.scala b/core/src/test/scala/cromwell/core/simpleton/WdlValueBuilderSpec.scala index 0d126e1bd..1e558ceb1 100644 --- a/core/src/test/scala/cromwell/core/simpleton/WdlValueBuilderSpec.scala +++ b/core/src/test/scala/cromwell/core/simpleton/WdlValueBuilderSpec.scala @@ -5,7 +5,7 @@ import org.scalatest.{FlatSpec, Matchers} import org.specs2.mock.Mockito import wdl4s.parser.WdlParser.Ast import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlMapType, WdlStringType} -import wdl4s.values.{WdlArray, WdlInteger, WdlMap, WdlString} +import wdl4s.values.{WdlArray, WdlInteger, WdlMap, WdlPair, WdlString, WdlValue} import wdl4s.{TaskOutput, WdlExpression} object WdlValueBuilderSpec { @@ -15,39 +15,116 @@ object WdlValueBuilderSpec { class WdlValueBuilderSpec extends FlatSpec with Matchers with Mockito { - "Builder" should "build" in { - - val wdlValues = Map( - "foo" -> WdlString("none"), - "bar" -> WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(1), WdlInteger(2))), - "baz" -> WdlArray(WdlArrayType(WdlArrayType(WdlIntegerType)), List( + case class SimpletonConversion(name: String, wdlValue: WdlValue, simpletons: Seq[WdlValueSimpleton]) + val simpletonConversions = List( + SimpletonConversion("foo", WdlString("none"), List(WdlValueSimpleton("foo", WdlString("none")))), + SimpletonConversion("bar", WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(1), WdlInteger(2))), List(WdlValueSimpleton("bar[0]", WdlInteger(1)), WdlValueSimpleton("bar[1]", WdlInteger(2)))), + SimpletonConversion( + "baz", + WdlArray(WdlArrayType(WdlArrayType(WdlIntegerType)), List( WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(0), WdlInteger(1))), WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(2), WdlInteger(3))))), - "map" -> WdlMap(WdlMapType(WdlStringType, WdlStringType), Map( + List(WdlValueSimpleton("baz[0][0]", WdlInteger(0)), WdlValueSimpleton("baz[0][1]", WdlInteger(1)), WdlValueSimpleton("baz[1][0]", WdlInteger(2)), WdlValueSimpleton("baz[1][1]", WdlInteger(3))) + ), + SimpletonConversion( + "map", + WdlMap(WdlMapType(WdlStringType, WdlStringType), Map( WdlString("foo") -> WdlString("foo"), - WdlString("bar") -> WdlString("bar")) - ), - "map2" -> WdlMap(WdlMapType(WdlStringType, WdlMapType(WdlStringType, WdlStringType)), Map( - WdlString("foo") -> - WdlMap(WdlMapType(WdlStringType, WdlStringType), Map(WdlString("foo2") -> WdlString("foo"))), - WdlString("bar") -> - WdlMap(WdlMapType(WdlStringType, WdlStringType), Map(WdlString("bar2") -> WdlString("bar"))) - )), - "map3" -> WdlMap(WdlMapType(WdlStringType, WdlArrayType(WdlIntegerType)), Map( + WdlString("bar") -> WdlString("bar"))), + List(WdlValueSimpleton("map:foo", WdlString("foo")), WdlValueSimpleton("map:bar", WdlString("bar"))) + ), + SimpletonConversion( + "mapOfMaps", + WdlMap(WdlMapType(WdlStringType, WdlMapType(WdlStringType, WdlStringType)), Map( + WdlString("foo") -> WdlMap(WdlMapType(WdlStringType, WdlStringType), Map(WdlString("foo2") -> WdlString("foo"))), + WdlString("bar") ->WdlMap(WdlMapType(WdlStringType, WdlStringType), Map(WdlString("bar2") -> WdlString("bar"))))), + List(WdlValueSimpleton("mapOfMaps:foo:foo2", WdlString("foo")), WdlValueSimpleton("mapOfMaps:bar:bar2", WdlString("bar"))) + ), + SimpletonConversion( + "simplePair1", + WdlPair(WdlInteger(1), WdlString("hello")), + List(WdlValueSimpleton("simplePair1:left", WdlInteger(1)), WdlValueSimpleton("simplePair1:right", WdlString("hello"))) + ), + SimpletonConversion( + "simplePair2", + WdlPair(WdlString("left"), WdlInteger(5)), + List(WdlValueSimpleton("simplePair2:left", WdlString("left")), WdlValueSimpleton("simplePair2:right", WdlInteger(5))) + ), + SimpletonConversion( + "pairOfPairs", + WdlPair( + WdlPair(WdlInteger(1), WdlString("one")), + WdlPair(WdlString("two"), WdlInteger(2))), + List( + WdlValueSimpleton("pairOfPairs:left:left", WdlInteger(1)), + WdlValueSimpleton("pairOfPairs:left:right", WdlString("one")), + WdlValueSimpleton("pairOfPairs:right:left", WdlString("two")), + WdlValueSimpleton("pairOfPairs:right:right", WdlInteger(2))) + ), + SimpletonConversion( + "pairOfArrayAndMap", + WdlPair( + WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(1), WdlInteger(2))), + WdlMap(WdlMapType(WdlStringType, WdlIntegerType), Map(WdlString("left") -> WdlInteger(100), WdlString("right") -> WdlInteger(200)))), + List( + WdlValueSimpleton("pairOfArrayAndMap:left[0]", WdlInteger(1)), + WdlValueSimpleton("pairOfArrayAndMap:left[1]", WdlInteger(2)), + WdlValueSimpleton("pairOfArrayAndMap:right:left", WdlInteger(100)), + WdlValueSimpleton("pairOfArrayAndMap:right:right", WdlInteger(200))) + ), + SimpletonConversion( + "mapOfArrays", + WdlMap(WdlMapType(WdlStringType, WdlArrayType(WdlIntegerType)), Map( WdlString("foo") -> WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(0), WdlInteger(1))), - WdlString("bar") -> WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(2), WdlInteger(3)))) - ), - "map4" -> WdlMap(WdlMapType(WdlStringType, WdlStringType), Map( + WdlString("bar") -> WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(2), WdlInteger(3))))), + List(WdlValueSimpleton("mapOfArrays:foo[0]", WdlInteger(0)), WdlValueSimpleton("mapOfArrays:foo[1]", WdlInteger(1)), + WdlValueSimpleton("mapOfArrays:bar[0]", WdlInteger(2)), WdlValueSimpleton("mapOfArrays:bar[1]", WdlInteger(3))) + ), + SimpletonConversion( + "escapology", + WdlMap(WdlMapType(WdlStringType, WdlStringType), Map( WdlString("foo[1]") -> WdlString("foo"), WdlString("bar[[") -> WdlString("bar"), - WdlString("baz:qux") -> WdlString("baz:qux") - )) + WdlString("baz:qux") -> WdlString("baz:qux"))), + List(WdlValueSimpleton("escapology:foo\\[1\\]", WdlString("foo")), + WdlValueSimpleton("escapology:bar\\[\\[", WdlString("bar")), + WdlValueSimpleton("escapology:baz\\:qux", WdlString("baz:qux"))) ) + ) + + behavior of "WdlValueSimpleton and WdlValueBuilder" + + simpletonConversions foreach { case SimpletonConversion(name, wdlValue, simpletons) => + it should s"decompose WdlValues into simpletons ($name)" in { + import WdlValueSimpleton._ + val map = Map(name -> wdlValue) + map.simplify should contain theSameElementsAs simpletons + } + + it should s"build simpletons back into WdlValues ($name)" in { + // The task output is used to tell us the type of output we're expecting: + val taskOutputs = List(TaskOutput(name, wdlValue.wdlType, IgnoredExpression, mock[Ast], None)) + val rebuiltValues = WdlValueBuilder.toWdlValues(taskOutputs, simpletons) + rebuiltValues.size should be(1) + rebuiltValues(name) should be(wdlValue) + } + + } + + + it should "round trip everything together with no losses" in { + + val wdlValues = (simpletonConversions map { case SimpletonConversion(name, wdlValue, simpletons) => name -> wdlValue }).toMap val taskOutputs = wdlValues map { case (k, wv) => TaskOutput(k, wv.wdlType, IgnoredExpression, mock[Ast], None) } + val allSimpletons = simpletonConversions flatMap { case SimpletonConversion(name, wdlValue, simpletons) => simpletons } import WdlValueSimpleton._ - val actual = WdlValueBuilder.toWdlValues(taskOutputs, wdlValues.simplify) + + val actualSimpletons = wdlValues.simplify + actualSimpletons should contain theSameElementsAs allSimpletons + + val actual = WdlValueBuilder.toWdlValues(taskOutputs, actualSimpletons) actual shouldEqual wdlValues } } diff --git a/engine/src/main/scala/cromwell/engine/WdlFunctions.scala b/engine/src/main/scala/cromwell/engine/WdlFunctions.scala index b715f5a49..9fe346c50 100644 --- a/engine/src/main/scala/cromwell/engine/WdlFunctions.scala +++ b/engine/src/main/scala/cromwell/engine/WdlFunctions.scala @@ -1,13 +1,13 @@ package cromwell.engine -import cromwell.backend.wdl.{PureFunctions, ReadLikeFunctions} +import cromwell.backend.wdl.ReadLikeFunctions +import wdl4s.expression.PureStandardLibraryFunctionsLike import cromwell.core.path.PathBuilder -import wdl4s.expression.WdlStandardLibraryFunctions import wdl4s.values.{WdlFile, WdlValue} import scala.util.{Failure, Try} -class WdlFunctions(val pathBuilders: List[PathBuilder]) extends WdlStandardLibraryFunctions with ReadLikeFunctions with PureFunctions { +class WdlFunctions(val pathBuilders: List[PathBuilder]) extends PureStandardLibraryFunctionsLike with ReadLikeFunctions { private def fail(name: String) = Failure(new NotImplementedError(s"$name() not supported at the workflow level yet")) override def write_json(params: Seq[Try[WdlValue]]): Try[WdlFile] = fail("write_json") diff --git a/engine/src/test/scala/cromwell/engine/EngineFunctionsSpec.scala b/engine/src/test/scala/cromwell/engine/EngineFunctionsSpec.scala index 0160bc6f1..14ccbd6bd 100644 --- a/engine/src/test/scala/cromwell/engine/EngineFunctionsSpec.scala +++ b/engine/src/test/scala/cromwell/engine/EngineFunctionsSpec.scala @@ -2,21 +2,22 @@ package cromwell.engine import java.nio.file.Path -import cromwell.backend.wdl.{PureFunctions, ReadLikeFunctions, WriteFunctions} +import cromwell.backend.wdl.{ReadLikeFunctions, WriteFunctions} import cromwell.core.path.{DefaultPathBuilder, PathBuilder} import org.scalatest.prop.TableDrivenPropertyChecks._ import org.scalatest.prop.Tables.Table import org.scalatest.{FlatSpec, Matchers} -import wdl4s.expression.{NoFunctions, WdlStandardLibraryFunctions} +import wdl4s.expression.{NoFunctions, PureStandardLibraryFunctionsLike, WdlStandardLibraryFunctions} import wdl4s.values.{WdlFile, WdlInteger, WdlString, WdlValue} import scala.util.{Failure, Success, Try} class EngineFunctionsSpec extends FlatSpec with Matchers { - trait WdlStandardLibraryImpl extends WdlStandardLibraryFunctions with ReadLikeFunctions with WriteFunctions with PureFunctions { + trait WdlStandardLibraryImpl extends WdlStandardLibraryFunctions with ReadLikeFunctions with WriteFunctions with PureStandardLibraryFunctionsLike { private def fail(name: String) = Failure(new NotImplementedError(s"$name() not implemented yet")) + override def writeTempFile(path: String, prefix: String, suffix: String, content: String): String = super[WriteFunctions].writeTempFile(path, prefix, suffix, content) override def stdout(params: Seq[Try[WdlValue]]): Try[WdlFile] = fail("stdout") override def stderr(params: Seq[Try[WdlValue]]): Try[WdlFile] = fail("stderr") } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 016b70483..ce5013f89 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -2,7 +2,7 @@ import sbt._ object Dependencies { lazy val lenthallV = "0.19" - lazy val wdl4sV = "0.7-5302cbf-SNAPSHOT" + lazy val wdl4sV = "0.7-1852aea-SNAPSHOT" lazy val sprayV = "1.3.3" /* spray-json is an independent project from the "spray suite" diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsJobExecutionActor.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsJobExecutionActor.scala index a81a6cf7e..97c58bc85 100644 --- a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsJobExecutionActor.scala +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsJobExecutionActor.scala @@ -8,7 +8,8 @@ import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, B import com.amazonaws.services.ecs.model._ import cromwell.backend.impl.aws.util.AwsSdkAsyncHandler import cromwell.backend.impl.aws.util.AwsSdkAsyncHandler.AwsSdkAsyncResult -import cromwell.backend.wdl.{Command, OnlyPureFunctions} +import cromwell.backend.wdl.Command +import wdl4s.expression.PureStandardLibraryFunctions import net.ceedubs.ficus.Ficus._ import scala.collection.JavaConverters._ @@ -31,7 +32,7 @@ class AwsJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, override def execute: Future[BackendJobExecutionResponse] = { - val instantiatedCommand = Command.instantiate(jobDescriptor, OnlyPureFunctions).get + val instantiatedCommand = Command.instantiate(jobDescriptor, PureStandardLibraryFunctions).get val commandOverride = new ContainerOverride().withName("simple-app").withCommand(instantiatedCommand) val runRequest: RunTaskRequest = new RunTaskRequest() diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala index b2260f5c8..8951b5a99 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala @@ -2,11 +2,11 @@ package cromwell.backend.impl.jes import java.nio.file.{Files, Path} -import cromwell.backend.wdl.{PureFunctions, ReadLikeFunctions, WriteFunctions} +import cromwell.backend.wdl.{ReadLikeFunctions, WriteFunctions} import cromwell.core.CallContext import cromwell.core.path.PathBuilder import cromwell.filesystems.gcs.GcsPathBuilder -import wdl4s.expression.WdlStandardLibraryFunctions +import wdl4s.expression.{PureStandardLibraryFunctionsLike, WdlStandardLibraryFunctions} import wdl4s.values._ import scala.collection.JavaConverters._ @@ -14,8 +14,9 @@ import scala.language.postfixOps import scala.util.{Success, Try} class JesExpressionFunctions(override val pathBuilders: List[PathBuilder], context: CallContext) - extends WdlStandardLibraryFunctions with PureFunctions with ReadLikeFunctions with WriteFunctions { + extends WdlStandardLibraryFunctions with PureStandardLibraryFunctionsLike with ReadLikeFunctions with WriteFunctions { + override def writeTempFile(path: String, prefix: String, suffix: String, content: String): String = super[WriteFunctions].writeTempFile(path, prefix, suffix, content) private def globDirectory(glob: String): String = s"glob-${glob.md5Sum}/" override def globPath(glob: String): String = context.root.resolve(globDirectory(glob)).toString diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala index 417760a84..7464c2e3f 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala @@ -7,7 +7,7 @@ import cromwell.backend.wdl._ import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptorKey, BackendWorkflowDescriptor} import cromwell.core.CallContext import cromwell.core.path.PathBuilder -import wdl4s.expression.WdlStandardLibraryFunctions +import wdl4s.expression.PureStandardLibraryFunctionsLike import wdl4s.values.{WdlFile, WdlValue} import scala.language.postfixOps @@ -57,10 +57,11 @@ object SharedFileSystemExpressionFunctions { class SharedFileSystemExpressionFunctions(override val pathBuilders: List[PathBuilder], context: CallContext - ) extends WdlStandardLibraryFunctions with PureFunctions with ReadLikeFunctions with WriteFunctions { + ) extends PureStandardLibraryFunctionsLike with ReadLikeFunctions with WriteFunctions { import SharedFileSystemExpressionFunctions._ import better.files._ + override def writeTempFile(path: String, prefix: String, suffix: String, content: String): String = super[WriteFunctions].writeTempFile(path, prefix, suffix, content) override def globPath(glob: String) = context.root.toString override def glob(path: String, pattern: String): Seq[String] = { File(context.root).glob(s"**/$pattern") map { _.pathAsString } toSeq From 0e005653ad049e019c9c01b9eeb2e925a39f62b5 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Thu, 3 Nov 2016 11:28:01 -0400 Subject: [PATCH 051/375] Encrypting and clearing workflow options. --- .../migration/src/main/resources/changelog.xml | 1 + .../encrypt_and_clear_workflow_options.xml | 14 +++ .../migration/custom/BatchedTaskChange.scala | 140 +++++++++++++++++++++ .../migration/custom/MigrationTaskChange.scala | 48 +++++++ .../ClearMetadataEntryWorkflowOptions.scala | 15 +++ .../EncryptWorkflowStoreEntryWorkflowOptions.scala | 14 +++ .../workflowoptions/WorkflowOptionsChange.scala | 69 ++++++++++ .../workflowstore/WorkflowStoreActor.scala | 44 ++++++- .../cromwell/engine/WorkflowStoreActorSpec.scala | 65 +++++++++- 9 files changed, 402 insertions(+), 8 deletions(-) create mode 100644 database/migration/src/main/resources/changesets/encrypt_and_clear_workflow_options.xml create mode 100644 database/migration/src/main/scala/cromwell/database/migration/custom/BatchedTaskChange.scala create mode 100644 database/migration/src/main/scala/cromwell/database/migration/custom/MigrationTaskChange.scala create mode 100644 database/migration/src/main/scala/cromwell/database/migration/workflowoptions/ClearMetadataEntryWorkflowOptions.scala create mode 100644 database/migration/src/main/scala/cromwell/database/migration/workflowoptions/EncryptWorkflowStoreEntryWorkflowOptions.scala create mode 100644 database/migration/src/main/scala/cromwell/database/migration/workflowoptions/WorkflowOptionsChange.scala diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml index 162b9cbcf..41d8fba10 100644 --- a/database/migration/src/main/resources/changelog.xml +++ b/database/migration/src/main/resources/changelog.xml @@ -50,6 +50,7 @@ + +* [Firecloud](#firecloud) +* [Security by sysadmin](#security) + * [Multi-tenant](#multi-tenant) + + +# Firecloud + +TODO + +# Security by sysadmin +__Warning!__ + +__This section is community-contributed. It is intended as helpful guidance only, and is not endorsed by the Broad Institute.__ + +Cromwell running in server mode accepts all connections on the configured webservice port. The simplest way to restrict access is by putting an authenticating proxy server in between users and the cromwell server: + 1. Configure a firewall rule on the cromwell server host to deny access to the webservice port (e.g. 8000) from all addresses except a secure proxy host. + 1. Configure `` on the proxy host with ``, to proxy authenticated traffic from the world to the cromwell server. Using Apache `httpd` web server for example with basic htpassword file-based authentication, the configuration might look something like: + + ```Apache + + Order deny,allow + Allow from all + AuthType Basic + AuthName "Password Required" + AuthUserFile /path/to/my/htpasswdfile + Require user someone someoneelse + ProxyPass http://101.101.234.567:8000 # address of cromwell server web service + +``` + + 1. That's it. Users now hit `http://my.proxy.org/cromwell` with authenticated requests, and they're forwarded to port 8000 on the cromwell server host. + +## Multi-tenant +The above scheme extends easily to multiple cromwell instances, for use by different groups within an organization for example. If the instances are running on the same host then each instance should be run as its own dedicated service account user, e.g. `cromwell1`, `cromwell2` etc. so that processes running under one cromwell instance cannot access the files of another; different webservice ports must also be configured. If persistent database storage is being used then each instance should be configured with its own database and database user. The proxy configuration above is extended simply by adding another `Location`: + +```Apache + + Order deny,allow + Allow from all + AuthType Basic + AuthName "Password Required" + AuthUserFile /path/to/my/htpasswdfile1 + Require user stillanotherperson andanother + ProxyPass http://101.101.234.567:8001 + +``` + From 863c01b9d1141cdb08a0ef491dcc6ddba10c5238 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Fri, 4 Nov 2016 12:21:02 -0400 Subject: [PATCH 058/375] Better JES globbing --- .../jes/JesAsyncBackendJobExecutionActor.scala | 90 ++++++++++++++++------ .../backend/impl/jes/JesExpressionFunctions.scala | 17 ++-- .../jes/JesAsyncBackendJobExecutionActorSpec.scala | 4 +- 3 files changed, 75 insertions(+), 36 deletions(-) diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala index 3888fc972..ec42adec2 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala @@ -65,7 +65,7 @@ object JesAsyncBackendJobExecutionActor { * ask them for results. */ case class JesPendingExecutionHandle(jobDescriptor: BackendJobDescriptor, - jesOutputs: Seq[JesFileOutput], + jesOutputs: Set[JesFileOutput], run: Run, previousStatus: Option[RunStatus]) extends ExecutionHandle { override val isDone = false @@ -126,8 +126,6 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes override def receive: Receive = pollingActorClientReceive orElse jesReceiveBehavior orElse super.receive - private def globOutputPath(glob: String) = callRootPath.resolve(s"glob-${glob.md5Sum}/") - private def gcsAuthParameter: Option[JesInput] = { if (jesAttributes.auths.gcs.requiresAuthFile || dockerConfiguration.isDefined) Option(JesLiteralInput(ExtraConfigParamName, jesCallPaths.gcsAuthFilePath.toUri.toString)) @@ -163,15 +161,14 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes */ private def relativeLocalizationPath(file: WdlFile): WdlFile = { getPath(file.value) match { - case Success(path) => { + case Success(path) => val value: WdlSource = path.toUri.getHost + path.toUri.getPath WdlFile(value, file.isGlob) - } case _ => file } } - private[jes] def generateJesInputs(jobDescriptor: BackendJobDescriptor): Iterable[JesInput] = { + private[jes] def generateJesInputs(jobDescriptor: BackendJobDescriptor): Set[JesInput] = { val writeFunctionFiles = call.task.evaluateFilesFromCommand(jobDescriptor.fullyQualifiedInputs, callEngineFunctions) map { case (expression, file) => expression.toWdlString.md5SumShort -> Seq(file) @@ -180,9 +177,11 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes /** Collect all WdlFiles from inputs to the call */ val callInputFiles: Map[FullyQualifiedName, Seq[WdlFile]] = jobDescriptor.fullyQualifiedInputs mapValues { _.collectAsSeq { case w: WdlFile => w } } - (callInputFiles ++ writeFunctionFiles) flatMap { + val inputs = (callInputFiles ++ writeFunctionFiles) flatMap { case (name, files) => jesInputsFromWdlFiles(name, files, files.map(relativeLocalizationPath), jobDescriptor) } + + inputs.toSet } /** @@ -212,18 +211,48 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes if (referenceName.length <= 127) referenceName else referenceName.md5Sum } - private[jes] def generateJesOutputs(jobDescriptor: BackendJobDescriptor): Seq[JesFileOutput] = { + private[jes] def findGlobOutputs(jobDescriptor: BackendJobDescriptor): Set[WdlGlobFile] = { + val globOutputs = (call.task.findOutputFiles(jobDescriptor.fullyQualifiedInputs, NoFunctions) map relativeLocalizationPath) collect { + case glob: WdlGlobFile => glob + } + globOutputs.distinct.toSet + } + + private[jes] def generateJesOutputs(jobDescriptor: BackendJobDescriptor): Set[JesFileOutput] = { val wdlFileOutputs = call.task.findOutputFiles(jobDescriptor.fullyQualifiedInputs, NoFunctions) map relativeLocalizationPath - // Create the mappings. GLOB mappings require special treatment (i.e. stick everything matching the glob in a folder) - wdlFileOutputs.distinct map { wdlFile => - val destination = wdlFile match { - case WdlSingleFile(filePath) => callRootPath.resolve(filePath.stripPrefix("/")).toUri.toString - case WdlGlobFile(filePath) => globOutputPath(filePath).toUri.toString + val outputs = wdlFileOutputs.distinct flatMap { wdlFile => + wdlFile match { + case singleFile: WdlSingleFile => List(generateJesSingleFileOutputs(singleFile)) + case globFile: WdlGlobFile => generateJesGlobFileOutputs(globFile) } - val (relpath, disk) = relativePathAndAttachedDisk(wdlFile.value, runtimeAttributes.disks) - JesFileOutput(makeSafeJesReferenceName(wdlFile.value), destination, relpath, disk) } + + outputs.toSet + } + + private def generateJesSingleFileOutputs(wdlFile: WdlSingleFile): JesFileOutput = { + val destination = callRootPath.resolve(wdlFile.value.stripPrefix("/")).toUri.toString + val (relpath, disk) = relativePathAndAttachedDisk(wdlFile.value, runtimeAttributes.disks) + JesFileOutput(makeSafeJesReferenceName(wdlFile.value), destination, relpath, disk) + } + + private def generateJesGlobFileOutputs(wdlFile: WdlGlobFile): List[JesFileOutput] = { + val globName = callEngineFunctions.globName(wdlFile.value) + val globDirectory = globName + "/" + val globListFile = globName + ".list" + val gcsGlobDirectoryDestinationPath = callRootPath.resolve(globDirectory).toUri.toString + val gcsGlobListFileDestinationPath = callRootPath.resolve(globListFile).toUri.toString + + val (_, globDirectoryDisk) = relativePathAndAttachedDisk(wdlFile.value, runtimeAttributes.disks) + + // We need both the glob directory and the glob list: + List( + // The glob directory: + JesFileOutput(makeSafeJesReferenceName(globDirectory), gcsGlobDirectoryDestinationPath, Paths.get(globDirectory + "*"), globDirectoryDisk), + // The glob list file: + JesFileOutput(makeSafeJesReferenceName(globListFile), gcsGlobListFileDestinationPath, Paths.get(globListFile), globDirectoryDisk) + ) } private def instantiateCommand: Try[String] = { @@ -231,7 +260,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes jobDescriptor.call.task.instantiateCommand(backendInputs, callEngineFunctions, valueMapper = gcsPathToLocal) } - private def uploadCommandScript(command: String, withMonitoring: Boolean): Future[Unit] = { + private def uploadCommandScript(command: String, withMonitoring: Boolean, globFiles: Set[WdlGlobFile]): Future[Unit] = { val monitoring = if (withMonitoring) { s"""|touch $JesMonitoringLogFile |chmod u+x $JesMonitoringScript @@ -241,6 +270,22 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes val tmpDir = File(JesWorkingDisk.MountPoint)./("tmp").path val rcPath = File(JesWorkingDisk.MountPoint)./(returnCodeFilename).path + def globManipulation(globFile: WdlGlobFile) = { + + val globDir = callEngineFunctions.globName(globFile.value) + val (_, disk) = relativePathAndAttachedDisk(globFile.value, runtimeAttributes.disks) + val globDirectory = Paths.get(s"${disk.mountPoint.toAbsolutePath}/$globDir/") + val globList = Paths.get(s"${disk.mountPoint.toAbsolutePath}/$globDir.list") + + s""" + |mkdir $globDirectory + |ln ${globFile.value} $globDirectory + |ls -1 $globDirectory > $globList + """.stripMargin + } + + val globManipulations = globFiles.map(globManipulation).mkString("\n") + val fileContent = s""" |#!/bin/bash @@ -250,6 +295,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes |( |cd ${JesWorkingDisk.MountPoint} |$command + |$globManipulations |) |echo $$? > $rcPath """.stripMargin.trim @@ -294,8 +340,8 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes } protected def runWithJes(command: String, - jesInputs: Seq[JesInput], - jesOutputs: Seq[JesFileOutput], + jesInputs: Set[JesInput], + jesOutputs: Set[JesFileOutput], runIdForResumption: Option[String], withMonitoring: Boolean): Future[ExecutionHandle] = { @@ -304,7 +350,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes val jesParameters = standardParameters ++ gcsAuthParameter ++ jesInputs ++ jesOutputs val jesJobSetup = for { - _ <- uploadCommandScript(command, withMonitoring) + _ <- uploadCommandScript(command, withMonitoring, findGlobOutputs(jobDescriptor)) run <- createJesRun(jesParameters, runIdForResumption) _ = tellMetadata(Map(CallMetadataKeys.JobId -> run.runId)) } yield run @@ -321,8 +367,8 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes } private def startExecuting(monitoringOutput: Option[JesFileOutput], mode: ExecutionMode): Future[ExecutionHandle] = { - val jesInputs: Seq[JesInput] = generateJesInputs(jobDescriptor).toSeq ++ monitoringScript :+ cmdInput - val jesOutputs: Seq[JesFileOutput] = generateJesOutputs(jobDescriptor) ++ monitoringOutput + val jesInputs: Set[JesInput] = generateJesInputs(jobDescriptor) ++ monitoringScript + cmdInput + val jesOutputs: Set[JesFileOutput] = generateJesOutputs(jobDescriptor) ++ monitoringOutput instantiateCommand match { case Success(command) => runWithJes(command, jesInputs, jesOutputs, mode.jobId.collectFirst { case j: JesJobId => j.operationId }, monitoringScript.isDefined) @@ -419,7 +465,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), metadataKeyValues) } - private[jes] def wdlValueToGcsPath(jesOutputs: Seq[JesFileOutput])(value: WdlValue): WdlValue = { + private[jes] def wdlValueToGcsPath(jesOutputs: Set[JesFileOutput])(value: WdlValue): WdlValue = { def toGcsPath(wdlFile: WdlFile) = jesOutputs collectFirst { case o if o.name == makeSafeJesReferenceName(wdlFile.valueString) => WdlFile(o.gcs) } getOrElse value diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala index 8951b5a99..bf29a387f 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala @@ -10,28 +10,21 @@ import wdl4s.expression.{PureStandardLibraryFunctionsLike, WdlStandardLibraryFun import wdl4s.values._ import scala.collection.JavaConverters._ -import scala.language.postfixOps import scala.util.{Success, Try} class JesExpressionFunctions(override val pathBuilders: List[PathBuilder], context: CallContext) extends WdlStandardLibraryFunctions with PureStandardLibraryFunctionsLike with ReadLikeFunctions with WriteFunctions { override def writeTempFile(path: String, prefix: String, suffix: String, content: String): String = super[WriteFunctions].writeTempFile(path, prefix, suffix, content) - private def globDirectory(glob: String): String = s"glob-${glob.md5Sum}/" + private[jes] def globDirectory(glob: String): String = globName(glob) + "/" + private[jes] def globName(glob: String) = s"glob-${glob.md5Sum}" override def globPath(glob: String): String = context.root.resolve(globDirectory(glob)).toString override def glob(path: String, pattern: String): Seq[String] = { - /* Globbing is not implemented in the gcs-nio implementation yet (specifically PathMatcher) at this time (09/2016) - * which is fine here since all files in the globing directory have already be matched against the glob pattern by JES. - * - * Also, better.file.File can't be used here because it calls toAbsolutePath on the path in File.apply, which adds a leading "/" in the path. - * This makes the newDirectoryStream implementation of CloudStorageFileSystemProvider return nothing - * because it doesn't call toRealPath for this method before doing I/O and hence does not remove the "/" prefix (bug ?) - * See com.google.cloud.storage.contrib.nio.CloudStorageConfiguration#stripPrefixSlash() - */ - val directory = context.root.resolve(s"glob-${pattern.md5Sum}/").toRealPath() - Files.list(directory).iterator().asScala filterNot { Files.isDirectory(_) } map { _.toUri.toString } toSeq + val name = globName(pattern) + val listFile = context.root.resolve(s"$name.list").toRealPath() + Files.readAllLines(listFile).asScala map { fileName => context.root.resolve(s"$name/$fileName").toUri.toString } } override def preMapping(str: String): String = if (!GcsPathBuilder.isValidGcsUrl(str)) { diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala index 6da2c7fa3..c1608c517 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala @@ -129,7 +129,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend // Mock/stub out the bits that would reach out to JES. val run = mock[Run] - val handle = JesPendingExecutionHandle(jobDescriptor, Seq.empty, run, None) + val handle = JesPendingExecutionHandle(jobDescriptor, Set.empty, run, None) class ExecuteOrRecoverActor extends TestableJesJobExecutionActor(jobDescriptor, promise, jesConfiguration, jesSingletonActor = jesSingletonActor) { override def executeOrRecover(mode: ExecutionMode)(implicit ec: ExecutionContext): Future[ExecutionHandle] = Future.successful(handle) @@ -488,7 +488,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend } it should "convert local Paths back to corresponding GCS paths in JesOutputs" in { - val jesOutputs = Seq( + val jesOutputs = Set( JesFileOutput("/cromwell_root/path/to/file1", "gs://path/to/file1", Paths.get("/cromwell_root/path/to/file1"), workingDisk), JesFileOutput("/cromwell_root/path/to/file2", "gs://path/to/file2", From 445b60c0bed28ab5c64d583d1a496ec5570bdfec Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Wed, 9 Nov 2016 14:05:17 -0500 Subject: [PATCH 059/375] Default the SWRA to abortJobsOnTerminate = true. While in aborting state, `WorkflowExecutionActor` will accept any exit message instead of just abort, including a successful execution. When a `WorkflowManagerActor` has stopped, it will no longer run its optional shutdown hook. `EngineJobHashingActorSpec` refactored to use the `CromwellTestkitSpec`. Removed dead code from `CromwellTestkitSpec.scala`. `CromwellTestkitSpec.TestWorkflowManagerSystem` using unique system names for easier debugging. --- CHANGELOG.md | 1 + README.md | 2 + core/src/main/resources/reference.conf | 2 +- .../workflow/SingleWorkflowRunnerActor.scala | 1 + .../engine/workflow/WorkflowManagerActor.scala | 72 ++++++++++++++-------- .../execution/WorkflowExecutionActor.scala | 9 +-- .../scala/cromwell/server/CromwellRootActor.scala | 5 +- .../scala/cromwell/server/CromwellServer.scala | 2 + .../test/scala/cromwell/CromwellTestkitSpec.scala | 18 ++---- .../workflow/SingleWorkflowRunnerActorSpec.scala | 6 +- .../callcaching/EngineJobHashingActorSpec.scala | 14 ++--- 11 files changed, 75 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a63c8d97..5d38dc798 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ * `zip: (Array[X], Array[Y]) => Array[Pair[X, Y]]` - align items in the two arrays by index and return them as WDL pairs * `cross: (Array[X], Array[Y]) => Array[Pair[X, Y]]` - create every possible pair from the two input arrays and return them all as WDL pairs * `transpose: (Array[Array[X]]) => Array[Array[X]]` compute the matrix transpose for a 2D array. Assumes each inner array has the same length. +* By default, `system.abort-jobs-on-terminate` is false when running `java -jar cromwell.jar server`, and true when running `java -jar cromwell.jar run `. ## 0.22 diff --git a/README.md b/README.md index e1a82db33..365c96ca3 100644 --- a/README.md +++ b/README.md @@ -328,6 +328,8 @@ system { Or, via `-Dsystem.abort-jobs-on-terminate=true` command line option. +By default, this value is false when running `java -jar cromwell.jar server`, and true when running `java -jar cromwell.jar run `. + # Security - Cromwell is NOT on its own a security appliance! diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index f9cffbde3..b3693ded7 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -56,7 +56,7 @@ akka { system { # If 'true', a SIGINT will trigger Cromwell to attempt to abort all currently running jobs before exiting - abort-jobs-on-terminate = false + #abort-jobs-on-terminate = false # Max number of retries per job that the engine will attempt in case of a retryable failure received from the backend max-retries = 10 diff --git a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala index 5300c1eb5..22fbd498f 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala @@ -37,6 +37,7 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: import SingleWorkflowRunnerActor._ private val backoff = SimpleExponentialBackoff(1 second, 1 minute, 1.2) + override val abortJobsOnTerminate = true override lazy val workflowStore = new InMemoryWorkflowStore() override lazy val jobStoreActor = context.actorOf(EmptyJobStoreActor.props) diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala index f8c4eefb5..d963a2b41 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala @@ -18,6 +18,7 @@ import net.ceedubs.ficus.Ficus._ import org.apache.commons.lang3.exception.ExceptionUtils import scala.concurrent.duration._ +import scala.sys.ShutdownHookThread object WorkflowManagerActor { val DefaultMaxWorkflowsToRun = 5000 @@ -44,10 +45,12 @@ object WorkflowManagerActor { jobStoreActor: ActorRef, callCacheReadActor: ActorRef, jobTokenDispenserActor: ActorRef, - backendSingletonCollection: BackendSingletonCollection): Props = { - Props(new WorkflowManagerActor( - workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection) - ).withDispatcher(EngineDispatcher) + backendSingletonCollection: BackendSingletonCollection, + abortJobsOnTerminate: Boolean): Props = { + val params = WorkflowManagerActorParams(ConfigFactory.load, workflowStore, serviceRegistryActor, + workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection, + abortJobsOnTerminate) + Props(new WorkflowManagerActor(params)).withDispatcher(EngineDispatcher) } /** @@ -78,24 +81,20 @@ object WorkflowManagerActor { } } -class WorkflowManagerActor(config: Config, - val workflowStore: ActorRef, - val serviceRegistryActor: ActorRef, - val workflowLogCopyRouter: ActorRef, - val jobStoreActor: ActorRef, - val callCacheReadActor: ActorRef, - val jobTokenDispenserActor: ActorRef, - val backendSingletonCollection: BackendSingletonCollection) +case class WorkflowManagerActorParams(config: Config, + workflowStore: ActorRef, + serviceRegistryActor: ActorRef, + workflowLogCopyRouter: ActorRef, + jobStoreActor: ActorRef, + callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection, + abortJobsOnTerminate: Boolean) + +class WorkflowManagerActor(params: WorkflowManagerActorParams) extends LoggingFSM[WorkflowManagerState, WorkflowManagerData] { - def this(workflowStore: ActorRef, - serviceRegistryActor: ActorRef, - workflowLogCopyRouter: ActorRef, - jobStoreActor: ActorRef, - callCacheReadActor: ActorRef, - jobTokenDispenserActor: ActorRef, - backendSingletonCollection: BackendSingletonCollection) = this( - ConfigFactory.load, workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection) + private val config = params.config private val maxWorkflowsRunning = config.getConfig("system").as[Option[Int]]("max-concurrent-workflows").getOrElse(DefaultMaxWorkflowsToRun) private val maxWorkflowsToLaunch = config.getConfig("system").as[Option[Int]]("max-workflow-launch-count").getOrElse(DefaultMaxWorkflowsToLaunch) @@ -105,6 +104,7 @@ class WorkflowManagerActor(config: Config, private val tag = self.path.name private var abortingWorkflowToReplyTo = Map.empty[WorkflowId, ActorRef] + private var shutdownHookThreadOption: Option[ShutdownHookThread] = None override def preStart(): Unit = { addShutdownHook() @@ -112,13 +112,19 @@ class WorkflowManagerActor(config: Config, self ! RetrieveNewWorkflows } + override def postStop() = { + // If the actor is stopping, especially during error tests, then there's nothing to wait for later at JVM shutdown. + tryRemoveShutdownHook() + super.postStop() + } + private def addShutdownHook() = { // Only abort jobs on SIGINT if the config explicitly sets system.abort-jobs-on-terminate = true. val abortJobsOnTerminate = - config.getConfig("system").as[Option[Boolean]]("abort-jobs-on-terminate").getOrElse(false) + config.getConfig("system").as[Option[Boolean]]("abort-jobs-on-terminate").getOrElse(params.abortJobsOnTerminate) if (abortJobsOnTerminate) { - sys.addShutdownHook { + val shutdownHookThread = sys.addShutdownHook { logger.info(s"$tag: Received shutdown signal.") self ! AbortAllWorkflowsCommand while (stateData != null && stateData.workflows.nonEmpty) { @@ -126,7 +132,18 @@ class WorkflowManagerActor(config: Config, Thread.sleep(1000) } } + shutdownHookThreadOption = Option(shutdownHookThread) + } + } + + private def tryRemoveShutdownHook() = { + try { + shutdownHookThreadOption.foreach(_.remove()) + } catch { + case _: IllegalStateException => /* ignore, we're probably shutting down */ + case exception: Exception => log.error(exception, "Error while removing shutdown hook: {}", exception.getMessage) } + shutdownHookThreadOption = None } startWith(Running, WorkflowManagerData(workflows = Map.empty)) @@ -141,7 +158,7 @@ class WorkflowManagerActor(config: Config, Determine the number of available workflow slots and request the smaller of that number of maxWorkflowsToLaunch. */ val maxNewWorkflows = maxWorkflowsToLaunch min (maxWorkflowsRunning - stateData.workflows.size) - workflowStore ! WorkflowStoreActor.FetchRunnableWorkflows(maxNewWorkflows) + params.workflowStore ! WorkflowStoreActor.FetchRunnableWorkflows(maxNewWorkflows) stay() case Event(WorkflowStoreActor.NoNewWorkflowsToStart, stateData) => log.debug("WorkflowStore provided no new workflows to start") @@ -188,13 +205,13 @@ class WorkflowManagerActor(config: Config, log.info(s"$tag ${workflowActor.path.name} is in a terminal state: $toState") // This silently fails if idFromActor is None, but data.without call right below will as well data.idFromActor(workflowActor) foreach { workflowId => - jobStoreActor ! RegisterWorkflowCompleted(workflowId) + params.jobStoreActor ! RegisterWorkflowCompleted(workflowId) if (toState.workflowState == WorkflowAborted) { val replyTo = abortingWorkflowToReplyTo(workflowId) replyTo ! WorkflowStoreActor.WorkflowAborted(workflowId) abortingWorkflowToReplyTo -= workflowId } else { - workflowStore ! WorkflowStoreActor.RemoveWorkflow(workflowId) + params.workflowStore ! WorkflowStoreActor.RemoveWorkflow(workflowId) } } stay using data.without(workflowActor) @@ -264,8 +281,9 @@ class WorkflowManagerActor(config: Config, StartNewWorkflow } - val wfProps = WorkflowActor.props(workflowId, startMode, workflow.sources, config, serviceRegistryActor, - workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection) + val wfProps = WorkflowActor.props(workflowId, startMode, workflow.sources, config, params.serviceRegistryActor, + params.workflowLogCopyRouter, params.jobStoreActor, params.callCacheReadActor, params.jobTokenDispenserActor, + params.backendSingletonCollection) val wfActor = context.actorOf(wfProps, name = s"WorkflowActor-$workflowId") wfActor ! SubscribeTransitionCallBack(self) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index 201ae99a4..a30dc7500 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -6,7 +6,7 @@ import akka.actor.SupervisorStrategy.{Escalate, Stop} import akka.actor._ import cats.data.NonEmptyList import com.typesafe.config.ConfigFactory -import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, FailedNonRetryableResponse, FailedRetryableResponse, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor._ import cromwell.backend.BackendLifecycleActor.AbortJobCommand import cromwell.backend.{AllBackendInitializationData, BackendJobDescriptor, BackendJobDescriptorKey} import cromwell.core.Dispatcher.EngineDispatcher @@ -390,11 +390,12 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, } when(WorkflowExecutionAbortingState) { - case Event(AbortedResponse(jobKey), stateData) => - workflowLogger.info(s"$tag job aborted: ${jobKey.tag}") + case Event(response: BackendJobExecutionResponse, stateData) => + val jobKey = response.jobKey + workflowLogger.info(s"$tag job exited with ${response.getClass.getSimpleName}: ${jobKey.tag}") val newStateData = stateData.removeBackendJobExecutionActor(jobKey) if (newStateData.backendJobExecutionActors.isEmpty) { - workflowLogger.info(s"$tag all jobs aborted") + workflowLogger.info(s"$tag all jobs exited") goto(WorkflowExecutionAbortedState) } else { stay() using newStateData diff --git a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala index cc0f5f4aa..2e3d4a315 100644 --- a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala +++ b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala @@ -56,9 +56,12 @@ import net.ceedubs.ficus.Ficus._ lazy val jobExecutionTokenDispenserActor = context.actorOf(JobExecutionTokenDispenserActor.props) + def abortJobsOnTerminate: Boolean + lazy val workflowManagerActor = context.actorOf( WorkflowManagerActor.props( - workflowStoreActor, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobExecutionTokenDispenserActor, backendSingletonCollection), + workflowStoreActor, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, + jobExecutionTokenDispenserActor, backendSingletonCollection, abortJobsOnTerminate), "WorkflowManagerActor") override def receive = { diff --git a/engine/src/main/scala/cromwell/server/CromwellServer.scala b/engine/src/main/scala/cromwell/server/CromwellServer.scala index dca31a625..5cefe87c5 100644 --- a/engine/src/main/scala/cromwell/server/CromwellServer.scala +++ b/engine/src/main/scala/cromwell/server/CromwellServer.scala @@ -53,6 +53,8 @@ object CromwellServer { class CromwellServerActor(config: Config) extends CromwellRootActor with CromwellApiService with SwaggerService { implicit def executionContext = actorRefFactory.dispatcher + override val abortJobsOnTerminate = false + override def actorRefFactory = context override def receive = handleTimeouts orElse runRoute(possibleRoutes) diff --git a/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala b/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala index 8dba0882a..c88d48979 100644 --- a/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala +++ b/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala @@ -2,6 +2,7 @@ package cromwell import java.nio.file.Paths import java.util.UUID +import java.util.concurrent.atomic.AtomicInteger import akka.actor.{Actor, ActorRef, ActorSystem, Props, Terminated} import akka.pattern.ask @@ -121,8 +122,10 @@ object CromwellTestkitSpec { val TimeoutDuration = 60 seconds + private val testWorkflowManagerSystemCount = new AtomicInteger() + class TestWorkflowManagerSystem extends CromwellSystem { - override protected def systemName: String = "test-system" + override protected def systemName: String = "test-system-" + testWorkflowManagerSystemCount.incrementAndGet() override protected def newActorSystem() = ActorSystem(systemName, ConfigFactory.parseString(CromwellTestkitSpec.ConfigText)) /** * Do NOT shut down the test actor system inside the normal flow. @@ -135,18 +138,6 @@ object CromwellTestkitSpec { } /** - * Loans a test actor system. NOTE: This should be run OUTSIDE of a wait block, never within one. - */ - def withTestWorkflowManagerSystem[T](block: CromwellSystem => T): T = { - val testWorkflowManagerSystem = new CromwellTestkitSpec.TestWorkflowManagerSystem - try { - block(testWorkflowManagerSystem) - } finally { - TestKit.shutdownActorSystem(testWorkflowManagerSystem.actorSystem, TimeoutDuration) - } - } - - /** * Wait for exactly one occurrence of the specified info pattern in the specified block. The block is in its own * parameter list for usage syntax reasons. */ @@ -266,6 +257,7 @@ object CromwellTestkitSpec { class TestCromwellRootActor(config: Config) extends CromwellRootActor { override lazy val serviceRegistryActor = ServiceRegistryActorInstance override lazy val workflowStore = new InMemoryWorkflowStore + override val abortJobsOnTerminate = false def submitWorkflow(sources: WorkflowSourceFiles): WorkflowId = { val submitMessage = WorkflowStoreActor.SubmitWorkflow(sources) val result = Await.result(workflowStoreActor.ask(submitMessage)(TimeoutDuration), Duration.Inf).asInstanceOf[WorkflowSubmittedToStore].workflowId diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala index 983c188df..16b427642 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala @@ -61,14 +61,16 @@ abstract class SingleWorkflowRunnerActorSpec extends CromwellTestkitSpec { def workflowManagerActor(): ActorRef = { - system.actorOf(Props(new WorkflowManagerActor(ConfigFactory.load(), + val params = WorkflowManagerActorParams(ConfigFactory.load(), workflowStore, dummyServiceRegistryActor, dummyLogCopyRouter, jobStore, callCacheReadActor, jobTokenDispenserActor, - BackendSingletonCollection(Map.empty))), "WorkflowManagerActor") + BackendSingletonCollection(Map.empty), + abortJobsOnTerminate = false) + system.actorOf(Props(new WorkflowManagerActor(params)), "WorkflowManagerActor") } def createRunnerActor(sampleWdl: SampleWdl = ThreeStep, managerActor: => ActorRef = workflowManagerActor(), diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala index a4feebb01..6da9bb2d1 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala @@ -1,23 +1,23 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching import akka.actor.{ActorRef, ActorSystem, Props} -import akka.testkit.{ImplicitSender, TestKit, TestProbe} +import akka.testkit.{ImplicitSender, TestProbe} import cats.data.NonEmptyList import cromwell.CromwellTestkitSpec -import cromwell.backend.callcaching.FileHashingActor.{FileHashResponse, SingleFileHashRequest} import cromwell.backend._ +import cromwell.backend.callcaching.FileHashingActor.{FileHashResponse, SingleFileHashRequest} import cromwell.core.callcaching._ import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CacheMiss, CallCacheHashes} import org.scalatest.mockito.MockitoSugar -import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} +import org.scalatest.{Matchers, WordSpecLike} import wdl4s._ import wdl4s.values.{WdlFile, WdlValue} import scala.concurrent.duration._ import scala.language.postfixOps -class EngineJobHashingActorSpec extends TestKit(new CromwellTestkitSpec.TestWorkflowManagerSystem().actorSystem) - with ImplicitSender with WordSpecLike with Matchers with MockitoSugar with BeforeAndAfterAll { +class EngineJobHashingActorSpec extends CromwellTestkitSpec + with ImplicitSender with WordSpecLike with Matchers with MockitoSugar { import EngineJobHashingActorSpec._ @@ -162,10 +162,6 @@ class EngineJobHashingActorSpec extends TestKit(new CromwellTestkitSpec.TestWork } } } - - override def afterAll() = { - TestKit.shutdownActorSystem(system) - } } object EngineJobHashingActorSpec extends BackendSpec { From 1213274a50c45bf0c54cc193390d7c3b98f0a849 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Thu, 10 Nov 2016 14:44:45 -0500 Subject: [PATCH 060/375] Fixed typo in `CromwellTestKitSpec`. --- .../test/scala/cromwell/ArrayOfArrayCoercionSpec.scala | 2 +- engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala | 2 +- .../test/scala/cromwell/CallCachingWorkflowSpec.scala | 2 +- .../test/scala/cromwell/CopyWorkflowOutputsSpec.scala | 6 +++--- ...wellTestkitSpec.scala => CromwellTestKitSpec.scala} | 8 ++++---- .../test/scala/cromwell/FilePassingWorkflowSpec.scala | 2 +- engine/src/test/scala/cromwell/MapWorkflowSpec.scala | 2 +- .../MultipleFilesWithSameNameWorkflowSpec.scala | 2 +- .../scala/cromwell/PostfixQuantifierWorkflowSpec.scala | 2 +- .../src/test/scala/cromwell/RestartWorkflowSpec.scala | 4 ++-- .../src/test/scala/cromwell/ScatterWorkflowSpec.scala | 2 +- .../test/scala/cromwell/SimpleWorkflowActorSpec.scala | 4 ++-- .../cromwell/WdlFunctionsAtWorkflowLevelSpec.scala | 2 +- .../src/test/scala/cromwell/WorkflowFailSlowSpec.scala | 2 +- .../src/test/scala/cromwell/WorkflowOutputsSpec.scala | 4 ++-- .../test/scala/cromwell/engine/WorkflowAbortSpec.scala | 4 ++-- .../cromwell/engine/WorkflowManagerActorSpec.scala | 4 ++-- .../scala/cromwell/engine/WorkflowStoreActorSpec.scala | 18 +++++++++--------- .../workflow/SingleWorkflowRunnerActorSpec.scala | 8 ++++---- .../cromwell/engine/workflow/WorkflowActorSpec.scala | 8 ++++---- .../engine/workflow/WorkflowDescriptorBuilder.scala | 4 ++-- .../MaterializeWorkflowDescriptorActorSpec.scala | 4 ++-- .../execution/WorkflowExecutionActorSpec.scala | 4 ++-- .../callcaching/EngineJobHashingActorSpec.scala | 4 ++-- .../execution/ejea/EngineJobExecutionActorSpec.scala | 4 ++-- .../scala/cromwell/jobstore/JobStoreServiceSpec.scala | 4 ++-- .../scala/cromwell/jobstore/JobStoreWriterSpec.scala | 4 ++-- .../cromwell/webservice/CromwellApiServiceSpec.scala | 4 ++-- 28 files changed, 60 insertions(+), 60 deletions(-) rename engine/src/test/scala/cromwell/{CromwellTestkitSpec.scala => CromwellTestKitSpec.scala} (98%) diff --git a/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala b/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala index 00a530f31..e0315d92f 100644 --- a/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala +++ b/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala @@ -6,7 +6,7 @@ import wdl4s.values.{WdlArray, WdlString} import cromwell.util.SampleWdl -class ArrayOfArrayCoercionSpec extends CromwellTestkitSpec { +class ArrayOfArrayCoercionSpec extends CromwellTestKitSpec { "A workflow that has an Array[Array[File]] input " should { "accept an Array[Array[String]] as the value for the input" in { runWdlAndAssertOutputs( diff --git a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala index 44393924f..ad6cef3d9 100644 --- a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala @@ -11,7 +11,7 @@ import wdl4s.types.{WdlArrayType, WdlFileType, WdlStringType} import wdl4s.values.{WdlArray, WdlFile, WdlInteger, WdlString} -class ArrayWorkflowSpec extends CromwellTestkitSpec { +class ArrayWorkflowSpec extends CromwellTestKitSpec { val tmpDir = Files.createTempDirectory("ArrayWorkflowSpec") val ns = WdlNamespaceWithWorkflow.load(SampleWdl.ArrayLiteral(tmpDir).wdlSource("")) val expectedArray = WdlArray(WdlArrayType(WdlFileType), Seq(WdlFile("f1"), WdlFile("f2"), WdlFile("f3"))) diff --git a/engine/src/test/scala/cromwell/CallCachingWorkflowSpec.scala b/engine/src/test/scala/cromwell/CallCachingWorkflowSpec.scala index cea47fe8d..202dcecf1 100644 --- a/engine/src/test/scala/cromwell/CallCachingWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/CallCachingWorkflowSpec.scala @@ -11,7 +11,7 @@ import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType} import wdl4s.values.{WdlArray, WdlFile, WdlInteger, WdlString} -class CallCachingWorkflowSpec extends CromwellTestkitSpec { +class CallCachingWorkflowSpec extends CromwellTestKitSpec { def cacheHitMessageForCall(name: String) = s"Call Caching: Cache hit. Using UUID\\(.{8}\\):$name\\.*" val expectedOutputs = Map( diff --git a/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala b/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala index c43346330..d8d910913 100644 --- a/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala +++ b/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala @@ -9,7 +9,7 @@ import org.scalatest.prop.Tables.Table import scala.language.postfixOps -class CopyWorkflowOutputsSpec extends CromwellTestkitSpec { +class CopyWorkflowOutputsSpec extends CromwellTestKitSpec { "CopyWorkflowOutputsCall" should { "copy workflow outputs" in { @@ -31,7 +31,7 @@ class CopyWorkflowOutputsSpec extends CromwellTestkitSpec { pattern = "transition from FinalizingWorkflowState to WorkflowSucceededState", occurrences = 1), runtime = "", workflowOptions = s""" { "final_workflow_outputs_dir": "$tmpDir" } """, - expectedOutputs = Seq("A.out", "A.out2", "B.outs") map { o => ("wfoutputs." + o) -> CromwellTestkitSpec.AnyValueIsFine } toMap, + expectedOutputs = Seq("A.out", "A.out2", "B.outs") map { o => ("wfoutputs." + o) -> CromwellTestKitSpec.AnyValueIsFine } toMap, allowOtherOutputs = false ) @@ -64,7 +64,7 @@ class CopyWorkflowOutputsSpec extends CromwellTestkitSpec { pattern = "transition from FinalizingWorkflowState to WorkflowSucceededState", occurrences = 1), runtime = "", workflowOptions = s""" { "final_workflow_outputs_dir": "$tmpDir" } """, - expectedOutputs = Map("wfoutputs.A.outs" -> CromwellTestkitSpec.AnyValueIsFine), + expectedOutputs = Map("wfoutputs.A.outs" -> CromwellTestKitSpec.AnyValueIsFine), allowOtherOutputs = false ) diff --git a/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala similarity index 98% rename from engine/src/test/scala/cromwell/CromwellTestkitSpec.scala rename to engine/src/test/scala/cromwell/CromwellTestKitSpec.scala index c88d48979..3c0ef26e7 100644 --- a/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala +++ b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala @@ -8,7 +8,7 @@ import akka.actor.{Actor, ActorRef, ActorSystem, Props, Terminated} import akka.pattern.ask import akka.testkit._ import com.typesafe.config.{Config, ConfigFactory} -import cromwell.CromwellTestkitSpec._ +import cromwell.CromwellTestKitSpec._ import cromwell.backend._ import cromwell.core._ import cromwell.engine.backend.BackendConfigurationEntry @@ -64,7 +64,7 @@ case class TestBackendLifecycleActorFactory(configurationDescriptor: BackendConf case class OutputNotFoundException(outputFqn: String, actualOutputs: String) extends RuntimeException(s"Expected output $outputFqn was not found in: '$actualOutputs'") case class LogNotFoundException(log: String) extends RuntimeException(s"Expected log $log was not found") -object CromwellTestkitSpec { +object CromwellTestKitSpec { val ConfigText = """ |akka { @@ -126,7 +126,7 @@ object CromwellTestkitSpec { class TestWorkflowManagerSystem extends CromwellSystem { override protected def systemName: String = "test-system-" + testWorkflowManagerSystemCount.incrementAndGet() - override protected def newActorSystem() = ActorSystem(systemName, ConfigFactory.parseString(CromwellTestkitSpec.ConfigText)) + override protected def newActorSystem() = ActorSystem(systemName, ConfigFactory.parseString(CromwellTestKitSpec.ConfigText)) /** * Do NOT shut down the test actor system inside the normal flow. * The actor system will be externally shutdown outside the block. @@ -267,7 +267,7 @@ object CromwellTestkitSpec { } } -abstract class CromwellTestkitSpec(val twms: TestWorkflowManagerSystem = new CromwellTestkitSpec.TestWorkflowManagerSystem()) extends TestKit(twms.actorSystem) +abstract class CromwellTestKitSpec(val twms: TestWorkflowManagerSystem = new CromwellTestKitSpec.TestWorkflowManagerSystem()) extends TestKit(twms.actorSystem) with DefaultTimeout with ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll with ScalaFutures with OneInstancePerTest with Eventually { override protected def afterAll() = { twms.shutdownTestActorSystem(); () } diff --git a/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala b/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala index 1aaafaf2f..d6d059467 100644 --- a/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala @@ -6,7 +6,7 @@ import wdl4s.values.{WdlFile, WdlString} import scala.concurrent.duration._ -class FilePassingWorkflowSpec extends CromwellTestkitSpec { +class FilePassingWorkflowSpec extends CromwellTestKitSpec { "A workflow that passes files between tasks" should { "pass files properly" in { runWdlAndAssertOutputs( diff --git a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala index 03ec39426..ee6b088c7 100644 --- a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala @@ -10,7 +10,7 @@ import wdl4s.values._ import scala.util.{Success, Try} -class MapWorkflowSpec extends CromwellTestkitSpec { +class MapWorkflowSpec extends CromwellTestKitSpec { private val pwd = File(".") private val sampleWdl = SampleWdl.MapLiteral(pwd.path) val ns = WdlNamespaceWithWorkflow.load(sampleWdl.wdlSource("")) diff --git a/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala b/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala index f0b7a70af..aa05c08f6 100644 --- a/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala @@ -5,7 +5,7 @@ import cromwell.util.SampleWdl import wdl4s.values.WdlString -class MultipleFilesWithSameNameWorkflowSpec extends CromwellTestkitSpec { +class MultipleFilesWithSameNameWorkflowSpec extends CromwellTestKitSpec { "A workflow with two file inputs that have the same name" should { "not clobber one file with the contents of another" in { runWdlAndAssertOutputs( diff --git a/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala b/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala index 8530dd6d2..a436a6287 100644 --- a/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala @@ -5,7 +5,7 @@ import wdl4s.values.WdlString import cromwell.util.SampleWdl -class PostfixQuantifierWorkflowSpec extends CromwellTestkitSpec { +class PostfixQuantifierWorkflowSpec extends CromwellTestKitSpec { "A task which contains a parameter with a zero-or-more postfix quantifier" should { "accept an array of size 3" in { runWdlAndAssertOutputs( diff --git a/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala b/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala index 6b706fbc6..18949dc80 100644 --- a/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala @@ -6,9 +6,9 @@ import cromwell.core.Tags._ import cromwell.core._ import cromwell.engine.workflow.WorkflowDescriptorBuilder -class RestartWorkflowSpec extends CromwellTestkitSpec with WorkflowDescriptorBuilder { +class RestartWorkflowSpec extends CromwellTestKitSpec with WorkflowDescriptorBuilder { - val actorSystem = ActorSystem("RestartWorkflowSpec", ConfigFactory.parseString(CromwellTestkitSpec.ConfigText)) + val actorSystem = ActorSystem("RestartWorkflowSpec", ConfigFactory.parseString(CromwellTestKitSpec.ConfigText)) //val localBackend = new OldStyleLocalBackend(CromwellTestkitSpec.DefaultLocalBackendConfigEntry, actorSystem) val sources = WorkflowSourceFiles( wdlSource="""task a {command{}} diff --git a/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala b/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala index 0d8847a27..347c21923 100644 --- a/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala @@ -6,7 +6,7 @@ import wdl4s.types.{WdlArrayType, WdlFileType, WdlIntegerType, WdlStringType} import wdl4s.values.{WdlArray, WdlFile, WdlInteger, WdlString} import cromwell.util.SampleWdl -class ScatterWorkflowSpec extends CromwellTestkitSpec { +class ScatterWorkflowSpec extends CromwellTestKitSpec { "A workflow with a stand-alone scatter block in it" should { "run properly" in { runWdlAndAssertOutputs( diff --git a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala index d2d9929f7..a636e3f46 100644 --- a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala @@ -28,7 +28,7 @@ object SimpleWorkflowActorSpec { promise: Promise[Unit]) } -class SimpleWorkflowActorSpec extends CromwellTestkitSpec with BeforeAndAfter { +class SimpleWorkflowActorSpec extends CromwellTestKitSpec with BeforeAndAfter { private def buildWorkflowActor(sampleWdl: SampleWdl, rawInputsOverride: String, @@ -148,7 +148,7 @@ class SimpleWorkflowActorSpec extends CromwellTestkitSpec with BeforeAndAfter { } private def startingCallsFilter[T](callNames: String*)(block: => T): T = { - import CromwellTestkitSpec.waitForInfo + import CromwellTestKitSpec.waitForInfo within(TestExecutionTimeout) { waitForInfo(s"Starting calls: ${callNames.mkString("", ":NA:1, ", ":NA:1")}$$", 1) { block diff --git a/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala b/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala index 72c618fca..186bd0714 100644 --- a/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala +++ b/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala @@ -6,7 +6,7 @@ import wdl4s.types.{WdlMapType, WdlStringType} import wdl4s.values.{WdlMap, WdlString} -class WdlFunctionsAtWorkflowLevelSpec extends CromwellTestkitSpec { +class WdlFunctionsAtWorkflowLevelSpec extends CromwellTestKitSpec { val outputMap = WdlMap(WdlMapType(WdlStringType, WdlStringType), Map( WdlString("k1") -> WdlString("v1"), WdlString("k2") -> WdlString("v2"), diff --git a/engine/src/test/scala/cromwell/WorkflowFailSlowSpec.scala b/engine/src/test/scala/cromwell/WorkflowFailSlowSpec.scala index 1cd7a7ef3..2093cf198 100644 --- a/engine/src/test/scala/cromwell/WorkflowFailSlowSpec.scala +++ b/engine/src/test/scala/cromwell/WorkflowFailSlowSpec.scala @@ -5,7 +5,7 @@ import cromwell.util.SampleWdl // TODO: These tests are (and were) somewhat unsatisfactory. They'd be much better if we use TestFSMRefs and TestProbes to simulate job completions against the WorkflowActor and make sure it only completes the workflow at the appropriate time. -class WorkflowFailSlowSpec extends CromwellTestkitSpec { +class WorkflowFailSlowSpec extends CromwellTestKitSpec { val FailFastOptions = """ |{ diff --git a/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala b/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala index 18df31795..b821dd742 100644 --- a/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala +++ b/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala @@ -2,10 +2,10 @@ package cromwell import akka.testkit._ import cromwell.util.SampleWdl -import cromwell.CromwellTestkitSpec.AnyValueIsFine +import cromwell.CromwellTestKitSpec.AnyValueIsFine -class WorkflowOutputsSpec extends CromwellTestkitSpec { +class WorkflowOutputsSpec extends CromwellTestKitSpec { "Workflow outputs" should { "use all outputs if none are specified" in { runWdlAndAssertOutputs( diff --git a/engine/src/test/scala/cromwell/engine/WorkflowAbortSpec.scala b/engine/src/test/scala/cromwell/engine/WorkflowAbortSpec.scala index 6a4b0a077..37ed641c8 100644 --- a/engine/src/test/scala/cromwell/engine/WorkflowAbortSpec.scala +++ b/engine/src/test/scala/cromwell/engine/WorkflowAbortSpec.scala @@ -1,8 +1,8 @@ package cromwell.engine -import cromwell.CromwellTestkitSpec +import cromwell.CromwellTestKitSpec -class WorkflowAbortSpec extends CromwellTestkitSpec { +class WorkflowAbortSpec extends CromwellTestKitSpec { // TODO: When re-enabled, this test also needs to check that child processes have actually been stopped. "A WorkflowManagerActor" should { diff --git a/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala b/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala index 1d8fbb2db..b22ebc13f 100644 --- a/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala @@ -1,11 +1,11 @@ package cromwell.engine -import cromwell.CromwellTestkitSpec +import cromwell.CromwellTestKitSpec import cromwell.engine.workflow.WorkflowDescriptorBuilder import cromwell.util.SampleWdl -class WorkflowManagerActorSpec extends CromwellTestkitSpec with WorkflowDescriptorBuilder { +class WorkflowManagerActorSpec extends CromwellTestKitSpec with WorkflowDescriptorBuilder { override implicit val actorSystem = system "A WorkflowManagerActor" should { diff --git a/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala b/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala index 8b3d0c5e5..d8bd649c8 100644 --- a/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala @@ -1,7 +1,7 @@ package cromwell.engine import cats.data.NonEmptyList -import cromwell.CromwellTestkitSpec +import cromwell.CromwellTestKitSpec import cromwell.core.{WorkflowId, WorkflowSourceFiles} import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._ import cromwell.engine.workflow.workflowstore._ @@ -15,7 +15,7 @@ import org.scalatest.Matchers import scala.concurrent.duration._ import scala.language.postfixOps -class WorkflowStoreActorSpec extends CromwellTestkitSpec with Matchers { +class WorkflowStoreActorSpec extends CromwellTestKitSpec with Matchers { val helloWorldSourceFiles = HelloWorld.asWorkflowSources() /** @@ -43,14 +43,14 @@ class WorkflowStoreActorSpec extends CromwellTestkitSpec with Matchers { "The WorkflowStoreActor" should { "return an ID for a submitted workflow" in { val store = new InMemoryWorkflowStore - val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestkitSpec.ServiceRegistryActorInstance)) + val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance)) storeActor ! SubmitWorkflow(helloWorldSourceFiles) expectMsgType[WorkflowSubmittedToStore](10 seconds) } "return 3 IDs for a batch submission of 3" in { val store = new InMemoryWorkflowStore - val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestkitSpec.ServiceRegistryActorInstance)) + val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance)) storeActor ! BatchSubmitWorkflows(NonEmptyList.of(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles)) expectMsgPF(10 seconds) { case WorkflowsBatchSubmittedToStore(ids) => ids.toList.size shouldBe 3 @@ -59,7 +59,7 @@ class WorkflowStoreActorSpec extends CromwellTestkitSpec with Matchers { "fetch exactly N workflows" in { val store = new InMemoryWorkflowStore - val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestkitSpec.ServiceRegistryActorInstance)) + val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance)) storeActor ! BatchSubmitWorkflows(NonEmptyList.of(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles)) val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.toList @@ -90,7 +90,7 @@ class WorkflowStoreActorSpec extends CromwellTestkitSpec with Matchers { val store = new InMemoryWorkflowStore - val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestkitSpec.ServiceRegistryActorInstance)) + val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance)) val readMetadataActor = system.actorOf(ReadMetadataActor.props()) storeActor ! BatchSubmitWorkflows(NonEmptyList.of(optionedSourceFiles)) val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.toList @@ -130,7 +130,7 @@ class WorkflowStoreActorSpec extends CromwellTestkitSpec with Matchers { "return only the remaining workflows if N is larger than size" in { val store = new InMemoryWorkflowStore - val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestkitSpec.ServiceRegistryActorInstance)) + val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance)) storeActor ! BatchSubmitWorkflows(NonEmptyList.of(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles)) val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.toList @@ -151,7 +151,7 @@ class WorkflowStoreActorSpec extends CromwellTestkitSpec with Matchers { "remove workflows which exist" in { val store = new InMemoryWorkflowStore - val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestkitSpec.ServiceRegistryActorInstance)) + val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance)) storeActor ! SubmitWorkflow(helloWorldSourceFiles) val id = expectMsgType[WorkflowSubmittedToStore](10 seconds).workflowId storeActor ! RemoveWorkflow(id) @@ -164,7 +164,7 @@ class WorkflowStoreActorSpec extends CromwellTestkitSpec with Matchers { "remain responsive if you ask to remove a workflow it doesn't have" in { val store = new InMemoryWorkflowStore - val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestkitSpec.ServiceRegistryActorInstance)) + val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance)) val id = WorkflowId.randomId() storeActor ! RemoveWorkflow(id) diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala index 16b427642..6753d3d2d 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala @@ -9,7 +9,7 @@ import akka.testkit.TestKit import akka.util.Timeout import better.files._ import com.typesafe.config.ConfigFactory -import cromwell.CromwellTestkitSpec._ +import cromwell.CromwellTestKitSpec._ import cromwell.core.WorkflowSourceFiles import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.SingleWorkflowRunnerActor.RunWorkflow @@ -18,7 +18,7 @@ import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreActor} import cromwell.util.SampleWdl import cromwell.util.SampleWdl.{ExpressionsInInputs, GoodbyeWorld, ThreeStep} -import cromwell.{AlwaysHappyJobStoreActor, CromwellTestkitSpec, EmptyCallCacheReadActor} +import cromwell.{AlwaysHappyJobStoreActor, CromwellTestKitSpec, EmptyCallCacheReadActor} import org.scalatest.prop.{TableDrivenPropertyChecks, TableFor3} import spray.json._ @@ -49,11 +49,11 @@ object SingleWorkflowRunnerActorSpec { class TestSingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: Option[Path]) extends SingleWorkflowRunnerActor(source, metadataOutputPath) { - override lazy val serviceRegistryActor = CromwellTestkitSpec.ServiceRegistryActorInstance + override lazy val serviceRegistryActor = CromwellTestKitSpec.ServiceRegistryActorInstance } } -abstract class SingleWorkflowRunnerActorSpec extends CromwellTestkitSpec { +abstract class SingleWorkflowRunnerActorSpec extends CromwellTestKitSpec { private val workflowStore = system.actorOf(WorkflowStoreActor.props(new InMemoryWorkflowStore, dummyServiceRegistryActor)) private val jobStore = system.actorOf(AlwaysHappyJobStoreActor.props) private val callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props) diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala index b98c5657f..19f1fd06d 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala @@ -13,13 +13,13 @@ import cromwell.engine.workflow.lifecycle.WorkflowFinalizationActor.{StartFinali import cromwell.engine.workflow.lifecycle.WorkflowInitializationActor.{WorkflowInitializationAbortedResponse, WorkflowInitializationFailedResponse} import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{WorkflowExecutionAbortedResponse, WorkflowExecutionFailedResponse, WorkflowExecutionSucceededResponse} import cromwell.util.SampleWdl.ThreeStep -import cromwell.{AlwaysHappyJobStoreActor, CromwellTestkitSpec, EmptyCallCacheReadActor} +import cromwell.{AlwaysHappyJobStoreActor, CromwellTestKitSpec, EmptyCallCacheReadActor} import org.scalatest.BeforeAndAfter import org.scalatest.concurrent.Eventually import scala.concurrent.duration._ -class WorkflowActorSpec extends CromwellTestkitSpec with WorkflowDescriptorBuilder with BeforeAndAfter with Eventually { +class WorkflowActorSpec extends CromwellTestKitSpec with WorkflowDescriptorBuilder with BeforeAndAfter with Eventually { override implicit val actorSystem = system val mockServiceRegistryActor = TestActorRef(new Actor { @@ -62,7 +62,7 @@ class WorkflowActorSpec extends CromwellTestkitSpec with WorkflowDescriptorBuild actor } - implicit val TimeoutDuration = CromwellTestkitSpec.TimeoutDuration + implicit val TimeoutDuration = CromwellTestKitSpec.TimeoutDuration "WorkflowActor" should { @@ -108,7 +108,7 @@ class WorkflowActorSpec extends CromwellTestkitSpec with WorkflowDescriptorBuild deathwatch watch actor actor ! AbortWorkflowCommand eventually { actor.stateName should be(WorkflowAbortingState) } - currentLifecycleActor.expectMsgPF(CromwellTestkitSpec.TimeoutDuration) { + currentLifecycleActor.expectMsgPF(CromwellTestKitSpec.TimeoutDuration) { case EngineLifecycleActorAbortCommand => actor ! WorkflowExecutionAbortedResponse(ExecutionStore.empty, OutputStore.empty) } diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowDescriptorBuilder.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowDescriptorBuilder.scala index 73013e712..a6b02620d 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/WorkflowDescriptorBuilder.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowDescriptorBuilder.scala @@ -2,7 +2,7 @@ package cromwell.engine.workflow import akka.actor.{ActorSystem, Props} import com.typesafe.config.ConfigFactory -import cromwell.CromwellTestkitSpec +import cromwell.CromwellTestKitSpec import cromwell.core.{WorkflowId, WorkflowSourceFiles} import cromwell.engine.EngineWorkflowDescriptor import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor @@ -12,7 +12,7 @@ import scala.concurrent.Await trait WorkflowDescriptorBuilder { - implicit val awaitTimeout = CromwellTestkitSpec.TimeoutDuration + implicit val awaitTimeout = CromwellTestKitSpec.TimeoutDuration implicit val actorSystem: ActorSystem def createMaterializedEngineWorkflowDescriptor(id: WorkflowId, workflowSources: WorkflowSourceFiles): EngineWorkflowDescriptor = { diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala index 057fa5456..bdaca25a7 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala @@ -3,7 +3,7 @@ package cromwell.engine.workflow.lifecycle import akka.actor.Props import akka.testkit.TestDuration import com.typesafe.config.ConfigFactory -import cromwell.CromwellTestkitSpec +import cromwell.CromwellTestKitSpec import cromwell.core.{WorkflowId, WorkflowOptions, WorkflowSourceFiles} import cromwell.engine.backend.{BackendConfigurationEntry, CromwellBackends} import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.{MaterializeWorkflowDescriptorCommand, MaterializeWorkflowDescriptorFailureResponse, MaterializeWorkflowDescriptorSuccessResponse} @@ -16,7 +16,7 @@ import wdl4s.values.{WdlInteger, WdlString} import scala.concurrent.duration._ -class MaterializeWorkflowDescriptorActorSpec extends CromwellTestkitSpec with BeforeAndAfter with MockitoSugar { +class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with BeforeAndAfter with MockitoSugar { val workflowId = WorkflowId.randomId() val minimumConf = ConfigFactory.parseString( diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala index f2136a69f..cc9978e19 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala @@ -12,13 +12,13 @@ import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor import cromwell.services.ServiceRegistryActor import cromwell.services.metadata.MetadataService import cromwell.util.SampleWdl -import cromwell.{AlwaysHappyJobStoreActor, CromwellTestkitSpec, EmptyCallCacheReadActor, MetadataWatchActor} +import cromwell.{AlwaysHappyJobStoreActor, CromwellTestKitSpec, EmptyCallCacheReadActor, MetadataWatchActor} import org.scalatest.BeforeAndAfter import scala.concurrent.{Await, Promise} import scala.concurrent.duration._ -class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter with WorkflowDescriptorBuilder { +class WorkflowExecutionActorSpec extends CromwellTestKitSpec with BeforeAndAfter with WorkflowDescriptorBuilder { override implicit val actorSystem = system implicit val DefaultDuration = 20.seconds.dilated diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala index 6da9bb2d1..e69085c9e 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala @@ -3,7 +3,7 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching import akka.actor.{ActorRef, ActorSystem, Props} import akka.testkit.{ImplicitSender, TestProbe} import cats.data.NonEmptyList -import cromwell.CromwellTestkitSpec +import cromwell.CromwellTestKitSpec import cromwell.backend._ import cromwell.backend.callcaching.FileHashingActor.{FileHashResponse, SingleFileHashRequest} import cromwell.core.callcaching._ @@ -16,7 +16,7 @@ import wdl4s.values.{WdlFile, WdlValue} import scala.concurrent.duration._ import scala.language.postfixOps -class EngineJobHashingActorSpec extends CromwellTestkitSpec +class EngineJobHashingActorSpec extends CromwellTestKitSpec with ImplicitSender with WordSpecLike with Matchers with MockitoSugar { import EngineJobHashingActorSpec._ diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala index cbba49811..105776781 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala @@ -4,7 +4,7 @@ import akka.actor.Actor import akka.testkit.TestFSMRef import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ import cromwell.jobstore.{Pending => _} -import cromwell.CromwellTestkitSpec +import cromwell.CromwellTestKitSpec import cromwell.backend.BackendJobExecutionActor import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionActorCommand import cromwell.core.callcaching._ @@ -15,7 +15,7 @@ import scala.concurrent.duration._ import scala.language.postfixOps -trait EngineJobExecutionActorSpec extends CromwellTestkitSpec +trait EngineJobExecutionActorSpec extends CromwellTestKitSpec with Matchers with Mockito with BeforeAndAfterAll with BeforeAndAfter { // If we WANT something to happen, make sure it happens within this window: diff --git a/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala b/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala index abab45a69..4ac29abca 100644 --- a/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala +++ b/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala @@ -1,6 +1,6 @@ package cromwell.jobstore -import cromwell.CromwellTestkitSpec +import cromwell.CromwellTestKitSpec import cromwell.backend.BackendJobDescriptorKey import cromwell.core.{JobOutput, WorkflowId} import cromwell.jobstore.JobStoreActor._ @@ -21,7 +21,7 @@ object JobStoreServiceSpec { val EmptyExpression = WdlExpression.fromString(""" "" """) } -class JobStoreServiceSpec extends CromwellTestkitSpec with Matchers with Mockito { +class JobStoreServiceSpec extends CromwellTestKitSpec with Matchers with Mockito { "JobStoreService" should { "work" in { diff --git a/engine/src/test/scala/cromwell/jobstore/JobStoreWriterSpec.scala b/engine/src/test/scala/cromwell/jobstore/JobStoreWriterSpec.scala index a6e59679a..eabc86a5e 100644 --- a/engine/src/test/scala/cromwell/jobstore/JobStoreWriterSpec.scala +++ b/engine/src/test/scala/cromwell/jobstore/JobStoreWriterSpec.scala @@ -1,7 +1,7 @@ package cromwell.jobstore import akka.testkit.TestFSMRef -import cromwell.CromwellTestkitSpec +import cromwell.CromwellTestKitSpec import cromwell.core.WorkflowId import cromwell.jobstore.JobStoreActor.{JobStoreWriteSuccess, RegisterJobCompleted, RegisterWorkflowCompleted} import org.scalatest.{BeforeAndAfter, Matchers} @@ -11,7 +11,7 @@ import scala.concurrent.duration._ import scala.concurrent.{ExecutionContext, Future, Promise} import scala.language.postfixOps -class JobStoreWriterSpec extends CromwellTestkitSpec with Matchers with BeforeAndAfter { +class JobStoreWriterSpec extends CromwellTestKitSpec with Matchers with BeforeAndAfter { var database: WriteCountingJobStore = _ var jobStoreWriter: TestFSMRef[JobStoreWriterState, JobStoreWriterData, JobStoreWriterActor] = _ diff --git a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala index 9904835dc..44dd81f08 100644 --- a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala @@ -7,7 +7,7 @@ import akka.actor.{Actor, Props} import akka.pattern.ask import akka.util.Timeout import com.typesafe.config.ConfigFactory -import cromwell.CromwellTestkitSpec +import cromwell.CromwellTestKitSpec import cromwell.core._ import cromwell.engine.workflow.workflowstore.WorkflowStoreActor import cromwell.engine.workflow.workflowstore.WorkflowStoreActor.{WorkflowAborted => _, _} @@ -67,7 +67,7 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala implicit val defaultTimeout = RouteTestTimeout(30.seconds.dilated) override def actorRefFactory = system - override val serviceRegistryActor = CromwellTestkitSpec.ServiceRegistryActorInstance + override val serviceRegistryActor = CromwellTestKitSpec.ServiceRegistryActorInstance override val workflowStoreActor = actorRefFactory.actorOf(Props(new MockWorkflowStoreActor())) override val workflowManagerActor = actorRefFactory.actorOf(Props.empty) From 67ca3aea6252d37c87bfa91d5d5b328564b31642 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Tue, 15 Nov 2016 12:53:08 -0500 Subject: [PATCH 061/375] Publish hotfix branches to docker. --- .travis.yml | 2 +- project/Settings.scala | 7 +++++-- project/Version.scala | 2 +- src/bin/travis/afterSuccess.sh | 20 ++++++++++++++++++++ src/bin/travis/publishSnapshot.sh | 11 ----------- 5 files changed, 27 insertions(+), 15 deletions(-) create mode 100755 src/bin/travis/afterSuccess.sh delete mode 100755 src/bin/travis/publishSnapshot.sh diff --git a/.travis.yml b/.travis.yml index 2c207083e..f4d0d9b2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ env: script: - src/bin/travis/test.sh after_success: - - src/bin/travis/publishSnapshot.sh + - src/bin/travis/afterSuccess.sh deploy: provider: script script: src/bin/travis/publishRelease.sh diff --git a/project/Settings.scala b/project/Settings.scala index c42e082db..f59d102e8 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -70,8 +70,11 @@ object Settings { ImageName( namespace = Option("broadinstitute"), repository = name.value, - tag = Some(s"${version.value}") - ) + tag = Option(cromwellVersion)), + ImageName( + namespace = Option("broadinstitute"), + repository = name.value, + tag = Option(version.value)) ), dockerfile in docker := { // The assembly task generates a fat JAR file diff --git a/project/Version.scala b/project/Version.scala index d72b63ac4..dabbc5717 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -3,7 +3,7 @@ import sbt.Keys._ import sbt._ object Version { - // Upcoming release, or current if we're on the master branch + // Upcoming release, or current if we're on a master / hotfix branch val cromwellVersion = "23" // Adapted from SbtGit.versionWithGit diff --git a/src/bin/travis/afterSuccess.sh b/src/bin/travis/afterSuccess.sh new file mode 100755 index 000000000..e8bd24aae --- /dev/null +++ b/src/bin/travis/afterSuccess.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -e + +echo "BUILD_TYPE='$BUILD_TYPE'" +echo "TRAVIS_BRANCH='$TRAVIS_BRANCH'" +echo "TRAVIS_PULL_REQUEST='$TRAVIS_PULL_REQUEST'" + +if [ "$BUILD_TYPE" == "sbt" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then + + if [ "$TRAVIS_BRANCH" == "develop" ]; then + sbt 'set test in Test := {}' publish + + elif [[ "$TRAVIS_BRANCH" =~ ^[0-9\.]+_hotfix$ ]]; then + docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" + sbt 'set test in Test := {}' -Dproject.isSnapshot=false dockerBuildAndPush + + fi + +fi diff --git a/src/bin/travis/publishSnapshot.sh b/src/bin/travis/publishSnapshot.sh deleted file mode 100755 index 9c18a97fd..000000000 --- a/src/bin/travis/publishSnapshot.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set -e - -echo "BUILD_TYPE='$BUILD_TYPE'" -echo "TRAVIS_BRANCH='$TRAVIS_BRANCH'" -echo "TRAVIS_PULL_REQUEST='$TRAVIS_PULL_REQUEST'" - -if [ "$BUILD_TYPE" == "sbt" ] && [ "$TRAVIS_BRANCH" == "develop" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then - sbt 'set test in Test := {}' publish -fi From 09ccaaa59aaee2b2f623b13d2e21bdb9b0e5e8c2 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Wed, 16 Nov 2016 11:44:47 -0500 Subject: [PATCH 062/375] Ignore locally broken test --- .../scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala index 9b83c3d4a..45a312b65 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala @@ -7,10 +7,11 @@ import com.typesafe.config.{Config, ConfigFactory} import cromwell.backend.BackendWorkflowInitializationActor.{InitializationFailed, InitializationSuccess, Initialize} import cromwell.backend.impl.jes.authentication.GcsLocalizing import cromwell.backend.{BackendConfigurationDescriptor, BackendSpec, BackendWorkflowDescriptor} +import cromwell.core.Tags.IntegrationTest import cromwell.core.logging.LoggingTest._ import cromwell.core.{TestKitSuite, WorkflowOptions} import cromwell.filesystems.gcs.GoogleConfiguration -import cromwell.filesystems.gcs.auth.{SimpleClientSecrets, RefreshTokenMode} +import cromwell.filesystems.gcs.auth.{RefreshTokenMode, SimpleClientSecrets} import cromwell.util.{EncryptionSpec, SampleWdl} import org.scalatest.{FlatSpecLike, Matchers} import org.specs2.mock.Mockito @@ -143,7 +144,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe behavior of "JesInitializationActor" - it should "log a warning message when there are unsupported runtime attributes" in { + it should "log a warning message when there are unsupported runtime attributes" taggedAs IntegrationTest in { within(Timeout) { val workflowDescriptor = buildWorkflowDescriptor(HelloWorld, runtime = """runtime { docker: "ubuntu/latest" test: true }""") From 03e258ab4ffe8074dd444289ae00660fdf9d14d2 Mon Sep 17 00:00:00 2001 From: Ruchi Date: Thu, 17 Nov 2016 15:31:26 -0500 Subject: [PATCH 063/375] SingleWorkflowMode support for WDL import directories closes #1525 (#1652) Support passing in zipped/non zipped directories of WDLs through SingleWorkflowRunnerMode, add tests, and turn WorkflowSourceFiles into a trait with 2 case classes - one with an imports File and one without. --- CHANGELOG.md | 1 + README.md | 33 ++++++++++++++++++ .../scala/cromwell/core/WorkflowMetadataKeys.scala | 1 + .../scala/cromwell/core/WorkflowSourceFiles.scala | 24 +++++++++++-- core/src/test/scala/cromwell/util/SampleWdl.scala | 2 +- .../workflow/SingleWorkflowRunnerActor.scala | 12 +++++-- .../cromwell/engine/workflow/WorkflowActor.scala | 4 +-- .../MaterializeWorkflowDescriptorActor.scala | 15 +++++---- .../workflowstore/InMemoryWorkflowStore.scala | 6 ++-- .../workflow/workflowstore/SqlWorkflowStore.scala | 6 ++-- .../workflow/workflowstore/WorkflowStore.scala | 4 +-- .../workflowstore/WorkflowStoreActor.scala | 31 +++++++++-------- .../workflow/workflowstore/workflowstore_.scala | 4 +-- .../cromwell/webservice/CromwellApiHandler.scala | 8 +++-- .../scala/cromwell/OptionalParamWorkflowSpec.scala | 2 +- .../cromwell/engine/WorkflowStoreActorSpec.scala | 6 ++-- .../workflow/SingleWorkflowRunnerActorSpec.scala | 4 +-- .../engine/workflow/WorkflowActorSpec.scala | 4 +-- .../workflow/WorkflowDescriptorBuilder.scala | 4 +-- project/Dependencies.scala | 2 +- src/main/scala/cromwell/CromwellCommandLine.scala | 37 +++++++++++++++++--- src/main/scala/cromwell/Main.scala | 12 ++++--- .../scala/cromwell/CromwellCommandLineSpec.scala | 39 ++++++++++++++++++++-- .../impl/sfs/config/ConfigWdlNamespace.scala | 4 +-- 24 files changed, 203 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d38dc798..a9058f66e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ * `cross: (Array[X], Array[Y]) => Array[Pair[X, Y]]` - create every possible pair from the two input arrays and return them all as WDL pairs * `transpose: (Array[Array[X]]) => Array[Array[X]]` compute the matrix transpose for a 2D array. Assumes each inner array has the same length. * By default, `system.abort-jobs-on-terminate` is false when running `java -jar cromwell.jar server`, and true when running `java -jar cromwell.jar run `. +* Enable WDL imports when running in Single Workflow Runner Mode. ## 0.22 diff --git a/README.md b/README.md index 365c96ca3..a49a4c774 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,39 @@ $ cat my_wf.metadata.json } ``` +The fifth, optional parameter to the 'run' subcommand is a zip file which contains WDL source files. This zip file can be passed +and your primary workflow can import any WDL's from that collection and re-use those tasks. + +For example, consider you have a directory of WDL files: +``` +my_WDLs +└──cgrep.wdl +└──ps.wdl +└──wc.wdl +``` + +If you zip that directory to my_WDLs.zip, you have the option to pass it in as the last parameter in your run command +and be able to reference these WDLs as imports in your primary WDL. For example, your primary WDL can look like this: +``` +import "ps.wdl" as ps +import "cgrep.wdl" +import "wc.wdl" as wordCount + +workflow threestep { + +call ps.ps as getStatus +call cgrep.cgrep { input: str = getStatus.x } +call wordCount { input: str = ... } + +} + +``` +The command to run this WDL, without needing any inputs, workflow options or metadata files would look like: + +``` +$ java -jar cromwell.jar run threestep.wdl - - - /path/to/my_WDLs.zip +``` + ## server Start a server on port 8000, the API for the server is described in the [REST API](#rest-api) section. diff --git a/core/src/main/scala/cromwell/core/WorkflowMetadataKeys.scala b/core/src/main/scala/cromwell/core/WorkflowMetadataKeys.scala index 80ade0a4d..e5a4dd7a6 100644 --- a/core/src/main/scala/cromwell/core/WorkflowMetadataKeys.scala +++ b/core/src/main/scala/cromwell/core/WorkflowMetadataKeys.scala @@ -18,4 +18,5 @@ object WorkflowMetadataKeys { val SubmissionSection_Workflow = "workflow" val SubmissionSection_Inputs = "inputs" val SubmissionSection_Options = "options" + val SubmissionSection_Imports = "imports" } diff --git a/core/src/main/scala/cromwell/core/WorkflowSourceFiles.scala b/core/src/main/scala/cromwell/core/WorkflowSourceFiles.scala index ed03f1733..44bd23cc7 100644 --- a/core/src/main/scala/cromwell/core/WorkflowSourceFiles.scala +++ b/core/src/main/scala/cromwell/core/WorkflowSourceFiles.scala @@ -1,9 +1,29 @@ package cromwell.core +import better.files.File import wdl4s.{WdlJson, WdlSource} /** * Represents the collection of source files that a user submits to run a workflow */ -final case class WorkflowSourceFiles(wdlSource: WdlSource, inputsJson: WdlJson, - workflowOptionsJson: WorkflowOptionsJson) + +sealed trait WorkflowSourceFilesCollection { + def wdlSource: WdlSource + def inputsJson: WdlJson + def workflowOptionsJson: WorkflowOptionsJson + + def copyOptions(workflowOptions: WorkflowOptionsJson) = this match { + case w: WorkflowSourceFiles => WorkflowSourceFiles(w.wdlSource, w.inputsJson, workflowOptions) + case w: WorkflowSourceFilesWithImports => WorkflowSourceFilesWithImports(w.wdlSource, w.inputsJson, workflowOptions, w.importsFile) + } + +} + +final case class WorkflowSourceFiles(wdlSource: WdlSource, + inputsJson: WdlJson, + workflowOptionsJson: WorkflowOptionsJson) extends WorkflowSourceFilesCollection + +final case class WorkflowSourceFilesWithImports(wdlSource: WdlSource, + inputsJson: WdlJson, + workflowOptionsJson: WorkflowOptionsJson, + importsFile: File) extends WorkflowSourceFilesCollection diff --git a/core/src/test/scala/cromwell/util/SampleWdl.scala b/core/src/test/scala/cromwell/util/SampleWdl.scala index 28718d1e9..937521ece 100644 --- a/core/src/test/scala/cromwell/util/SampleWdl.scala +++ b/core/src/test/scala/cromwell/util/SampleWdl.scala @@ -4,7 +4,7 @@ import java.nio.file.{Files, Path} import java.util.UUID import better.files._ -import cromwell.core.WorkflowSourceFiles +import cromwell.core.{WorkflowSourceFiles} import spray.json._ import wdl4s._ import wdl4s.types.{WdlArrayType, WdlStringType} diff --git a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala index 22fbd498f..506626965 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala @@ -31,7 +31,7 @@ import scala.util.{Failure, Try} * Designed explicitly for the use case of the 'run' functionality in Main. This Actor will start a workflow, * print out the outputs when complete and reply with a result. */ -class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: Option[Path]) +class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, metadataOutputPath: Option[Path]) extends CromwellRootActor with LoggingFSM[RunnerState, SwraData] { import SingleWorkflowRunnerActor._ @@ -144,7 +144,15 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: stay() } + private def cleanUpTempFiles = source match { + case w: WorkflowSourceFilesWithImports => w.importsFile.delete(swallowIOExceptions = true) + case w: WorkflowSourceFiles => // + } + private def issueReply(data: TerminalSwraData) = { + + cleanUpTempFiles + data match { case s: SucceededSwraData => issueSuccessReply(s.replyTo) case f: FailedSwraData => issueFailureReply(f.replyTo, f.failure) @@ -190,7 +198,7 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: } object SingleWorkflowRunnerActor { - def props(source: WorkflowSourceFiles, metadataOutputFile: Option[Path]): Props = { + def props(source: WorkflowSourceFilesCollection, metadataOutputFile: Option[Path]): Props = { Props(new SingleWorkflowRunnerActor(source, metadataOutputFile)) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala index 5802c7697..90390fbe3 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala @@ -136,7 +136,7 @@ object WorkflowActor { def props(workflowId: WorkflowId, startMode: StartMode, - wdlSource: WorkflowSourceFiles, + wdlSource: WorkflowSourceFilesCollection, conf: Config, serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, @@ -154,7 +154,7 @@ object WorkflowActor { */ class WorkflowActor(val workflowId: WorkflowId, startMode: StartMode, - workflowSources: WorkflowSourceFiles, + workflowSources: WorkflowSourceFilesCollection, conf: Config, serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala index fcd187133..b4fe9d45a 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala @@ -49,7 +49,7 @@ object MaterializeWorkflowDescriptorActor { Commands */ sealed trait MaterializeWorkflowDescriptorActorMessage - case class MaterializeWorkflowDescriptorCommand(workflowSourceFiles: WorkflowSourceFiles, + case class MaterializeWorkflowDescriptorCommand(workflowSourceFiles: WorkflowSourceFilesCollection, conf: Config) extends MaterializeWorkflowDescriptorActorMessage case object MaterializeWorkflowDescriptorAbortCommand @@ -159,9 +159,9 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor } private def buildWorkflowDescriptor(id: WorkflowId, - sourceFiles: WorkflowSourceFiles, + sourceFiles: WorkflowSourceFilesCollection, conf: Config): ErrorOr[EngineWorkflowDescriptor] = { - val namespaceValidation = validateNamespace(sourceFiles.wdlSource) + val namespaceValidation = validateNamespace(sourceFiles) val workflowOptionsValidation = validateWorkflowOptions(sourceFiles.workflowOptionsJson) (namespaceValidation |@| workflowOptionsValidation) map { (_, _) @@ -181,7 +181,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor } private def buildWorkflowDescriptor(id: WorkflowId, - sourceFiles: WorkflowSourceFiles, + sourceFiles: WorkflowSourceFilesCollection, namespace: WdlNamespaceWithWorkflow, workflowOptions: WorkflowOptions, conf: Config, @@ -299,9 +299,12 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor } } - private def validateNamespace(source: WdlSource): ErrorOr[WdlNamespaceWithWorkflow] = { + private def validateNamespace(source: WorkflowSourceFilesCollection): ErrorOr[WdlNamespaceWithWorkflow] = { try { - WdlNamespaceWithWorkflow.load(source).validNel + source match { + case w: WorkflowSourceFilesWithImports => WdlNamespaceWithWorkflow.load(w.wdlSource, w.importsFile).validNel + case w: WorkflowSourceFiles => WdlNamespaceWithWorkflow.load(w.wdlSource).validNel + } } catch { case e: Exception => s"Unable to load namespace from workflow: ${e.getMessage}".invalidNel } diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala index a24be2d32..0f04212f8 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala @@ -1,7 +1,7 @@ package cromwell.engine.workflow.workflowstore import cats.data.NonEmptyList -import cromwell.core.{WorkflowId, WorkflowSourceFiles} +import cromwell.core.{WorkflowId, WorkflowSourceFilesCollection} import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState import scala.concurrent.{ExecutionContext, Future} @@ -14,7 +14,7 @@ class InMemoryWorkflowStore extends WorkflowStore { * Adds the requested WorkflowSourceFiles to the store and returns a WorkflowId for each one (in order) * for tracking purposes. */ - override def add(sources: NonEmptyList[WorkflowSourceFiles])(implicit ec: ExecutionContext): Future[NonEmptyList[WorkflowId]] = { + override def add(sources: NonEmptyList[WorkflowSourceFilesCollection])(implicit ec: ExecutionContext): Future[NonEmptyList[WorkflowId]] = { val submittedWorkflows = sources map { SubmittedWorkflow(WorkflowId.randomId(), _, WorkflowStoreState.Submitted) } workflowStore = workflowStore ++ submittedWorkflows.toList Future.successful(submittedWorkflows map { _.id }) @@ -44,7 +44,7 @@ class InMemoryWorkflowStore extends WorkflowStore { override def initialize(implicit ec: ExecutionContext): Future[Unit] = Future.successful(()) } -final case class SubmittedWorkflow(id: WorkflowId, sources: WorkflowSourceFiles, state: WorkflowStoreState) { +final case class SubmittedWorkflow(id: WorkflowId, sources: WorkflowSourceFilesCollection, state: WorkflowStoreState) { def toWorkflowToStart: WorkflowToStart = { state match { case r: StartableState => WorkflowToStart(id, sources, r) diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala index 7056137c3..7613c9950 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala @@ -3,7 +3,7 @@ package cromwell.engine.workflow.workflowstore import java.time.OffsetDateTime import cats.data.NonEmptyList -import cromwell.core.{WorkflowId, WorkflowSourceFiles} +import cromwell.core.{WorkflowId, WorkflowSourceFiles, WorkflowSourceFilesCollection} import cromwell.database.sql.SqlConverters._ import cromwell.database.sql.WorkflowStoreSqlDatabase import cromwell.database.sql.tables.WorkflowStoreEntry @@ -36,7 +36,7 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase) extends Workf * Adds the requested WorkflowSourceFiles to the store and returns a WorkflowId for each one (in order) * for tracking purposes. */ - override def add(sources: NonEmptyList[WorkflowSourceFiles])(implicit ec: ExecutionContext): Future[NonEmptyList[WorkflowId]] = { + override def add(sources: NonEmptyList[WorkflowSourceFilesCollection])(implicit ec: ExecutionContext): Future[NonEmptyList[WorkflowId]] = { val asStoreEntries = sources map toWorkflowStoreEntry val returnValue = asStoreEntries map { workflowStore => WorkflowId.fromString(workflowStore.workflowExecutionUuid) } @@ -56,7 +56,7 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase) extends Workf fromDbStateStringToStartableState(workflowStoreEntry.workflowState)) } - private def toWorkflowStoreEntry(workflowSourceFiles: WorkflowSourceFiles): WorkflowStoreEntry = { + private def toWorkflowStoreEntry(workflowSourceFiles: WorkflowSourceFilesCollection): WorkflowStoreEntry = { WorkflowStoreEntry( WorkflowId.randomId().toString, workflowSourceFiles.wdlSource.toClob, diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala index e3d7b44be..f4734f7bb 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala @@ -1,7 +1,7 @@ package cromwell.engine.workflow.workflowstore import cats.data.NonEmptyList -import cromwell.core.{WorkflowId, WorkflowSourceFiles} +import cromwell.core.{WorkflowId, WorkflowSourceFilesCollection} import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState import scala.concurrent.{ExecutionContext, Future} @@ -14,7 +14,7 @@ trait WorkflowStore { * Adds the requested WorkflowSourceFiles to the store and returns a WorkflowId for each one (in order) * for tracking purposes. */ - def add(sources: NonEmptyList[WorkflowSourceFiles])(implicit ec: ExecutionContext): Future[NonEmptyList[WorkflowId]] + def add(sources: NonEmptyList[WorkflowSourceFilesCollection])(implicit ec: ExecutionContext): Future[NonEmptyList[WorkflowId]] /** * Retrieves up to n workflows which have not already been pulled into the engine and sets their pickedUp diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala index f38b31d93..f1e06e2e4 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala @@ -117,20 +117,20 @@ case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor: ActorR goto(Working) using nextData } - private def storeWorkflowSources(sources: NonEmptyList[WorkflowSourceFiles]): Future[NonEmptyList[WorkflowId]] = { + private def storeWorkflowSources(sources: NonEmptyList[WorkflowSourceFilesCollection]): Future[NonEmptyList[WorkflowId]] = { for { processedSources <- Future.fromTry(processSources(sources, _.asPrettyJson)) workflowIds <- store.add(processedSources) } yield workflowIds } - private def processSources(sources: NonEmptyList[WorkflowSourceFiles], + private def processSources(sources: NonEmptyList[WorkflowSourceFilesCollection], processOptions: WorkflowOptions => WorkflowOptionsJson): - Try[NonEmptyList[WorkflowSourceFiles]] = { - val nelTries: NonEmptyList[Try[WorkflowSourceFiles]] = sources map processSource(processOptions) - val seqTries: Seq[Try[WorkflowSourceFiles]] = nelTries.toList - val trySeqs: Try[Seq[WorkflowSourceFiles]] = TryUtil.sequence(seqTries) - val tryNel: Try[NonEmptyList[WorkflowSourceFiles]] = trySeqs.map(seq => NonEmptyList.fromList(seq.toList).get) + Try[NonEmptyList[WorkflowSourceFilesCollection]] = { + val nelTries: NonEmptyList[Try[WorkflowSourceFilesCollection]] = sources map processSource(processOptions) + val seqTries: Seq[Try[WorkflowSourceFilesCollection]] = nelTries.toList + val trySeqs: Try[Seq[WorkflowSourceFilesCollection]] = TryUtil.sequence(seqTries) + val tryNel: Try[NonEmptyList[WorkflowSourceFilesCollection]] = trySeqs.map(seq => NonEmptyList.fromList(seq.toList).get) tryNel } @@ -142,10 +142,10 @@ case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor: ActorR * @return Attempted updated workflow source */ private def processSource(processOptions: WorkflowOptions => WorkflowOptionsJson) - (source: WorkflowSourceFiles): Try[WorkflowSourceFiles] = { + (source: WorkflowSourceFilesCollection): Try[WorkflowSourceFilesCollection] = { for { processedWorkflowOptions <- WorkflowOptions.fromJsonString(source.workflowOptionsJson) - } yield source.copy(workflowOptionsJson = processOptions(processedWorkflowOptions)) + } yield source.copyOptions(processOptions(processedWorkflowOptions)) } private def addWorkCompletionHooks[A](command: WorkflowStoreActorCommand, work: Future[A]) = { @@ -184,7 +184,7 @@ case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor: ActorR /** * Takes the workflow id and sends it over to the metadata service w/ default empty values for inputs/outputs */ - private def registerSubmissionWithMetadataService(id: WorkflowId, originalSourceFiles: WorkflowSourceFiles): Unit = { + private def registerSubmissionWithMetadataService(id: WorkflowId, originalSourceFiles: WorkflowSourceFilesCollection): Unit = { val sourceFiles = processSource(_.clearEncryptedValues)(originalSourceFiles).get val submissionEvents = List( @@ -197,7 +197,12 @@ case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor: ActorR MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Options), MetadataValue(sourceFiles.workflowOptionsJson)) ) - serviceRegistryActor ! PutMetadataAction(submissionEvents) + val wfImportEvent = sourceFiles match { + case w: WorkflowSourceFilesWithImports => MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Imports), MetadataValue(w.importsFile.pathAsString)) + case w: WorkflowSourceFiles => MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Imports), MetadataValue("None")) + } + + serviceRegistryActor ! PutMetadataAction(submissionEvents :+ wfImportEvent) } } @@ -220,8 +225,8 @@ object WorkflowStoreActor { private[workflowstore] case object Idle extends WorkflowStoreActorState sealed trait WorkflowStoreActorCommand - final case class SubmitWorkflow(source: WorkflowSourceFiles) extends WorkflowStoreActorCommand - final case class BatchSubmitWorkflows(sources: NonEmptyList[WorkflowSourceFiles]) extends WorkflowStoreActorCommand + final case class SubmitWorkflow(source: WorkflowSourceFilesCollection) extends WorkflowStoreActorCommand + final case class BatchSubmitWorkflows(sources: NonEmptyList[WorkflowSourceFilesCollection]) extends WorkflowStoreActorCommand final case class FetchRunnableWorkflows(n: Int) extends WorkflowStoreActorCommand final case class RemoveWorkflow(id: WorkflowId) extends WorkflowStoreActorCommand final case class AbortWorkflow(id: WorkflowId, manager: ActorRef) extends WorkflowStoreActorCommand diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/workflowstore_.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/workflowstore_.scala index 0d9481c47..61bc37ee0 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/workflowstore_.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/workflowstore_.scala @@ -1,6 +1,6 @@ package cromwell.engine.workflow.workflowstore -import cromwell.core.{WorkflowId, WorkflowSourceFiles} +import cromwell.core.{WorkflowId, WorkflowSourceFilesCollection} import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState sealed trait WorkflowStoreState {def isStartable: Boolean} @@ -12,4 +12,4 @@ object WorkflowStoreState { case object Restartable extends StartableState } -final case class WorkflowToStart(id: WorkflowId, sources: WorkflowSourceFiles, state: StartableState) +final case class WorkflowToStart(id: WorkflowId, sources: WorkflowSourceFilesCollection, state: StartableState) diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala index a2441abb4..343de5f8e 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala @@ -61,12 +61,16 @@ class CromwellApiHandler(requestHandlerActor: ActorRef) extends Actor with Workf case _ => RequestComplete((StatusCodes.InternalServerError, APIResponse.error(e))) } - case ApiHandlerWorkflowSubmit(source) => requestHandlerActor ! WorkflowStoreActor.SubmitWorkflow(source) + case ApiHandlerWorkflowSubmit(source) => requestHandlerActor ! WorkflowStoreActor.SubmitWorkflow(WorkflowSourceFiles(source.wdlSource, + source.inputsJson, + source.workflowOptionsJson)) case WorkflowStoreActor.WorkflowSubmittedToStore(id) => context.parent ! RequestComplete((StatusCodes.Created, WorkflowSubmitResponse(id.toString, WorkflowSubmitted.toString))) - case ApiHandlerWorkflowSubmitBatch(sources) => requestHandlerActor ! WorkflowStoreActor.BatchSubmitWorkflows(sources) + case ApiHandlerWorkflowSubmitBatch(sources) => requestHandlerActor ! + WorkflowStoreActor.BatchSubmitWorkflows(sources.map(x => WorkflowSourceFiles(x.wdlSource,x.inputsJson,x.workflowOptionsJson))) + case WorkflowStoreActor.WorkflowsBatchSubmittedToStore(ids) => val responses = ids map { id => WorkflowSubmitResponse(id.toString, WorkflowSubmitted.toString) } diff --git a/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala b/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala index cded78cc4..73347b794 100644 --- a/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala @@ -22,7 +22,7 @@ class OptionalParamWorkflowSpec extends Matchers with WordSpecLike { | call find |} """.stripMargin - val ns = WdlNamespace.load(wf) + val ns = WdlNamespace.loadUsingSource(wf, None, None) val findTask = ns.findTask("find") getOrElse { fail("Expected to find task 'find'") } diff --git a/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala b/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala index d8bd649c8..d6719fe85 100644 --- a/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala @@ -2,7 +2,7 @@ package cromwell.engine import cats.data.NonEmptyList import cromwell.CromwellTestKitSpec -import cromwell.core.{WorkflowId, WorkflowSourceFiles} +import cromwell.core.{WorkflowId, WorkflowSourceFilesCollection} import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._ import cromwell.engine.workflow.workflowstore._ import cromwell.services.metadata.MetadataQuery @@ -35,9 +35,9 @@ class WorkflowStoreActorSpec extends CromwellTestKitSpec with Matchers { list.foldLeft((List.empty[WorkflowToStart], true))(folderFunction)._2 } - private def prettyOptions(workflowSourceFiles: WorkflowSourceFiles): WorkflowSourceFiles = { + private def prettyOptions(workflowSourceFiles: WorkflowSourceFilesCollection): WorkflowSourceFilesCollection = { import spray.json._ - workflowSourceFiles.copy(workflowOptionsJson = workflowSourceFiles.workflowOptionsJson.parseJson.prettyPrint) + workflowSourceFiles.copyOptions(workflowSourceFiles.workflowOptionsJson.parseJson.prettyPrint) } "The WorkflowStoreActor" should { diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala index 6753d3d2d..543b8e502 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala @@ -10,7 +10,7 @@ import akka.util.Timeout import better.files._ import com.typesafe.config.ConfigFactory import cromwell.CromwellTestKitSpec._ -import cromwell.core.WorkflowSourceFiles +import cromwell.core.{WorkflowSourceFilesCollection} import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.SingleWorkflowRunnerActor.RunWorkflow import cromwell.engine.workflow.SingleWorkflowRunnerActorSpec._ @@ -46,7 +46,7 @@ object SingleWorkflowRunnerActorSpec { def toFields = jsValue.get.asJsObject.fields } - class TestSingleWorkflowRunnerActor(source: WorkflowSourceFiles, + class TestSingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, metadataOutputPath: Option[Path]) extends SingleWorkflowRunnerActor(source, metadataOutputPath) { override lazy val serviceRegistryActor = CromwellTestKitSpec.ServiceRegistryActorInstance diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala index 19f1fd06d..23087f529 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala @@ -4,7 +4,7 @@ import akka.actor.{Actor, ActorRef} import akka.testkit.{TestActorRef, TestFSMRef, TestProbe} import com.typesafe.config.{Config, ConfigFactory} import cromwell.backend.AllBackendInitializationData -import cromwell.core.{ExecutionStore, OutputStore, WorkflowId, WorkflowSourceFiles} +import cromwell.core._ import cromwell.engine.EngineWorkflowDescriptor import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.WorkflowActor._ @@ -151,7 +151,7 @@ class WorkflowActorSpec extends CromwellTestKitSpec with WorkflowDescriptorBuild class MockWorkflowActor(val finalizationProbe: TestProbe, workflowId: WorkflowId, startMode: StartMode, - workflowSources: WorkflowSourceFiles, + workflowSources: WorkflowSourceFilesCollection, conf: Config, serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowDescriptorBuilder.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowDescriptorBuilder.scala index a6b02620d..2cabf349d 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/WorkflowDescriptorBuilder.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowDescriptorBuilder.scala @@ -3,7 +3,7 @@ package cromwell.engine.workflow import akka.actor.{ActorSystem, Props} import com.typesafe.config.ConfigFactory import cromwell.CromwellTestKitSpec -import cromwell.core.{WorkflowId, WorkflowSourceFiles} +import cromwell.core.{WorkflowId, WorkflowSourceFilesCollection} import cromwell.engine.EngineWorkflowDescriptor import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.{MaterializeWorkflowDescriptorCommand, MaterializeWorkflowDescriptorFailureResponse, MaterializeWorkflowDescriptorSuccessResponse, WorkflowDescriptorMaterializationResult} @@ -15,7 +15,7 @@ trait WorkflowDescriptorBuilder { implicit val awaitTimeout = CromwellTestKitSpec.TimeoutDuration implicit val actorSystem: ActorSystem - def createMaterializedEngineWorkflowDescriptor(id: WorkflowId, workflowSources: WorkflowSourceFiles): EngineWorkflowDescriptor = { + def createMaterializedEngineWorkflowDescriptor(id: WorkflowId, workflowSources: WorkflowSourceFilesCollection): EngineWorkflowDescriptor = { import akka.pattern.ask implicit val timeout = akka.util.Timeout(awaitTimeout) implicit val ec = actorSystem.dispatcher diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 406e7df26..04c3f02b0 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -2,7 +2,7 @@ import sbt._ object Dependencies { lazy val lenthallV = "0.19" - lazy val wdl4sV = "0.7-1852aea-SNAPSHOT" + lazy val wdl4sV = "0.7-b9c6c98-SNAPSHOT" lazy val sprayV = "1.3.3" /* spray-json is an independent project from the "spray suite" diff --git a/src/main/scala/cromwell/CromwellCommandLine.scala b/src/main/scala/cromwell/CromwellCommandLine.scala index c52ebbd66..6b8fc960e 100644 --- a/src/main/scala/cromwell/CromwellCommandLine.scala +++ b/src/main/scala/cromwell/CromwellCommandLine.scala @@ -6,7 +6,7 @@ import better.files._ import cats.data.Validated._ import cats.syntax.cartesian._ import cats.syntax.validated._ -import cromwell.core.WorkflowSourceFiles +import cromwell.core.{WorkflowSourceFiles, WorkflowSourceFilesCollection, WorkflowSourceFilesWithImports} import cromwell.util.FileUtil._ import lenthall.exception.MessageAggregation import cromwell.core.ErrorOr._ @@ -17,7 +17,7 @@ sealed abstract class CromwellCommandLine case object UsageAndExit extends CromwellCommandLine case object RunServer extends CromwellCommandLine final case class RunSingle(wdlPath: Path, - sourceFiles: WorkflowSourceFiles, + sourceFiles: WorkflowSourceFilesCollection, inputsPath: Option[Path], optionsPath: Option[Path], metadataPath: Option[Path]) extends CromwellCommandLine @@ -26,7 +26,7 @@ object CromwellCommandLine { def apply(args: Seq[String]): CromwellCommandLine = { args.headOption match { case Some("server") if args.size == 1 => RunServer - case Some("run") if args.size >= 2 && args.size <= 5 => RunSingle(args.tail) + case Some("run") if args.size >= 2 && args.size <= 6 => RunSingle(args.tail) case _ => UsageAndExit } } @@ -38,15 +38,19 @@ object RunSingle { val inputsPath = argPath(args, 1, Option(".inputs"), checkDefaultExists = false) val optionsPath = argPath(args, 2, Option(".options"), checkDefaultExists = true) val metadataPath = argPath(args, 3, None) + val importPath = argPath(args, 4, None) val wdl = readContent("WDL file", wdlPath) val inputsJson = readJson("Inputs", inputsPath) val optionsJson = readJson("Workflow Options", optionsPath) - val sourceFiles = (wdl |@| inputsJson |@| optionsJson) map { WorkflowSourceFiles.apply } + val sourceFileCollection = importPath match { + case Some(p) => (wdl |@| inputsJson |@| optionsJson |@| validateImportsDirectory(p)) map WorkflowSourceFilesWithImports.apply + case None => (wdl |@| inputsJson |@| optionsJson) map WorkflowSourceFiles.apply + } val runSingle = for { - sources <- sourceFiles + sources <- sourceFileCollection _ <- writeableMetadataPath(metadataPath) } yield RunSingle(wdlPath, sources, inputsPath, optionsPath, metadataPath) @@ -59,6 +63,29 @@ object RunSingle { } } + private def validateImportsDirectory(path: Path): ErrorOr[File] = { + + def unZipFile(f: File): File = { + val unzippedFile = f.unzip() + val unzippedFileContents = unzippedFile.toJava.listFiles().head + + if (unzippedFileContents.isDirectory) File(unzippedFileContents.getPath) + else unzippedFile + } + + val importsFile: File = File(path).extension match { + case Some(".zip") => unZipFile(File(path)) + case _ => File(path) + } + + importsFile match { + case file if !file.isDirectory => s"Unable to import workflows as the given path is not a directory: ${file.pathAsString}".invalidNel + case file if file.isDirectory && file.isEmpty => s"Unable to import workflows as the given path is an empty directory: ${file.pathAsString}".invalidNel + case file => file.validNel + } + } + + private def writeableMetadataPath(path: Option[Path]): ErrorOr[Unit] = { path match { case Some(p) if !metadataPathIsWriteable(p) => s"Unable to write to metadata directory: $p".invalidNel diff --git a/src/main/scala/cromwell/Main.scala b/src/main/scala/cromwell/Main.scala index 45bd9e838..3f630a498 100644 --- a/src/main/scala/cromwell/Main.scala +++ b/src/main/scala/cromwell/Main.scala @@ -98,17 +98,21 @@ object Main extends App { |java -jar cromwell.jar | |Actions: - |run [ [ - | []]] + |run [] [] + | [] [] | | Given a WDL file and JSON file containing the value of the | workflow inputs, this will run the workflow locally and | print out the outputs in JSON format. The workflow | options file specifies some runtime configuration for the | workflow (see README for details). The workflow metadata - | output is an optional file path to output the metadata. + | output is an optional file path to output the metadata. The + | directory of WDL files is optional. However, it is required + | if the primary workflow imports workflows that are outside + | of the root directory of the Cromwell project. + | | Use a single dash ("-") to skip optional files. Ex: - | run noinputs.wdl - - metadata.json + | run noinputs.wdl - - metadata.json - | |server | diff --git a/src/test/scala/cromwell/CromwellCommandLineSpec.scala b/src/test/scala/cromwell/CromwellCommandLineSpec.scala index 0c0593db7..a53e7d68d 100644 --- a/src/test/scala/cromwell/CromwellCommandLineSpec.scala +++ b/src/test/scala/cromwell/CromwellCommandLineSpec.scala @@ -3,7 +3,7 @@ package cromwell import better.files._ import cromwell.core.path.PathImplicits._ import cromwell.util.SampleWdl -import cromwell.util.SampleWdl.ThreeStep +import cromwell.util.SampleWdl.{FileClobber, FilePassingWorkflow, GoodbyeWorld, ThreeStep} import org.scalatest.{FlatSpec, Matchers} import scala.util.Try @@ -30,7 +30,7 @@ class CromwellCommandLineSpec extends FlatSpec with Matchers { } it should "fail with too many arguments to run" in { - CromwellCommandLine(List("run", "bork", "bork", "bork", "bork", "bork")) + CromwellCommandLine(List("run", "bork", "bork", "bork", "bork", "bork", "blerg")) } it should "RunSingle when supplying wdl and inputs" in { @@ -76,6 +76,41 @@ class CromwellCommandLineSpec extends FlatSpec with Matchers { ccl.isFailure shouldBe true ccl.failed.get.getMessage should include("Unable to write to metadata directory:") } + + it should "fail if imports path is not a direcotry" in { + val goodBye = WdlAndInputs(GoodbyeWorld) + val badDirectory = goodBye.wdlFile + val ccl = Try(CromwellCommandLine(List("run", goodBye.wdl, "-", "-", "-", badDirectory.toString))) + ccl.isFailure shouldBe true + ccl.failed.get.getMessage should include("Unable to import workflows as the given path is not a directory:") + } + + it should "fail if imports directory is empty" in { + val goodBye = WdlAndInputs(GoodbyeWorld) + val emptyDirectory = File.newTemporaryDirectory(s"temp_empty_dir") + val ccl = Try(CromwellCommandLine(List("run", goodBye.wdl, "-", "-", "-", emptyDirectory.pathAsString))) + ccl.failed.get.getMessage should include("Unable to import workflows as the given path is an empty directory:") + + goodBye.deleteTempFiles() + emptyDirectory.delete(swallowIOExceptions = true) + } + + it should "run if imports directory is a .zip file" in { + val wdlDir = File.newTemporaryDirectory("wdlDirectory") + + val filePassing = File.newTemporaryFile("filePassing", ".wdl", Option(wdlDir)) + val fileClobber = File.newTemporaryFile("fileClobber", ".wdl", Option(wdlDir)) + filePassing write FilePassingWorkflow.wdlSource() + fileClobber write FileClobber.wdlSource() + + val zippedDir = wdlDir.zip() + val zippedPath = zippedDir.pathAsString + + val ccl = Try(CromwellCommandLine(List("run", filePassing.pathAsString, "-", "-", "-", zippedPath))) + ccl.isFailure shouldBe false + + zippedDir.delete(swallowIOExceptions = true) + } } object CromwellCommandLineSpec { diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala index cb56e35a0..161768da1 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala @@ -44,7 +44,7 @@ class ConfigWdlNamespace(backendConfig: Config) { */ val wdlNamespace = { try { - WdlNamespace.load(wdlSource) + WdlNamespace.loadUsingSource(wdlSource, None, None) } catch { case exception: Exception => throw new RuntimeException(s"Error parsing generated wdl:\n$wdlSource".stripMargin, exception) @@ -74,7 +74,7 @@ object ConfigWdlNamespace { private def makeTask(taskName: String, command: String, declarations: String): Task = { val wdlSource = makeWdlSource(taskName, command, declarations) - val wdlNamespace = WdlNamespace.load(wdlSource) + val wdlNamespace = WdlNamespace.loadUsingSource(wdlSource, None, None) wdlNamespace.findTask(taskName).getOrElse(throw new RuntimeException(s"Couldn't find task $taskName")) } From fb9c8d4e8db831454c9e0999834856bd5d60edc1 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Wed, 16 Nov 2016 16:26:14 -0500 Subject: [PATCH 064/375] Better error reporting if an EJEA crashes unexpectedly --- .../backend/BackendJobExecutionActor.scala | 8 ++-- .../scala/cromwell/util/StopAndLogSupervisor.scala | 24 +++++++++++ .../execution/EngineJobExecutionActor.scala | 2 +- .../execution/WorkflowExecutionActor.scala | 46 +++++++++++++--------- .../execution/WorkflowExecutionActorData.scala | 15 +++++-- .../tokens/JobExecutionTokenDispenserActor.scala | 2 +- .../execution/ejea/EjeaPreparingJobSpec.scala | 8 ++-- .../lifecycle/execution/ejea/PerTestHelper.scala | 2 +- 8 files changed, 75 insertions(+), 32 deletions(-) create mode 100644 core/src/main/scala/cromwell/util/StopAndLogSupervisor.scala diff --git a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala index 7bbdc27b2..f5b31f776 100644 --- a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala @@ -7,7 +7,7 @@ import akka.event.LoggingReceive import cromwell.backend.BackendJobExecutionActor._ import cromwell.backend.BackendLifecycleActor._ import cromwell.backend.wdl.OutputEvaluator -import cromwell.core.{ExecutionEvent, JobOutputs} +import cromwell.core.{ExecutionEvent, JobKey, JobOutputs} import wdl4s.expression.WdlStandardLibraryFunctions import wdl4s.values.WdlValue @@ -24,11 +24,11 @@ object BackendJobExecutionActor { // Responses sealed trait BackendJobExecutionActorResponse extends BackendWorkflowLifecycleActorResponse - sealed trait BackendJobExecutionResponse extends BackendJobExecutionActorResponse { def jobKey: BackendJobDescriptorKey } + sealed trait BackendJobExecutionResponse extends BackendJobExecutionActorResponse { def jobKey: JobKey } case class SucceededResponse(jobKey: BackendJobDescriptorKey, returnCode: Option[Int], jobOutputs: JobOutputs, jobDetritusFiles: Option[Map[String, Path]], executionEvents: Seq[ExecutionEvent]) extends BackendJobExecutionResponse - case class AbortedResponse(jobKey: BackendJobDescriptorKey) extends BackendJobExecutionResponse + case class AbortedResponse(jobKey: JobKey) extends BackendJobExecutionResponse sealed trait BackendJobFailedResponse extends BackendJobExecutionResponse { def throwable: Throwable; def returnCode: Option[Int] } - case class FailedNonRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse + case class FailedNonRetryableResponse(jobKey: JobKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse case class FailedRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse } diff --git a/core/src/main/scala/cromwell/util/StopAndLogSupervisor.scala b/core/src/main/scala/cromwell/util/StopAndLogSupervisor.scala new file mode 100644 index 000000000..4b64e5d8b --- /dev/null +++ b/core/src/main/scala/cromwell/util/StopAndLogSupervisor.scala @@ -0,0 +1,24 @@ +package cromwell.util + +import akka.actor.SupervisorStrategy.{Decider, Stop} +import akka.actor.{Actor, ActorRef, OneForOneStrategy, SupervisorStrategy} +import cromwell.core.logging.WorkflowLogging + +trait StopAndLogSupervisor { this: Actor with WorkflowLogging => + + private var failureLog: Map[ActorRef, Throwable] = Map.empty + + final val stopAndLogStrategy: SupervisorStrategy = { + def stoppingDecider: Decider = { + case e: Exception => + val failer = sender() + failureLog += failer -> e + Stop + } + OneForOneStrategy()(stoppingDecider) + } + + final def getFailureCause(actorRef: ActorRef): Option[Throwable] = failureLog.get(actorRef) + + override final val supervisorStrategy = stopAndLogStrategy +} diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala index b6c991724..4f7f0878a 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala @@ -122,7 +122,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, case CallCachingOff => runJob(updatedData) } case Event(response: BackendJobPreparationFailed, NoData) => - forwardAndStop(response) + respondAndStop(FailedNonRetryableResponse(response.jobKey, response.throwable, None)) } private val callCachingReadResultMetadataKey = "Call caching read result" diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index a30dc7500..64e23672d 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -2,7 +2,6 @@ package cromwell.engine.workflow.lifecycle.execution import java.time.OffsetDateTime -import akka.actor.SupervisorStrategy.{Escalate, Stop} import akka.actor._ import cats.data.NonEmptyList import com.typesafe.config.ConfigFactory @@ -19,12 +18,12 @@ import cromwell.core._ import cromwell.core.logging.WorkflowLogging import cromwell.engine.backend.{BackendSingletonCollection, CromwellBackends} import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{JobRunning, JobStarting} -import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor.BackendJobPreparationFailed import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.WorkflowExecutionActorState import cromwell.engine.workflow.lifecycle.{EngineLifecycleActorAbortCommand, EngineLifecycleActorAbortedResponse} import cromwell.engine.{ContinueWhilePossible, EngineWorkflowDescriptor} import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata._ +import cromwell.util.StopAndLogSupervisor import cromwell.webservice.EngineStatsActor import lenthall.exception.ThrowableAggregation import net.ceedubs.ficus.Ficus._ @@ -251,18 +250,10 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, backendSingletonCollection: BackendSingletonCollection, initializationData: AllBackendInitializationData, restarting: Boolean) - extends LoggingFSM[WorkflowExecutionActorState, WorkflowExecutionActorData] with WorkflowLogging { + extends LoggingFSM[WorkflowExecutionActorState, WorkflowExecutionActorData] with WorkflowLogging with StopAndLogSupervisor { import WorkflowExecutionActor._ - override def supervisorStrategy = AllForOneStrategy() { - case ex: ActorInitializationException => - context.parent ! WorkflowExecutionFailedResponse(stateData.executionStore, stateData.outputStore, List(ex)) - context.stop(self) - Stop - case t => super.supervisorStrategy.decider.applyOrElse(t, (_: Any) => Escalate) - } - val tag = s"WorkflowExecutionActor [UUID(${workflowId.shortString})]" private lazy val DefaultMaxRetriesFallbackValue = 10 @@ -288,6 +279,7 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, workflowDescriptor, executionStore = buildInitialExecutionStore(), backendJobExecutionActors = Map.empty, + engineJobExecutionActors = Map.empty, outputStore = OutputStore.empty ) ) @@ -333,17 +325,14 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, // The EJEA is telling us that the job is now Starting. Update the metadata and our execution store. val statusChange = MetadataEvent(metadataKey(jobKey, CallMetadataKeys.ExecutionStatus), MetadataValue(ExecutionStatus.Starting)) serviceRegistryActor ! PutMetadataAction(statusChange) - stay() using stateData.mergeExecutionDiff(WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Starting))) + stay() using stateData + .mergeExecutionDiff(WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Starting))) case Event(JobRunning(jobDescriptor, backendJobExecutionActor), stateData) => // The EJEA is telling us that the job is now Running. Update the metadata and our execution store. pushRunningJobMetadata(jobDescriptor) stay() using stateData .addBackendJobExecutionActor(jobDescriptor.key, backendJobExecutionActor) .mergeExecutionDiff(WorkflowExecutionDiff(Map(jobDescriptor.key -> ExecutionStatus.Running))) - case Event(BackendJobPreparationFailed(jobKey, throwable), stateData) => - pushFailedJobMetadata(jobKey, None, throwable, retryableFailure = false) - context.parent ! WorkflowExecutionFailedResponse(stateData.executionStore, stateData.outputStore, List(throwable)) - goto(WorkflowExecutionFailedState) using stateData.mergeExecutionDiff(WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Failed))) case Event(SucceededResponse(jobKey, returnCode, callOutputs, _, _), stateData) => pushSuccessfulJobMetadata(jobKey, returnCode, callOutputs) handleJobSuccessful(jobKey, callOutputs, stateData) @@ -402,7 +391,25 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, } } + def handleTerminated(actorRef: ActorRef) = { + // Both of these Should Never Happen (tm), assuming the state data is set correctly on EJEA creation. + // If they do, it's a big programmer error and the workflow execution fails. + val jobKey = stateData.engineJobExecutionActors.getOrElse(actorRef, throw new RuntimeException("Programmer Error: An EJEA has terminated but was not assigned a jobKey")) + val jobStatus = stateData.executionStore.store.getOrElse(jobKey, throw new RuntimeException("Programmer Error: An EJEA representing a jobKey which this workflow is not running has sent up a terminated message.")) + + if (!jobStatus.isTerminal) { + val terminationException = getFailureCause(actorRef) match { + case Some(e) => new RuntimeException("Unexpected failure in EJEA.", e) + case None => new RuntimeException("Unexpected failure in EJEA (root cause not captured).") + } + self ! FailedNonRetryableResponse(jobKey, terminationException, None) + } + + stay + } + whenUnhandled { + case Event(Terminated(actorRef), stateData) => handleTerminated(actorRef) using stateData.removeEngineJobExecutionActor(actorRef) case Event(MetadataPutFailed(action, error), _) => // Do something useful here?? workflowLogger.warn(s"$tag Put failed for Metadata action $action : ${error.getMessage}") @@ -586,7 +593,7 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, private def pushQueuedJobMetadata(diffs: Seq[WorkflowExecutionDiff]) = { val startingEvents = for { diff <- diffs - (jobKey, executionState) <- diff.executionStore if jobKey.isInstanceOf[BackendJobDescriptorKey] && executionState == ExecutionStatus.QueuedInCromwell + (jobKey, executionState) <- diff.executionStoreChanges if jobKey.isInstanceOf[BackendJobDescriptorKey] && executionState == ExecutionStatus.QueuedInCromwell } yield MetadataEvent(metadataKey(jobKey, CallMetadataKeys.ExecutionStatus), MetadataValue(ExecutionStatus.QueuedInCromwell)) serviceRegistryActor ! PutMetadataAction(startingEvents) } @@ -623,9 +630,12 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, self, jobKey, data, factory, initializationData.get(backendName), restarting, serviceRegistryActor, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingleton, backendName, workflowDescriptor.callCachingMode) val ejeaRef = context.actorOf(ejeaProps, ejeaName) + context watch ejeaRef pushNewJobMetadata(jobKey, backendName) ejeaRef ! EngineJobExecutionActor.Execute - Success(WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.QueuedInCromwell))) + Success(WorkflowExecutionDiff( + executionStoreChanges = Map(jobKey -> ExecutionStatus.QueuedInCromwell), + engineJobExecutionActorAdditions = Map(ejeaRef -> jobKey))) case None => throw WorkflowExecutionException(NonEmptyList.of(new Exception(s"Could not get BackendLifecycleActor for backend $backendName"))) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala index 96e4f1658..d06514284 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala @@ -8,18 +8,19 @@ import cromwell.engine.{EngineWorkflowDescriptor, WdlFunctions} import cromwell.util.JsonFormatting.WdlValueJsonFormatter import wdl4s.{GraphNode, Scope} - object WorkflowExecutionDiff { def empty = WorkflowExecutionDiff(Map.empty) } /** Data differential between current execution data, and updates performed in a method that needs to be merged. */ -final case class WorkflowExecutionDiff(executionStore: Map[JobKey, ExecutionStatus]) { - def containsNewEntry = executionStore.exists(_._2 == NotStarted) +final case class WorkflowExecutionDiff(executionStoreChanges: Map[JobKey, ExecutionStatus], + engineJobExecutionActorAdditions: Map[ActorRef, JobKey] = Map.empty) { + def containsNewEntry = executionStoreChanges.exists(_._2 == NotStarted) } case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescriptor, executionStore: ExecutionStore, backendJobExecutionActors: Map[JobKey, ActorRef], + engineJobExecutionActors: Map[ActorRef, JobKey], outputStore: OutputStore) { val expressionLanguageFunctions = new WdlFunctions(workflowDescriptor.pathBuilders) @@ -68,6 +69,10 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript executionStore.store.values.exists(_ == ExecutionStatus.Failed) } + def removeEngineJobExecutionActor(actorRef: ActorRef) = { + this.copy(engineJobExecutionActors = engineJobExecutionActors - actorRef) + } + def addBackendJobExecutionActor(jobKey: JobKey, actor: Option[ActorRef]): WorkflowExecutionActorData = actor match { case Some(actorRef) => this.copy(backendJobExecutionActors = backendJobExecutionActors + (jobKey -> actorRef)) case None => this @@ -91,7 +96,9 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript } def mergeExecutionDiff(diff: WorkflowExecutionDiff): WorkflowExecutionActorData = { - this.copy(executionStore = executionStore.add(diff.executionStore)) + this.copy( + executionStore = executionStore.add(diff.executionStoreChanges), + engineJobExecutionActors = engineJobExecutionActors ++ diff.engineJobExecutionActorAdditions) } def mergeExecutionDiffs(diffs: Traversable[WorkflowExecutionDiff]): WorkflowExecutionActorData = { diff --git a/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala b/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala index b2afd6b5e..99d10221d 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala @@ -84,7 +84,7 @@ class JobExecutionTokenDispenserActor extends Actor with ActorLogging { private def onTerminate(terminee: ActorRef): Unit = { tokenAssignments.get(terminee) match { case Some(token) => - log.error("Actor {} stopped without returning its Job Execution Token. Reclaiming it!", terminee) + log.debug("Actor {} stopped without returning its Job Execution Token. Reclaiming it!", terminee) self.tell(msg = JobExecutionTokenReturn(token), sender = terminee) case None => log.debug("Actor {} stopped while we were still watching it... but it doesn't have a token. Removing it from any queues if necessary", terminee) diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPreparingJobSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPreparingJobSpec.scala index 1d6f2ccbe..8e3288865 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPreparingJobSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPreparingJobSpec.scala @@ -2,6 +2,7 @@ package cromwell.engine.workflow.lifecycle.execution.ejea import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ import EngineJobExecutionActorSpec._ +import cromwell.backend.BackendJobExecutionActor.FailedNonRetryableResponse import cromwell.core.callcaching.CallCachingMode import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor.{BackendJobPreparationFailed, BackendJobPreparationSucceeded} import org.scalatest.concurrent.Eventually @@ -34,10 +35,11 @@ class EjeaPreparingJobSpec extends EngineJobExecutionActorSpec with CanExpectHas } s"Not proceed if Job Preparation fails ($mode)" in { - val prepFailedResponse = BackendJobPreparationFailed(helper.jobDescriptorKey, new Exception("The goggles! They do nothing!")) + val prepActorResponse = BackendJobPreparationFailed(helper.jobDescriptorKey, new Exception("The goggles! They do nothing!")) + val prepFailedEjeaResponse = FailedNonRetryableResponse(prepActorResponse.jobKey, prepActorResponse.throwable, None) ejea = ejeaInPreparingState(mode) - ejea ! prepFailedResponse - helper.replyToProbe.expectMsg(prepFailedResponse) + ejea ! prepActorResponse + helper.replyToProbe.expectMsg(prepFailedEjeaResponse) helper.deathwatch.expectTerminated(ejea, awaitTimeout) } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala index c3178e27f..6896bcc00 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala @@ -119,7 +119,7 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock jobPreparationProbe = jobPreparationProbe, replyTo = replyToProbe.ref, jobDescriptorKey = jobDescriptorKey, - executionData = WorkflowExecutionActorData(descriptor, ExecutionStore(Map.empty), Map.empty, OutputStore(Map.empty)), + executionData = WorkflowExecutionActorData(descriptor, ExecutionStore(Map.empty), Map.empty, Map.empty, OutputStore(Map.empty)), factory = factory, initializationData = None, restarting = restarting, From 0c49459ba0182a38db0ca348daf8ed1047ade800 Mon Sep 17 00:00:00 2001 From: kcibul Date: Mon, 21 Nov 2016 19:33:55 -0500 Subject: [PATCH 065/375] allow specification of compute service account in JES closes #1656 (#1693) * allow specification of compute service account in JES * address PR comments --- README.md | 5 ++++- core/src/main/resources/reference.conf | 6 ++++++ .../impl/jes/JesAsyncBackendJobExecutionActor.scala | 6 ++++++ .../cromwell/backend/impl/jes/JesAttributes.scala | 5 ++++- .../cromwell/backend/impl/jes/JesConfiguration.scala | 1 + .../main/scala/cromwell/backend/impl/jes/Run.scala | 5 +++-- .../backend/impl/jes/JesAttributesSpec.scala | 20 +++++++++++++++----- 7 files changed, 39 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a49a4c774..b9fc082ff 100644 --- a/README.md +++ b/README.md @@ -1197,6 +1197,8 @@ Creating the account will cause the JSON file to be downloaded. The structure o Most importantly, the value of the `client_email` field should go into the `service-account-id` field in the configuration (see below). The `private_key` portion needs to be pulled into its own file (e.g. `my-key.pem`). The `\n`s in the string need to be converted to newline characters. +While technically not part of Service Account authorization mode, one can also override the default service account that the compute VM is started with via the configuration option `JES.config.genomics.compute-service-account` or through the workflow options parameter `google_compute_service_account`. It's important that this service account, and the service account specified in `JES.config.genomics.auth` can both read/write the location specified by `JES.config.root` + #### Refresh Token A **refresh_token** field must be specified in the [workflow options](#workflow-options) when submitting the job. Omitting this field will cause the workflow to fail. @@ -1567,7 +1569,8 @@ Valid keys and their meanings: * The default is `NoNewCalls` but this can be changed using the `workflow-options.workflow-failure-mode` configuration option. * **backend** - Override the default backend specified in the Cromwell configuration for this workflow only. * JES Backend Only - * **jes_gcs_root** - (JES backend only) Specifies where outputs of the workflow will be written. Expects this to be a GCS URL (e.g. `gs://my-bucket/workflows`). If this is not set, this defaults to the value within `backend.jes.root` in the [configuration](#configuring-cromwell). + * **jes_gcs_root** - (JES backend only) Specifies where outputs of the workflow will be written. Expects this to be a GCS URL (e.g. `gs://my-bucket/workflows`). If this is not set, this defaults to the value within `backend.jes.config.root` in the [configuration](#configuring-cromwell). + * **google_compute_service_account** - (JES backend only) Specifies an alternate service account to use on the compute instance (e.g. my-new-svcacct@my-google-project.iam.gserviceaccount.com). If this is not set, this defaults to the value within `backend.jes.config.genomics.compute-service-account` in the [configuration](#configuring-cromwell) if specified or `default` otherwise. * **google_project** - (JES backend only) Specifies which google project to execute this workflow. * **refresh_token** - (JES backend only) Only used if `localizeWithRefreshToken` is specified in the [configuration file](#configuring-cromwell). * **auth_bucket** - (JES backend only) defaults to the the value in **jes_gcs_root**. This should represent a GCS URL that only Cromwell can write to. The Cromwell account is determined by the `google.authScheme` (and the corresponding `google.userAuth` and `google.serviceAuth`) diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index b3693ded7..87bc710b9 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -333,6 +333,12 @@ backend { # # A reference to an auth defined in the `google` stanza at the top. This auth is used to create # # Pipelines and manipulate auth JSONs. # auth = "application-default" + # + # // alternative service account to use on the launched compute instance + # // NOTE: If combined with service account authorization, both that serivce account and this service account + # // must be able to read and write to the 'root' GCS path + # compute-service-account = "default" + # # # Endpoint for APIs, no reason to change this unless directed by Google. # endpoint-url = "https://genomics.googleapis.com/" # } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala index 31286ed68..1e86782c8 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala @@ -55,6 +55,7 @@ object JesAsyncBackendJobExecutionActor { object WorkflowOptionKeys { val MonitoringScript = "monitoring_script" val GoogleProject = "google_project" + val GoogleComputeServiceAccount = "google_compute_service_account" } @@ -307,6 +308,10 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes descriptor.workflowOptions.getOrElse(WorkflowOptionKeys.GoogleProject, jesAttributes.project) } + private def computeServiceAccount(descriptor: BackendWorkflowDescriptor): String = { + descriptor.workflowOptions.getOrElse(WorkflowOptionKeys.GoogleComputeServiceAccount, jesAttributes.computeServiceAccount) + } + private def createJesRun(jesParameters: Seq[JesParameter], runIdForResumption: Option[String]): Future[Run] = { def createRun() = Future(Run( @@ -318,6 +323,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes logFileName = jesLogFilename, jesParameters, googleProject(jobDescriptor.workflowDescriptor), + computeServiceAccount(jobDescriptor.workflowDescriptor), retryable, initializationData.genomics )) diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala index 066a4cb52..be7bcffa3 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala @@ -14,6 +14,7 @@ import net.ceedubs.ficus.Ficus._ import wdl4s.ExceptionWithErrors case class JesAttributes(project: String, + computeServiceAccount: String, auths: JesAuths, executionBucket: String, endpointUrl: URL, @@ -25,6 +26,7 @@ object JesAttributes { "project", "root", "maximum-polling-interval", + "compute-service-account", "dockerhub", "genomics", "filesystems", @@ -42,6 +44,7 @@ object JesAttributes { val executionBucket: ValidatedNel[String, String] = backendConfig.validateString("root") val endpointUrl: ErrorOr[URL] = backendConfig.validateURL("genomics.endpoint-url") val maxPollingInterval: Int = backendConfig.as[Option[Int]]("maximum-polling-interval").getOrElse(600) + val computeServiceAccount: String = backendConfig.as[Option[String]]("genomics.compute-service-account").getOrElse("default") val genomicsAuthName: ErrorOr[String] = backendConfig.validateString("genomics.auth") val gcsFilesystemAuthName: ErrorOr[String] = backendConfig.validateString("filesystems.gcs.auth") @@ -49,7 +52,7 @@ object JesAttributes { (_, _, _, _, _) } flatMap { case (p, b, u, genomicsName, gcsName) => (googleConfig.auth(genomicsName) |@| googleConfig.auth(gcsName)) map { case (genomicsAuth, gcsAuth) => - JesAttributes(p, JesAuths(genomicsAuth, gcsAuth), b, u, maxPollingInterval) + JesAttributes(p, computeServiceAccount, JesAuths(genomicsAuth, gcsAuth), b, u, maxPollingInterval) } } match { case Valid(r) => r diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala index 88ece5bec..bd0e14b1c 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala @@ -28,6 +28,7 @@ class JesConfiguration(val configurationDescriptor: BackendConfigurationDescript val root = configurationDescriptor.backendConfig.getString("root") val jesAttributes = JesAttributes(googleConfig, configurationDescriptor.backendConfig) val jesAuths = jesAttributes.auths + val jesComputeServiceAccount = jesAttributes.computeServiceAccount val gcsPathBuilderFactory = RetryableGcsPathBuilderFactory(jesAuths.gcs, customRetryParams = JesConfiguration.GcsRetryParams) val genomicsFactory = GenomicsFactory(googleConfig.applicationName, jesAuths.genomics, jesAttributes.endpointUrl) val dockerCredentials = DockerConfiguration.build(configurationDescriptor.backendConfig).dockerCredentials map JesDockerCredentials.apply diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala index b5a8b5f9f..2bcad42d3 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala @@ -21,7 +21,6 @@ object Run { "https://www.googleapis.com/auth/compute" ).asJava - private val JesServiceAccount = new ServiceAccount().setEmail("default").setScopes(GenomicsScopes) private val AcceptableEvents = Set("start", "pulling-image", "localizing-files", "running-docker", "delocalizing-files", "ok", "fail", "start-shutdown", "preempted") val NoAddressFieldName = "noAddress" @@ -36,6 +35,7 @@ object Run { logFileName: String, jesParameters: Seq[JesParameter], projectId: String, + computeServiceAccount: String, preemptible: Boolean, genomicsInterface: Genomics): Run = { val logger = new JobLogger("JesRun", jobDescriptor.workflowDescriptor.id, jobDescriptor.key.tag, None, Set(slf4jLogger)) @@ -60,7 +60,8 @@ object Run { } def runPipeline: String = { - val rpargs = new RunPipelineArgs().setProjectId(projectId).setServiceAccount(JesServiceAccount).setResources(runtimePipelineResources) + val svcAccount = new ServiceAccount().setEmail(computeServiceAccount).setScopes(GenomicsScopes) + val rpargs = new RunPipelineArgs().setProjectId(projectId).setServiceAccount(svcAccount).setResources(runtimePipelineResources) rpargs.setInputs(jesParameters.collect({ case i: JesInput => i.name -> i.toGoogleRunParameter }).toMap.asJava) logger.debug(s"Inputs:\n${stringifyMap(rpargs.getInputs.asScala.toMap)}") diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala index 17c33d0de..0eec23cbe 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala @@ -16,18 +16,19 @@ class JesAttributesSpec extends FlatSpec with Matchers { it should "parse correct JES config" taggedAs IntegrationTest in { val googleConfig = GoogleConfiguration(JesGlobalConfig) - val backendConfig = ConfigFactory.parseString(configString.replace("[PREEMPTIBLE]", "")) + val backendConfig = ConfigFactory.parseString(configString()) val jesAttributes = JesAttributes(googleConfig, backendConfig) jesAttributes.endpointUrl should be(new URL("http://myEndpoint")) jesAttributes.project should be("myProject") jesAttributes.executionBucket should be("gs://myBucket") jesAttributes.maxPollingInterval should be(600) + jesAttributes.computeServiceAccount should be("default") } it should "parse correct preemptible config" taggedAs IntegrationTest in { val googleConfig = GoogleConfiguration(JesGlobalConfig) - val backendConfig = ConfigFactory.parseString(configString.replace("[PREEMPTIBLE]", "preemptible = 3")) + val backendConfig = ConfigFactory.parseString(configString(preemptible = "preemptible = 3")) val jesAttributes = JesAttributes(googleConfig, backendConfig) jesAttributes.endpointUrl should be(new URL("http://myEndpoint")) @@ -36,6 +37,14 @@ class JesAttributesSpec extends FlatSpec with Matchers { jesAttributes.maxPollingInterval should be(600) } + it should "parse compute service account" taggedAs IntegrationTest in { + val googleConfig = GoogleConfiguration(JesGlobalConfig) + val backendConfig = ConfigFactory.parseString(configString(genomics = """compute-service-account = "testing" """)) + + val jesAttributes = JesAttributes(googleConfig, backendConfig) + jesAttributes.computeServiceAccount should be("testing") + } + it should "not parse invalid config" taggedAs IntegrationTest in { val nakedConfig = ConfigFactory.parseString( @@ -60,17 +69,18 @@ class JesAttributesSpec extends FlatSpec with Matchers { errorsList should contain("no protocol: myEndpoint") } - val configString = - """ + def configString(preemptible: String = "", genomics: String = "") = + s""" |{ | project = "myProject" | root = "gs://myBucket" | maximum-polling-interval = 600 - | [PREEMPTIBLE] + | $preemptible | genomics { | // A reference to an auth defined in the `google` stanza at the top. This auth is used to create | // Pipelines and manipulate auth JSONs. | auth = "application-default" + | $genomics | endpoint-url = "http://myEndpoint" | } | From 1fc26d134f61276aba968b6512cb92f0f071fc20 Mon Sep 17 00:00:00 2001 From: Thib Date: Tue, 22 Nov 2016 13:35:34 -0500 Subject: [PATCH 066/375] Sub Workflows closes #1532 #1473 (#1682) * Sub Workflows --- CHANGELOG.md | 1 + README.md | 456 +++++++++++++ .../backend/BackendCacheHitCopyingActor.scala | 4 +- .../cromwell/backend/BackendJobBreadCrumb.scala | 14 + .../backend/BackendJobExecutionActor.scala | 14 +- .../cromwell/backend/BackendLifecycleActor.scala | 6 +- .../backend/BackendLifecycleActorFactory.scala | 23 +- .../BackendWorkflowInitializationActor.scala | 7 +- .../async/AsyncBackendJobExecutionActor.scala | 10 +- .../cromwell/backend/async/ExecutionHandle.scala | 4 +- .../cromwell/backend/async/ExecutionResult.scala | 4 +- .../src/main/scala/cromwell/backend/backend.scala | 27 +- .../backend/callcaching/CacheHitDuplicating.scala | 4 +- .../main/scala/cromwell/backend/io/JobPaths.scala | 70 +- .../cromwell/backend/io/JobPathsWithDocker.scala | 39 ++ .../scala/cromwell/backend/io/WorkflowPaths.scala | 34 +- .../backend/io/WorkflowPathsWithDocker.scala | 16 + .../src/main/scala/cromwell/backend/package.scala | 12 +- .../cromwell/backend/wfs/WorkflowPathBuilder.scala | 4 +- .../test/scala/cromwell/backend/BackendSpec.scala | 18 +- .../scala/cromwell/backend/io/JobPathsSpec.scala | 12 +- .../cromwell/backend/io/WorkflowPathsSpec.scala | 44 +- core/src/main/scala/cromwell/core/CallKey.scala | 7 + .../main/scala/cromwell/core/ExecutionStore.scala | 13 - core/src/main/scala/cromwell/core/JobKey.scala | 2 + .../scala/cromwell/core/WorkflowMetadataKeys.scala | 1 + .../scala/cromwell/core/logging/JobLogger.scala | 2 +- .../cromwell/core/logging/WorkflowLogger.scala | 8 +- core/src/main/scala/cromwell/core/package.scala | 2 +- .../cromwell/core/simpleton/WdlValueBuilder.scala | 4 +- core/src/test/scala/cromwell/util/SampleWdl.scala | 3 +- .../migration/src/main/resources/changelog.xml | 1 + .../resources/changesets/sub_workflow_store.xml | 62 ++ .../cromwell/database/slick/SlickDatabase.scala | 3 +- .../slick/SubWorkflowStoreSlickDatabase.scala | 67 ++ .../slick/tables/DataAccessComponent.scala | 6 +- .../tables/SubWorkflowStoreEntryComponent.scala | 62 ++ .../scala/cromwell/database/sql/SqlDatabase.scala | 3 +- .../database/sql/SubWorkflowStoreSqlDatabase.scala | 21 + .../sql/tables/SubWorkflowStoreEntry.scala | 12 + engine/src/main/resources/swagger/cromwell.yaml | 7 + .../resources/workflowTimings/workflowTimings.html | 181 +++-- .../cromwell/engine/EngineWorkflowDescriptor.scala | 21 +- .../workflow/SingleWorkflowRunnerActor.scala | 12 +- .../cromwell/engine/workflow/WorkflowActor.scala | 68 +- .../engine/workflow/WorkflowManagerActor.scala | 6 +- .../lifecycle/CopyWorkflowOutputsActor.scala | 39 +- .../MaterializeWorkflowDescriptorActor.scala | 26 +- .../lifecycle/WorkflowFinalizationActor.scala | 28 +- .../lifecycle/WorkflowInitializationActor.scala | 2 +- .../lifecycle/execution/CallMetadataHelper.scala | 135 ++++ .../execution/EngineJobExecutionActor.scala | 116 ++-- .../lifecycle/execution/ExecutionStore.scala | 103 +++ .../lifecycle/execution/JobPreparationActor.scala | 101 ++- .../lifecycle/execution}/OutputStore.scala | 30 +- .../execution/SubWorkflowExecutionActor.scala | 272 ++++++++ .../execution/WorkflowExecutionActor.scala | 750 +++++++++------------ .../execution/WorkflowExecutionActorData.scala | 74 +- .../execution/WorkflowMetadataHelper.scala | 37 + .../execution/callcaching/CallCache.scala | 4 +- .../callcaching/CallCacheWriteActor.scala | 6 +- .../workflow/lifecycle/execution/package.scala | 12 +- .../main/scala/cromwell/jobstore/jobstore_.scala | 2 +- .../scala/cromwell/server/CromwellRootActor.scala | 6 +- .../EmptySubWorkflowStoreActor.scala | 17 + .../subworkflowstore/SqlSubWorkflowStore.scala | 31 + .../subworkflowstore/SubWorkflowStore.scala | 19 + .../subworkflowstore/SubWorkflowStoreActor.scala | 72 ++ .../cromwell/webservice/CromwellApiService.scala | 19 +- .../cromwell/webservice/EngineStatsActor.scala | 10 +- .../webservice/metadata/IndexedJsonValue.scala | 22 +- .../webservice/metadata/MetadataBuilderActor.scala | 122 +++- .../scala/cromwell/ArrayOfArrayCoercionSpec.scala | 2 +- .../test/scala/cromwell/ArrayWorkflowSpec.scala | 8 +- .../scala/cromwell/CopyWorkflowOutputsSpec.scala | 4 +- .../test/scala/cromwell/CromwellTestKitSpec.scala | 11 +- .../scala/cromwell/FilePassingWorkflowSpec.scala | 12 +- .../src/test/scala/cromwell/MapWorkflowSpec.scala | 6 +- .../MultipleFilesWithSameNameWorkflowSpec.scala | 4 +- .../cromwell/PostfixQuantifierWorkflowSpec.scala | 10 +- .../test/scala/cromwell/ScatterWorkflowSpec.scala | 38 +- .../scala/cromwell/SimpleWorkflowActorSpec.scala | 1 + .../cromwell/WdlFunctionsAtWorkflowLevelSpec.scala | 4 +- .../test/scala/cromwell/WorkflowOutputsSpec.scala | 16 +- .../cromwell/engine/WorkflowManagerActorSpec.scala | 2 +- .../mock/DefaultBackendJobExecutionActor.scala | 8 +- .../mock/RetryableBackendJobExecutionActor.scala | 6 +- .../RetryableBackendLifecycleActorFactory.scala | 4 +- .../workflow/SingleWorkflowRunnerActorSpec.scala | 4 +- .../engine/workflow/WorkflowActorSpec.scala | 16 +- .../MaterializeWorkflowDescriptorActorSpec.scala | 2 +- .../execution/SubWorkflowExecutionActorSpec.scala | 213 ++++++ .../execution/WorkflowExecutionActorSpec.scala | 10 +- .../callcaching/EngineJobHashingActorSpec.scala | 2 +- .../execution/ejea/EjeaCheckingJobStoreSpec.scala | 14 +- .../execution/ejea/EjeaPreparingJobSpec.scala | 12 +- .../ejea/EjeaRequestingExecutionTokenSpec.scala | 4 +- .../execution/ejea/EjeaUpdatingJobStoreSpec.scala | 4 +- .../ejea/EngineJobExecutionActorSpecUtil.scala | 10 +- .../lifecycle/execution/ejea/PerTestHelper.scala | 61 +- .../engine/workflow/mocks/DeclarationMock.scala | 21 + .../cromwell/engine/workflow/mocks/TaskMock.scala | 27 + .../engine/workflow/mocks/WdlExpressionMock.scala | 32 + .../cromwell/jobstore/JobStoreServiceSpec.scala | 6 +- .../subworkflowstore/SubWorkflowStoreSpec.scala | 87 +++ .../cromwell/webservice/EngineStatsActorSpec.scala | 10 +- .../webservice/MetadataBuilderActorSpec.scala | 110 ++- project/Dependencies.scala | 2 +- .../services/metadata/CallMetadataKeys.scala | 2 + .../cromwell/services/metadata/MetadataQuery.scala | 11 +- .../services/metadata/MetadataService.scala | 3 +- .../metadata/impl/MetadataDatabaseAccess.scala | 14 +- .../services/metadata/impl/ReadMetadataActor.scala | 11 +- .../impl/htcondor/HtCondorBackendFactory.scala | 8 +- .../htcondor/HtCondorInitializationActor.scala | 6 +- .../impl/htcondor/HtCondorJobExecutionActor.scala | 28 +- .../backend/impl/htcondor/caching/CacheActor.scala | 6 +- .../localization/CachedResultLocalization.scala | 2 +- .../caching/model/CachedExecutionResult.scala | 4 +- .../caching/provider/mongodb/MongoCacheActor.scala | 4 +- .../htcondor/HtCondorInitializationActorSpec.scala | 6 +- .../htcondor/HtCondorJobExecutionActorSpec.scala | 30 +- .../htcondor/HtCondorRuntimeAttributesSpec.scala | 2 +- .../CachedResultLocalizationSpec.scala | 4 +- .../provider/mongodb/MongoCacheActorSpec.scala | 6 +- .../jes/JesAsyncBackendJobExecutionActor.scala | 34 +- .../impl/jes/JesBackendLifecycleActorFactory.scala | 25 +- .../backend/impl/jes/JesCacheHitCopyingActor.scala | 4 +- .../cromwell/backend/impl/jes/JesCallPaths.scala | 78 --- .../backend/impl/jes/JesFinalizationActor.scala | 30 +- .../backend/impl/jes/JesInitializationActor.scala | 10 +- .../impl/jes/JesJobCachingActorHelper.scala | 41 +- .../cromwell/backend/impl/jes/JesJobPaths.scala | 60 ++ .../backend/impl/jes/JesWorkflowPaths.scala | 42 +- .../main/scala/cromwell/backend/impl/jes/Run.scala | 2 +- .../jes/JesAsyncBackendJobExecutionActorSpec.scala | 81 ++- .../backend/impl/jes/JesCallPathsSpec.scala | 12 +- .../impl/jes/JesInitializationActorSpec.scala | 10 +- .../impl/jes/JesJobExecutionActorSpec.scala | 6 +- .../backend/impl/jes/JesWorkflowPathsSpec.scala | 4 +- .../SharedFileSystemAsyncJobExecutionActor.scala | 4 +- ...redFileSystemBackendLifecycleActorFactory.scala | 6 +- .../sfs/SharedFileSystemExpressionFunctions.scala | 6 +- .../sfs/SharedFileSystemInitializationActor.scala | 8 +- .../SharedFileSystemJobCachingActorHelper.scala | 4 +- .../SharedFileSystemInitializationActorSpec.scala | 6 +- .../SharedFileSystemJobExecutionActorSpec.scala | 36 +- .../sfs/TestLocalAsyncJobExecutionActor.scala | 4 +- .../backend/impl/spark/SparkBackendFactory.scala | 8 +- .../impl/spark/SparkInitializationActor.scala | 6 +- .../impl/spark/SparkJobExecutionActor.scala | 24 +- .../impl/spark/SparkInitializationActorSpec.scala | 5 +- .../impl/spark/SparkJobExecutionActorSpec.scala | 42 +- .../impl/spark/SparkRuntimeAttributesSpec.scala | 4 +- 154 files changed, 3531 insertions(+), 1453 deletions(-) create mode 100644 backend/src/main/scala/cromwell/backend/BackendJobBreadCrumb.scala create mode 100644 backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala create mode 100644 backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala create mode 100644 core/src/main/scala/cromwell/core/CallKey.scala delete mode 100644 core/src/main/scala/cromwell/core/ExecutionStore.scala create mode 100644 database/migration/src/main/resources/changesets/sub_workflow_store.xml create mode 100644 database/sql/src/main/scala/cromwell/database/slick/SubWorkflowStoreSlickDatabase.scala create mode 100644 database/sql/src/main/scala/cromwell/database/slick/tables/SubWorkflowStoreEntryComponent.scala create mode 100644 database/sql/src/main/scala/cromwell/database/sql/SubWorkflowStoreSqlDatabase.scala create mode 100644 database/sql/src/main/scala/cromwell/database/sql/tables/SubWorkflowStoreEntry.scala create mode 100644 engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala create mode 100644 engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala rename {core/src/main/scala/cromwell/core => engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution}/OutputStore.scala (51%) create mode 100644 engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala create mode 100644 engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowMetadataHelper.scala create mode 100644 engine/src/main/scala/cromwell/subworkflowstore/EmptySubWorkflowStoreActor.scala create mode 100644 engine/src/main/scala/cromwell/subworkflowstore/SqlSubWorkflowStore.scala create mode 100644 engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStore.scala create mode 100644 engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStoreActor.scala create mode 100644 engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala create mode 100644 engine/src/test/scala/cromwell/engine/workflow/mocks/DeclarationMock.scala create mode 100644 engine/src/test/scala/cromwell/engine/workflow/mocks/TaskMock.scala create mode 100644 engine/src/test/scala/cromwell/engine/workflow/mocks/WdlExpressionMock.scala create mode 100644 engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala delete mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCallPaths.scala create mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala diff --git a/CHANGELOG.md b/CHANGELOG.md index a9058f66e..b4e83829a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ * `transpose: (Array[Array[X]]) => Array[Array[X]]` compute the matrix transpose for a 2D array. Assumes each inner array has the same length. * By default, `system.abort-jobs-on-terminate` is false when running `java -jar cromwell.jar server`, and true when running `java -jar cromwell.jar run `. * Enable WDL imports when running in Single Workflow Runner Mode. +* Support for sub workflows ## 0.22 diff --git a/README.md b/README.md index b9fc082ff..782d790e8 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management * [Configuring Call Caching](#configuring-call-caching) * [Call Caching Workflow Options](#call-caching-workflow-options) * [Local Filesystem Options](#local-filesystem-options) +* [Sub Workflows](#sub-workflows) * [REST API](#rest-api) * [REST API Versions](#rest-api-versions) * [POST /api/workflows/:version](#post-apiworkflowsversion) @@ -1642,6 +1643,461 @@ When running a job on the Config (Shared Filesystem) backend, Cromwell provides } } ``` +# Sub Workflows + +WDL allows the execution of an entire workflow as a step in a larger workflow (see WDL SPEC for more details), which is what will be referred to as a sub workflow going forward. +Cromwell supports execution of such workflows. Note that sub workflows can themselves contain sub workflows, etc... There is no limitation as to how deeply workflows can be nested. + +## Execution + +Sub workflows are executed exactly as a task would be. +*This means that if another call depends on an output of a sub workflow, this call will run when the whole sub workflow completes (successfully).* +For example, in the following case : + +`main.wdl` +``` +import "sub_wdl.wdl" as sub + +workflow main_workflow { + + call sub.hello_and_goodbye { input: hello_and_goodbye_input = "sub world" } + + # call myTask { input: hello_and_goodbye.hello_output } + + output { + String main_output = hello_and_goodbye.hello_output + } +} +``` + +`sub_wdl.wdl` +``` +task hello { + String addressee + command { + echo "Hello ${addressee}!" + } + runtime { + docker: "ubuntu:latest" + } + output { + String salutation = read_string(stdout()) + } +} + +task goodbye { + String addressee + command { + echo "Goodbye ${addressee}!" + } + runtime { + docker: "ubuntu:latest" + } + output { + String salutation = read_string(stdout()) + } +} + +workflow hello_and_goodbye { + String hello_and_goodbye_input + + call hello {input: addressee = hello_and_goodbye_input } + call goodbye {input: addressee = hello_and_goodbye_input } + + output { + String hello_output = hello.salutation + String goodbye_output = goodbye.salutation + } +} +``` + +`myTask` will start only when hello_and_goodbye completes (which means all of its calls are done), even though `myTask` only needs the output of hello in the hello_and_goodbye sub workflow. +If hello_and_goodbye fails, then `myTask` won't be executed. +Only workflow outputs are visible outside a workflow, which means that references to outputs produced by a sub workflow will only be valid if those outputs are exposed in the workflow output section. + +Sub workflows are executed in the context of a main workflow, which means that operations that are normally executed once per workflow (set up, clean up, outputs copying, log copying, etc...) +will NOT be re-executed for each sub workflow. For instance if a resource is created during workflow initialization, sub workflows will need to share this same resource. +Workflow outputs will be copied for the main root workflow but not for intermediate sub workflows. + +Restarts, aborts, and call-caching work exactly as they would with tasks. +All tasks run by a sub workflow are eligible for call caching under the same rules as any other task. +However, workflows themselves are not cached as such. Which means that running the exact same workflow twice with call caching on will trigger each task to cache individually, +but not the workflow itself. + +The root path for sub workflow execution files (scripts, output files, logs) will be under the parent workflow call directory. +For example, the execution directory for the above main workflow would look like the following: + +``` +cromwell-executions/main_workflow/1d919bd4-d046-43b0-9918-9964509689dd/ <- main workflow id +└── call-hello_and_goodbye <- call directory for call hello_and_goodbye in the main workflow + └── hello_and_goodbye <- name of the sub workflow + └── a6365f91-c807-465a-9186-a5d3da98fe11 <- sub workflow id + ├── call-goodbye + │   └── execution + │   ├── rc + │   ├── script + │   ├── script.background + │   ├── script.submit + │   ├── stderr + │   ├── stderr.background + │   ├── stdout + │   └── stdout.background + └── call-hello + └── execution + ├── rc + ├── script + ├── script.background + ├── script.submit + ├── stderr + ├── stderr.background + ├── stdout + └── stdout.background + +``` + +## Metadata +Each sub workflow will have its own workflow ID. This ID will appear in the metadata of the parent workflow, in the call section corresponding to the sub workflow, under the "subWorkflowId" attribute. +For example, querying the `main_workflow` metadata above (minus the `myTask` call) , could result in something like this: + +`GET /api/workflows/v2/1d919bd4-d046-43b0-9918-9964509689dd/metadata` + +``` +{ + "workflowName": "main_workflow", + "submittedFiles": { + "inputs": "{}", + "workflow": "import \"sub_wdl.wdl\" as sub\n\nworkflow main_workflow {\n\n call sub.hello_and_goodbye { input: hello_and_goodbye_input = \"sub world\" }\n \n # call myTask { input: hello_and_goodbye.hello_output }\n \n output {\n String main_output = hello_and_goodbye.hello_output\n }\n}", + "options": "{\n\n}" + }, + "calls": { + "main_workflow.hello_and_goodbye": [ + { + "executionStatus": "Done", + "shardIndex": -1, + "outputs": { + "goodbye_output": "Goodbye sub world!", + "hello_output": "Hello sub world!" + }, + "inputs": { + "hello_and_goodbye_input": "sub world" + }, + "end": "2016-11-17T14:13:41.117-05:00", + "attempt": 1, + "start": "2016-11-17T14:13:39.236-05:00", + "subWorkflowId": "a6365f91-c807-465a-9186-a5d3da98fe11" + } + ] + }, + "outputs": { + "main_output": "Hello sub world!" + }, + "workflowRoot": "/cromwell-executions/main_workflow/1d919bd4-d046-43b0-9918-9964509689dd", + "id": "1d919bd4-d046-43b0-9918-9964509689dd", + "inputs": {}, + "submission": "2016-11-17T14:13:39.104-05:00", + "status": "Succeeded", + "end": "2016-11-17T14:13:41.120-05:00", + "start": "2016-11-17T14:13:39.204-05:00" +} +``` + +The sub workflow ID can be queried separately: + +`GET /api/workflows/v2/a6365f91-c807-465a-9186-a5d3da98fe11/metadata` + +``` +{ + "workflowName": "hello_and_goodbye", + "calls": { + "sub.hello_and_goodbye.hello": [ + { + "executionStatus": "Done", + "stdout": "/cromwell-executions/main_workflow/1d919bd4-d046-43b0-9918-9964509689dd/call-hello_and_goodbye/hello_and_goodbye/a6365f91-c807-465a-9186-a5d3da98fe11/call-hello/execution/stdout", + "shardIndex": -1, + "outputs": { + "salutation": "Hello sub world!" + }, + "runtimeAttributes": { + "docker": "ubuntu:latest", + "failOnStderr": false, + "continueOnReturnCode": "0" + }, + "cache": { + "allowResultReuse": true + }, + "Effective call caching mode": "CallCachingOff", + "inputs": { + "addressee": "sub world" + }, + "returnCode": 0, + "jobId": "49830", + "backend": "Local", + "end": "2016-11-17T14:13:40.712-05:00", + "stderr": "/cromwell-executions/main_workflow/1d919bd4-d046-43b0-9918-9964509689dd/call-hello_and_goodbye/hello_and_goodbye/a6365f91-c807-465a-9186-a5d3da98fe11/call-hello/execution/stderr", + "callRoot": "/cromwell/cromwell-executions/main_workflow/1d919bd4-d046-43b0-9918-9964509689dd/call-hello_and_goodbye/hello_and_goodbye/a6365f91-c807-465a-9186-a5d3da98fe11/call-hello", + "attempt": 1, + "executionEvents": [ + { + "startTime": "2016-11-17T14:13:39.240-05:00", + "description": "Pending", + "endTime": "2016-11-17T14:13:39.240-05:00" + }, + { + "startTime": "2016-11-17T14:13:39.240-05:00", + "description": "RequestingExecutionToken", + "endTime": "2016-11-17T14:13:39.240-05:00" + }, + { + "startTime": "2016-11-17T14:13:39.240-05:00", + "description": "PreparingJob", + "endTime": "2016-11-17T14:13:39.243-05:00" + }, + { + "startTime": "2016-11-17T14:13:39.243-05:00", + "description": "RunningJob", + "endTime": "2016-11-17T14:13:40.704-05:00" + }, + { + "startTime": "2016-11-17T14:13:40.704-05:00", + "description": "UpdatingJobStore", + "endTime": "2016-11-17T14:13:40.712-05:00" + } + ], + "start": "2016-11-17T14:13:39.239-05:00" + } + ], + "sub.hello_and_goodbye.goodbye": [ + { + "executionStatus": "Done", + "stdout": "/cromwell-executions/main_workflow/1d919bd4-d046-43b0-9918-9964509689dd/call-hello_and_goodbye/hello_and_goodbye/a6365f91-c807-465a-9186-a5d3da98fe11/call-goodbye/execution/stdout", + "shardIndex": -1, + "outputs": { + "salutation": "Goodbye sub world!" + }, + "runtimeAttributes": { + "docker": "ubuntu:latest", + "failOnStderr": false, + "continueOnReturnCode": "0" + }, + "cache": { + "allowResultReuse": true + }, + "Effective call caching mode": "CallCachingOff", + "inputs": { + "addressee": "sub world" + }, + "returnCode": 0, + "jobId": "49831", + "backend": "Local", + "end": "2016-11-17T14:13:41.115-05:00", + "stderr": "/cromwell-executions/main_workflow/1d919bd4-d046-43b0-9918-9964509689dd/call-hello_and_goodbye/hello_and_goodbye/a6365f91-c807-465a-9186-a5d3da98fe11/call-goodbye/execution/stderr", + "callRoot": "/cromwell-executions/main_workflow/1d919bd4-d046-43b0-9918-9964509689dd/call-hello_and_goodbye/hello_and_goodbye/a6365f91-c807-465a-9186-a5d3da98fe11/call-goodbye", + "attempt": 1, + "executionEvents": [ + { + "startTime": "2016-11-17T14:13:39.240-05:00", + "description": "Pending", + "endTime": "2016-11-17T14:13:39.240-05:00" + }, + { + "startTime": "2016-11-17T14:13:39.240-05:00", + "description": "RequestingExecutionToken", + "endTime": "2016-11-17T14:13:39.240-05:00" + }, + { + "startTime": "2016-11-17T14:13:39.240-05:00", + "description": "PreparingJob", + "endTime": "2016-11-17T14:13:39.243-05:00" + }, + { + "startTime": "2016-11-17T14:13:39.243-05:00", + "description": "RunningJob", + "endTime": "2016-11-17T14:13:41.112-05:00" + }, + { + "startTime": "2016-11-17T14:13:41.112-05:00", + "description": "UpdatingJobStore", + "endTime": "2016-11-17T14:13:41.115-05:00" + } + ], + "start": "2016-11-17T14:13:39.239-05:00" + } + ] + }, + "outputs": { + "goodbye_output": "Goodbye sub world!", + "hello_output": "Hello sub world!" + }, + "workflowRoot": "/cromwell-executions/main_workflow/1d919bd4-d046-43b0-9918-9964509689dd/call-hello_and_goodbye/hello_and_goodbye/a6365f91-c807-465a-9186-a5d3da98fe11", + "id": "a6365f91-c807-465a-9186-a5d3da98fe11", + "inputs": { + "hello_and_goodbye_input": "sub world" + }, + "status": "Succeeded", + "parentWorkflowId": "1d919bd4-d046-43b0-9918-9964509689dd", + "end": "2016-11-17T14:13:41.116-05:00", + "start": "2016-11-17T14:13:39.236-05:00" +} +``` + +It's also possible to set the URL query parameter `expandSubWorkflows` to `true` to automatically include sub workflows metadata (`false` by default). + +`GET api/workflows/v2/1d919bd4-d046-43b0-9918-9964509689dd/metadata?expandSubWorkflows=true` + +``` +{ + "workflowName": "main_workflow", + "submittedFiles": { + "inputs": "{}", + "workflow": "import \"sub_wdl.wdl\" as sub\n\nworkflow main_workflow {\n\n call sub.hello_and_goodbye { input: hello_and_goodbye_input = \"sub world\" }\n \n # call myTask { input: hello_and_goodbye.hello_output }\n \n output {\n String main_output = hello_and_goodbye.hello_output\n }\n}", + "options": "{\n\n}" + }, + "calls": { + "main_workflow.hello_and_goodbye": [{ + "executionStatus": "Done", + "subWorkflowMetadata": { + "workflowName": "hello_and_goodbye", + "calls": { + "sub.hello_and_goodbye.hello": [{ + "executionStatus": "Done", + "stdout": "/cromwell-executions/main_workflow/1d919bd4-d046-43b0-9918-9964509689dd/call-hello_and_goodbye/hello_and_goodbye/a6365f91-c807-465a-9186-a5d3da98fe11/call-hello/execution/stdout", + "shardIndex": -1, + "outputs": { + "salutation": "Hello sub world!" + }, + "runtimeAttributes": { + "docker": "ubuntu:latest", + "failOnStderr": false, + "continueOnReturnCode": "0" + }, + "cache": { + "allowResultReuse": true + }, + "Effective call caching mode": "CallCachingOff", + "inputs": { + "addressee": "sub world" + }, + "returnCode": 0, + "jobId": "49830", + "backend": "Local", + "end": "2016-11-17T14:13:40.712-05:00", + "stderr": "/cromwell-executions/main_workflow/1d919bd4-d046-43b0-9918-9964509689dd/call-hello_and_goodbye/hello_and_goodbye/a6365f91-c807-465a-9186-a5d3da98fe11/call-hello/execution/stderr", + "callRoot": "cromwell-executions/main_workflow/1d919bd4-d046-43b0-9918-9964509689dd/call-hello_and_goodbye/hello_and_goodbye/a6365f91-c807-465a-9186-a5d3da98fe11/call-hello", + "attempt": 1, + "executionEvents": [{ + "startTime": "2016-11-17T14:13:39.240-05:00", + "description": "Pending", + "endTime": "2016-11-17T14:13:39.240-05:00" + }, { + "startTime": "2016-11-17T14:13:39.240-05:00", + "description": "RequestingExecutionToken", + "endTime": "2016-11-17T14:13:39.240-05:00" + }, { + "startTime": "2016-11-17T14:13:39.240-05:00", + "description": "PreparingJob", + "endTime": "2016-11-17T14:13:39.243-05:00" + }, { + "startTime": "2016-11-17T14:13:39.243-05:00", + "description": "RunningJob", + "endTime": "2016-11-17T14:13:40.704-05:00" + }, { + "startTime": "2016-11-17T14:13:40.704-05:00", + "description": "UpdatingJobStore", + "endTime": "2016-11-17T14:13:40.712-05:00" + }], + "start": "2016-11-17T14:13:39.239-05:00" + }], + "sub.hello_and_goodbye.goodbye": [{ + "executionStatus": "Done", + "stdout": "/cromwell-executions/main_workflow/1d919bd4-d046-43b0-9918-9964509689dd/call-hello_and_goodbye/hello_and_goodbye/a6365f91-c807-465a-9186-a5d3da98fe11/call-goodbye/execution/stdout", + "shardIndex": -1, + "outputs": { + "salutation": "Goodbye sub world!" + }, + "runtimeAttributes": { + "docker": "ubuntu:latest", + "failOnStderr": false, + "continueOnReturnCode": "0" + }, + "cache": { + "allowResultReuse": true + }, + "Effective call caching mode": "CallCachingOff", + "inputs": { + "addressee": "sub world" + }, + "returnCode": 0, + "jobId": "49831", + "backend": "Local", + "end": "2016-11-17T14:13:41.115-05:00", + "stderr": "/cromwell-executions/main_workflow/1d919bd4-d046-43b0-9918-9964509689dd/call-hello_and_goodbye/hello_and_goodbye/a6365f91-c807-465a-9186-a5d3da98fe11/call-goodbye/execution/stderr", + "callRoot": "/cromwell-executions/main_workflow/1d919bd4-d046-43b0-9918-9964509689dd/call-hello_and_goodbye/hello_and_goodbye/a6365f91-c807-465a-9186-a5d3da98fe11/call-goodbye", + "attempt": 1, + "executionEvents": [{ + "startTime": "2016-11-17T14:13:39.240-05:00", + "description": "Pending", + "endTime": "2016-11-17T14:13:39.240-05:00" + }, { + "startTime": "2016-11-17T14:13:39.240-05:00", + "description": "RequestingExecutionToken", + "endTime": "2016-11-17T14:13:39.240-05:00" + }, { + "startTime": "2016-11-17T14:13:39.240-05:00", + "description": "PreparingJob", + "endTime": "2016-11-17T14:13:39.243-05:00" + }, { + "startTime": "2016-11-17T14:13:39.243-05:00", + "description": "RunningJob", + "endTime": "2016-11-17T14:13:41.112-05:00" + }, { + "startTime": "2016-11-17T14:13:41.112-05:00", + "description": "UpdatingJobStore", + "endTime": "2016-11-17T14:13:41.115-05:00" + }], + "start": "2016-11-17T14:13:39.239-05:00" + }] + }, + "outputs": { + "goodbye_output": "Goodbye sub world!", + "hello_output": "Hello sub world!" + }, + "workflowRoot": "/cromwell-executions/main_workflow/1d919bd4-d046-43b0-9918-9964509689dd/call-hello_and_goodbye/hello_and_goodbye/a6365f91-c807-465a-9186-a5d3da98fe11", + "id": "a6365f91-c807-465a-9186-a5d3da98fe11", + "inputs": { + "hello_and_goodbye_input": "sub world" + }, + "status": "Succeeded", + "parentWorkflowId": "1d919bd4-d046-43b0-9918-9964509689dd", + "end": "2016-11-17T14:13:41.116-05:00", + "start": "2016-11-17T14:13:39.236-05:00" + }, + "shardIndex": -1, + "outputs": { + "goodbye_output": "Goodbye sub world!", + "hello_output": "Hello sub world!" + }, + "inputs": { + "hello_and_goodbye_input": "sub world" + }, + "end": "2016-11-17T14:13:41.117-05:00", + "attempt": 1, + "start": "2016-11-17T14:13:39.236-05:00" + }] + }, + "outputs": { + "main_output": "Hello sub world!" + }, + "workflowRoot": "/cromwell-executions/main_workflow/1d919bd4-d046-43b0-9918-9964509689dd", + "id": "1d919bd4-d046-43b0-9918-9964509689dd", + "inputs": { + + }, + "submission": "2016-11-17T14:13:39.104-05:00", + "status": "Succeeded", + "end": "2016-11-17T14:13:41.120-05:00", + "start": "2016-11-17T14:13:39.204-05:00" +} +``` # REST API diff --git a/backend/src/main/scala/cromwell/backend/BackendCacheHitCopyingActor.scala b/backend/src/main/scala/cromwell/backend/BackendCacheHitCopyingActor.scala index 11a95df39..f1bf38386 100644 --- a/backend/src/main/scala/cromwell/backend/BackendCacheHitCopyingActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendCacheHitCopyingActor.scala @@ -3,7 +3,7 @@ package cromwell.backend import akka.actor.{Actor, ActorLogging} import akka.event.LoggingReceive import cromwell.backend.BackendCacheHitCopyingActor.CopyOutputsCommand -import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse, FailedNonRetryableResponse} +import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse, JobFailedNonRetryableResponse} import cromwell.backend.BackendLifecycleActor._ import cromwell.core.simpleton.WdlValueSimpleton @@ -29,6 +29,6 @@ trait BackendCacheHitCopyingActor extends Actor with ActorLogging with BackendJo def abort(): Unit = log.warning("{}: Abort not supported during cache hit copying", jobTag) private def cachingFailed(t: Throwable) = { - FailedNonRetryableResponse(jobKey = jobDescriptor.key, throwable = t, returnCode = None) + JobFailedNonRetryableResponse(jobKey = jobDescriptor.key, throwable = t, returnCode = None) } } diff --git a/backend/src/main/scala/cromwell/backend/BackendJobBreadCrumb.scala b/backend/src/main/scala/cromwell/backend/BackendJobBreadCrumb.scala new file mode 100644 index 000000000..1dbc9ca50 --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/BackendJobBreadCrumb.scala @@ -0,0 +1,14 @@ +package cromwell.backend + +import java.nio.file.Path + +import cromwell.backend.io.JobPaths +import cromwell.core.{JobKey, WorkflowId} +import wdl4s.Workflow + +case class BackendJobBreadCrumb(workflow: Workflow, id: WorkflowId, jobKey: JobKey) { + def toPath(root: Path): Path = { + val workflowPart = root.resolve(workflow.unqualifiedName).resolve(id.toString) + JobPaths.callPathBuilder(workflowPart, jobKey) + } +} diff --git a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala index f5b31f776..897e782f2 100644 --- a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala @@ -7,7 +7,7 @@ import akka.event.LoggingReceive import cromwell.backend.BackendJobExecutionActor._ import cromwell.backend.BackendLifecycleActor._ import cromwell.backend.wdl.OutputEvaluator -import cromwell.core.{ExecutionEvent, JobKey, JobOutputs} +import cromwell.core.{CallOutputs, ExecutionEvent} import wdl4s.expression.WdlStandardLibraryFunctions import wdl4s.values.WdlValue @@ -24,12 +24,12 @@ object BackendJobExecutionActor { // Responses sealed trait BackendJobExecutionActorResponse extends BackendWorkflowLifecycleActorResponse - sealed trait BackendJobExecutionResponse extends BackendJobExecutionActorResponse { def jobKey: JobKey } - case class SucceededResponse(jobKey: BackendJobDescriptorKey, returnCode: Option[Int], jobOutputs: JobOutputs, jobDetritusFiles: Option[Map[String, Path]], executionEvents: Seq[ExecutionEvent]) extends BackendJobExecutionResponse - case class AbortedResponse(jobKey: JobKey) extends BackendJobExecutionResponse + sealed trait BackendJobExecutionResponse extends BackendJobExecutionActorResponse { def jobKey: BackendJobDescriptorKey } + case class JobSucceededResponse(jobKey: BackendJobDescriptorKey, returnCode: Option[Int], jobOutputs: CallOutputs, jobDetritusFiles: Option[Map[String, Path]], executionEvents: Seq[ExecutionEvent]) extends BackendJobExecutionResponse + case class AbortedResponse(jobKey: BackendJobDescriptorKey) extends BackendJobExecutionResponse sealed trait BackendJobFailedResponse extends BackendJobExecutionResponse { def throwable: Throwable; def returnCode: Option[Int] } - case class FailedNonRetryableResponse(jobKey: JobKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse - case class FailedRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse + case class JobFailedNonRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse + case class JobFailedRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse } /** @@ -48,7 +48,7 @@ trait BackendJobExecutionActor extends BackendJobLifecycleActor with ActorLoggin // We need this for receive because we can't do `onFailure = ExecutionFailure` directly - because BackendJobDescriptor =/= BackendJobDescriptorKey private def executionFailed = (t: Throwable) => - FailedNonRetryableResponse(jobKey = jobDescriptor.key, throwable = t, returnCode = None) + JobFailedNonRetryableResponse(jobKey = jobDescriptor.key, throwable = t, returnCode = None) /** * Execute a new job. diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala index 1856f1d85..a6a09cff4 100644 --- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala @@ -3,7 +3,7 @@ package cromwell.backend import akka.actor.{Actor, ActorRef} import cromwell.backend.BackendLifecycleActor._ import cromwell.core.logging.{JobLogging, WorkflowLogging} -import wdl4s.Call +import wdl4s.TaskCall import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success} @@ -55,7 +55,7 @@ trait BackendLifecycleActor extends Actor { trait BackendWorkflowLifecycleActor extends BackendLifecycleActor with WorkflowLogging { //For Logging and boilerplate - override lazy final val workflowId = workflowDescriptor.id + override lazy final val workflowIdForLogging = workflowDescriptor.id /** * The workflow descriptor for the workflow in which this Backend is being used @@ -65,7 +65,7 @@ trait BackendWorkflowLifecycleActor extends BackendLifecycleActor with WorkflowL /** * The subset of calls which this backend will be expected to run */ - protected def calls: Set[Call] + protected def calls: Set[TaskCall] } trait BackendJobLifecycleActor extends BackendLifecycleActor with JobLogging { diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala index f8adf04dd..cb625a78b 100644 --- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala +++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala @@ -6,17 +6,16 @@ import akka.actor.{ActorRef, Props} import com.typesafe.config.Config import cromwell.backend.callcaching.FileHashingActor import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction -import cromwell.backend.io.WorkflowPaths -import wdl4s.expression.PureStandardLibraryFunctions +import cromwell.backend.io.WorkflowPathsWithDocker +import cromwell.core.CallOutputs import cromwell.core.JobExecutionToken.JobExecutionTokenType -import cromwell.core.{ExecutionStore, OutputStore} -import wdl4s.Call -import wdl4s.expression.WdlStandardLibraryFunctions +import wdl4s.TaskCall +import wdl4s.expression.{PureStandardLibraryFunctions, WdlStandardLibraryFunctions} trait BackendLifecycleActorFactory { def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Set[Call], + calls: Set[TaskCall], serviceRegistryActor: ActorRef): Option[Props] = None def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, @@ -38,9 +37,9 @@ trait BackendLifecycleActorFactory { def backendSingletonActorProps: Option[Props] = None def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Set[Call], - executionStore: ExecutionStore, - outputStore: OutputStore, + calls: Set[TaskCall], + jobExecutionMap: JobExecutionMap, + workflowOutputs: CallOutputs, initializationData: Option[BackendInitializationData]): Option[Props] = None def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor, @@ -48,7 +47,11 @@ trait BackendLifecycleActorFactory { initializationData: Option[BackendInitializationData]): WdlStandardLibraryFunctions = PureStandardLibraryFunctions def getExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config, initializationData: Option[BackendInitializationData]): Path = { - new WorkflowPaths(workflowDescriptor, backendConfig).executionRoot + new WorkflowPathsWithDocker(workflowDescriptor, backendConfig).executionRoot + } + + def getWorkflowExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config, initializationData: Option[BackendInitializationData]): Path = { + new WorkflowPathsWithDocker(workflowDescriptor, backendConfig).workflowRoot } def runtimeAttributeDefinitions(initializationDataOption: Option[BackendInitializationData]): Set[RuntimeAttributeDefinition] = Set.empty diff --git a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala index 8895064be..feaf5720b 100644 --- a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala @@ -10,7 +10,7 @@ import cromwell.services.metadata.MetadataService.PutMetadataAction import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} import wdl4s.types._ import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString, WdlValue} -import wdl4s.{Call, NoLookup, Task, WdlExpression} +import wdl4s._ import scala.concurrent.Future import scala.util.{Failure, Success, Try} @@ -36,7 +36,7 @@ object BackendWorkflowInitializationActor { trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor with ActorLogging { val serviceRegistryActor: ActorRef - def calls: Set[Call] + def calls: Set[TaskCall] /** * This method is meant only as a "pre-flight check" validation of runtime attribute expressions during workflow @@ -91,6 +91,9 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w protected def runtimeAttributeValidators: Map[String, Option[WdlValue] => Boolean] + // FIXME: If a workflow executes jobs using multiple backends, + // each backend will try to write its own workflow root and override any previous one. + // They should be structured differently or at least be prefixed by the backend name protected def publishWorkflowRoot(workflowRoot: String) = { serviceRegistryActor ! PutMetadataAction(MetadataEvent(MetadataKey(workflowDescriptor.id, None, WorkflowMetadataKeys.WorkflowRoot), MetadataValue(workflowRoot))) } diff --git a/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala index bbbfbf82b..759127d67 100644 --- a/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala @@ -2,7 +2,7 @@ package cromwell.backend.async import akka.actor.{Actor, ActorLogging, ActorRef} import cromwell.backend.BackendJobDescriptor -import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, SucceededResponse, _} +import cromwell.backend.BackendJobExecutionActor._ import cromwell.backend.async.AsyncBackendJobExecutionActor._ import cromwell.core.CromwellFatalException import cromwell.core.retry.{Retry, SimpleExponentialBackoff} @@ -60,7 +60,7 @@ trait AsyncBackendJobExecutionActor { this: Actor with ActorLogging => } private def failAndStop(t: Throwable) = { - val responseBuilder = if (retryable) FailedRetryableResponse else FailedNonRetryableResponse + val responseBuilder = if (retryable) JobFailedRetryableResponse else JobFailedNonRetryableResponse completionPromise.success(responseBuilder.apply(jobDescriptor.key, t, None)) context.stop(self) } @@ -75,13 +75,13 @@ trait AsyncBackendJobExecutionActor { this: Actor with ActorLogging => context.system.scheduler.scheduleOnce(pollBackOff.backoffMillis.millis, self, IssuePollRequest(handle)) () case Finish(SuccessfulExecutionHandle(outputs, returnCode, jobDetritusFiles, executionEvents, resultsClonedFrom)) => - completionPromise.success(SucceededResponse(jobDescriptor.key, Some(returnCode), outputs, Option(jobDetritusFiles), executionEvents)) + completionPromise.success(JobSucceededResponse(jobDescriptor.key, Some(returnCode), outputs, Option(jobDetritusFiles), executionEvents)) context.stop(self) case Finish(FailedNonRetryableExecutionHandle(throwable, returnCode)) => - completionPromise.success(FailedNonRetryableResponse(jobDescriptor.key, throwable, returnCode)) + completionPromise.success(JobFailedNonRetryableResponse(jobDescriptor.key, throwable, returnCode)) context.stop(self) case Finish(FailedRetryableExecutionHandle(throwable, returnCode)) => - completionPromise.success(FailedRetryableResponse(jobDescriptor.key, throwable, returnCode)) + completionPromise.success(JobFailedRetryableResponse(jobDescriptor.key, throwable, returnCode)) context.stop(self) case Finish(cromwell.backend.async.AbortedExecutionHandle) => completionPromise.success(AbortedResponse(jobDescriptor.key)) diff --git a/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala b/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala index 7c626b153..1e4238014 100644 --- a/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala +++ b/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala @@ -3,7 +3,7 @@ package cromwell.backend.async import java.nio.file.Path import cromwell.backend.BackendJobDescriptor -import cromwell.core.{ExecutionEvent, JobOutputs} +import cromwell.core.{ExecutionEvent, CallOutputs} /** * Trait to encapsulate whether an execution is complete and if so provide a result. Useful in conjunction @@ -14,7 +14,7 @@ trait ExecutionHandle { def result: ExecutionResult } -final case class SuccessfulExecutionHandle(outputs: JobOutputs, returnCode: Int, jobDetritusFiles: Map[String, Path], executionEvents: Seq[ExecutionEvent], resultsClonedFrom: Option[BackendJobDescriptor] = None) extends ExecutionHandle { +final case class SuccessfulExecutionHandle(outputs: CallOutputs, returnCode: Int, jobDetritusFiles: Map[String, Path], executionEvents: Seq[ExecutionEvent], resultsClonedFrom: Option[BackendJobDescriptor] = None) extends ExecutionHandle { override val isDone = true override val result = SuccessfulExecution(outputs, returnCode, jobDetritusFiles, executionEvents, resultsClonedFrom) } diff --git a/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala b/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala index fb4614129..ff9722337 100644 --- a/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala +++ b/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala @@ -3,7 +3,7 @@ package cromwell.backend.async import java.nio.file.Path import cromwell.backend.BackendJobDescriptor -import cromwell.core.{ExecutionEvent, JobOutputs} +import cromwell.core.{ExecutionEvent, CallOutputs} /** * ADT representing the result of an execution of a BackendCall. @@ -13,7 +13,7 @@ sealed trait ExecutionResult /** * A successful execution with resolved outputs. */ -final case class SuccessfulExecution(outputs: JobOutputs, +final case class SuccessfulExecution(outputs: CallOutputs, returnCode: Int, jobDetritusFiles: Map[String, Path], executionEvents: Seq[ExecutionEvent], diff --git a/backend/src/main/scala/cromwell/backend/backend.scala b/backend/src/main/scala/cromwell/backend/backend.scala index d950eb8ff..e1addfe30 100644 --- a/backend/src/main/scala/cromwell/backend/backend.scala +++ b/backend/src/main/scala/cromwell/backend/backend.scala @@ -2,20 +2,19 @@ package cromwell.backend import com.typesafe.config.Config import cromwell.core.WorkflowOptions.WorkflowOption -import cromwell.core.{JobKey, WorkflowId, WorkflowOptions} +import cromwell.core.{CallKey, WorkflowId, WorkflowOptions} +import wdl4s._ import wdl4s.values.WdlValue -import wdl4s.{Call, WdlNamespaceWithWorkflow, _} import scala.util.Try /** * For uniquely identifying a job which has been or will be sent to the backend. */ -case class BackendJobDescriptorKey(call: Call, index: Option[Int], attempt: Int) extends JobKey { +case class BackendJobDescriptorKey(call: TaskCall, index: Option[Int], attempt: Int) extends CallKey { def scope = call private val indexString = index map { _.toString } getOrElse "NA" val tag = s"${call.fullyQualifiedName}:$indexString:$attempt" - val isShard = index.isDefined def mkTag(workflowId: WorkflowId) = s"$workflowId:$this" } @@ -31,14 +30,28 @@ case class BackendJobDescriptor(workflowDescriptor: BackendWorkflowDescriptor, override val toString = s"${key.mkTag(workflowDescriptor.id)}" } +object BackendWorkflowDescriptor { + def apply(id: WorkflowId, + workflow: Workflow, + inputs: Map[FullyQualifiedName, WdlValue], + workflowOptions: WorkflowOptions) = { + new BackendWorkflowDescriptor(id, workflow, inputs, workflowOptions, List.empty) + } +} + /** * For passing to a BackendActor construction time */ case class BackendWorkflowDescriptor(id: WorkflowId, - workflowNamespace: WdlNamespaceWithWorkflow, + workflow: Workflow, inputs: Map[FullyQualifiedName, WdlValue], - workflowOptions: WorkflowOptions) { - override def toString: String = s"[BackendWorkflowDescriptor id=${id.shortString} workflowName=${workflowNamespace.workflow.unqualifiedName}]" + workflowOptions: WorkflowOptions, + breadCrumbs: List[BackendJobBreadCrumb]) { + + val rootWorkflow = breadCrumbs.headOption.map(_.workflow).getOrElse(workflow) + val rootWorkflowId = breadCrumbs.headOption.map(_.id).getOrElse(id) + + override def toString: String = s"[BackendWorkflowDescriptor id=${id.shortString} workflowName=${workflow.unqualifiedName}]" def getWorkflowOption(key: WorkflowOption) = workflowOptions.get(key).toOption } diff --git a/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala b/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala index 140881094..48a6d590e 100644 --- a/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala +++ b/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala @@ -4,7 +4,7 @@ import java.nio.file.Path import akka.actor.ActorRef import cromwell.backend.BackendCacheHitCopyingActor -import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobSucceededResponse} import cromwell.backend.io.JobPaths import cromwell.core.path.PathCopier import cromwell.core.simpleton.{WdlValueBuilder, WdlValueSimpleton} @@ -99,6 +99,6 @@ trait CacheHitDuplicating { import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), metadataKeyValues) - SucceededResponse(jobDescriptor.key, returnCodeOption, destinationJobOutputs, Option(destinationJobDetritusFiles), Seq.empty) + JobSucceededResponse(jobDescriptor.key, returnCodeOption, destinationJobOutputs, Option(destinationJobDetritusFiles), Seq.empty) } } diff --git a/backend/src/main/scala/cromwell/backend/io/JobPaths.scala b/backend/src/main/scala/cromwell/backend/io/JobPaths.scala index 8170488ea..5c3a0c9f7 100644 --- a/backend/src/main/scala/cromwell/backend/io/JobPaths.scala +++ b/backend/src/main/scala/cromwell/backend/io/JobPaths.scala @@ -2,8 +2,7 @@ package cromwell.backend.io import java.nio.file.Path -import com.typesafe.config.Config -import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} +import cromwell.core.JobKey import cromwell.services.metadata.CallMetadataKeys object JobPaths { @@ -15,65 +14,50 @@ object JobPaths { val StdErrPathKey = "stderr" val ReturnCodePathKey = "returnCode" val CallRootPathKey = "callRootPath" -} - -class JobPaths(workflowDescriptor: BackendWorkflowDescriptor, - config: Config, - jobKey: BackendJobDescriptorKey) extends WorkflowPaths(workflowDescriptor, config) { - import JobPaths._ - private def callPathBuilder(root: Path) = { - val callName = jobKey.call.fullyQualifiedName.split('.').last + def callPathBuilder(root: Path, jobKey: JobKey) = { + val callName = jobKey.scope.unqualifiedName val call = s"$CallPrefix-$callName" val shard = jobKey.index map { s => s"$ShardPrefix-$s" } getOrElse "" val retry = if (jobKey.attempt > 1) s"$AttemptPrefix-${jobKey.attempt}" else "" List(call, shard, retry).foldLeft(root)((path, dir) => path.resolve(dir)) } +} - def toDockerPath(path: Path): Path = { - path.toAbsolutePath match { - case p if p.startsWith(WorkflowPaths.DockerRoot) => p - case p => - /* For example: - * - * p = /abs/path/to/cromwell-executions/three-step/f00ba4/call-ps/stdout.txt - * localExecutionRoot = /abs/path/to/cromwell-executions - * subpath = three-step/f00ba4/call-ps/stdout.txt - * - * return value = /root/three-step/f00ba4/call-ps/stdout.txt - * - * TODO: this assumes that p.startsWith(localExecutionRoot) - */ - val subpath = p.subpath(executionRoot.getNameCount, p.getNameCount) - WorkflowPaths.DockerRoot.resolve(subpath) - } - } - - val callRoot = callPathBuilder(workflowRoot) - val callDockerRoot = callPathBuilder(dockerWorkflowRoot) - - val callExecutionRoot = callRoot.resolve("execution") - val callExecutionDockerRoot = callDockerRoot.resolve("execution") - - val callInputsRoot = callRoot.resolve("inputs") - - val stdout = callExecutionRoot.resolve("stdout") - val stderr = callExecutionRoot.resolve("stderr") - val script = callExecutionRoot.resolve("script") - val returnCode = callExecutionRoot.resolve("rc") +trait JobPaths { this: WorkflowPaths => + import JobPaths._ - lazy val metadataPaths: Map[String, Path] = Map( + def returnCodeFilename: String = "rc" + def stdoutFilename: String = "stdout" + def stderrFilename: String = "stderr" + def scriptFilename: String = "script" + + def jobKey: JobKey + lazy val callRoot = callPathBuilder(workflowRoot, jobKey) + lazy val callExecutionRoot = callRoot + lazy val stdout = callExecutionRoot.resolve(stdoutFilename) + lazy val stderr = callExecutionRoot.resolve(stderrFilename) + lazy val script = callExecutionRoot.resolve(scriptFilename) + lazy val returnCode = callExecutionRoot.resolve(returnCodeFilename) + + private lazy val commonMetadataPaths: Map[String, Path] = Map( CallMetadataKeys.CallRoot -> callRoot, CallMetadataKeys.Stdout -> stdout, CallMetadataKeys.Stderr -> stderr ) - lazy val detritusPaths: Map[String, Path] = Map( + private lazy val commonDetritusPaths: Map[String, Path] = Map( JobPaths.CallRootPathKey -> callRoot, JobPaths.ScriptPathKey -> script, JobPaths.StdoutPathKey -> stdout, JobPaths.StdErrPathKey -> stderr, JobPaths.ReturnCodePathKey -> returnCode ) + + protected lazy val customMetadataPaths: Map[String, Path] = Map.empty + protected lazy val customDetritusPaths: Map[String, Path] = Map.empty + + lazy val metadataPaths = commonMetadataPaths ++ customMetadataPaths + lazy val detritusPaths = commonDetritusPaths ++ customDetritusPaths } diff --git a/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala b/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala new file mode 100644 index 000000000..49c748e88 --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala @@ -0,0 +1,39 @@ +package cromwell.backend.io + +import java.nio.file.Path + +import com.typesafe.config.Config +import cromwell.backend.{BackendWorkflowDescriptor, BackendJobDescriptorKey} +import cromwell.core.path.PathBuilder + +class JobPathsWithDocker(val jobKey: BackendJobDescriptorKey, + workflowDescriptor: BackendWorkflowDescriptor, + config: Config, + pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends WorkflowPathsWithDocker( + workflowDescriptor, config, pathBuilders) with JobPaths { + import JobPaths._ + + override lazy val callExecutionRoot = { callRoot.resolve("execution") } + val callDockerRoot = callPathBuilder(dockerWorkflowRoot, jobKey) + val callExecutionDockerRoot = callDockerRoot.resolve("execution") + val callInputsRoot = callRoot.resolve("inputs") + + def toDockerPath(path: Path): Path = { + path.toAbsolutePath match { + case p if p.startsWith(WorkflowPathsWithDocker.DockerRoot) => p + case p => + /* For example: + * + * p = /abs/path/to/cromwell-executions/three-step/f00ba4/call-ps/stdout.txt + * localExecutionRoot = /abs/path/to/cromwell-executions + * subpath = three-step/f00ba4/call-ps/stdout.txt + * + * return value = /root/three-step/f00ba4/call-ps/stdout.txt + * + * TODO: this assumes that p.startsWith(localExecutionRoot) + */ + val subpath = p.subpath(executionRoot.getNameCount, p.getNameCount) + WorkflowPathsWithDocker.DockerRoot.resolve(subpath) + } + } +} \ No newline at end of file diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala index c61ea9108..be959aec5 100644 --- a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala +++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala @@ -1,27 +1,37 @@ package cromwell.backend.io -import java.nio.file.{Path, Paths} +import java.nio.file.Path import com.typesafe.config.Config import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} +import cromwell.core.WorkflowOptions.FinalCallLogsDir +import cromwell.core.path.{DefaultPathBuilder, PathFactory} import net.ceedubs.ficus.Ficus._ -import cromwell.core.path.{DefaultPathBuilder, PathBuilder, PathFactory} -object WorkflowPaths{ - val DockerRoot = Paths.get("/root") +import scala.util.Try + +object WorkflowPaths { val DefaultPathBuilders = List(DefaultPathBuilder) } -class WorkflowPaths(workflowDescriptor: BackendWorkflowDescriptor, config: Config, val pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends PathFactory { - val executionRoot = Paths.get(config.as[Option[String]]("root").getOrElse("cromwell-executions")).toAbsolutePath - - private def workflowPathBuilder(root: Path) = { - root.resolve(workflowDescriptor.workflowNamespace.workflow.unqualifiedName) - .resolve(workflowDescriptor.id.toString) +trait WorkflowPaths extends PathFactory { + def workflowDescriptor: BackendWorkflowDescriptor + def config: Config + + protected lazy val executionRootString = config.as[Option[String]]("root").getOrElse("cromwell-executions") + + def getPath(url: String): Try[Path] = Try(PathFactory.buildPath(url, pathBuilders)) + + // Rebuild potential intermediate call directories in case of a sub workflow + protected def workflowPathBuilder(root: Path) = { + workflowDescriptor.breadCrumbs.foldLeft(root)((acc, breadCrumb) => { + breadCrumb.toPath(acc) + }).resolve(workflowDescriptor.workflow.unqualifiedName).resolve(workflowDescriptor.id.toString + "/") } + lazy val executionRoot = PathFactory.buildPath(executionRootString, pathBuilders).toAbsolutePath lazy val workflowRoot = workflowPathBuilder(executionRoot) - lazy val dockerWorkflowRoot = workflowPathBuilder(WorkflowPaths.DockerRoot) + lazy val finalCallLogsPath = workflowDescriptor.getWorkflowOption(FinalCallLogsDir) map getPath map { _.get } - def toJobPaths(jobKey: BackendJobDescriptorKey) = new JobPaths(workflowDescriptor, config, jobKey) + def toJobPaths(jobKey: BackendJobDescriptorKey): JobPaths } diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala new file mode 100644 index 000000000..c10e66972 --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala @@ -0,0 +1,16 @@ +package cromwell.backend.io + +import java.nio.file.Paths + +import com.typesafe.config.Config +import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} +import cromwell.core.path.PathBuilder + +object WorkflowPathsWithDocker { + val DockerRoot = Paths.get("/root") +} + +class WorkflowPathsWithDocker(val workflowDescriptor: BackendWorkflowDescriptor, val config: Config, val pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends WorkflowPaths { + val dockerWorkflowRoot = workflowPathBuilder(WorkflowPathsWithDocker.DockerRoot) + override def toJobPaths(jobKey: BackendJobDescriptorKey): JobPaths = new JobPathsWithDocker(jobKey, workflowDescriptor, config, pathBuilders) +} \ No newline at end of file diff --git a/backend/src/main/scala/cromwell/backend/package.scala b/backend/src/main/scala/cromwell/backend/package.scala index 3bad6f61f..132fcf578 100644 --- a/backend/src/main/scala/cromwell/backend/package.scala +++ b/backend/src/main/scala/cromwell/backend/package.scala @@ -1,14 +1,6 @@ package cromwell -import wdl4s.values.WdlValue - -import scala.language.postfixOps -import scala.util.Success - package object backend { - implicit class AugmentedAttemptedLookupSequence(s: Seq[AttemptedLookupResult]) { - def toLookupMap: Map[String, WdlValue] = s collect { - case AttemptedLookupResult(name, Success(value)) => (name, value) - } toMap - } + /** Represents the jobKeys executed by a (potentially sub-) workflow at a given point in time */ + type JobExecutionMap = Map[BackendWorkflowDescriptor, List[BackendJobDescriptorKey]] } diff --git a/backend/src/main/scala/cromwell/backend/wfs/WorkflowPathBuilder.scala b/backend/src/main/scala/cromwell/backend/wfs/WorkflowPathBuilder.scala index a593ed6fe..bd39ae2c9 100644 --- a/backend/src/main/scala/cromwell/backend/wfs/WorkflowPathBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/wfs/WorkflowPathBuilder.scala @@ -1,7 +1,7 @@ package cromwell.backend.wfs import com.typesafe.config.Config -import cromwell.backend.io.WorkflowPaths +import cromwell.backend.io.{WorkflowPathsWithDocker, WorkflowPaths} import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor} import cromwell.core.WorkflowOptions import cromwell.core.path.PathBuilder @@ -12,7 +12,7 @@ object WorkflowPathBuilder { def workflowPaths(configurationDescriptor: BackendConfigurationDescriptor, workflowDescriptor: BackendWorkflowDescriptor, pathBuilders: List[PathBuilder]): WorkflowPaths = { - new WorkflowPaths(workflowDescriptor, configurationDescriptor.backendConfig, pathBuilders) + new WorkflowPathsWithDocker(workflowDescriptor, configurationDescriptor.backendConfig, pathBuilders) } } diff --git a/backend/src/test/scala/cromwell/backend/BackendSpec.scala b/backend/src/test/scala/cromwell/backend/BackendSpec.scala index 522fa54a1..cfd73f084 100644 --- a/backend/src/test/scala/cromwell/backend/BackendSpec.scala +++ b/backend/src/test/scala/cromwell/backend/BackendSpec.scala @@ -1,7 +1,7 @@ package cromwell.backend import com.typesafe.config.ConfigFactory -import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, FailedNonRetryableResponse, FailedRetryableResponse, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobFailedNonRetryableResponse, JobFailedRetryableResponse, JobSucceededResponse} import cromwell.backend.io.TestWorkflows._ import cromwell.core.{WorkflowId, WorkflowOptions} import org.scalatest.Matchers @@ -27,7 +27,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito { runtime: String = "") = { BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime)), + WdlNamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime)).workflow, inputs, options ) @@ -47,7 +47,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito { inputs: Map[String, WdlValue], options: WorkflowOptions, runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition]): BackendJobDescriptor = { - val call = workflowDescriptor.workflowNamespace.workflow.calls.head + val call = workflowDescriptor.workflow.taskCalls.head val jobKey = BackendJobDescriptorKey(call, None, 1) val inputDeclarations = call.evaluateTaskInputs(inputs, NoFunctions) val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, inputDeclarations).get // .get is OK here because this is a test @@ -59,7 +59,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito { options: WorkflowOptions, runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition]): BackendJobDescriptor = { val workflowDescriptor = buildWorkflowDescriptor(wdl) - val call = workflowDescriptor.workflowNamespace.workflow.calls.head + val call = workflowDescriptor.workflow.taskCalls.head val jobKey = BackendJobDescriptorKey(call, None, 1) val inputDeclarations = fqnMapToDeclarationMap(workflowDescriptor.inputs) val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, inputDeclarations).get // .get is OK here because this is a test @@ -73,7 +73,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito { options: WorkflowOptions, runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition]): BackendJobDescriptor = { val workflowDescriptor = buildWorkflowDescriptor(wdl, runtime = runtime) - val call = workflowDescriptor.workflowNamespace.workflow.calls.head + val call = workflowDescriptor.workflow.taskCalls.head val jobKey = BackendJobDescriptorKey(call, None, attempt) val inputDeclarations = fqnMapToDeclarationMap(workflowDescriptor.inputs) val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, inputDeclarations).get // .get is OK here because this is a test @@ -83,7 +83,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito { def assertResponse(executionResponse: BackendJobExecutionResponse, expectedResponse: BackendJobExecutionResponse) = { (executionResponse, expectedResponse) match { - case (SucceededResponse(_, _, responseOutputs, _, _), SucceededResponse(_, _, expectedOutputs, _, _)) => + case (JobSucceededResponse(_, _, responseOutputs, _, _), JobSucceededResponse(_, _, expectedOutputs, _, _)) => responseOutputs.size shouldBe expectedOutputs.size responseOutputs foreach { case (fqn, out) => @@ -91,10 +91,10 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito { expectedOut.isDefined shouldBe true expectedOut.get.wdlValue.valueString shouldBe out.wdlValue.valueString } - case (FailedNonRetryableResponse(_, failure, _), FailedNonRetryableResponse(_, expectedFailure, _)) => + case (JobFailedNonRetryableResponse(_, failure, _), JobFailedNonRetryableResponse(_, expectedFailure, _)) => failure.getClass shouldBe expectedFailure.getClass failure.getMessage should include(expectedFailure.getMessage) - case (FailedRetryableResponse(_, failure, _), FailedRetryableResponse(_, expectedFailure, _)) => + case (JobFailedRetryableResponse(_, failure, _), JobFailedRetryableResponse(_, expectedFailure, _)) => failure.getClass shouldBe expectedFailure.getClass case (response, expectation) => fail(s"Execution response $response wasn't conform to expectation $expectation") @@ -111,7 +111,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito { ConfigFactory.parseString("{}"), ConfigFactory.load()) def firstJobDescriptorKey(workflowDescriptor: BackendWorkflowDescriptor): BackendJobDescriptorKey = { - val call = workflowDescriptor.workflowNamespace.workflow.calls.head + val call = workflowDescriptor.workflow.taskCalls.head BackendJobDescriptorKey(call, None, 1) } diff --git a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala index 6de5450f7..dffd64884 100644 --- a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala +++ b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala @@ -6,7 +6,7 @@ import better.files._ import com.typesafe.config.ConfigFactory import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptorKey, BackendSpec} import org.scalatest.{FlatSpec, Matchers} -import wdl4s.Call +import wdl4s.TaskCall class JobPathsSpec extends FlatSpec with Matchers with BackendSpec { @@ -32,9 +32,9 @@ class JobPathsSpec extends FlatSpec with Matchers with BackendSpec { "JobPaths" should "provide correct paths for a job" in { val wd = buildWorkflowDescriptor(TestWorkflows.HelloWorld) - val call: Call = wd.workflowNamespace.workflow.calls.head + val call: TaskCall = wd.workflow.taskCalls.head val jobKey = BackendJobDescriptorKey(call, None, 1) - val jobPaths = new JobPaths(wd, backendConfig, jobKey) + val jobPaths = new JobPathsWithDocker(jobKey, wd, backendConfig) val id = wd.id jobPaths.callRoot.toString shouldBe File(s"local-cromwell-executions/wf_hello/$id/call-hello").pathAsString @@ -60,17 +60,17 @@ class JobPathsSpec extends FlatSpec with Matchers with BackendSpec { File("/root/dock/path").pathAsString val jobKeySharded = BackendJobDescriptorKey(call, Option(0), 1) - val jobPathsSharded = new JobPaths(wd, backendConfig, jobKeySharded) + val jobPathsSharded = new JobPathsWithDocker(jobKeySharded, wd, backendConfig) jobPathsSharded.callExecutionRoot.toString shouldBe File(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/execution").pathAsString val jobKeyAttempt = BackendJobDescriptorKey(call, None, 2) - val jobPathsAttempt = new JobPaths(wd, backendConfig, jobKeyAttempt) + val jobPathsAttempt = new JobPathsWithDocker(jobKeyAttempt, wd, backendConfig) jobPathsAttempt.callExecutionRoot.toString shouldBe File(s"local-cromwell-executions/wf_hello/$id/call-hello/attempt-2/execution").pathAsString val jobKeyShardedAttempt = BackendJobDescriptorKey(call, Option(0), 2) - val jobPathsShardedAttempt = new JobPaths(wd, backendConfig, jobKeyShardedAttempt) + val jobPathsShardedAttempt = new JobPathsWithDocker(jobKeyShardedAttempt, wd, backendConfig) jobPathsShardedAttempt.callExecutionRoot.toString shouldBe File(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/attempt-2/execution").pathAsString } diff --git a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala index d2dfd1b20..edd86e500 100644 --- a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala +++ b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala @@ -2,9 +2,11 @@ package cromwell.backend.io import better.files._ import com.typesafe.config.Config -import cromwell.backend.BackendSpec +import cromwell.backend.{BackendJobBreadCrumb, BackendSpec, BackendWorkflowDescriptor} +import cromwell.core.{JobKey, WorkflowId} import org.mockito.Mockito._ import org.scalatest.{FlatSpec, Matchers} +import wdl4s.{Call, Scope, Workflow} class WorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec { @@ -14,11 +16,49 @@ class WorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec { when(backendConfig.hasPath(any[String])).thenReturn(true) when(backendConfig.getString(any[String])).thenReturn("local-cromwell-executions") // This is the folder defined in the config as the execution root dir val wd = buildWorkflowDescriptor(TestWorkflows.HelloWorld) - val workflowPaths = new WorkflowPaths(wd, backendConfig) + val workflowPaths = new WorkflowPathsWithDocker(wd, backendConfig) val id = wd.id workflowPaths.workflowRoot.toString shouldBe File(s"local-cromwell-executions/wf_hello/$id").pathAsString workflowPaths.dockerWorkflowRoot.toString shouldBe s"/root/wf_hello/$id" } + + "WorkflowPaths" should "provide correct paths for a sub workflow" in { + when(backendConfig.hasPath(any[String])).thenReturn(true) + when(backendConfig.getString(any[String])).thenReturn("local-cromwell-executions") // This is the folder defined in the config as the execution root dir + + val rootWd = mock[BackendWorkflowDescriptor] + val rootWorkflow = mock[Workflow] + val rootWorkflowId = WorkflowId.randomId() + rootWorkflow.unqualifiedName returns "rootWorkflow" + rootWd.workflow returns rootWorkflow + rootWd.id returns rootWorkflowId + + val subWd = mock[BackendWorkflowDescriptor] + val subWorkflow = mock[Workflow] + val subWorkflowId = WorkflowId.randomId() + subWorkflow.unqualifiedName returns "subWorkflow" + subWd.workflow returns subWorkflow + subWd.id returns subWorkflowId + + val call1 = mock[Call] + call1.unqualifiedName returns "call1" + val call2 = mock[Call] + call2.unqualifiedName returns "call2" + + val jobKey = new JobKey { + override def scope: Scope = call1 + override def tag: String = "tag1" + override def index: Option[Int] = Option(1) + override def attempt: Int = 2 + } + + subWd.breadCrumbs returns List(BackendJobBreadCrumb(rootWorkflow, rootWorkflowId, jobKey)) + subWd.id returns subWorkflowId + + val workflowPaths = new WorkflowPathsWithDocker(subWd, backendConfig) + workflowPaths.workflowRoot.toString shouldBe File(s"local-cromwell-executions/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId").pathAsString + workflowPaths.dockerWorkflowRoot.toString shouldBe s"/root/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId" + } } diff --git a/core/src/main/scala/cromwell/core/CallKey.scala b/core/src/main/scala/cromwell/core/CallKey.scala new file mode 100644 index 000000000..547eadabd --- /dev/null +++ b/core/src/main/scala/cromwell/core/CallKey.scala @@ -0,0 +1,7 @@ +package cromwell.core + +import wdl4s.Call + +trait CallKey extends JobKey { + def scope: Call +} diff --git a/core/src/main/scala/cromwell/core/ExecutionStore.scala b/core/src/main/scala/cromwell/core/ExecutionStore.scala deleted file mode 100644 index 1632061ce..000000000 --- a/core/src/main/scala/cromwell/core/ExecutionStore.scala +++ /dev/null @@ -1,13 +0,0 @@ -package cromwell.core - -import cromwell.core.ExecutionStatus._ - - -object ExecutionStore { - def empty = ExecutionStore(Map.empty) - type ExecutionStoreEntry = (JobKey, ExecutionStatus) -} - -case class ExecutionStore(store: Map[JobKey, ExecutionStatus]) { - def add(values: Map[JobKey, ExecutionStatus]) = this.copy(store = store ++ values) -} diff --git a/core/src/main/scala/cromwell/core/JobKey.scala b/core/src/main/scala/cromwell/core/JobKey.scala index 374e3d0eb..a28b7150e 100644 --- a/core/src/main/scala/cromwell/core/JobKey.scala +++ b/core/src/main/scala/cromwell/core/JobKey.scala @@ -12,4 +12,6 @@ trait JobKey { import ExecutionIndex.IndexEnhancedIndex s"${scope.fullyQualifiedName}:${index.fromIndex}:$attempt" } + + def isShard = index.isDefined } diff --git a/core/src/main/scala/cromwell/core/WorkflowMetadataKeys.scala b/core/src/main/scala/cromwell/core/WorkflowMetadataKeys.scala index e5a4dd7a6..922be7ffb 100644 --- a/core/src/main/scala/cromwell/core/WorkflowMetadataKeys.scala +++ b/core/src/main/scala/cromwell/core/WorkflowMetadataKeys.scala @@ -13,6 +13,7 @@ object WorkflowMetadataKeys { val WorkflowLog = "workflowLog" val Failures = "failures" val WorkflowRoot = "workflowRoot" + val ParentWorkflowId = "parentWorkflowId" val SubmissionSection = "submittedFiles" val SubmissionSection_Workflow = "workflow" diff --git a/core/src/main/scala/cromwell/core/logging/JobLogger.scala b/core/src/main/scala/cromwell/core/logging/JobLogger.scala index 3eb96d4b5..1388f5f6b 100644 --- a/core/src/main/scala/cromwell/core/logging/JobLogger.scala +++ b/core/src/main/scala/cromwell/core/logging/JobLogger.scala @@ -9,7 +9,7 @@ trait JobLogging extends ActorLogging { this: Actor => def workflowId: WorkflowId def jobTag: String - lazy val jobLogger: Logger = new JobLogger(self.path.name, workflowId, jobTag, Option(log)) + lazy val jobLogger = new JobLogger(self.path.name, workflowId, jobTag, Option(log)) } /** diff --git a/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala b/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala index b79e9f7b6..ba339f011 100644 --- a/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala +++ b/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala @@ -15,9 +15,9 @@ import org.slf4j.helpers.NOPLogger import org.slf4j.{Logger, LoggerFactory} trait WorkflowLogging extends ActorLogging { this: Actor => - def workflowId: WorkflowId + def workflowIdForLogging: WorkflowId - lazy val workflowLogger = new WorkflowLogger(self.path.name, workflowId, Option(log)) + lazy val workflowLogger = new WorkflowLogger(self.path.name, workflowIdForLogging, Option(log)) } object WorkflowLogger { @@ -113,10 +113,10 @@ class WorkflowLogger(loggerName: String, import WorkflowLogger._ - val workflowLogPath = workflowLogConfiguration.map(workflowLogConfigurationActual => + lazy val workflowLogPath = workflowLogConfiguration.map(workflowLogConfigurationActual => File(workflowLogConfigurationActual.dir).createDirectories() / s"workflow.$workflowId.log").map(_.path) - val fileLogger = workflowLogPath match { + lazy val fileLogger = workflowLogPath match { case Some(path) => makeFileLogger(path, Level.toLevel(sys.props.getOrElse("LOG_LEVEL", "debug"))) case None => NOPLogger.NOP_LOGGER } diff --git a/core/src/main/scala/cromwell/core/package.scala b/core/src/main/scala/cromwell/core/package.scala index 3334bfa24..def878003 100644 --- a/core/src/main/scala/cromwell/core/package.scala +++ b/core/src/main/scala/cromwell/core/package.scala @@ -7,7 +7,7 @@ package object core { type FullyQualifiedName = String type WorkflowOutputs = Map[FullyQualifiedName, JobOutput] type WorkflowOptionsJson = String - type JobOutputs = Map[LocallyQualifiedName, JobOutput] + type CallOutputs = Map[LocallyQualifiedName, JobOutput] type HostInputs = Map[String, WdlValue] type EvaluatedRuntimeAttributes = Map[String, WdlValue] } diff --git a/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala b/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala index c192f9123..18b6529fb 100644 --- a/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala +++ b/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala @@ -5,7 +5,7 @@ import wdl4s.types._ import wdl4s.values.{WdlArray, WdlMap, WdlPair, WdlValue} import scala.language.postfixOps -import cromwell.core.{JobOutput, JobOutputs} +import cromwell.core.{JobOutput, CallOutputs} import cromwell.core.simpleton.WdlValueSimpleton._ @@ -123,7 +123,7 @@ object WdlValueBuilder { */ private case class SimpletonComponent(path: String, value: WdlValue) - def toJobOutputs(taskOutputs: Traversable[TaskOutput], simpletons: Traversable[WdlValueSimpleton]): JobOutputs = { + def toJobOutputs(taskOutputs: Traversable[TaskOutput], simpletons: Traversable[WdlValueSimpleton]): CallOutputs = { toWdlValues(taskOutputs, simpletons) mapValues JobOutput.apply } diff --git a/core/src/test/scala/cromwell/util/SampleWdl.scala b/core/src/test/scala/cromwell/util/SampleWdl.scala index 937521ece..677dfebf5 100644 --- a/core/src/test/scala/cromwell/util/SampleWdl.scala +++ b/core/src/test/scala/cromwell/util/SampleWdl.scala @@ -147,6 +147,7 @@ object SampleWdl { | output { | String empty = read_string(stdout()) | } + | RUNTIME |} | |workflow wf_hello { @@ -661,7 +662,7 @@ object SampleWdl { class ScatterWdl extends SampleWdl { val tasks = s"""task A { | command { - | echo -n -e "jeff\nchris\nmiguel\nthibault\nkhalid\nscott" + | echo -n -e "jeff\nchris\nmiguel\nthibault\nkhalid\nruchi" | } | RUNTIME | output { diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml index 41d8fba10..2575316f5 100644 --- a/database/migration/src/main/resources/changelog.xml +++ b/database/migration/src/main/resources/changelog.xml @@ -51,6 +51,7 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala index 80a4413fd..173de0e89 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala @@ -58,7 +58,8 @@ class SlickDatabase(override val originalDatabaseConfig: Config) extends SqlData with JobKeyValueSlickDatabase with JobStoreSlickDatabase with CallCachingSlickDatabase - with SummaryStatusSlickDatabase { + with SummaryStatusSlickDatabase + with SubWorkflowStoreSlickDatabase { override val urlKey = SlickDatabase.urlKey(originalDatabaseConfig) private val slickConfig = DatabaseConfig.forConfig[JdbcProfile]("", databaseConfig) diff --git a/database/sql/src/main/scala/cromwell/database/slick/SubWorkflowStoreSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SubWorkflowStoreSlickDatabase.scala new file mode 100644 index 000000000..05d216141 --- /dev/null +++ b/database/sql/src/main/scala/cromwell/database/slick/SubWorkflowStoreSlickDatabase.scala @@ -0,0 +1,67 @@ +package cromwell.database.slick + +import cats.instances.future._ +import cats.syntax.functor._ +import cromwell.database.sql.SubWorkflowStoreSqlDatabase +import cromwell.database.sql.tables.SubWorkflowStoreEntry + +import scala.concurrent.{ExecutionContext, Future} +import scala.language.postfixOps + +trait SubWorkflowStoreSlickDatabase extends SubWorkflowStoreSqlDatabase { + this: SlickDatabase => + + import dataAccess.driver.api._ + + def addSubWorkflowStoreEntry(rootWorkflowExecutionUuid: String, + parentWorkflowExecutionUuid: String, + callFullyQualifiedName: String, + jobIndex: Int, + jobAttempt: Int, + subWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Unit] = { + val action = for { + workflowStoreEntry <- dataAccess.workflowStoreEntriesForWorkflowExecutionUuid(rootWorkflowExecutionUuid).result.headOption + _ <- workflowStoreEntry match { + case Some(rootWorkflow) => + dataAccess.subWorkflowStoreEntryIdsAutoInc += + SubWorkflowStoreEntry( + rootWorkflow.workflowStoreEntryId, + parentWorkflowExecutionUuid, + callFullyQualifiedName, + jobIndex, + jobAttempt, + subWorkflowExecutionUuid + ) + case None => DBIO.failed(new IllegalArgumentException(s"Could not find root workflow with UUID $rootWorkflowExecutionUuid")) + } + } yield () + + runTransaction(action) void + } + + override def querySubWorkflowStore(parentWorkflowExecutionUuid: String, callFqn: String, jobIndex: Int, jobAttempt: Int) + (implicit ec: ExecutionContext): Future[Option[SubWorkflowStoreEntry]] = { + val action = for { + subWorkflowStoreEntryOption <- dataAccess.subWorkflowStoreEntriesForJobKey( + (parentWorkflowExecutionUuid, callFqn, jobIndex, jobAttempt) + ).result.headOption + } yield subWorkflowStoreEntryOption + + runTransaction(action) + } + + override def removeSubWorkflowStoreEntries(rootWorkflowExecutionUuid: String) + (implicit ec: ExecutionContext): Future[Int] = { + val action = for { + workflowStoreEntry <- dataAccess.workflowStoreEntriesForWorkflowExecutionUuid(rootWorkflowExecutionUuid).result.headOption + deleted <- workflowStoreEntry match { + case Some(rootWorkflow) => + dataAccess.subWorkflowStoreEntriesForRootWorkflowId(rootWorkflow.workflowStoreEntryId.get).delete + case None => + DBIO.successful(0) + } + } yield deleted + + runTransaction(action) + } +} diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/DataAccessComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/DataAccessComponent.scala index cdcce5a7a..b0c70abf3 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/DataAccessComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/DataAccessComponent.scala @@ -14,7 +14,8 @@ class DataAccessComponent(val driver: JdbcProfile) with MetadataEntryComponent with SummaryStatusEntryComponent with WorkflowMetadataSummaryEntryComponent - with WorkflowStoreEntryComponent { + with WorkflowStoreEntryComponent + with SubWorkflowStoreEntryComponent { import driver.api._ @@ -29,5 +30,6 @@ class DataAccessComponent(val driver: JdbcProfile) metadataEntries.schema ++ summaryStatusEntries.schema ++ workflowMetadataSummaryEntries.schema ++ - workflowStoreEntries.schema + workflowStoreEntries.schema ++ + subWorkflowStoreEntries.schema } diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/SubWorkflowStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/SubWorkflowStoreEntryComponent.scala new file mode 100644 index 000000000..848c60c4b --- /dev/null +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/SubWorkflowStoreEntryComponent.scala @@ -0,0 +1,62 @@ +package cromwell.database.slick.tables + +import cromwell.database.sql.tables.SubWorkflowStoreEntry +import slick.model.ForeignKeyAction.Cascade + +trait SubWorkflowStoreEntryComponent { + + this: DriverComponent with WorkflowStoreEntryComponent => + + import driver.api._ + + class SubWorkflowStoreEntries(tag: Tag) extends Table[SubWorkflowStoreEntry](tag, "SUB_WORKFLOW_STORE_ENTRY") { + def subWorkflowStoreEntryId = column[Int]("SUB_WORKFLOW_STORE_ENTRY_ID", O.PrimaryKey, O.AutoInc) + + def rootWorkflowId = column[Int]("ROOT_WORKFLOW_ID") + + def parentWorkflowExecutionUuid = column[String]("PARENT_WORKFLOW_EXECUTION_UUID") + + def callFullyQualifiedName = column[String]("CALL_FULLY_QUALIFIED_NAME") + + def callIndex = column[Int]("CALL_INDEX") + + def callAttempt = column[Int]("CALL_ATTEMPT") + + def subWorkflowExecutionUuid = column[String]("SUB_WORKFLOW_EXECUTION_UUID") + + override def * = (rootWorkflowId.?, parentWorkflowExecutionUuid, callFullyQualifiedName, callIndex, callAttempt, subWorkflowExecutionUuid, subWorkflowStoreEntryId.?) <> (SubWorkflowStoreEntry.tupled, SubWorkflowStoreEntry.unapply) + + def ucSubWorkflowStoreEntryPweuCfqnJiJa = index("UC_SUB_WORKFLOW_STORE_ENTRY_PWEU_CFQN_CI_CA", + (parentWorkflowExecutionUuid, callFullyQualifiedName, callIndex, callAttempt), unique = true) + + def fkSubWorkflowStoreRootWorkflowStoreEntryId = foreignKey("FK_SUB_WORKFLOW_STORE_ROOT_WORKFLOW_ID_WORKFLOW_STORE_ENTRY_ID", + rootWorkflowId, workflowStoreEntries)(_.workflowStoreEntryId, onDelete = Cascade) + + def ixSubWorkflowStoreEntryPweu = index("IX_SUB_WORKFLOW_STORE_ENTRY_PWEU", parentWorkflowExecutionUuid, unique = false) + } + + protected val subWorkflowStoreEntries = TableQuery[SubWorkflowStoreEntries] + + val subWorkflowStoreEntryIdsAutoInc = subWorkflowStoreEntries returning subWorkflowStoreEntries.map(_.subWorkflowStoreEntryId) + + val subWorkflowStoreEntriesForRootWorkflowId = Compiled( + (rootWorkflowId: Rep[Int]) => for { + subWorkflowStoreEntry <- subWorkflowStoreEntries + if subWorkflowStoreEntry.rootWorkflowId === rootWorkflowId + } yield subWorkflowStoreEntry + ) + + /** + * Useful for finding the unique sub workflow entry for a given job key + */ + val subWorkflowStoreEntriesForJobKey = Compiled( + (parentWorkflowExecutionUuid: Rep[String], callFullyQualifiedName: Rep[String], jobIndex: Rep[Int], + jobAttempt: Rep[Int]) => + for { + subWorkflowStoreEntry <- subWorkflowStoreEntries + if subWorkflowStoreEntry.parentWorkflowExecutionUuid === parentWorkflowExecutionUuid && + subWorkflowStoreEntry.callFullyQualifiedName === callFullyQualifiedName && + subWorkflowStoreEntry.callIndex === jobIndex && subWorkflowStoreEntry.callAttempt === jobAttempt + } yield subWorkflowStoreEntry + ) +} diff --git a/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala index c6c29479b..e1431de76 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala @@ -10,7 +10,8 @@ trait SqlDatabase extends AutoCloseable with CallCachingSqlDatabase with JobStoreSqlDatabase with MetadataSqlDatabase - with WorkflowStoreSqlDatabase { + with WorkflowStoreSqlDatabase + with SubWorkflowStoreSqlDatabase { protected val urlKey: String protected val originalDatabaseConfig: Config diff --git a/database/sql/src/main/scala/cromwell/database/sql/SubWorkflowStoreSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/SubWorkflowStoreSqlDatabase.scala new file mode 100644 index 000000000..10707dc90 --- /dev/null +++ b/database/sql/src/main/scala/cromwell/database/sql/SubWorkflowStoreSqlDatabase.scala @@ -0,0 +1,21 @@ +package cromwell.database.sql + +import cromwell.database.sql.tables.SubWorkflowStoreEntry + +import scala.concurrent.{ExecutionContext, Future} + +trait SubWorkflowStoreSqlDatabase { + this: SqlDatabase => + + def addSubWorkflowStoreEntry(rootWorkflowExecutionUuid: String, + parentWorkflowExecutionUuid: String, + callFullyQualifiedName: String, + jobIndex: Int, + jobAttempt: Int, + subWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Unit] + + def querySubWorkflowStore(parentWorkflowExecutionUuid: String, callFqn: String, jobIndex: Int, jobAttempt: Int) + (implicit ec: ExecutionContext): Future[Option[SubWorkflowStoreEntry]] + + def removeSubWorkflowStoreEntries(parentWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Int] +} diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/SubWorkflowStoreEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/SubWorkflowStoreEntry.scala new file mode 100644 index 000000000..2e718179a --- /dev/null +++ b/database/sql/src/main/scala/cromwell/database/sql/tables/SubWorkflowStoreEntry.scala @@ -0,0 +1,12 @@ +package cromwell.database.sql.tables + +case class SubWorkflowStoreEntry +( + rootWorkflowId: Option[Int], + parentWorkflowExecutionUuid: String, + callFullyQualifiedName: String, + jobIndex: Int, + jobAttempt: Int, + subWorkflowExecutionUuid: String, + subWorkflowStoreEntryId: Option[Int] = None +) diff --git a/engine/src/main/resources/swagger/cromwell.yaml b/engine/src/main/resources/swagger/cromwell.yaml index 7960bb937..1a2851f18 100644 --- a/engine/src/main/resources/swagger/cromwell.yaml +++ b/engine/src/main/resources/swagger/cromwell.yaml @@ -386,6 +386,13 @@ paths: type: string collectionFormat: multi in: query + - name: expandSubWorkflows + description: > + When true, metadata for sub workflows will be fetched and inserted automatically in the metadata response. + required: false + type: boolean + default: false + in: query tags: - Workflows responses: diff --git a/engine/src/main/resources/workflowTimings/workflowTimings.html b/engine/src/main/resources/workflowTimings/workflowTimings.html index 626f4cf14..3e1df152c 100644 --- a/engine/src/main/resources/workflowTimings/workflowTimings.html +++ b/engine/src/main/resources/workflowTimings/workflowTimings.html @@ -1,40 +1,33 @@ - - - + - + + var parentWorkflow; + if (selectedRow) parentWorkflow = chartView.getValue(selectedRow, 0); + + var indexOfParentWorkflow = expandedParentWorkflows.indexOf(parentWorkflow); + + if (indexOfParentWorkflow != -1) { + // Remove the parent workflow from the list if it's in it + expandedParentWorkflows.splice(indexOfParentWorkflow, 1); + } else if (parentWorkflow && parentWorkflowNames.indexOf(parentWorkflow) != -1) { + // Add it if it's not + expandedParentWorkflows.push(parentWorkflow); + } + + var rowsToDisplay = dt.getFilteredRows([filter]); + var view = new google.visualization.DataView(dt); + view.setRows(rowsToDisplay); + return view; + } + + function hideAllSubWorkflows(dt) { + var view = new google.visualization.DataView(dt); + function filterFunction(cell, row, column, table) { + return table.getRowProperty(row, "ancestry").length != 0; + } + + view.hideRows(dt.getFilteredRows([{column: 0, test: filterFunction}])); + return view; + } + + + +
diff --git a/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala b/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala index accda4c8a..d8aa2a44d 100644 --- a/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala +++ b/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala @@ -6,14 +6,23 @@ import cromwell.core.callcaching.CallCachingMode import cromwell.core.path.PathBuilder import wdl4s._ -final case class EngineWorkflowDescriptor(backendDescriptor: BackendWorkflowDescriptor, +final case class EngineWorkflowDescriptor(namespace: WdlNamespaceWithWorkflow, + backendDescriptor: BackendWorkflowDescriptor, workflowInputs: WorkflowCoercedInputs, - backendAssignments: Map[Call, String], + backendAssignments: Map[TaskCall, String], failureMode: WorkflowFailureMode, pathBuilders: List[PathBuilder], - callCachingMode: CallCachingMode) { - def id = backendDescriptor.id - def namespace = backendDescriptor.workflowNamespace - def name = namespace.workflow.unqualifiedName + callCachingMode: CallCachingMode, + parentWorkflow: Option[EngineWorkflowDescriptor] = None) { + + val rootWorkflow: EngineWorkflowDescriptor = parentWorkflow match { + case Some(parent) => parent.rootWorkflow + case None => this + } + + val id = backendDescriptor.id + lazy val workflow = backendDescriptor.workflow + lazy val name = workflow.unqualifiedName + val inputs = backendDescriptor.inputs def getWorkflowOption(key: WorkflowOption) = backendDescriptor.getWorkflowOption(key) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala index 506626965..373ade011 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala @@ -9,7 +9,7 @@ import better.files._ import cats.instances.try_._ import cats.syntax.functor._ import cromwell.core.retry.SimpleExponentialBackoff -import cromwell.core.{ExecutionStore => _, _} +import cromwell.core._ import cromwell.engine.workflow.SingleWorkflowRunnerActor._ import cromwell.engine.workflow.WorkflowManagerActor.RetrieveNewWorkflows import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreActor} @@ -17,6 +17,7 @@ import cromwell.engine.workflow.workflowstore.WorkflowStoreActor.SubmitWorkflow import cromwell.jobstore.EmptyJobStoreActor import cromwell.server.CromwellRootActor import cromwell.services.metadata.MetadataService.{GetSingleWorkflowMetadataAction, GetStatus, WorkflowOutputs} +import cromwell.subworkflowstore.EmptySubWorkflowStoreActor import cromwell.webservice.PerRequest.RequestComplete import cromwell.webservice.metadata.MetadataBuilderActor import spray.http.StatusCodes @@ -40,6 +41,7 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, metadataO override val abortJobsOnTerminate = true override lazy val workflowStore = new InMemoryWorkflowStore() override lazy val jobStoreActor = context.actorOf(EmptyJobStoreActor.props) + override lazy val subWorkflowStoreActor = context.actorOf(EmptySubWorkflowStoreActor.props) startWith(NotStarted, EmptySwraData) @@ -111,11 +113,11 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, metadataO } private def requestMetadataOrIssueReply(newData: TerminalSwraData) = if (metadataOutputPath.isDefined) requestMetadata(newData) else issueReply(newData) - - private def requestMetadata(newData: TerminalSwraData) = { + + private def requestMetadata(newData: TerminalSwraData): State = { val metadataBuilder = context.actorOf(MetadataBuilderActor.props(serviceRegistryActor), s"MetadataRequest-Workflow-${newData.id}") - metadataBuilder ! GetSingleWorkflowMetadataAction(newData.id, None, None) - goto(RequestingMetadata) using newData + metadataBuilder ! GetSingleWorkflowMetadataAction(newData.id, None, None, expandSubWorkflows = true) + goto (RequestingMetadata) using newData } private def schedulePollRequest(): Unit = { diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala index 90390fbe3..9a9f90748 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala @@ -1,11 +1,9 @@ package cromwell.engine.workflow -import java.time.OffsetDateTime - import akka.actor.SupervisorStrategy.Escalate import akka.actor._ import com.typesafe.config.Config -import cromwell.backend.AllBackendInitializationData +import cromwell.backend._ import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.WorkflowOptions.FinalWorkflowLogDir import cromwell.core._ @@ -18,13 +16,12 @@ import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.{Ma import cromwell.engine.workflow.lifecycle.WorkflowFinalizationActor.{StartFinalizationCommand, WorkflowFinalizationFailedResponse, WorkflowFinalizationSucceededResponse} import cromwell.engine.workflow.lifecycle.WorkflowInitializationActor.{StartInitializationCommand, WorkflowInitializationFailedResponse, WorkflowInitializationSucceededResponse} import cromwell.engine.workflow.lifecycle._ -import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor +import cromwell.engine.workflow.lifecycle.execution.{WorkflowExecutionActor, WorkflowMetadataHelper} import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor._ import cromwell.services.metadata.MetadataService._ -import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} +import cromwell.subworkflowstore.SubWorkflowStoreActor.WorkflowComplete import cromwell.webservice.EngineStatsActor - -import scala.util.Random +import wdl4s.{LocallyQualifiedName => _} object WorkflowActor { @@ -141,11 +138,12 @@ object WorkflowActor { serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, + subWorkflowStoreActor: ActorRef, callCacheReadActor: ActorRef, jobTokenDispenserActor: ActorRef, backendSingletonCollection: BackendSingletonCollection): Props = { Props(new WorkflowActor(workflowId, startMode, wdlSource, conf, serviceRegistryActor, workflowLogCopyRouter, - jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection)).withDispatcher(EngineDispatcher) + jobStoreActor, subWorkflowStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection)).withDispatcher(EngineDispatcher) } } @@ -156,28 +154,29 @@ class WorkflowActor(val workflowId: WorkflowId, startMode: StartMode, workflowSources: WorkflowSourceFilesCollection, conf: Config, - serviceRegistryActor: ActorRef, + override val serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, + subWorkflowStoreActor: ActorRef, callCacheReadActor: ActorRef, jobTokenDispenserActor: ActorRef, backendSingletonCollection: BackendSingletonCollection) - extends LoggingFSM[WorkflowActorState, WorkflowActorData] with WorkflowLogging { + extends LoggingFSM[WorkflowActorState, WorkflowActorData] with WorkflowLogging with WorkflowMetadataHelper { implicit val ec = context.dispatcher + override val workflowIdForLogging = workflowId startWith(WorkflowUnstartedState, WorkflowActorData.empty) - pushCurrentStateToMetadataService(WorkflowUnstartedState.workflowState) - + pushCurrentStateToMetadataService(workflowId, WorkflowUnstartedState.workflowState) + override def supervisorStrategy: SupervisorStrategy = OneForOneStrategy() { case _ => Escalate } when(WorkflowUnstartedState) { case Event(StartWorkflowCommand, _) => val actor = context.actorOf(MaterializeWorkflowDescriptorActor.props(serviceRegistryActor, workflowId), "MaterializeWorkflowDescriptorActor") - val startEvent = MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.StartTime), MetadataValue(OffsetDateTime.now.toString)) - serviceRegistryActor ! PutMetadataAction(startEvent) + pushWorkflowStart(workflowId) actor ! MaterializeWorkflowDescriptorCommand(workflowSources, conf) goto(MaterializingWorkflowDescriptorState) using stateData.copy(currentLifecycleStateActor = Option(actor)) @@ -204,10 +203,11 @@ class WorkflowActor(val workflowId: WorkflowId, case RestartExistingWorkflow => true } - val executionActor = context.actorOf(WorkflowExecutionActor.props(workflowId, + val executionActor = context.actorOf(WorkflowExecutionActor.props( workflowDescriptor, serviceRegistryActor, jobStoreActor, + subWorkflowStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection, @@ -218,16 +218,16 @@ class WorkflowActor(val workflowId: WorkflowId, goto(ExecutingWorkflowState) using data.copy(currentLifecycleStateActor = Option(executionActor), initializationData = initializationData) case Event(WorkflowInitializationFailedResponse(reason), data @ WorkflowActorData(_, Some(workflowDescriptor), _, _)) => - finalizeWorkflow(data, workflowDescriptor, ExecutionStore.empty, OutputStore.empty, Option(reason.toList)) + finalizeWorkflow(data, workflowDescriptor, Map.empty, Map.empty, Option(reason.toList)) } when(ExecutingWorkflowState) { - case Event(WorkflowExecutionSucceededResponse(executionStore, outputStore), + case Event(WorkflowExecutionSucceededResponse(jobKeys, outputs), data @ WorkflowActorData(_, Some(workflowDescriptor), _, _)) => - finalizeWorkflow(data, workflowDescriptor, executionStore, outputStore, None) - case Event(WorkflowExecutionFailedResponse(executionStore, outputStore, failures), + finalizeWorkflow(data, workflowDescriptor, jobKeys, outputs, None) + case Event(WorkflowExecutionFailedResponse(jobKeys, failures), data @ WorkflowActorData(_, Some(workflowDescriptor), _, _)) => - finalizeWorkflow(data, workflowDescriptor, executionStore, outputStore, Option(failures.toList)) + finalizeWorkflow(data, workflowDescriptor, jobKeys, Map.empty, Option(List(failures))) case Event(msg @ EngineStatsActor.JobCountQuery, data) => data.currentLifecycleStateActor match { case Some(a) => a forward msg @@ -246,7 +246,7 @@ class WorkflowActor(val workflowId: WorkflowId, when(WorkflowAbortingState) { case Event(x: EngineLifecycleStateCompleteResponse, data @ WorkflowActorData(_, Some(workflowDescriptor), _, _)) => - finalizeWorkflow(data, workflowDescriptor, ExecutionStore.empty, OutputStore.empty, failures = None) + finalizeWorkflow(data, workflowDescriptor, Map.empty, Map.empty, failures = None) case _ => stay() } @@ -279,22 +279,19 @@ class WorkflowActor(val workflowId: WorkflowId, // Only publish "External" state to metadata service // workflowState maps a state to an "external" state (e.g all states extending WorkflowActorRunningState map to WorkflowRunning) if (fromState.workflowState != toState.workflowState) { - pushCurrentStateToMetadataService(toState.workflowState) + pushCurrentStateToMetadataService(workflowId, toState.workflowState) } } onTransition { case (oldState, terminalState: WorkflowActorTerminalState) => workflowLogger.debug(s"transition from {} to {}. Stopping self.", arg1 = oldState, arg2 = terminalState) - // Add the end time of the workflow in the MetadataService - val now = OffsetDateTime.now - val metadataEventMsg = MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.EndTime), MetadataValue(now)) - serviceRegistryActor ! PutMetadataAction(metadataEventMsg) + pushWorkflowEnd(workflowId) + subWorkflowStoreActor ! WorkflowComplete(workflowId) terminalState match { case WorkflowFailedState => val failures = nextStateData.lastStateReached.failures.getOrElse(List.empty) - val failureEvents = failures flatMap { r => throwableToMetadataEvents(MetadataKey(workflowId, None, s"${WorkflowMetadataKeys.Failures}[${Random.nextInt(Int.MaxValue)}]"), r) } - serviceRegistryActor ! PutMetadataAction(failureEvents) + pushWorkflowFailures(workflowId, failures) context.parent ! WorkflowFailedResponse(workflowId, nextStateData.lastStateReached.state, failures) case _ => // The WMA is watching state transitions and needs no further info } @@ -324,24 +321,17 @@ class WorkflowActor(val workflowId: WorkflowId, goto(finalState) using data.copy(currentLifecycleStateActor = None) } - private[workflow] def makeFinalizationActor(workflowDescriptor: EngineWorkflowDescriptor, executionStore: ExecutionStore, outputStore: OutputStore) = { - context.actorOf(WorkflowFinalizationActor.props(workflowId, workflowDescriptor, executionStore, outputStore, stateData.initializationData), name = s"WorkflowFinalizationActor") + private[workflow] def makeFinalizationActor(workflowDescriptor: EngineWorkflowDescriptor, jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs) = { + context.actorOf(WorkflowFinalizationActor.props(workflowId, workflowDescriptor, jobExecutionMap, workflowOutputs, stateData.initializationData), name = s"WorkflowFinalizationActor") } /** * Run finalization actor and transition to FinalizingWorkflowState. */ private def finalizeWorkflow(data: WorkflowActorData, workflowDescriptor: EngineWorkflowDescriptor, - executionStore: ExecutionStore, outputStore: OutputStore, + jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs, failures: Option[List[Throwable]]) = { - val finalizationActor = makeFinalizationActor(workflowDescriptor, executionStore, outputStore) + val finalizationActor = makeFinalizationActor(workflowDescriptor, jobExecutionMap, workflowOutputs) finalizationActor ! StartFinalizationCommand goto(FinalizingWorkflowState) using data.copy(lastStateReached = StateCheckpoint(stateName, failures)) } - - // Update the current State of the Workflow (corresponding to the FSM state) in the Metadata service - private def pushCurrentStateToMetadataService(workflowState: WorkflowState): Unit = { - val metadataEventMsg = MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.Status), - MetadataValue(workflowState)) - serviceRegistryActor ! PutMetadataAction(metadataEventMsg) - } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala index d963a2b41..af39d207f 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala @@ -43,12 +43,13 @@ object WorkflowManagerActor { serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, + subWorkflowStoreActor: ActorRef, callCacheReadActor: ActorRef, jobTokenDispenserActor: ActorRef, backendSingletonCollection: BackendSingletonCollection, abortJobsOnTerminate: Boolean): Props = { val params = WorkflowManagerActorParams(ConfigFactory.load, workflowStore, serviceRegistryActor, - workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection, + workflowLogCopyRouter, jobStoreActor, subWorkflowStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection, abortJobsOnTerminate) Props(new WorkflowManagerActor(params)).withDispatcher(EngineDispatcher) } @@ -86,6 +87,7 @@ case class WorkflowManagerActorParams(config: Config, serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, + subWorkflowStoreActor: ActorRef, callCacheReadActor: ActorRef, jobTokenDispenserActor: ActorRef, backendSingletonCollection: BackendSingletonCollection, @@ -282,7 +284,7 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams) } val wfProps = WorkflowActor.props(workflowId, startMode, workflow.sources, config, params.serviceRegistryActor, - params.workflowLogCopyRouter, params.jobStoreActor, params.callCacheReadActor, params.jobTokenDispenserActor, + params.workflowLogCopyRouter, params.jobStoreActor, params.subWorkflowStoreActor, params.callCacheReadActor, params.jobTokenDispenserActor, params.backendSingletonCollection) val wfActor = context.actorOf(wfProps, name = s"WorkflowActor-$workflowId") diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala index d9056200e..1dbf8b270 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala @@ -11,19 +11,18 @@ import cromwell.core._ import cromwell.core.path.{PathCopier, PathFactory} import cromwell.engine.EngineWorkflowDescriptor import cromwell.engine.backend.{BackendConfiguration, CromwellBackends} -import wdl4s.ReportableSymbol -import wdl4s.values.WdlSingleFile +import wdl4s.values.{WdlArray, WdlMap, WdlSingleFile, WdlValue} import scala.concurrent.{ExecutionContext, Future} object CopyWorkflowOutputsActor { - def props(workflowId: WorkflowId, workflowDescriptor: EngineWorkflowDescriptor, outputStore: OutputStore, + def props(workflowId: WorkflowId, workflowDescriptor: EngineWorkflowDescriptor, workflowOutputs: CallOutputs, initializationData: AllBackendInitializationData) = Props( - new CopyWorkflowOutputsActor(workflowId, workflowDescriptor, outputStore, initializationData) + new CopyWorkflowOutputsActor(workflowId, workflowDescriptor, workflowOutputs, initializationData) ).withDispatcher(IoDispatcher) } -class CopyWorkflowOutputsActor(workflowId: WorkflowId, val workflowDescriptor: EngineWorkflowDescriptor, outputStore: OutputStore, +class CopyWorkflowOutputsActor(workflowId: WorkflowId, val workflowDescriptor: EngineWorkflowDescriptor, workflowOutputs: CallOutputs, initializationData: AllBackendInitializationData) extends EngineWorkflowFinalizationActor with PathFactory { @@ -32,9 +31,7 @@ class CopyWorkflowOutputsActor(workflowId: WorkflowId, val workflowDescriptor: E private def copyWorkflowOutputs(workflowOutputsFilePath: String): Unit = { val workflowOutputsPath = buildPath(workflowOutputsFilePath) - val reportableOutputs = workflowDescriptor.backendDescriptor.workflowNamespace.workflow.outputs - - val outputFilePaths = getOutputFilePaths(reportableOutputs) + val outputFilePaths = getOutputFilePaths outputFilePaths foreach { case (workflowRootPath, srcPath) => @@ -43,23 +40,23 @@ class CopyWorkflowOutputsActor(workflowId: WorkflowId, val workflowDescriptor: E } } - private def getOutputFilePaths(reportableOutputs: Seq[ReportableSymbol]): Seq[(Path, Path)] = { + private def findFiles(values: Seq[WdlValue]): Seq[WdlSingleFile] = { + values flatMap { + case file: WdlSingleFile => Seq(file) + case array: WdlArray => findFiles(array.value) + case map: WdlMap => findFiles(map.value.values.toSeq) + case _ => Seq.empty + } + } + + private def getOutputFilePaths: Seq[(Path, Path)] = { for { - reportableOutput <- reportableOutputs // NOTE: Without .toSeq, outputs in arrays only yield the last output - (backend, calls) <- workflowDescriptor.backendAssignments.groupBy(_._2).mapValues(_.keys.toSeq).toSeq + backend <- workflowDescriptor.backendAssignments.values.toSeq config <- BackendConfiguration.backendConfigurationDescriptor(backend).toOption.toSeq rootPath <- getBackendRootPath(backend, config).toSeq - call <- calls - // NOTE: Without .toSeq, outputs in arrays only yield the last output - (outputCallKey, outputEntries) <- outputStore.store.toSeq - // Only get paths for the original scatter call, not the indexed entries - if outputCallKey.call == call && outputCallKey.index.isEmpty - outputEntry <- outputEntries - if reportableOutput.fullyQualifiedName == s"${call.fullyQualifiedName}.${outputEntry.name}" - wdlValue <- outputEntry.wdlValue.toSeq - collected = wdlValue collectAsSeq { case f: WdlSingleFile => f } - wdlFile <- collected + outputFiles = findFiles(workflowOutputs.values.map(_.wdlValue).toSeq) + wdlFile <- outputFiles wdlPath = rootPath.getFileSystem.getPath(wdlFile.value) } yield (rootPath, wdlPath) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala index b4fe9d45a..0b567a962 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala @@ -109,7 +109,7 @@ object MaterializeWorkflowDescriptorActor { } } -class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val workflowId: WorkflowId, cromwellBackends: => CromwellBackends) extends LoggingFSM[MaterializeWorkflowDescriptorActorState, MaterializeWorkflowDescriptorActorData] with LazyLogging with WorkflowLogging { +class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val workflowIdForLogging: WorkflowId, cromwellBackends: => CromwellBackends) extends LoggingFSM[MaterializeWorkflowDescriptorActorState, MaterializeWorkflowDescriptorActorData] with LazyLogging with WorkflowLogging { import MaterializeWorkflowDescriptorActor._ @@ -121,7 +121,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor when(ReadyToMaterializeState) { case Event(MaterializeWorkflowDescriptorCommand(workflowSourceFiles, conf), _) => - buildWorkflowDescriptor(workflowId, workflowSourceFiles, conf) match { + buildWorkflowDescriptor(workflowIdForLogging, workflowSourceFiles, conf) match { case Valid(descriptor) => sender() ! MaterializeWorkflowDescriptorSuccessResponse(descriptor) goto(MaterializationSuccessfulState) @@ -175,7 +175,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor private def pushWfNameMetadataService(name: String): Unit = { // Workflow name: - val nameEvent = MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.Name), MetadataValue(name)) + val nameEvent = MetadataEvent(MetadataKey(workflowIdForLogging, None, WorkflowMetadataKeys.Name), MetadataValue(name)) serviceRegistryActor ! PutMetadataAction(nameEvent) } @@ -189,7 +189,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor val defaultBackendName = conf.as[Option[String]]("backend.default") val rawInputsValidation = validateRawInputs(sourceFiles.inputsJson) val failureModeValidation = validateWorkflowFailureMode(workflowOptions, conf) - val backendAssignmentsValidation = validateBackendAssignments(namespace.workflow.calls, workflowOptions, defaultBackendName) + val backendAssignmentsValidation = validateBackendAssignments(namespace.taskCalls, workflowOptions, defaultBackendName) val callCachingModeValidation = validateCallCachingMode(workflowOptions, conf) (rawInputsValidation |@| failureModeValidation |@| backendAssignmentsValidation |@| callCachingModeValidation ) map { @@ -202,7 +202,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor private def buildWorkflowDescriptor(id: WorkflowId, namespace: WdlNamespaceWithWorkflow, rawInputs: Map[String, JsValue], - backendAssignments: Map[Call, String], + backendAssignments: Map[TaskCall, String], workflowOptions: WorkflowOptions, failureMode: WorkflowFailureMode, pathBuilders: List[PathBuilder], @@ -225,27 +225,27 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor for { coercedInputs <- validateCoercedInputs(rawInputs, namespace) _ = pushWfInputsToMetadataService(coercedInputs) - declarations <- validateDeclarations(namespace, workflowOptions, coercedInputs, pathBuilders) - declarationsAndInputs <- checkTypes(declarations ++ coercedInputs) - backendDescriptor = BackendWorkflowDescriptor(id, namespace, declarationsAndInputs, workflowOptions) - } yield EngineWorkflowDescriptor(backendDescriptor, coercedInputs, backendAssignments, failureMode, pathBuilders, callCachingMode) + evaluatedWorkflowsDeclarations <- validateDeclarations(namespace, workflowOptions, coercedInputs, pathBuilders) + declarationsAndInputs <- checkTypes(evaluatedWorkflowsDeclarations ++ coercedInputs) + backendDescriptor = BackendWorkflowDescriptor(id, namespace.workflow, declarationsAndInputs, workflowOptions) + } yield EngineWorkflowDescriptor(namespace, backendDescriptor, coercedInputs, backendAssignments, failureMode, pathBuilders, callCachingMode) } private def pushWfInputsToMetadataService(workflowInputs: WorkflowCoercedInputs): Unit = { // Inputs val inputEvents = workflowInputs match { case empty if empty.isEmpty => - List(MetadataEvent.empty(MetadataKey(workflowId, None,WorkflowMetadataKeys.Inputs))) + List(MetadataEvent.empty(MetadataKey(workflowIdForLogging, None,WorkflowMetadataKeys.Inputs))) case inputs => inputs flatMap { case (inputName, wdlValue) => - wdlValueToMetadataEvents(MetadataKey(workflowId, None, s"${WorkflowMetadataKeys.Inputs}:$inputName"), wdlValue) + wdlValueToMetadataEvents(MetadataKey(workflowIdForLogging, None, s"${WorkflowMetadataKeys.Inputs}:$inputName"), wdlValue) } } serviceRegistryActor ! PutMetadataAction(inputEvents) } - private def validateBackendAssignments(calls: Set[Call], workflowOptions: WorkflowOptions, defaultBackendName: Option[String]): ErrorOr[Map[Call, String]] = { + private def validateBackendAssignments(calls: Set[TaskCall], workflowOptions: WorkflowOptions, defaultBackendName: Option[String]): ErrorOr[Map[TaskCall, String]] = { val callToBackendMap = Try { calls map { call => val backendPriorities = Seq( @@ -274,7 +274,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor /** * Map a call to a backend name depending on the runtime attribute key */ - private def assignBackendUsingRuntimeAttrs(call: Call): Option[String] = { + private def assignBackendUsingRuntimeAttrs(call: TaskCall): Option[String] = { val runtimeAttributesMap = call.task.runtimeAttributes.attrs runtimeAttributesMap.get(RuntimeBackendKey) map { wdlExpr => evaluateBackendNameExpression(call.fullyQualifiedName, wdlExpr) } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala index d237bbc7b..5c4b777a5 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala @@ -1,14 +1,15 @@ package cromwell.engine.workflow.lifecycle import akka.actor.{FSM, Props} -import cromwell.backend.AllBackendInitializationData import cromwell.backend.BackendWorkflowFinalizationActor.{FinalizationFailed, FinalizationSuccess, Finalize} +import cromwell.backend._ import cromwell.core.Dispatcher.EngineDispatcher -import cromwell.core.{ExecutionStore, OutputStore, WorkflowId} +import cromwell.core.{CallOutputs, WorkflowId} import cromwell.engine.EngineWorkflowDescriptor import cromwell.engine.backend.CromwellBackends import cromwell.engine.workflow.lifecycle.WorkflowFinalizationActor._ import cromwell.engine.workflow.lifecycle.WorkflowLifecycleActor._ +import wdl4s.TaskCall import scala.util.{Failure, Success, Try} @@ -37,14 +38,14 @@ object WorkflowFinalizationActor { case object WorkflowFinalizationSucceededResponse extends WorkflowLifecycleSuccessResponse final case class WorkflowFinalizationFailedResponse(reasons: Seq[Throwable]) extends WorkflowLifecycleFailureResponse - def props(workflowId: WorkflowId, workflowDescriptor: EngineWorkflowDescriptor, executionStore: ExecutionStore, - outputStore: OutputStore, initializationData: AllBackendInitializationData): Props = { - Props(new WorkflowFinalizationActor(workflowId, workflowDescriptor, executionStore, outputStore, initializationData)).withDispatcher(EngineDispatcher) + def props(workflowId: WorkflowId, workflowDescriptor: EngineWorkflowDescriptor, jobExecutionMap: JobExecutionMap, + workflowOutputs: CallOutputs, initializationData: AllBackendInitializationData): Props = { + Props(new WorkflowFinalizationActor(workflowId, workflowDescriptor, jobExecutionMap, workflowOutputs, initializationData)).withDispatcher(EngineDispatcher) } } -case class WorkflowFinalizationActor(workflowId: WorkflowId, workflowDescriptor: EngineWorkflowDescriptor, - executionStore: ExecutionStore, outputStore: OutputStore, initializationData: AllBackendInitializationData) +case class WorkflowFinalizationActor(workflowIdForLogging: WorkflowId, workflowDescriptor: EngineWorkflowDescriptor, + jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs, initializationData: AllBackendInitializationData) extends WorkflowLifecycleActor[WorkflowFinalizationActorState] { val tag = self.path.name @@ -64,14 +65,14 @@ case class WorkflowFinalizationActor(workflowId: WorkflowId, workflowDescriptor: for { (backend, calls) <- workflowDescriptor.backendAssignments.groupBy(_._2).mapValues(_.keySet) props <- CromwellBackends.backendLifecycleFactoryActorByName(backend).map( - _.workflowFinalizationActorProps(workflowDescriptor.backendDescriptor, calls, executionStore, outputStore, initializationData.get(backend)) + _.workflowFinalizationActorProps(workflowDescriptor.backendDescriptor, calls, filterJobExecutionsForBackend(calls), workflowOutputs, initializationData.get(backend)) ).get actor = context.actorOf(props, backend) } yield actor } val engineFinalizationActor = Try { - context.actorOf(CopyWorkflowOutputsActor.props(workflowId, workflowDescriptor, outputStore, initializationData), + context.actorOf(CopyWorkflowOutputsActor.props(workflowIdForLogging, workflowDescriptor, workflowOutputs, initializationData), "CopyWorkflowOutputsActor") } @@ -95,6 +96,15 @@ case class WorkflowFinalizationActor(workflowId: WorkflowId, workflowDescriptor: goto(WorkflowFinalizationFailedState) } } + + // Only send to each backend the jobs that it executed + private def filterJobExecutionsForBackend(calls: Set[TaskCall]): JobExecutionMap = { + jobExecutionMap map { + case (wd, executedKeys) => wd -> (executedKeys filter { jobKey => calls.contains(jobKey.call) }) + } filter { + case (wd, keys) => keys.nonEmpty + } + } when(FinalizationInProgressState) { case Event(FinalizationSuccess, stateData) => checkForDoneAndTransition(stateData.withSuccess(sender)) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala index 98ebf2564..14e8a31a4 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala @@ -50,7 +50,7 @@ object WorkflowInitializationActor { case class BackendActorAndBackend(actor: ActorRef, backend: String) } -case class WorkflowInitializationActor(workflowId: WorkflowId, +case class WorkflowInitializationActor(workflowIdForLogging: WorkflowId, workflowDescriptor: EngineWorkflowDescriptor, serviceRegistryActor: ActorRef) extends AbortableWorkflowLifecycleActor[WorkflowInitializationActorState] { diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala new file mode 100644 index 000000000..80748f6dd --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala @@ -0,0 +1,135 @@ +package cromwell.engine.workflow.lifecycle.execution + +import java.time.OffsetDateTime + +import akka.actor.ActorRef +import cromwell.backend.BackendJobDescriptorKey +import cromwell.core.ExecutionStatus._ +import cromwell.core._ +import cromwell.services.metadata.MetadataService._ +import cromwell.services.metadata._ +import wdl4s._ +import wdl4s.values.WdlValue + +import scala.util.Random + +trait CallMetadataHelper { + + def workflowIdForCallMetadata: WorkflowId + def serviceRegistryActor: ActorRef + + def pushNewCallMetadata(callKey: CallKey, backendName: Option[String]) = { + val startEvents = List( + Option(MetadataEvent(metadataKeyForCall(callKey, CallMetadataKeys.Start), MetadataValue(OffsetDateTime.now))), + backendName map { name => MetadataEvent(metadataKeyForCall(callKey, CallMetadataKeys.Backend), MetadataValue(name)) } + ).flatten + + serviceRegistryActor ! PutMetadataAction(startEvents) + } + + def pushQueuedCallMetadata(diffs: Seq[WorkflowExecutionDiff]) = { + val startingEvents = for { + diff <- diffs + (jobKey, executionState) <- diff.executionStoreChanges if jobKey.isInstanceOf[BackendJobDescriptorKey] && executionState == ExecutionStatus.QueuedInCromwell + } yield MetadataEvent(metadataKeyForCall(jobKey, CallMetadataKeys.ExecutionStatus), MetadataValue(ExecutionStatus.QueuedInCromwell)) + serviceRegistryActor ! PutMetadataAction(startingEvents) + } + + def pushStartingCallMetadata(callKey: CallKey) = { + val statusChange = MetadataEvent(metadataKeyForCall(callKey, CallMetadataKeys.ExecutionStatus), MetadataValue(ExecutionStatus.Starting)) + serviceRegistryActor ! PutMetadataAction(statusChange) + } + + def pushRunningCallMetadata(key: CallKey, evaluatedInputs: EvaluatedTaskInputs) = { + val inputEvents = evaluatedInputs match { + case empty if empty.isEmpty => + List(MetadataEvent.empty(metadataKeyForCall(key, s"${CallMetadataKeys.Inputs}"))) + case inputs => + inputs flatMap { + case (inputName, inputValue) => + wdlValueToMetadataEvents(metadataKeyForCall(key, s"${CallMetadataKeys.Inputs}:${inputName.unqualifiedName}"), inputValue) + } + } + + val runningEvent = List(MetadataEvent(metadataKeyForCall(key, CallMetadataKeys.ExecutionStatus), MetadataValue(ExecutionStatus.Running))) + + serviceRegistryActor ! PutMetadataAction(runningEvent ++ inputEvents) + } + + def pushWorkflowOutputMetadata(outputs: Map[LocallyQualifiedName, WdlValue]) = { + val events = outputs match { + case empty if empty.isEmpty => List(MetadataEvent.empty(MetadataKey(workflowIdForCallMetadata, None, WorkflowMetadataKeys.Outputs))) + case _ => outputs flatMap { + case (outputName, outputValue) => + wdlValueToMetadataEvents(MetadataKey(workflowIdForCallMetadata, None, s"${WorkflowMetadataKeys.Outputs}:$outputName"), outputValue) + } + } + + serviceRegistryActor ! PutMetadataAction(events) + } + + def pushSuccessfulCallMetadata(jobKey: JobKey, returnCode: Option[Int], outputs: CallOutputs) = { + val completionEvents = completedCallMetadataEvents(jobKey, ExecutionStatus.Done, returnCode) + + val outputEvents = outputs match { + case empty if empty.isEmpty => + List(MetadataEvent.empty(metadataKeyForCall(jobKey, s"${CallMetadataKeys.Outputs}"))) + case _ => + outputs flatMap { case (lqn, outputValue) => wdlValueToMetadataEvents(metadataKeyForCall(jobKey, s"${CallMetadataKeys.Outputs}:$lqn"), outputValue.wdlValue) } + } + + serviceRegistryActor ! PutMetadataAction(completionEvents ++ outputEvents) + } + + def pushFailedCallMetadata(jobKey: JobKey, returnCode: Option[Int], failure: Throwable, retryableFailure: Boolean) = { + val failedState = if (retryableFailure) ExecutionStatus.Preempted else ExecutionStatus.Failed + val completionEvents = completedCallMetadataEvents(jobKey, failedState, returnCode) + val retryableFailureEvent = MetadataEvent(metadataKeyForCall(jobKey, CallMetadataKeys.RetryableFailure), MetadataValue(retryableFailure)) + val failureEvents = throwableToMetadataEvents(metadataKeyForCall(jobKey, s"${CallMetadataKeys.Failures}[$randomNumberString]"), failure).+:(retryableFailureEvent) + + serviceRegistryActor ! PutMetadataAction(completionEvents ++ failureEvents) + } + + def pushExecutionEventsToMetadataService(jobKey: JobKey, eventList: Seq[ExecutionEvent]) = { + def metadataEvent(k: String, value: Any) = { + val metadataValue = MetadataValue(value) + val metadataKey = metadataKeyForCall(jobKey, k) + MetadataEvent(metadataKey, metadataValue) + } + + eventList.headOption foreach { firstEvent => + // The final event is only used as the book-end for the final pairing so the name is never actually used... + val offset = firstEvent.offsetDateTime.getOffset + val now = OffsetDateTime.now.withOffsetSameInstant(offset) + val lastEvent = ExecutionEvent("!!Bring Back the Monarchy!!", now) + val tailedEventList = eventList :+ lastEvent + val events = tailedEventList.sliding(2).zipWithIndex flatMap { + case (Seq(eventCurrent, eventNext), index) => + val eventKey = s"executionEvents[$index]" + List( + metadataEvent(s"$eventKey:description", eventCurrent.name), + metadataEvent(s"$eventKey:startTime", eventCurrent.offsetDateTime), + metadataEvent(s"$eventKey:endTime", eventNext.offsetDateTime) + ) + } + + serviceRegistryActor ! PutMetadataAction(events.toIterable) + } + } + + private def completedCallMetadataEvents(jobKey: JobKey, executionStatus: ExecutionStatus, returnCode: Option[Int]) = { + val returnCodeEvent = returnCode map { rc => + List(MetadataEvent(metadataKeyForCall(jobKey, CallMetadataKeys.ReturnCode), MetadataValue(rc))) + } + + List( + MetadataEvent(metadataKeyForCall(jobKey, CallMetadataKeys.ExecutionStatus), MetadataValue(executionStatus)), + MetadataEvent(metadataKeyForCall(jobKey, CallMetadataKeys.End), MetadataValue(OffsetDateTime.now)) + ) ++ returnCodeEvent.getOrElse(List.empty) + } + + private def metadataKeyForCall(jobKey: JobKey, myKey: String) = MetadataKey(workflowIdForCallMetadata, Option(MetadataJobKey(jobKey.scope.fullyQualifiedName, jobKey.index, jobKey.attempt)), myKey) + + private def randomNumberString: String = Random.nextInt.toString.stripPrefix("-") + +} diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala index 4f7f0878a..afa48474d 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala @@ -1,7 +1,5 @@ package cromwell.engine.workflow.lifecycle.execution -import java.time.OffsetDateTime - import akka.actor.{ActorRef, ActorRefFactory, LoggingFSM, Props} import akka.routing.RoundRobinPool import cats.data.NonEmptyList @@ -14,8 +12,8 @@ import cromwell.core._ import cromwell.core.callcaching._ import cromwell.core.logging.WorkflowLogging import cromwell.core.simpleton.WdlValueSimpleton +import cromwell.engine.workflow.lifecycle.execution.CallPreparationActor.{BackendJobPreparationSucceeded, CallPreparationFailed} import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ -import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor.{BackendJobPreparationFailed, BackendJobPreparationSucceeded} import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CacheMiss, CallCacheHashes, HashError} import cromwell.engine.workflow.lifecycle.execution.callcaching.FetchCachedResultsActor.{CachedOutputLookupFailed, CachedOutputLookupSucceeded} import cromwell.engine.workflow.lifecycle.execution.callcaching._ @@ -23,8 +21,6 @@ import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecu import cromwell.jobstore.JobStoreActor._ import cromwell.jobstore.{Pending => _, _} import cromwell.services.SingletonServicesStore -import cromwell.services.metadata.MetadataService.PutMetadataAction -import cromwell.services.metadata.{MetadataEvent, MetadataJobKey, MetadataKey, MetadataValue} import wdl4s.TaskOutput import scala.concurrent.ExecutionContext @@ -36,17 +32,18 @@ class EngineJobExecutionActor(replyTo: ActorRef, factory: BackendLifecycleActorFactory, initializationData: Option[BackendInitializationData], restarting: Boolean, - serviceRegistryActor: ActorRef, + val serviceRegistryActor: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, jobTokenDispenserActor: ActorRef, backendSingletonActor: Option[ActorRef], backendName: String, - callCachingMode: CallCachingMode) extends LoggingFSM[EngineJobExecutionActorState, EJEAData] with WorkflowLogging { + callCachingMode: CallCachingMode) extends LoggingFSM[EngineJobExecutionActorState, EJEAData] with WorkflowLogging with CallMetadataHelper { - override val workflowId = executionData.workflowDescriptor.id + override val workflowIdForLogging = executionData.workflowDescriptor.id + override val workflowIdForCallMetadata = executionData.workflowDescriptor.id - val jobTag = s"${workflowId.shortString}:${jobDescriptorKey.call.fullyQualifiedName}:${jobDescriptorKey.index.fromIndex}:${jobDescriptorKey.attempt}" + val jobTag = s"${workflowIdForLogging.shortString}:${jobDescriptorKey.call.fullyQualifiedName}:${jobDescriptorKey.index.fromIndex}:${jobDescriptorKey.attempt}" val tag = s"EJEA_$jobTag" // There's no need to check for a cache hit again if we got preempted, or if there's no result copying actor defined @@ -80,7 +77,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, executionToken = Option(jobExecutionToken) replyTo ! JobStarting(jobDescriptorKey) if (restarting) { - val jobStoreKey = jobDescriptorKey.toJobStoreKey(workflowId) + val jobStoreKey = jobDescriptorKey.toJobStoreKey(workflowIdForLogging) jobStoreActor ! QueryJobCompletion(jobStoreKey, jobDescriptorKey.call.task.outputs) goto(CheckingJobStore) } else { @@ -97,9 +94,9 @@ class EngineJobExecutionActor(replyTo: ActorRef, prepareJob() case Event(JobComplete(jobResult), NoData) => val response = jobResult match { - case JobResultSuccess(returnCode, jobOutputs) => SucceededResponse(jobDescriptorKey, returnCode, jobOutputs, None, Seq.empty) - case JobResultFailure(returnCode, reason, false) => FailedNonRetryableResponse(jobDescriptorKey, reason, returnCode) - case JobResultFailure(returnCode, reason, true) => FailedRetryableResponse(jobDescriptorKey, reason, returnCode) + case JobResultSuccess(returnCode, jobOutputs) => JobSucceededResponse(jobDescriptorKey, returnCode, jobOutputs, None, Seq.empty) + case JobResultFailure(returnCode, reason, false) => JobFailedNonRetryableResponse(jobDescriptorKey, reason, returnCode) + case JobResultFailure(returnCode, reason, true) => JobFailedRetryableResponse(jobDescriptorKey, reason, returnCode) } respondAndStop(response) case Event(f: JobStoreReadFailure, NoData) => @@ -121,8 +118,8 @@ class EngineJobExecutionActor(replyTo: ActorRef, runJob(updatedData) case CallCachingOff => runJob(updatedData) } - case Event(response: BackendJobPreparationFailed, NoData) => - respondAndStop(FailedNonRetryableResponse(response.jobKey, response.throwable, None)) + case Event(CallPreparationFailed(jobKey: BackendJobDescriptorKey, throwable), NoData) => + respondAndStop(JobFailedNonRetryableResponse(jobKey, throwable, None)) } private val callCachingReadResultMetadataKey = "Call caching read result" @@ -157,12 +154,12 @@ class EngineJobExecutionActor(replyTo: ActorRef, when(BackendIsCopyingCachedOutputs) { // Backend copying response: - case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, Some(Success(hashes)), _)) => + case Event(response: JobSucceededResponse, data @ ResponsePendingData(_, _, Some(Success(hashes)), _)) => saveCacheResults(hashes, data.withSuccessResponse(response)) - case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, None, _)) if effectiveCallCachingMode.writeToCache => + case Event(response: JobSucceededResponse, data @ ResponsePendingData(_, _, None, _)) if effectiveCallCachingMode.writeToCache => // Wait for the CallCacheHashes stay using data.withSuccessResponse(response) - case Event(response: SucceededResponse, data: ResponsePendingData) => // bad hashes or cache write off + case Event(response: JobSucceededResponse, data: ResponsePendingData) => // bad hashes or cache write off saveJobCompletionToJobStore(data.withSuccessResponse(response)) case Event(response: BackendJobExecutionResponse, data @ ResponsePendingData(_, _, _, Some(cacheHit))) => response match { @@ -214,10 +211,10 @@ class EngineJobExecutionActor(replyTo: ActorRef, disableCallCaching(t) stay using data.copy(hashes = Option(Failure(t))) - case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, Some(Success(hashes)), _)) if effectiveCallCachingMode.writeToCache => + case Event(response: JobSucceededResponse, data @ ResponsePendingData(_, _, Some(Success(hashes)), _)) if effectiveCallCachingMode.writeToCache => eventList ++= response.executionEvents saveCacheResults(hashes, data.withSuccessResponse(response)) - case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, None, _)) if effectiveCallCachingMode.writeToCache => + case Event(response: JobSucceededResponse, data @ ResponsePendingData(_, _, None, _)) if effectiveCallCachingMode.writeToCache => log.debug(s"Got job result for {}, awaiting hashes", jobTag) stay using data.withSuccessResponse(response) case Event(response: BackendJobExecutionResponse, data: ResponsePendingData) => @@ -238,7 +235,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, case Event(JobStoreWriteSuccess(_), data: ResponseData) => forwardAndStop(data.response) case Event(JobStoreWriteFailure(t), data: ResponseData) => - respondAndStop(FailedNonRetryableResponse(jobDescriptorKey, new Exception(s"JobStore write failure: ${t.getMessage}", t), None)) + respondAndStop(JobFailedNonRetryableResponse(jobDescriptorKey, new Exception(s"JobStore write failure: ${t.getMessage}", t), None)) } onTransition { @@ -264,7 +261,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, private def forwardAndStop(response: Any): State = { replyTo forward response returnExecutionToken() - tellEventMetadata() + pushExecutionEventsToMetadataService(jobDescriptorKey, eventList) context stop self stay() } @@ -272,7 +269,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, private def respondAndStop(response: Any): State = { replyTo ! response returnExecutionToken() - tellEventMetadata() + pushExecutionEventsToMetadataService(jobDescriptorKey, eventList) context stop self stay() } @@ -300,7 +297,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, val jobPreparationActorName = s"BackendPreparationActor_for_$jobTag" val jobPrepProps = JobPreparationActor.props(executionData, jobDescriptorKey, factory, initializationData, serviceRegistryActor, backendSingletonActor) val jobPreparationActor = createJobPreparationActor(jobPrepProps, jobPreparationActorName) - jobPreparationActor ! JobPreparationActor.Start + jobPreparationActor ! CallPreparationActor.Start goto(PreparingJob) } @@ -333,13 +330,13 @@ class EngineJobExecutionActor(replyTo: ActorRef, val backendCacheHitCopyingActorProps = propsMaker(data.jobDescriptor, initializationData, serviceRegistryActor) val cacheHitCopyActor = context.actorOf(backendCacheHitCopyingActorProps, buildCacheHitCopyingActorName(data.jobDescriptor)) cacheHitCopyActor ! CopyOutputsCommand(wdlValueSimpletons, jobDetritusFiles, returnCode) - replyTo ! JobRunning(data.jobDescriptor, None) + replyTo ! JobRunning(data.jobDescriptor.key, data.jobDescriptor.inputDeclarations, None) goto(BackendIsCopyingCachedOutputs) case None => // This should be impossible with the FSM, but luckily, we CAN recover if some foolish future programmer makes this happen: val errorMessage = "Call caching copying should never have even been attempted with no copy actor props! (Programmer error!)" log.error(errorMessage) - self ! FailedNonRetryableResponse(data.jobDescriptor.key, new RuntimeException(errorMessage), None) + self ! JobFailedNonRetryableResponse(data.jobDescriptor.key, new RuntimeException(errorMessage), None) goto(BackendIsCopyingCachedOutputs) } } @@ -348,7 +345,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, val backendJobExecutionActor = context.actorOf(data.bjeaProps, buildJobExecutionActorName(data.jobDescriptor)) val message = if (restarting) RecoverJobCommand else ExecuteJobCommand backendJobExecutionActor ! message - replyTo ! JobRunning(data.jobDescriptor, Option(backendJobExecutionActor)) + replyTo ! JobRunning(data.jobDescriptor.key, data.jobDescriptor.inputDeclarations, Option(backendJobExecutionActor)) goto(RunningJob) using data } @@ -369,16 +366,16 @@ class EngineJobExecutionActor(replyTo: ActorRef, } private def buildJobExecutionActorName(jobDescriptor: BackendJobDescriptor) = { - s"$workflowId-BackendJobExecutionActor-$jobTag" + s"$workflowIdForLogging-BackendJobExecutionActor-$jobTag" } private def buildCacheHitCopyingActorName(jobDescriptor: BackendJobDescriptor) = { - s"$workflowId-BackendCacheHitCopyingActor-$jobTag" + s"$workflowIdForLogging-BackendCacheHitCopyingActor-$jobTag" } - protected def createSaveCacheResultsActor(hashes: CallCacheHashes, success: SucceededResponse): Unit = { + protected def createSaveCacheResultsActor(hashes: CallCacheHashes, success: JobSucceededResponse): Unit = { val callCache = new CallCache(SingletonServicesStore.databaseInterface) - context.actorOf(CallCacheWriteActor.props(callCache, workflowId, hashes, success), s"CallCacheWriteActor-$tag") + context.actorOf(CallCacheWriteActor.props(callCache, workflowIdForLogging, hashes, success), s"CallCacheWriteActor-$tag") () } @@ -410,71 +407,37 @@ class EngineJobExecutionActor(replyTo: ActorRef, private def saveJobCompletionToJobStore(updatedData: ResponseData) = { updatedData.response match { - case SucceededResponse(jobKey: BackendJobDescriptorKey, returnCode: Option[Int], jobOutputs: JobOutputs, _, _) => saveSuccessfulJobResults(jobKey, returnCode, jobOutputs) + case JobSucceededResponse(jobKey: BackendJobDescriptorKey, returnCode: Option[Int], jobOutputs: CallOutputs, _, _) => saveSuccessfulJobResults(jobKey, returnCode, jobOutputs) case AbortedResponse(jobKey: BackendJobDescriptorKey) => log.debug("{}: Won't save aborted job response to JobStore", jobTag) forwardAndStop(updatedData.response) - case FailedNonRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) => saveUnsuccessfulJobResults(jobKey, returnCode, throwable, retryable = false) - case FailedRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) => saveUnsuccessfulJobResults(jobKey, returnCode, throwable, retryable = true) + case JobFailedNonRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) => saveUnsuccessfulJobResults(jobKey, returnCode, throwable, retryable = false) + case JobFailedRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) => saveUnsuccessfulJobResults(jobKey, returnCode, throwable, retryable = true) } goto(UpdatingJobStore) using updatedData } - private def saveSuccessfulJobResults(jobKey: JobKey, returnCode: Option[Int], outputs: JobOutputs) = { - val jobStoreKey = jobKey.toJobStoreKey(workflowId) + private def saveSuccessfulJobResults(jobKey: JobKey, returnCode: Option[Int], outputs: CallOutputs) = { + val jobStoreKey = jobKey.toJobStoreKey(workflowIdForLogging) val jobStoreResult = JobResultSuccess(returnCode, outputs) jobStoreActor ! RegisterJobCompleted(jobStoreKey, jobStoreResult) } private def saveUnsuccessfulJobResults(jobKey: JobKey, returnCode: Option[Int], reason: Throwable, retryable: Boolean) = { - val jobStoreKey = jobKey.toJobStoreKey(workflowId) + val jobStoreKey = jobKey.toJobStoreKey(workflowIdForLogging) val jobStoreResult = JobResultFailure(returnCode, reason, retryable) jobStoreActor ! RegisterJobCompleted(jobStoreKey, jobStoreResult) } private def writeToMetadata(keyValues: Map[String, String]) = { import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter - serviceRegistryActor.putMetadata(workflowId, Option(jobDescriptorKey), keyValues) + serviceRegistryActor.putMetadata(workflowIdForLogging, Option(jobDescriptorKey), keyValues) } private def addHashesAndStay(data: ResponsePendingData, hashes: CallCacheHashes): State = { val updatedData = data.copy(hashes = Option(Success(hashes))) stay using updatedData } - - /** - * Fire and forget events to the metadata service - */ - private def tellEventMetadata(): Unit = { - eventList.headOption foreach { firstEvent => - // The final event is only used as the book-end for the final pairing so the name is never actually used... - val offset = firstEvent.offsetDateTime.getOffset - val now = OffsetDateTime.now.withOffsetSameInstant(offset) - val lastEvent = ExecutionEvent("!!Bring Back the Monarchy!!", now) - val tailedEventList = eventList :+ lastEvent - val events = tailedEventList.sliding(2).zipWithIndex flatMap { - case (Seq(eventCurrent, eventNext), index) => - val eventKey = s"executionEvents[$index]" - List( - metadataEvent(s"$eventKey:description", eventCurrent.name), - metadataEvent(s"$eventKey:startTime", eventCurrent.offsetDateTime), - metadataEvent(s"$eventKey:endTime", eventNext.offsetDateTime) - ) - } - - serviceRegistryActor ! PutMetadataAction(events.toIterable) - } - } - - private def metadataEvent(key: String, value: Any) = { - val metadataValue = MetadataValue(value) - MetadataEvent(metadataKey(key), metadataValue) - } - - private lazy val metadataJobKey = { - MetadataJobKey(jobDescriptorKey.call.fullyQualifiedName, jobDescriptorKey.index, jobDescriptorKey.attempt) - } - private def metadataKey(key: String) = MetadataKey(workflowId, Option(metadataJobKey), key) } object EngineJobExecutionActor { @@ -496,9 +459,6 @@ object EngineJobExecutionActor { sealed trait EngineJobExecutionActorCommand case object Execute extends EngineJobExecutionActorCommand - final case class JobRunning(jobDescriptor: BackendJobDescriptor, backendJobExecutionActor: Option[ActorRef]) - final case class JobStarting(jobKey: JobKey) - def props(replyTo: ActorRef, jobDescriptorKey: BackendJobDescriptorKey, executionData: WorkflowExecutionActorData, @@ -539,10 +499,10 @@ object EngineJobExecutionActor { hashes: Option[Try[CallCacheHashes]] = None, cacheHit: Option[CacheHit] = None) extends EJEAData { - def withSuccessResponse(success: SucceededResponse) = SucceededResponseData(success, hashes) + def withSuccessResponse(success: JobSucceededResponse) = SucceededResponseData(success, hashes) def withResponse(response: BackendJobExecutionResponse) = response match { - case success: SucceededResponse => SucceededResponseData(success, hashes) + case success: JobSucceededResponse => SucceededResponseData(success, hashes) case failure => NotSucceededResponseData(failure, hashes) } @@ -559,7 +519,7 @@ object EngineJobExecutionActor { def hashes: Option[Try[CallCacheHashes]] } - private[execution] case class SucceededResponseData(successResponse: SucceededResponse, + private[execution] case class SucceededResponseData(successResponse: JobSucceededResponse, hashes: Option[Try[CallCacheHashes]] = None) extends ResponseData { override def response = successResponse } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala new file mode 100644 index 000000000..390657fef --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala @@ -0,0 +1,103 @@ +package cromwell.engine.workflow.lifecycle.execution + +import cromwell.backend.BackendJobDescriptorKey +import cromwell.core.ExecutionStatus._ +import cromwell.core.{CallKey, ExecutionStatus, JobKey} +import cromwell.engine.workflow.lifecycle.execution.ExecutionStore.ExecutionStoreEntry +import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{CollectorKey, ScatterKey, SubWorkflowKey} +import wdl4s._ + + +object ExecutionStore { + def empty = ExecutionStore(Map.empty[JobKey, ExecutionStatus]) + type ExecutionStoreEntry = (JobKey, ExecutionStatus) + def apply(workflow: Workflow) = { + // Only add direct children to the store, the rest is dynamically created when necessary + val keys = workflow.children map { + case call: TaskCall => Option(BackendJobDescriptorKey(call, None, 1)) + case call: WorkflowCall => Option(SubWorkflowKey(call, None, 1)) + case scatter: Scatter => Option(ScatterKey(scatter)) + case _ => None // FIXME there are other types of scopes now (Declarations, Ifs) Figure out what to do with those + } + + new ExecutionStore(keys.flatten.map(_ -> NotStarted).toMap) + } +} + +case class ExecutionStore(store: Map[JobKey, ExecutionStatus]) { + def add(values: Map[JobKey, ExecutionStatus]) = this.copy(store = store ++ values) + + // Convert the store to a `List` before `collect`ing to sidestep expensive and pointless hashing of `Scope`s when + // assembling the result. + def runnableScopes = store.toList collect { case entry if isRunnable(entry) => entry._1 } + + def backendJobKeys = store.keys.toList collect { case k: BackendJobDescriptorKey => k } + + private def isRunnable(entry: ExecutionStoreEntry) = { + entry match { + case (key, ExecutionStatus.NotStarted) => arePrerequisitesDone(key) + case _ => false + } + } + + def findShardEntries(key: CollectorKey): List[ExecutionStoreEntry] = store.toList filter { + case (k: CallKey, v) => k.scope == key.scope && k.isShard + case _ => false + } + + private def arePrerequisitesDone(key: JobKey): Boolean = { + val upstream = key.scope match { + case node: GraphNode => node.upstream collect { + // Only scatters and calls are in the execution store for now (not declarations) + // However declarations are nodes so they can be an upstream dependency + // We don't want to look for those in the execution store (yet ?) since upstreamEntry would return None + case n: Call => upstreamEntry(key, n) + case n: Scatter => upstreamEntry(key, n) + } + case _ => Set.empty + } + + val downstream: List[(JobKey, ExecutionStatus)] = key match { + case collector: CollectorKey => findShardEntries(collector) + case _ => Nil + } + + val dependencies = upstream.flatten ++ downstream + val dependenciesResolved = dependencies forall { case (_, s) => s == ExecutionStatus.Done } + + /* + * We need to make sure that all prerequisiteScopes have been resolved to some entry before going forward. + * If a scope cannot be resolved it may be because it is in a scatter that has not been populated yet, + * therefore there is no entry in the executionStore for this scope. + * If that's the case this prerequisiteScope has not been run yet, hence the (upstream forall {_.nonEmpty}) + */ + (upstream forall { _.nonEmpty }) && dependenciesResolved + } + + private def upstreamEntry(entry: JobKey, prerequisiteScope: Scope): Option[ExecutionStoreEntry] = { + prerequisiteScope.closestCommonAncestor(entry.scope) match { + /* + * If this entry refers to a Scope which has a common ancestor with prerequisiteScope + * and that common ancestor is a Scatter block, then find the shard with the same index + * as 'entry'. In other words, if you're in the same scatter block as your pre-requisite + * scope, then depend on the shard (with same index). + * + * NOTE: this algorithm was designed for ONE-LEVEL of scattering and probably does not + * work as-is for nested scatter blocks + */ + case Some(ancestor: Scatter) => + store find { + case (k, _) => k.scope == prerequisiteScope && k.index == entry.index + } + + /* + * Otherwise, simply refer to the collector entry. This means that 'entry' depends + * on every shard of the pre-requisite scope to finish. + */ + case _ => + store find { + case (k, _) => k.scope == prerequisiteScope && k.index.isEmpty + } + } + } +} diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala index 4cdc50a78..3cfe95075 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala @@ -3,44 +3,36 @@ package cromwell.engine.workflow.lifecycle.execution import akka.actor.{Actor, ActorRef, Props} import cromwell.backend._ import cromwell.core.logging.WorkflowLogging -import cromwell.core.{ExecutionStore, JobKey, OutputStore} +import cromwell.core.{CallKey, JobKey, WorkflowId} import cromwell.engine.EngineWorkflowDescriptor -import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor._ +import cromwell.engine.workflow.lifecycle.execution.CallPreparationActor._ +import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.SubWorkflowKey import wdl4s._ import wdl4s.expression.WdlStandardLibraryFunctions import wdl4s.values.WdlValue import scala.util.{Failure, Success, Try} -final case class JobPreparationActor(executionData: WorkflowExecutionActorData, - jobKey: BackendJobDescriptorKey, - factory: BackendLifecycleActorFactory, - initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef, - backendSingletonActor: Option[ActorRef]) - extends Actor with WorkflowLogging { - - lazy val workflowDescriptor: EngineWorkflowDescriptor = executionData.workflowDescriptor - lazy val workflowId = workflowDescriptor.id - lazy val executionStore: ExecutionStore = executionData.executionStore - lazy val outputStore: OutputStore = executionData.outputStore - lazy val expressionLanguageFunctions = factory.expressionLanguageFunctions( - workflowDescriptor.backendDescriptor, jobKey, initializationData) - +abstract class CallPreparationActor(val workflowDescriptor: EngineWorkflowDescriptor, + val outputStore: OutputStore, + callKey: CallKey) extends Actor with WorkflowLogging { + lazy val workflowIdForLogging = workflowDescriptor.id + def expressionLanguageFunctions: WdlStandardLibraryFunctions + def prepareExecutionActor(inputEvaluation: Map[Declaration, WdlValue]): CallPreparationActorResponse + override def receive = { case Start => - val response = resolveAndEvaluateInputs(jobKey, expressionLanguageFunctions) map { prepareJobExecutionActor } - context.parent ! (response recover { case f => BackendJobPreparationFailed(jobKey, f) }).get + val response = resolveAndEvaluateInputs() map { prepareExecutionActor } + context.parent ! (response recover { case f => CallPreparationFailed(callKey, f) }).get context stop self case unhandled => workflowLogger.warn(self.path.name + " received an unhandled message: " + unhandled) } - def resolveAndEvaluateInputs(jobKey: BackendJobDescriptorKey, - wdlFunctions: WdlStandardLibraryFunctions): Try[Map[Declaration, WdlValue]] = { + def resolveAndEvaluateInputs(): Try[Map[Declaration, WdlValue]] = { Try { - val call = jobKey.call - val scatterMap = jobKey.index flatMap { i => + val call = callKey.scope + val scatterMap = callKey.index flatMap { i => // Will need update for nested scatters call.upstream collectFirst { case s: Scatter => Map(s -> i) } } getOrElse Map.empty[Scatter, Int] @@ -53,8 +45,19 @@ final case class JobPreparationActor(executionData: WorkflowExecutionActorData, ) } } +} + +final case class JobPreparationActor(executionData: WorkflowExecutionActorData, + jobKey: BackendJobDescriptorKey, + factory: BackendLifecycleActorFactory, + initializationData: Option[BackendInitializationData], + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]) + extends CallPreparationActor(executionData.workflowDescriptor, executionData.outputStore, jobKey) { - private def prepareJobExecutionActor(inputEvaluation: Map[Declaration, WdlValue]): JobPreparationActorResponse = { + override lazy val expressionLanguageFunctions = factory.expressionLanguageFunctions(workflowDescriptor.backendDescriptor, jobKey, initializationData) + + override def prepareExecutionActor(inputEvaluation: Map[Declaration, WdlValue]): CallPreparationActorResponse = { import RuntimeAttributeDefinition.{addDefaultsToAttributes, evaluateRuntimeAttributes} val curriedAddDefaultsToAttributes = addDefaultsToAttributes(factory.runtimeAttributeDefinitions(initializationData), workflowDescriptor.backendDescriptor.workflowOptions) _ @@ -65,19 +68,45 @@ final case class JobPreparationActor(executionData: WorkflowExecutionActorData, jobDescriptor = BackendJobDescriptor(workflowDescriptor.backendDescriptor, jobKey, attributesWithDefault, inputEvaluation) } yield BackendJobPreparationSucceeded(jobDescriptor, factory.jobExecutionActorProps(jobDescriptor, initializationData, serviceRegistryActor, backendSingletonActor))) match { case Success(s) => s - case Failure(f) => BackendJobPreparationFailed(jobKey, f) + case Failure(f) => CallPreparationFailed(jobKey, f) } } } -object JobPreparationActor { - sealed trait JobPreparationActorCommands - case object Start extends JobPreparationActorCommands +final case class SubWorkflowPreparationActor(executionData: WorkflowExecutionActorData, + key: SubWorkflowKey, + subWorkflowId: WorkflowId) + extends CallPreparationActor(executionData.workflowDescriptor, executionData.outputStore, key) { + + override lazy val expressionLanguageFunctions = executionData.expressionLanguageFunctions + + override def prepareExecutionActor(inputEvaluation: Map[Declaration, WdlValue]): CallPreparationActorResponse = { + val oldBackendDescriptor = workflowDescriptor.backendDescriptor + + val newBackendDescriptor = oldBackendDescriptor.copy( + id = subWorkflowId, + workflow = key.scope.calledWorkflow, + inputs = workflowDescriptor.inputs ++ (inputEvaluation map { case (k, v) => k.fullyQualifiedName -> v }), + breadCrumbs = oldBackendDescriptor.breadCrumbs :+ BackendJobBreadCrumb(workflowDescriptor.workflow, workflowDescriptor.id, key) + ) + val engineDescriptor = workflowDescriptor.copy(backendDescriptor = newBackendDescriptor, parentWorkflow = Option(workflowDescriptor)) + SubWorkflowPreparationSucceeded(engineDescriptor, inputEvaluation) + } +} - sealed trait JobPreparationActorResponse - case class BackendJobPreparationSucceeded(jobDescriptor: BackendJobDescriptor, bjeaProps: Props) extends JobPreparationActorResponse - case class BackendJobPreparationFailed(jobKey: JobKey, throwable: Throwable) extends JobPreparationActorResponse +object CallPreparationActor { + sealed trait CallPreparationActorCommands + case object Start extends CallPreparationActorCommands + + sealed trait CallPreparationActorResponse + + case class BackendJobPreparationSucceeded(jobDescriptor: BackendJobDescriptor, bjeaProps: Props) extends CallPreparationActorResponse + case class SubWorkflowPreparationSucceeded(workflowDescriptor: EngineWorkflowDescriptor, inputs: EvaluatedTaskInputs) extends CallPreparationActorResponse + case class JobCallPreparationFailed(jobKey: JobKey, throwable: Throwable) extends CallPreparationActorResponse + case class CallPreparationFailed(jobKey: JobKey, throwable: Throwable) extends CallPreparationActorResponse +} +object JobPreparationActor { def props(executionData: WorkflowExecutionActorData, jobKey: BackendJobDescriptorKey, factory: BackendLifecycleActorFactory, @@ -89,3 +118,13 @@ object JobPreparationActor { Props(new JobPreparationActor(executionData, jobKey, factory, initializationData, serviceRegistryActor, backendSingletonActor)) } } + +object SubWorkflowPreparationActor { + def props(executionData: WorkflowExecutionActorData, + key: SubWorkflowKey, + subWorkflowId: WorkflowId) = { + // Note that JobPreparationActor doesn't run on the engine dispatcher as it mostly executes backend-side code + // (WDL expression evaluation using Backend's expressionLanguageFunctions) + Props(new SubWorkflowPreparationActor(executionData, key, subWorkflowId)) + } +} diff --git a/core/src/main/scala/cromwell/core/OutputStore.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala similarity index 51% rename from core/src/main/scala/cromwell/core/OutputStore.scala rename to engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala index 38bee68db..b85c467e5 100644 --- a/core/src/main/scala/cromwell/core/OutputStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala @@ -1,10 +1,12 @@ -package cromwell.core +package cromwell.engine.workflow.lifecycle.execution import cromwell.core.ExecutionIndex._ -import cromwell.core.OutputStore.{OutputCallKey, OutputEntry} -import wdl4s.types.WdlType +import cromwell.core._ +import cromwell.engine.workflow.lifecycle.execution.OutputStore.{OutputCallKey, OutputEntry} +import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.CollectorKey +import wdl4s.types.{WdlArrayType, WdlType} import wdl4s.util.TryUtil -import wdl4s.values.{WdlCallOutputsObject, WdlValue} +import wdl4s.values.{WdlArray, WdlCallOutputsObject, WdlValue} import wdl4s.{Call, Scope} import scala.language.postfixOps @@ -37,4 +39,24 @@ case class OutputStore(store: Map[OutputCallKey, Traversable[OutputEntry]]) { case None => Failure(new RuntimeException(s"Could not find call ${call.unqualifiedName}")) } } + + /** + * Try to generate output for a collector call, by collecting outputs for all of its shards. + * It's fail-fast on shard output retrieval + */ + def generateCollectorOutput(collector: CollectorKey, + shards: Iterable[CallKey]): Try[CallOutputs] = Try { + val shardsOutputs = shards.toSeq sortBy { _.index.fromIndex } map { e => + fetchCallOutputEntries(e.scope, e.index) map { + _.outputs + } getOrElse(throw new RuntimeException(s"Could not retrieve output for shard ${e.scope} #${e.index}")) + } + collector.scope.outputs map { taskOutput => + val wdlValues = shardsOutputs.map( + _.getOrElse(taskOutput.unqualifiedName, throw new RuntimeException(s"Could not retrieve output ${taskOutput.unqualifiedName}"))) + val arrayOfValues = new WdlArray(WdlArrayType(taskOutput.wdlType), wdlValues) + taskOutput.unqualifiedName -> JobOutput(arrayOfValues) + } toMap + } + } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala new file mode 100644 index 000000000..599d0cd1e --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala @@ -0,0 +1,272 @@ +package cromwell.engine.workflow.lifecycle.execution + +import akka.actor.{ActorRef, FSM, LoggingFSM, Props} +import cromwell.backend.{AllBackendInitializationData, BackendLifecycleActorFactory, BackendWorkflowDescriptor} +import cromwell.core._ +import cromwell.core.logging.JobLogging +import cromwell.engine.EngineWorkflowDescriptor +import cromwell.engine.backend.{BackendConfiguration, BackendSingletonCollection} +import cromwell.engine.workflow.lifecycle.execution.CallPreparationActor.{CallPreparationFailed, Start, SubWorkflowPreparationSucceeded} +import cromwell.engine.workflow.lifecycle.execution.SubWorkflowExecutionActor._ +import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor._ +import cromwell.services.metadata.MetadataService._ +import cromwell.services.metadata._ +import cromwell.subworkflowstore.SubWorkflowStoreActor._ +import wdl4s.EvaluatedTaskInputs + +class SubWorkflowExecutionActor(key: SubWorkflowKey, + data: WorkflowExecutionActorData, + factories: Map[String, BackendLifecycleActorFactory], + override val serviceRegistryActor: ActorRef, + jobStoreActor: ActorRef, + subWorkflowStoreActor: ActorRef, + callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection, + initializationData: AllBackendInitializationData, + restarting: Boolean) extends LoggingFSM[SubWorkflowExecutionActorState, SubWorkflowExecutionActorData] with JobLogging with WorkflowMetadataHelper with CallMetadataHelper { + + private val parentWorkflow = data.workflowDescriptor + override val workflowId = parentWorkflow.id + override val workflowIdForCallMetadata = parentWorkflow.id + override def jobTag: String = key.tag + + startWith(SubWorkflowPendingState, SubWorkflowExecutionActorData.empty) + + private var eventList: Seq[ExecutionEvent] = Seq(ExecutionEvent(stateName.toString)) + + when(SubWorkflowPendingState) { + case Event(Execute, _) => + if (restarting) { + subWorkflowStoreActor ! QuerySubWorkflow(parentWorkflow.id, key) + goto(SubWorkflowCheckingStoreState) + } else { + prepareSubWorkflow(createSubWorkflowId()) + } + } + + when(SubWorkflowCheckingStoreState) { + case Event(SubWorkflowFound(entry), _) => + prepareSubWorkflow(WorkflowId.fromString(entry.subWorkflowExecutionUuid)) + case Event(_: SubWorkflowNotFound, _) => + prepareSubWorkflow(createSubWorkflowId()) + case Event(SubWorkflowStoreFailure(command, reason), _) => + jobLogger.error(reason, s"SubWorkflowStore failure for command $command, starting sub workflow with fresh ID.") + prepareSubWorkflow(createSubWorkflowId()) + } + + when(SubWorkflowPreparingState) { + case Event(SubWorkflowPreparationSucceeded(subWorkflowEngineDescriptor, inputs), _) => + startSubWorkflow(subWorkflowEngineDescriptor, inputs) + case Event(failure: CallPreparationFailed, _) => + context.parent ! SubWorkflowFailedResponse(key, Map.empty, failure.throwable) + context stop self + stay() + } + + when(SubWorkflowRunningState) { + case Event(WorkflowExecutionSucceededResponse(executedJobKeys, outputs), _) => + context.parent ! SubWorkflowSucceededResponse(key, executedJobKeys, outputs) + goto(SubWorkflowSucceededState) + case Event(WorkflowExecutionFailedResponse(executedJobKeys, reason), _) => + context.parent ! SubWorkflowFailedResponse(key, executedJobKeys, reason) + goto(SubWorkflowFailedState) + case Event(WorkflowExecutionAbortedResponse(executedJobKeys), _) => + context.parent ! SubWorkflowAbortedResponse(key, executedJobKeys) + goto(SubWorkflowAbortedState) + } + + when(SubWorkflowSucceededState) { FSM.NullFunction } + when(SubWorkflowFailedState) { FSM.NullFunction } + when(SubWorkflowAbortedState) { FSM.NullFunction } + + whenUnhandled { + case Event(SubWorkflowStoreRegisterSuccess(command), _) => + // Nothing to do here + stay() + case Event(SubWorkflowStoreCompleteSuccess(command), _) => + // Nothing to do here + stay() + case Event(SubWorkflowStoreFailure(command, reason), _) => + jobLogger.error(reason, s"SubWorkflowStore failure for command $command") + stay() + case Event(MetadataPutFailed(action, error), _) => + jobLogger.warn(s"Put failed for Metadata action $action", error) + stay() + case Event(MetadataPutAcknowledgement(_), _) => stay() + } + + onTransition { + case (fromState, toState) => + stateData.subWorkflowId foreach { id => pushCurrentStateToMetadataService(id, toState.workflowState) } + } + + onTransition { + case (fromState, subWorkflowTerminalState: SubWorkflowTerminalState) => + stateData.subWorkflowId match { + case Some(id) => + pushWorkflowEnd(id) + pushExecutionEventsToMetadataService(key, eventList) + case None => jobLogger.error("Sub workflow completed without a Sub Workflow UUID.") + } + context stop self + } + + onTransition { + case fromState -> toState => eventList :+= ExecutionEvent(toState.toString) + } + + private def startSubWorkflow(subWorkflowEngineDescriptor: EngineWorkflowDescriptor, inputs: EvaluatedTaskInputs) = { + val subWorkflowActor = createSubWorkflowActor(subWorkflowEngineDescriptor) + + subWorkflowActor ! WorkflowExecutionActor.ExecuteWorkflowCommand + context.parent ! JobRunning(key, inputs, Option(subWorkflowActor)) + pushWorkflowRunningMetadata(subWorkflowEngineDescriptor.backendDescriptor, inputs) + + goto(SubWorkflowRunningState) + } + + private def prepareSubWorkflow(subWorkflowId: WorkflowId) = { + createSubWorkflowPreparationActor(subWorkflowId) ! Start + context.parent ! JobStarting(key) + pushCurrentStateToMetadataService(subWorkflowId, WorkflowRunning) + pushWorkflowStart(subWorkflowId) + goto(SubWorkflowPreparingState) using SubWorkflowExecutionActorData(Option(subWorkflowId)) + } + + def createSubWorkflowPreparationActor(subWorkflowId: WorkflowId) = { + context.actorOf( + SubWorkflowPreparationActor.props(data, key, subWorkflowId), + s"$subWorkflowId-SubWorkflowPreparationActor-${key.tag}" + ) + } + + def createSubWorkflowActor(subWorkflowEngineDescriptor: EngineWorkflowDescriptor) = { + context.actorOf( + WorkflowExecutionActor.props( + subWorkflowEngineDescriptor, + serviceRegistryActor, + jobStoreActor, + subWorkflowStoreActor, + callCacheReadActor, + jobTokenDispenserActor, + backendSingletonCollection, + initializationData, + restarting + ), + s"${subWorkflowEngineDescriptor.id}-SubWorkflowActor-${key.tag}" + ) + } + + private def pushWorkflowRunningMetadata(subWorkflowDescriptor: BackendWorkflowDescriptor, workflowInputs: EvaluatedTaskInputs) = { + val subWorkflowId = subWorkflowDescriptor.id + val parentWorkflowMetadataKey = MetadataKey(parentWorkflow.id, Option(MetadataJobKey(key.scope.fullyQualifiedName, key.index, key.attempt)), CallMetadataKeys.SubWorkflowId) + + val events = List( + MetadataEvent(parentWorkflowMetadataKey, MetadataValue(subWorkflowId)), + MetadataEvent(MetadataKey(subWorkflowId, None, WorkflowMetadataKeys.Name), MetadataValue(key.scope.callable.unqualifiedName)), + MetadataEvent(MetadataKey(subWorkflowId, None, WorkflowMetadataKeys.ParentWorkflowId), MetadataValue(parentWorkflow.id)) + ) + + val inputEvents = workflowInputs match { + case empty if empty.isEmpty => + List(MetadataEvent.empty(MetadataKey(subWorkflowId, None,WorkflowMetadataKeys.Inputs))) + case inputs => + inputs flatMap { case (inputName, wdlValue) => + wdlValueToMetadataEvents(MetadataKey(subWorkflowId, None, s"${WorkflowMetadataKeys.Inputs}:${inputName.unqualifiedName}"), wdlValue) + } + } + + val workflowRootEvents = buildWorkflowRootMetadataEvents(subWorkflowDescriptor) + + serviceRegistryActor ! PutMetadataAction(events ++ inputEvents ++ workflowRootEvents) + } + + private def buildWorkflowRootMetadataEvents(subWorkflowDescriptor: BackendWorkflowDescriptor) = { + val subWorkflowId = subWorkflowDescriptor.id + + factories flatMap { + case (backendName, factory) => + BackendConfiguration.backendConfigurationDescriptor(backendName).toOption map { config => + backendName -> factory.getWorkflowExecutionRootPath(subWorkflowDescriptor, config.backendConfig, initializationData.get(backendName)) + } + } map { + case (backend, wfRoot) => + MetadataEvent(MetadataKey(subWorkflowId, None, s"${WorkflowMetadataKeys.WorkflowRoot}[$backend]"), MetadataValue(wfRoot.toAbsolutePath)) + } + } + + private def createSubWorkflowId() = { + val subWorkflowId = WorkflowId.randomId() + // Register ID to the sub workflow store + subWorkflowStoreActor ! RegisterSubWorkflow(parentWorkflow.rootWorkflow.id, parentWorkflow.id, key, subWorkflowId) + subWorkflowId + } +} + +object SubWorkflowExecutionActor { + sealed trait SubWorkflowExecutionActorState { + def workflowState: WorkflowState + } + sealed trait SubWorkflowTerminalState extends SubWorkflowExecutionActorState + + case object SubWorkflowPendingState extends SubWorkflowExecutionActorState { + override val workflowState = WorkflowRunning + } + case object SubWorkflowCheckingStoreState extends SubWorkflowExecutionActorState { + override val workflowState = WorkflowRunning + } + case object SubWorkflowPreparingState extends SubWorkflowExecutionActorState { + override val workflowState = WorkflowRunning + } + case object SubWorkflowRunningState extends SubWorkflowExecutionActorState { + override val workflowState = WorkflowRunning + } + case object SubWorkflowAbortingState extends SubWorkflowExecutionActorState { + override val workflowState = WorkflowAborting + } + + case object SubWorkflowSucceededState extends SubWorkflowTerminalState { + override val workflowState = WorkflowSucceeded + } + case object SubWorkflowAbortedState extends SubWorkflowTerminalState { + override val workflowState = WorkflowAborted + } + case object SubWorkflowFailedState extends SubWorkflowTerminalState { + override val workflowState = WorkflowFailed + } + + object SubWorkflowExecutionActorData { + def empty = SubWorkflowExecutionActorData(None) + } + case class SubWorkflowExecutionActorData(subWorkflowId: Option[WorkflowId]) + + sealed trait EngineWorkflowExecutionActorCommand + case object Execute + + def props(key: SubWorkflowKey, + data: WorkflowExecutionActorData, + factories: Map[String, BackendLifecycleActorFactory], + serviceRegistryActor: ActorRef, + jobStoreActor: ActorRef, + subWorkflowStoreActor: ActorRef, + callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection, + initializationData: AllBackendInitializationData, + restarting: Boolean) = { + Props(new SubWorkflowExecutionActor( + key, + data, + factories, + serviceRegistryActor, + jobStoreActor, + subWorkflowStoreActor, + callCacheReadActor, + jobTokenDispenserActor, + backendSingletonCollection, + initializationData, + restarting) + ) + } +} \ No newline at end of file diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index 64e23672d..8eefb8c72 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -1,319 +1,76 @@ package cromwell.engine.workflow.lifecycle.execution -import java.time.OffsetDateTime - import akka.actor._ import cats.data.NonEmptyList import com.typesafe.config.ConfigFactory -import cromwell.backend.BackendJobExecutionActor._ +import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, JobFailedNonRetryableResponse, JobFailedRetryableResponse, JobSucceededResponse} import cromwell.backend.BackendLifecycleActor.AbortJobCommand -import cromwell.backend.{AllBackendInitializationData, BackendJobDescriptor, BackendJobDescriptorKey} -import cromwell.core.Dispatcher.EngineDispatcher +import cromwell.backend.{AllBackendInitializationData, BackendJobDescriptorKey, JobExecutionMap} +import cromwell.core.Dispatcher._ import cromwell.core.ExecutionIndex._ import cromwell.core.ExecutionStatus._ -import cromwell.core.ExecutionStore.ExecutionStoreEntry -import cromwell.core.OutputStore.OutputEntry import cromwell.core.WorkflowOptions.WorkflowFailureMode import cromwell.core._ import cromwell.core.logging.WorkflowLogging import cromwell.engine.backend.{BackendSingletonCollection, CromwellBackends} -import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{JobRunning, JobStarting} -import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.WorkflowExecutionActorState +import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{apply => _, _} import cromwell.engine.workflow.lifecycle.{EngineLifecycleActorAbortCommand, EngineLifecycleActorAbortedResponse} import cromwell.engine.{ContinueWhilePossible, EngineWorkflowDescriptor} -import cromwell.services.metadata.MetadataService._ -import cromwell.services.metadata._ -import cromwell.util.StopAndLogSupervisor +import cromwell.services.metadata.MetadataService.{MetadataPutAcknowledgement, MetadataPutFailed} +import cromwell.util.{StopAndLogSupervisor, TryUtil} import cromwell.webservice.EngineStatsActor import lenthall.exception.ThrowableAggregation import net.ceedubs.ficus.Ficus._ -import wdl4s.types.WdlArrayType -import wdl4s.util.TryUtil import wdl4s.values.{WdlArray, WdlValue} import wdl4s.{Scope, _} import scala.annotation.tailrec import scala.language.postfixOps -import scala.util.{Failure, Random, Success, Try} - -object WorkflowExecutionActor { - - /** - * States - */ - sealed trait WorkflowExecutionActorState { def terminal = false } - sealed trait WorkflowExecutionActorTerminalState extends WorkflowExecutionActorState { override val terminal = true } - - case object WorkflowExecutionPendingState extends WorkflowExecutionActorState - case object WorkflowExecutionInProgressState extends WorkflowExecutionActorState - case object WorkflowExecutionAbortingState extends WorkflowExecutionActorState - case object WorkflowExecutionSuccessfulState extends WorkflowExecutionActorTerminalState - case object WorkflowExecutionFailedState extends WorkflowExecutionActorTerminalState - case object WorkflowExecutionAbortedState extends WorkflowExecutionActorTerminalState - - /** - * Commands - */ - sealed trait WorkflowExecutionActorCommand - case object ExecuteWorkflowCommand extends WorkflowExecutionActorCommand - case object RestartExecutingWorkflowCommand extends WorkflowExecutionActorCommand - - /** - * Responses - */ - sealed trait WorkflowExecutionActorResponse { - def executionStore: ExecutionStore - - def outputStore: OutputStore - } - - case class WorkflowExecutionSucceededResponse(executionStore: ExecutionStore, outputStore: OutputStore) - extends WorkflowExecutionActorResponse { - override def toString = "WorkflowExecutionSucceededResponse" - } - - case class WorkflowExecutionAbortedResponse(executionStore: ExecutionStore, outputStore: OutputStore) - extends WorkflowExecutionActorResponse with EngineLifecycleActorAbortedResponse { - override def toString = "WorkflowExecutionAbortedResponse" - } - - final case class WorkflowExecutionFailedResponse(executionStore: ExecutionStore, outputStore: OutputStore, - reasons: Seq[Throwable]) extends WorkflowExecutionActorResponse { - override def toString = "WorkflowExecutionFailedResponse" - } - - /** - * Internal control flow messages - */ - private case class JobInitializationFailed(jobKey: JobKey, throwable: Throwable) - private case class ScatterCollectionFailedResponse(collectorKey: CollectorKey, throwable: Throwable) - private case class ScatterCollectionSucceededResponse(collectorKey: CollectorKey, outputs: JobOutputs) - - /** - * Internal ADTs - */ - case class ScatterKey(scope: Scatter) extends JobKey { - override val index = None // When scatters are nested, this might become Some(_) - override val attempt = 1 - override val tag = scope.unqualifiedName - - /** - * Creates a sub-ExecutionStore with Starting entries for each of the scoped children. - * - * @param count Number of ways to scatter the children. - * @return ExecutionStore of scattered children. - */ - def populate(count: Int): Map[JobKey, ExecutionStatus.Value] = { - val keys = this.scope.children flatMap { explode(_, count) } - keys map { _ -> ExecutionStatus.NotStarted } toMap - } - - private def explode(scope: Scope, count: Int): Seq[JobKey] = { - scope match { - case call: Call => - val shards = (0 until count) map { i => BackendJobDescriptorKey(call, Option(i), 1) } - shards :+ CollectorKey(call) - case scatter: Scatter => - throw new UnsupportedOperationException("Nested Scatters are not supported (yet).") - case e => - throw new UnsupportedOperationException(s"Scope ${e.getClass.getName} is not supported.") - } - } - } - - // Represents a scatter collection for a call in the execution store - case class CollectorKey(scope: Call) extends JobKey { - override val index = None - override val attempt = 1 - override val tag = s"Collector-${scope.unqualifiedName}" - } - - case class WorkflowExecutionException[T <: Throwable](exceptions: NonEmptyList[T]) extends ThrowableAggregation { - override val throwables = exceptions.toList - override val exceptionContext = s"WorkflowExecutionActor" - } - - def props(workflowId: WorkflowId, - workflowDescriptor: EngineWorkflowDescriptor, - serviceRegistryActor: ActorRef, - jobStoreActor: ActorRef, - callCacheReadActor: ActorRef, - jobTokenDispenserActor: ActorRef, - backendSingletonCollection: BackendSingletonCollection, - initializationData: AllBackendInitializationData, - restarting: Boolean): Props = { - Props(WorkflowExecutionActor(workflowId, workflowDescriptor, serviceRegistryActor, jobStoreActor, - callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection, initializationData, restarting)).withDispatcher(EngineDispatcher) - } - - implicit class EnhancedExecutionStore(val executionStore: ExecutionStore) extends AnyVal { - // Convert the store to a `List` before `collect`ing to sidestep expensive and pointless hashing of `Scope`s when - // assembling the result. - def runnableScopes = executionStore.store.toList collect { case entry if isRunnable(entry) => entry._1 } - - private def isRunnable(entry: ExecutionStoreEntry) = { - entry match { - case (key, ExecutionStatus.NotStarted) => arePrerequisitesDone(key) - case _ => false - } - } - - def findShardEntries(key: CollectorKey): List[ExecutionStoreEntry] = executionStore.store.toList collect { - case (k: BackendJobDescriptorKey, v) if k.scope == key.scope && k.isShard => (k, v) - } - - private def arePrerequisitesDone(key: JobKey): Boolean = { - val upstream = key.scope match { - case node: GraphNode => node.upstream collect { - // Only scatters and calls are in the execution store for now (not declarations) - // However declarations are nodes so they can be an upstream dependency - // We don't want to look for those in the execution store (yet ?) since upstreamEntry would return None - case n: Call => upstreamEntry(key, n) - case n: Scatter => upstreamEntry(key, n) - } - case _ => Set.empty - } - - val downstream: List[(JobKey, ExecutionStatus)] = key match { - case collector: CollectorKey => findShardEntries(collector) - case _ => Nil - } - - val dependencies = upstream.flatten ++ downstream - val dependenciesResolved = dependencies forall { case (_, s) => s == ExecutionStatus.Done } - - /* - * We need to make sure that all prerequisiteScopes have been resolved to some entry before going forward. - * If a scope cannot be resolved it may be because it is in a scatter that has not been populated yet, - * therefore there is no entry in the executionStore for this scope. - * If that's the case this prerequisiteScope has not been run yet, hence the (upstream forall {_.nonEmpty}) - */ - (upstream forall { _.nonEmpty }) && dependenciesResolved - } - - private def upstreamEntry(entry: JobKey, prerequisiteScope: Scope): Option[ExecutionStoreEntry] = { - prerequisiteScope.closestCommonAncestor(entry.scope) match { - /* - * If this entry refers to a Scope which has a common ancestor with prerequisiteScope - * and that common ancestor is a Scatter block, then find the shard with the same index - * as 'entry'. In other words, if you're in the same scatter block as your pre-requisite - * scope, then depend on the shard (with same index). - * - * NOTE: this algorithm was designed for ONE-LEVEL of scattering and probably does not - * work as-is for nested scatter blocks - */ - case Some(ancestor: Scatter) => - executionStore.store find { - case (k, _) => k.scope == prerequisiteScope && k.index == entry.index - } - - /* - * Otherwise, simply refer to the entry the collector entry. This means that 'entry' depends - * on every shard of the pre-requisite scope to finish. - */ - case _ => - executionStore.store find { - case (k, _) => k.scope == prerequisiteScope && k.index.isEmpty - } - } - } - } - - implicit class EnhancedOutputStore(val outputStore: OutputStore) extends AnyVal { - /** - * Try to generate output for a collector call, by collecting outputs for all of its shards. - * It's fail-fast on shard output retrieval - */ - def generateCollectorOutput(collector: CollectorKey, - shards: Iterable[BackendJobDescriptorKey]): Try[JobOutputs] = Try { - val shardsOutputs = shards.toSeq sortBy { _.index.fromIndex } map { e => - outputStore.fetchCallOutputEntries(e.scope, e.index) map { - _.outputs - } getOrElse(throw new RuntimeException(s"Could not retrieve output for shard ${e.scope} #${e.index}")) - } - collector.scope.task.outputs map { taskOutput => - val wdlValues = shardsOutputs.map( - _.getOrElse(taskOutput.unqualifiedName, throw new RuntimeException(s"Could not retrieve output ${taskOutput.unqualifiedName}"))) - val arrayOfValues = new WdlArray(WdlArrayType(taskOutput.wdlType), wdlValues) - taskOutput.unqualifiedName -> JobOutput(arrayOfValues) - } toMap - } - } -} - -final case class WorkflowExecutionActor(workflowId: WorkflowId, - workflowDescriptor: EngineWorkflowDescriptor, - serviceRegistryActor: ActorRef, - jobStoreActor: ActorRef, - callCacheReadActor: ActorRef, - jobTokenDispenserActor: ActorRef, - backendSingletonCollection: BackendSingletonCollection, - initializationData: AllBackendInitializationData, - restarting: Boolean) - extends LoggingFSM[WorkflowExecutionActorState, WorkflowExecutionActorData] with WorkflowLogging with StopAndLogSupervisor { - - import WorkflowExecutionActor._ - - val tag = s"WorkflowExecutionActor [UUID(${workflowId.shortString})]" - private lazy val DefaultMaxRetriesFallbackValue = 10 - +import scala.util.{Failure, Success, Try} + +case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, + serviceRegistryActor: ActorRef, + jobStoreActor: ActorRef, + subWorkflowStoreActor: ActorRef, + callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection, + initializationData: AllBackendInitializationData, + restarting: Boolean) + extends LoggingFSM[WorkflowExecutionActorState, WorkflowExecutionActorData] with WorkflowLogging with CallMetadataHelper with StopAndLogSupervisor { + implicit val ec = context.dispatcher + + override val workflowIdForLogging = workflowDescriptor.id + override val workflowIdForCallMetadata = workflowDescriptor.id - val MaxRetries = ConfigFactory.load().as[Option[Int]]("system.max-retries") match { + private val tag = s"WorkflowExecutionActor [UUID(${workflowDescriptor.id.shortString})]" + private val MaxRetries = ConfigFactory.load().as[Option[Int]]("system.max-retries") match { case Some(value) => value case None => workflowLogger.warn(s"Failed to load the max-retries value from the configuration. Defaulting back to a value of '$DefaultMaxRetriesFallbackValue'.") DefaultMaxRetriesFallbackValue } - - private val factories = TryUtil.sequenceMap(workflowDescriptor.backendAssignments.values.toSet[String] map { backendName => + + private val backendFactories = TryUtil.sequenceMap(workflowDescriptor.backendAssignments.values.toSet[String] map { backendName => backendName -> CromwellBackends.backendLifecycleFactoryActorByName(backendName) } toMap) recover { case e => throw new RuntimeException("Could not instantiate backend factories", e) } get - // Initialize the StateData with ExecutionStore (all calls as NotStarted) and SymbolStore startWith( WorkflowExecutionPendingState, WorkflowExecutionActorData( workflowDescriptor, - executionStore = buildInitialExecutionStore(), + executionStore = ExecutionStore(workflowDescriptor.backendDescriptor.workflow), backendJobExecutionActors = Map.empty, engineJobExecutionActors = Map.empty, + subWorkflowExecutionActors = Map.empty, + downstreamExecutionMap = Map.empty, outputStore = OutputStore.empty ) ) - private def buildInitialExecutionStore(): ExecutionStore = { - val workflow = workflowDescriptor.backendDescriptor.workflowNamespace.workflow - // Only add direct children to the store, the rest is dynamically created when necessary - val keys = workflow.children map { - case call: Call => Option(BackendJobDescriptorKey(call, None, 1)) - case scatter: Scatter => Option(ScatterKey(scatter)) - case _ => None // FIXME there are other types of scopes now (Declarations, Ifs) Figure out what to do with those - } - - ExecutionStore(keys.flatten.map(_ -> NotStarted).toMap) - } - - private def handleNonRetryableFailure(stateData: WorkflowExecutionActorData, failedJobKey: JobKey, reason: Throwable) = { - val mergedStateData = stateData.mergeExecutionDiff(WorkflowExecutionDiff(Map(failedJobKey -> ExecutionStatus.Failed))) - .removeBackendJobExecutionActor(failedJobKey) - - if (workflowDescriptor.getWorkflowOption(WorkflowFailureMode).contains(ContinueWhilePossible.toString)) { - mergedStateData.workflowCompletionStatus match { - case Some(completionStatus) if completionStatus == Failed => - context.parent ! WorkflowExecutionFailedResponse(stateData.executionStore, stateData.outputStore, List(reason)) - goto(WorkflowExecutionFailedState) using mergedStateData - case _ => - stay() using startRunnableScopes(mergedStateData) - } - } else { - context.parent ! WorkflowExecutionFailedResponse(stateData.executionStore, stateData.outputStore, List(reason)) - goto(WorkflowExecutionFailedState) using mergedStateData - } - } - when(WorkflowExecutionPendingState) { case Event(ExecuteWorkflowCommand, stateData) => val data = startRunnableScopes(stateData) @@ -322,34 +79,58 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, when(WorkflowExecutionInProgressState) { case Event(JobStarting(jobKey), stateData) => - // The EJEA is telling us that the job is now Starting. Update the metadata and our execution store. - val statusChange = MetadataEvent(metadataKey(jobKey, CallMetadataKeys.ExecutionStatus), MetadataValue(ExecutionStatus.Starting)) - serviceRegistryActor ! PutMetadataAction(statusChange) + pushStartingCallMetadata(jobKey) stay() using stateData .mergeExecutionDiff(WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Starting))) - case Event(JobRunning(jobDescriptor, backendJobExecutionActor), stateData) => - // The EJEA is telling us that the job is now Running. Update the metadata and our execution store. - pushRunningJobMetadata(jobDescriptor) + case Event(JobRunning(key, inputs, callExecutionActor), stateData) => + pushRunningCallMetadata(key, inputs) stay() using stateData - .addBackendJobExecutionActor(jobDescriptor.key, backendJobExecutionActor) - .mergeExecutionDiff(WorkflowExecutionDiff(Map(jobDescriptor.key -> ExecutionStatus.Running))) - case Event(SucceededResponse(jobKey, returnCode, callOutputs, _, _), stateData) => - pushSuccessfulJobMetadata(jobKey, returnCode, callOutputs) - handleJobSuccessful(jobKey, callOutputs, stateData) - case Event(FailedNonRetryableResponse(jobKey, reason, returnCode), stateData) => - pushFailedJobMetadata(jobKey, returnCode, reason, retryableFailure = false) - handleNonRetryableFailure(stateData, jobKey, reason) - case Event(FailedRetryableResponse(jobKey, reason, returnCode), stateData) => - workflowLogger.warn(s"Job ${jobKey.tag} failed with a retryable failure: ${reason.getMessage}") - pushFailedJobMetadata(jobKey, None, reason, retryableFailure = true) - handleRetryableFailure(jobKey, reason, returnCode) - case Event(JobInitializationFailed(jobKey, reason), stateData) => - pushFailedJobMetadata(jobKey, None, reason, retryableFailure = false) - handleNonRetryableFailure(stateData, jobKey, reason) + .addCallExecutionActor(key, callExecutionActor) + .mergeExecutionDiff(WorkflowExecutionDiff(Map(key -> ExecutionStatus.Running))) + + //Success + // Job + case Event(JobSucceededResponse(jobKey, returnCode, callOutputs, _, _), stateData) => + pushSuccessfulCallMetadata(jobKey, returnCode, callOutputs) + handleCallSuccessful(jobKey, callOutputs, stateData, Map.empty) + // Sub Workflow + case Event(SubWorkflowSucceededResponse(jobKey, descendantJobKeys, callOutputs), stateData) => + pushSuccessfulCallMetadata(jobKey, None, callOutputs) + handleCallSuccessful(jobKey, callOutputs, stateData, descendantJobKeys) + // Scatter case Event(ScatterCollectionSucceededResponse(jobKey, callOutputs), stateData) => - handleJobSuccessful(jobKey, callOutputs, stateData) + handleCallSuccessful(jobKey, callOutputs, stateData, Map.empty) + + // Failure + // Initialization + case Event(JobInitializationFailed(jobKey, reason), stateData) => + pushFailedCallMetadata(jobKey, None, reason, retryableFailure = false) + handleNonRetryableFailure(stateData, jobKey, reason, Map.empty) + // Job Non Retryable + case Event(JobFailedNonRetryableResponse(jobKey, reason, returnCode), stateData) => + pushFailedCallMetadata(jobKey, returnCode, reason, retryableFailure = false) + handleNonRetryableFailure(stateData, jobKey, reason, Map.empty) + // Job Retryable + case Event(JobFailedRetryableResponse(jobKey, reason, returnCode), stateData) => + pushFailedCallMetadata(jobKey, None, reason, retryableFailure = true) + handleRetryableFailure(jobKey, reason, returnCode) + // Sub Workflow - sub workflow failures are always non retryable + case Event(SubWorkflowFailedResponse(jobKey, descendantJobKeys, reason), stateData) => + pushFailedCallMetadata(jobKey, None, reason, retryableFailure = false) + handleNonRetryableFailure(stateData, jobKey, reason, descendantJobKeys) } + when(WorkflowExecutionAbortingState) { + case Event(AbortedResponse(jobKey), stateData) => + handleCallAborted(stateData, jobKey, Map.empty) + case Event(SubWorkflowAbortedResponse(jobKey, executedKeys), stateData) => + handleCallAborted(stateData, jobKey, executedKeys) + case Event(SubWorkflowSucceededResponse(subKey, executedKeys, _), stateData) => + handleCallAborted(stateData, subKey, executedKeys) + case Event(JobSucceededResponse(jobKey, returnCode, callOutputs, _, _), stateData) => + handleCallAborted(stateData, jobKey, Map.empty) + } + when(WorkflowExecutionSuccessfulState) { FSM.NullFunction } @@ -365,31 +146,19 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, */ private def alreadyFailedMopUp: StateFunction = { case Event(JobInitializationFailed(jobKey, reason), stateData) => - pushFailedJobMetadata(jobKey, None, reason, retryableFailure = false) + pushFailedCallMetadata(jobKey, None, reason, retryableFailure = false) stay - case Event(FailedNonRetryableResponse(jobKey, reason, returnCode), stateData) => - pushFailedJobMetadata(jobKey, returnCode, reason, retryableFailure = false) + case Event(JobFailedNonRetryableResponse(jobKey, reason, returnCode), stateData) => + pushFailedCallMetadata(jobKey, returnCode, reason, retryableFailure = false) stay - case Event(FailedRetryableResponse(jobKey, reason, returnCode), stateData) => - pushFailedJobMetadata(jobKey, returnCode, reason, retryableFailure = true) + case Event(JobFailedRetryableResponse(jobKey, reason, returnCode), stateData) => + pushFailedCallMetadata(jobKey, returnCode, reason, retryableFailure = true) stay - case Event(SucceededResponse(jobKey, returnCode, callOutputs, _, _), stateData) => - pushSuccessfulJobMetadata(jobKey, returnCode, callOutputs) + case Event(JobSucceededResponse(jobKey, returnCode, callOutputs, _, _), stateData) => + pushSuccessfulCallMetadata(jobKey, returnCode, callOutputs) stay } - when(WorkflowExecutionAbortingState) { - case Event(response: BackendJobExecutionResponse, stateData) => - val jobKey = response.jobKey - workflowLogger.info(s"$tag job exited with ${response.getClass.getSimpleName}: ${jobKey.tag}") - val newStateData = stateData.removeBackendJobExecutionActor(jobKey) - if (newStateData.backendJobExecutionActors.isEmpty) { - workflowLogger.info(s"$tag all jobs exited") - goto(WorkflowExecutionAbortedState) - } else { - stay() using newStateData - } - } def handleTerminated(actorRef: ActorRef) = { // Both of these Should Never Happen (tm), assuming the state data is set correctly on EJEA creation. @@ -402,7 +171,7 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, case Some(e) => new RuntimeException("Unexpected failure in EJEA.", e) case None => new RuntimeException("Unexpected failure in EJEA (root cause not captured).") } - self ! FailedNonRetryableResponse(jobKey, terminationException, None) + self ! JobFailedNonRetryableResponse(jobKey, terminationException, None) } stay @@ -412,23 +181,28 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, case Event(Terminated(actorRef), stateData) => handleTerminated(actorRef) using stateData.removeEngineJobExecutionActor(actorRef) case Event(MetadataPutFailed(action, error), _) => // Do something useful here?? - workflowLogger.warn(s"$tag Put failed for Metadata action $action : ${error.getMessage}") - stay + workflowLogger.warn(s"$tag Put failed for Metadata action $action", error) + stay() case Event(MetadataPutAcknowledgement(_), _) => stay() case Event(EngineLifecycleActorAbortCommand, stateData) => - if (stateData.backendJobExecutionActors.nonEmpty) { - log.info(s"$tag: Abort received. Aborting ${stateData.backendJobExecutionActors.size} EJEAs") - stateData.backendJobExecutionActors.values foreach {_ ! AbortJobCommand} + if (stateData.hasRunningActors) { + log.info(s"$tag: Abort received. " + + s"Aborting ${stateData.backendJobExecutionActors.size} Job Execution Actors" + + s"and ${stateData.subWorkflowExecutionActors.size} Sub Workflow Execution Actors" + ) + stateData.backendJobExecutionActors.values foreach { _ ! AbortJobCommand } + stateData.subWorkflowExecutionActors.values foreach { _ ! EngineLifecycleActorAbortCommand } goto(WorkflowExecutionAbortingState) } else { goto(WorkflowExecutionAbortedState) } case Event(EngineStatsActor.JobCountQuery, data) => sender ! EngineStatsActor.JobCount(data.backendJobExecutionActors.size) + data.subWorkflowExecutionActors.values foreach { _ forward EngineStatsActor.JobCountQuery } stay() case unhandledMessage => workflowLogger.warn(s"$tag received an unhandled message: ${unhandledMessage.event} in state: $stateName") - stay + stay() } onTransition { @@ -440,11 +214,51 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, } onTransition { - case _ -> WorkflowExecutionSuccessfulState => - pushWorkflowOutputMetadata(nextStateData) - context.parent ! WorkflowExecutionSucceededResponse(nextStateData.executionStore, nextStateData.outputStore) case _ -> WorkflowExecutionAbortedState => - context.parent ! WorkflowExecutionAbortedResponse(nextStateData.executionStore, nextStateData.outputStore) + context.parent ! WorkflowExecutionAbortedResponse(nextStateData.jobExecutionMap) + } + + private def handleNonRetryableFailure(stateData: WorkflowExecutionActorData, failedJobKey: JobKey, reason: Throwable, jobExecutionMap: JobExecutionMap) = { + val mergedStateData = stateData.mergeExecutionDiff(WorkflowExecutionDiff(Map(failedJobKey -> ExecutionStatus.Failed))) + .removeCallExecutionActor(failedJobKey) + .addExecutions(jobExecutionMap) + + if (workflowDescriptor.getWorkflowOption(WorkflowFailureMode).contains(ContinueWhilePossible.toString)) { + mergedStateData.workflowCompletionStatus match { + case Some(completionStatus) if completionStatus == Failed => + context.parent ! WorkflowExecutionFailedResponse(stateData.jobExecutionMap, reason) + goto(WorkflowExecutionFailedState) using mergedStateData + case _ => + stay() using startRunnableScopes(mergedStateData) + } + } else { + context.parent ! WorkflowExecutionFailedResponse(stateData.jobExecutionMap, reason) + goto(WorkflowExecutionFailedState) using mergedStateData + } + } + + private def handleWorkflowSuccessful(data: WorkflowExecutionActorData) = { + import cromwell.util.JsonFormatting.WdlValueJsonFormatter._ + import spray.json._ + + val (response, finalState) = workflowDescriptor.workflow.evaluateOutputs( + workflowDescriptor.inputs, + data.expressionLanguageFunctions, + data.outputStore.fetchCallOutputEntries + ) map { workflowOutputs => + workflowLogger.info( + s"""Workflow ${workflowDescriptor.workflow.unqualifiedName} complete. Final Outputs: + |${workflowOutputs.toJson.prettyPrint}""".stripMargin + ) + pushWorkflowOutputMetadata(workflowOutputs) + (WorkflowExecutionSucceededResponse(data.jobExecutionMap, workflowOutputs mapValues JobOutput.apply), WorkflowExecutionSuccessfulState) + } recover { + case ex => + (WorkflowExecutionFailedResponse(data.jobExecutionMap, ex), WorkflowExecutionFailedState) + } get + + context.parent ! response + goto(finalState) using data } private def handleRetryableFailure(jobKey: BackendJobDescriptorKey, reason: Throwable, returnCode: Option[Int]) = { @@ -455,100 +269,43 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, /* Currently, we update the status of the old key to Preempted, and add a new entry (with the #attempts incremented by 1) * to the execution store with status as NotStarted. This allows startRunnableCalls to re-execute this job */ val executionDiff = WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Preempted, newJobKey -> ExecutionStatus.NotStarted)) - val newData = stateData.mergeExecutionDiff(executionDiff) + val newData = stateData.mergeExecutionDiff(executionDiff).removeCallExecutionActor(jobKey) stay() using startRunnableScopes(newData) } else { workflowLogger.warn(s"Exhausted maximum number of retries for job ${jobKey.tag}. Failing.") - goto(WorkflowExecutionFailedState) using stateData.mergeExecutionDiff(WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Failed))) + goto(WorkflowExecutionFailedState) using stateData.mergeExecutionDiff(WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Failed))).removeCallExecutionActor(jobKey) } } - private def handleJobSuccessful(jobKey: JobKey, outputs: JobOutputs, data: WorkflowExecutionActorData) = { + private def handleCallSuccessful(jobKey: JobKey, outputs: CallOutputs, data: WorkflowExecutionActorData, jobExecutionMap: JobExecutionMap) = { workflowLogger.debug(s"Job ${jobKey.tag} succeeded!") - val newData = data.jobExecutionSuccess(jobKey, outputs) + val newData = data.callExecutionSuccess(jobKey, outputs).addExecutions(jobExecutionMap) newData.workflowCompletionStatus match { case Some(ExecutionStatus.Done) => - workflowLogger.info(newData.outputsJson()) - goto(WorkflowExecutionSuccessfulState) using newData + handleWorkflowSuccessful(newData) case Some(sts) => - context.parent ! WorkflowExecutionFailedResponse(stateData.executionStore, stateData.outputStore, List(new Exception("One or more jobs failed in fail-slow mode"))) + context.parent ! WorkflowExecutionFailedResponse(data.jobExecutionMap, new Exception("One or more jobs failed in fail-slow mode")) goto(WorkflowExecutionFailedState) using newData case _ => stay() using startRunnableScopes(newData) } } - - private def pushWorkflowOutputMetadata(data: WorkflowExecutionActorData) = { - val reportableOutputs = workflowDescriptor.backendDescriptor.workflowNamespace.workflow.outputs - val keyValues = data.outputStore.store filterKeys { - _.index.isEmpty - } flatMap { - case (key, value) => - value collect { - case entry if isReportableOutput(key.call, entry, reportableOutputs) => - s"${key.call.fullyQualifiedName}.${entry.name}" -> entry.wdlValue - } - } collect { - case (key, Some(wdlValue)) => (key, wdlValue) - } - - val events = keyValues match { - case empty if empty.isEmpty => List(MetadataEvent.empty(MetadataKey(workflowId, None, WorkflowMetadataKeys.Outputs))) - case _ => keyValues flatMap { - case (outputName, outputValue) => - wdlValueToMetadataEvents(MetadataKey(workflowId, None, s"${WorkflowMetadataKeys.Outputs}:$outputName"), outputValue) - } - } - - serviceRegistryActor ! PutMetadataAction(events) - } - - private def isReportableOutput(scope: Scope, entry: OutputEntry, - reportableOutputs: Seq[ReportableSymbol]): Boolean = { - reportableOutputs exists { reportableOutput => - reportableOutput.fullyQualifiedName == s"${scope.fullyQualifiedName}.${entry.name}" - } - } - - private def pushSuccessfulJobMetadata(jobKey: JobKey, returnCode: Option[Int], outputs: JobOutputs) = { - val completionEvents = completedJobMetadataEvents(jobKey, ExecutionStatus.Done, returnCode) - - val outputEvents = outputs match { - case empty if empty.isEmpty => - List(MetadataEvent.empty(metadataKey(jobKey, s"${CallMetadataKeys.Outputs}"))) - case _ => - outputs flatMap { case (lqn, value) => wdlValueToMetadataEvents(metadataKey(jobKey, s"${CallMetadataKeys.Outputs}:$lqn"), value.wdlValue) } - } - - serviceRegistryActor ! PutMetadataAction(completionEvents ++ outputEvents) - } - - private def pushFailedJobMetadata(jobKey: JobKey, returnCode: Option[Int], failure: Throwable, retryableFailure: Boolean) = { - val failedState = if (retryableFailure) ExecutionStatus.Preempted else ExecutionStatus.Failed - val completionEvents = completedJobMetadataEvents(jobKey, failedState, returnCode) - val retryableFailureEvent = MetadataEvent(metadataKey(jobKey, CallMetadataKeys.RetryableFailure), MetadataValue(retryableFailure)) - val failureEvents = throwableToMetadataEvents(metadataKey(jobKey, s"${CallMetadataKeys.Failures}[$randomNumberString]"), failure).+:(retryableFailureEvent) - - serviceRegistryActor ! PutMetadataAction(completionEvents ++ failureEvents) - } - - private def randomNumberString: String = Random.nextInt.toString.stripPrefix("-") - - private def completedJobMetadataEvents(jobKey: JobKey, executionStatus: ExecutionStatus, returnCode: Option[Int]) = { - val returnCodeEvent = returnCode map { rc => - List(MetadataEvent(metadataKey(jobKey, CallMetadataKeys.ReturnCode), MetadataValue(rc))) + + private def handleCallAborted(data: WorkflowExecutionActorData, jobKey: JobKey, jobExecutionMap: JobExecutionMap) = { + workflowLogger.info(s"$tag job aborted: ${jobKey.tag}") + val newStateData = data.removeCallExecutionActor(jobKey).addExecutions(jobExecutionMap) + if (!newStateData.hasRunningActors) { + workflowLogger.info(s"$tag all jobs aborted") + goto(WorkflowExecutionAbortedState) + } else { + stay() using newStateData } - - List( - MetadataEvent(metadataKey(jobKey, CallMetadataKeys.ExecutionStatus), MetadataValue(executionStatus)), - MetadataEvent(metadataKey(jobKey, CallMetadataKeys.End), MetadataValue(OffsetDateTime.now)) - ) ++ returnCodeEvent.getOrElse(List.empty) } /** * Attempt to start all runnable jobs and return updated state data. This will create a new copy - * of the state data including new pending persists. + * of the state data. */ @tailrec private def startRunnableScopes(data: WorkflowExecutionActorData): WorkflowExecutionActorData = { @@ -562,6 +319,7 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, case k: BackendJobDescriptorKey => processRunnableJob(k, data) case k: ScatterKey => processRunnableScatter(k, data) case k: CollectorKey => processRunnableCollector(k, data) + case k: SubWorkflowKey => processRunnableSubWorkflow(k, data) case k => val exception = new UnsupportedOperationException(s"Unknown entry in execution store: ${k.tag}") self ! JobInitializationFailed(k, exception) @@ -571,47 +329,14 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, TryUtil.sequence(executionDiffs) match { case Success(diffs) => // Update the metadata for the jobs we just sent to EJEAs (they'll start off queued up waiting for tokens): - pushQueuedJobMetadata(diffs) + pushQueuedCallMetadata(diffs) if (diffs.exists(_.containsNewEntry)) { startRunnableScopes(data.mergeExecutionDiffs(diffs)) } else { data.mergeExecutionDiffs(diffs) } - case Failure(e) => data - } - } - - private def pushNewJobMetadata(jobKey: BackendJobDescriptorKey, backendName: String) = { - val startEvents = List( - MetadataEvent(metadataKey(jobKey, CallMetadataKeys.Start), MetadataValue(OffsetDateTime.now)), - MetadataEvent(metadataKey(jobKey, CallMetadataKeys.Backend), MetadataValue(backendName)) - ) - - serviceRegistryActor ! PutMetadataAction(startEvents) - } - - private def pushQueuedJobMetadata(diffs: Seq[WorkflowExecutionDiff]) = { - val startingEvents = for { - diff <- diffs - (jobKey, executionState) <- diff.executionStoreChanges if jobKey.isInstanceOf[BackendJobDescriptorKey] && executionState == ExecutionStatus.QueuedInCromwell - } yield MetadataEvent(metadataKey(jobKey, CallMetadataKeys.ExecutionStatus), MetadataValue(ExecutionStatus.QueuedInCromwell)) - serviceRegistryActor ! PutMetadataAction(startingEvents) - } - - private def pushRunningJobMetadata(jobDescriptor: BackendJobDescriptor) = { - val inputEvents = jobDescriptor.inputDeclarations match { - case empty if empty.isEmpty => - List(MetadataEvent.empty(metadataKey(jobDescriptor.key, s"${CallMetadataKeys.Inputs}"))) - case inputs => - inputs flatMap { - case (inputName, inputValue) => - wdlValueToMetadataEvents(metadataKey(jobDescriptor.key, s"${CallMetadataKeys.Inputs}:${inputName.unqualifiedName}"), inputValue) - } + case Failure(e) => throw new RuntimeException("Unexpected engine failure", e) } - - val runningEvent = List(MetadataEvent(metadataKey(jobDescriptor.key, CallMetadataKeys.ExecutionStatus), MetadataValue(ExecutionStatus.Running))) - - serviceRegistryActor ! PutMetadataAction(runningEvent ++ inputEvents) } private def processRunnableJob(jobKey: BackendJobDescriptorKey, data: WorkflowExecutionActorData): Try[WorkflowExecutionDiff] = { @@ -622,7 +347,7 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, workflowLogger.error(exception, s"$tag $message") throw exception case Some(backendName) => - factories.get(backendName) match { + backendFactories.get(backendName) match { case Some(factory) => val ejeaName = s"${workflowDescriptor.id}-EngineJobExecutionActor-${jobKey.tag}" val backendSingleton = backendSingletonCollection.backendSingletonActors(backendName) @@ -631,7 +356,7 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingleton, backendName, workflowDescriptor.callCachingMode) val ejeaRef = context.actorOf(ejeaProps, ejeaName) context watch ejeaRef - pushNewJobMetadata(jobKey, backendName) + pushNewCallMetadata(jobKey, Option(backendName)) ejeaRef ! EngineJobExecutionActor.Execute Success(WorkflowExecutionDiff( executionStoreChanges = Map(jobKey -> ExecutionStatus.QueuedInCromwell), @@ -641,6 +366,19 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, } } } + + private def processRunnableSubWorkflow(key: SubWorkflowKey, data: WorkflowExecutionActorData): Try[WorkflowExecutionDiff] = { + val sweaRef = context.actorOf( + SubWorkflowExecutionActor.props(key, data, backendFactories, serviceRegistryActor, jobStoreActor, subWorkflowStoreActor, + callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection, initializationData, restarting), + s"SubWorkflowExecutionActor-${key.tag}" + ) + + pushNewCallMetadata(key, None) + sweaRef ! SubWorkflowExecutionActor.Execute + + Success(WorkflowExecutionDiff(Map(key -> ExecutionStatus.QueuedInCromwell))) + } private def processRunnableScatter(scatterKey: ScatterKey, data: WorkflowExecutionActorData): Try[WorkflowExecutionDiff] = { val lookup = scatterKey.scope.lookupFunction( @@ -656,13 +394,151 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, } private def processRunnableCollector(collector: CollectorKey, data: WorkflowExecutionActorData): Try[WorkflowExecutionDiff] = { - val shards = data.executionStore.findShardEntries(collector) collect { case (k: BackendJobDescriptorKey, v) if v == ExecutionStatus.Done => k } + val shards = data.executionStore.findShardEntries(collector) collect { case (k: CallKey, v) if v == ExecutionStatus.Done => k } data.outputStore.generateCollectorOutput(collector, shards) match { case Failure(e) => Failure(new RuntimeException(s"Failed to collect output shards for call ${collector.tag}")) case Success(outputs) => self ! ScatterCollectionSucceededResponse(collector, outputs) Success(WorkflowExecutionDiff(Map(collector -> ExecutionStatus.Starting))) } } - - private def metadataKey(jobKey: JobKey, myKey: String) = MetadataKey(workflowDescriptor.id, Option(MetadataJobKey(jobKey.scope.fullyQualifiedName, jobKey.index, jobKey.attempt)), myKey) } + +object WorkflowExecutionActor { + + /** + * States + */ + sealed trait WorkflowExecutionActorState { + def terminal = false + } + + sealed trait WorkflowExecutionActorTerminalState extends WorkflowExecutionActorState { + override val terminal = true + } + + case object WorkflowExecutionPendingState extends WorkflowExecutionActorState + + case object WorkflowExecutionInProgressState extends WorkflowExecutionActorState + + case object WorkflowExecutionAbortingState extends WorkflowExecutionActorState + + case object WorkflowExecutionSuccessfulState extends WorkflowExecutionActorTerminalState + + case object WorkflowExecutionFailedState extends WorkflowExecutionActorTerminalState + + case object WorkflowExecutionAbortedState extends WorkflowExecutionActorTerminalState + + /** + * Commands + */ + sealed trait WorkflowExecutionActorCommand + + case object ExecuteWorkflowCommand extends WorkflowExecutionActorCommand + + /** + * Responses + */ + sealed trait WorkflowExecutionActorResponse { + def jobExecutionMap: JobExecutionMap + } + + case class WorkflowExecutionSucceededResponse(jobExecutionMap: JobExecutionMap, outputs: CallOutputs) + extends WorkflowExecutionActorResponse { + override def toString = "WorkflowExecutionSucceededResponse" + } + + case class WorkflowExecutionAbortedResponse(jobExecutionMap: JobExecutionMap) + extends WorkflowExecutionActorResponse with EngineLifecycleActorAbortedResponse { + override def toString = "WorkflowExecutionAbortedResponse" + } + + final case class WorkflowExecutionFailedResponse(jobExecutionMap: JobExecutionMap, reason: Throwable) extends WorkflowExecutionActorResponse { + override def toString = "WorkflowExecutionFailedResponse" + } + + /** + * Internal control flow messages + */ + private case class JobInitializationFailed(jobKey: JobKey, throwable: Throwable) + + private case class ScatterCollectionFailedResponse(collectorKey: CollectorKey, throwable: Throwable) + + private case class ScatterCollectionSucceededResponse(collectorKey: CollectorKey, outputs: CallOutputs) + + case class SubWorkflowSucceededResponse(key: SubWorkflowKey, jobExecutionMap: JobExecutionMap, outputs: CallOutputs) + + case class SubWorkflowFailedResponse(key: SubWorkflowKey, jobExecutionMap: JobExecutionMap, reason: Throwable) + + case class SubWorkflowAbortedResponse(key: SubWorkflowKey, jobExecutionMap: JobExecutionMap) + + /** + * Internal ADTs + */ + case class ScatterKey(scope: Scatter) extends JobKey { + override val index = None + // When scatters are nested, this might become Some(_) + override val attempt = 1 + override val tag = scope.unqualifiedName + + /** + * Creates a sub-ExecutionStore with Starting entries for each of the scoped children. + * + * @param count Number of ways to scatter the children. + * @return ExecutionStore of scattered children. + */ + def populate(count: Int): Map[JobKey, ExecutionStatus.Value] = { + val keys = this.scope.children flatMap { + explode(_, count) + } + keys map { + _ -> ExecutionStatus.NotStarted + } toMap + } + + private def explode(scope: Scope, count: Int): Seq[JobKey] = { + scope match { + case call: TaskCall => + val shards = (0 until count) map { i => BackendJobDescriptorKey(call, Option(i), 1) } + shards :+ CollectorKey(call) + case call: WorkflowCall => + val shards = (0 until count) map { i => SubWorkflowKey(call, Option(i), 1) } + shards :+ CollectorKey(call) + case scatter: Scatter => + throw new UnsupportedOperationException("Nested Scatters are not supported (yet) ... but you might try a sub workflow to achieve the same effect!") + case e => + throw new UnsupportedOperationException(s"Scope ${e.getClass.getName} is not supported.") + } + } + } + + // Represents a scatter collection for a call in the execution store + case class CollectorKey(scope: Call) extends JobKey { + override val index = None + override val attempt = 1 + override val tag = s"Collector-${scope.unqualifiedName}" + } + + case class SubWorkflowKey(scope: WorkflowCall, index: ExecutionIndex, attempt: Int) extends CallKey { + override val tag = s"SubWorkflow-${scope.unqualifiedName}:${index.fromIndex}:$attempt" + } + + case class WorkflowExecutionException[T <: Throwable](exceptions: NonEmptyList[T]) extends ThrowableAggregation { + override val throwables = exceptions.toList + override val exceptionContext = s"WorkflowExecutionActor" + } + + private lazy val DefaultMaxRetriesFallbackValue = 10 + + def props(workflowDescriptor: EngineWorkflowDescriptor, + serviceRegistryActor: ActorRef, + jobStoreActor: ActorRef, + subWorkflowStoreActor: ActorRef, + callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection, + initializationData: AllBackendInitializationData, + restarting: Boolean): Props = { + Props(WorkflowExecutionActor(workflowDescriptor, serviceRegistryActor, jobStoreActor, subWorkflowStoreActor, + callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection, initializationData, restarting)).withDispatcher(EngineDispatcher) + } +} \ No newline at end of file diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala index d06514284..665ef93e9 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala @@ -1,9 +1,11 @@ package cromwell.engine.workflow.lifecycle.execution import akka.actor.ActorRef +import cromwell.backend._ import cromwell.core.ExecutionStatus._ -import cromwell.core.OutputStore.{OutputCallKey, OutputEntry} import cromwell.core._ +import cromwell.engine.workflow.lifecycle.execution.OutputStore.{OutputCallKey, OutputEntry} +import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.SubWorkflowKey import cromwell.engine.{EngineWorkflowDescriptor, WdlFunctions} import cromwell.util.JsonFormatting.WdlValueJsonFormatter import wdl4s.{GraphNode, Scope} @@ -13,26 +15,51 @@ object WorkflowExecutionDiff { } /** Data differential between current execution data, and updates performed in a method that needs to be merged. */ final case class WorkflowExecutionDiff(executionStoreChanges: Map[JobKey, ExecutionStatus], - engineJobExecutionActorAdditions: Map[ActorRef, JobKey] = Map.empty) { + engineJobExecutionActorAdditions: Map[ActorRef, BackendJobDescriptorKey] = Map.empty) { def containsNewEntry = executionStoreChanges.exists(_._2 == NotStarted) } +object WorkflowExecutionActorData { + def empty(workflowDescriptor: EngineWorkflowDescriptor) = { + new WorkflowExecutionActorData( + workflowDescriptor, + ExecutionStore.empty, + Map.empty, + Map.empty, + Map.empty, + Map.empty, + OutputStore.empty + ) + } +} + case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescriptor, executionStore: ExecutionStore, backendJobExecutionActors: Map[JobKey, ActorRef], - engineJobExecutionActors: Map[ActorRef, JobKey], + engineJobExecutionActors: Map[ActorRef, BackendJobDescriptorKey], + subWorkflowExecutionActors: Map[SubWorkflowKey, ActorRef], + downstreamExecutionMap: JobExecutionMap, outputStore: OutputStore) { val expressionLanguageFunctions = new WdlFunctions(workflowDescriptor.pathBuilders) - def jobExecutionSuccess(jobKey: JobKey, outputs: JobOutputs) = this.copy( - executionStore = executionStore.add(Map(jobKey -> Done)), - backendJobExecutionActors = backendJobExecutionActors - jobKey, - outputStore = outputStore.add(updateSymbolStoreEntry(jobKey, outputs)) - ) + def callExecutionSuccess(jobKey: JobKey, outputs: CallOutputs) = { + val (newJobExecutionActors, newSubWorkflowExecutionActors) = jobKey match { + case jobKey: BackendJobDescriptorKey => (backendJobExecutionActors - jobKey, subWorkflowExecutionActors) + case swKey: SubWorkflowKey => (backendJobExecutionActors, subWorkflowExecutionActors - swKey) + case _ => (backendJobExecutionActors, subWorkflowExecutionActors) + } + + this.copy( + executionStore = executionStore.add(Map(jobKey -> Done)), + backendJobExecutionActors = newJobExecutionActors, + subWorkflowExecutionActors = newSubWorkflowExecutionActors, + outputStore = outputStore.add(updateSymbolStoreEntry(jobKey, outputs)) + ) + } /** Add the outputs for the specified `JobKey` to the symbol cache. */ - private def updateSymbolStoreEntry(jobKey: JobKey, outputs: JobOutputs) = { + private def updateSymbolStoreEntry(jobKey: JobKey, outputs: CallOutputs) = { val newOutputEntries = outputs map { case (name, value) => OutputEntry(name, value.wdlValue.wdlType, Option(value.wdlValue)) } @@ -73,13 +100,26 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript this.copy(engineJobExecutionActors = engineJobExecutionActors - actorRef) } - def addBackendJobExecutionActor(jobKey: JobKey, actor: Option[ActorRef]): WorkflowExecutionActorData = actor match { - case Some(actorRef) => this.copy(backendJobExecutionActors = backendJobExecutionActors + (jobKey -> actorRef)) + def addCallExecutionActor(jobKey: JobKey, actor: Option[ActorRef]): WorkflowExecutionActorData = actor match { + case Some(actorRef) => + jobKey match { + case jobKey: BackendJobDescriptorKey => this.copy(backendJobExecutionActors = backendJobExecutionActors + (jobKey -> actorRef)) + case swKey: SubWorkflowKey => this.copy(subWorkflowExecutionActors = subWorkflowExecutionActors + (swKey -> actorRef)) + case _ => this + } case None => this } - def removeBackendJobExecutionActor(jobKey: JobKey): WorkflowExecutionActorData = { - this.copy(backendJobExecutionActors = backendJobExecutionActors - jobKey) + def removeCallExecutionActor(jobKey: JobKey): WorkflowExecutionActorData = { + jobKey match { + case jobKey: BackendJobDescriptorKey => this.copy(backendJobExecutionActors = backendJobExecutionActors - jobKey) + case swKey: SubWorkflowKey => this.copy(subWorkflowExecutionActors = subWorkflowExecutionActors - swKey) + case _ => this + } + } + + def addExecutions(jobExecutionMap: JobExecutionMap): WorkflowExecutionActorData = { + this.copy(downstreamExecutionMap = downstreamExecutionMap ++ jobExecutionMap) } def outputsJson(): String = { @@ -104,5 +144,11 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript def mergeExecutionDiffs(diffs: Traversable[WorkflowExecutionDiff]): WorkflowExecutionActorData = { diffs.foldLeft(this)((newData, diff) => newData.mergeExecutionDiff(diff)) } - + + def jobExecutionMap: JobExecutionMap = { + val keys = executionStore.store.collect({case (k: BackendJobDescriptorKey, status) if status != ExecutionStatus.NotStarted => k }).toList + downstreamExecutionMap updated (workflowDescriptor.backendDescriptor, keys) + } + + def hasRunningActors = backendJobExecutionActors.nonEmpty || subWorkflowExecutionActors.nonEmpty } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowMetadataHelper.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowMetadataHelper.scala new file mode 100644 index 000000000..d569f1fae --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowMetadataHelper.scala @@ -0,0 +1,37 @@ +package cromwell.engine.workflow.lifecycle.execution + +import java.time.OffsetDateTime + +import akka.actor.ActorRef +import cromwell.core.{WorkflowId, WorkflowMetadataKeys, WorkflowState} +import cromwell.services.metadata.MetadataService._ +import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} + +import scala.util.Random + +trait WorkflowMetadataHelper { + + def serviceRegistryActor: ActorRef + + def pushWorkflowStart(workflowId: WorkflowId) = { + val startEvent = MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.StartTime), MetadataValue(OffsetDateTime.now.toString)) + serviceRegistryActor ! PutMetadataAction(startEvent) + } + + def pushWorkflowEnd(workflowId: WorkflowId) = { + val metadataEventMsg = MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.EndTime), MetadataValue(OffsetDateTime.now.toString)) + serviceRegistryActor ! PutMetadataAction(metadataEventMsg) + } + + def pushWorkflowFailures(workflowId: WorkflowId, failures: List[Throwable]) = { + val failureEvents = failures flatMap { r => throwableToMetadataEvents(MetadataKey(workflowId, None, s"${WorkflowMetadataKeys.Failures}[${Random.nextInt(Int.MaxValue)}]"), r) } + serviceRegistryActor ! PutMetadataAction(failureEvents) + } + + def pushCurrentStateToMetadataService(workflowId: WorkflowId, workflowState: WorkflowState): Unit = { + val metadataEventMsg = MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.Status), + MetadataValue(workflowState)) + serviceRegistryActor ! PutMetadataAction(metadataEventMsg) + } + +} diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala index 7683eba92..674b1ee88 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala @@ -3,7 +3,7 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching import java.nio.file.Path import cats.data.NonEmptyList -import cromwell.backend.BackendJobExecutionActor.SucceededResponse +import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse import cromwell.core.ExecutionIndex.IndexEnhancedIndex import cromwell.core.WorkflowId import cromwell.core.callcaching.HashResult @@ -21,7 +21,7 @@ final case class CallCachingEntryId(id: Int) * Given a database-layer CallCacheStore, this accessor can access the database with engine-friendly data types. */ class CallCache(database: CallCachingSqlDatabase) { - def addToCache(workflowId: WorkflowId, callCacheHashes: CallCacheHashes, response: SucceededResponse)(implicit ec: ExecutionContext): Future[Unit] = { + def addToCache(workflowId: WorkflowId, callCacheHashes: CallCacheHashes, response: JobSucceededResponse)(implicit ec: ExecutionContext): Future[Unit] = { val metaInfo = CallCachingEntry( workflowExecutionUuid = workflowId.toString, callFullyQualifiedName = response.jobKey.call.fullyQualifiedName, diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheWriteActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheWriteActor.scala index f0e9c0186..c6e42b5cc 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheWriteActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheWriteActor.scala @@ -2,14 +2,14 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching import akka.actor.{Actor, ActorLogging, Props} import cromwell.backend.BackendJobExecutionActor -import cromwell.backend.BackendJobExecutionActor.SucceededResponse +import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse import cromwell.core.WorkflowId import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes import scala.concurrent.ExecutionContext import scala.util.{Failure, Success} -case class CallCacheWriteActor(callCache: CallCache, workflowId: WorkflowId, callCacheHashes: CallCacheHashes, succeededResponse: BackendJobExecutionActor.SucceededResponse) extends Actor with ActorLogging { +case class CallCacheWriteActor(callCache: CallCache, workflowId: WorkflowId, callCacheHashes: CallCacheHashes, succeededResponse: BackendJobExecutionActor.JobSucceededResponse) extends Actor with ActorLogging { implicit val ec: ExecutionContext = context.dispatcher @@ -30,7 +30,7 @@ case class CallCacheWriteActor(callCache: CallCache, workflowId: WorkflowId, cal } object CallCacheWriteActor { - def props(callCache: CallCache, workflowId: WorkflowId, callCacheHashes: CallCacheHashes, succeededResponse: SucceededResponse): Props = + def props(callCache: CallCache, workflowId: WorkflowId, callCacheHashes: CallCacheHashes, succeededResponse: JobSucceededResponse): Props = Props(CallCacheWriteActor(callCache, workflowId, callCacheHashes, succeededResponse)) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/package.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/package.scala index 1d3eedd9f..d0350e662 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/package.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/package.scala @@ -1,10 +1,12 @@ package cromwell.engine.workflow.lifecycle +import akka.actor.ActorRef import wdl4s._ -package object execution { - def splitFqn(fullyQualifiedName: FullyQualifiedName): (String, String) = { - val lastIndex = fullyQualifiedName.lastIndexOf(".") - (fullyQualifiedName.substring(0, lastIndex), fullyQualifiedName.substring(lastIndex + 1)) - } +package execution { + + import cromwell.core.CallKey + + final case class JobRunning(key: CallKey, inputs: EvaluatedTaskInputs, executionActor: Option[ActorRef]) + final case class JobStarting(callKey: CallKey) } diff --git a/engine/src/main/scala/cromwell/jobstore/jobstore_.scala b/engine/src/main/scala/cromwell/jobstore/jobstore_.scala index 7fbdd0107..921183d35 100644 --- a/engine/src/main/scala/cromwell/jobstore/jobstore_.scala +++ b/engine/src/main/scala/cromwell/jobstore/jobstore_.scala @@ -5,6 +5,6 @@ import cromwell.core.{WorkflowId, _} case class JobStoreKey(workflowId: WorkflowId, callFqn: String, index: Option[Int], attempt: Int) sealed trait JobResult -case class JobResultSuccess(returnCode: Option[Int], jobOutputs: JobOutputs) extends JobResult +case class JobResultSuccess(returnCode: Option[Int], jobOutputs: CallOutputs) extends JobResult case class JobResultFailure(returnCode: Option[Int], reason: Throwable, retryable: Boolean) extends JobResult diff --git a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala index 2e3d4a315..7aefc3d1c 100644 --- a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala +++ b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala @@ -13,6 +13,7 @@ import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor import cromwell.engine.workflow.workflowstore.{SqlWorkflowStore, WorkflowStore, WorkflowStoreActor} import cromwell.jobstore.{JobStore, JobStoreActor, SqlJobStore} import cromwell.services.{ServiceRegistryActor, SingletonServicesStore} +import cromwell.subworkflowstore.{SqlSubWorkflowStore, SubWorkflowStoreActor} import net.ceedubs.ficus.Ficus._ /** * An actor which serves as the lord protector for the rest of Cromwell, allowing us to have more fine grain @@ -44,6 +45,9 @@ import net.ceedubs.ficus.Ficus._ lazy val jobStore: JobStore = new SqlJobStore(SingletonServicesStore.databaseInterface) lazy val jobStoreActor = context.actorOf(JobStoreActor.props(jobStore), "JobStoreActor") + lazy val subWorkflowStore = new SqlSubWorkflowStore(SingletonServicesStore.databaseInterface) + lazy val subWorkflowStoreActor = context.actorOf(SubWorkflowStoreActor.props(subWorkflowStore), "SubWorkflowStoreActor") + lazy val callCache: CallCache = new CallCache(SingletonServicesStore.databaseInterface) lazy val callCacheReadActor = context.actorOf(RoundRobinPool(25) .props(CallCacheReadActor.props(callCache)), @@ -60,7 +64,7 @@ import net.ceedubs.ficus.Ficus._ lazy val workflowManagerActor = context.actorOf( WorkflowManagerActor.props( - workflowStoreActor, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, + workflowStoreActor, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, subWorkflowStoreActor, callCacheReadActor, jobExecutionTokenDispenserActor, backendSingletonCollection, abortJobsOnTerminate), "WorkflowManagerActor") diff --git a/engine/src/main/scala/cromwell/subworkflowstore/EmptySubWorkflowStoreActor.scala b/engine/src/main/scala/cromwell/subworkflowstore/EmptySubWorkflowStoreActor.scala new file mode 100644 index 000000000..166d7d685 --- /dev/null +++ b/engine/src/main/scala/cromwell/subworkflowstore/EmptySubWorkflowStoreActor.scala @@ -0,0 +1,17 @@ +package cromwell.subworkflowstore + +import akka.actor.{Actor, ActorLogging, Props} +import cromwell.subworkflowstore.SubWorkflowStoreActor._ + +class EmptySubWorkflowStoreActor extends Actor with ActorLogging { + override def receive: Receive = { + case register: RegisterSubWorkflow => sender() ! SubWorkflowStoreRegisterSuccess(register) + case query: QuerySubWorkflow => sender() ! SubWorkflowNotFound(query) + case complete: WorkflowComplete =>sender() ! SubWorkflowStoreCompleteSuccess(complete) + case unknown => log.error(s"SubWorkflowStoreActor received unknown message: $unknown") + } +} + +object EmptySubWorkflowStoreActor { + def props: Props = Props(new EmptySubWorkflowStoreActor()) +} diff --git a/engine/src/main/scala/cromwell/subworkflowstore/SqlSubWorkflowStore.scala b/engine/src/main/scala/cromwell/subworkflowstore/SqlSubWorkflowStore.scala new file mode 100644 index 000000000..64f21275f --- /dev/null +++ b/engine/src/main/scala/cromwell/subworkflowstore/SqlSubWorkflowStore.scala @@ -0,0 +1,31 @@ +package cromwell.subworkflowstore +import cromwell.database.sql.SubWorkflowStoreSqlDatabase +import cromwell.database.sql.tables.SubWorkflowStoreEntry + +import scala.concurrent.{ExecutionContext, Future} + +class SqlSubWorkflowStore(subWorkflowStoreSqlDatabase: SubWorkflowStoreSqlDatabase) extends SubWorkflowStore { + override def addSubWorkflowStoreEntry(rootWorkflowExecutionUuid: String, + parentWorkflowExecutionUuid: String, + callFullyQualifiedName: String, + jobIndex: Int, + jobAttempt: Int, + subWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Unit] = { + subWorkflowStoreSqlDatabase.addSubWorkflowStoreEntry( + rootWorkflowExecutionUuid, + parentWorkflowExecutionUuid, + callFullyQualifiedName, + jobIndex, + jobAttempt, + subWorkflowExecutionUuid + ) + } + + override def querySubWorkflowStore(parentWorkflowExecutionUuid: String, callFqn: String, jobIndex: Int, jobAttempt: Int)(implicit ec: ExecutionContext): Future[Option[SubWorkflowStoreEntry]] = { + subWorkflowStoreSqlDatabase.querySubWorkflowStore(parentWorkflowExecutionUuid, callFqn, jobIndex, jobAttempt) + } + + override def removeSubWorkflowStoreEntries(parentWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Int] = { + subWorkflowStoreSqlDatabase.removeSubWorkflowStoreEntries(parentWorkflowExecutionUuid) + } +} diff --git a/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStore.scala b/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStore.scala new file mode 100644 index 000000000..8ad92fa9b --- /dev/null +++ b/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStore.scala @@ -0,0 +1,19 @@ +package cromwell.subworkflowstore + +import cromwell.database.sql.tables.SubWorkflowStoreEntry + +import scala.concurrent.{ExecutionContext, Future} + +trait SubWorkflowStore { + def addSubWorkflowStoreEntry(rootWorkflowExecutionUuid: String, + parentWorkflowExecutionUuid: String, + callFullyQualifiedName: String, + jobIndex: Int, + jobAttempt: Int, + subWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Unit] + + def querySubWorkflowStore(parentWorkflowExecutionUuid: String, callFqn: String, jobIndex: Int, jobAttempt: Int) + (implicit ec: ExecutionContext): Future[Option[SubWorkflowStoreEntry]] + + def removeSubWorkflowStoreEntries(parentWorkflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Int] +} diff --git a/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStoreActor.scala b/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStoreActor.scala new file mode 100644 index 000000000..cf7624087 --- /dev/null +++ b/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStoreActor.scala @@ -0,0 +1,72 @@ +package cromwell.subworkflowstore + +import akka.actor.{Actor, ActorLogging, ActorRef, Props} +import cromwell.core.ExecutionIndex._ +import cromwell.core.{JobKey, WorkflowId} +import cromwell.database.sql.tables.SubWorkflowStoreEntry +import cromwell.subworkflowstore.SubWorkflowStoreActor._ + +import scala.concurrent.ExecutionContext +import scala.util.{Failure, Success} + +class SubWorkflowStoreActor(database: SubWorkflowStore) extends Actor with ActorLogging { + + implicit val ec: ExecutionContext = context.dispatcher + + override def receive = { + case register: RegisterSubWorkflow => registerSubWorkflow(sender(), register) + case query: QuerySubWorkflow => querySubWorkflow(sender(), query) + case complete: WorkflowComplete => workflowComplete(sender(), complete) + case unknown => log.error(s"SubWorkflowStoreActor received unknown message: $unknown") + } + + private def registerSubWorkflow(replyTo: ActorRef, command: RegisterSubWorkflow) = { + database.addSubWorkflowStoreEntry( + command.rootWorkflowExecutionUuid.toString, + command.parentWorkflowExecutionUuid.toString, + command.jobKey.scope.fullyQualifiedName, + command.jobKey.index.fromIndex, + command.jobKey.attempt, + command.subWorkflowExecutionUuid.toString + ) onComplete { + case Success(_) => replyTo ! SubWorkflowStoreRegisterSuccess(command) + case Failure(ex) => replyTo ! SubWorkflowStoreFailure(command, ex) + } + } + + private def querySubWorkflow(replyTo: ActorRef, command: QuerySubWorkflow) = { + val jobKey = command.jobKey + database.querySubWorkflowStore(command.parentWorkflowExecutionUuid.toString, jobKey.scope.fullyQualifiedName, jobKey.index.fromIndex, jobKey.attempt) onComplete { + case Success(Some(result)) => replyTo ! SubWorkflowFound(result) + case Success(None) => replyTo ! SubWorkflowNotFound(command) + case Failure(ex) => replyTo ! SubWorkflowStoreFailure(command, ex) + } + } + + private def workflowComplete(replyTo: ActorRef, command: WorkflowComplete) = { + database.removeSubWorkflowStoreEntries(command.workflowExecutionUuid.toString) onComplete { + case Success(_) => replyTo ! SubWorkflowStoreCompleteSuccess(command) + case Failure(ex) => replyTo ! SubWorkflowStoreFailure(command, ex) + } + } + +} + +object SubWorkflowStoreActor { + sealed trait SubWorkflowStoreActorCommand + case class RegisterSubWorkflow(rootWorkflowExecutionUuid: WorkflowId, parentWorkflowExecutionUuid: WorkflowId, jobKey: JobKey, subWorkflowExecutionUuid: WorkflowId) extends SubWorkflowStoreActorCommand + case class QuerySubWorkflow(parentWorkflowExecutionUuid: WorkflowId, jobKey: JobKey) extends SubWorkflowStoreActorCommand + case class WorkflowComplete(workflowExecutionUuid: WorkflowId) extends SubWorkflowStoreActorCommand + + sealed trait SubWorkflowStoreActorResponse + case class SubWorkflowStoreRegisterSuccess(command: RegisterSubWorkflow) extends SubWorkflowStoreActorResponse + case class SubWorkflowFound(subWorkflowStoreEntry: SubWorkflowStoreEntry) extends SubWorkflowStoreActorResponse + case class SubWorkflowNotFound(command: QuerySubWorkflow) extends SubWorkflowStoreActorResponse + case class SubWorkflowStoreCompleteSuccess(command: SubWorkflowStoreActorCommand) extends SubWorkflowStoreActorResponse + + case class SubWorkflowStoreFailure(command: SubWorkflowStoreActorCommand, failure: Throwable) extends SubWorkflowStoreActorResponse + + def props(database: SubWorkflowStore) = Props( + new SubWorkflowStoreActor(database) + ) +} diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala index 54ddf2786..bbebdecd1 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala @@ -2,6 +2,7 @@ package cromwell.webservice import akka.actor._ import java.lang.Throwable + import cats.data.NonEmptyList import cromwell.core.{WorkflowId, WorkflowSourceFiles} import cromwell.engine.backend.BackendConfiguration @@ -15,6 +16,8 @@ import spray.httpx.SprayJsonSupport._ import spray.json._ import spray.routing._ +import scala.util.{Failure, Success, Try} + trait SwaggerService extends SwaggerUiResourceHttpService { override def swaggerServiceName = "cromwell" @@ -183,13 +186,21 @@ trait CromwellApiService extends HttpService with PerRequestCreator { parameterMultiMap { parameters => val includeKeysOption = NonEmptyList.fromList(parameters.getOrElse("includeKey", List.empty)) val excludeKeysOption = NonEmptyList.fromList(parameters.getOrElse("excludeKey", List.empty)) - (includeKeysOption, excludeKeysOption) match { - case (Some(_), Some(_)) => + val expandSubWorkflowsOption = { + parameters.get("expandSubWorkflows") match { + case Some(v :: Nil) => Try(v.toBoolean) + case _ => Success(false) + } + } + + (includeKeysOption, excludeKeysOption, expandSubWorkflowsOption) match { + case (Some(_), Some(_), _) => failBadRequest(new IllegalArgumentException("includeKey and excludeKey may not be specified together")) - case _ => + case (_, _, Success(expandSubWorkflows)) => withRecognizedWorkflowId(possibleWorkflowId) { id => - handleMetadataRequest(GetSingleWorkflowMetadataAction(id, includeKeysOption, excludeKeysOption)) + handleMetadataRequest(GetSingleWorkflowMetadataAction(id, includeKeysOption, excludeKeysOption, expandSubWorkflows)) } + case (_, _, Failure(ex)) => failBadRequest(new IllegalArgumentException(ex)) } } } diff --git a/engine/src/main/scala/cromwell/webservice/EngineStatsActor.scala b/engine/src/main/scala/cromwell/webservice/EngineStatsActor.scala index 3b83955a3..047eeccb9 100644 --- a/engine/src/main/scala/cromwell/webservice/EngineStatsActor.scala +++ b/engine/src/main/scala/cromwell/webservice/EngineStatsActor.scala @@ -19,9 +19,10 @@ final case class EngineStatsActor(workflowActors: List[ActorRef], replyTo: Actor private var jobCounts = Map.empty[ActorRef, Int] /* - It's possible that WorkflowActors might disappear behind us and never manage to write us back. - Instead of waiting longingly, watching a mailbox which might never receive some love instead wait - a specified period of time and assume anything which was going to reply already has + * FIXME + * Because of sub workflows there is currently no reliable way to know if we received responses from all running WEAs. + * For now, we always wait for the timeout duration before responding to give a chance to all WEAs to respond (even nested ones). + * This could be improved by having WEAs wait for their sub WEAs before sending back the response. */ val scheduledMsg = context.system.scheduler.scheduleOnce(timeout, self, ShutItDown) @@ -31,7 +32,6 @@ final case class EngineStatsActor(workflowActors: List[ActorRef], replyTo: Actor override def receive = { case JobCount(count) => jobCounts += (sender -> count) - if (jobCounts.size == workflowActors.size) reportStats() case ShutItDown => reportStats() case wompWomp => log.error("Unexpected message to EngineStatsActor: {}", wompWomp) @@ -59,5 +59,5 @@ object EngineStatsActor { final case class EngineStats(workflows: Int, jobs: Int) - val MaxTimeToWait = 30 seconds + val MaxTimeToWait = 3 seconds } diff --git a/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala b/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala index d9ed77438..f51e64187 100644 --- a/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala +++ b/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala @@ -4,6 +4,7 @@ import java.time.OffsetDateTime import cats.{Monoid, Semigroup} import cats.instances.map._ +import cromwell.services.metadata.CallMetadataKeys import spray.json._ @@ -30,20 +31,33 @@ object IndexedJsonValue { /** Customized version of Json data structure, to account for timestamped values and lazy array creation */ sealed trait TimestampedJsValue { - def toJson: JsValue + def toJson(expandedValues: Map[String, JsValue]): JsValue def timestamp: OffsetDateTime } private case class TimestampedJsList(v: Map[Int, TimestampedJsValue], timestamp: OffsetDateTime) extends TimestampedJsValue { - override val toJson = JsArray(v.values.toVector map { _.toJson }) + override def toJson(expandedValues: Map[String, JsValue]) = JsArray(v.values.toVector map { _.toJson(expandedValues) }) } private case class TimestampedJsObject(v: Map[String, TimestampedJsValue], timestamp: OffsetDateTime) extends TimestampedJsValue { - override val toJson = JsObject(v mapValues { _.toJson }) + override def toJson(expandedValues: Map[String, JsValue]) = { + val mappedValues = v map { + case (key, subWorkflowId: TimestampedJsPrimitive) if key == CallMetadataKeys.SubWorkflowId => + val subId = subWorkflowId.v.asInstanceOf[JsString] + expandedValues.get(subId.value) map { subMetadata => + CallMetadataKeys.SubWorkflowMetadata -> subMetadata + } getOrElse { + key -> subWorkflowId.v + } + case (key, value) => key -> value.toJson(expandedValues) + } + + JsObject(mappedValues) + } } private class TimestampedJsPrimitive(val v: JsValue, val timestamp: OffsetDateTime) extends TimestampedJsValue { - override val toJson = v + override def toJson(expandedValues: Map[String, JsValue]) = v } private case class TimestampedEmptyJson(override val timestamp: OffsetDateTime) extends TimestampedJsPrimitive(JsObject(Map.empty[String, JsValue]), timestamp) \ No newline at end of file diff --git a/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala b/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala index 0653be425..272e94c75 100644 --- a/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala +++ b/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala @@ -13,8 +13,8 @@ import cromwell.services.ServiceRegistryActor.ServiceRegistryFailure import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata._ import cromwell.webservice.PerRequest.{RequestComplete, RequestCompleteWithHeaders} -import cromwell.webservice.metadata.MetadataBuilderActor.{Idle, MetadataBuilderActorState, WaitingForMetadataService} -import cromwell.webservice.{APIResponse, WorkflowJsonSupport} +import cromwell.webservice.metadata.MetadataBuilderActor.{Idle, MetadataBuilderActorData, MetadataBuilderActorState, WaitingForMetadataService, WaitingForSubWorkflows} +import cromwell.webservice.{APIResponse, PerRequestCreator, WorkflowJsonSupport} import org.slf4j.LoggerFactory import spray.http.{StatusCodes, Uri} import spray.httpx.SprayJsonSupport._ @@ -29,7 +29,21 @@ object MetadataBuilderActor { sealed trait MetadataBuilderActorState case object Idle extends MetadataBuilderActorState case object WaitingForMetadataService extends MetadataBuilderActorState - + case object WaitingForSubWorkflows extends MetadataBuilderActorState + + case class MetadataBuilderActorData( + originalQuery: MetadataQuery, + originalEvents: Seq[MetadataEvent], + subWorkflowsMetadata: Map[String, JsValue], + waitFor: Int + ) { + def withSubWorkflow(id: String, metadata: JsValue) = { + this.copy(subWorkflowsMetadata = subWorkflowsMetadata + ((id, metadata))) + } + + def isComplete = subWorkflowsMetadata.size == waitFor + } + def props(serviceRegistryActor: ActorRef) = { Props(new MetadataBuilderActor(serviceRegistryActor)).withDispatcher(ApiDispatcher) } @@ -138,8 +152,8 @@ object MetadataBuilderActor { events.toList map { e => keyValueToIndexedJson(e.key.key, e.value, e.offsetDateTime) } combineAll } - private def eventsToAttemptMetadata(attempt: Int, events: Seq[MetadataEvent]) = { - val withAttemptField = JsObject(eventsToIndexedJson(events).toJson.asJsObject.fields + (AttemptKey -> JsNumber(attempt))) + private def eventsToAttemptMetadata(expandedValues: Map[String, JsValue])(attempt: Int, events: Seq[MetadataEvent]) = { + val withAttemptField = JsObject(eventsToIndexedJson(events).toJson(expandedValues).asJsObject.fields + (AttemptKey -> JsNumber(attempt))) MetadataForAttempt(attempt, withAttemptField) } @@ -160,10 +174,10 @@ object MetadataBuilderActor { workflowNonStatusEvents ++ sortedStateEvents.headOption.toList } - private def parseWorkflowEventsToTimestampedJsValue(events: Seq[MetadataEvent], includeCallsIfEmpty: Boolean): JsObject = { + private def parseWorkflowEventsToTimestampedJsValue(events: Seq[MetadataEvent], includeCallsIfEmpty: Boolean, expandedValues: Map[String, JsValue]): JsObject = { // Partition if sequence of events in a pair of (Workflow level events, Call level events) val (workflowLevel, callLevel) = events partition { _.key.jobKey.isEmpty } - val foldedWorkflowValues = eventsToIndexedJson(reduceWorkflowEvents(workflowLevel)).toJson.asJsObject + val foldedWorkflowValues = eventsToIndexedJson(reduceWorkflowEvents(workflowLevel)).toJson(expandedValues).asJsObject val callsGroupedByFQN = callLevel groupBy { _.key.jobKey.get.callFqn } val callsGroupedByFQNAndIndex = callsGroupedByFQN mapValues { _ groupBy { _.key.jobKey.get.index } } @@ -171,7 +185,7 @@ object MetadataBuilderActor { val callsMap = callsGroupedByFQNAndIndexAndAttempt mapValues { eventsForIndex => eventsForIndex mapValues { eventsForAttempt => - eventsForAttempt map Function.tupled(eventsToAttemptMetadata) + eventsForAttempt map Function.tupled(eventsToAttemptMetadata(expandedValues)) } map { Function.tupled(attemptMetadataToIndexMetadata) } } mapValues { md => JsArray(md.toVector.sortBy(_.index) flatMap { _.metadata }) } @@ -180,13 +194,13 @@ object MetadataBuilderActor { JsObject(foldedWorkflowValues.fields ++ callData) } - private def parseWorkflowEvents(includeCallsIfEmpty: Boolean)(events: Seq[MetadataEvent]): JsObject = parseWorkflowEventsToTimestampedJsValue(events, includeCallsIfEmpty) + private def parseWorkflowEvents(includeCallsIfEmpty: Boolean, expandedValues: Map[String, JsValue])(events: Seq[MetadataEvent]): JsObject = parseWorkflowEventsToTimestampedJsValue(events, includeCallsIfEmpty, expandedValues) /** * Parse a Seq of MetadataEvent into a full Json metadata response. */ - private def parse(events: Seq[MetadataEvent]): JsObject = { - JsObject(events.groupBy(_.key.workflowId.toString) mapValues parseWorkflowEvents(includeCallsIfEmpty = true)) + private def parse(events: Seq[MetadataEvent], expandedValues: Map[String, JsValue]): JsObject = { + JsObject(events.groupBy(_.key.workflowId.toString) mapValues parseWorkflowEvents(includeCallsIfEmpty = true, expandedValues)) } implicit class EnhancedMetadataValue(val value: MetadataValue) extends AnyVal { @@ -194,12 +208,12 @@ object MetadataBuilderActor { } } -class MetadataBuilderActor(serviceRegistryActor: ActorRef) extends LoggingFSM[MetadataBuilderActorState, Unit] +class MetadataBuilderActor(serviceRegistryActor: ActorRef) extends LoggingFSM[MetadataBuilderActorState, Option[MetadataBuilderActorData]] with DefaultJsonProtocol with WorkflowQueryPagination { import WorkflowJsonSupport._ - startWith(Idle, ()) + startWith(Idle, None) val tag = self.path.name when(Idle) { @@ -214,9 +228,8 @@ class MetadataBuilderActor(serviceRegistryActor: ActorRef) extends LoggingFSM[Me } when(WaitingForMetadataService) { - case Event(MetadataLookupResponse(query, metadata), _) => - context.parent ! RequestComplete((StatusCodes.OK, processMetadataResponse(query, metadata))) - allDone + case Event(MetadataLookupResponse(query, metadata), None) => + processMetadataResponse(query, metadata) case Event(StatusLookupResponse(w, status), _) => context.parent ! RequestComplete((StatusCodes.OK, processStatusResponse(w, status))) allDone @@ -225,7 +238,6 @@ class MetadataBuilderActor(serviceRegistryActor: ActorRef) extends LoggingFSM[Me context.parent ! RequestComplete((StatusCodes.InternalServerError, response)) allDone case Event(WorkflowQuerySuccess(uri: Uri, response, metadata), _) => - import WorkflowJsonSupport._ context.parent ! RequestCompleteWithHeaders(response, generateLinkHeaders(uri, metadata):_*) allDone case Event(failure: WorkflowQueryFailure, _) => @@ -235,10 +247,10 @@ class MetadataBuilderActor(serviceRegistryActor: ActorRef) extends LoggingFSM[Me // Add in an empty output event if there aren't already any output events. val hasOutputs = events exists { _.key.key.startsWith(WorkflowMetadataKeys.Outputs + ":") } val updatedEvents = if (hasOutputs) events else MetadataEvent.empty(MetadataKey(id, None, WorkflowMetadataKeys.Outputs)) +: events - context.parent ! RequestComplete((StatusCodes.OK, workflowMetadataResponse(id, updatedEvents, includeCallsIfEmpty = false))) + context.parent ! RequestComplete((StatusCodes.OK, workflowMetadataResponse(id, updatedEvents, includeCallsIfEmpty = false, Map.empty))) allDone case Event(LogsResponse(w, l), _) => - context.parent ! RequestComplete((StatusCodes.OK, workflowMetadataResponse(w, l, includeCallsIfEmpty = false))) + context.parent ! RequestComplete((StatusCodes.OK, workflowMetadataResponse(w, l, includeCallsIfEmpty = false, Map.empty))) allDone case Event(failure: MetadataServiceFailure, _) => context.parent ! RequestComplete((StatusCodes.InternalServerError, APIResponse.error(failure.reason))) @@ -249,14 +261,76 @@ class MetadataBuilderActor(serviceRegistryActor: ActorRef) extends LoggingFSM[Me context stop self stay() } + + when(WaitingForSubWorkflows) { + case Event(RequestComplete(metadata), Some(data)) => + processSubWorkflowMetadata(metadata, data) + } + + whenUnhandled { + case Event(message, data) => + log.error(s"Received unexpected message $message in state $stateName with data $data") + stay() + } + + def processSubWorkflowMetadata(metadataResponse: Any, data: MetadataBuilderActorData) = { + metadataResponse match { + case (StatusCodes.OK, js: JsObject) => + js.fields.get(WorkflowMetadataKeys.Id) match { + case Some(subId: JsString) => + val newData = data.withSubWorkflow(subId.value, js) + + if (newData.isComplete) { + buildAndStop(data.originalQuery, data.originalEvents, newData.subWorkflowsMetadata) + } else { + stay() using Option(newData) + } + case _ => failAndDie(new RuntimeException("Received unexpected response while waiting for sub workflow metadata.")) + } + case _ => failAndDie(new RuntimeException("Failed to retrieve metadata for a sub workflow.")) + } + } + + def failAndDie(reason: Throwable) = { + context.parent ! RequestComplete((StatusCodes.InternalServerError, APIResponse.error(reason))) + context stop self + stay() + } + + def buildAndStop(query: MetadataQuery, eventsList: Seq[MetadataEvent], expandedValues: Map[String, JsValue]) = { + context.parent ! RequestComplete((StatusCodes.OK, processMetadataEvents(query, eventsList, expandedValues))) + allDone + } + + def processMetadataResponse(query: MetadataQuery, eventsList: Seq[MetadataEvent]) = { + if (query.expandSubWorkflows) { + // Scan events for sub workflow ids + val subWorkflowIds = eventsList.collect({ + case MetadataEvent(key, value, _) if key.key.endsWith(CallMetadataKeys.SubWorkflowId) => value map { _.value } + }).flatten + + // If none is found just proceed to build metadata + if (subWorkflowIds.isEmpty) buildAndStop(query, eventsList, Map.empty) + else { + // Otherwise spin up a metadata builder actor for each sub workflow + subWorkflowIds foreach { subId => + val subMetadataBuilder = context.actorOf(MetadataBuilderActor.props(serviceRegistryActor), PerRequestCreator.endpointActorName) + subMetadataBuilder ! GetMetadataQueryAction(query.copy(workflowId = WorkflowId.fromString(subId))) + } + goto(WaitingForSubWorkflows) using Option(MetadataBuilderActorData(query, eventsList, Map.empty, subWorkflowIds.size)) + } + } else { + buildAndStop(query, eventsList, Map.empty) + } + } - def processMetadataResponse(query: MetadataQuery, eventsList: Seq[MetadataEvent]): JsObject = { + def processMetadataEvents(query: MetadataQuery, eventsList: Seq[MetadataEvent], expandedValues: Map[String, JsValue]): JsObject = { // Should we send back some message ? Or even fail the request instead ? if (eventsList.isEmpty) JsObject(Map.empty[String, JsValue]) else { query match { - case MetadataQuery(w, _, _, _, _) => workflowMetadataResponse(w, eventsList) - case _ => MetadataBuilderActor.parse(eventsList) + case MetadataQuery(w, _, _, _, _, _) => workflowMetadataResponse(w, eventsList, includeCallsIfEmpty = true, expandedValues) + case _ => MetadataBuilderActor.parse(eventsList, expandedValues) } } } @@ -268,7 +342,7 @@ class MetadataBuilderActor(serviceRegistryActor: ActorRef) extends LoggingFSM[Me )) } - private def workflowMetadataResponse(workflowId: WorkflowId, eventsList: Seq[MetadataEvent], includeCallsIfEmpty: Boolean = true) = { - JsObject(MetadataBuilderActor.parseWorkflowEvents(includeCallsIfEmpty)(eventsList).fields + ("id" -> JsString(workflowId.toString))) + private def workflowMetadataResponse(workflowId: WorkflowId, eventsList: Seq[MetadataEvent], includeCallsIfEmpty: Boolean, expandedValues: Map[String, JsValue]) = { + JsObject(MetadataBuilderActor.parseWorkflowEvents(includeCallsIfEmpty, expandedValues)(eventsList).fields + ("id" -> JsString(workflowId.toString))) } } diff --git a/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala b/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala index e0315d92f..537440935 100644 --- a/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala +++ b/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala @@ -13,7 +13,7 @@ class ArrayOfArrayCoercionSpec extends CromwellTestKitSpec { sampleWdl = SampleWdl.ArrayOfArrays, eventFilter = EventFilter.info(pattern = "Workflow complete", occurrences = 1), expectedOutputs = Map( - "wf.subtask.concatenated" -> WdlArray(WdlArrayType(WdlStringType), Seq( + "wf_subtask_concatenated" -> WdlArray(WdlArrayType(WdlStringType), Seq( WdlString("foo\nbar\nbaz"), WdlString("third\nfourth") )) diff --git a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala index ad6cef3d9..f8f5d432a 100644 --- a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala @@ -22,9 +22,9 @@ class ArrayWorkflowSpec extends CromwellTestKitSpec { sampleWdl = SampleWdl.ArrayIO, eventFilter = EventFilter.info(pattern = "Workflow complete", occurrences = 1), expectedOutputs = Map( - "wf.count_lines.count" -> WdlInteger(3), - "wf.count_lines_array.count" -> WdlInteger(3), - "wf.serialize.contents" -> WdlString("str1\nstr2\nstr3") + "wf_count_lines_count" -> WdlInteger(3), + "wf_count_lines_array_count" -> WdlInteger(3), + "wf_serialize_contents" -> WdlString("str1\nstr2\nstr3") ) ) } @@ -54,7 +54,7 @@ class ArrayWorkflowSpec extends CromwellTestKitSpec { } "Coerce Array[String] to Array[File] when running the workflow" in { val outputs = Map( - "wf.cat.lines" -> WdlArray(WdlArrayType(WdlStringType), Seq( + "wf_cat_lines" -> WdlArray(WdlArrayType(WdlStringType), Seq( WdlString("line1"), WdlString("line2"), WdlString("line3"), diff --git a/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala b/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala index d8d910913..b6465a8ac 100644 --- a/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala +++ b/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala @@ -31,7 +31,7 @@ class CopyWorkflowOutputsSpec extends CromwellTestKitSpec { pattern = "transition from FinalizingWorkflowState to WorkflowSucceededState", occurrences = 1), runtime = "", workflowOptions = s""" { "final_workflow_outputs_dir": "$tmpDir" } """, - expectedOutputs = Seq("A.out", "A.out2", "B.outs") map { o => ("wfoutputs." + o) -> CromwellTestKitSpec.AnyValueIsFine } toMap, + expectedOutputs = Seq("A_out", "A_out2", "B_outs") map { o => ("wfoutputs_" + o) -> CromwellTestKitSpec.AnyValueIsFine } toMap, allowOtherOutputs = false ) @@ -64,7 +64,7 @@ class CopyWorkflowOutputsSpec extends CromwellTestKitSpec { pattern = "transition from FinalizingWorkflowState to WorkflowSucceededState", occurrences = 1), runtime = "", workflowOptions = s""" { "final_workflow_outputs_dir": "$tmpDir" } """, - expectedOutputs = Map("wfoutputs.A.outs" -> CromwellTestKitSpec.AnyValueIsFine), + expectedOutputs = Map("wfoutputs_A_outs" -> CromwellTestKitSpec.AnyValueIsFine), allowOtherOutputs = false ) diff --git a/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala index 3c0ef26e7..4d02a5ec8 100644 --- a/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala +++ b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala @@ -22,6 +22,7 @@ import cromwell.server.{CromwellRootActor, CromwellSystem} import cromwell.services.ServiceRegistryActor import cromwell.services.metadata.MetadataQuery import cromwell.services.metadata.MetadataService._ +import cromwell.subworkflowstore.EmptySubWorkflowStoreActor import cromwell.util.SampleWdl import cromwell.webservice.PerRequest.RequestComplete import cromwell.webservice.metadata.MetadataBuilderActor @@ -31,7 +32,7 @@ import org.scalatest.time.{Millis, Seconds, Span} import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike} import spray.http.StatusCode import spray.json._ -import wdl4s.Call +import wdl4s.TaskCall import wdl4s.expression.{NoFunctions, WdlStandardLibraryFunctions} import wdl4s.types._ import wdl4s.values._ @@ -44,7 +45,7 @@ import scala.util.matching.Regex case class TestBackendLifecycleActorFactory(configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Set[Call], + calls: Set[TaskCall], serviceRegistryActor: ActorRef): Option[Props] = None override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, @@ -374,7 +375,7 @@ abstract class CromwellTestKitSpec(val twms: TestWorkflowManagerSystem = new Cro def getWorkflowMetadata(workflowId: WorkflowId, serviceRegistryActor: ActorRef, key: Option[String] = None)(implicit ec: ExecutionContext): JsObject = { // MetadataBuilderActor sends its response to context.parent, so we can't just use an ask to talk to it here - val message = GetMetadataQueryAction(MetadataQuery(workflowId, None, key, None, None)) + val message = GetMetadataQueryAction(MetadataQuery(workflowId, None, key, None, None, expandSubWorkflows = false)) val parentProbe = TestProbe() TestActorRef(MetadataBuilderActor.props(serviceRegistryActor), parentProbe.ref, s"MetadataActor-${UUID.randomUUID()}") ! message @@ -436,6 +437,10 @@ class AlwaysHappyJobStoreActor extends Actor { } } +object AlwaysHappySubWorkflowStoreActor { + def props: Props = Props(new EmptySubWorkflowStoreActor) +} + object AlwaysHappyJobStoreActor { def props: Props = Props(new AlwaysHappyJobStoreActor) } diff --git a/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala b/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala index d6d059467..40eb03624 100644 --- a/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala @@ -13,12 +13,12 @@ class FilePassingWorkflowSpec extends CromwellTestKitSpec { sampleWdl = SampleWdl.FilePassingWorkflow, EventFilter.info(pattern = "Workflow complete", occurrences = 1), expectedOutputs = Map( - "file_passing.a.out" -> WdlFile("out"), - "file_passing.a.out_interpolation" -> WdlFile("out"), - "file_passing.a.contents" -> WdlString("foo bar baz"), - "file_passing.b.out" -> WdlFile("out"), - "file_passing.b.out_interpolation" -> WdlFile("out"), - "file_passing.b.contents" -> WdlString("foo bar baz") + "file_passing_a_out" -> WdlFile("out"), + "file_passing_a_out_interpolation" -> WdlFile("out"), + "file_passing_a_contents" -> WdlString("foo bar baz"), + "file_passing_b_out" -> WdlFile("out"), + "file_passing_b_out_interpolation" -> WdlFile("out"), + "file_passing_b_contents" -> WdlString("foo bar baz") ), patienceConfig = PatienceConfig(2.minutes.dilated) ) diff --git a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala index ee6b088c7..bb1f8b658 100644 --- a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala @@ -28,12 +28,12 @@ class MapWorkflowSpec extends CromwellTestKitSpec { sampleWdl = sampleWdl, EventFilter.info(pattern = "Starting calls: wf.read_map:NA:1, wf.write_map:NA:1", occurrences = 1), expectedOutputs = Map( - "wf.read_map.out_map" -> WdlMap(WdlMapType(WdlStringType, WdlIntegerType), Map( + "wf_read_map_out_map" -> WdlMap(WdlMapType(WdlStringType, WdlIntegerType), Map( WdlString("x") -> WdlInteger(500), WdlString("y") -> WdlInteger(600), WdlString("z") -> WdlInteger(700) )), - "wf.write_map.contents" -> WdlString("f1\talice\nf2\tbob\nf3\tchuck") + "wf_write_map_contents" -> WdlString("f1\talice\nf2\tbob\nf3\tchuck") ) ) sampleWdl.cleanup() @@ -75,7 +75,7 @@ class MapWorkflowSpec extends CromwellTestKitSpec { sampleWdl, eventFilter = EventFilter.info(pattern = "Starting calls: wf.read_map:NA:1, wf.write_map:NA:1", occurrences = 1), expectedOutputs = Map( - "wf.read_map.out_map" -> WdlMap(WdlMapType(WdlStringType, WdlIntegerType), Map( + "wf_read_map_out_map" -> WdlMap(WdlMapType(WdlStringType, WdlIntegerType), Map( WdlString("x") -> WdlInteger(500), WdlString("y") -> WdlInteger(600), WdlString("z") -> WdlInteger(700) diff --git a/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala b/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala index aa05c08f6..4fc24c56e 100644 --- a/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala @@ -12,8 +12,8 @@ class MultipleFilesWithSameNameWorkflowSpec extends CromwellTestKitSpec { sampleWdl = SampleWdl.FileClobber, EventFilter.info(pattern = "Starting calls: two.x:NA:1, two.y:NA:1", occurrences = 1), expectedOutputs = Map( - "two.x.out" -> WdlString("first file.txt"), - "two.y.out" -> WdlString("second file.txt") + "two_x_out" -> WdlString("first file.txt"), + "two_y_out" -> WdlString("second file.txt") ) ) } diff --git a/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala b/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala index a436a6287..c72d98758 100644 --- a/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala @@ -11,21 +11,21 @@ class PostfixQuantifierWorkflowSpec extends CromwellTestKitSpec { runWdlAndAssertOutputs( sampleWdl = SampleWdl.ZeroOrMorePostfixQuantifierWorkflowWithArrayInput, EventFilter.info(pattern = "Starting calls: postfix.hello", occurrences = 1), - expectedOutputs = Map("postfix.hello.greeting" -> WdlString("hello alice,bob,charles")) + expectedOutputs = Map("postfix_hello_greeting" -> WdlString("hello alice,bob,charles")) ) } "accept an array of size 1" in { runWdlAndAssertOutputs( sampleWdl = SampleWdl.ZeroOrMorePostfixQuantifierWorkflowWithOneElementArrayInput, EventFilter.info(pattern = "Starting calls: postfix.hello", occurrences = 1), - expectedOutputs = Map("postfix.hello.greeting" -> WdlString("hello alice")) + expectedOutputs = Map("postfix_hello_greeting" -> WdlString("hello alice")) ) } "accept an array of size 0" in { runWdlAndAssertOutputs( sampleWdl = SampleWdl.ZeroOrMorePostfixQuantifierWorkflowWithZeroElementArrayInput, EventFilter.info(pattern = "Starting calls: postfix.hello", occurrences = 1), - expectedOutputs = Map("postfix.hello.greeting" -> WdlString("hello")) + expectedOutputs = Map("postfix_hello_greeting" -> WdlString("hello")) ) } } @@ -35,14 +35,14 @@ class PostfixQuantifierWorkflowSpec extends CromwellTestKitSpec { runWdlAndAssertOutputs( sampleWdl = SampleWdl.OneOrMorePostfixQuantifierWorkflowWithArrayInput, EventFilter.info(pattern = "Starting calls: postfix.hello", occurrences = 1), - expectedOutputs = Map("postfix.hello.greeting" -> WdlString("hello alice,bob,charles")) + expectedOutputs = Map("postfix_hello_greeting" -> WdlString("hello alice,bob,charles")) ) } "accept a scalar for the value" in { runWdlAndAssertOutputs( sampleWdl = SampleWdl.OneOrMorePostfixQuantifierWorkflowWithScalarInput, EventFilter.info(pattern = "Starting calls: postfix.hello", occurrences = 1), - expectedOutputs = Map("postfix.hello.greeting" -> WdlString("hello alice")) + expectedOutputs = Map("postfix_hello_greeting" -> WdlString("hello alice")) ) } } diff --git a/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala b/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala index 347c21923..c1ca1ad9e 100644 --- a/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala @@ -13,8 +13,8 @@ class ScatterWorkflowSpec extends CromwellTestKitSpec { sampleWdl = SampleWdl.SimpleScatterWdl, eventFilter = EventFilter.info(pattern = "Workflow complete", occurrences = 1), expectedOutputs = Map( - "scatter0.outside_scatter.out" -> WdlInteger(8000), - "scatter0.inside_scatter.out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(1, 2, 3, 4, 5).map(WdlInteger(_))) + "scatter0_outside_scatter_out" -> WdlInteger(8000), + "scatter0_inside_scatter_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(1, 2, 3, 4, 5).map(WdlInteger(_))) ) ) } @@ -25,11 +25,11 @@ class ScatterWorkflowSpec extends CromwellTestKitSpec { sampleWdl = new SampleWdl.ScatterWdl, eventFilter = EventFilter.info(pattern = "Workflow complete", occurrences = 1), expectedOutputs = Map( - "w.E.E_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(9, 9, 9, 9, 9, 9).map(WdlInteger(_))), - "w.C.C_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(400, 500, 600, 800, 600, 500).map(WdlInteger(_))), - "w.A.A_out" -> WdlArray(WdlArrayType(WdlStringType), Seq("jeff", "chris", "miguel", "thibault", "khalid", "scott").map(WdlString)), - "w.D.D_out" -> WdlInteger(34), - "w.B.B_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(4, 5, 6, 8, 6, 5).map(WdlInteger(_))) + "w_E_E_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(9, 9, 9, 9, 9, 9).map(WdlInteger(_))), + "w_C_C_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(400, 500, 600, 800, 600, 500).map(WdlInteger(_))), + "w_A_A_out" -> WdlArray(WdlArrayType(WdlStringType), Seq("jeff", "chris", "miguel", "thibault", "khalid", "ruchi").map(WdlString)), + "w_D_D_out" -> WdlInteger(34), + "w_B_B_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(4, 5, 6, 8, 6, 5).map(WdlInteger(_))) ) ) } @@ -40,12 +40,12 @@ class ScatterWorkflowSpec extends CromwellTestKitSpec { sampleWdl = SampleWdl.SiblingsScatterWdl, eventFilter = EventFilter.info(pattern = "Workflow complete", occurrences = 1), expectedOutputs = Map( - "w.E.E_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(9, 9, 9, 9, 9, 9).map(WdlInteger(_))), - "w.F.B_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(4, 5, 6, 8, 6, 5).map(WdlInteger(_))), - "w.C.C_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(400, 500, 600, 800, 600, 500).map(WdlInteger(_))), - "w.A.A_out" -> WdlArray(WdlArrayType(WdlStringType), Seq("jeff", "chris", "miguel", "thibault", "khalid", "scott").map(WdlString)), - "w.D.D_out" -> WdlInteger(34), - "w.B.B_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(4, 5, 6, 8, 6, 5).map(WdlInteger(_))) + "w_E_E_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(9, 9, 9, 9, 9, 9).map(WdlInteger(_))), + "w_F_B_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(4, 5, 6, 8, 6, 5).map(WdlInteger(_))), + "w_C_C_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(400, 500, 600, 800, 600, 500).map(WdlInteger(_))), + "w_A_A_out" -> WdlArray(WdlArrayType(WdlStringType), Seq("jeff", "chris", "miguel", "thibault", "khalid", "ruchi").map(WdlString)), + "w_D_D_out" -> WdlInteger(34), + "w_B_B_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(4, 5, 6, 8, 6, 5).map(WdlInteger(_))) ) ) } @@ -57,9 +57,9 @@ class ScatterWorkflowSpec extends CromwellTestKitSpec { sampleWdl = SampleWdl.PrepareScatterGatherWdl(), eventFilter = EventFilter.info(pattern = "Workflow complete", occurrences = 1), expectedOutputs = Map( - "sc_test.do_gather.sum" -> WdlInteger(11), - "sc_test.do_prepare.split_files" -> WdlArray(WdlArrayType(WdlFileType), Seq("temp_aa", "temp_ab", "temp_ac", "temp_ad").map(WdlFile(_))), - "sc_test.do_scatter.count_file" -> WdlArray(WdlArrayType(WdlFileType), (1 to 4).map(_ => WdlFile("output.txt"))) + "sc_test_do_gather_sum" -> WdlInteger(11), + "sc_test_do_prepare_split_files" -> WdlArray(WdlArrayType(WdlFileType), Seq("temp_aa", "temp_ab", "temp_ac", "temp_ad").map(WdlFile(_))), + "sc_test_do_scatter_count_file" -> WdlArray(WdlArrayType(WdlFileType), (1 to 4).map(_ => WdlFile("output.txt"))) ) ) } @@ -74,9 +74,9 @@ class ScatterWorkflowSpec extends CromwellTestKitSpec { |} """.stripMargin, expectedOutputs = Map( - "sc_test.do_gather.sum" -> WdlInteger(11), - "sc_test.do_prepare.split_files" -> WdlArray(WdlArrayType(WdlFileType), Seq("temp_aa", "temp_ab", "temp_ac", "temp_ad").map(WdlFile(_))), - "sc_test.do_scatter.count_file" -> WdlArray(WdlArrayType(WdlFileType), (1 to 4).map(_ => WdlFile("output.txt"))) + "sc_test_do_gather_sum" -> WdlInteger(11), + "sc_test_do_prepare_split_files" -> WdlArray(WdlArrayType(WdlFileType), Seq("temp_aa", "temp_ab", "temp_ac", "temp_ad").map(WdlFile(_))), + "sc_test_do_scatter_count_file" -> WdlArray(WdlArrayType(WdlFileType), (1 to 4).map(_ => WdlFile("output.txt"))) ) ) } diff --git a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala index a636e3f46..c99b93b0a 100644 --- a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala @@ -43,6 +43,7 @@ class SimpleWorkflowActorSpec extends CromwellTestKitSpec with BeforeAndAfter { serviceRegistryActor = watchActor, workflowLogCopyRouter = system.actorOf(Props.empty, s"workflow-copy-log-router-$workflowId-${UUID.randomUUID()}"), jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props), + subWorkflowStoreActor = system.actorOf(AlwaysHappySubWorkflowStoreActor.props), callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props), jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props), backendSingletonCollection = BackendSingletonCollection(Map("Local" -> None))), diff --git a/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala b/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala index 186bd0714..3052e3370 100644 --- a/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala +++ b/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala @@ -19,8 +19,8 @@ class WdlFunctionsAtWorkflowLevelSpec extends CromwellTestKitSpec { sampleWdl = SampleWdl.WdlFunctionsAtWorkflowLevel, eventFilter = EventFilter.info(pattern = "Starting calls: w.a", occurrences = 1), expectedOutputs = Map( - "w.a.x" -> WdlString("one two three four five"), - "w.a.y" -> outputMap + "w_a_x" -> WdlString("one two three four five"), + "w_a_y" -> outputMap ) ) } diff --git a/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala b/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala index b821dd742..0f210fd1e 100644 --- a/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala +++ b/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala @@ -13,9 +13,9 @@ class WorkflowOutputsSpec extends CromwellTestKitSpec { eventFilter = EventFilter.info(pattern = s"is in a terminal state: WorkflowSucceededState", occurrences = 1), runtime = "", expectedOutputs = Map( - "three_step.ps.procs" -> AnyValueIsFine, - "three_step.cgrep.count" -> AnyValueIsFine, - "three_step.wc.count" -> AnyValueIsFine + "three_step_ps_procs" -> AnyValueIsFine, + "three_step_cgrep_count" -> AnyValueIsFine, + "three_step_wc_count" -> AnyValueIsFine ), allowOtherOutputs = false ) @@ -27,8 +27,8 @@ class WorkflowOutputsSpec extends CromwellTestKitSpec { eventFilter = EventFilter.info(pattern = s"is in a terminal state: WorkflowSucceededState", occurrences = 1), runtime = "", expectedOutputs = Map( - "three_step.cgrep.count" -> AnyValueIsFine, - "three_step.wc.count" -> AnyValueIsFine + "three_step_cgrep_count" -> AnyValueIsFine, + "three_step_wc_count" -> AnyValueIsFine ), allowOtherOutputs = false ) @@ -40,8 +40,8 @@ class WorkflowOutputsSpec extends CromwellTestKitSpec { eventFilter = EventFilter.info(pattern = s"is in a terminal state: WorkflowSucceededState", occurrences = 1), runtime = "", expectedOutputs = Map( - "scatter0.outside_scatter.out" -> AnyValueIsFine, - "scatter0.inside_scatter.out" -> AnyValueIsFine + "scatter0_outside_scatter_out" -> AnyValueIsFine, + "scatter0_inside_scatter_out" -> AnyValueIsFine ), allowOtherOutputs = false ) @@ -53,7 +53,7 @@ class WorkflowOutputsSpec extends CromwellTestKitSpec { eventFilter = EventFilter.info(pattern = s"is in a terminal state: WorkflowSucceededState", occurrences = 1), runtime = "", expectedOutputs = Map( - "scatter0.inside_scatter.out" -> AnyValueIsFine + "scatter0_inside_scatter_out" -> AnyValueIsFine ), allowOtherOutputs = false ) diff --git a/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala b/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala index b22ebc13f..1d7d313b6 100644 --- a/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala @@ -13,7 +13,7 @@ class WorkflowManagerActorSpec extends CromwellTestKitSpec with WorkflowDescript "run workflows in the correct directory" in { val outputs = runWdl(sampleWdl = SampleWdl.CurrentDirectory) - val outputName = "wf_whereami.whereami.pwd" + val outputName = "wf_whereami_whereami_pwd" val salutation = outputs(outputName) val actualOutput = salutation.valueString.trim actualOutput should endWith("/call-whereami/execution") diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala b/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala index 223b715bf..0763a1c30 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala @@ -1,9 +1,9 @@ package cromwell.engine.backend.mock import akka.actor.{ActorRef, Props} -import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobSucceededResponse} import cromwell.backend._ -import wdl4s.Call +import wdl4s.TaskCall import wdl4s.expression.{NoFunctions, WdlStandardLibraryFunctions} import scala.concurrent.Future @@ -14,7 +14,7 @@ object DefaultBackendJobExecutionActor { case class DefaultBackendJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, override val configurationDescriptor: BackendConfigurationDescriptor) extends BackendJobExecutionActor { override def execute: Future[BackendJobExecutionResponse] = { - Future.successful(SucceededResponse(jobDescriptor.key, Some(0), (jobDescriptor.call.task.outputs map taskOutputToJobOutput).toMap, None, Seq.empty)) + Future.successful(JobSucceededResponse(jobDescriptor.key, Some(0), (jobDescriptor.call.task.outputs map taskOutputToJobOutput).toMap, None, Seq.empty)) } override def recover = execute @@ -25,7 +25,7 @@ case class DefaultBackendJobExecutionActor(override val jobDescriptor: BackendJo class DefaultBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Set[Call], + calls: Set[TaskCall], serviceRegistryActor: ActorRef): Option[Props] = None override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendJobExecutionActor.scala b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendJobExecutionActor.scala index 60617f468..eaaa04abb 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendJobExecutionActor.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendJobExecutionActor.scala @@ -2,7 +2,7 @@ package cromwell.engine.backend.mock import akka.actor.Props import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobExecutionActor} -import cromwell.backend.BackendJobExecutionActor.{FailedRetryableResponse, BackendJobExecutionResponse, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{JobFailedRetryableResponse, BackendJobExecutionResponse, JobSucceededResponse} import scala.concurrent.Future @@ -16,9 +16,9 @@ case class RetryableBackendJobExecutionActor(override val jobDescriptor: Backend override def execute: Future[BackendJobExecutionResponse] = { if (jobDescriptor.key.attempt < attempts) - Future.successful(FailedRetryableResponse(jobDescriptor.key, new RuntimeException("An apparent transient Exception!"), None)) + Future.successful(JobFailedRetryableResponse(jobDescriptor.key, new RuntimeException("An apparent transient Exception!"), None)) else - Future.successful(SucceededResponse(jobDescriptor.key, Some(0), (jobDescriptor.call.task.outputs map taskOutputToJobOutput).toMap, None, Seq.empty)) + Future.successful(JobSucceededResponse(jobDescriptor.key, Some(0), (jobDescriptor.call.task.outputs map taskOutputToJobOutput).toMap, None, Seq.empty)) } override def recover = execute diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala index 48636f014..c88481610 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala @@ -2,13 +2,13 @@ package cromwell.engine.backend.mock import akka.actor.{ActorRef, Props} import cromwell.backend._ -import wdl4s.Call +import wdl4s.TaskCall import wdl4s.expression.{NoFunctions, WdlStandardLibraryFunctions} class RetryableBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Set[Call], + calls: Set[TaskCall], serviceRegistryActor: ActorRef): Option[Props] = None override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala index 543b8e502..3243582a3 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala @@ -18,7 +18,7 @@ import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreActor} import cromwell.util.SampleWdl import cromwell.util.SampleWdl.{ExpressionsInInputs, GoodbyeWorld, ThreeStep} -import cromwell.{AlwaysHappyJobStoreActor, CromwellTestKitSpec, EmptyCallCacheReadActor} +import cromwell.{AlwaysHappyJobStoreActor, AlwaysHappySubWorkflowStoreActor, CromwellTestKitSpec, EmptyCallCacheReadActor} import org.scalatest.prop.{TableDrivenPropertyChecks, TableFor3} import spray.json._ @@ -56,6 +56,7 @@ object SingleWorkflowRunnerActorSpec { abstract class SingleWorkflowRunnerActorSpec extends CromwellTestKitSpec { private val workflowStore = system.actorOf(WorkflowStoreActor.props(new InMemoryWorkflowStore, dummyServiceRegistryActor)) private val jobStore = system.actorOf(AlwaysHappyJobStoreActor.props) + private val subWorkflowStore = system.actorOf(AlwaysHappySubWorkflowStoreActor.props) private val callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props) private val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props) @@ -66,6 +67,7 @@ abstract class SingleWorkflowRunnerActorSpec extends CromwellTestKitSpec { dummyServiceRegistryActor, dummyLogCopyRouter, jobStore, + subWorkflowStore, callCacheReadActor, jobTokenDispenserActor, BackendSingletonCollection(Map.empty), diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala index 23087f529..bacfc4da2 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala @@ -3,7 +3,7 @@ package cromwell.engine.workflow import akka.actor.{Actor, ActorRef} import akka.testkit.{TestActorRef, TestFSMRef, TestProbe} import com.typesafe.config.{Config, ConfigFactory} -import cromwell.backend.AllBackendInitializationData +import cromwell.backend.{AllBackendInitializationData, JobExecutionMap} import cromwell.core._ import cromwell.engine.EngineWorkflowDescriptor import cromwell.engine.backend.BackendSingletonCollection @@ -13,7 +13,7 @@ import cromwell.engine.workflow.lifecycle.WorkflowFinalizationActor.{StartFinali import cromwell.engine.workflow.lifecycle.WorkflowInitializationActor.{WorkflowInitializationAbortedResponse, WorkflowInitializationFailedResponse} import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{WorkflowExecutionAbortedResponse, WorkflowExecutionFailedResponse, WorkflowExecutionSucceededResponse} import cromwell.util.SampleWdl.ThreeStep -import cromwell.{AlwaysHappyJobStoreActor, CromwellTestKitSpec, EmptyCallCacheReadActor} +import cromwell.{AlwaysHappyJobStoreActor, AlwaysHappySubWorkflowStoreActor, CromwellTestKitSpec, EmptyCallCacheReadActor} import org.scalatest.BeforeAndAfter import org.scalatest.concurrent.Eventually @@ -53,6 +53,7 @@ class WorkflowActorSpec extends CromwellTestKitSpec with WorkflowDescriptorBuild serviceRegistryActor = mockServiceRegistryActor, workflowLogCopyRouter = TestProbe().ref, jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props), + subWorkflowStoreActor = system.actorOf(AlwaysHappySubWorkflowStoreActor.props), callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props), jobTokenDispenserActor = TestProbe().ref ), @@ -95,7 +96,7 @@ class WorkflowActorSpec extends CromwellTestKitSpec with WorkflowDescriptorBuild "run Finalization if Execution fails" in { val actor = createWorkflowActor(ExecutingWorkflowState) deathwatch watch actor - actor ! WorkflowExecutionFailedResponse(ExecutionStore.empty, OutputStore.empty, Seq(new Exception("Execution Failed"))) + actor ! WorkflowExecutionFailedResponse(Map.empty, new Exception("Execution Failed")) finalizationProbe.expectMsg(StartFinalizationCommand) actor.stateName should be(FinalizingWorkflowState) actor ! WorkflowFinalizationSucceededResponse @@ -110,7 +111,7 @@ class WorkflowActorSpec extends CromwellTestKitSpec with WorkflowDescriptorBuild eventually { actor.stateName should be(WorkflowAbortingState) } currentLifecycleActor.expectMsgPF(CromwellTestKitSpec.TimeoutDuration) { case EngineLifecycleActorAbortCommand => - actor ! WorkflowExecutionAbortedResponse(ExecutionStore.empty, OutputStore.empty) + actor ! WorkflowExecutionAbortedResponse(Map.empty) } finalizationProbe.expectMsg(StartFinalizationCommand) actor.stateName should be(FinalizingWorkflowState) @@ -122,7 +123,7 @@ class WorkflowActorSpec extends CromwellTestKitSpec with WorkflowDescriptorBuild "run Finalization actor if Execution succeeds" in { val actor = createWorkflowActor(ExecutingWorkflowState) deathwatch watch actor - actor ! WorkflowExecutionSucceededResponse(ExecutionStore.empty, OutputStore.empty) + actor ! WorkflowExecutionSucceededResponse(Map.empty, Map.empty) finalizationProbe.expectMsg(StartFinalizationCommand) actor.stateName should be(FinalizingWorkflowState) actor ! WorkflowFinalizationSucceededResponse @@ -156,8 +157,9 @@ class MockWorkflowActor(val finalizationProbe: TestProbe, serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, + subWorkflowStoreActor: ActorRef, callCacheReadActor: ActorRef, - jobTokenDispenserActor: ActorRef) extends WorkflowActor(workflowId, startMode, workflowSources, conf, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, BackendSingletonCollection(Map.empty)) { + jobTokenDispenserActor: ActorRef) extends WorkflowActor(workflowId, startMode, workflowSources, conf, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, subWorkflowStoreActor, callCacheReadActor, jobTokenDispenserActor, BackendSingletonCollection(Map.empty)) { - override def makeFinalizationActor(workflowDescriptor: EngineWorkflowDescriptor, executionStore: ExecutionStore, outputStore: OutputStore) = finalizationProbe.ref + override def makeFinalizationActor(workflowDescriptor: EngineWorkflowDescriptor, jobExecutionMap: JobExecutionMap, worfklowOutputs: CallOutputs) = finalizationProbe.ref } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala index bdaca25a7..9c8369bf9 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala @@ -203,7 +203,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be within(Timeout) { expectMsgPF() { case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => - wfDesc.namespace.workflow.calls foreach { + wfDesc.namespace.workflow.taskCalls foreach { case call if call.task.name.equals("a") => wfDesc.backendAssignments(call) shouldBe "SpecifiedBackend" case call if call.task.name.equals("b") => diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala new file mode 100644 index 000000000..1f61772e6 --- /dev/null +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala @@ -0,0 +1,213 @@ +package cromwell.engine.workflow.lifecycle.execution + +import java.util.UUID + +import akka.actor.Props +import akka.testkit.{TestFSMRef, TestProbe} +import cromwell.backend.{AllBackendInitializationData, BackendWorkflowDescriptor, JobExecutionMap} +import cromwell.core._ +import cromwell.core.callcaching.CallCachingOff +import cromwell.database.sql.tables.SubWorkflowStoreEntry +import cromwell.engine.backend.BackendSingletonCollection +import cromwell.engine.workflow.lifecycle.execution.CallPreparationActor.{CallPreparationFailed, SubWorkflowPreparationSucceeded} +import cromwell.engine.workflow.lifecycle.execution.SubWorkflowExecutionActor._ +import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor._ +import cromwell.engine.{ContinueWhilePossible, EngineWorkflowDescriptor} +import cromwell.subworkflowstore.SubWorkflowStoreActor.{QuerySubWorkflow, SubWorkflowFound, SubWorkflowNotFound} +import org.scalatest.concurrent.Eventually +import org.scalatest.{FlatSpecLike, Matchers} +import org.specs2.mock.Mockito +import wdl4s.{WdlNamespaceWithWorkflow, Workflow, WorkflowCall} + +import scala.concurrent.duration._ +import scala.language.postfixOps + +class SubWorkflowExecutionActorSpec extends TestKitSuite with FlatSpecLike with Matchers with Mockito with Eventually { + + behavior of "SubWorkflowExecutionActor" + + val serviceRegistryProbe = TestProbe() + val jobStoreProbe = TestProbe() + val subWorkflowStoreProbe = TestProbe() + val callCacheReadActorProbe = TestProbe() + val jobTokenDispenserProbe = TestProbe() + val preparationActor = TestProbe() + val subWorkflowActor = TestProbe() + val deathWatch = TestProbe() + val parentProbe = TestProbe() + val parentBackendDescriptor = mock[BackendWorkflowDescriptor] + val parentWorkflowId: WorkflowId = WorkflowId.randomId() + parentBackendDescriptor.id returns parentWorkflowId + val parentWorkflowDescriptor = EngineWorkflowDescriptor( + mock[WdlNamespaceWithWorkflow], + parentBackendDescriptor, + Map.empty, + Map.empty, + ContinueWhilePossible, + List.empty, + CallCachingOff + ) + val subWorkflow = mock[Workflow] + subWorkflow.unqualifiedName returns "sub_wf" + val subWorkflowCall = mock[WorkflowCall] + subWorkflowCall.fullyQualifiedName returns "foo.bar" + subWorkflowCall.callable returns subWorkflow + val subKey = SubWorkflowKey(subWorkflowCall, None, 1) + + val awaitTimeout: FiniteDuration = 10 seconds + + def buildEWEA(restart: Boolean = false) = { + new TestFSMRef[SubWorkflowExecutionActorState, SubWorkflowExecutionActorData, SubWorkflowExecutionActor](system, Props( + new SubWorkflowExecutionActor( + subKey, + WorkflowExecutionActorData.empty(parentWorkflowDescriptor), + Map.empty, + serviceRegistryProbe.ref, + jobStoreProbe.ref, + subWorkflowStoreProbe.ref, + callCacheReadActorProbe.ref, + jobTokenDispenserProbe.ref, + BackendSingletonCollection(Map.empty), + AllBackendInitializationData(Map.empty), + restart + ) { + override def createSubWorkflowPreparationActor(subWorkflowId: WorkflowId) = preparationActor.ref + override def createSubWorkflowActor(createSubWorkflowActor: EngineWorkflowDescriptor) = subWorkflowActor.ref + }), parentProbe.ref, s"SubWorkflowExecutionActorSpec-${UUID.randomUUID()}") + } + + it should "Check the sub workflow store when restarting" in { + val ewea = buildEWEA(restart = true) + ewea.setState(SubWorkflowPendingState) + + ewea ! Execute + subWorkflowStoreProbe.expectMsg(QuerySubWorkflow(parentWorkflowId, subKey)) + eventually { + ewea.stateName shouldBe SubWorkflowCheckingStoreState + } + } + + it should "Reuse sub workflow id if found in the store" in { + import cromwell.core.ExecutionIndex._ + + val ewea = buildEWEA(restart = true) + ewea.setState(SubWorkflowCheckingStoreState) + + val subWorkflowUuid = WorkflowId.randomId() + ewea ! SubWorkflowFound(SubWorkflowStoreEntry(Option(0), parentWorkflowId.toString, subKey.scope.fullyQualifiedName, subKey.index.fromIndex, subKey.attempt, subWorkflowUuid.toString, None)) + preparationActor.expectMsg(CallPreparationActor.Start) + parentProbe.expectMsg(JobStarting(subKey)) + + eventually { + ewea.stateName shouldBe SubWorkflowPreparingState + ewea.stateData.subWorkflowId shouldBe Some(subWorkflowUuid) + } + } + + it should "Fall back to a random Id if the sub workflow id is not found in the store" in { + val ewea = buildEWEA(restart = true) + ewea.setState(SubWorkflowCheckingStoreState) + + ewea ! SubWorkflowNotFound(QuerySubWorkflow(parentWorkflowId, subKey)) + preparationActor.expectMsg(CallPreparationActor.Start) + parentProbe.expectMsg(JobStarting(subKey)) + + eventually { + ewea.stateName shouldBe SubWorkflowPreparingState + ewea.stateData.subWorkflowId should not be empty + } + } + + it should "Prepare a sub workflow" in { + val ewea = buildEWEA() + ewea.setState(SubWorkflowPendingState) + + ewea ! Execute + preparationActor.expectMsg(CallPreparationActor.Start) + parentProbe.expectMsg(JobStarting(subKey)) + eventually { + ewea.stateName shouldBe SubWorkflowPreparingState + } + } + + it should "Run a sub workflow" in { + val ewea = buildEWEA() + ewea.setState(SubWorkflowPreparingState, SubWorkflowExecutionActorData(Some(WorkflowId.randomId()))) + + val subWorkflowId = WorkflowId.randomId() + val subBackendDescriptor = mock[BackendWorkflowDescriptor] + subBackendDescriptor.id returns subWorkflowId + val subWorkflowDescriptor = EngineWorkflowDescriptor( + mock[WdlNamespaceWithWorkflow], + subBackendDescriptor, + Map.empty, + Map.empty, + ContinueWhilePossible, + List.empty, + CallCachingOff + ) + + ewea ! SubWorkflowPreparationSucceeded(subWorkflowDescriptor, Map.empty) + subWorkflowActor.expectMsg(WorkflowExecutionActor.ExecuteWorkflowCommand) + parentProbe.expectMsg(JobRunning(subKey, Map.empty, Option(subWorkflowActor.ref))) + eventually { + ewea.stateName shouldBe SubWorkflowRunningState + } + } + + it should "Fail a sub workflow if preparation failed" in { + val ewea = buildEWEA() + ewea.setState(SubWorkflowPreparingState) + deathWatch watch ewea + + val subWorkflowKey = mock[SubWorkflowKey] + val throwable: Exception = new Exception("Expected test exception") + val preparationFailedMessage: CallPreparationFailed = CallPreparationFailed(subWorkflowKey, throwable) + ewea ! preparationFailedMessage + parentProbe.expectMsg(SubWorkflowFailedResponse(subKey, Map.empty, throwable)) + deathWatch.expectTerminated(ewea, awaitTimeout) + } + + it should "Relay Workflow Successful message" in { + val ewea = buildEWEA() + ewea.setState(SubWorkflowRunningState, SubWorkflowExecutionActorData(Some(WorkflowId.randomId()))) + + deathWatch watch ewea + + val jobExecutionMap: JobExecutionMap = Map.empty + val outputs: CallOutputs = Map.empty[LocallyQualifiedName, JobOutput] + val workflowSuccessfulMessage = WorkflowExecutionSucceededResponse(jobExecutionMap, outputs) + ewea ! workflowSuccessfulMessage + parentProbe.expectMsg(SubWorkflowSucceededResponse(subKey, jobExecutionMap, outputs)) + deathWatch.expectTerminated(ewea, awaitTimeout) + } + + it should "Relay Workflow Failed message" in { + val ewea = buildEWEA() + ewea.setState(SubWorkflowRunningState, SubWorkflowExecutionActorData(Some(WorkflowId.randomId()))) + + deathWatch watch ewea + + val jobExecutionMap: JobExecutionMap = Map.empty + val expectedException: Exception = new Exception("Expected test exception") + + val workflowSuccessfulMessage = WorkflowExecutionFailedResponse(jobExecutionMap, expectedException) + ewea ! workflowSuccessfulMessage + parentProbe.expectMsg(SubWorkflowFailedResponse(subKey, jobExecutionMap, expectedException)) + deathWatch.expectTerminated(ewea, awaitTimeout) + } + + it should "Relay Workflow Aborted message" in { + val ewea = buildEWEA() + ewea.setState(SubWorkflowRunningState, SubWorkflowExecutionActorData(Some(WorkflowId.randomId()))) + + deathWatch watch ewea + + val jobExecutionMap: JobExecutionMap = Map.empty + val workflowAbortedMessage = WorkflowExecutionAbortedResponse(jobExecutionMap) + ewea ! workflowAbortedMessage + parentProbe.expectMsg(SubWorkflowAbortedResponse(subKey, jobExecutionMap)) + deathWatch.expectTerminated(ewea, awaitTimeout) + } + +} diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala index cc9978e19..51007a2df 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala @@ -12,11 +12,11 @@ import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor import cromwell.services.ServiceRegistryActor import cromwell.services.metadata.MetadataService import cromwell.util.SampleWdl -import cromwell.{AlwaysHappyJobStoreActor, CromwellTestKitSpec, EmptyCallCacheReadActor, MetadataWatchActor} +import cromwell._ import org.scalatest.BeforeAndAfter -import scala.concurrent.{Await, Promise} import scala.concurrent.duration._ +import scala.concurrent.{Await, Promise} class WorkflowExecutionActorSpec extends CromwellTestKitSpec with BeforeAndAfter with WorkflowDescriptorBuilder { @@ -53,6 +53,7 @@ class WorkflowExecutionActorSpec extends CromwellTestKitSpec with BeforeAndAfter val metadataWatcherProps = Props(MetadataWatchActor(metadataSuccessPromise, requiredMetadataMatchers: _*)) val serviceRegistryActor = system.actorOf(ServiceRegistryActor.props(ConfigFactory.load(), overrides = Map(MetadataService.MetadataServiceName -> metadataWatcherProps))) val jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props) + val subWorkflowStoreActor = system.actorOf(AlwaysHappySubWorkflowStoreActor.props) val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props) val MockBackendConfigEntry = BackendConfigurationEntry( name = "Mock", @@ -66,7 +67,7 @@ class WorkflowExecutionActorSpec extends CromwellTestKitSpec with BeforeAndAfter val callCacheReadActor = TestProbe() val workflowExecutionActor = system.actorOf( - WorkflowExecutionActor.props(workflowId, engineWorkflowDescriptor, serviceRegistryActor, jobStoreActor, + WorkflowExecutionActor.props(engineWorkflowDescriptor, serviceRegistryActor, jobStoreActor, subWorkflowStoreActor, callCacheReadActor.ref, jobTokenDispenserActor, MockBackendSingletonCollection, AllBackendInitializationData.empty, restarting = false), "WorkflowExecutionActor") @@ -86,6 +87,7 @@ class WorkflowExecutionActorSpec extends CromwellTestKitSpec with BeforeAndAfter "execute a workflow with scatters" in { val serviceRegistry = mockServiceRegistryActor val jobStore = system.actorOf(AlwaysHappyJobStoreActor.props) + val subWorkflowStoreActor = system.actorOf(AlwaysHappySubWorkflowStoreActor.props) val callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props) val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props) @@ -99,7 +101,7 @@ class WorkflowExecutionActorSpec extends CromwellTestKitSpec with BeforeAndAfter val workflowId = WorkflowId.randomId() val engineWorkflowDescriptor = createMaterializedEngineWorkflowDescriptor(workflowId, SampleWdl.SimpleScatterWdl.asWorkflowSources(runtime = runtimeSection)) val workflowExecutionActor = system.actorOf( - WorkflowExecutionActor.props(workflowId, engineWorkflowDescriptor, serviceRegistry, jobStore, + WorkflowExecutionActor.props(engineWorkflowDescriptor, serviceRegistry, jobStore, subWorkflowStoreActor, callCacheReadActor, jobTokenDispenserActor, MockBackendSingletonCollection, AllBackendInitializationData.empty, restarting = false), "WorkflowExecutionActor") diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala index e69085c9e..34ac835cd 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala @@ -194,7 +194,7 @@ object EngineJobHashingActorSpec extends BackendSpec { def templateJobDescriptor(inputs: Map[LocallyQualifiedName, WdlValue] = Map.empty) = { val task = mock[Task] - val call = mock[Call] + val call = mock[TaskCall] when(task.commandTemplateString).thenReturn("Do the stuff... now!!") when(task.outputs).thenReturn(List.empty) when(call.task).thenReturn(task) diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala index a8f2bdf46..5a9bb4f5c 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala @@ -1,9 +1,9 @@ package cromwell.engine.workflow.lifecycle.execution.ejea -import cromwell.backend.BackendJobExecutionActor.{FailedNonRetryableResponse, FailedRetryableResponse, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{JobFailedNonRetryableResponse, JobFailedRetryableResponse, JobSucceededResponse} import cromwell.core._ +import cromwell.engine.workflow.lifecycle.execution.CallPreparationActor import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{CheckingJobStore, NoData, PreparingJob} -import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec.EnhancedTestEJEA import cromwell.jobstore.JobStoreActor.{JobComplete, JobNotComplete} import cromwell.jobstore.{JobResultFailure, JobResultSuccess} @@ -17,12 +17,12 @@ class EjeaCheckingJobStoreSpec extends EngineJobExecutionActorSpec { createCheckingJobStoreEjea() ejea.setState(CheckingJobStore) val returnCode: Option[Int] = Option(0) - val jobOutputs: JobOutputs = Map.empty + val jobOutputs: CallOutputs = Map.empty ejea ! JobComplete(JobResultSuccess(returnCode, jobOutputs)) helper.replyToProbe.expectMsgPF(awaitTimeout) { - case response: SucceededResponse => + case response: JobSucceededResponse => response.returnCode shouldBe returnCode response.jobOutputs shouldBe jobOutputs } @@ -40,11 +40,11 @@ class EjeaCheckingJobStoreSpec extends EngineJobExecutionActorSpec { ejea ! JobComplete(JobResultFailure(returnCode, reason, retryable)) helper.replyToProbe.expectMsgPF(awaitTimeout) { - case response: FailedNonRetryableResponse => + case response: JobFailedNonRetryableResponse => false should be(retryable) response.returnCode shouldBe returnCode response.throwable shouldBe reason - case response: FailedRetryableResponse => + case response: JobFailedRetryableResponse => true should be(retryable) response.returnCode shouldBe returnCode response.throwable shouldBe reason @@ -59,7 +59,7 @@ class EjeaCheckingJobStoreSpec extends EngineJobExecutionActorSpec { ejea.setState(CheckingJobStore) ejea ! JobNotComplete - helper.jobPreparationProbe.expectMsg(awaitTimeout, "expecting RecoverJobCommand", JobPreparationActor.Start) + helper.jobPreparationProbe.expectMsg(awaitTimeout, "expecting RecoverJobCommand", CallPreparationActor.Start) ejea.stateName should be(PreparingJob) ejea.stop() diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPreparingJobSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPreparingJobSpec.scala index 8e3288865..bdb426b60 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPreparingJobSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPreparingJobSpec.scala @@ -1,10 +1,10 @@ package cromwell.engine.workflow.lifecycle.execution.ejea -import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ -import EngineJobExecutionActorSpec._ -import cromwell.backend.BackendJobExecutionActor.FailedNonRetryableResponse +import cromwell.backend.BackendJobExecutionActor.JobFailedNonRetryableResponse import cromwell.core.callcaching.CallCachingMode -import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor.{BackendJobPreparationFailed, BackendJobPreparationSucceeded} +import cromwell.engine.workflow.lifecycle.execution.CallPreparationActor.{BackendJobPreparationSucceeded, CallPreparationFailed} +import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ +import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec._ import org.scalatest.concurrent.Eventually class EjeaPreparingJobSpec extends EngineJobExecutionActorSpec with CanExpectHashingInitialization with Eventually { @@ -35,8 +35,8 @@ class EjeaPreparingJobSpec extends EngineJobExecutionActorSpec with CanExpectHas } s"Not proceed if Job Preparation fails ($mode)" in { - val prepActorResponse = BackendJobPreparationFailed(helper.jobDescriptorKey, new Exception("The goggles! They do nothing!")) - val prepFailedEjeaResponse = FailedNonRetryableResponse(prepActorResponse.jobKey, prepActorResponse.throwable, None) + val prepActorResponse = CallPreparationFailed(helper.jobDescriptorKey, new Exception("The goggles! They do nothing!")) + val prepFailedEjeaResponse = JobFailedNonRetryableResponse(helper.jobDescriptorKey, prepActorResponse.throwable, None) ejea = ejeaInPreparingState(mode) ejea ! prepActorResponse helper.replyToProbe.expectMsg(prepFailedEjeaResponse) diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRequestingExecutionTokenSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRequestingExecutionTokenSpec.scala index 83ae08835..3047de560 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRequestingExecutionTokenSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRequestingExecutionTokenSpec.scala @@ -1,7 +1,7 @@ package cromwell.engine.workflow.lifecycle.execution.ejea +import cromwell.engine.workflow.lifecycle.execution.CallPreparationActor import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ -import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed} import cromwell.jobstore.JobStoreActor.QueryJobCompletion import org.scalatest.concurrent.Eventually @@ -45,7 +45,7 @@ class EjeaRequestingExecutionTokenSpec extends EngineJobExecutionActorSpec with ejea = helper.buildEJEA(restarting = false) ejea ! JobExecutionTokenDispensed(helper.executionToken) - helper.jobPreparationProbe.expectMsg(max = awaitTimeout, hint = "Awaiting job preparation", JobPreparationActor.Start) + helper.jobPreparationProbe.expectMsg(max = awaitTimeout, hint = "Awaiting job preparation", CallPreparationActor.Start) helper.jobStoreProbe.expectNoMsg(awaitAlmostNothing) ejea.stateName should be(PreparingJob) } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaUpdatingJobStoreSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaUpdatingJobStoreSpec.scala index 1b783a69e..73a224a00 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaUpdatingJobStoreSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaUpdatingJobStoreSpec.scala @@ -1,7 +1,7 @@ package cromwell.engine.workflow.lifecycle.execution.ejea import EngineJobExecutionActorSpec._ -import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, FailedNonRetryableResponse} +import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobFailedNonRetryableResponse} import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ import cromwell.jobstore.JobStoreActor.{JobStoreWriteFailure, JobStoreWriteSuccess} import cromwell.engine.workflow.lifecycle.execution.ejea.HasJobSuccessResponse.SuccessfulCallCacheHashes @@ -33,7 +33,7 @@ class EjeaUpdatingJobStoreSpec extends EngineJobExecutionActorSpec with HasJobSu val exception = new Exception("I loved Ophelia: forty thousand brothers\\ Could not, with all their quantity of love,\\ Make up my sum. What wilt thou do for her?") ejea ! JobStoreWriteFailure(exception) helper.replyToProbe.expectMsgPF(awaitTimeout) { - case FailedNonRetryableResponse(jobDescriptorKey, reason, None) => + case JobFailedNonRetryableResponse(jobDescriptorKey, reason, None) => jobDescriptorKey should be(helper.jobDescriptorKey) reason.getCause should be(exception) } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala index 8ef607c5b..b6ddb5601 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala @@ -1,6 +1,6 @@ package cromwell.engine.workflow.lifecycle.execution.ejea -import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, FailedNonRetryableResponse, FailedRetryableResponse, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, JobFailedNonRetryableResponse, JobFailedRetryableResponse, JobSucceededResponse} import cromwell.core.JobOutput import cromwell.core.callcaching._ import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{EJEAData, SucceededResponseData, UpdatingCallCache, UpdatingJobStore} @@ -23,7 +23,7 @@ private[ejea] trait CanValidateJobStoreKey { self: EngineJobExecutionActorSpec = } private[ejea] trait CanExpectCacheWrites extends Eventually { self: EngineJobExecutionActorSpec => - def expectCacheWrite(expectedResponse: SucceededResponse, expectedCallCacheHashes: CallCacheHashes): Unit = { + def expectCacheWrite(expectedResponse: JobSucceededResponse, expectedCallCacheHashes: CallCacheHashes): Unit = { eventually { ejea.stateName should be(UpdatingCallCache) } ejea.stateData should be(SucceededResponseData(expectedResponse, Some(Success(expectedCallCacheHashes)))) helper.callCacheWriteActorCreations match { @@ -83,7 +83,7 @@ private[ejea] trait CanExpectCacheInvalidation extends Eventually { self: Engine private[ejea] trait HasJobSuccessResponse { self: EngineJobExecutionActorSpec => val successRc = Option(171) val successOutputs = Map("a" -> JobOutput(WdlInteger(3)), "b" -> JobOutput(WdlString("bee"))) - def successResponse = SucceededResponse(helper.jobDescriptorKey, successRc, successOutputs, None, Seq.empty) + def successResponse = JobSucceededResponse(helper.jobDescriptorKey, successRc, successOutputs, None, Seq.empty) } private[ejea] object HasJobSuccessResponse { val SuccessfulCallCacheHashes = CallCacheHashes(Set(HashResult(HashKey("whatever you want"), HashValue("whatever you need")))) @@ -93,7 +93,7 @@ private[ejea] trait HasJobFailureResponses { self: EngineJobExecutionActorSpec = val failedRc = Option(12) val failureReason = new Exception("The sixth sheik's sheep is sick!") // Need to delay making the response because job descriptors come from the per-test "helper", which is null outside tests! - def failureRetryableResponse = FailedRetryableResponse(helper.jobDescriptorKey, failureReason, failedRc) - def failureNonRetryableResponse = FailedNonRetryableResponse(helper.jobDescriptorKey, failureReason, Option(12)) + def failureRetryableResponse = JobFailedRetryableResponse(helper.jobDescriptorKey, failureReason, failedRc) + def failureNonRetryableResponse = JobFailedNonRetryableResponse(helper.jobDescriptorKey, failureReason, Option(12)) def abortedResponse = AbortedResponse(helper.jobDescriptorKey) } \ No newline at end of file diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala index 6896bcc00..ef53291ca 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala @@ -4,30 +4,27 @@ import java.util.UUID import akka.actor.{ActorRef, ActorSystem, Props} import akka.testkit.{TestFSMRef, TestProbe} -import cromwell.backend.BackendJobExecutionActor.SucceededResponse -import cromwell.backend.{BackendInitializationData, BackendJobDescriptor, BackendJobDescriptorKey, BackendLifecycleActorFactory, BackendWorkflowDescriptor} +import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse +import cromwell.backend._ import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.core.callcaching.{CallCachingActivity, CallCachingMode, CallCachingOff} -import cromwell.core.{ExecutionStore, JobExecutionToken, OutputStore, WorkflowId} +import cromwell.core.{CallOutputs, JobExecutionToken, WorkflowId} import cromwell.engine.EngineWorkflowDescriptor import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{EJEAData, EngineJobExecutionActorState} import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec._ import cromwell.engine.workflow.lifecycle.execution.{EngineJobExecutionActor, WorkflowExecutionActorData} +import cromwell.engine.workflow.mocks.{DeclarationMock, TaskMock, WdlExpressionMock} import cromwell.util.AkkaTestUtil._ import org.specs2.mock.Mockito -import wdl4s.WdlExpression.ScopedLookupFunction import wdl4s._ -import wdl4s.expression.{NoFunctions, WdlFunctions, WdlStandardLibraryFunctions} +import wdl4s.expression.{NoFunctions, WdlStandardLibraryFunctions} import wdl4s.parser.WdlParser.Ast import wdl4s.types.{WdlIntegerType, WdlStringType} -import wdl4s.values.{WdlInteger, WdlString, WdlValue} -import scala.util.Success - -private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mockito { +private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mockito with TaskMock with WdlExpressionMock with DeclarationMock { val workflowId = WorkflowId.randomId() val workflowName = "wf" @@ -38,28 +35,14 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock val executionToken = JobExecutionToken(JobExecutionTokenType("test", None), UUID.randomUUID()) - val task = mock[Task] - task.declarations returns Seq.empty - task.runtimeAttributes returns new RuntimeAttributes(Map.empty) - task.commandTemplateString returns "!!shazam!!" - task.name returns taskName - val stringOutputExpression = mock[WdlExpression] - stringOutputExpression.valueString returns "hello" - stringOutputExpression.evaluate(any[ScopedLookupFunction], any[ WdlFunctions[WdlValue]]) returns Success(WdlString("hello")) - task.outputs returns Seq(TaskOutput("outString", WdlStringType, stringOutputExpression, mock[Ast], Option(task))) - - val intInputExpression = mock[WdlExpression] - intInputExpression.valueString returns "543" - intInputExpression.evaluate(any[ScopedLookupFunction], any[WdlFunctions[WdlValue]]) returns Success(WdlInteger(543)) - - val intInputDeclaration = mock[Declaration] - intInputDeclaration.unqualifiedName returns "inInt" - intInputDeclaration.expression returns Option(intInputExpression) - intInputDeclaration.wdlType returns WdlIntegerType - task.declarations returns Seq(intInputDeclaration) - - val workflow = new Workflow(workflowName, Seq.empty, mock[Ast]) - val call: Call = new Call(None, task, Map.empty, mock[Ast]) + val task = mockTask( + taskName, + declarations = Seq(mockDeclaration("inInt", WdlIntegerType, mockIntExpression(543))), + outputs = Seq(("outString", WdlStringType, mockStringExpression("hello"))) + ) + + val workflow = new Workflow(workflowName, Seq.empty, mock[WdlSyntaxErrorFormatter], mock[Ast]) + val call: TaskCall = TaskCall(None, task, Map.empty, mock[Ast]) call.parent_=(workflow) val jobDescriptorKey = BackendJobDescriptorKey(call, jobIndex, jobAttempt) @@ -68,7 +51,7 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock var fetchCachedResultsActorCreations: ExpectOne[(CallCachingEntryId, Seq[TaskOutput])] = NothingYet var jobHashingInitializations: ExpectOne[(BackendJobDescriptor, CallCachingActivity)] = NothingYet - var callCacheWriteActorCreations: ExpectOne[(CallCacheHashes, SucceededResponse)] = NothingYet + var callCacheWriteActorCreations: ExpectOne[(CallCacheHashes, JobSucceededResponse)] = NothingYet var invalidateCacheActorCreations: ExpectOne[CallCachingEntryId] = NothingYet val deathwatch = TestProbe() @@ -98,12 +81,12 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock // These two factory methods should never be called from EJEA or any of its descendants: override def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Set[Call], - executionStore: ExecutionStore, - outputStore: OutputStore, + calls: Set[TaskCall], + jobExecutionMap: JobExecutionMap, + workflowOutputs: CallOutputs, initializationData: Option[BackendInitializationData]): Option[Props] = throw new UnsupportedOperationException("Unexpected finalization actor creation!") override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Set[Call], + calls: Set[TaskCall], serviceRegistryActor: ActorRef): Option[Props] = throw new UnsupportedOperationException("Unexpected finalization actor creation!") } @@ -112,14 +95,14 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock (implicit startingState: EngineJobExecutionActorState): TestFSMRef[EngineJobExecutionActorState, EJEAData, MockEjea] = { val factory: BackendLifecycleActorFactory = buildFactory() - val descriptor = EngineWorkflowDescriptor(backendWorkflowDescriptor, Map.empty, null, null, null, callCachingMode) + val descriptor = EngineWorkflowDescriptor(mock[WdlNamespaceWithWorkflow], backendWorkflowDescriptor, Map.empty, null, null, null, callCachingMode) val myBrandNewEjea = new TestFSMRef[EngineJobExecutionActorState, EJEAData, MockEjea](system, Props(new MockEjea( helper = this, jobPreparationProbe = jobPreparationProbe, replyTo = replyToProbe.ref, jobDescriptorKey = jobDescriptorKey, - executionData = WorkflowExecutionActorData(descriptor, ExecutionStore(Map.empty), Map.empty, Map.empty, OutputStore(Map.empty)), + executionData = WorkflowExecutionActorData.empty(descriptor), factory = factory, initializationData = None, restarting = restarting, @@ -153,7 +136,7 @@ private[ejea] class MockEjea(helper: PerTestHelper, override def makeFetchCachedResultsActor(cacheId: CallCachingEntryId, taskOutputs: Seq[TaskOutput]) = helper.fetchCachedResultsActorCreations = helper.fetchCachedResultsActorCreations.foundOne((cacheId, taskOutputs)) override def initializeJobHashing(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity) = helper.jobHashingInitializations = helper.jobHashingInitializations.foundOne((jobDescriptor, activity)) - override def createSaveCacheResultsActor(hashes: CallCacheHashes, success: SucceededResponse) = helper.callCacheWriteActorCreations = helper.callCacheWriteActorCreations.foundOne((hashes, success)) + override def createSaveCacheResultsActor(hashes: CallCacheHashes, success: JobSucceededResponse) = helper.callCacheWriteActorCreations = helper.callCacheWriteActorCreations.foundOne((hashes, success)) override def invalidateCacheHit(cacheId: CallCachingEntryId): Unit = { helper.invalidateCacheActorCreations = helper.invalidateCacheActorCreations.foundOne(cacheId) } override def createJobPreparationActor(jobPrepProps: Props, name: String) = jobPreparationProbe.ref } diff --git a/engine/src/test/scala/cromwell/engine/workflow/mocks/DeclarationMock.scala b/engine/src/test/scala/cromwell/engine/workflow/mocks/DeclarationMock.scala new file mode 100644 index 000000000..471d00777 --- /dev/null +++ b/engine/src/test/scala/cromwell/engine/workflow/mocks/DeclarationMock.scala @@ -0,0 +1,21 @@ +package cromwell.engine.workflow.mocks + +import org.specs2.mock.Mockito +import wdl4s.{Declaration, WdlExpression} +import wdl4s.types.WdlType + +object DeclarationMock { + type DeclarationMockType = (String, WdlType, WdlExpression) +} + +trait DeclarationMock extends Mockito { + def mockDeclaration(name: String, + wdlType: WdlType, + expression: WdlExpression) = { + val declaration = mock[Declaration] + declaration.unqualifiedName returns name + declaration.expression returns Option(expression) + declaration.wdlType returns wdlType + declaration + } +} diff --git a/engine/src/test/scala/cromwell/engine/workflow/mocks/TaskMock.scala b/engine/src/test/scala/cromwell/engine/workflow/mocks/TaskMock.scala new file mode 100644 index 000000000..4d8ef9c1d --- /dev/null +++ b/engine/src/test/scala/cromwell/engine/workflow/mocks/TaskMock.scala @@ -0,0 +1,27 @@ +package cromwell.engine.workflow.mocks + +import cromwell.engine.workflow.mocks.DeclarationMock.DeclarationMockType +import org.specs2.mock.Mockito +import wdl4s._ +import wdl4s.parser.WdlParser.Ast + +trait TaskMock extends Mockito { + + def mockTask(name: String, + declarations: Seq[Declaration] = Seq.empty, + runtimeAttributes: RuntimeAttributes = new RuntimeAttributes(Map.empty), + commandTemplateString: String = "!!shazam!!", + outputs: Seq[DeclarationMockType] = Seq.empty + ) = { + val task = mock[Task] + task.declarations returns declarations + task.runtimeAttributes returns runtimeAttributes + task.commandTemplateString returns commandTemplateString + task.name returns name + task.unqualifiedName returns name + task.outputs returns (outputs map { + case (outputName, wdlType, expression) => TaskOutput(outputName, wdlType, expression, mock[Ast], Option(task)) + }) + task + } +} diff --git a/engine/src/test/scala/cromwell/engine/workflow/mocks/WdlExpressionMock.scala b/engine/src/test/scala/cromwell/engine/workflow/mocks/WdlExpressionMock.scala new file mode 100644 index 000000000..6bb15b306 --- /dev/null +++ b/engine/src/test/scala/cromwell/engine/workflow/mocks/WdlExpressionMock.scala @@ -0,0 +1,32 @@ +package cromwell.engine.workflow.mocks + +import org.specs2.mock.Mockito +import wdl4s.WdlExpression +import wdl4s.WdlExpression._ +import wdl4s.expression.WdlFunctions +import wdl4s.values.{WdlInteger, WdlString, WdlValue} + +import scala.util.Success + +trait WdlExpressionMock extends Mockito { + val helloStringExpression = { + val expression = mock[WdlExpression] + expression.valueString returns "hello" + expression.evaluate(any[ScopedLookupFunction], any[ WdlFunctions[WdlValue]]) returns Success(WdlString("hello")) + expression + } + + def mockStringExpression(value: String) = { + val expression = mock[WdlExpression] + expression.valueString returns value + expression.evaluate(any[ScopedLookupFunction], any[ WdlFunctions[WdlValue]]) returns Success(WdlString(value)) + expression + } + + def mockIntExpression(value: Int) = { + val expression = mock[WdlExpression] + expression.valueString returns value.toString + expression.evaluate(any[ScopedLookupFunction], any[ WdlFunctions[WdlValue]]) returns Success(WdlInteger(value)) + expression + } +} diff --git a/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala b/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala index 4ac29abca..ac84a6c45 100644 --- a/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala +++ b/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala @@ -11,7 +11,7 @@ import org.specs2.mock.Mockito import wdl4s.parser.WdlParser.Ast import wdl4s.types.WdlStringType import wdl4s.values.WdlString -import wdl4s.{Call, Task, TaskOutput, WdlExpression} +import wdl4s._ import scala.concurrent.duration._ import scala.language.postfixOps @@ -29,7 +29,7 @@ class JobStoreServiceSpec extends CromwellTestKitSpec with Matchers with Mockito val jobStoreService = system.actorOf(JobStoreActor.props(jobStore)) val workflowId = WorkflowId.randomId() - val successCall = mock[Call] + val successCall = mock[TaskCall] successCall.fullyQualifiedName returns "foo.bar" val mockTask = mock[Task] mockTask.outputs returns Seq(TaskOutput("baz", WdlStringType, EmptyExpression, mock[Ast], Option(mockTask))) @@ -50,7 +50,7 @@ class JobStoreServiceSpec extends CromwellTestKitSpec with Matchers with Mockito case JobComplete(JobResultSuccess(Some(0), os)) if os == outputs => } - val failureCall = mock[Call] + val failureCall = mock[TaskCall] failureCall.fullyQualifiedName returns "baz.qux" val failureKey = BackendJobDescriptorKey(failureCall, None, 1).toJobStoreKey(workflowId) diff --git a/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala b/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala new file mode 100644 index 000000000..7c11e4e6e --- /dev/null +++ b/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala @@ -0,0 +1,87 @@ +package cromwell.subworkflowstore + +import cromwell.CromwellTestKitSpec +import cromwell.core.{JobKey, WorkflowId, WorkflowSourceFiles} +import cromwell.services.SingletonServicesStore +import cromwell.subworkflowstore.SubWorkflowStoreActor._ +import org.scalatest.Matchers +import org.specs2.mock.Mockito +import wdl4s.{Scope, TaskCall, WdlExpression} +import cromwell.core.ExecutionIndex._ + +import scala.concurrent.duration._ +import SubWorkflowStoreSpec._ +import akka.testkit.TestProbe +import cromwell.database.sql.tables.SubWorkflowStoreEntry +import cromwell.engine.workflow.workflowstore.WorkflowStoreActor.{SubmitWorkflow, WorkflowSubmittedToStore} +import cromwell.engine.workflow.workflowstore.{SqlWorkflowStore, WorkflowStoreActor} + +import scala.language.postfixOps + +object SubWorkflowStoreSpec { + val MaxWait = 5 seconds + val EmptyExpression = WdlExpression.fromString(""" "" """) +} + +class SubWorkflowStoreSpec extends CromwellTestKitSpec with Matchers with Mockito { + "SubWorkflowStore" should { + "work" in { + lazy val subWorkflowStore = new SqlSubWorkflowStore(SingletonServicesStore.databaseInterface) + val subWorkflowStoreService = system.actorOf(SubWorkflowStoreActor.props(subWorkflowStore)) + + lazy val workflowStore = SqlWorkflowStore(SingletonServicesStore.databaseInterface) + val workflowStoreService = system.actorOf(WorkflowStoreActor.props(workflowStore, TestProbe().ref)) + + val parentWorkflowId = WorkflowId.randomId() + val subWorkflowId = WorkflowId.randomId() + val subSubWorkflowId = WorkflowId.randomId() + val call = mock[TaskCall] + call.fullyQualifiedName returns "foo.bar" + val jobKey = new JobKey { + override def scope: Scope = call + override def index: Option[Int] = None + override def attempt: Int = 0 + override def tag: String = "foobar" + } + + workflowStoreService ! SubmitWorkflow(WorkflowSourceFiles("", "{}", "{}")) + val rootWorkflowId = expectMsgType[WorkflowSubmittedToStore](10 seconds).workflowId + + // Query for non existing sub workflow + subWorkflowStoreService ! QuerySubWorkflow(parentWorkflowId, jobKey) + expectMsgType[SubWorkflowNotFound](MaxWait) + + // Register sub workflow + subWorkflowStoreService ! RegisterSubWorkflow(rootWorkflowId, parentWorkflowId, jobKey, subWorkflowId) + expectMsgType[SubWorkflowStoreRegisterSuccess](MaxWait) + + // Query for sub workflow + subWorkflowStoreService ! QuerySubWorkflow(parentWorkflowId, jobKey) + val subWorkflowEntry = SubWorkflowStoreEntry(Option(0), parentWorkflowId.toString, jobKey.scope.fullyQualifiedName, jobKey.index.fromIndex, jobKey.attempt, subWorkflowId.toString, Some(0)) + expectMsg[SubWorkflowFound](SubWorkflowFound(subWorkflowEntry)) + + // Register sub sub workflow + subWorkflowStoreService ! RegisterSubWorkflow(rootWorkflowId, subWorkflowId, jobKey, subSubWorkflowId) + expectMsgType[SubWorkflowStoreRegisterSuccess](MaxWait) + + // Query for sub sub workflow + subWorkflowStoreService ! QuerySubWorkflow(subWorkflowId, jobKey) + val subSubWorkflowEntry = SubWorkflowStoreEntry(Option(0), subWorkflowId.toString, jobKey.scope.fullyQualifiedName, jobKey.index.fromIndex, jobKey.attempt, subSubWorkflowId.toString, Some(1)) + expectMsg[SubWorkflowFound](SubWorkflowFound(subSubWorkflowEntry)) + + // Delete root workflow + subWorkflowStoreService ! WorkflowComplete(rootWorkflowId) + expectMsgType[SubWorkflowStoreCompleteSuccess](MaxWait) + + // Verify that everything is gone + subWorkflowStoreService ! QuerySubWorkflow(rootWorkflowId, jobKey) + expectMsgType[SubWorkflowNotFound](MaxWait) + + subWorkflowStoreService ! QuerySubWorkflow(parentWorkflowId, jobKey) + expectMsgType[SubWorkflowNotFound](MaxWait) + + subWorkflowStoreService ! QuerySubWorkflow(subWorkflowId, jobKey) + expectMsgType[SubWorkflowNotFound](MaxWait) + } + } +} diff --git a/engine/src/test/scala/cromwell/webservice/EngineStatsActorSpec.scala b/engine/src/test/scala/cromwell/webservice/EngineStatsActorSpec.scala index 021e4a43c..b32ceb1c9 100644 --- a/engine/src/test/scala/cromwell/webservice/EngineStatsActorSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/EngineStatsActorSpec.scala @@ -14,22 +14,22 @@ class EngineStatsActorSpec extends TestKitSuite("EngineStatsActor") with FlatSpe behavior of "EngineStatsActor" val replyTo = TestProbe() - val defaultTimeout = 100 millis + val defaultTimeout = 500 millis it should "return double zeros with no WorkflowActors" in { - TestActorRef(EngineStatsActor.props(List.empty[ActorRef], replyTo.ref)) + TestActorRef(EngineStatsActor.props(List.empty[ActorRef], replyTo.ref, timeout = 200 millis)) replyTo.expectMsg(defaultTimeout, EngineStats(0, 0)) } it should "return snakeyes with a single workflow with one job" in { val workflowActors = List(Props(FakeWorkflowActor(1))) map { TestActorRef(_) } - TestActorRef(EngineStatsActor.props(workflowActors, replyTo.ref)) + TestActorRef(EngineStatsActor.props(workflowActors, replyTo.ref, timeout = 200 millis)) replyTo.expectMsg(defaultTimeout, EngineStats(1, 1)) } it should "return an unemployed workflow when that's the world it lives in" in { val workflowActors = List(Props(FakeWorkflowActor(0))) map { TestActorRef(_) } - TestActorRef(EngineStatsActor.props(workflowActors, replyTo.ref)) + TestActorRef(EngineStatsActor.props(workflowActors, replyTo.ref, timeout = 200 millis)) replyTo.expectMsg(defaultTimeout, EngineStats(1, 0)) } @@ -41,7 +41,7 @@ class EngineStatsActorSpec extends TestKitSuite("EngineStatsActor") with FlatSpe it should "return the summation of jobs for all WorkflowActors" in { val workflowActors = List(Props(FakeWorkflowActor(1)), Props(FakeWorkflowActor(2))) map { TestActorRef(_) } - TestActorRef(EngineStatsActor.props(workflowActors, replyTo.ref)) + TestActorRef(EngineStatsActor.props(workflowActors, replyTo.ref, timeout = 200 millis)) replyTo.expectMsg(defaultTimeout, EngineStats(2, 3)) } } diff --git a/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala b/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala index 49b631876..9ecfeea42 100644 --- a/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala @@ -95,7 +95,7 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik | "id": "$workflowA" |}""".stripMargin - val mdQuery = MetadataQuery(workflowA, None, None, None, None) + val mdQuery = MetadataQuery(workflowA, None, None, None, None, expandSubWorkflows = false) val queryAction = GetMetadataQueryAction(mdQuery) assertMetadataResponse(queryAction, mdQuery, workflowAEvents, expectedRes) } @@ -112,8 +112,8 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik val events = eventList map { e => (e._1, MetadataValue(e._2), e._3) } map Function.tupled(makeEvent(workflow)) val expectedRes = s"""{ "calls": {}, $expectedJson, "id":"$workflow" }""" - val mdQuery = MetadataQuery(workflow, None, None, None, None) - val queryAction = GetSingleWorkflowMetadataAction(workflow, None, None) + val mdQuery = MetadataQuery(workflow, None, None, None, None, expandSubWorkflows = false) + val queryAction = GetSingleWorkflowMetadataAction(workflow, None, None, expandSubWorkflows = false) assertMetadataResponse(queryAction, mdQuery, events, expectedRes) } @@ -304,7 +304,7 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik | } """.stripMargin - val mdQuery = MetadataQuery(workflowId, None, None, None, None) + val mdQuery = MetadataQuery(workflowId, None, None, None, None, expandSubWorkflows = false) val queryAction = GetMetadataQueryAction(mdQuery) assertMetadataResponse(queryAction, mdQuery, events, expectedResponse) } @@ -325,7 +325,7 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik |} """.stripMargin - val mdQuery = MetadataQuery(workflowId, None, None, None, None) + val mdQuery = MetadataQuery(workflowId, None, None, None, None, expandSubWorkflows = false) val queryAction = GetMetadataQueryAction(mdQuery) assertMetadataResponse(queryAction, mdQuery, events, expectedResponse) } @@ -345,14 +345,14 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik |} """.stripMargin - val mdQuery = MetadataQuery(workflowId, None, None, None, None) + val mdQuery = MetadataQuery(workflowId, None, None, None, None, expandSubWorkflows = false) val queryAction = GetMetadataQueryAction(mdQuery) assertMetadataResponse(queryAction, mdQuery, events, expectedResponse) } it should "render empty Json" in { val workflowId = WorkflowId.randomId() - val mdQuery = MetadataQuery(workflowId, None, None, None, None) + val mdQuery = MetadataQuery(workflowId, None, None, None, None, expandSubWorkflows = false) val queryAction = GetMetadataQueryAction(mdQuery) val expectedEmptyResponse = """{}""" assertMetadataResponse(queryAction, mdQuery, List.empty, expectedEmptyResponse) @@ -382,7 +382,7 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik |} """.stripMargin - val mdQuery = MetadataQuery(workflowId, None, None, None, None) + val mdQuery = MetadataQuery(workflowId, None, None, None, None, expandSubWorkflows = false) val queryAction = GetMetadataQueryAction(mdQuery) assertMetadataResponse(queryAction, mdQuery, emptyEvents, expectedEmptyResponse) @@ -397,4 +397,98 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik assertMetadataResponse(queryAction, mdQuery, valueEvents, expectedNonEmptyResponse) } + + it should "expand sub workflow metadata when asked for" in { + val mainWorkflowId = WorkflowId.randomId() + val subWorkflowId = WorkflowId.randomId() + + val mainEvents = List( + MetadataEvent(MetadataKey(mainWorkflowId, Option(MetadataJobKey("callA", None, 1)), "subWorkflowId"), MetadataValue(subWorkflowId)) + ) + + val subEvents = List( + MetadataEvent(MetadataKey(mainWorkflowId, None, "some"), MetadataValue("sub workflow info")) + ) + + val mainQuery = MetadataQuery(mainWorkflowId, None, None, None, None, expandSubWorkflows = true) + val mainQueryAction = GetMetadataQueryAction(mainQuery) + + val subQuery = MetadataQuery(subWorkflowId, None, None, None, None, expandSubWorkflows = true) + val subQueryAction = GetMetadataQueryAction(subQuery) + + val parentProbe = TestProbe() + val metadataBuilder = TestActorRef(MetadataBuilderActor.props(mockServiceRegistry.ref), parentProbe.ref, s"MetadataActor-${UUID.randomUUID()}") + metadataBuilder ! mainQueryAction + mockServiceRegistry.expectMsg(defaultTimeout, mainQueryAction) + mockServiceRegistry.reply(MetadataLookupResponse(mainQuery, mainEvents)) + mockServiceRegistry.expectMsg(defaultTimeout, subQueryAction) + mockServiceRegistry.reply(MetadataLookupResponse(subQuery, subEvents)) + + val expandedRes = + s""" + |{ + | "calls": { + | "callA": [ + | { + | "subWorkflowMetadata": { + | "some": "sub workflow info", + | "calls": {}, + | "id": "$subWorkflowId" + | }, + | "attempt": 1, + | "shardIndex": -1 + | } + | ] + | }, + | "id": "$mainWorkflowId" + |} + """.stripMargin + + parentProbe.expectMsgPF(defaultTimeout) { + case response: RequestComplete[(StatusCode, JsObject)] @unchecked => + response.response._1 shouldBe StatusCodes.OK + response.response._2 shouldBe expandedRes.parseJson + } + } + + it should "NOT expand sub workflow metadata when NOT asked for" in { + val mainWorkflowId = WorkflowId.randomId() + val subWorkflowId = WorkflowId.randomId() + + val mainEvents = List( + MetadataEvent(MetadataKey(mainWorkflowId, Option(MetadataJobKey("callA", None, 1)), "subWorkflowId"), MetadataValue(subWorkflowId)) + ) + + val queryNoExpand = MetadataQuery(mainWorkflowId, None, None, None, None, expandSubWorkflows = false) + val queryNoExpandAction = GetMetadataQueryAction(queryNoExpand) + + val parentProbe = TestProbe() + val metadataBuilder = TestActorRef(MetadataBuilderActor.props(mockServiceRegistry.ref), parentProbe.ref, s"MetadataActor-${UUID.randomUUID()}") + metadataBuilder ! queryNoExpandAction + mockServiceRegistry.expectMsg(defaultTimeout, queryNoExpandAction) + mockServiceRegistry.reply(MetadataLookupResponse(queryNoExpand, mainEvents)) + + + val nonExpandedRes = + s""" + |{ + | "calls": { + | "callA": [ + | { + | "subWorkflowId": "$subWorkflowId", + | "attempt": 1, + | "shardIndex": -1 + | } + | ] + | }, + | "id": "$mainWorkflowId" + |} + """.stripMargin + + parentProbe.expectMsgPF(defaultTimeout) { + case response: RequestComplete[(StatusCode, JsObject)] @unchecked => + response.response._1 shouldBe StatusCodes.OK + response.response._2 shouldBe nonExpandedRes.parseJson + } + } } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 04c3f02b0..fd8f8891d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -2,7 +2,7 @@ import sbt._ object Dependencies { lazy val lenthallV = "0.19" - lazy val wdl4sV = "0.7-b9c6c98-SNAPSHOT" + lazy val wdl4sV = "0.7-4a9e61e-SNAP" lazy val sprayV = "1.3.3" /* spray-json is an independent project from the "spray suite" diff --git a/services/src/main/scala/cromwell/services/metadata/CallMetadataKeys.scala b/services/src/main/scala/cromwell/services/metadata/CallMetadataKeys.scala index 5284a9a8e..bf1df98d9 100644 --- a/services/src/main/scala/cromwell/services/metadata/CallMetadataKeys.scala +++ b/services/src/main/scala/cromwell/services/metadata/CallMetadataKeys.scala @@ -17,4 +17,6 @@ object CallMetadataKeys { val BackendLogsPrefix = "backendLogs" val JobId = "jobId" val CallRoot = "callRoot" + val SubWorkflowId = "subWorkflowId" + val SubWorkflowMetadata = "subWorkflowMetadata" } diff --git a/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala b/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala index 374169d05..6542c2082 100644 --- a/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala +++ b/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala @@ -36,7 +36,7 @@ case object MetadataBoolean extends MetadataType { override val typeName = "bool object MetadataValue { def apply(value: Any) = { - value match { + Option(value).getOrElse("") match { case WdlInteger(i) => new MetadataValue(i.toString, MetadataInt) case WdlFloat(f) => new MetadataValue(f.toString, MetadataNumber) case WdlBoolean(b) => new MetadataValue(b.toString, MetadataBoolean) @@ -75,16 +75,17 @@ object MetadataQueryJobKey { case class MetadataQuery(workflowId: WorkflowId, jobKey: Option[MetadataQueryJobKey], key: Option[String], includeKeysOption: Option[NonEmptyList[String]], - excludeKeysOption: Option[NonEmptyList[String]]) + excludeKeysOption: Option[NonEmptyList[String]], + expandSubWorkflows: Boolean) object MetadataQuery { - def forWorkflow(workflowId: WorkflowId) = MetadataQuery(workflowId, None, None, None, None) + def forWorkflow(workflowId: WorkflowId) = MetadataQuery(workflowId, None, None, None, None, expandSubWorkflows = false) def forJob(workflowId: WorkflowId, jobKey: MetadataJobKey) = { - MetadataQuery(workflowId, Option(MetadataQueryJobKey.forMetadataJobKey(jobKey)), None, None, None) + MetadataQuery(workflowId, Option(MetadataQueryJobKey.forMetadataJobKey(jobKey)), None, None, None, expandSubWorkflows = false) } def forKey(key: MetadataKey) = { - MetadataQuery(key.workflowId, key.jobKey map MetadataQueryJobKey.forMetadataJobKey, Option(key.key), None, None) + MetadataQuery(key.workflowId, key.jobKey map MetadataQueryJobKey.forMetadataJobKey, Option(key.key), None, None, expandSubWorkflows = false) } } diff --git a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala index 672f68580..e2a62c784 100644 --- a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala +++ b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala @@ -50,7 +50,8 @@ object MetadataService { case class PutMetadataAction(events: Iterable[MetadataEvent]) extends MetadataServiceAction case class GetSingleWorkflowMetadataAction(workflowId: WorkflowId, includeKeysOption: Option[NonEmptyList[String]], - excludeKeysOption: Option[NonEmptyList[String]]) + excludeKeysOption: Option[NonEmptyList[String]], + expandSubWorkflows: Boolean) extends ReadAction case class GetMetadataQueryAction(key: MetadataQuery) extends ReadAction case class GetStatus(workflowId: WorkflowId) extends ReadAction diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala index 76476d3fd..5250107b6 100644 --- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala +++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala @@ -107,19 +107,19 @@ trait MetadataDatabaseAccess { val uuid = query.workflowId.id.toString val futureMetadata: Future[Seq[MetadataEntry]] = query match { - case MetadataQuery(_, None, None, None, None) => databaseInterface.queryMetadataEntries(uuid) - case MetadataQuery(_, None, Some(key), None, None) => databaseInterface.queryMetadataEntries(uuid, key) - case MetadataQuery(_, Some(jobKey), None, None, None) => + case MetadataQuery(_, None, None, None, None, _) => databaseInterface.queryMetadataEntries(uuid) + case MetadataQuery(_, None, Some(key), None, None, _) => databaseInterface.queryMetadataEntries(uuid, key) + case MetadataQuery(_, Some(jobKey), None, None, None, _) => databaseInterface.queryMetadataEntries(uuid, jobKey.callFqn, jobKey.index, jobKey.attempt) - case MetadataQuery(_, Some(jobKey), Some(key), None, None) => + case MetadataQuery(_, Some(jobKey), Some(key), None, None, _) => databaseInterface.queryMetadataEntries(uuid, key, jobKey.callFqn, jobKey.index, jobKey.attempt) - case MetadataQuery(_, None, None, Some(includeKeys), None) => + case MetadataQuery(_, None, None, Some(includeKeys), None, _) => databaseInterface. queryMetadataEntriesLikeMetadataKeys(uuid, includeKeys.map(_ + "%"), requireEmptyJobKey = false) - case MetadataQuery(_, None, None, None, Some(excludeKeys)) => + case MetadataQuery(_, None, None, None, Some(excludeKeys), _) => databaseInterface. queryMetadataEntryNotLikeMetadataKeys(uuid, excludeKeys.map(_ + "%"), requireEmptyJobKey = false) - case MetadataQuery(_, None, None, Some(includeKeys), Some(excludeKeys)) => Future.failed( + case MetadataQuery(_, None, None, Some(includeKeys), Some(excludeKeys), _) => Future.failed( new IllegalArgumentException( s"Include/Exclude keys may not be mixed: include = $includeKeys, exclude = $excludeKeys")) case invalidQuery => Future.failed(new IllegalArgumentException( diff --git a/services/src/main/scala/cromwell/services/metadata/impl/ReadMetadataActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/ReadMetadataActor.scala index ebdc500de..5308d69f2 100644 --- a/services/src/main/scala/cromwell/services/metadata/impl/ReadMetadataActor.scala +++ b/services/src/main/scala/cromwell/services/metadata/impl/ReadMetadataActor.scala @@ -5,7 +5,7 @@ import cromwell.core.Dispatcher.ApiDispatcher import cromwell.core.{WorkflowId, WorkflowSubmitted} import cromwell.services.SingletonServicesStore import cromwell.services.metadata.MetadataService._ -import cromwell.services.metadata.{MetadataQuery, WorkflowQueryParameters} +import cromwell.services.metadata.{CallMetadataKeys, MetadataQuery, WorkflowQueryParameters} import scala.concurrent.Future import scala.util.{Failure, Success, Try} @@ -19,9 +19,12 @@ class ReadMetadataActor extends Actor with ActorLogging with MetadataDatabaseAcc implicit val ec = context.dispatcher def receive = { - case GetSingleWorkflowMetadataAction(workflowId, includeKeysOption, excludeKeysOption) => - queryAndRespond(MetadataQuery(workflowId, None, None, includeKeysOption, excludeKeysOption)) - case GetMetadataQueryAction(query@MetadataQuery(_, _, _, _, _)) => queryAndRespond(query) + case GetSingleWorkflowMetadataAction(workflowId, includeKeysOption, excludeKeysOption, expandSubWorkflows) => + val includeKeys = if (expandSubWorkflows) { + includeKeysOption map { _.::(CallMetadataKeys.SubWorkflowId) } + } else includeKeysOption + queryAndRespond(MetadataQuery(workflowId, None, None, includeKeys, excludeKeysOption, expandSubWorkflows)) + case GetMetadataQueryAction(query@MetadataQuery(_, _, _, _, _, _)) => queryAndRespond(query) case GetStatus(workflowId) => queryStatusAndRespond(workflowId) case GetLogs(workflowId) => queryLogsAndRespond(workflowId) case query: WorkflowQuery[_] => queryWorkflowsAndRespond(query.uri, query.parameters) diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala index 4c83e9dd3..3c8e48fd4 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala @@ -5,10 +5,10 @@ import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import cromwell.backend._ import cromwell.backend.impl.htcondor.caching.CacheActorFactory -import cromwell.backend.io.JobPaths +import cromwell.backend.io.JobPathsWithDocker import cromwell.backend.sfs.SharedFileSystemExpressionFunctions import cromwell.core.{CallContext, WorkflowOptions} -import wdl4s.Call +import wdl4s.TaskCall import wdl4s.expression.WdlStandardLibraryFunctions import scala.util.{Failure, Success, Try} @@ -17,7 +17,7 @@ case class HtCondorBackendFactory(name: String, configurationDescriptor: Backend extends BackendLifecycleActorFactory with StrictLogging { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Set[Call], + calls: Set[TaskCall], serviceRegistryActor: ActorRef): Option[Props] = { Option(HtCondorInitializationActor.props(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor)) } @@ -32,7 +32,7 @@ case class HtCondorBackendFactory(name: String, configurationDescriptor: Backend override def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor, jobKey: BackendJobDescriptorKey, initializationData: Option[BackendInitializationData]): WdlStandardLibraryFunctions = { - val jobPaths = new JobPaths(workflowDescriptor, configurationDescriptor.backendConfig, jobKey) + val jobPaths = new JobPathsWithDocker(jobKey, workflowDescriptor, configurationDescriptor.backendConfig) val callContext = CallContext( jobPaths.callExecutionRoot, jobPaths.stdout.toAbsolutePath.toString, diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala index c661acf19..1212876b6 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala @@ -7,9 +7,9 @@ import cromwell.backend.validation.RuntimeAttributesDefault import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor} import cromwell.core.WorkflowOptions +import wdl4s.TaskCall import wdl4s.types.{WdlBooleanType, WdlIntegerType, WdlStringType} import wdl4s.values.WdlValue -import wdl4s.Call import scala.concurrent.Future import scala.util.Try @@ -19,14 +19,14 @@ object HtCondorInitializationActor { ContinueOnReturnCodeKey, CpuKey, MemoryKey, DiskKey) def props(workflowDescriptor: BackendWorkflowDescriptor, - calls: Set[Call], + calls: Set[TaskCall], configurationDescriptor: BackendConfigurationDescriptor, serviceRegistryActor: ActorRef): Props = Props(new HtCondorInitializationActor(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor)) } class HtCondorInitializationActor(override val workflowDescriptor: BackendWorkflowDescriptor, - override val calls: Set[Call], + override val calls: Set[TaskCall], override val configurationDescriptor: BackendConfigurationDescriptor, override val serviceRegistryActor: ActorRef) extends BackendWorkflowInitializationActor { diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala index dbecacdde..d005016ca 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala @@ -4,11 +4,11 @@ import java.nio.file.attribute.PosixFilePermission import java.util.UUID import akka.actor.{ActorRef, Props} -import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, FailedNonRetryableResponse, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobFailedNonRetryableResponse, JobSucceededResponse} import cromwell.backend._ import cromwell.backend.impl.htcondor.caching.CacheActor._ import cromwell.backend.impl.htcondor.caching.localization.CachedResultLocalization -import cromwell.backend.io.JobPaths +import cromwell.backend.io.JobPathsWithDocker import cromwell.backend.sfs.{SharedFileSystem, SharedFileSystemExpressionFunctions} import cromwell.backend.wdl.Command import cromwell.core.path.JavaWriterImplicits._ @@ -55,7 +55,7 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor private val fileSystemsConfig = configurationDescriptor.backendConfig.getConfig("filesystems") override val sharedFileSystemConfig = fileSystemsConfig.getConfig("local") private val workflowDescriptor = jobDescriptor.workflowDescriptor - private val jobPaths = new JobPaths(workflowDescriptor, configurationDescriptor.backendConfig, jobDescriptor.key) + private val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, configurationDescriptor.backendConfig) // Files private val executionDir = jobPaths.callExecutionRoot @@ -204,18 +204,18 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor condorJobId = Option(overallJobIdentifier) self ! TrackTaskStatus(overallJobIdentifier) - case _ => self ! JobExecutionResponse(FailedNonRetryableResponse(jobDescriptor.key, + case _ => self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key, new IllegalStateException("Failed to retrieve job(id) and cluster id"), Option(condorReturnCode))) } case 0 => log.error(s"Unexpected! Received return code for condor submission as 0, although stderr file is non-empty: {}", File(submitFileStderr).lines) - self ! JobExecutionResponse(FailedNonRetryableResponse(jobDescriptor.key, + self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key, new IllegalStateException(s"Execution process failed. HtCondor returned zero status code but non empty stderr file: $condorReturnCode"), Option(condorReturnCode))) case nonZeroExitCode: Int => - self ! JobExecutionResponse(FailedNonRetryableResponse(jobDescriptor.key, + self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key, new IllegalStateException(s"Execution process failed. HtCondor returned non zero status code: $condorReturnCode"), Option(condorReturnCode))) } } @@ -231,16 +231,16 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor context.system.scheduler.scheduleOnce(pollingInterval.seconds, self, TrackTaskStatus(jobIdentifier)) () case Success(Some(rc)) if runtimeAttributes.continueOnReturnCode.continueFor(rc) => self ! JobExecutionResponse(processSuccess(rc)) - case Success(Some(rc)) => self ! JobExecutionResponse(FailedNonRetryableResponse(jobDescriptor.key, + case Success(Some(rc)) => self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key, new IllegalStateException("Job exited with invalid return code: " + rc), Option(rc))) - case Failure(error) => self ! JobExecutionResponse(FailedNonRetryableResponse(jobDescriptor.key, error, None)) + case Failure(error) => self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key, error, None)) } } private def processSuccess(rc: Int): BackendJobExecutionResponse = { evaluateOutputs(callEngineFunction, outputMapper(jobPaths)) match { case Success(outputs) => - val succeededResponse = SucceededResponse(jobDescriptor.key, Some(rc), outputs, None, Seq.empty) + val succeededResponse = JobSucceededResponse(jobDescriptor.key, Some(rc), outputs, None, Seq.empty) log.debug("{} Storing data into cache for hash {}.", tag, jobHash) // If cache fails to store data for any reason it should not stop the workflow/task execution but log the issue. cacheActor foreach { _ ! StoreExecutionResult(jobHash, succeededResponse) } @@ -249,7 +249,7 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor val message = Option(e.getMessage) map { ": " + _ } getOrElse "" - FailedNonRetryableResponse(jobDescriptor.key, new Throwable("Failed post processing of outputs" + message, e), Option(rc)) + JobFailedNonRetryableResponse(jobDescriptor.key, new Throwable("Failed post processing of outputs" + message, e), Option(rc)) } } @@ -364,16 +364,16 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor createExecutionFolderAndScript() executeTask() } catch { - case e: Exception => self ! JobExecutionResponse(FailedNonRetryableResponse(jobDescriptor.key, e, None)) + case e: Exception => self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key, e, None)) } } - private def localizeCachedResponse(succeededResponse: SucceededResponse): BackendJobExecutionResponse = { + private def localizeCachedResponse(succeededResponse: JobSucceededResponse): BackendJobExecutionResponse = { Try(localizeCachedOutputs(executionDir, succeededResponse.jobOutputs)) match { case Success(outputs) => executionDir.toString.toFile.createIfNotExists(asDirectory = true, createParents = true) - SucceededResponse(jobDescriptor.key, succeededResponse.returnCode, outputs, None, Seq.empty) - case Failure(exception) => FailedNonRetryableResponse(jobDescriptor.key, exception, None) + JobSucceededResponse(jobDescriptor.key, succeededResponse.returnCode, outputs, None, Seq.empty) + case Failure(exception) => JobFailedNonRetryableResponse(jobDescriptor.key, exception, None) } } } diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActor.scala index 70799bd67..74fb3ade5 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActor.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActor.scala @@ -1,7 +1,7 @@ package cromwell.backend.impl.htcondor.caching import akka.actor.{Actor, ActorLogging} -import cromwell.backend.BackendJobExecutionActor.SucceededResponse +import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse import cromwell.backend.impl.htcondor.caching.CacheActor._ import cromwell.backend.impl.htcondor.caching.exception.{CachedResultAlreadyExistException, CachedResultNotFoundException} import cromwell.backend.impl.htcondor.caching.model.CachedExecutionResult @@ -10,10 +10,10 @@ object CacheActor { trait CacheActorCommand case class ReadExecutionResult(hash: String) extends CacheActorCommand - case class StoreExecutionResult(hash: String, succeededResponse: SucceededResponse) extends CacheActorCommand + case class StoreExecutionResult(hash: String, succeededResponse: JobSucceededResponse) extends CacheActorCommand trait CacheActorResponse - case class ExecutionResultFound(succeededResponse: SucceededResponse) extends CacheActorResponse + case class ExecutionResultFound(succeededResponse: JobSucceededResponse) extends CacheActorResponse case object ExecutionResultNotFound extends CacheActorResponse case class ExecutionResultStored(hash: String) extends CacheActorResponse case object ExecutionResultAlreadyExist extends CacheActorResponse diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala index 2f8254e5c..127141c5b 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala @@ -23,7 +23,7 @@ trait CachedResultLocalization { WdlSingleFile(slPath.toString) } - def localizeCachedOutputs(executionPath: Path, outputs: JobOutputs): JobOutputs = { + def localizeCachedOutputs(executionPath: Path, outputs: CallOutputs): CallOutputs = { outputs map { case (lqn, jobOutput) => jobOutput.wdlValue.wdlType match { case WdlFileType => (lqn -> JobOutput(localizeCachedFile(executionPath, jobOutput.wdlValue))) diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/model/CachedExecutionResult.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/model/CachedExecutionResult.scala index fdff70804..1b023fd41 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/model/CachedExecutionResult.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/model/CachedExecutionResult.scala @@ -1,6 +1,6 @@ package cromwell.backend.impl.htcondor.caching.model -import cromwell.backend.BackendJobExecutionActor.SucceededResponse +import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse -case class CachedExecutionResult(hash: String, succeededResponse: SucceededResponse) +case class CachedExecutionResult(hash: String, succeededResponse: JobSucceededResponse) diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActor.scala index 7f99e1565..cf6cf1151 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActor.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActor.scala @@ -4,7 +4,7 @@ import com.mongodb.DBObject import com.mongodb.casbah.MongoCollection import com.mongodb.casbah.commons.{MongoDBObject, TypeImports} import com.mongodb.util.JSON -import cromwell.backend.BackendJobExecutionActor.SucceededResponse +import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse import cromwell.backend.impl.htcondor.caching.CacheActor import cromwell.backend.impl.htcondor.caching.exception.{CachedResultAlreadyExistException, CachedResultNotFoundException} import cromwell.backend.impl.htcondor.caching.model.CachedExecutionResult @@ -55,7 +55,7 @@ class MongoCacheActor(collection: MongoCollection, private def deserializeSucceededResponse(mongoDbObject: TypeImports.DBObject): CachedExecutionResult = { val cachedResult = JsonParser(mongoDbObject.toString).convertTo[MongoCachedExecutionResult] - val succeededResponse = deserialize(cachedResult.succeededResponse.byteArray, classOf[SucceededResponse]) + val succeededResponse = deserialize(cachedResult.succeededResponse.byteArray, classOf[JobSucceededResponse]) CachedExecutionResult(cachedResult.hash, succeededResponse) } diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala index e564cf002..29d19d117 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala @@ -5,7 +5,7 @@ import cromwell.backend.BackendWorkflowInitializationActor.Initialize import cromwell.backend.{BackendConfigurationDescriptor, BackendSpec, BackendWorkflowDescriptor} import cromwell.core.TestKitSuite import org.scalatest.{Matchers, WordSpecLike} -import wdl4s.Call +import wdl4s.TaskCall import scala.concurrent.duration._ @@ -34,7 +34,7 @@ class HtCondorInitializationActorSpec extends TestKitSuite("HtCondorInitializati |} """.stripMargin - private def getHtCondorBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[Call], conf: BackendConfigurationDescriptor) = { + private def getHtCondorBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall], conf: BackendConfigurationDescriptor) = { system.actorOf(HtCondorInitializationActor.props(workflowDescriptor, calls, conf, emptyActor)) } @@ -43,7 +43,7 @@ class HtCondorInitializationActorSpec extends TestKitSuite("HtCondorInitializati within(Timeout) { EventFilter.warning(message = s"Key/s [proc] is/are not supported by HtCondorBackend. Unsupported attributes will not be part of jobs executions.", occurrences = 1) intercept { val workflowDescriptor = buildWorkflowDescriptor(HelloWorld, runtime = """runtime { proc: 1 }""") - val backend = getHtCondorBackend(workflowDescriptor, workflowDescriptor.workflowNamespace.workflow.calls, + val backend = getHtCondorBackend(workflowDescriptor, workflowDescriptor.workflow.taskCalls, emptyBackendConfig) backend ! Initialize } diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala index 8109908f5..8dcecba21 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala @@ -7,11 +7,11 @@ import akka.actor.{Actor, Props} import akka.testkit.{ImplicitSender, TestActorRef} import better.files._ import com.typesafe.config.ConfigFactory -import cromwell.backend.BackendJobExecutionActor.{FailedNonRetryableResponse, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{JobFailedNonRetryableResponse, JobSucceededResponse} import cromwell.backend.impl.htcondor.caching.CacheActor import cromwell.backend.impl.htcondor.caching.exception.CachedResultNotFoundException import cromwell.backend.impl.htcondor.caching.model.CachedExecutionResult -import cromwell.backend.io.JobPaths +import cromwell.backend.io.JobPathsWithDocker import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendSpec} import cromwell.core._ import cromwell.core.path.{PathWriter, TailedWriter, UntailedWriter} @@ -147,7 +147,7 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc }).underlyingActor whenReady(backend.execute, timeout) { response => - response shouldBe a[SucceededResponse] + response shouldBe a[JobSucceededResponse] verify(htCondorProcess, times(1)).externalProcess(any[Seq[String]], any[ProcessLogger]) verify(htCondorProcess, times(1)).tailedWriter(any[Int], any[Path]) verify(htCondorProcess, times(1)).untailedWriter(any[Path]) @@ -179,7 +179,7 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc }).underlyingActor whenReady(backend.recover, timeout) { response => - response shouldBe a[SucceededResponse] + response shouldBe a[JobSucceededResponse] } cleanUpJob(jobPaths) @@ -207,7 +207,7 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc }).underlyingActor whenReady(backend.execute, timeout) { response => - response shouldBe a[SucceededResponse] + response shouldBe a[JobSucceededResponse] verify(htCondorProcess, times(1)).externalProcess(any[Seq[String]], any[ProcessLogger]) verify(htCondorProcess, times(1)).tailedWriter(any[Int], any[Path]) verify(htCondorProcess, times(1)).untailedWriter(any[Path]) @@ -237,8 +237,8 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(-1)) whenReady(backend.execute, timeout) { response => - response shouldBe a[FailedNonRetryableResponse] - assert(response.asInstanceOf[FailedNonRetryableResponse].throwable.getMessage.contains("Job exited with invalid return code")) + response shouldBe a[JobFailedNonRetryableResponse] + assert(response.asInstanceOf[JobFailedNonRetryableResponse].throwable.getMessage.contains("Job exited with invalid return code")) } cleanUpJob(jobPaths) @@ -271,7 +271,7 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(911)) whenReady(backend.execute, timeout) { response => - response shouldBe a[SucceededResponse] + response shouldBe a[JobSucceededResponse] } cleanUpJob(jobPaths) @@ -310,7 +310,7 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(0)) whenReady(backend.execute) { response => - response shouldBe a[SucceededResponse] + response shouldBe a[JobSucceededResponse] } val bashScript = Source.fromFile(jobPaths.script.toFile).getLines.mkString @@ -346,8 +346,8 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(-1)) whenReady(backend.execute, timeout) { response => - response shouldBe a[FailedNonRetryableResponse] - assert(response.asInstanceOf[FailedNonRetryableResponse].throwable.getMessage.contains("Could not write the file.")) + response shouldBe a[JobFailedNonRetryableResponse] + assert(response.asInstanceOf[JobFailedNonRetryableResponse].throwable.getMessage.contains("Could not write the file.")) } cleanUpJob(jobPaths) @@ -394,7 +394,7 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(0)) whenReady(backend.execute) { response => - response shouldBe a[SucceededResponse] + response shouldBe a[JobSucceededResponse] } val bashScript = Source.fromFile(jobPaths.script.toFile).getLines.mkString @@ -408,7 +408,7 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc cleanUpJob(jobPaths) } - private def cleanUpJob(jobPaths: JobPaths): Unit = { + private def cleanUpJob(jobPaths: JobPathsWithDocker): Unit = { File(jobPaths.workflowRoot).delete(true) () } @@ -425,7 +425,7 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc val backendWorkflowDescriptor = buildWorkflowDescriptor(wdl = source, inputs = inputFiles.getOrElse(Map.empty), runtime = runtimeString) val backendConfigurationDescriptor = BackendConfigurationDescriptor(backendConfig, ConfigFactory.load) val jobDesc = jobDescriptorFromSingleCallWorkflow(backendWorkflowDescriptor, inputFiles.getOrElse(Map.empty), emptyWorkflowOptions, Set.empty) - val jobPaths = new JobPaths(backendWorkflowDescriptor, backendConfig, jobDesc.key) + val jobPaths = new JobPathsWithDocker(jobDesc.key, backendWorkflowDescriptor, backendConfig) val executionDir = File(jobPaths.callExecutionRoot) val stdout = File(executionDir.pathAsString, "stdout") stdout.createIfNotExists(asDirectory = false, createParents = true) @@ -440,7 +440,7 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc TestJobDescriptor(jobDesc, jobPaths, backendConfigurationDescriptor) } - private case class TestJobDescriptor(jobDescriptor: BackendJobDescriptor, jobPaths: JobPaths, backendConfigurationDescriptor: BackendConfigurationDescriptor) + private case class TestJobDescriptor(jobDescriptor: BackendJobDescriptor, jobPaths: JobPathsWithDocker, backendConfigurationDescriptor: BackendConfigurationDescriptor) trait MockWriter extends Writer { var closed = false diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala index 85e9185a0..659d700fa 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala @@ -295,7 +295,7 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { call.lookupFunction(knownInputs, NoFunctions) } - workflowDescriptor.workflowNamespace.workflow.calls.toSeq map { + workflowDescriptor.workflow.taskCalls.toSeq map { call => val ra = call.task.runtimeAttributes.attrs mapValues { _.evaluate(createLookup(call), NoFunctions) } TryUtil.sequenceMap(ra, "Runtime attributes evaluation").get diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalizationSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalizationSpec.scala index c2c4d101f..e0e847e56 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalizationSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalizationSpec.scala @@ -2,7 +2,7 @@ package cromwell.backend.impl.htcondor.caching.localization import java.nio.file.Files -import cromwell.core.{JobOutput, JobOutputs} +import cromwell.core.{JobOutput, CallOutputs} import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} import wdl4s.types.{WdlArrayType, WdlFileType} import wdl4s.values.{WdlArray, WdlSingleFile, WdlString} @@ -33,7 +33,7 @@ class CachedResultLocalizationSpec extends WordSpecLike with Matchers with Befor } "localize cached job outputs which are WDL files using symbolic link" in { - val outputs: JobOutputs = Map("File1" -> JobOutput(WdlSingleFile(defaultCachedFile.toAbsolutePath.toString))) + val outputs: CallOutputs = Map("File1" -> JobOutput(WdlSingleFile(defaultCachedFile.toAbsolutePath.toString))) val newJobOutputs = cachedResults.localizeCachedOutputs(newTmpDir, outputs) newJobOutputs foreach { case (lqn, jobOutput) => assert(jobOutput.wdlValue.valueString == newTmpFile.toString) diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala index 6790b5bd4..d638b82d9 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala @@ -8,7 +8,7 @@ import com.mongodb.util.JSON import com.mongodb.{DBObject, WriteResult} import com.typesafe.config.{Config, ConfigFactory} import cromwell.backend.{MemorySize, BackendJobDescriptorKey} -import cromwell.backend.BackendJobExecutionActor.SucceededResponse +import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse import cromwell.backend.impl.htcondor.HtCondorRuntimeAttributes import cromwell.backend.impl.htcondor.caching.CacheActor._ import cromwell.backend.impl.htcondor.caching.exception.CachedResultNotFoundException @@ -20,7 +20,7 @@ import org.mockito.Mockito import org.mockito.Mockito._ import org.scalatest.mockito.MockitoSugar import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, MustMatchers, WordSpecLike} -import wdl4s.Call +import wdl4s.TaskCall import wdl4s.values.WdlString class MongoCacheActorSpec extends TestKit(ActorSystem("MongoCacheProviderActorSpecSystem")) with WordSpecLike with MustMatchers @@ -36,7 +36,7 @@ class MongoCacheActorSpec extends TestKit(ActorSystem("MongoCacheProviderActorSp val runtimeConfig = HtCondorRuntimeAttributes(ContinueOnReturnCodeSet(Set(0)), Some("tool-name"), Some("/workingDir"), Some("/outputDir"), true, 1, memorySize, diskSize, None) val jobHash = "88dde49db10f1551299fb9937f313c10" val taskStatus = "done" - val succeededResponseMock = SucceededResponse(BackendJobDescriptorKey(Call(Option("taskName"), null, null, null), None, 0), None, Map("test" -> JobOutput(WdlString("Test"))), None, Seq.empty) + val succeededResponseMock = JobSucceededResponse(BackendJobDescriptorKey(TaskCall(Option("taskName"), null, null, null), None, 0), None, Map("test" -> JobOutput(WdlString("Test"))), None, Seq.empty) val serSucceededRespMock = KryoSerializedObject(serialize(succeededResponseMock)) val cachedExecutionResult = MongoCachedExecutionResult(jobHash, serSucceededRespMock) val cachedExecutionResultDbObject = JSON.parse(cachedExecutionResult.toJson.toString).asInstanceOf[DBObject] diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala index 1e86782c8..84da1efbc 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala @@ -95,13 +95,13 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes override lazy val executeOrRecoverBackOff = SimpleExponentialBackoff( initialInterval = 3 seconds, maxInterval = 20 seconds, multiplier = 1.1) - private lazy val workflowDescriptor = jobDescriptor.workflowDescriptor + override lazy val workflowDescriptor = jobDescriptor.workflowDescriptor private lazy val call = jobDescriptor.key.call override lazy val retryable = jobDescriptor.key.attempt <= runtimeAttributes.preemptible private lazy val cmdInput = - JesFileInput(ExecParamName, jesCallPaths.gcsExecPath.toUri.toString, Paths.get(jesCallPaths.gcsExecFilename), workingDisk) + JesFileInput(ExecParamName, jesCallPaths.script.toUri.toString, Paths.get(jesCallPaths.scriptFilename), workingDisk) private lazy val jesCommandLine = s"/bin/bash ${cmdInput.containerPath}" private lazy val rcJesOutput = JesFileOutput(returnCodeFilename, returnCodeGcsPath.toUri.toString, Paths.get(returnCodeFilename), workingDisk) @@ -301,7 +301,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes |echo $$? > $rcPath """.stripMargin.trim - Future(File(jesCallPaths.gcsExecPath).write(fileContent)) void + Future(File(jesCallPaths.script).write(fileContent)) void } private def googleProject(descriptor: BackendWorkflowDescriptor): String = { @@ -439,29 +439,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes /** * Fire and forget start info to the metadata service */ - private def tellStartMetadata(): Unit = { - val runtimeAttributesMetadata: Map[String, Any] = runtimeAttributes.asMap map { - case (key, value) => s"runtimeAttributes:$key" -> value - } - - var fileMetadata: Map[String, Any] = jesCallPaths.metadataPaths - if (monitoringOutput.nonEmpty) { - // TODO: Move this to JesCallPaths - fileMetadata += JesMetadataKeys.MonitoringLog -> monitoringOutput.get.gcs - } - - val otherMetadata: Map[String, Any] = Map( - JesMetadataKeys.GoogleProject -> jesAttributes.project, - JesMetadataKeys.ExecutionBucket -> jesAttributes.executionBucket, - JesMetadataKeys.EndpointUrl -> jesAttributes.endpointUrl, - "preemptible" -> preemptible, - "cache:allowResultReuse" -> true - ) - - val metadataKeyValues = runtimeAttributesMetadata ++ fileMetadata ++ otherMetadata - - tellMetadata(metadataKeyValues) - } + private def tellStartMetadata() = tellMetadata(metadataKeyValues) /** * Fire and forget info to the metadata service @@ -486,7 +464,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes } } - private def postProcess: Try[JobOutputs] = { + private def postProcess: Try[CallOutputs] = { def wdlValueToSuccess(value: WdlValue): Try[WdlValue] = Success(value) OutputEvaluator.evaluateOutputs( @@ -496,7 +474,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes ) } - private def handleSuccess(outputMappings: Try[JobOutputs], returnCode: Int, jobDetritusFiles: Map[String, Path], executionHandle: ExecutionHandle, events: Seq[ExecutionEvent]): ExecutionHandle = { + private def handleSuccess(outputMappings: Try[CallOutputs], returnCode: Int, jobDetritusFiles: Map[String, Path], executionHandle: ExecutionHandle, events: Seq[ExecutionEvent]): ExecutionHandle = { outputMappings match { case Success(outputs) => SuccessfulExecutionHandle(outputs, returnCode, jobDetritusFiles, events) case Failure(ex: CromwellAggregatedException) if ex.throwables collectFirst { case s: SocketTimeoutException => s } isDefined => diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala index 8e9a261a4..37271286c 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala @@ -8,9 +8,9 @@ import cromwell.backend._ import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction import cromwell.backend.impl.jes.callcaching.JesBackendFileHashing import cromwell.backend.validation.RuntimeAttributesKeys +import cromwell.core.CallOutputs import cromwell.core.Dispatcher.BackendDispatcher -import cromwell.core.{ExecutionStore, OutputStore} -import wdl4s.Call +import wdl4s.TaskCall import wdl4s.expression.WdlStandardLibraryFunctions @@ -21,7 +21,7 @@ case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor val jesConfiguration = new JesConfiguration(configurationDescriptor) override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Set[Call], + calls: Set[TaskCall], serviceRegistryActor: ActorRef): Option[Props] = { Option(JesInitializationActor.props(workflowDescriptor, calls, jesConfiguration, serviceRegistryActor).withDispatcher(BackendDispatcher)) } @@ -46,15 +46,15 @@ case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor } override def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor, - calls: Set[Call], - executionStore: ExecutionStore, - outputStore: OutputStore, + calls: Set[TaskCall], + jobExecutionMap: JobExecutionMap, + workflowOutputs: CallOutputs, initializationData: Option[BackendInitializationData]) = { // The `JesInitializationActor` will only return a non-`Empty` `JesBackendInitializationData` from a successful `beforeAll` // invocation. HOWEVER, the finalization actor is created regardless of whether workflow initialization was successful // or not. So the finalization actor must be able to handle an empty `JesBackendInitializationData` option, and there is no // `.get` on the initialization data as there is with the execution or cache hit copying actor methods. - Option(JesFinalizationActor.props(workflowDescriptor, calls, jesConfiguration, executionStore, outputStore, initializationData.toJes).withDispatcher(BackendDispatcher)) + Option(JesFinalizationActor.props(workflowDescriptor, calls, jesConfiguration, jobExecutionMap, workflowOutputs, initializationData.toJes).withDispatcher(BackendDispatcher)) } override def runtimeAttributeDefinitions(initializationDataOption: Option[BackendInitializationData]) = staticRuntimeAttributeDefinitions @@ -63,13 +63,18 @@ case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor jobKey: BackendJobDescriptorKey, initializationData: Option[BackendInitializationData]): WdlStandardLibraryFunctions = { - val jesCallPaths = initializationData.toJes.get.workflowPaths.toJesCallPaths(jobKey) + val jesCallPaths = initializationData.toJes.get.workflowPaths.toJobPaths(jobKey) new JesExpressionFunctions(List(jesCallPaths.gcsPathBuilder), jesCallPaths.callContext) } override def getExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config, initializationData: Option[BackendInitializationData]): Path = { - initializationData.toJes.get.workflowPaths.rootPath + initializationData.toJes.get.workflowPaths.executionRoot + } + + override def getWorkflowExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config, + initializationData: Option[BackendInitializationData]): Path = { + initializationData.toJes.get.workflowPaths.workflowRoot } override def backendSingletonActorProps = Option(JesBackendSingletonActor.props()) @@ -86,8 +91,8 @@ object JesBackendLifecycleActorFactory { } val staticRuntimeAttributeDefinitions = { - import RuntimeAttributesKeys._ import JesRuntimeAttributes._ + import RuntimeAttributesKeys._ Set( RuntimeAttributeDefinition(DockerKey, None, usedInCallCaching = true), diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala index e84e4d3ec..a77dd66c7 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala @@ -15,9 +15,11 @@ case class JesCacheHitCopyingActor(override val jobDescriptor: BackendJobDescrip extends BackendCacheHitCopyingActor with CacheHitDuplicating with JesJobCachingActorHelper with JobLogging { override protected def duplicate(source: Path, destination: Path) = PathCopier.copy(source, destination).get - override protected def destinationCallRootPath = jesCallPaths.callRootPath + override protected def destinationCallRootPath = jesCallPaths.callExecutionRoot override protected def destinationJobDetritusPaths = jesCallPaths.detritusPaths + + override val workflowDescriptor = jobDescriptor.workflowDescriptor } object JesCacheHitCopyingActor { diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCallPaths.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCallPaths.scala deleted file mode 100644 index 97dc3be39..000000000 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCallPaths.scala +++ /dev/null @@ -1,78 +0,0 @@ -package cromwell.backend.impl.jes - -import java.nio.file.Path - -import akka.actor.ActorSystem -import cromwell.backend.io.JobPaths -import cromwell.backend.io.JobPaths._ -import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} -import cromwell.core.CallContext -import cromwell.services.metadata.CallMetadataKeys - -object JesCallPaths { - def apply(jobKey: BackendJobDescriptorKey, workflowDescriptor: BackendWorkflowDescriptor, - jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem): JesCallPaths = { - new JesCallPaths(jobKey, workflowDescriptor, jesConfiguration) - } - - val JesLogPathKey = "jesLog" - val GcsExecPathKey = "gcsExec" -} - -class JesCallPaths(jobKey: BackendJobDescriptorKey, workflowDescriptor: BackendWorkflowDescriptor, - jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem) extends - JesWorkflowPaths(workflowDescriptor, jesConfiguration)(actorSystem) { - - val jesLogBasename = { - val index = jobKey.index.map(s => s"-$s").getOrElse("") - s"${jobKey.scope.unqualifiedName}$index" - } - - val callRootPath: Path = { - val callName = jobKey.call.fullyQualifiedName.split('.').last - val call = s"$CallPrefix-$callName" - val shard = jobKey.index map { s => s"$ShardPrefix-$s" } getOrElse "" - val retry = if (jobKey.attempt > 1) s"$AttemptPrefix-${jobKey.attempt}" else "" - - List(call, shard, retry).foldLeft(workflowRootPath)((path, dir) => path.resolve(dir)) - } - - val returnCodeFilename: String = s"$jesLogBasename-rc.txt" - val stdoutFilename: String = s"$jesLogBasename-stdout.log" - val stderrFilename: String = s"$jesLogBasename-stderr.log" - val jesLogFilename: String = s"$jesLogBasename.log" - val gcsExecFilename: String = "exec.sh" - - lazy val returnCodePath: Path = callRootPath.resolve(returnCodeFilename) - lazy val stdoutPath: Path = callRootPath.resolve(stdoutFilename) - lazy val stderrPath: Path = callRootPath.resolve(stderrFilename) - lazy val jesLogPath: Path = callRootPath.resolve(jesLogFilename) - lazy val gcsExecPath: Path = callRootPath.resolve(gcsExecFilename) - lazy val callContext = CallContext(callRootPath, stdoutFilename, stderrFilename) - - /* - TODO: Move various monitoring files path generation here. - - "/cromwell_root" is a well known path, called in the regular JobPaths callDockerRoot. - This JesCallPaths should know about that root, and be able to create the monitoring file paths. - Instead of the AsyncActor creating the paths, the paths could then be shared with the CachingActor. - - Those monitoring paths could then be returned by metadataFiles and detritusFiles. - */ - - lazy val metadataPaths: Map[String, Path] = Map( - CallMetadataKeys.CallRoot -> callRootPath, - CallMetadataKeys.Stdout -> stdoutPath, - CallMetadataKeys.Stderr -> stderrPath, - CallMetadataKeys.BackendLogsPrefix + ":log" -> jesLogPath - ) - - lazy val detritusPaths: Map[String, Path] = Map( - JobPaths.CallRootPathKey -> callRootPath, - JesCallPaths.GcsExecPathKey -> gcsExecPath, - JesCallPaths.JesLogPathKey -> jesLogPath, - JobPaths.StdoutPathKey -> stdoutPath, - JobPaths.StdErrPathKey -> stderrPath, - JobPaths.ReturnCodePathKey -> returnCodePath - ) -} diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala index 195e34e8b..fc2d141be 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala @@ -6,26 +6,26 @@ import akka.actor.Props import better.files._ import cats.instances.future._ import cats.syntax.functor._ -import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor, BackendWorkflowFinalizationActor} +import cromwell.backend.{BackendWorkflowDescriptor, BackendWorkflowFinalizationActor, JobExecutionMap} +import cromwell.core.CallOutputs import cromwell.core.Dispatcher.IoDispatcher import cromwell.core.path.PathCopier -import cromwell.core.{ExecutionStore, OutputStore} -import wdl4s.Call +import wdl4s.TaskCall import scala.concurrent.Future import scala.language.postfixOps object JesFinalizationActor { - def props(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[Call], jesConfiguration: JesConfiguration, - executionStore: ExecutionStore, outputStore: OutputStore, initializationData: Option[JesBackendInitializationData]) = { - Props(new JesFinalizationActor(workflowDescriptor, calls, jesConfiguration, executionStore, outputStore, initializationData)) + def props(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall], jesConfiguration: JesConfiguration, + jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs, initializationData: Option[JesBackendInitializationData]) = { + Props(new JesFinalizationActor(workflowDescriptor, calls, jesConfiguration, jobExecutionMap, workflowOutputs, initializationData)) } } class JesFinalizationActor (override val workflowDescriptor: BackendWorkflowDescriptor, - override val calls: Set[Call], - jesConfiguration: JesConfiguration, executionStore: ExecutionStore, - outputStore: OutputStore, + override val calls: Set[TaskCall], + jesConfiguration: JesConfiguration, jobExecutionMap: JobExecutionMap, + workflowOutputs: CallOutputs, initializationData: Option[JesBackendInitializationData]) extends BackendWorkflowFinalizationActor { override val configurationDescriptor = jesConfiguration.configurationDescriptor @@ -69,19 +69,19 @@ class JesFinalizationActor (override val workflowDescriptor: BackendWorkflowDesc } private lazy val logPaths: Seq[Path] = { - val allCallPaths = executionStore.store.toSeq collect { - case (backendJobDescriptorKey: BackendJobDescriptorKey, _) => - initializationData map { _.workflowPaths.toJesCallPaths(backendJobDescriptorKey) } + val allCallPaths = jobExecutionMap flatMap { + case (backendJobDescriptor, keys) => + keys map { JesWorkflowPaths(backendJobDescriptor, jesConfiguration)(context.system).toJobPaths(_) } } - allCallPaths.flatten flatMap { callPaths => - Seq(callPaths.stdoutPath, callPaths.stderrPath, callPaths.jesLogPath) + allCallPaths.toSeq flatMap { callPaths => + Seq(callPaths.stdout, callPaths.stderr, callPaths.jesLogPath) } } private def copyLogs(callLogsDirPath: Path, logPaths: Seq[Path]): Unit = { workflowPaths match { - case Some(paths) => logPaths.foreach(PathCopier.copy(paths.rootPath, _, callLogsDirPath)) + case Some(paths) => logPaths.foreach(PathCopier.copy(paths.executionRoot, _, callLogsDirPath)) case None => } } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala index 607c8c8c4..03cae0dff 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala @@ -15,7 +15,7 @@ import cromwell.backend.{BackendInitializationData, BackendWorkflowDescriptor, B import cromwell.core.WorkflowOptions import cromwell.filesystems.gcs.auth.{ClientSecrets, GoogleAuthMode} import spray.json.JsObject -import wdl4s.Call +import wdl4s.TaskCall import wdl4s.types.{WdlBooleanType, WdlFloatType, WdlIntegerType, WdlStringType} import wdl4s.values.WdlValue @@ -28,14 +28,14 @@ object JesInitializationActor { JesRuntimeAttributes.PreemptibleKey, JesRuntimeAttributes.BootDiskSizeKey, JesRuntimeAttributes.DisksKey) def props(workflowDescriptor: BackendWorkflowDescriptor, - calls: Set[Call], + calls: Set[TaskCall], jesConfiguration: JesConfiguration, serviceRegistryActor: ActorRef): Props = Props(new JesInitializationActor(workflowDescriptor, calls, jesConfiguration, serviceRegistryActor: ActorRef)) } class JesInitializationActor(override val workflowDescriptor: BackendWorkflowDescriptor, - override val calls: Set[Call], + override val calls: Set[TaskCall], private[jes] val jesConfiguration: JesConfiguration, override val serviceRegistryActor: ActorRef) extends BackendWorkflowInitializationActor { @@ -79,14 +79,14 @@ class JesInitializationActor(override val workflowDescriptor: BackendWorkflowDes genomics <- buildGenomics workflowPaths = new JesWorkflowPaths(workflowDescriptor, jesConfiguration)(context.system) _ <- if (jesConfiguration.needAuthFileUpload) writeAuthenticationFile(workflowPaths) else Future.successful(()) - _ = publishWorkflowRoot(workflowPaths.workflowRootPath.toString) + _ = publishWorkflowRoot(workflowPaths.workflowRoot.toString) } yield Option(JesBackendInitializationData(workflowPaths, genomics)) } private def writeAuthenticationFile(workflowPath: JesWorkflowPaths): Future[Unit] = { generateAuthJson(jesConfiguration.dockerCredentials, refreshTokenAuth) map { content => val path = workflowPath.gcsAuthFilePath - workflowLogger.info(s"Creating authentication file for workflow ${workflowDescriptor.id} at \n ${path.toString}") + workflowLogger.info(s"Creating authentication file for workflow ${workflowDescriptor.id} at \n ${path.toUri}") Future(path.writeAsJson(content)).void.recoverWith { case failure => Future.failed(new IOException("Failed to upload authentication file", failure)) } void diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala index e30aa4e6f..fe39df840 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala @@ -4,12 +4,11 @@ import java.nio.file.Path import akka.actor.{Actor, ActorRef} import better.files._ +import cromwell.backend.BackendWorkflowDescriptor import cromwell.backend.callcaching.JobCachingActorHelper -import cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.WorkflowOptionKeys import cromwell.backend.impl.jes.io.{JesAttachedDisk, JesWorkingDisk} import cromwell.core.logging.JobLogging -import scala.language.postfixOps import scala.util.Try trait JesJobCachingActorHelper extends JobCachingActorHelper { @@ -26,23 +25,33 @@ trait JesJobCachingActorHelper extends JobCachingActorHelper { def initializationData: JesBackendInitializationData def serviceRegistryActor: ActorRef + + def workflowDescriptor: BackendWorkflowDescriptor def getPath(str: String): Try[Path] = jesCallPaths.getPath(str) override lazy val configurationDescriptor = jesConfiguration.configurationDescriptor - lazy val jesCallPaths = initializationData.workflowPaths.toJesCallPaths(jobDescriptor.key) + lazy val jesCallPaths = { + val workflowPaths = if (workflowDescriptor.breadCrumbs.isEmpty) { + initializationData.workflowPaths + } else { + new JesWorkflowPaths(workflowDescriptor, jesConfiguration)(context.system) + } + + workflowPaths.toJobPaths(jobDescriptor.key) + } lazy val runtimeAttributes = JesRuntimeAttributes(jobDescriptor.runtimeAttributes, jobLogger) lazy val retryable = jobDescriptor.key.attempt <= runtimeAttributes.preemptible lazy val workingDisk: JesAttachedDisk = runtimeAttributes.disks.find(_.name == JesWorkingDisk.Name).get - lazy val callRootPath: Path = jesCallPaths.callRootPath + lazy val callRootPath: Path = jesCallPaths.callExecutionRoot lazy val returnCodeFilename = jesCallPaths.returnCodeFilename - lazy val returnCodeGcsPath = jesCallPaths.returnCodePath - lazy val jesStdoutFile = jesCallPaths.stdoutPath - lazy val jesStderrFile = jesCallPaths.stderrPath + lazy val returnCodeGcsPath = jesCallPaths.returnCode + lazy val jesStdoutFile = jesCallPaths.stdout + lazy val jesStderrFile = jesCallPaths.stderr lazy val jesLogFilename = jesCallPaths.jesLogFilename lazy val defaultMonitoringOutputPath = callRootPath.resolve(JesMonitoringLogFile) @@ -50,28 +59,22 @@ trait JesJobCachingActorHelper extends JobCachingActorHelper { lazy val preemptible: Boolean = jobDescriptor.key.attempt <= maxPreemption lazy val jesAttributes = jesConfiguration.jesAttributes - // TODO: Move monitoring paths to JesCallPaths lazy val monitoringScript: Option[JesInput] = { - jobDescriptor.workflowDescriptor.workflowOptions.get(WorkflowOptionKeys.MonitoringScript) map { path => - JesFileInput(s"$MonitoringParamName-in", getPath(path).get.toUri.toString, + jesCallPaths.monitoringPath map { path => + JesFileInput(s"$MonitoringParamName-in", path.toUri.toString, JesWorkingDisk.MountPoint.resolve(JesMonitoringScript), workingDisk) - } toOption + } } lazy val monitoringOutput = monitoringScript map { _ => JesFileOutput(s"$MonitoringParamName-out", defaultMonitoringOutputPath.toString, File(JesMonitoringLogFile).path, workingDisk) } + // Implements CacheHitDuplicating.metadataKeyValues lazy val metadataKeyValues: Map[String, Any] = { val runtimeAttributesMetadata: Map[String, Any] = runtimeAttributes.asMap map { case (key, value) => s"runtimeAttributes:$key" -> value } - - var fileMetadata: Map[String, Any] = jesCallPaths.metadataPaths - if (monitoringOutput.nonEmpty) { - // TODO: Move this to JesCallPaths - fileMetadata += JesMetadataKeys.MonitoringLog -> monitoringOutput.get.gcs - } - + val otherMetadata: Map[String, Any] = Map( JesMetadataKeys.GoogleProject -> jesAttributes.project, JesMetadataKeys.ExecutionBucket -> jesAttributes.executionBucket, @@ -80,6 +83,6 @@ trait JesJobCachingActorHelper extends JobCachingActorHelper { "cache:allowResultReuse" -> true ) - runtimeAttributesMetadata ++ fileMetadata ++ otherMetadata + runtimeAttributesMetadata ++ jesCallPaths.metadataPaths ++ otherMetadata } } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala new file mode 100644 index 000000000..fdbdb1dc7 --- /dev/null +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala @@ -0,0 +1,60 @@ +package cromwell.backend.impl.jes + +import java.nio.file.Path + +import akka.actor.ActorSystem +import cromwell.backend.io.JobPaths +import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} +import cromwell.core.CallContext +import cromwell.services.metadata.CallMetadataKeys + +object JesJobPaths { + def apply(jobKey: BackendJobDescriptorKey, workflowDescriptor: BackendWorkflowDescriptor, + jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem): JesJobPaths = { + new JesJobPaths(jobKey, workflowDescriptor, jesConfiguration) + } + + val JesLogPathKey = "jesLog" + val GcsExecPathKey = "gcsExec" +} + +class JesJobPaths(val jobKey: BackendJobDescriptorKey, workflowDescriptor: BackendWorkflowDescriptor, + jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem) extends + JesWorkflowPaths(workflowDescriptor, jesConfiguration)(actorSystem) with JobPaths { + + val jesLogBasename = { + val index = jobKey.index.map(s => s"-$s").getOrElse("") + s"${jobKey.scope.unqualifiedName}$index" + } + + override val returnCodeFilename: String = s"$jesLogBasename-rc.txt" + override val stdoutFilename: String = s"$jesLogBasename-stdout.log" + override val stderrFilename: String = s"$jesLogBasename-stderr.log" + override val scriptFilename: String = "exec.sh" + + val jesLogFilename: String = s"$jesLogBasename.log" + lazy val jesLogPath: Path = callExecutionRoot.resolve(jesLogFilename) + + lazy val callContext = CallContext(callExecutionRoot, stdoutFilename, stderrFilename) + + /* + TODO: Move various monitoring files path generation here. + + "/cromwell_root" is a well known path, called in the regular JobPaths callDockerRoot. + This JesCallPaths should know about that root, and be able to create the monitoring file paths. + Instead of the AsyncActor creating the paths, the paths could then be shared with the CachingActor. + + Those monitoring paths could then be returned by metadataFiles and detritusFiles. + */ + + override lazy val customMetadataPaths = Map( + CallMetadataKeys.BackendLogsPrefix + ":log" -> jesLogPath + ) ++ ( + monitoringPath map { p => Map(JesMetadataKeys.MonitoringLog -> p) } getOrElse Map.empty + ) + + override lazy val customDetritusPaths: Map[String, Path] = Map( + JesJobPaths.GcsExecPathKey -> script, + JesJobPaths.JesLogPathKey -> jesLogPath + ) +} diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala index 6c3d2b27c..a7ac5e50a 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala @@ -3,13 +3,15 @@ package cromwell.backend.impl.jes import java.nio.file.Path import akka.actor.ActorSystem +import com.typesafe.config.Config +import cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.WorkflowOptionKeys +import cromwell.backend.io.WorkflowPaths import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} import cromwell.core.WorkflowOptions -import cromwell.core.WorkflowOptions.FinalCallLogsDir -import cromwell.filesystems.gcs.{RetryableGcsPathBuilder, GcsPathBuilderFactory} +import cromwell.core.path.PathBuilder +import cromwell.filesystems.gcs.{GcsPathBuilderFactory, RetryableGcsPathBuilder} import scala.language.postfixOps -import scala.util.Try object JesWorkflowPaths { private val GcsRootOptionKey = "jes_gcs_root" @@ -21,38 +23,40 @@ object JesWorkflowPaths { } } -class JesWorkflowPaths(workflowDescriptor: BackendWorkflowDescriptor, - jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem) { +class JesWorkflowPaths(val workflowDescriptor: BackendWorkflowDescriptor, + jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem) extends WorkflowPaths { - private val rootString = workflowDescriptor.workflowOptions.getOrElse(JesWorkflowPaths.GcsRootOptionKey, jesConfiguration.root) + override lazy val executionRootString = workflowDescriptor.workflowOptions.getOrElse(JesWorkflowPaths.GcsRootOptionKey, jesConfiguration.root) private val workflowOptions: WorkflowOptions = workflowDescriptor.workflowOptions - val gcsPathBuilder: RetryableGcsPathBuilder = jesConfiguration.gcsPathBuilderFactory.withOptions(workflowOptions) - def getPath(gcsUrl: String): Try[Path] = gcsPathBuilder.build(gcsUrl) def getHash(gcsUrl: Path) = gcsPathBuilder.getHash(gcsUrl) - val rootPath: Path = getPath(rootString) recover { - case ex => throw new Exception(s"Failed to : $rootString", ex) - } get - - val workflowRootPath: Path = rootPath.resolve(workflowDescriptor.workflowNamespace.workflow.unqualifiedName).resolve(s"${workflowDescriptor.id.toString}/") - - val finalCallLogsPath = workflowDescriptor.getWorkflowOption(FinalCallLogsDir) map getPath map { _.get } - val gcsAuthFilePath: Path = { /* * This is an "exception". The filesystem used here is built from genomicsAuth * unlike everywhere else where the filesystem used is built from gcsFileSystemAuth */ val genomicsCredentials = jesConfiguration.jesAuths.genomics - val bucket = workflowDescriptor.workflowOptions.get(JesWorkflowPaths.AuthFilePathOptionKey) getOrElse workflowRootPath.toUri.toString + + // The default auth file bucket is always at the root of the root workflow + val defaultBucket = executionRoot.resolve(workflowDescriptor.rootWorkflow.unqualifiedName).resolve(workflowDescriptor.rootWorkflowId.toString) + + val bucket = workflowDescriptor.workflowOptions.get(JesWorkflowPaths.AuthFilePathOptionKey) getOrElse defaultBucket.toUri.toString val authBucket = GcsPathBuilderFactory(genomicsCredentials).withOptions(workflowOptions).build(bucket) recover { case ex => throw new Exception(s"Invalid gcs auth_bucket path $bucket", ex) } get - authBucket.resolve(s"${workflowDescriptor.id}_auth.json") + authBucket.resolve(s"${workflowDescriptor.rootWorkflowId}_auth.json") + } + + + val monitoringPath = workflowOptions.get(WorkflowOptionKeys.MonitoringScript).toOption map { path => + // Fail here if the path exists but can't be built + getPath(path).get } - def toJesCallPaths(jobKey: BackendJobDescriptorKey) = JesCallPaths(jobKey, workflowDescriptor, jesConfiguration) + override def toJobPaths(jobKey: BackendJobDescriptorKey) = JesJobPaths(jobKey, workflowDescriptor, jesConfiguration) + override def config: Config = jesConfiguration.configurationDescriptor.backendConfig + override def pathBuilders: List[PathBuilder] = List(gcsPathBuilder) } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala index 2bcad42d3..05078c7f2 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala @@ -48,7 +48,7 @@ object Run { .setProjectId(projectId) .setDocker(pipelineInfo.docker) .setResources(pipelineInfo.resources) - .setName(workflow.workflowNamespace.workflow.unqualifiedName) + .setName(workflow.workflow.unqualifiedName) .setInputParameters(jesParameters.collect({ case i: JesInput => i.toGooglePipelineParameter }).toVector.asJava) .setOutputParameters(jesParameters.collect({ case i: JesFileOutput => i.toGooglePipelineParameter }).toVector.asJava) diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala index c1608c517..a92b6289a 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala @@ -4,18 +4,17 @@ import java.nio.file.Paths import java.util.UUID import akka.actor.{ActorRef, Props} -import akka.event.LoggingAdapter import akka.testkit.{ImplicitSender, TestActorRef, TestDuration, TestProbe} import com.google.cloud.storage.contrib.nio.CloudStoragePath import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse +import cromwell.backend._ import cromwell.backend.async.AsyncBackendJobExecutionActor.{Execute, ExecutionMode} import cromwell.backend.async.{AbortedExecutionHandle, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle} import cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.JesPendingExecutionHandle import cromwell.backend.impl.jes.RunStatus.Failed import cromwell.backend.impl.jes.io.{DiskType, JesWorkingDisk} import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll -import cromwell.backend._ -import cromwell.core.logging.LoggerWrapper +import cromwell.core.logging.JobLogger import cromwell.core.{WorkflowId, WorkflowOptions, _} import cromwell.filesystems.gcs.GcsPathBuilderFactory import cromwell.filesystems.gcs.auth.GoogleAuthMode.NoAuthMode @@ -27,12 +26,11 @@ import org.specs2.mock.Mockito import spray.json.{JsObject, JsValue} import wdl4s.types.{WdlArrayType, WdlFileType, WdlMapType, WdlStringType} import wdl4s.values.{WdlArray, WdlFile, WdlMap, WdlString, WdlValue} -import wdl4s.{Call, LocallyQualifiedName, WdlNamespaceWithWorkflow} +import wdl4s.{LocallyQualifiedName, TaskCall, WdlNamespaceWithWorkflow} import scala.concurrent.duration._ import scala.concurrent.{Await, ExecutionContext, Future, Promise} import scala.util.{Success, Try} -import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackendJobExecutionActorSpec") with FlatSpecLike with Matchers with ImplicitSender with Mockito with BackendSpec { @@ -86,12 +84,9 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend jesSingletonActor: ActorRef = emptyActor) extends JesAsyncBackendJobExecutionActor(jobDescriptor, promise, jesConfiguration, buildInitializationData(jobDescriptor, jesConfiguration), emptyActor, jesSingletonActor) { - override lazy val jobLogger = new LoggerWrapper { - override def akkaLogger: Option[LoggingAdapter] = Option(log) - + override lazy val jobLogger = new JobLogger("TestLogger", workflowId, jobTag, akkaLogger = Option(log)) { override def tag: String = s"$name [UUID(${workflowId.shortString})$jobTag]" - - override def slf4jLoggers: Set[Logger] = Set.empty + override val slf4jLoggers: Set[Logger] = Set.empty } override lazy val callEngineFunctions = functions @@ -111,12 +106,12 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend private def buildPreemptibleJobDescriptor(attempt: Int, preemptible: Int): BackendJobDescriptor = { val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(YoSup.replace("[PREEMPTIBLE]", s"preemptible: $preemptible")), + WdlNamespaceWithWorkflow.load(YoSup.replace("[PREEMPTIBLE]", s"preemptible: $preemptible")).workflow, Inputs, NoOptions ) - val job = workflowDescriptor.workflowNamespace.workflow.calls.head + val job = workflowDescriptor.workflow.taskCalls.head val key = BackendJobDescriptorKey(job, None, attempt) val runtimeAttributes = makeRuntimeAttributes(job) BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(Inputs)) @@ -188,8 +183,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend expectations foreach { case (attempt, preemptible, errorCode, innerErrorCode, shouldRetry) => it should s"handle call failures appropriately with respect to preemption (attempt=$attempt, preemptible=$preemptible, errorCode=$errorCode, innerErrorCode=$innerErrorCode)" in { runAndFail(attempt, preemptible, errorCode, innerErrorCode).getClass.getSimpleName match { - case "FailedNonRetryableResponse" => false shouldBe shouldRetry - case "FailedRetryableResponse" => true shouldBe shouldRetry + case "JobFailedNonRetryableResponse" => false shouldBe shouldRetry + case "JobFailedRetryableResponse" => true shouldBe shouldRetry case huh => fail(s"Unexpected response class name: '$huh'") } } @@ -285,12 +280,12 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(YoSup.replace("[PREEMPTIBLE]", "")), + WdlNamespaceWithWorkflow.load(YoSup.replace("[PREEMPTIBLE]", "")).workflow, inputs, NoOptions ) - val call = workflowDescriptor.workflowNamespace.workflow.calls.head + val call = workflowDescriptor.workflow.taskCalls.head val key = BackendJobDescriptorKey(call, None, 1) val runtimeAttributes = makeRuntimeAttributes(call) val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs)) @@ -339,12 +334,12 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource), + WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource).workflow, inputs, NoOptions ) - val job = workflowDescriptor.workflowNamespace.workflow.calls.head + val job = workflowDescriptor.workflow.taskCalls.head val runtimeAttributes = makeRuntimeAttributes(job) val key = BackendJobDescriptorKey(job, None, 1) val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs)) @@ -378,12 +373,12 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend TestActorRef[TestableJesJobExecutionActor] = { val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(sampleWdl.asWorkflowSources(DockerAndDiskRuntime).wdlSource), + WdlNamespaceWithWorkflow.load(sampleWdl.asWorkflowSources(DockerAndDiskRuntime).wdlSource).workflow, inputs, NoOptions ) - val call = workflowDescriptor.workflowNamespace.workflow.findCallByName(callName).get + val call = workflowDescriptor.workflow.findCallByName(callName).get.asInstanceOf[TaskCall] val key = BackendJobDescriptorKey(call, None, 1) val runtimeAttributes = makeRuntimeAttributes(call) val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs)) @@ -439,12 +434,12 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource), + WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource).workflow, inputs, NoOptions ) - val job = workflowDescriptor.workflowNamespace.workflow.calls.head + val job = workflowDescriptor.workflow.taskCalls.head val runtimeAttributes = makeRuntimeAttributes(job) val key = BackendJobDescriptorKey(job, None, 1) val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs)) @@ -467,12 +462,12 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource), + WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource).workflow, inputs, NoOptions ) - val job = workflowDescriptor.workflowNamespace.workflow.calls.head + val job = workflowDescriptor.workflow.taskCalls.head val runtimeAttributes = makeRuntimeAttributes(job) val key = BackendJobDescriptorKey(job, None, 1) val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs)) @@ -511,12 +506,12 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource), + WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource).workflow, Map.empty, NoOptions ) - val call = workflowDescriptor.workflowNamespace.workflow.calls.head + val call = workflowDescriptor.workflow.taskCalls.head val key = BackendJobDescriptorKey(call, None, 1) val runtimeAttributes = makeRuntimeAttributes(call) val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty) @@ -539,12 +534,12 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend it should "create a JesFileInput for the monitoring script, when specified" in { val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource), + WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource).workflow, Map.empty, WorkflowOptions.fromJsonString("""{"monitoring_script": "gs://path/to/script"}""").get ) - val job = workflowDescriptor.workflowNamespace.workflow.calls.head + val job = workflowDescriptor.workflow.taskCalls.head val runtimeAttributes = makeRuntimeAttributes(job) val key = BackendJobDescriptorKey(job, None, 1) val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty) @@ -560,12 +555,12 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend it should "not create a JesFileInput for the monitoring script, when not specified" in { val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource), + WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource).workflow, Map.empty, NoOptions ) - val job = workflowDescriptor.workflowNamespace.workflow.calls.head + val job = workflowDescriptor.workflow.taskCalls.head val key = BackendJobDescriptorKey(job, None, 1) val runtimeAttributes = makeRuntimeAttributes(job) val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty) @@ -581,12 +576,12 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId(UUID.fromString("e6236763-c518-41d0-9688-432549a8bf7c")), WdlNamespaceWithWorkflow.load( - SampleWdl.HelloWorld.asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).wdlSource), + SampleWdl.HelloWorld.asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).wdlSource).workflow, Map.empty, WorkflowOptions.fromJsonString(""" {"jes_gcs_root": "gs://path/to/gcs_root"} """).get ) - val call = workflowDescriptor.workflowNamespace.workflow.findCallByName("hello").get + val call = workflowDescriptor.workflow.findCallByName("hello").get.asInstanceOf[TaskCall] val key = BackendJobDescriptorKey(call, None, 1) val runtimeAttributes = makeRuntimeAttributes(call) val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty) @@ -597,11 +592,11 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val jesBackend = testActorRef.underlyingActor - jesBackend.jesCallPaths.stdoutPath should be(a[CloudStoragePath]) - jesBackend.jesCallPaths.stdoutPath.toUri.toString shouldBe + jesBackend.jesCallPaths.stdout should be(a[CloudStoragePath]) + jesBackend.jesCallPaths.stdout.toUri.toString shouldBe "gs://path/to/gcs_root/wf_hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello-stdout.log" - jesBackend.jesCallPaths.stderrPath should be(a[CloudStoragePath]) - jesBackend.jesCallPaths.stderrPath.toUri.toString shouldBe + jesBackend.jesCallPaths.stderr should be(a[CloudStoragePath]) + jesBackend.jesCallPaths.stderr.toUri.toString shouldBe "gs://path/to/gcs_root/wf_hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello-stderr.log" jesBackend.jesCallPaths.jesLogPath should be(a[CloudStoragePath]) jesBackend.jesCallPaths.jesLogPath.toUri.toString shouldBe @@ -612,12 +607,12 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val workflowDescriptor = BackendWorkflowDescriptor( WorkflowId(UUID.fromString("e6236763-c518-41d0-9688-432549a8bf7d")), WdlNamespaceWithWorkflow.load( - new SampleWdl.ScatterWdl().asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).wdlSource), + new SampleWdl.ScatterWdl().asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).wdlSource).workflow, Map.empty, WorkflowOptions.fromJsonString(""" {"jes_gcs_root": "gs://path/to/gcs_root"} """).get ) - val call = workflowDescriptor.workflowNamespace.workflow.findCallByName("B").get + val call = workflowDescriptor.workflow.findCallByName("B").get.asInstanceOf[TaskCall] val key = BackendJobDescriptorKey(call, Option(2), 1) val runtimeAttributes = makeRuntimeAttributes(call) val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty) @@ -628,11 +623,11 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val jesBackend = testActorRef.underlyingActor - jesBackend.jesCallPaths.stdoutPath should be(a[CloudStoragePath]) - jesBackend.jesCallPaths.stdoutPath.toUri.toString shouldBe + jesBackend.jesCallPaths.stdout should be(a[CloudStoragePath]) + jesBackend.jesCallPaths.stdout.toUri.toString shouldBe "gs://path/to/gcs_root/w/e6236763-c518-41d0-9688-432549a8bf7d/call-B/shard-2/B-2-stdout.log" - jesBackend.jesCallPaths.stderrPath should be(a[CloudStoragePath]) - jesBackend.jesCallPaths.stderrPath.toUri.toString shouldBe + jesBackend.jesCallPaths.stderr should be(a[CloudStoragePath]) + jesBackend.jesCallPaths.stderr.toUri.toString shouldBe "gs://path/to/gcs_root/w/e6236763-c518-41d0-9688-432549a8bf7d/call-B/shard-2/B-2-stderr.log" jesBackend.jesCallPaths.jesLogPath should be(a[CloudStoragePath]) jesBackend.jesCallPaths.jesLogPath.toUri.toString shouldBe @@ -662,7 +657,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend descriptorWithMax2AndKey2.preemptible shouldBe true } - private def makeRuntimeAttributes(job: Call) = { + private def makeRuntimeAttributes(job: TaskCall) = { val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(job.task.runtimeAttributes, TestableJesExpressionFunctions, Map.empty) RuntimeAttributeDefinition.addDefaultsToAttributes(JesBackendLifecycleActorFactory.staticRuntimeAttributeDefinitions, NoOptions)(evaluatedAttributes.get) // Fine to throw the exception if this "get" fails. This is a test after all! } diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala index 60b3f2e5e..58841f756 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala @@ -18,7 +18,7 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with val jobDescriptorKey = firstJobDescriptorKey(workflowDescriptor) val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) - val callPaths = JesCallPaths(jobDescriptorKey, workflowDescriptor, + val callPaths = JesJobPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration) callPaths.returnCodeFilename should be("hello-rc.txt") callPaths.stderrFilename should be("hello-stderr.log") @@ -31,12 +31,12 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with val jobDescriptorKey = firstJobDescriptorKey(workflowDescriptor) val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) - val callPaths = JesCallPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration) - callPaths.returnCodePath.toUri.toString should + val callPaths = JesJobPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration) + callPaths.returnCode.toUri.toString should be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello-rc.txt") - callPaths.stdoutPath.toUri.toString should + callPaths.stdout.toUri.toString should be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello-stdout.log") - callPaths.stderrPath.toUri.toString should + callPaths.stderr.toUri.toString should be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello-stderr.log") callPaths.jesLogPath.toUri.toString should be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello.log") @@ -47,7 +47,7 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with val jobDescriptorKey = firstJobDescriptorKey(workflowDescriptor) val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) - val callPaths = JesCallPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration) + val callPaths = JesJobPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration) callPaths.callContext.root.toUri.toString should be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello") callPaths.callContext.stdout should be("hello-stdout.log") diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala index 45a312b65..73b8cb6de 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala @@ -16,7 +16,7 @@ import cromwell.util.{EncryptionSpec, SampleWdl} import org.scalatest.{FlatSpecLike, Matchers} import org.specs2.mock.Mockito import spray.json._ -import wdl4s.Call +import wdl4s.TaskCall import scala.concurrent.duration._ @@ -138,7 +138,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe val refreshTokenConfig = ConfigFactory.parseString(refreshTokenConfigTemplate) - private def getJesBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[Call], conf: BackendConfigurationDescriptor) = { + private def getJesBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall], conf: BackendConfigurationDescriptor) = { system.actorOf(JesInitializationActor.props(workflowDescriptor, calls, new JesConfiguration(conf), emptyActor)) } @@ -148,7 +148,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe within(Timeout) { val workflowDescriptor = buildWorkflowDescriptor(HelloWorld, runtime = """runtime { docker: "ubuntu/latest" test: true }""") - val backend = getJesBackend(workflowDescriptor, workflowDescriptor.workflowNamespace.workflow.calls, + val backend = getJesBackend(workflowDescriptor, workflowDescriptor.workflow.taskCalls, defaultBackendConfig) val eventPattern = "Key/s [test] is/are not supported by JesBackend. Unsupported attributes will not be part of jobs executions." @@ -165,7 +165,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe it should "return InitializationFailed when docker runtime attribute key is not present" in { within(Timeout) { val workflowDescriptor = buildWorkflowDescriptor(HelloWorld, runtime = """runtime { }""") - val backend = getJesBackend(workflowDescriptor, workflowDescriptor.workflowNamespace.workflow.calls, + val backend = getJesBackend(workflowDescriptor, workflowDescriptor.workflow.taskCalls, defaultBackendConfig) backend ! Initialize expectMsgPF() { @@ -184,7 +184,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe private def buildJesInitializationTestingBits(backendConfig: Config = dockerBackendConfig): TestingBits = { val workflowOptions = WorkflowOptions.fromMap(Map("refresh_token" -> "mytoken")).get val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.wdlSource(), options = workflowOptions) - val calls = workflowDescriptor.workflowNamespace.workflow.calls + val calls = workflowDescriptor.workflow.taskCalls val backendConfigurationDescriptor = BackendConfigurationDescriptor(backendConfig, globalConfig) val jesConfiguration = new JesConfiguration(backendConfigurationDescriptor) diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala index 20e1c5d85..2c0853718 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala @@ -9,7 +9,7 @@ import org.specs2.mock.Mockito import scala.concurrent.duration._ import akka.testkit._ -import cromwell.backend.BackendJobExecutionActor.{ExecuteJobCommand, FailedNonRetryableResponse} +import cromwell.backend.BackendJobExecutionActor.{ExecuteJobCommand, JobFailedNonRetryableResponse} import cromwell.backend.impl.jes.ControllableFailingJabjea.JabjeaExplode import scala.concurrent.{ExecutionContext, Promise} @@ -43,7 +43,7 @@ class JesJobExecutionActorSpec extends TestKitSuite("JesJobExecutionActorSpec") testJJEA.tell(msg = ExecuteJobCommand, sender = parent.ref) parent.expectMsgPF(max = TimeoutDuration) { - case FailedNonRetryableResponse(jobKey, e, errorCode) => + case JobFailedNonRetryableResponse(jobKey, e, errorCode) => e.getMessage should be("JesAsyncBackendJobExecutionActor failed and didn't catch its exception.") } } @@ -75,7 +75,7 @@ class JesJobExecutionActorSpec extends TestKitSuite("JesJobExecutionActorSpec") jabjeaConstructionPromise.future foreach { _ ! JabjeaExplode } parent.expectMsgPF(max = TimeoutDuration) { - case FailedNonRetryableResponse(jobKey, e, errorCode) => + case JobFailedNonRetryableResponse(jobKey, e, errorCode) => e.getMessage should be("JesAsyncBackendJobExecutionActor failed and didn't catch its exception.") } } diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala index b2fe12336..48dd3d74c 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala @@ -17,8 +17,8 @@ class JesWorkflowPathsSpec extends TestKitSuite with FlatSpecLike with Matchers val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) val workflowPaths = JesWorkflowPaths(workflowDescriptor, jesConfiguration)(system) - workflowPaths.rootPath.toUri.toString should be("gs://my-cromwell-workflows-bucket/") - workflowPaths.workflowRootPath.toUri.toString should + workflowPaths.executionRoot.toUri.toString should be("gs://my-cromwell-workflows-bucket/") + workflowPaths.workflowRoot.toUri.toString should be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/") workflowPaths.gcsAuthFilePath.toUri.toString should be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/${workflowDescriptor.id}_auth.json") diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala index 5b8fc0a8c..ef5ef5e89 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala @@ -14,7 +14,7 @@ import cromwell.backend.sfs.SharedFileSystem._ import cromwell.backend.validation._ import cromwell.backend.wdl.{OutputEvaluator, Command} import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor} -import cromwell.core.JobOutputs +import cromwell.core.CallOutputs import cromwell.core.logging.JobLogging import cromwell.core.path.DefaultPathBuilder import cromwell.core.retry.SimpleExponentialBackoff @@ -386,7 +386,7 @@ mv $rcTmpPath $rcPath } } - private def processOutputs(): Try[JobOutputs] = { + private def processOutputs(): Try[CallOutputs] = { OutputEvaluator.evaluateOutputs(jobDescriptor, callEngineFunction, sharedFileSystem.outputMapper(jobPaths)) } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala index ad03963b6..1ac3093dc 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala @@ -6,11 +6,11 @@ import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, BackendJobDescriptorKey, BackendLifecycleActorFactory, BackendWorkflowDescriptor} import cromwell.core.Dispatcher import cromwell.core.Dispatcher._ -import cromwell.core.path.{PathBuilderFactory, DefaultPathBuilderFactory} +import cromwell.core.path.{DefaultPathBuilderFactory, PathBuilderFactory} import cromwell.filesystems.gcs.{GcsPathBuilderFactory, GoogleConfiguration} import lenthall.exception.MessageAggregation import net.ceedubs.ficus.Ficus._ -import wdl4s.Call +import wdl4s.TaskCall import wdl4s.expression.WdlStandardLibraryFunctions import scala.concurrent.Promise @@ -63,7 +63,7 @@ trait SharedFileSystemBackendLifecycleActorFactory extends BackendLifecycleActor */ def asyncJobExecutionActorClass: Class[_ <: SharedFileSystemAsyncJobExecutionActor] - override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[Call], + override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall], serviceRegistryActor: ActorRef) = { val params = SharedFileSystemInitializationActorParams(serviceRegistryActor, workflowDescriptor, configurationDescriptor, calls, pathBuilderFactories) diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala index 7464c2e3f..7dc5172ba 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala @@ -2,7 +2,7 @@ package cromwell.backend.sfs import java.nio.file.Path -import cromwell.backend.io.{JobPaths, WorkflowPathsBackendInitializationData} +import cromwell.backend.io.{JobPaths, JobPathsWithDocker, WorkflowPathsBackendInitializationData} import cromwell.backend.wdl._ import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptorKey, BackendWorkflowDescriptor} import cromwell.core.CallContext @@ -22,7 +22,7 @@ object SharedFileSystemExpressionFunctions { jobKey: BackendJobDescriptorKey, configurationDescriptor: BackendConfigurationDescriptor, pathBuilders: List[PathBuilder]): SharedFileSystemExpressionFunctions = { - val jobPaths = new JobPaths(workflowDescriptor, configurationDescriptor.backendConfig, jobKey) + val jobPaths = new JobPathsWithDocker(jobKey, workflowDescriptor, configurationDescriptor.backendConfig) val callContext = CallContext( jobPaths.callExecutionRoot, jobPaths.stdout.toString, @@ -44,7 +44,7 @@ object SharedFileSystemExpressionFunctions { configurationDescriptor: BackendConfigurationDescriptor, jobKey: BackendJobDescriptorKey, initializationData: Option[BackendInitializationData]) = { - val jobPaths = new JobPaths(workflowDescriptor, configurationDescriptor.backendConfig, jobKey) + val jobPaths = new JobPathsWithDocker(jobKey, workflowDescriptor, configurationDescriptor.backendConfig) val callContext = CallContext( jobPaths.callExecutionRoot, jobPaths.stdout.toString, diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala index b73797d88..52c8fbeaf 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala @@ -6,9 +6,9 @@ import cromwell.backend.io.{WorkflowPaths, WorkflowPathsBackendInitializationDat import cromwell.backend.validation.RuntimeAttributesDefault import cromwell.backend.wfs.WorkflowPathBuilder import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor} -import cromwell.core.path.PathBuilderFactory import cromwell.core.WorkflowOptions -import wdl4s.Call +import cromwell.core.path.PathBuilderFactory +import wdl4s.TaskCall import wdl4s.values.WdlValue import scala.concurrent.Future @@ -19,7 +19,7 @@ case class SharedFileSystemInitializationActorParams serviceRegistryActor: ActorRef, workflowDescriptor: BackendWorkflowDescriptor, configurationDescriptor: BackendConfigurationDescriptor, - calls: Set[Call], + calls: Set[TaskCall], pathBuilderFactories: List[PathBuilderFactory] ) @@ -39,7 +39,7 @@ class SharedFileSystemInitializationActor(params: SharedFileSystemInitialization override lazy val workflowDescriptor: BackendWorkflowDescriptor = params.workflowDescriptor override lazy val configurationDescriptor: BackendConfigurationDescriptor = params.configurationDescriptor - override lazy val calls: Set[Call] = params.calls + override lazy val calls: Set[TaskCall] = params.calls override lazy val serviceRegistryActor: ActorRef = params.serviceRegistryActor def runtimeAttributesBuilder: SharedFileSystemValidatedRuntimeAttributesBuilder = diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala index b958f3c6f..54453ab75 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala @@ -4,7 +4,7 @@ import akka.actor.{Actor, ActorRef} import com.typesafe.config.{Config, ConfigFactory} import cromwell.backend.BackendInitializationData import cromwell.backend.callcaching.JobCachingActorHelper -import cromwell.backend.io.{WorkflowPathsBackendInitializationData, JobPaths} +import cromwell.backend.io.{JobPathsWithDocker, WorkflowPathsBackendInitializationData} import cromwell.backend.validation.{RuntimeAttributesValidation, ValidatedRuntimeAttributes} import cromwell.core.logging.JobLogging import net.ceedubs.ficus.Ficus._ @@ -17,7 +17,7 @@ trait SharedFileSystemJobCachingActorHelper extends JobCachingActorHelper { def serviceRegistryActor: ActorRef lazy val jobPaths = - new JobPaths(jobDescriptor.workflowDescriptor, configurationDescriptor.backendConfig, jobDescriptor.key) + new JobPathsWithDocker(jobDescriptor.key, jobDescriptor.workflowDescriptor, configurationDescriptor.backendConfig) lazy val initializationData = BackendInitializationData. as[SharedFileSystemBackendInitializationData](backendInitializationDataOption) diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala index b39e3609c..1cbfd2cfe 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala @@ -8,7 +8,7 @@ import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescript import cromwell.core.TestKitSuite import cromwell.core.logging.LoggingTest._ import org.scalatest.{Matchers, WordSpecLike} -import wdl4s.Call +import wdl4s.TaskCall import scala.concurrent.duration._ @@ -35,7 +35,7 @@ class SharedFileSystemInitializationActorSpec extends TestKitSuite("SharedFileSy |} """.stripMargin - private def getActorRef(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[Call], + private def getActorRef(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall], conf: BackendConfigurationDescriptor) = { val params = SharedFileSystemInitializationActorParams(emptyActor, workflowDescriptor, conf, calls, List.empty) val props = Props(new SharedFileSystemInitializationActor(params)) @@ -47,7 +47,7 @@ class SharedFileSystemInitializationActorSpec extends TestKitSuite("SharedFileSy within(Timeout) { val workflowDescriptor = buildWorkflowDescriptor(HelloWorld, runtime = """runtime { unsupported: 1 }""") val conf = emptyBackendConfig - val backend = getActorRef(workflowDescriptor, workflowDescriptor.workflowNamespace.workflow.calls, conf) + val backend = getActorRef(workflowDescriptor, workflowDescriptor.workflow.taskCalls, conf) val pattern = "Key/s [unsupported] is/are not supported by backend. " + "Unsupported attributes will not be part of jobs executions." EventFilter.warning(pattern = escapePattern(pattern), occurrences = 1) intercept { diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala index 3144fef17..49f4ccf6c 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala @@ -5,10 +5,10 @@ import java.nio.file.{Files, Paths} import akka.testkit.TestDuration import better.files._ import com.typesafe.config.ConfigFactory -import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, FailedNonRetryableResponse, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, JobFailedNonRetryableResponse, JobSucceededResponse} import cromwell.backend.BackendLifecycleActor.AbortJobCommand import cromwell.backend.io.TestWorkflows._ -import cromwell.backend.io.{JobPaths, TestWorkflows} +import cromwell.backend.io.{JobPathsWithDocker, TestWorkflows} import cromwell.backend.sfs.TestLocalAsyncJobExecutionActor._ import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobDescriptorKey, BackendSpec, RuntimeAttributeDefinition} import cromwell.core.Tags._ @@ -31,10 +31,10 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst lazy val runtimeAttributeDefinitions = SharedFileSystemValidatedRuntimeAttributesBuilder.default.definitions.toSet def executeSpec(docker: Boolean) = { - val expectedOutputs: JobOutputs = Map( + val expectedOutputs: CallOutputs = Map( "salutation" -> JobOutput(WdlString("Hello you !")) ) - val expectedResponse = SucceededResponse(mock[BackendJobDescriptorKey], Some(0), expectedOutputs, None, Seq.empty) + val expectedResponse = JobSucceededResponse(mock[BackendJobDescriptorKey], Some(0), expectedOutputs, None, Seq.empty) val runtime = if (docker) """runtime { docker: "ubuntu:latest" }""" else "" val workflowDescriptor = buildWorkflowDescriptor(HelloWorld, runtime = runtime) val workflow = TestWorkflow(workflowDescriptor, emptyBackendConfig, expectedResponse) @@ -51,7 +51,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst } it should "send back an execution failure if the task fails" in { - val expectedResponse = FailedNonRetryableResponse(mock[BackendJobDescriptorKey], new Exception(""), Option(1)) + val expectedResponse = JobFailedNonRetryableResponse(mock[BackendJobDescriptorKey], new Exception(""), Option(1)) val workflow = TestWorkflow(buildWorkflowDescriptor(GoodbyeWorld), emptyBackendConfig, expectedResponse) val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflow.workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions), workflow.config) testWorkflow(workflow, backend) @@ -85,7 +85,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst "wf_localize.localize.inputFileFromJson" -> WdlFile(jsonInputFile) ) - val expectedOutputs: JobOutputs = Map( + val expectedOutputs: CallOutputs = Map( "out" -> JobOutput(WdlArray(WdlArrayType(WdlStringType), Array( WdlString("content from json inputs"), @@ -106,9 +106,9 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst val workflowDescriptor = buildWorkflowDescriptor(InputFiles, inputs, runtime = runtime) val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflowDescriptor, inputs, WorkflowOptions.empty, runtimeAttributeDefinitions), conf) val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, inputs, WorkflowOptions.empty, runtimeAttributeDefinitions) - val expectedResponse = SucceededResponse(jobDescriptor.key, Some(0), expectedOutputs, None, Seq.empty) + val expectedResponse = JobSucceededResponse(jobDescriptor.key, Some(0), expectedOutputs, None, Seq.empty) - val jobPaths = new JobPaths(workflowDescriptor, conf.backendConfig, jobDescriptor.key) + val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, conf.backendConfig) whenReady(backend.execute) { executionResponse => assertResponse(executionResponse, expectedResponse) @@ -156,7 +156,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst val backendRef = createBackendRef(jobDescriptor, emptyBackendConfig) val backend = backendRef.underlyingActor - val jobPaths = new JobPaths(workflowDescriptor, ConfigFactory.empty, jobDescriptor.key) + val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, ConfigFactory.empty) File(jobPaths.callExecutionRoot).createDirectories() File(jobPaths.stdout).write("Hello stubby ! ") File(jobPaths.stderr).touch() @@ -188,13 +188,13 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst whenReady(execute, Timeout(10.seconds.dilated)) { executionResponse => if (writeReturnCode) { - executionResponse should be(a[SucceededResponse]) - val succeededResponse = executionResponse.asInstanceOf[SucceededResponse] + executionResponse should be(a[JobSucceededResponse]) + val succeededResponse = executionResponse.asInstanceOf[JobSucceededResponse] succeededResponse.returnCode.value should be(0) succeededResponse.jobOutputs should be(Map("salutation" -> JobOutput(WdlString("Hello stubby !")))) } else { - executionResponse should be(a[FailedNonRetryableResponse]) - val failedResponse = executionResponse.asInstanceOf[FailedNonRetryableResponse] + executionResponse should be(a[JobFailedNonRetryableResponse]) + val failedResponse = executionResponse.asInstanceOf[JobFailedNonRetryableResponse] failedResponse.returnCode should be(empty) failedResponse.throwable should be(a[RuntimeException]) failedResponse.throwable.getMessage should startWith("Unable to determine that 0 is alive, and") @@ -218,7 +218,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst it should "execute shards from a scatter" in { val workflowDescriptor = buildWorkflowDescriptor(TestWorkflows.Scatter) - val call = workflowDescriptor.workflowNamespace.workflow.calls.head + val call = workflowDescriptor.workflow.taskCalls.head 0 to 2 foreach { shard => // This assumes that engine will give us the evaluated value of the scatter item at the correct index @@ -231,7 +231,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst BackendJobDescriptor(workflowDescriptor, BackendJobDescriptorKey(call, Option(shard), 1), runtimeAttributes, fqnMapToDeclarationMap(symbolMaps)) val backend = createBackend(jobDescriptor, emptyBackendConfig) val response = - SucceededResponse(mock[BackendJobDescriptorKey], Some(0), Map("out" -> JobOutput(WdlInteger(shard))), None, Seq.empty) + JobSucceededResponse(mock[BackendJobDescriptorKey], Some(0), Map("out" -> JobOutput(WdlInteger(shard))), None, Seq.empty) executeJobAndAssertOutputs(backend, response) } } @@ -244,7 +244,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst val workflowDescriptor = buildWorkflowDescriptor(OutputProcess, inputs) val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, inputs, WorkflowOptions.empty, runtimeAttributeDefinitions) val backend = createBackend(jobDescriptor, emptyBackendConfig) - val jobPaths = new JobPaths(workflowDescriptor, emptyBackendConfig.backendConfig, jobDescriptor.key) + val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, emptyBackendConfig.backendConfig) val expectedA = WdlFile(jobPaths.callExecutionRoot.resolve("a").toAbsolutePath.toString) val expectedB = WdlFile(jobPaths.callExecutionRoot.resolve("dir").toAbsolutePath.resolve("b").toString) val expectedOutputs = Map( @@ -252,13 +252,13 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst "o2" -> JobOutput(WdlArray(WdlArrayType(WdlFileType), Seq(expectedA, expectedB))), "o3" -> JobOutput(WdlFile(inputFile)) ) - val expectedResponse = SucceededResponse(jobDescriptor.key, Some(0), expectedOutputs, None, Seq.empty) + val expectedResponse = JobSucceededResponse(jobDescriptor.key, Some(0), expectedOutputs, None, Seq.empty) executeJobAndAssertOutputs(backend, expectedResponse) } it should "fail post processing if an output fail is not found" in { - val expectedResponse = FailedNonRetryableResponse(mock[BackendJobDescriptorKey], + val expectedResponse = JobFailedNonRetryableResponse(mock[BackendJobDescriptorKey], AggregatedException(Seq.empty, "Could not process output, file not found"), Option(0)) val workflow = TestWorkflow(buildWorkflowDescriptor(MissingOutputProcess), emptyBackendConfig, expectedResponse) val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflow.workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions), workflow.config) diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala index 0ae091367..72d2c6e99 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala @@ -3,7 +3,7 @@ package cromwell.backend.sfs import akka.actor.{ActorSystem, Props} import akka.testkit.TestActorRef import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse -import cromwell.backend.io.WorkflowPaths +import cromwell.backend.io.WorkflowPathsWithDocker import cromwell.backend.validation.{DockerValidation, RuntimeAttributesValidation} import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor} @@ -34,7 +34,7 @@ object TestLocalAsyncJobExecutionActor { def createBackendRef(jobDescriptor: BackendJobDescriptor, configurationDescriptor: BackendConfigurationDescriptor) (implicit system: ActorSystem): TestActorRef[SharedFileSystemJobExecutionActor] = { val emptyActor = system.actorOf(Props.empty) - val workflowPaths = new WorkflowPaths(jobDescriptor.workflowDescriptor, configurationDescriptor.backendConfig) + val workflowPaths = new WorkflowPathsWithDocker(jobDescriptor.workflowDescriptor, configurationDescriptor.backendConfig) val initializationData = new SharedFileSystemBackendInitializationData(workflowPaths, SharedFileSystemValidatedRuntimeAttributesBuilder.default.withValidation(DockerValidation.optional)) diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala index fd6e424b0..6fd75323e 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala @@ -2,14 +2,14 @@ package cromwell.backend.impl.spark import akka.actor.{ActorRef, ActorSystem, Props} import cromwell.backend._ +import cromwell.backend.io.JobPathsWithDocker import cromwell.backend.sfs.SharedFileSystemExpressionFunctions -import cromwell.backend.io.JobPaths import cromwell.core.CallContext -import wdl4s.Call +import wdl4s.TaskCall import wdl4s.expression.WdlStandardLibraryFunctions case class SparkBackendFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor, actorSystem: ActorSystem) extends BackendLifecycleActorFactory { - override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[Call], serviceRegistryActor: ActorRef): Option[Props] = { + override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall], serviceRegistryActor: ActorRef): Option[Props] = { Option(SparkInitializationActor.props(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor)) } @@ -22,7 +22,7 @@ case class SparkBackendFactory(name: String, configurationDescriptor: BackendCon override def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor, jobKey: BackendJobDescriptorKey, initializationData: Option[BackendInitializationData]): WdlStandardLibraryFunctions = { - val jobPaths = new JobPaths(workflowDescriptor, configurationDescriptor.backendConfig, jobKey) + val jobPaths = new JobPathsWithDocker(jobKey, workflowDescriptor, configurationDescriptor.backendConfig) val callContext = new CallContext( jobPaths.callExecutionRoot, jobPaths.stdout.toAbsolutePath.toString, diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala index 4e00ee954..1ff66aa24 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala @@ -6,7 +6,7 @@ import cromwell.backend.validation.RuntimeAttributesDefault import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor} import cromwell.core.WorkflowOptions -import wdl4s.Call +import wdl4s.TaskCall import wdl4s.types.{WdlBooleanType, WdlIntegerType, WdlStringType} import wdl4s.values.WdlValue @@ -18,14 +18,14 @@ object SparkInitializationActor { SparkRuntimeAttributes.NumberOfExecutorsKey, SparkRuntimeAttributes.AppMainClassKey) def props(workflowDescriptor: BackendWorkflowDescriptor, - calls: Set[Call], + calls: Set[TaskCall], configurationDescriptor: BackendConfigurationDescriptor, serviceRegistryActor: ActorRef): Props = Props(new SparkInitializationActor(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor)) } class SparkInitializationActor(override val workflowDescriptor: BackendWorkflowDescriptor, - override val calls: Set[Call], + override val calls: Set[TaskCall], override val configurationDescriptor: BackendConfigurationDescriptor, override val serviceRegistryActor: ActorRef) extends BackendWorkflowInitializationActor { diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala index 9b2656ca5..927ac7d45 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala @@ -3,14 +3,14 @@ package cromwell.backend.impl.spark import java.nio.file.attribute.PosixFilePermission import akka.actor.Props -import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, FailedNonRetryableResponse, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobFailedNonRetryableResponse, JobSucceededResponse} import cromwell.backend.impl.spark.SparkClusterProcess._ -import cromwell.backend.io.JobPaths +import cromwell.backend.io.JobPathsWithDocker import cromwell.backend.sfs.{SharedFileSystem, SharedFileSystemExpressionFunctions} import cromwell.backend.wdl.Command import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobExecutionActor} -import cromwell.core.path.{DefaultPathBuilder, TailedWriter, UntailedWriter} import cromwell.core.path.JavaWriterImplicits._ +import cromwell.core.path.{DefaultPathBuilder, TailedWriter, UntailedWriter} import wdl4s.parser.MemoryUnit import wdl4s.util.TryUtil @@ -44,7 +44,7 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, private val sparkDeployMode = configurationDescriptor.backendConfig.getString("deployMode").toLowerCase override val sharedFileSystemConfig = fileSystemsConfig.getConfig("local") private val workflowDescriptor = jobDescriptor.workflowDescriptor - private val jobPaths = new JobPaths(workflowDescriptor, configurationDescriptor.backendConfig, jobDescriptor.key) + private val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, configurationDescriptor.backendConfig) // Files private val executionDir = jobPaths.callExecutionRoot @@ -108,12 +108,12 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, private def resolveExecutionResult(jobReturnCode: Try[Int], failedOnStderr: Boolean): Future[BackendJobExecutionResponse] = { (jobReturnCode, failedOnStderr) match { case (Success(0), true) if File(jobPaths.stderr).lines.toList.nonEmpty => - Future.successful(FailedNonRetryableResponse(jobDescriptor.key, + Future.successful(JobFailedNonRetryableResponse(jobDescriptor.key, new IllegalStateException(s"Execution process failed although return code is zero but stderr is not empty"), Option(0))) case (Success(0), _) => resolveExecutionProcess - case (Success(rc), _) => Future.successful(FailedNonRetryableResponse(jobDescriptor.key, + case (Success(rc), _) => Future.successful(JobFailedNonRetryableResponse(jobDescriptor.key, new IllegalStateException(s"Execution process failed. Spark returned non zero status code: $rc"), Option(rc))) - case (Failure(error), _) => Future.successful(FailedNonRetryableResponse(jobDescriptor.key, error, None)) + case (Failure(error), _) => Future.successful(JobFailedNonRetryableResponse(jobDescriptor.key, error, None)) } } @@ -123,9 +123,9 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, case true => clusterExtProcess.startMonitoringSparkClusterJob(jobPaths.callExecutionRoot, SubmitJobJson.format(sparkDeployMode)) collect { case Finished => processSuccess(0) - case Failed(error: Throwable) => FailedNonRetryableResponse(jobDescriptor.key, error, None) + case Failed(error: Throwable) => JobFailedNonRetryableResponse(jobDescriptor.key, error, None) } recover { - case error: Throwable => FailedNonRetryableResponse(jobDescriptor.key, error, None) + case error: Throwable => JobFailedNonRetryableResponse(jobDescriptor.key, error, None) } case false => Future.successful(processSuccess(0)) } @@ -133,12 +133,12 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, private def processSuccess(rc: Int) = { evaluateOutputs(callEngineFunction, outputMapper(jobPaths)) match { - case Success(outputs) => SucceededResponse(jobDescriptor.key, Some(rc), outputs, None, Seq.empty) + case Success(outputs) => JobSucceededResponse(jobDescriptor.key, Some(rc), outputs, None, Seq.empty) case Failure(e) => val message = Option(e.getMessage) map { ": " + _ } getOrElse "" - FailedNonRetryableResponse(jobDescriptor.key, new Throwable("Failed post processing of outputs" + message, e), Option(rc)) + JobFailedNonRetryableResponse(jobDescriptor.key, new Throwable("Failed post processing of outputs" + message, e), Option(rc)) } } @@ -206,7 +206,7 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, case false => executionResponse completeWith executeTask(extProcess, stdoutWriter, stderrWriter) } } recover { - case exception => executionResponse success FailedNonRetryableResponse(jobDescriptor.key, exception, None) + case exception => executionResponse success JobFailedNonRetryableResponse(jobDescriptor.key, exception, None) } } diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkInitializationActorSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkInitializationActorSpec.scala index aafd665ab..de58b7c06 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkInitializationActorSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkInitializationActorSpec.scala @@ -7,6 +7,7 @@ import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescript import cromwell.core.TestKitSuite import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} import wdl4s._ + import scala.concurrent.duration._ class SparkInitializationActorSpec extends TestKitSuite("SparkInitializationActorSpec") @@ -32,7 +33,7 @@ class SparkInitializationActorSpec extends TestKitSuite("SparkInitializationAc |} """.stripMargin - private def getSparkBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[Call], conf: BackendConfigurationDescriptor) = { + private def getSparkBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall], conf: BackendConfigurationDescriptor) = { system.actorOf(SparkInitializationActor.props(workflowDescriptor, calls, conf, emptyActor)) } @@ -41,7 +42,7 @@ class SparkInitializationActorSpec extends TestKitSuite("SparkInitializationAc within(Timeout) { EventFilter.warning(message = s"Key/s [memory] is/are not supported by SparkBackend. Unsupported attributes will not be part of jobs executions.", occurrences = 1) intercept { val workflowDescriptor = buildWorkflowDescriptor(HelloWorld, runtime = """runtime { memory: 1 %s: "%s"}""".format("appMainClass", "test")) - val backend = getSparkBackend(workflowDescriptor, workflowDescriptor.workflowNamespace.workflow.calls, emptyBackendConfig) + val backend = getSparkBackend(workflowDescriptor, workflowDescriptor.workflow.taskCalls, emptyBackendConfig) backend ! Initialize } } diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala index 02e65ab4d..3a7873fc5 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala @@ -6,7 +6,7 @@ import java.nio.file.Path import akka.testkit.{ImplicitSender, TestActorRef} import better.files._ import com.typesafe.config.ConfigFactory -import cromwell.backend.BackendJobExecutionActor.{FailedNonRetryableResponse, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{JobFailedNonRetryableResponse, JobSucceededResponse} import cromwell.backend.impl.spark.SparkClusterProcess._ import cromwell.backend.io._ import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendSpec} @@ -171,7 +171,7 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") when(sparkClusterProcess.startMonitoringSparkClusterJob(any[Path], any[String])).thenReturn(Future.successful(Finished)) whenReady(backend.execute, timeout) { response => - response shouldBe a[SucceededResponse] + response shouldBe a[JobSucceededResponse] verify(sparkClusterProcess, times(1)).externalProcess(any[Seq[String]], any[ProcessLogger]) verify(sparkClusterProcess, times(1)).tailedWriter(any[Int], any[Path]) verify(sparkClusterProcess, times(1)).untailedWriter(any[Path]) @@ -201,8 +201,8 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") when(sparkClusterProcess.startMonitoringSparkClusterJob(any[Path], any[String])).thenReturn(Future.successful(Failed(new Throwable("failed to monitor")))) whenReady(backend.execute, timeout) { response => - response shouldBe a[FailedNonRetryableResponse] - assert(response.asInstanceOf[FailedNonRetryableResponse].throwable.getMessage.contains("failed to monitor")) + response shouldBe a[JobFailedNonRetryableResponse] + assert(response.asInstanceOf[JobFailedNonRetryableResponse].throwable.getMessage.contains("failed to monitor")) verify(sparkClusterProcess, times(1)).externalProcess(any[Seq[String]], any[ProcessLogger]) verify(sparkClusterProcess, times(1)).tailedWriter(any[Int], any[Path]) verify(sparkClusterProcess, times(1)).untailedWriter(any[Path]) @@ -232,8 +232,8 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") when(sparkClusterProcess.startMonitoringSparkClusterJob(any[Path], any[String])).thenReturn(Future.failed(new IllegalStateException("failed to start monitoring process"))) whenReady(backend.execute, timeout) { response => - response shouldBe a[FailedNonRetryableResponse] - assert(response.asInstanceOf[FailedNonRetryableResponse].throwable.getMessage.contains("failed to start monitoring process")) + response shouldBe a[JobFailedNonRetryableResponse] + assert(response.asInstanceOf[JobFailedNonRetryableResponse].throwable.getMessage.contains("failed to start monitoring process")) verify(sparkClusterProcess, times(1)).externalProcess(any[Seq[String]], any[ProcessLogger]) verify(sparkClusterProcess, times(1)).tailedWriter(any[Int], any[Path]) verify(sparkClusterProcess, times(1)).untailedWriter(any[Path]) @@ -263,8 +263,8 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") when(sparkClusterProcess.processStderr).thenReturn(sampleSubmissionResponse) whenReady(backend.execute, timeout) { response => - response shouldBe a[FailedNonRetryableResponse] - assert(response.asInstanceOf[FailedNonRetryableResponse].throwable.getMessage.contains(s"Execution process failed although return code is zero but stderr is not empty")) + response shouldBe a[JobFailedNonRetryableResponse] + assert(response.asInstanceOf[JobFailedNonRetryableResponse].throwable.getMessage.contains(s"Execution process failed although return code is zero but stderr is not empty")) verify(sparkClusterProcess, times(1)).externalProcess(any[Seq[String]], any[ProcessLogger]) verify(sparkClusterProcess, times(1)).tailedWriter(any[Int], any[Path]) verify(sparkClusterProcess, times(1)).untailedWriter(any[Path]) @@ -292,8 +292,8 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") when(sparkClusterProcess.processStderr).thenReturn(stderrResult) whenReady(backend.execute, timeout) { response => - response shouldBe a[FailedNonRetryableResponse] - assert(response.asInstanceOf[FailedNonRetryableResponse].throwable.getMessage.contains(s"Execution process failed. Spark returned non zero status code:")) + response shouldBe a[JobFailedNonRetryableResponse] + assert(response.asInstanceOf[JobFailedNonRetryableResponse].throwable.getMessage.contains(s"Execution process failed. Spark returned non zero status code:")) } cleanUpJob(jobPaths) } @@ -318,8 +318,8 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") when(sparkClusterProcess.processStderr).thenReturn(stderrResult) whenReady(backend.execute, timeout) { response => - response shouldBe a[FailedNonRetryableResponse] - assert(response.asInstanceOf[FailedNonRetryableResponse].throwable.getMessage.contains(s"submit job process exitValue method failed")) + response shouldBe a[JobFailedNonRetryableResponse] + assert(response.asInstanceOf[JobFailedNonRetryableResponse].throwable.getMessage.contains(s"submit job process exitValue method failed")) } cleanUpJob(jobPaths) } @@ -347,7 +347,7 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") }).underlyingActor whenReady(backend.execute, timeout) { response => - response shouldBe a[SucceededResponse] + response shouldBe a[JobSucceededResponse] verify(sparkProcess, times(1)).externalProcess(any[Seq[String]], any[ProcessLogger]) verify(sparkProcess, times(1)).tailedWriter(any[Int], any[Path]) verify(sparkProcess, times(1)).untailedWriter(any[Path]) @@ -376,8 +376,8 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") when(sparkProcess.processStderr).thenReturn(stderrResult) whenReady(backend.execute, timeout) { response => - response shouldBe a[FailedNonRetryableResponse] - assert(response.asInstanceOf[FailedNonRetryableResponse].throwable.getMessage.contains(s"Execution process failed. Spark returned non zero status code:")) + response shouldBe a[JobFailedNonRetryableResponse] + assert(response.asInstanceOf[JobFailedNonRetryableResponse].throwable.getMessage.contains(s"Execution process failed. Spark returned non zero status code:")) } cleanUpJob(jobPaths) @@ -402,8 +402,8 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") when(sparkProcess.untailedWriter(any[Path])).thenReturn(stubUntailed) whenReady(backend.execute, timeout) { response => - response shouldBe a[FailedNonRetryableResponse] - assert(response.asInstanceOf[FailedNonRetryableResponse].throwable.getMessage.contains(s"Execution process failed although return code is zero but stderr is not empty")) + response shouldBe a[JobFailedNonRetryableResponse] + assert(response.asInstanceOf[JobFailedNonRetryableResponse].throwable.getMessage.contains(s"Execution process failed although return code is zero but stderr is not empty")) } cleanUpJob(jobPaths) @@ -427,7 +427,7 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") when(sparkProcess.untailedWriter(any[Path])).thenReturn(stubUntailed) whenReady(backend.execute, timeout) { response => - response shouldBe a[SucceededResponse] + response shouldBe a[JobSucceededResponse] verify(sparkProcess, times(1)).externalProcess(any[Seq[String]], any[ProcessLogger]) verify(sparkProcess, times(1)).tailedWriter(any[Int], any[Path]) verify(sparkProcess, times(1)).untailedWriter(any[Path]) @@ -438,7 +438,7 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") } - private def cleanUpJob(jobPaths: JobPaths): Unit = { + private def cleanUpJob(jobPaths: JobPathsWithDocker): Unit = { File(jobPaths.workflowRoot).delete(true) () } @@ -447,7 +447,7 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") val backendWorkflowDescriptor = buildWorkflowDescriptor(wdl = wdlSource, inputs = inputFiles.getOrElse(Map.empty), runtime = runtimeString) val backendConfigurationDescriptor = if (isCluster) BackendConfigurationDescriptor(backendClusterConfig, ConfigFactory.load) else BackendConfigurationDescriptor(backendClientConfig, ConfigFactory.load) val jobDesc = jobDescriptorFromSingleCallWorkflow(backendWorkflowDescriptor, inputFiles.getOrElse(Map.empty), WorkflowOptions.empty, Set.empty) - val jobPaths = if (isCluster) new JobPaths(backendWorkflowDescriptor, backendClusterConfig, jobDesc.key) else new JobPaths(backendWorkflowDescriptor, backendClientConfig, jobDesc.key) + val jobPaths = if (isCluster) new JobPathsWithDocker(jobDesc.key, backendWorkflowDescriptor, backendClusterConfig) else new JobPathsWithDocker(jobDesc.key, backendWorkflowDescriptor, backendClientConfig) val executionDir = jobPaths.callExecutionRoot val stdout = File(executionDir.toString, "stdout") stdout.createIfNotExists(asDirectory = false, createParents = true) @@ -456,7 +456,7 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") TestJobDescriptor(jobDesc, jobPaths, backendConfigurationDescriptor) } - private case class TestJobDescriptor(jobDescriptor: BackendJobDescriptor, jobPaths: JobPaths, backendConfigurationDescriptor: BackendConfigurationDescriptor) + private case class TestJobDescriptor(jobDescriptor: BackendJobDescriptor, jobPaths: JobPathsWithDocker, backendConfigurationDescriptor: BackendConfigurationDescriptor) trait MockWriter extends Writer { var closed = false diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala index 8d7888adb..36af37e3d 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala @@ -90,7 +90,7 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { runtime: String) = { BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime.format("appMainClass", "com.test.spark"))), + WdlNamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime.format("appMainClass", "com.test.spark"))).workflow, inputs, options ) @@ -104,7 +104,7 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { call.lookupFunction(knownInputs, NoFunctions) } - workflowDescriptor.workflowNamespace.workflow.calls map { + workflowDescriptor.workflow.taskCalls map { call => val ra = call.task.runtimeAttributes.attrs mapValues { _.evaluate(createLookup(call), NoFunctions) } TryUtil.sequenceMap(ra, "Runtime attributes evaluation").get From 1aeeb9d75bff455a34000712d3355b09b2a41b35 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Thu, 17 Nov 2016 17:43:44 -0500 Subject: [PATCH 067/375] Publish `SNAPSHOT`s obfuscated as `SNAP` releases. --- project/Version.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project/Version.scala b/project/Version.scala index dabbc5717..70adedb03 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -39,6 +39,7 @@ object Version { // The project isSnapshot string passed in via command line settings, if desired. val isSnapshot = sys.props.get("project.isSnapshot").forall(_.toBoolean) - if (isSnapshot) s"$version-SNAPSHOT" else version + // For now, obfuscate SNAPSHOTs from sbt's developers: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241 + if (isSnapshot) s"$version-SNAP" else version } } From 59c6baadd318927e13258a5bda01cb15f19e9179 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Wed, 23 Nov 2016 00:15:40 -0500 Subject: [PATCH 068/375] Exclude scalacheck from cats and mouse. --- project/Dependencies.scala | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index fd8f8891d..6bf024beb 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -21,23 +21,28 @@ object Dependencies { // Internal collections of dependencies - private val baseDependencies = List( - "org.broadinstitute" %% "lenthall" % lenthallV, + private val catsDependencies = List( + "org.typelevel" %% "cats" % "0.7.2", + "com.github.benhutchison" %% "mouse" % "0.5" + ) map (_ /* Exclude test framework cats-laws and its transitive dependency scalacheck. If sbt detects scalacheck, it tries to run it. Explicitly excluding the two problematic artifacts instead of including the three (or four?). https://github.com/typelevel/cats/tree/v0.7.2#getting-started + Re "_2.11", see also: https://github.com/sbt/sbt/issues/1518 */ - "org.typelevel" %% "cats" % "0.7.2" - exclude("org.typelevel", "cats-laws_2.11") - exclude("org.typelevel", "cats-kernel-laws_2.11"), - "com.github.benhutchison" %% "mouse" % "0.5", + exclude("org.typelevel", "cats-laws_2.11") + exclude("org.typelevel", "cats-kernel-laws_2.11") + ) + + private val baseDependencies = List( + "org.broadinstitute" %% "lenthall" % lenthallV, "com.iheart" %% "ficus" % "1.3.0", "org.scalatest" %% "scalatest" % "3.0.0" % Test, "org.pegdown" % "pegdown" % "1.6.0" % Test, "org.specs2" %% "specs2-mock" % "3.8.5" % Test - ) + ) ++ catsDependencies private val slf4jBindingDependencies = List( // http://logback.qos.ch/dependencies.html From a7634955b24e83655320626c51d0989334d18783 Mon Sep 17 00:00:00 2001 From: Ruchi Date: Wed, 23 Nov 2016 14:41:19 -0500 Subject: [PATCH 069/375] Resolve downstream Dependencies develop (#1698) Resolve dependencies for preempted scatter jobs --- .../engine/workflow/lifecycle/execution/ExecutionStore.scala | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala index 390657fef..b3f29928c 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala @@ -62,8 +62,17 @@ case class ExecutionStore(store: Map[JobKey, ExecutionStatus]) { case _ => Nil } + /* + * We need to use an "exists" in this case because the execution store can contain a job attempt with the same + * fqn and index but a preempted status. We wouldn't want that preempted attempt to count against the completion + * of the scatter block. + */ + def isDone(e: JobKey): Boolean = store exists { + case (k, s) => k.scope.fullyQualifiedName == e.scope.fullyQualifiedName && k.index == e.index && s == ExecutionStatus.Done + } + val dependencies = upstream.flatten ++ downstream - val dependenciesResolved = dependencies forall { case (_, s) => s == ExecutionStatus.Done } + val dependenciesResolved = dependencies forall { case (k, _) => isDone(k) } /* * We need to make sure that all prerequisiteScopes have been resolved to some entry before going forward. From 3eefaa97b9a4a5500b35a2913a37b3c318890abd Mon Sep 17 00:00:00 2001 From: Jeff Gentry Date: Thu, 24 Nov 2016 20:19:02 -0500 Subject: [PATCH 070/375] Keep the transient google communication errors in the poll queue (#1686) --- .../jes/statuspolling/JesApiQueryManager.scala | 16 +++++-- .../impl/jes/statuspolling/JesPollingActor.scala | 1 - .../jes/statuspolling/JesPollingActorClient.scala | 5 +- .../jes/statuspolling/JesApiQueryManagerSpec.scala | 54 +++++++++------------- 4 files changed, 33 insertions(+), 43 deletions(-) diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala index f3a7739ac..2e2d254e2 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala @@ -13,13 +13,17 @@ import scala.collection.immutable.Queue */ class JesApiQueryManager extends Actor with ActorLogging { - private var workQueue: Queue[JesStatusPollQuery] = Queue.empty + // workQueue is protected for the unit tests, not intended to be generally overridden + protected[statuspolling] var workQueue: Queue[JesStatusPollQuery] = Queue.empty private var workInProgress: Map[ActorRef, JesPollingWorkBatch] = Map.empty // If the statusPoller dies, we want to stop it and handle the termination ourselves. override val supervisorStrategy = SupervisorStrategy.stoppingStrategy private def statusPollerProps = JesPollingActor.props(self) - private var statusPoller: ActorRef = _ + + // statusPoller is protected for the unit tests, not intended to be generally overridden + protected[statuspolling] var statusPoller: ActorRef = _ + resetStatusPoller() override def receive = { @@ -70,15 +74,17 @@ class JesApiQueryManager extends Actor with ActorLogging { // Currently we can assume this is a polling actor. Might change in a future update: workInProgress.get(terminee) match { case Some(work) => - // Ouch. We should tell all of its clients that it fell over. And then start a new one. - log.error(s"The JES polling actor $terminee unexpectedly terminated while conducting ${work.workBatch.tail.size + 1} polls. Making a new one...") - work.workBatch.toList foreach { _.requester ! JesPollingActor.JesPollError } + // Most likely due to an unexpected HTTP error, push the work back on the queue and keep going + log.info(s"The JES polling actor $terminee unexpectedly terminated while conducting ${work.workBatch.tail.size + 1} polls. Making a new one...") + workInProgress -= terminee + workQueue = workQueue ++ work.workBatch.toList case None => // It managed to die while doing absolutely nothing...!? // Maybe it deserves an entry in https://en.wikipedia.org/wiki/List_of_unusual_deaths // Oh well, in the mean time don't do anything, just start a new one log.error(s"The JES polling actor $terminee managed to unexpectedly terminate whilst doing absolutely nothing. This is probably a programming error. Making a new one...") } + resetStatusPoller() } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala index 4152d3933..39329513d 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala @@ -118,5 +118,4 @@ object JesPollingActor { def props(pollingManager: ActorRef) = Props(new JesPollingActor(pollingManager)) final case class JesPollFailed(e: GoogleJsonError, responseHeaders: HttpHeaders) - case object JesPollError } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorClient.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorClient.scala index 1bf7328f8..9070378c3 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorClient.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorClient.scala @@ -3,7 +3,7 @@ package cromwell.backend.impl.jes.statuspolling import java.io.IOException import akka.actor.{Actor, ActorLogging, ActorRef} -import cromwell.backend.impl.jes.statuspolling.JesPollingActor.{JesPollError, JesPollFailed} +import cromwell.backend.impl.jes.statuspolling.JesPollingActor.JesPollFailed import cromwell.backend.impl.jes.{Run, RunStatus} import scala.concurrent.{Future, Promise} @@ -28,9 +28,6 @@ trait JesPollingActorClient { this: Actor with ActorLogging => case JesPollFailed(e, responseHeaders) => log.debug("JES poll failed! Sad.") completePromise(Failure(new IOException(s"Google request failed: ${e.toPrettyString}"))) - case JesPollError => - log.debug("JES poll failed when polling actor died unexpectedly! Sad.") - completePromise(Failure(new RuntimeException("Unexpected actor death!"))) } private def completePromise(runStatus: Try[RunStatus]) = { diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala index 1eaa42297..d98316e81 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala @@ -65,49 +65,35 @@ class JesApiQueryManagerSpec extends TestKitSuite("JesApiQueryManagerSpec") with } AkkaTestUtil.actorDeathMethods(system) foreach { case (name, stopMethod) => - it should s"catch polling actors if they $name and then recreate them" in { - + /* + This test creates two statusPoller ActorRefs which are handed to the TestJesApiQueryManager. Work is added to that query + manager and then the first statusPoller requests work and is subsequently killed. The expectation is that: + + - The work will return to the workQueue of the query manager + - The query manager will have registered a new statusPoller + - That statusPoller is the second ActorRef (and artifact of TestJesApiQueryManager) + */ + it should s"catch polling actors if they $name, recreate them and add work back to the queue" in { val statusPoller1 = TestActorRef(Props(new AkkaTestUtil.DeathTestActor()), TestActorRef(new AkkaTestUtil.StoppingSupervisor())) - val statusPoller2 = TestActorRef(Props(new AkkaTestUtil.DeathTestActor())) + val statusPoller2 = TestActorRef(Props(new AkkaTestUtil.DeathTestActor()), TestActorRef(new AkkaTestUtil.StoppingSupervisor())) val jaqmActor: TestActorRef[TestJesApiQueryManager] = TestActorRef(TestJesApiQueryManager.props(statusPoller1, statusPoller2)) - val statusRequesters = ((0 until BatchSize * 2) map { i => i -> TestProbe(name = s"StatusRequester_$i") }).toMap + val emptyActor = system.actorOf(Props.empty) - // Send a few status poll requests: + // Send a few status poll requests: BatchSize indexedTimes { index => - val probe = statusRequesters(index) - jaqmActor.tell(msg = JesApiQueryManager.DoPoll(Run(index.toString, null)), sender = probe.ref) - } - BatchSize indexedTimes { i => - val index = i + BatchSize // For the second half of the statusRequester set - val probe = statusRequesters(index) - jaqmActor.tell(msg = JesApiQueryManager.DoPoll(Run(index.toString, null)), sender = probe.ref) + jaqmActor.tell(msg = JesApiQueryManager.DoPoll(Run(index.toString, null)), sender = emptyActor) } - // Request a set of work from the middle of the queue: - val batchOffset = 2 - jaqmActor.tell(msg = JesApiQueryManager.RequestJesPollingWork(batchOffset), sender = statusPoller1) jaqmActor.tell(msg = JesApiQueryManager.RequestJesPollingWork(BatchSize), sender = statusPoller1) - // Kill the original status poller: stopMethod(statusPoller1) - // Only the appropriate requesters get an error: - (0 until batchOffset) foreach { index => - val probe = statusRequesters(index) - probe.expectNoMsg(max = AwaitAlmostNothing) - } - (batchOffset until batchOffset + BatchSize) foreach { index => - val probe = statusRequesters(index) - probe.expectMsg(max = TestExecutionTimeout, hint = s"Polling error to requester #$index", obj = JesPollingActor.JesPollError) - } - (batchOffset + BatchSize until 2 * BatchSize) foreach { index => - val probe = statusRequesters(index) - probe.expectNoMsg(max = AwaitAlmostNothing) + eventually { + jaqmActor.underlyingActor.testPollerCreations should be (2) + jaqmActor.underlyingActor.queueSize should be (BatchSize) + jaqmActor.underlyingActor.statusPollerEquals(statusPoller2) should be (true) } - - // Check the next status poller gets created: - eventually { jaqmActor.underlyingActor.testPollerCreations should be(2) } } } } @@ -123,7 +109,6 @@ object JesApiQueryManagerSpec { * This test class allows us to hook into the JesApiQueryManager's makeStatusPoller and provide our own TestProbes instead */ class TestJesApiQueryManager(statusPollerProbes: ActorRef*) extends JesApiQueryManager { - var testProbes: Queue[ActorRef] = _ var testPollerCreations: Int = _ @@ -133,7 +118,7 @@ class TestJesApiQueryManager(statusPollerProbes: ActorRef*) extends JesApiQueryM } override private[statuspolling] def makeStatusPoller(): ActorRef = { - // Initialise the queue, if necessary: + // Initialize the queue, if necessary: if (testProbes == null) { init() } @@ -146,6 +131,9 @@ class TestJesApiQueryManager(statusPollerProbes: ActorRef*) extends JesApiQueryM testProbes = newQueue probe } + + def queueSize = workQueue.size + def statusPollerEquals(otherStatusPoller: ActorRef) = statusPoller == otherStatusPoller } object TestJesApiQueryManager { From 288f782b062b50d416a1a4c41786ac2a8524fd08 Mon Sep 17 00:00:00 2001 From: kcibul Date: Mon, 28 Nov 2016 20:55:14 -0500 Subject: [PATCH 071/375] Re-enable flag for workflow-restart Closes #1706 (#1707) * ability to enable/disable restart * addressed PR comments --- .../engine/workflow/workflowstore/SqlWorkflowStore.scala | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala index 7613c9950..be4ab6ff0 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala @@ -3,6 +3,8 @@ package cromwell.engine.workflow.workflowstore import java.time.OffsetDateTime import cats.data.NonEmptyList +import com.typesafe.config.ConfigFactory +import net.ceedubs.ficus.Ficus._ import cromwell.core.{WorkflowId, WorkflowSourceFiles, WorkflowSourceFilesCollection} import cromwell.database.sql.SqlConverters._ import cromwell.database.sql.WorkflowStoreSqlDatabase @@ -13,9 +15,13 @@ import scala.concurrent.{ExecutionContext, Future} case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase) extends WorkflowStore { override def initialize(implicit ec: ExecutionContext): Future[Unit] = { - sqlDatabase.updateWorkflowState( - WorkflowStoreState.Running.toString, - WorkflowStoreState.Restartable.toString) + if (ConfigFactory.load().as[Option[Boolean]]("system.workflow-restart").getOrElse(true)) { + sqlDatabase.updateWorkflowState( + WorkflowStoreState.Running.toString, + WorkflowStoreState.Restartable.toString) + } else { + Future.successful(()) + } } override def remove(id: WorkflowId)(implicit ec: ExecutionContext): Future[Boolean] = { From b7bd150c1065450115b1c52bf195f62f359ce7bb Mon Sep 17 00:00:00 2001 From: Jeff Gentry Date: Tue, 29 Nov 2016 11:25:06 -0500 Subject: [PATCH 072/375] meta & parameter_meta in wfs --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4e83829a..4c304a220 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 23 +* The `meta` and `parameter_meta` blocks are now valid within `workflow` blocks, not just `task` * Added an option `call-caching.invalidate-bad-cache-results` (default: `true`). If true, Cromwell will invalidate cached results which have failed to copy as part of a cache hit. * Timing diagrams and metadata now receive more fine grained workflow states between submission and Running. * Support for the Pair WDL type (e.g. `Pair[Int, File] floo = (3, "gs://blar/blaz/qlux.txt")`) From 164c82fb2c61d2cc622576fae8c1093f8b865820 Mon Sep 17 00:00:00 2001 From: kcibul Date: Tue, 29 Nov 2016 12:48:39 -0500 Subject: [PATCH 073/375] added support for a FileRoller logback configuration (#1692) (#1710) * added support for a FileRoller logback configuration * made both logback.xml files in repo the same * rearrange to single logback.xml --- core/src/main/resources/logback.xml | 88 +++++++++++++++++++++++ database/migration/src/main/resources/logback.xml | 36 ---------- engine/src/main/resources/logback.xml | 36 ---------- 3 files changed, 88 insertions(+), 72 deletions(-) create mode 100644 core/src/main/resources/logback.xml delete mode 100644 database/migration/src/main/resources/logback.xml delete mode 100644 engine/src/main/resources/logback.xml diff --git a/core/src/main/resources/logback.xml b/core/src/main/resources/logback.xml new file mode 100644 index 000000000..e165bc8b2 --- /dev/null +++ b/core/src/main/resources/logback.xml @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + %date %X{sourceThread} %-5level - %msg%n + + + + + + + + + + + + + + + + + + + + + + + + + + + ${FILEROLLER_DIR}/${FILEROLLER_NAME} + + + + + + ${FILEROLLER_DIR}/${FILEROLLER_NAMEPATTERN}-${FILEROLLER_NAME} + + + ${FILEROLLER_DIR}/%d{yyyyMMdd}-${FILEROLLER_NAME} + + + + ${FILEROLLER_MAXHISTORY} + + + + ${FILEROLLER_SIZECAP} + + + + + + %d{yyyy-MM-dd HH:mm:ss,SSS} [%thread] %-5level %logger{35} - %msg%n + + + + + + + + + + + + + + + + + diff --git a/database/migration/src/main/resources/logback.xml b/database/migration/src/main/resources/logback.xml deleted file mode 100644 index fa27b5dde..000000000 --- a/database/migration/src/main/resources/logback.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - %date %X{sourceThread} %-5level - %msg%n - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/engine/src/main/resources/logback.xml b/engine/src/main/resources/logback.xml deleted file mode 100644 index fa27b5dde..000000000 --- a/engine/src/main/resources/logback.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - %date %X{sourceThread} %-5level - %msg%n - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 8ab7476fa5aaa86d47fe912bdd474e6d949e3904 Mon Sep 17 00:00:00 2001 From: Thib Date: Tue, 29 Nov 2016 15:50:41 -0500 Subject: [PATCH 074/375] Rename workflow options in metadata --- .../migration/src/main/resources/changelog.xml | 1 + .../rename_workflow_options_in_metadata.xml | 10 ++++++ .../table/RenameWorkflowOptionKeysMigration.scala | 18 +--------- .../RenameWorkflowOptionsInMetadata.scala | 38 ++++++++++++++++++++++ .../workflowoptions/WorkflowOptionsRenaming.scala | 21 ++++++++++++ 5 files changed, 71 insertions(+), 17 deletions(-) create mode 100644 database/migration/src/main/resources/changesets/rename_workflow_options_in_metadata.xml create mode 100644 database/migration/src/main/scala/cromwell/database/migration/workflowoptions/RenameWorkflowOptionsInMetadata.scala create mode 100644 database/migration/src/main/scala/cromwell/database/migration/workflowoptions/WorkflowOptionsRenaming.scala diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml index 2575316f5..b2751b7cc 100644 --- a/database/migration/src/main/resources/changelog.xml +++ b/database/migration/src/main/resources/changelog.xml @@ -50,6 +50,7 @@ + diff --git a/database/migration/src/main/resources/changesets/rename_workflow_options_in_metadata.xml b/database/migration/src/main/resources/changesets/rename_workflow_options_in_metadata.xml new file mode 100644 index 000000000..b189e535a --- /dev/null +++ b/database/migration/src/main/resources/changesets/rename_workflow_options_in_metadata.xml @@ -0,0 +1,10 @@ + + + + + + + + diff --git a/database/migration/src/main/scala/cromwell/database/migration/restart/table/RenameWorkflowOptionKeysMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/restart/table/RenameWorkflowOptionKeysMigration.scala index 3852750ce..c0610ae36 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/restart/table/RenameWorkflowOptionKeysMigration.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/restart/table/RenameWorkflowOptionKeysMigration.scala @@ -1,19 +1,11 @@ package cromwell.database.migration.restart.table +import cromwell.database.migration.workflowoptions.WorkflowOptionsRenaming._ import cromwell.database.migration.restart.table.RenameWorkflowOptionKeysMigration._ import liquibase.database.jvm.JdbcConnection import spray.json._ - object RenameWorkflowOptionKeysMigration { - private val RenamedOptionKeys = Map( - "defaultRuntimeOptions" -> "default_runtime_attributes", - "workflowFailureMode" -> "workflow_failure_mode", - "workflow_log_dir" -> "final_workflow_log_dir", - "outputs_path" -> "final_workflow_outputs_dir", - "call_logs_dir" -> "final_call_logs_dir" - ) - private val QueryWorkflowStore = " SELECT WORKFLOW_STORE_ID, WORKFLOW_OPTIONS FROM WORKFLOW_STORE " private val UpdateWorkflowStore = " UPDATE WORKFLOW_STORE SET WORKFLOW_OPTIONS = ? WHERE WORKFLOW_STORE_ID = ? " @@ -25,14 +17,6 @@ class RenameWorkflowOptionKeysMigration extends AbstractRestartMigration { override protected def description: String = "Workflow option renaming" override protected def doMigration(connection: JdbcConnection): Unit = { - - def renameOptionKeys(field: JsField): JsField = { - field match { - case (oldName, value) if RenamedOptionKeys.contains(oldName) => RenamedOptionKeys(oldName) -> value - case noop => noop - } - } - val query = connection.createStatement() lazy val insert = connection.prepareStatement(UpdateWorkflowStore) query.execute(QueryWorkflowStore) diff --git a/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/RenameWorkflowOptionsInMetadata.scala b/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/RenameWorkflowOptionsInMetadata.scala new file mode 100644 index 000000000..e6842b459 --- /dev/null +++ b/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/RenameWorkflowOptionsInMetadata.scala @@ -0,0 +1,38 @@ +package cromwell.database.migration.workflowoptions + +import java.sql.{PreparedStatement, ResultSet} + +import cromwell.database.migration.custom.BatchedTaskChange +import cromwell.database.migration.workflowoptions.WorkflowOptionsRenaming._ +import spray.json.{JsObject, _} + +class RenameWorkflowOptionsInMetadata extends BatchedTaskChange { + val tableName = "METADATA_ENTRY" + val primaryKeyColumn = "METADATA_JOURNAL_ID" + val workflowOptionsColumn = "METADATA_VALUE" + val additionalReadBatchFilters = "AND METADATA_KEY = 'submittedFiles:options'" + + override def readCountQuery = s"SELECT MAX($primaryKeyColumn) FROM $tableName;" + + override def readBatchQuery = + s"""|SELECT $primaryKeyColumn, $workflowOptionsColumn + | FROM $tableName + | WHERE $primaryKeyColumn >= ? AND $primaryKeyColumn < ? $additionalReadBatchFilters; + |""".stripMargin + + override def migrateBatchQuery = s"UPDATE $tableName SET $workflowOptionsColumn = ? WHERE $primaryKeyColumn = ?;" + + override def migrateBatchRow(readRow: ResultSet, migrateStatement: PreparedStatement): Int = { + val rowId = readRow.getInt(1) + + val migratedJson = readRow.getString(2).parseJson match { + case JsObject(fields) => JsObject(fields map renameOptionKeys) + case other => other + } + + migrateStatement.setString(1, migratedJson.prettyPrint) + migrateStatement.setInt(2, rowId) + migrateStatement.addBatch() + 1 + } +} diff --git a/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/WorkflowOptionsRenaming.scala b/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/WorkflowOptionsRenaming.scala new file mode 100644 index 000000000..d148a570f --- /dev/null +++ b/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/WorkflowOptionsRenaming.scala @@ -0,0 +1,21 @@ +package cromwell.database.migration.workflowoptions + +import spray.json._ + +object WorkflowOptionsRenaming { + + private val RenamedOptionKeys = Map( + "defaultRuntimeOptions" -> "default_runtime_attributes", + "workflowFailureMode" -> "workflow_failure_mode", + "workflow_log_dir" -> "final_workflow_log_dir", + "outputs_path" -> "final_workflow_outputs_dir", + "call_logs_dir" -> "final_call_logs_dir" + ) + + def renameOptionKeys(field: JsField): JsField = { + field match { + case (oldName, value) if RenamedOptionKeys.contains(oldName) => RenamedOptionKeys(oldName) -> value + case noop => noop + } + } +} From 85216837ce62399b33c2b4d51c314d0bd98fe89a Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Fri, 28 Oct 2016 14:54:58 -0400 Subject: [PATCH 075/375] Included optional types from WDL4S --- .../cromwell/core/simpleton/WdlValueBuilder.scala | 10 ++++++++-- .../cromwell/core/simpleton/WdlValueSimpleton.scala | 1 + .../core/path/RetryableFileSystemProxySpec.scala | 2 +- .../cromwell/webservice/CromwellApiService.scala | 1 - .../scala/cromwell/DeclarationWorkflowSpec.scala | 8 ++++---- .../src/test/scala/cromwell/MetadataWatchActor.scala | 20 +++++++++++++++----- .../scala/cromwell/SimpleWorkflowActorSpec.scala | 10 ++++++++-- .../lifecycle/execution/ejea/PerTestHelper.scala | 8 +++++++- project/Dependencies.scala | 2 +- .../impl/sfs/config/DeclarationValidation.scala | 2 +- 10 files changed, 46 insertions(+), 18 deletions(-) diff --git a/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala b/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala index 18b6529fb..774c1b5ce 100644 --- a/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala +++ b/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala @@ -2,10 +2,10 @@ package cromwell.core.simpleton import wdl4s.TaskOutput import wdl4s.types._ -import wdl4s.values.{WdlArray, WdlMap, WdlPair, WdlValue} +import wdl4s.values.{WdlArray, WdlMap, WdlOptionalValue, WdlPair, WdlValue} import scala.language.postfixOps -import cromwell.core.{JobOutput, CallOutputs} +import cromwell.core.{CallOutputs, JobOutput} import cromwell.core.simpleton.WdlValueSimpleton._ @@ -93,6 +93,12 @@ object WdlValueBuilder { outputType match { case _: WdlPrimitiveType => components collectFirst { case SimpletonComponent(_, v) => v } get + case opt: WdlOptionalType => + if (components.isEmpty) { + WdlOptionalValue(opt.memberType, None) + } else { + WdlOptionalValue(toWdlValue(opt.memberType, components)) + } case arrayType: WdlArrayType => val groupedByArrayIndex: Map[Int, Traversable[SimpletonComponent]] = group(components map descendIntoArray) WdlArray(arrayType, groupedByArrayIndex.toList.sortBy(_._1) map { case (_, s) => toWdlValue(arrayType.memberType, s) }) diff --git a/core/src/main/scala/cromwell/core/simpleton/WdlValueSimpleton.scala b/core/src/main/scala/cromwell/core/simpleton/WdlValueSimpleton.scala index edcf4c2c9..1f5e04375 100644 --- a/core/src/main/scala/cromwell/core/simpleton/WdlValueSimpleton.scala +++ b/core/src/main/scala/cromwell/core/simpleton/WdlValueSimpleton.scala @@ -26,6 +26,7 @@ object WdlValueSimpleton { implicit class WdlValueSimplifier(wdlValue: WdlValue) { def simplify(name: String): Iterable[WdlValueSimpleton] = wdlValue match { case prim: WdlPrimitive => List(WdlValueSimpleton(name, prim)) + case opt: WdlOptionalValue => opt.value.map(_.simplify(name)).getOrElse(Seq.empty) case WdlArray(_, arrayValue) => arrayValue.zipWithIndex flatMap { case (arrayItem, index) => arrayItem.simplify(s"$name[$index]") } case WdlMap(_, mapValue) => mapValue flatMap { case (key, value) => value.simplify(s"$name:${key.valueString.escapeMeta}") } case WdlPair(left, right) => left.simplify(s"$name:left") ++ right.simplify(s"$name:right") diff --git a/core/src/test/scala/cromwell/core/path/RetryableFileSystemProxySpec.scala b/core/src/test/scala/cromwell/core/path/RetryableFileSystemProxySpec.scala index b6de83ee7..88dbf463d 100644 --- a/core/src/test/scala/cromwell/core/path/RetryableFileSystemProxySpec.scala +++ b/core/src/test/scala/cromwell/core/path/RetryableFileSystemProxySpec.scala @@ -117,7 +117,7 @@ class RetryableFileSystemProxySpec extends TestKitSuite with FlatSpecLike with M val pathMock = mock(classOf[Path]) - it should "timeout if the operation takes too long" in { + it should "timeout if the operation takes too long" ignore { val retryParams = testRetryParams.copy(timeout = 100 millis) val mockFs = mockFileSystem(delay = Option(200 millis)) val retryableFs = new RetryableFileSystemProviderProxy(mockFs, retryParams)(system) diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala index bbebdecd1..1472a5eec 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala @@ -1,7 +1,6 @@ package cromwell.webservice import akka.actor._ -import java.lang.Throwable import cats.data.NonEmptyList import cromwell.core.{WorkflowId, WorkflowSourceFiles} diff --git a/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala b/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala index 1786733be..390e8f6d8 100644 --- a/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala @@ -10,10 +10,10 @@ class DeclarationWorkflowSpec extends Matchers with WordSpecLike { "A workflow with declarations in it" should { "compute inputs properly" in { WdlNamespaceWithWorkflow.load(SampleWdl.DeclarationsWorkflow.wdlSource(runtime="")).workflow.inputs shouldEqual Map( - "two_step.cat.file" -> WorkflowInput("two_step.cat.file", WdlFileType, postfixQuantifier = None), - "two_step.cgrep.str_decl" -> WorkflowInput("two_step.cgrep.str_decl", WdlStringType, postfixQuantifier = None), - "two_step.cgrep.pattern" -> WorkflowInput("two_step.cgrep.pattern", WdlStringType, postfixQuantifier = None), - "two_step.flags_suffix" -> WorkflowInput("two_step.flags_suffix", WdlStringType, postfixQuantifier = None) + "two_step.cat.file" -> WorkflowInput("two_step.cat.file", WdlFileType), + "two_step.cgrep.str_decl" -> WorkflowInput("two_step.cgrep.str_decl", WdlStringType), + "two_step.cgrep.pattern" -> WorkflowInput("two_step.cgrep.pattern", WdlStringType), + "two_step.flags_suffix" -> WorkflowInput("two_step.flags_suffix", WdlStringType) ) } } diff --git a/engine/src/test/scala/cromwell/MetadataWatchActor.scala b/engine/src/test/scala/cromwell/MetadataWatchActor.scala index 691c4efc5..c0c294442 100644 --- a/engine/src/test/scala/cromwell/MetadataWatchActor.scala +++ b/engine/src/test/scala/cromwell/MetadataWatchActor.scala @@ -1,7 +1,7 @@ package cromwell import akka.actor.{Actor, Props} -import cromwell.services.metadata.{MetadataEvent, MetadataJobKey, MetadataString} +import cromwell.services.metadata.{MetadataEvent, MetadataJobKey, MetadataString, MetadataValue} import cromwell.services.metadata.MetadataService.PutMetadataAction import MetadataWatchActor._ @@ -32,26 +32,36 @@ object MetadataWatchActor { trait Matcher { def matches(events: Traversable[MetadataEvent]): Boolean + private var _nearMisses: List[String] = List.empty + protected def addNearMissInfo(miss: String) = _nearMisses :+= miss + def nearMissInformation = _nearMisses + + def checkMetadataValueContains(key: String, actual: MetadataValue, expected: String): Boolean = { + val result = actual.value.contains(expected) + if (!result) addNearMissInfo(s"Key $key had unexpected value.\nActual value: ${actual.value}\n\nDid not contain: $expected") + result + } } def metadataKeyAttemptChecker(attempt: Int): Option[MetadataJobKey] => Boolean = { case Some(jobKey) => jobKey.attempt == attempt case None => false } + final case class JobKeyMetadataKeyAndValueContainStringMatcher(jobKeyCheck: Option[MetadataJobKey] => Boolean, key: String, value: String) extends Matcher { def matches(events: Traversable[MetadataEvent]): Boolean = { - events.exists(e => e.key.key.contains(key) && jobKeyCheck(e.key.jobKey) && e.value.exists { v => v.valueType == MetadataString && v.value.contains(value) }) + events.exists(e => e.key.key.contains(key) && jobKeyCheck(e.key.jobKey) && e.value.exists { v => v.valueType == MetadataString && checkMetadataValueContains(e.key.key, v, value) }) } } abstract class KeyMatchesRegexAndValueContainsStringMatcher(keyTemplate: String, value: String) extends Matcher { val templateRegex = keyTemplate.r def matches(events: Traversable[MetadataEvent]): Boolean = { - events.exists(e => templateRegex.findFirstIn(e.key.key).isDefined && e.value.exists { v => v.value.contains(value) }) + events.exists(e => templateRegex.findFirstIn(e.key.key).isDefined && + e.value.exists { v => checkMetadataValueContains(e.key.key, v, value) }) } } val failurePattern = """failures\[\d*\].message""" - final case class FailureMatcher(value: String) extends KeyMatchesRegexAndValueContainsStringMatcher(failurePattern, value) { - } + final case class FailureMatcher(value: String) extends KeyMatchesRegexAndValueContainsStringMatcher(failurePattern, value) { } } diff --git a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala index c99b93b0a..145834473 100644 --- a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala @@ -93,7 +93,7 @@ class SimpleWorkflowActorSpec extends CromwellTestKitSpec with BeforeAndAfter { } "fail to construct with inputs of the wrong type" in { - val expectedError = "Could not coerce value for 'wf_hello.hello.addressee' into: WdlStringType" + val expectedError = "Could not coerce JsNumber value for 'wf_hello.hello.addressee' (3) into: WdlStringType" val failureMatcher = FailureMatcher(expectedError) val TestableWorkflowActorAndMetadataPromise(workflowActor, supervisor, promise) = buildWorkflowActor(SampleWdl.HelloWorld, s""" { "$Addressee" : 3} """, workflowId, failureMatcher) @@ -101,7 +101,13 @@ class SimpleWorkflowActorSpec extends CromwellTestKitSpec with BeforeAndAfter { val probe = TestProbe() probe watch workflowActor workflowActor ! StartWorkflowCommand - Await.result(promise.future, TestExecutionTimeout) + try { + Await.result(promise.future, TestExecutionTimeout) + } catch { + case e: Throwable => + val info = failureMatcher.nearMissInformation + fail(s"We didn't see the expected error message $expectedError within $TestExecutionTimeout. ${info.mkString(", ")}") + } probe.expectTerminated(workflowActor, AwaitAlmostNothing) supervisor.expectMsgPF(AwaitAlmostNothing, "parent should get a failed response") { case x: WorkflowFailedResponse => diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala index ef53291ca..9389362ad 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala @@ -41,7 +41,13 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock outputs = Seq(("outString", WdlStringType, mockStringExpression("hello"))) ) - val workflow = new Workflow(workflowName, Seq.empty, mock[WdlSyntaxErrorFormatter], mock[Ast]) + val workflow = new Workflow( + unqualifiedName = workflowName, + workflowOutputWildcards = Seq.empty, + wdlSyntaxErrorFormatter = mock[WdlSyntaxErrorFormatter], + meta = Map.empty, + parameterMeta = Map.empty, + ast = mock[Ast]) val call: TaskCall = TaskCall(None, task, Map.empty, mock[Ast]) call.parent_=(workflow) val jobDescriptorKey = BackendJobDescriptorKey(call, jobIndex, jobAttempt) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6bf024beb..a2d1b91a7 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -2,7 +2,7 @@ import sbt._ object Dependencies { lazy val lenthallV = "0.19" - lazy val wdl4sV = "0.7-4a9e61e-SNAP" + lazy val wdl4sV = "0.7-799567f-SNAP" lazy val sprayV = "1.3.3" /* spray-json is an independent project from the "spray suite" diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala index 11b69147e..37a42a5c3 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala @@ -74,7 +74,7 @@ class DeclarationValidation(declaration: Declaration, instanceValidation: Runtim val validationDefault = if (declaration.expression.isDefined) default(instanceValidation, declaration.expression.get) else instanceValidation - if (declaration.postfixQuantifier.contains("?")) validationDefault.optional else validationDefault + if (declaration.wdlType.isInstanceOf[WdlOptionalType]) validationDefault.optional else validationDefault } /** From 28c92c1299f6c63a2ee4d8a1d166aa85c164198c Mon Sep 17 00:00:00 2001 From: mcovarr Date: Wed, 30 Nov 2016 09:35:35 -0500 Subject: [PATCH 076/375] Workflow name expanded to 500 chars as consistent with 0.19 (#1714) --- .../migration/src/main/resources/changelog.xml | 1 + .../changesets/embiggen_summary_workflow_name.xml | 51 ++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 database/migration/src/main/resources/changesets/embiggen_summary_workflow_name.xml diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml index b2751b7cc..18a84c2ca 100644 --- a/database/migration/src/main/resources/changelog.xml +++ b/database/migration/src/main/resources/changelog.xml @@ -46,6 +46,7 @@ + diff --git a/database/migration/src/main/resources/changesets/embiggen_summary_workflow_name.xml b/database/migration/src/main/resources/changesets/embiggen_summary_workflow_name.xml new file mode 100644 index 000000000..8366a2481 --- /dev/null +++ b/database/migration/src/main/resources/changesets/embiggen_summary_workflow_name.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + Either WORKFLOW_METADATA_SUMMARY or WORKFLOW_METADATA_SUMMARY_ENTRY must exist, but not both (Liquibase doesn't have an xor). + + + + + + + + + The WORKFLOW_METADATA_SUMMARY_ENTRY version of the embiggener. This should gracefully handle the absence of the table + in the event the table is still called WORKFLOW_METADATA_SUMMARY when this runs. + + + + + + + + + + The WORKFLOW_METADATA_SUMMARY version of the embiggener. This should gracefully handle the absence of the table + in the event the table has been renamed to WORKFLOW_METADATA_SUMMARY_ENTRY when this runs. + + + + + From 94eb009948374e3fb34b4b3a5af284d1e670dde4 Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 1 Dec 2016 06:00:30 +1000 Subject: [PATCH 077/375] handle WdlPair - to fix #1703 (#1704) * handle WdlPair --- .../JsonFormatting/WdlValueJsonFormatter.scala | 1 + .../cromwell/util/WdlValueJsonFormatterSpec.scala | 28 ++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 core/src/test/scala/cromwell/util/WdlValueJsonFormatterSpec.scala diff --git a/core/src/main/scala/cromwell/util/JsonFormatting/WdlValueJsonFormatter.scala b/core/src/main/scala/cromwell/util/JsonFormatting/WdlValueJsonFormatter.scala index 9d8638be0..8a997efb0 100644 --- a/core/src/main/scala/cromwell/util/JsonFormatting/WdlValueJsonFormatter.scala +++ b/core/src/main/scala/cromwell/util/JsonFormatting/WdlValueJsonFormatter.scala @@ -17,6 +17,7 @@ object WdlValueJsonFormatter extends DefaultJsonProtocol { case a: WdlArray => new JsArray(a.value.map(write).toVector) case m: WdlMap => new JsObject(m.value map {case(k,v) => k.valueString -> write(v)}) case e: WdlExpression => JsString(e.toWdlString) + case q: WdlPair => new JsObject(Map("left" -> write(q.left), "right" -> write(q.right))) } // NOTE: This assumes a map's keys are strings. Since we're coming from JSON this is fine. diff --git a/core/src/test/scala/cromwell/util/WdlValueJsonFormatterSpec.scala b/core/src/test/scala/cromwell/util/WdlValueJsonFormatterSpec.scala new file mode 100644 index 000000000..91d678c01 --- /dev/null +++ b/core/src/test/scala/cromwell/util/WdlValueJsonFormatterSpec.scala @@ -0,0 +1,28 @@ +package cromwell.util + +import scala.Vector + +import org.scalatest.FlatSpec +import org.scalatest.Matchers + +import JsonFormatting.WdlValueJsonFormatter.WdlValueJsonFormat +import spray.json.{ JsObject, pimpString } +import wdl4s.types.{ WdlArrayType, WdlStringType } +import wdl4s.values.{ WdlArray, WdlPair, WdlString } + +class WdlValueJsonFormatterSpec extends FlatSpec with Matchers { + + behavior of "WdlValueJsonFormat" + + it should "write WdlPair to left/right structured JsObject" in { + val left = "sanders" + val right = Vector("rubio", "carson", "cruz") + val wdlPair = WdlPair(WdlString(left), WdlArray(WdlArrayType(WdlStringType), right.map { WdlString(_) })) + val ExpectedJson: JsObject = + """|{ + | "left": "sanders", + | "right": ["rubio", "carson", "cruz"] + |}""".stripMargin.parseJson.asJsObject + WdlValueJsonFormat.write(wdlPair) should matchPattern { case ExpectedJson => } + } +} From 1055719336f558e91a6ef1f6b489ddd6cff32b48 Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Wed, 30 Nov 2016 18:29:56 -0500 Subject: [PATCH 078/375] Revert "Workflow name expanded to 500 chars as consistent with 0.19 (#1714)" This reverts commit 4ff5e1011817b30146522c4e2472b186321a2f0d. --- .../migration/src/main/resources/changelog.xml | 1 - .../changesets/embiggen_summary_workflow_name.xml | 51 ---------------------- 2 files changed, 52 deletions(-) delete mode 100644 database/migration/src/main/resources/changesets/embiggen_summary_workflow_name.xml diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml index 18a84c2ca..b2751b7cc 100644 --- a/database/migration/src/main/resources/changelog.xml +++ b/database/migration/src/main/resources/changelog.xml @@ -46,7 +46,6 @@ - diff --git a/database/migration/src/main/resources/changesets/embiggen_summary_workflow_name.xml b/database/migration/src/main/resources/changesets/embiggen_summary_workflow_name.xml deleted file mode 100644 index 8366a2481..000000000 --- a/database/migration/src/main/resources/changesets/embiggen_summary_workflow_name.xml +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - Either WORKFLOW_METADATA_SUMMARY or WORKFLOW_METADATA_SUMMARY_ENTRY must exist, but not both (Liquibase doesn't have an xor). - - - - - - - - - The WORKFLOW_METADATA_SUMMARY_ENTRY version of the embiggener. This should gracefully handle the absence of the table - in the event the table is still called WORKFLOW_METADATA_SUMMARY when this runs. - - - - - - - - - - The WORKFLOW_METADATA_SUMMARY version of the embiggener. This should gracefully handle the absence of the table - in the event the table has been renamed to WORKFLOW_METADATA_SUMMARY_ENTRY when this runs. - - - - - From 4b4016315ab1444326b8260cfd78c2f84b312dc5 Mon Sep 17 00:00:00 2001 From: "Francisco M. Casares" Date: Wed, 30 Nov 2016 16:02:56 -0800 Subject: [PATCH 079/375] [HtCondor] Semicolon in WDL command allows execution on host system (#1719) * Fixed issue related to semicolon in WDL command allows execution on host system. * Addding missing change for one unit test. --- core/src/main/resources/reference.conf | 2 +- .../backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 87bc710b9..10711adbf 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -259,7 +259,7 @@ backend { # #6. Job command. # docker { # #Allow soft links in dockerized jobs - # cmd = "docker run -w %s %s %s %s --rm %s %s" + # cmd = "docker run -w %s %s %s %s --rm %s /bin/bash -c \"%s\"" # defaultWorkingDir = "/workingDir/" # defaultOutputDir = "/output/" # } diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala index 8dcecba21..33bc5c71c 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala @@ -102,7 +102,7 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc | root = "local-cromwell-executions" | | docker { - | cmd = "docker run -w %s %s %s %s --rm %s %s" + | cmd = "docker run -w %s %s %s %s --rm %s /bin/bash -c \\"%s\\"" | defaultWorkingDir = "/workingDir/" | defaultOutputDir = "/output/" | } @@ -318,7 +318,7 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc assert(bashScript.contains("docker run -w /workingDir/ -v")) assert(bashScript.contains(":/workingDir/")) assert(bashScript.contains(":ro")) - assert(bashScript.contains("/call-hello/execution:/outputDir/ --rm ubuntu/latest echo")) + assert(bashScript.contains("/call-hello/execution:/outputDir/ --rm ubuntu/latest /bin/bash -c \"echo")) cleanUpJob(jobPaths) } @@ -403,7 +403,7 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc assert(bashScript.contains(":/workingDir/")) assert(bashScript.contains(tempDir1.toAbsolutePath.toString)) assert(bashScript.contains(tempDir2.toAbsolutePath.toString)) - assert(bashScript.contains("/call-hello/execution:/outputDir/ --rm ubuntu/latest echo")) + assert(bashScript.contains("/call-hello/execution:/outputDir/ --rm ubuntu/latest /bin/bash -c \"echo")) cleanUpJob(jobPaths) } From 422a4113cccf113d239751433cb8b76ba2fd216b Mon Sep 17 00:00:00 2001 From: Thib Date: Thu, 1 Dec 2016 10:16:25 -0500 Subject: [PATCH 080/375] JIT Declaration evaluation Closes #1577 Closes #687 Closes #1513 (#1711) * JIT Declaration evaluation --- CHANGELOG.md | 1 + .../cromwell/backend/io/WorkflowPathsSpec.scala | 4 +- core/src/main/scala/cromwell/core/JobKey.scala | 4 +- .../lifecycle/execution/ExecutionStore.scala | 21 ++-- .../lifecycle/execution/JobPreparationActor.scala | 2 +- .../workflow/lifecycle/execution/OutputStore.scala | 78 ++++++++---- .../execution/WorkflowExecutionActor.scala | 132 +++++++++++++++++---- .../execution/WorkflowExecutionActorData.scala | 16 ++- .../subworkflowstore/SubWorkflowStoreSpec.scala | 4 +- project/Dependencies.scala | 2 +- 10 files changed, 195 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c304a220..1c0c1acab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ * By default, `system.abort-jobs-on-terminate` is false when running `java -jar cromwell.jar server`, and true when running `java -jar cromwell.jar run `. * Enable WDL imports when running in Single Workflow Runner Mode. * Support for sub workflows +* Support declarations as graph nodes ## 0.22 diff --git a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala index edd86e500..36d774bf0 100644 --- a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala +++ b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala @@ -6,7 +6,7 @@ import cromwell.backend.{BackendJobBreadCrumb, BackendSpec, BackendWorkflowDescr import cromwell.core.{JobKey, WorkflowId} import org.mockito.Mockito._ import org.scalatest.{FlatSpec, Matchers} -import wdl4s.{Call, Scope, Workflow} +import wdl4s.{Call, Workflow} class WorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec { @@ -48,7 +48,7 @@ class WorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec { call2.unqualifiedName returns "call2" val jobKey = new JobKey { - override def scope: Scope = call1 + override def scope = call1 override def tag: String = "tag1" override def index: Option[Int] = Option(1) override def attempt: Int = 2 diff --git a/core/src/main/scala/cromwell/core/JobKey.scala b/core/src/main/scala/cromwell/core/JobKey.scala index a28b7150e..9fd22b31e 100644 --- a/core/src/main/scala/cromwell/core/JobKey.scala +++ b/core/src/main/scala/cromwell/core/JobKey.scala @@ -1,9 +1,9 @@ package cromwell.core -import wdl4s.Scope +import wdl4s.{GraphNode, Scope} trait JobKey { - def scope: Scope + def scope: Scope with GraphNode def index: Option[Int] def attempt: Int def tag: String diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala index b3f29928c..3b9d1af1f 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala @@ -4,20 +4,21 @@ import cromwell.backend.BackendJobDescriptorKey import cromwell.core.ExecutionStatus._ import cromwell.core.{CallKey, ExecutionStatus, JobKey} import cromwell.engine.workflow.lifecycle.execution.ExecutionStore.ExecutionStoreEntry -import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{CollectorKey, ScatterKey, SubWorkflowKey} +import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{apply => _, _} import wdl4s._ object ExecutionStore { def empty = ExecutionStore(Map.empty[JobKey, ExecutionStatus]) type ExecutionStoreEntry = (JobKey, ExecutionStatus) - def apply(workflow: Workflow) = { + def apply(workflow: Workflow, workflowCoercedInputs: WorkflowCoercedInputs) = { // Only add direct children to the store, the rest is dynamically created when necessary val keys = workflow.children map { case call: TaskCall => Option(BackendJobDescriptorKey(call, None, 1)) case call: WorkflowCall => Option(SubWorkflowKey(call, None, 1)) case scatter: Scatter => Option(ScatterKey(scatter)) - case _ => None // FIXME there are other types of scopes now (Declarations, Ifs) Figure out what to do with those + case declaration: Declaration => Option(DeclarationKey(declaration, None, workflowCoercedInputs)) + case _ => None // Ifs will need to be added here when supported } new ExecutionStore(keys.flatten.map(_ -> NotStarted).toMap) @@ -42,19 +43,15 @@ case class ExecutionStore(store: Map[JobKey, ExecutionStatus]) { def findShardEntries(key: CollectorKey): List[ExecutionStoreEntry] = store.toList filter { case (k: CallKey, v) => k.scope == key.scope && k.isShard + case (k: DeclarationKey, v) => k.scope == key.scope && k.isShard case _ => false } private def arePrerequisitesDone(key: JobKey): Boolean = { - val upstream = key.scope match { - case node: GraphNode => node.upstream collect { - // Only scatters and calls are in the execution store for now (not declarations) - // However declarations are nodes so they can be an upstream dependency - // We don't want to look for those in the execution store (yet ?) since upstreamEntry would return None - case n: Call => upstreamEntry(key, n) - case n: Scatter => upstreamEntry(key, n) - } - case _ => Set.empty + val upstream = key.scope.upstream collect { + case n: Call => upstreamEntry(key, n) + case n: Scatter => upstreamEntry(key, n) + case n: Declaration => upstreamEntry(key, n) } val downstream: List[(JobKey, ExecutionStatus)] = key match { diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala index 3cfe95075..467439622 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala @@ -40,7 +40,7 @@ abstract class CallPreparationActor(val workflowDescriptor: EngineWorkflowDescri call.evaluateTaskInputs( workflowDescriptor.backendDescriptor.inputs, expressionLanguageFunctions, - outputStore.fetchCallOutputEntries, + outputStore.fetchNodeOutputEntries, scatterMap ) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala index b85c467e5..02c0bc113 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala @@ -7,22 +7,22 @@ import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.Colle import wdl4s.types.{WdlArrayType, WdlType} import wdl4s.util.TryUtil import wdl4s.values.{WdlArray, WdlCallOutputsObject, WdlValue} -import wdl4s.{Call, Scope} +import wdl4s.{Call, Declaration, GraphNode, Scope} import scala.language.postfixOps import scala.util.{Failure, Success, Try} object OutputStore { case class OutputEntry(name: String, wdlType: WdlType, wdlValue: Option[WdlValue]) - case class OutputCallKey(call: Scope, index: ExecutionIndex) + case class OutputCallKey(call: Scope with GraphNode, index: ExecutionIndex) def empty = OutputStore(Map.empty) } -case class OutputStore(store: Map[OutputCallKey, Traversable[OutputEntry]]) { - def add(values: Map[OutputCallKey, Traversable[OutputEntry]]) = this.copy(store = store ++ values) +case class OutputStore(store: Map[OutputCallKey, List[OutputEntry]]) { + def add(values: Map[OutputCallKey, List[OutputEntry]]) = this.copy(store = store ++ values) - def fetchCallOutputEntries(call: Call, index: ExecutionIndex): Try[WdlCallOutputsObject] = { - def outputEntriesToMap(outputs: Traversable[OutputEntry]): Map[String, Try[WdlValue]] = { + def fetchNodeOutputEntries(node: GraphNode, index: ExecutionIndex): Try[WdlValue] = { + def outputEntriesToMap(outputs: List[OutputEntry]): Map[String, Try[WdlValue]] = { outputs map { output => output.wdlValue match { case Some(wdlValue) => output.name -> Success(wdlValue) @@ -31,27 +31,39 @@ case class OutputStore(store: Map[OutputCallKey, Traversable[OutputEntry]]) { } toMap } - store.get(OutputCallKey(call, index)) match { + def callOutputs(call: Call, outputs: List[OutputEntry]) = { + TryUtil.sequenceMap(outputEntriesToMap(outputs), s"Output fetching for call ${node.unqualifiedName}") map { outputsMap => + WdlCallOutputsObject(call, outputsMap) + } + } + + def declarationOutputs(declaration: Declaration, outputs: List[OutputEntry]) = { + outputs match { + case OutputEntry(name, _, Some(value)) :: Nil => Success(value) + case _ => Failure(new RuntimeException(s"Could not find value for declaration ${declaration.fullyQualifiedName}")) + } + } + + store.get(OutputCallKey(node, index)) match { case Some(outputs) => - TryUtil.sequenceMap(outputEntriesToMap(outputs), s"Output fetching for call ${call.unqualifiedName}") map { outputsMap => - WdlCallOutputsObject(call, outputsMap) + node match { + case call: Call => callOutputs(call, outputs) + case declaration: Declaration => declarationOutputs(declaration, outputs) + case other => Failure(new RuntimeException(s"Only Calls and Declarations are allowed in the OutputStore, found ${other.getClass.getSimpleName}")) } - case None => Failure(new RuntimeException(s"Could not find call ${call.unqualifiedName}")) + case None => Failure(new RuntimeException(s"Could not find scope ${node.unqualifiedName}")) } } - - /** - * Try to generate output for a collector call, by collecting outputs for all of its shards. - * It's fail-fast on shard output retrieval - */ - def generateCollectorOutput(collector: CollectorKey, - shards: Iterable[CallKey]): Try[CallOutputs] = Try { - val shardsOutputs = shards.toSeq sortBy { _.index.fromIndex } map { e => - fetchCallOutputEntries(e.scope, e.index) map { - _.outputs + + def collectCall(call: Call, sortedShards: Seq[JobKey]) = Try { + val shardsOutputs = sortedShards map { e => + fetchNodeOutputEntries(call, e.index) map { + case callOutputs: WdlCallOutputsObject => callOutputs.outputs + case _ => throw new RuntimeException("Call outputs should be a WdlCallOutputsObject") } getOrElse(throw new RuntimeException(s"Could not retrieve output for shard ${e.scope} #${e.index}")) } - collector.scope.outputs map { taskOutput => + + call.outputs map { taskOutput => val wdlValues = shardsOutputs.map( _.getOrElse(taskOutput.unqualifiedName, throw new RuntimeException(s"Could not retrieve output ${taskOutput.unqualifiedName}"))) val arrayOfValues = new WdlArray(WdlArrayType(taskOutput.wdlType), wdlValues) @@ -59,4 +71,28 @@ case class OutputStore(store: Map[OutputCallKey, Traversable[OutputEntry]]) { } toMap } + def collectDeclaration(declaration: Declaration, sortedShards: Seq[JobKey]) = Try { + val shardsOutputs = sortedShards map { e => + fetchNodeOutputEntries(declaration, e.index) getOrElse { + throw new RuntimeException(s"Could not retrieve output for shard ${e.scope} #${e.index}") + } + } + + Map(declaration.unqualifiedName -> JobOutput(WdlArray(WdlArrayType(declaration.wdlType), shardsOutputs))) + } + + /** + * Try to generate output for a collector call, by collecting outputs for all of its shards. + * It's fail-fast on shard output retrieval + */ + def generateCollectorOutput(collector: CollectorKey, + shards: Iterable[JobKey]): Try[CallOutputs] = { + lazy val sortedShards = shards.toSeq sortBy { _.index.fromIndex } + + collector.scope match { + case call: Call => collectCall(call, sortedShards) + case declaration: Declaration => collectDeclaration(declaration, sortedShards) + case other => Failure(new RuntimeException(s"Cannot retrieve outputs for ${other.fullyQualifiedName}")) + } + } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index 8eefb8c72..c3fb16303 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -62,7 +62,7 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, WorkflowExecutionPendingState, WorkflowExecutionActorData( workflowDescriptor, - executionStore = ExecutionStore(workflowDescriptor.backendDescriptor.workflow), + executionStore = ExecutionStore(workflowDescriptor.backendDescriptor.workflow, workflowDescriptor.workflowInputs), backendJobExecutionActors = Map.empty, engineJobExecutionActors = Map.empty, subWorkflowExecutionActors = Map.empty, @@ -100,6 +100,9 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, // Scatter case Event(ScatterCollectionSucceededResponse(jobKey, callOutputs), stateData) => handleCallSuccessful(jobKey, callOutputs, stateData, Map.empty) + // Declaration + case Event(DeclarationEvaluationSucceededResponse(jobKey, callOutputs), stateData) => + handleDeclarationEvaluationSuccessful(jobKey, callOutputs, stateData) // Failure // Initialization @@ -118,6 +121,8 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, case Event(SubWorkflowFailedResponse(jobKey, descendantJobKeys, reason), stateData) => pushFailedCallMetadata(jobKey, None, reason, retryableFailure = false) handleNonRetryableFailure(stateData, jobKey, reason, descendantJobKeys) + case Event(DeclarationEvaluationFailedResponse(jobKey, reason), stateData) => + handleDeclarationEvaluationFailure(jobKey, reason, stateData) } when(WorkflowExecutionAbortingState) { @@ -219,21 +224,31 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, } private def handleNonRetryableFailure(stateData: WorkflowExecutionActorData, failedJobKey: JobKey, reason: Throwable, jobExecutionMap: JobExecutionMap) = { - val mergedStateData = stateData.mergeExecutionDiff(WorkflowExecutionDiff(Map(failedJobKey -> ExecutionStatus.Failed))) - .removeCallExecutionActor(failedJobKey) - .addExecutions(jobExecutionMap) - + val newData = stateData + .removeCallExecutionActor(failedJobKey) + .addExecutions(jobExecutionMap) + + handleExecutionFailure(failedJobKey, newData, reason, jobExecutionMap) + } + + private def handleDeclarationEvaluationFailure(declarationKey: DeclarationKey, reason: Throwable, stateData: WorkflowExecutionActorData) = { + handleExecutionFailure(declarationKey, stateData, reason, Map.empty) + } + + private def handleExecutionFailure(failedJobKey: JobKey, data: WorkflowExecutionActorData, reason: Throwable, jobExecutionMap: JobExecutionMap) = { + val newData = data.executionFailed(failedJobKey) + if (workflowDescriptor.getWorkflowOption(WorkflowFailureMode).contains(ContinueWhilePossible.toString)) { - mergedStateData.workflowCompletionStatus match { + newData.workflowCompletionStatus match { case Some(completionStatus) if completionStatus == Failed => - context.parent ! WorkflowExecutionFailedResponse(stateData.jobExecutionMap, reason) - goto(WorkflowExecutionFailedState) using mergedStateData + context.parent ! WorkflowExecutionFailedResponse(newData.jobExecutionMap, reason) + goto(WorkflowExecutionFailedState) using newData case _ => - stay() using startRunnableScopes(mergedStateData) + stay() using startRunnableScopes(newData) } } else { - context.parent ! WorkflowExecutionFailedResponse(stateData.jobExecutionMap, reason) - goto(WorkflowExecutionFailedState) using mergedStateData + context.parent ! WorkflowExecutionFailedResponse(newData.jobExecutionMap, reason) + goto(WorkflowExecutionFailedState) using newData } } @@ -244,7 +259,7 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, val (response, finalState) = workflowDescriptor.workflow.evaluateOutputs( workflowDescriptor.inputs, data.expressionLanguageFunctions, - data.outputStore.fetchCallOutputEntries + data.outputStore.fetchNodeOutputEntries ) map { workflowOutputs => workflowLogger.info( s"""Workflow ${workflowDescriptor.workflow.unqualifiedName} complete. Final Outputs: @@ -278,17 +293,22 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, } private def handleCallSuccessful(jobKey: JobKey, outputs: CallOutputs, data: WorkflowExecutionActorData, jobExecutionMap: JobExecutionMap) = { - workflowLogger.debug(s"Job ${jobKey.tag} succeeded!") - val newData = data.callExecutionSuccess(jobKey, outputs).addExecutions(jobExecutionMap) - - newData.workflowCompletionStatus match { + handleExecutionSuccess(data.callExecutionSuccess(jobKey, outputs).addExecutions(jobExecutionMap)) + } + + private def handleDeclarationEvaluationSuccessful(key: DeclarationKey, value: WdlValue, data: WorkflowExecutionActorData) = { + handleExecutionSuccess(data.declarationEvaluationSuccess(key, value)) + } + + private def handleExecutionSuccess(data: WorkflowExecutionActorData) = { + data.workflowCompletionStatus match { case Some(ExecutionStatus.Done) => - handleWorkflowSuccessful(newData) + handleWorkflowSuccessful(data) case Some(sts) => context.parent ! WorkflowExecutionFailedResponse(data.jobExecutionMap, new Exception("One or more jobs failed in fail-slow mode")) - goto(WorkflowExecutionFailedState) using newData + goto(WorkflowExecutionFailedState) using data case _ => - stay() using startRunnableScopes(newData) + stay() using startRunnableScopes(data) } } @@ -320,6 +340,8 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, case k: ScatterKey => processRunnableScatter(k, data) case k: CollectorKey => processRunnableCollector(k, data) case k: SubWorkflowKey => processRunnableSubWorkflow(k, data) + case k: StaticDeclarationKey => processRunnableStaticDeclaration(k) + case k: DynamicDeclarationKey => processRunnableDynamicDeclaration(k, data) case k => val exception = new UnsupportedOperationException(s"Unknown entry in execution store: ${k.tag}") self ! JobInitializationFailed(k, exception) @@ -339,6 +361,32 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, } } + def processRunnableStaticDeclaration(declaration: StaticDeclarationKey) = { + self ! DeclarationEvaluationSucceededResponse(declaration, declaration.value) + Success(WorkflowExecutionDiff(Map(declaration -> ExecutionStatus.Running))) + } + + def processRunnableDynamicDeclaration(declaration: DynamicDeclarationKey, data: WorkflowExecutionActorData) = { + val scatterMap = declaration.index flatMap { i => + // Will need update for nested scatters + declaration.scope.ancestry collectFirst { case s: Scatter => Map(s -> i) } + } getOrElse Map.empty[Scatter, Int] + + val lookup = declaration.scope.lookupFunction( + workflowDescriptor.workflowInputs, + data.expressionLanguageFunctions, + data.outputStore.fetchNodeOutputEntries, + scatterMap + ) + + declaration.requiredExpression.evaluate(lookup, data.expressionLanguageFunctions) match { + case Success(result) => self ! DeclarationEvaluationSucceededResponse(declaration, result) + case Failure(ex) => self ! DeclarationEvaluationFailedResponse(declaration, ex) + } + + Success(WorkflowExecutionDiff(Map(declaration -> ExecutionStatus.Running))) + } + private def processRunnableJob(jobKey: BackendJobDescriptorKey, data: WorkflowExecutionActorData): Try[WorkflowExecutionDiff] = { workflowDescriptor.backendAssignments.get(jobKey.call) match { case None => @@ -384,17 +432,20 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, val lookup = scatterKey.scope.lookupFunction( workflowDescriptor.workflowInputs, data.expressionLanguageFunctions, - data.outputStore.fetchCallOutputEntries + data.outputStore.fetchNodeOutputEntries ) scatterKey.scope.collection.evaluate(lookup, data.expressionLanguageFunctions) map { - case a: WdlArray => WorkflowExecutionDiff(scatterKey.populate(a.value.size) + (scatterKey -> ExecutionStatus.Done)) + case a: WdlArray => WorkflowExecutionDiff(scatterKey.populate(a.value.size, workflowDescriptor.workflowInputs) + (scatterKey -> ExecutionStatus.Done)) case v: WdlValue => throw new RuntimeException("Scatter collection must evaluate to an array") } } private def processRunnableCollector(collector: CollectorKey, data: WorkflowExecutionActorData): Try[WorkflowExecutionDiff] = { - val shards = data.executionStore.findShardEntries(collector) collect { case (k: CallKey, v) if v == ExecutionStatus.Done => k } + val shards = data.executionStore.findShardEntries(collector) collect { + case (k: CallKey, v) if v == ExecutionStatus.Done => k + case (k: DynamicDeclarationKey, v) if v == ExecutionStatus.Done => k + } data.outputStore.generateCollectorOutput(collector, shards) match { case Failure(e) => Failure(new RuntimeException(s"Failed to collect output shards for call ${collector.tag}")) case Success(outputs) => self ! ScatterCollectionSucceededResponse(collector, outputs) @@ -464,6 +515,10 @@ object WorkflowExecutionActor { private case class ScatterCollectionFailedResponse(collectorKey: CollectorKey, throwable: Throwable) private case class ScatterCollectionSucceededResponse(collectorKey: CollectorKey, outputs: CallOutputs) + + private case class DeclarationEvaluationSucceededResponse(declarationKey: DeclarationKey, value: WdlValue) + + private case class DeclarationEvaluationFailedResponse(declarationKey: DeclarationKey, reason: Throwable) case class SubWorkflowSucceededResponse(key: SubWorkflowKey, jobExecutionMap: JobExecutionMap, outputs: CallOutputs) @@ -486,16 +541,16 @@ object WorkflowExecutionActor { * @param count Number of ways to scatter the children. * @return ExecutionStore of scattered children. */ - def populate(count: Int): Map[JobKey, ExecutionStatus.Value] = { + def populate(count: Int, workflowCoercedInputs: WorkflowCoercedInputs): Map[JobKey, ExecutionStatus.Value] = { val keys = this.scope.children flatMap { - explode(_, count) + explode(_, count, workflowCoercedInputs) } keys map { _ -> ExecutionStatus.NotStarted } toMap } - private def explode(scope: Scope, count: Int): Seq[JobKey] = { + private def explode(scope: Scope, count: Int, workflowCoercedInputs: WorkflowCoercedInputs): Seq[JobKey] = { scope match { case call: TaskCall => val shards = (0 until count) map { i => BackendJobDescriptorKey(call, Option(i), 1) } @@ -503,6 +558,9 @@ object WorkflowExecutionActor { case call: WorkflowCall => val shards = (0 until count) map { i => SubWorkflowKey(call, Option(i), 1) } shards :+ CollectorKey(call) + case declaration: Declaration => + val shards = (0 until count) map { i => DeclarationKey(declaration, Option(i), workflowCoercedInputs) } + shards :+ CollectorKey(declaration) case scatter: Scatter => throw new UnsupportedOperationException("Nested Scatters are not supported (yet) ... but you might try a sub workflow to achieve the same effect!") case e => @@ -512,7 +570,7 @@ object WorkflowExecutionActor { } // Represents a scatter collection for a call in the execution store - case class CollectorKey(scope: Call) extends JobKey { + case class CollectorKey(scope: Scope with GraphNode) extends JobKey { override val index = None override val attempt = 1 override val tag = s"Collector-${scope.unqualifiedName}" @@ -522,6 +580,28 @@ object WorkflowExecutionActor { override val tag = s"SubWorkflow-${scope.unqualifiedName}:${index.fromIndex}:$attempt" } + object DeclarationKey { + def apply(declaration: Declaration, index: ExecutionIndex, inputs: WorkflowCoercedInputs): DeclarationKey = { + inputs.find(_._1 == declaration.fullyQualifiedName) match { + case Some((_, value)) => StaticDeclarationKey(declaration, index, value) + case None => declaration.expression map { expression => + DynamicDeclarationKey(declaration, index, expression) + } getOrElse { + throw new RuntimeException(s"Found a declaration ${declaration.fullyQualifiedName} without expression and without input value. This should have been a validation error.") + } + } + } + } + + sealed trait DeclarationKey extends JobKey { + override val attempt = 1 + override val tag = s"Declaration-${scope.unqualifiedName}:${index.fromIndex}:$attempt" + } + + case class StaticDeclarationKey(scope: Declaration, index: ExecutionIndex, value: WdlValue) extends DeclarationKey + + case class DynamicDeclarationKey(scope: Declaration, index: ExecutionIndex, requiredExpression: WdlExpression) extends DeclarationKey + case class WorkflowExecutionException[T <: Throwable](exceptions: NonEmptyList[T]) extends ThrowableAggregation { override val throwables = exceptions.toList override val exceptionContext = s"WorkflowExecutionActor" diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala index 665ef93e9..de5c975a7 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala @@ -5,9 +5,10 @@ import cromwell.backend._ import cromwell.core.ExecutionStatus._ import cromwell.core._ import cromwell.engine.workflow.lifecycle.execution.OutputStore.{OutputCallKey, OutputEntry} -import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.SubWorkflowKey +import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{DeclarationKey, SubWorkflowKey} import cromwell.engine.{EngineWorkflowDescriptor, WdlFunctions} import cromwell.util.JsonFormatting.WdlValueJsonFormatter +import wdl4s.values.WdlValue import wdl4s.{GraphNode, Scope} object WorkflowExecutionDiff { @@ -57,6 +58,17 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript outputStore = outputStore.add(updateSymbolStoreEntry(jobKey, outputs)) ) } + + def declarationEvaluationSuccess(declarationKey: DeclarationKey, value: WdlValue) = { + val outputStoreKey = OutputCallKey(declarationKey.scope, declarationKey.index) + val outputStoreValue = OutputEntry(declarationKey.scope.unqualifiedName, value.wdlType, Option(value)) + this.copy( + executionStore = executionStore.add(Map(declarationKey -> Done)), + outputStore = outputStore.add(Map(outputStoreKey -> List(outputStoreValue))) + ) + } + + def executionFailed(jobKey: JobKey) = mergeExecutionDiff(WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Failed))) /** Add the outputs for the specified `JobKey` to the symbol cache. */ private def updateSymbolStoreEntry(jobKey: JobKey, outputs: CallOutputs) = { @@ -64,7 +76,7 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript case (name, value) => OutputEntry(name, value.wdlValue.wdlType, Option(value.wdlValue)) } - Map(OutputCallKey(jobKey.scope, jobKey.index) -> newOutputEntries) + Map(OutputCallKey(jobKey.scope, jobKey.index) -> newOutputEntries.toList) } /** Checks if the workflow is completed by scanning through the executionStore. diff --git a/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala b/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala index 7c11e4e6e..6d1007eaa 100644 --- a/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala +++ b/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala @@ -6,7 +6,7 @@ import cromwell.services.SingletonServicesStore import cromwell.subworkflowstore.SubWorkflowStoreActor._ import org.scalatest.Matchers import org.specs2.mock.Mockito -import wdl4s.{Scope, TaskCall, WdlExpression} +import wdl4s.{TaskCall, WdlExpression} import cromwell.core.ExecutionIndex._ import scala.concurrent.duration._ @@ -38,7 +38,7 @@ class SubWorkflowStoreSpec extends CromwellTestKitSpec with Matchers with Mockit val call = mock[TaskCall] call.fullyQualifiedName returns "foo.bar" val jobKey = new JobKey { - override def scope: Scope = call + override def scope = call override def index: Option[Int] = None override def attempt: Int = 0 override def tag: String = "foobar" diff --git a/project/Dependencies.scala b/project/Dependencies.scala index a2d1b91a7..c5fc8320d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -2,7 +2,7 @@ import sbt._ object Dependencies { lazy val lenthallV = "0.19" - lazy val wdl4sV = "0.7-799567f-SNAP" + lazy val wdl4sV = "0.7-0a41be3-SNAP" lazy val sprayV = "1.3.3" /* spray-json is an independent project from the "spray suite" From 624f492a4110800a691d9b3ecf8ad46c25686be0 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 1 Dec 2016 13:53:26 -0500 Subject: [PATCH 081/375] Quieten down that localization switchover --- .../sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala index 6cba209c1..188391c45 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala @@ -72,7 +72,7 @@ object SharedFileSystem extends StrictLogging { } private def logOnFailure(action: Try[Unit], actionLabel: String): Try[Unit] = { - if (action.isFailure) logger.warn(s"Localization via $actionLabel has failed: ${action.failed.get.getMessage}", action.failed.get) + if (action.isFailure) logger.warn(s"Localization via $actionLabel has failed: ${action.failed.get.getMessage}") action } From ba2538e1ea2a40b1f347d3ad3347c77a63593bfb Mon Sep 17 00:00:00 2001 From: Thib Date: Thu, 1 Dec 2016 16:40:21 -0500 Subject: [PATCH 082/375] Remove hard link localization from centaur local conf (#1724) * remove hard link localization from centaur local conf * limit number of concurrent jobs to 20 --- src/bin/travis/resources/local_centaur.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/bin/travis/resources/local_centaur.conf b/src/bin/travis/resources/local_centaur.conf index 68ba866bf..ef6df3975 100644 --- a/src/bin/travis/resources/local_centaur.conf +++ b/src/bin/travis/resources/local_centaur.conf @@ -16,3 +16,6 @@ spray.can { call-caching { enabled = true } + +backend.providers.Local.config.filesystems.local.localization = ["soft-link", "copy"] +backend.providers.Local.config.concurrent-job-limit = 20 From 9ddbce01162e4a031d4452c3d94279cfc68d9f71 Mon Sep 17 00:00:00 2001 From: Jeff Gentry Date: Fri, 2 Dec 2016 10:51:36 -0500 Subject: [PATCH 083/375] Allow for JES QPS to be tunable (#1718) * Allow for JES QPS to be tunable --- CHANGELOG.md | 1 + README.md | 3 ++ core/src/main/resources/reference.conf | 9 +++++ .../cromwell/backend/impl/jes/JesAttributes.scala | 8 +++- .../impl/jes/JesBackendLifecycleActorFactory.scala | 2 +- .../impl/jes/JesBackendSingletonActor.scala | 8 ++-- .../backend/impl/jes/JesConfiguration.scala | 1 + .../jes/statuspolling/JesApiQueryManager.scala | 6 +-- .../impl/jes/statuspolling/JesPollingActor.scala | 45 ++++++++++++++++++---- .../cromwell/backend/impl/jes/JesTestConfig.scala | 2 +- .../jes/statuspolling/JesApiQueryManagerSpec.scala | 9 +++-- .../jes/statuspolling/JesPollingActorSpec.scala | 19 ++++++--- 12 files changed, 86 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c0c1acab..ae57ce073 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 23 * The `meta` and `parameter_meta` blocks are now valid within `workflow` blocks, not just `task` +* The JES backend configuration now has an option `genomics-api-queries-per-100-seconds` to help tune the rate of batch polling against the JES servers * Added an option `call-caching.invalidate-bad-cache-results` (default: `true`). If true, Cromwell will invalidate cached results which have failed to copy as part of a cache hit. * Timing diagrams and metadata now receive more fine grained workflow states between submission and Running. * Support for the Pair WDL type (e.g. `Pair[Int, File] floo = (3, "gs://blar/blaz/qlux.txt")`) diff --git a/README.md b/README.md index 782d790e8..b5147558c 100644 --- a/README.md +++ b/README.md @@ -1112,6 +1112,7 @@ backend { config { project = "my-project" root = "gs://my-bucket" + genomics-api-queries-per-100-seconds = 1000 . . . @@ -1121,6 +1122,8 @@ backend { } ``` +If your project has API quotas other than the defaults set the `genomics-api-queries-per-100-seconds` value to be the lesser of the `Queries per 100 seconds per user` and `Queries per 100 seconds` quotas. This value will be used to help tune Cromwell's rate of interaction with JES. + ### Configuring Authentication The `google` stanza in the Cromwell configuration file defines how to authenticate to Google. There are four different diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 10711adbf..52df9e771 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -319,6 +319,15 @@ backend { # # Base bucket for workflow executions # root = "gs://my-cromwell-workflows-bucket" # + # # Set this to the lower of the two values "Queries per 100 seconds" and "Queries per 100 seconds per user" for + # # your project. + # # + # # Used to help determine maximum throughput to the Google Genomics API. Setting this value too low will + # # cause a drop in performance. Setting this value too high will cause QPS based locks from Google. + # # 1000 is the default "Queries per 100 seconds per user", 50000 is the default "Queries per 100 seconds" + # # See https://cloud.google.com/genomics/quotas for more information + # genomics-api-queries-per-100-seconds = 1000 + # # # Polling for completion backs-off gradually for slower-running jobs. # # This is the maximum polling interval (in seconds): # maximum-polling-interval = 600 diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala index be7bcffa3..ac2475e64 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala @@ -18,9 +18,11 @@ case class JesAttributes(project: String, auths: JesAuths, executionBucket: String, endpointUrl: URL, - maxPollingInterval: Int) + maxPollingInterval: Int, + qps: Int) object JesAttributes { + val GenomicsApiDefaultQps = 1000 private val jesKeys = Set( "project", @@ -48,11 +50,13 @@ object JesAttributes { val genomicsAuthName: ErrorOr[String] = backendConfig.validateString("genomics.auth") val gcsFilesystemAuthName: ErrorOr[String] = backendConfig.validateString("filesystems.gcs.auth") + val qps = backendConfig.as[Option[Int]]("genomics-api-queries-per-100-seconds").getOrElse(GenomicsApiDefaultQps) / 100 + (project |@| executionBucket |@| endpointUrl |@| genomicsAuthName |@| gcsFilesystemAuthName) map { (_, _, _, _, _) } flatMap { case (p, b, u, genomicsName, gcsName) => (googleConfig.auth(genomicsName) |@| googleConfig.auth(gcsName)) map { case (genomicsAuth, gcsAuth) => - JesAttributes(p, computeServiceAccount, JesAuths(genomicsAuth, gcsAuth), b, u, maxPollingInterval) + JesAttributes(p, computeServiceAccount, JesAuths(genomicsAuth, gcsAuth), b, u, maxPollingInterval, qps) } } match { case Valid(r) => r diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala index 37271286c..f61769379 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala @@ -77,7 +77,7 @@ case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor initializationData.toJes.get.workflowPaths.workflowRoot } - override def backendSingletonActorProps = Option(JesBackendSingletonActor.props()) + override def backendSingletonActorProps = Option(JesBackendSingletonActor.props(jesConfiguration.qps)) override lazy val fileHashingFunction: Option[FileHashingFunction] = Option(FileHashingFunction(JesBackendFileHashing.getCrc32c)) } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala index 3b830107a..5c1275ae1 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala @@ -1,12 +1,12 @@ package cromwell.backend.impl.jes import akka.actor.{Actor, ActorLogging, Props} -import cromwell.backend.impl.jes.statuspolling.{JesApiQueryManager} +import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll -class JesBackendSingletonActor extends Actor with ActorLogging { +final case class JesBackendSingletonActor(qps: Int) extends Actor with ActorLogging { - val pollingActor = context.actorOf(JesApiQueryManager.props) + val pollingActor = context.actorOf(JesApiQueryManager.props(qps)) override def receive = { case poll: DoPoll => @@ -16,5 +16,5 @@ class JesBackendSingletonActor extends Actor with ActorLogging { } object JesBackendSingletonActor { - def props(): Props = Props(new JesBackendSingletonActor()) + def props(qps: Int): Props = Props(JesBackendSingletonActor(qps)) } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala index bd0e14b1c..20ec8130d 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala @@ -33,4 +33,5 @@ class JesConfiguration(val configurationDescriptor: BackendConfigurationDescript val genomicsFactory = GenomicsFactory(googleConfig.applicationName, jesAuths.genomics, jesAttributes.endpointUrl) val dockerCredentials = DockerConfiguration.build(configurationDescriptor.backendConfig).dockerCredentials map JesDockerCredentials.apply val needAuthFileUpload = jesAuths.gcs.requiresAuthFile || dockerCredentials.isDefined + val qps = jesAttributes.qps } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala index 2e2d254e2..12fe055dc 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala @@ -11,7 +11,7 @@ import scala.collection.immutable.Queue * Currently, just holds a set of JES status poll requests until a PollingActor pulls the work. * TODO: Could eventually move all of the JES queries into a single work-pulling model. */ -class JesApiQueryManager extends Actor with ActorLogging { +class JesApiQueryManager(val qps: Int) extends Actor with ActorLogging { // workQueue is protected for the unit tests, not intended to be generally overridden protected[statuspolling] var workQueue: Queue[JesStatusPollQuery] = Queue.empty @@ -19,7 +19,7 @@ class JesApiQueryManager extends Actor with ActorLogging { // If the statusPoller dies, we want to stop it and handle the termination ourselves. override val supervisorStrategy = SupervisorStrategy.stoppingStrategy - private def statusPollerProps = JesPollingActor.props(self) + private def statusPollerProps = JesPollingActor.props(self, qps) // statusPoller is protected for the unit tests, not intended to be generally overridden protected[statuspolling] var statusPoller: ActorRef = _ @@ -99,7 +99,7 @@ class JesApiQueryManager extends Actor with ActorLogging { object JesApiQueryManager { - def props: Props = Props(new JesApiQueryManager) + def props(qps: Int): Props = Props(new JesApiQueryManager(qps)) /** * Poll the job represented by the Run. diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala index 39329513d..31a9d114a 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala @@ -7,7 +7,7 @@ import com.google.api.client.googleapis.batch.json.JsonBatchCallback import com.google.api.client.googleapis.json.GoogleJsonError import com.google.api.client.http.HttpHeaders import com.google.api.services.genomics.model.Operation -import cromwell.backend.impl.jes.Run +import cromwell.backend.impl.jes.{JesAttributes, Run} import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.{JesPollingWorkBatch, JesStatusPollQuery, NoWorkToDo} import cromwell.backend.impl.jes.statuspolling.JesPollingActor._ @@ -19,12 +19,11 @@ import scala.concurrent.duration._ /** * Polls JES for status. Pipes the results back (so expect either a RunStatus or a akka.actor.Status.Failure). */ -class JesPollingActor(pollingManager: ActorRef) extends Actor with ActorLogging { +class JesPollingActor(val pollingManager: ActorRef, val qps: Int) extends Actor with ActorLogging { + // The interval to delay between submitting each batch + lazy val batchInterval = determineBatchInterval(determineEffectiveQps(qps)) + log.debug("JES batch polling interval is {}", batchInterval) - // We want to query at just under our fixed JES QPS limit of 20 per second. That should hopefully allow some room at the edges - // for things like new calls, etc. - val MaxBatchSize = 100 - val BatchInterval = 5.5.seconds self ! NoWorkToDo // Starts the check-for-work cycle implicit val ec: ExecutionContext = context.dispatcher @@ -109,13 +108,43 @@ class JesPollingActor(pollingManager: ActorRef) extends Actor with ActorLogging * Warning: Only use this from inside a receive method. */ private def scheduleCheckForWork(): Unit = { - context.system.scheduler.scheduleOnce(BatchInterval) { pollingManager ! JesApiQueryManager.RequestJesPollingWork(MaxBatchSize) } + context.system.scheduler.scheduleOnce(batchInterval) { pollingManager ! JesApiQueryManager.RequestJesPollingWork(MaxBatchSize) } () } + + /** + * We don't want to allow non-positive QPS values. Catch these instances and replace them with a sensible default. + * Here we're using the default value coming from JES itself + */ + private def determineEffectiveQps(qps: Int): Int = { + if (qps > 0) qps + else { + val defaultQps = JesAttributes.GenomicsApiDefaultQps + log.warning("Supplied QPS for Google Genomics API was not positive, value was {} using {} instead", qps, defaultQps) + defaultQps + } + } } object JesPollingActor { - def props(pollingManager: ActorRef) = Props(new JesPollingActor(pollingManager)) + def props(pollingManager: ActorRef, qps: Int) = Props(new JesPollingActor(pollingManager, qps)) + + // The Batch API limits us to 100 at a time + val MaxBatchSize = 100 + + /** + * Given the Genomics API queries per 100 seconds and given MaxBatchSize will determine a batch interval which + * is at 90% of the quota. The (still crude) delta is to provide some room at the edges for things like new + * calls, etc. + * + * Forcing the minimum value to be 1 second, for now it seems unlikely to matter and it makes testing a bit + * easier + */ + def determineBatchInterval(qps: Int): FiniteDuration = { + val maxInterval = MaxBatchSize / qps.toDouble // Force this to be floating point in case the value is < 1 + val interval = Math.max(maxInterval * 0.9, 1) + interval.seconds + } final case class JesPollFailed(e: GoogleJsonError, responseHeaders: HttpHeaders) } diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesTestConfig.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesTestConfig.scala index 03f17d65b..25f061387 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesTestConfig.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesTestConfig.scala @@ -48,5 +48,5 @@ object JesTestConfig { val JesBackendConfig = ConfigFactory.parseString(JesBackendConfigString) val JesGlobalConfig = ConfigFactory.parseString(JesGlobalConfigString) - val JesBackendConfigurationDescriptor = new BackendConfigurationDescriptor(JesBackendConfig, JesGlobalConfig) + val JesBackendConfigurationDescriptor = BackendConfigurationDescriptor(JesBackendConfig, JesGlobalConfig) } diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala index d98316e81..c7434419a 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala @@ -2,7 +2,7 @@ package cromwell.backend.impl.jes.statuspolling import akka.actor.{ActorRef, Props} import akka.testkit.{TestActorRef, TestProbe} -import cromwell.backend.impl.jes.Run +import cromwell.backend.impl.jes.{JesConfiguration, Run} import cromwell.core.TestKitSuite import org.scalatest.{FlatSpecLike, Matchers} @@ -108,7 +108,7 @@ object JesApiQueryManagerSpec { /** * This test class allows us to hook into the JesApiQueryManager's makeStatusPoller and provide our own TestProbes instead */ -class TestJesApiQueryManager(statusPollerProbes: ActorRef*) extends JesApiQueryManager { +class TestJesApiQueryManager(qps: Int, statusPollerProbes: ActorRef*) extends JesApiQueryManager(qps) { var testProbes: Queue[ActorRef] = _ var testPollerCreations: Int = _ @@ -137,5 +137,8 @@ class TestJesApiQueryManager(statusPollerProbes: ActorRef*) extends JesApiQueryM } object TestJesApiQueryManager { - def props(statusPollers: ActorRef*): Props = Props(new TestJesApiQueryManager(statusPollers: _*)) + import cromwell.backend.impl.jes.JesTestConfig.JesBackendConfigurationDescriptor + val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) + + def props(statusPollers: ActorRef*): Props = Props(new TestJesApiQueryManager(jesConfiguration.qps, statusPollers: _*)) } diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala index b861cbf0f..aa29b0df3 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala @@ -13,7 +13,7 @@ import com.google.api.client.googleapis.batch.BatchRequest import com.google.api.client.googleapis.batch.json.JsonBatchCallback import com.google.api.client.googleapis.json.GoogleJsonError import com.google.api.services.genomics.model.Operation -import cromwell.backend.impl.jes.{Run, RunStatus} +import cromwell.backend.impl.jes.{JesConfiguration, Run, RunStatus} import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.JesStatusPollQuery import cromwell.backend.impl.jes.statuspolling.JesPollingActor.JesPollFailed import cromwell.backend.impl.jes.statuspolling.TestJesPollingActor.{CallbackFailure, CallbackSuccess, JesBatchCallbackResponse} @@ -29,9 +29,17 @@ class JesPollingActorSpec extends TestKitSuite("JesPollingActor") with FlatSpecL implicit val DefaultPatienceConfig = PatienceConfig(TestExecutionTimeout) val AwaitAlmostNothing = 30.milliseconds.dilated + import cromwell.backend.impl.jes.JesTestConfig.JesBackendConfigurationDescriptor + val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) + var managerProbe: TestProbe = _ var jpActor: TestActorRef[TestJesPollingActor] = _ + it should "correctly calculate batch intervals" in { + JesPollingActor.determineBatchInterval(10) shouldBe 9.seconds + JesPollingActor.determineBatchInterval(100) shouldBe 1.second + } + it should "query for work and wait for a reply" in { managerProbe.expectMsgClass(max = TestExecutionTimeout, c = classOf[JesApiQueryManager.RequestJesPollingWork]) managerProbe.expectNoMsg(max = AwaitAlmostNothing) @@ -77,7 +85,7 @@ class JesPollingActorSpec extends TestKitSuite("JesPollingActor") with FlatSpecL before { managerProbe = TestProbe() - jpActor = TestActorRef(TestJesPollingActor.props(managerProbe.ref), managerProbe.ref) + jpActor = TestActorRef(TestJesPollingActor.props(managerProbe.ref, jesConfiguration), managerProbe.ref) } } @@ -94,8 +102,9 @@ object JesPollingActorSpec extends Mockito { * - Mocks out the methods which actually call out to JES, and allows the callbacks to be triggered in a testable way * - Also waits a **lot** less time before polls! */ -class TestJesPollingActor(manager: ActorRef) extends JesPollingActor(manager) with Mockito { - override val BatchInterval = 10.milliseconds +class TestJesPollingActor(manager: ActorRef, qps: Int) extends JesPollingActor(manager, qps) with Mockito { + + override lazy val batchInterval = 10.milliseconds var operationStatusResponses: Queue[RunStatus] = Queue.empty var resultHandlers: Queue[JsonBatchCallback[Operation]] = Queue.empty @@ -123,7 +132,7 @@ class TestJesPollingActor(manager: ActorRef) extends JesPollingActor(manager) wi } object TestJesPollingActor { - def props(manager: ActorRef) = Props(new TestJesPollingActor(manager)) + def props(manager: ActorRef, jesConfiguration: JesConfiguration) = Props(new TestJesPollingActor(manager, jesConfiguration.qps)) sealed trait JesBatchCallbackResponse case object CallbackSuccess extends JesBatchCallbackResponse From 3c732e058eab88086bf839d3eaf2dbc8ca55194e Mon Sep 17 00:00:00 2001 From: Thib Date: Fri, 2 Dec 2016 14:35:15 -0500 Subject: [PATCH 084/375] Add some docs regarding workflow outputs in the changelog (#1728) * add some docs regarding workflow outputs in the changelog --- CHANGELOG.md | 99 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae57ce073..ec608444f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,106 @@ * `transpose: (Array[Array[X]]) => Array[Array[X]]` compute the matrix transpose for a 2D array. Assumes each inner array has the same length. * By default, `system.abort-jobs-on-terminate` is false when running `java -jar cromwell.jar server`, and true when running `java -jar cromwell.jar run `. * Enable WDL imports when running in Single Workflow Runner Mode. -* Support for sub workflows +* Support for sub workflows (see [Annex A](#annex-a---workflow-outputs)) * Support declarations as graph nodes +### Annex A - Workflow outputs + +The WDL specification has changed regarding [workflow outputs](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#outputs) to accommodate sub workflows. +This change is backward compatible in terms of runnable WDLs (WDL files using the deprecated workflow outputs syntax will still run the same). +The only visible change lies in the metadata (as well as the console output in single workflow mode, when workflow outputs are printed out at the end of a successful workflow). + +TL;DR Unless you are parsing or manipulating the "key" by which workflow outputs are referenced in the metadata (and/or the console output for single workflow mode), you can skip the following explanation. + +*Metadata Response* +``` +{ + ... + outputs { + "task_output_1": "hello", + "task_output_2": "world" + ^ + If you don't manipulate this part of the metadata, then skip this section + } +} +``` + +In order to maintain backward compatibility, workflow outputs expressed with the deprecated syntax are "expanded" to the new syntax. Here is an example: + +``` +task t { + command { + #do something + } + output { + String out1 = "hello" + String out2 = "world" + } +} +``` + +``` + workflow old_syntax { + call t + output { + t.* + } + } +``` + +``` + workflow new_syntax { + call t + output { + String wf_out1 = t.out1 + String wf_out2 = t.out2 + } + } +``` + +The new syntax allows for type checking of the outputs as well as expressions. It also allows for explicitly naming to the outputs. +The old syntax doesn't give the ability to name workflow outputs. For consistency reasons, Cromwell will generate a "new syntax" workflow output for each task output, and name them. +Their name will be generated using their FQN, which would give + +``` +output { + String w.t.out1 = t.out1 + String w.t.out2 = t.out2 +} +``` + +However as the FQN separator is `.`, the name itself cannot contain any `.`. +For that reason, `.` are replaced with `_` : + +*Old syntax expanded to new syntax* +``` +output { + String w_t_out1 = t.out1 + String w_t_out2 = t.out2 +} +``` + +The consequence is that the workflow outputs section of the metadata for `old_syntax` would previously look like + + ``` + outputs { + "w.t.out1": "hello", + "w.t.out2": "hello" + } + ``` + +but it will now look like + +``` + outputs { + "w_t_out1": "hello", + "w_t_out2": "hello" + } +``` + +The same applies for the console output of a workflow run in single workflow mode. + + ## 0.22 * Improved retries for Call Caching and general bug fixes. From 42ca957057c52015df75fd4a45b2bdd5a27717c0 Mon Sep 17 00:00:00 2001 From: Jeff Gentry Date: Fri, 2 Dec 2016 14:52:51 -0500 Subject: [PATCH 085/375] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec608444f..73b2a1568 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## 23 * The `meta` and `parameter_meta` blocks are now valid within `workflow` blocks, not just `task` -* The JES backend configuration now has an option `genomics-api-queries-per-100-seconds` to help tune the rate of batch polling against the JES servers +* The JES backend configuration now has an option `genomics-api-queries-per-100-seconds` to help tune the rate of batch polling against the JES servers. Users with quotas larger than default should make sure to set this value. * Added an option `call-caching.invalidate-bad-cache-results` (default: `true`). If true, Cromwell will invalidate cached results which have failed to copy as part of a cache hit. * Timing diagrams and metadata now receive more fine grained workflow states between submission and Running. * Support for the Pair WDL type (e.g. `Pair[Int, File] floo = (3, "gs://blar/blaz/qlux.txt")`) From 2074a0eae5f2f547fc410c060b3dbe10f0b18943 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Mon, 21 Nov 2016 22:35:30 -0500 Subject: [PATCH 086/375] Imports in server mode --- .../backend/BackendJobExecutionActor.scala | 6 +- .../test/scala/cromwell/backend/BackendSpec.scala | 12 ++- .../scala/cromwell/core/WorkflowSourceFiles.scala | 29 ------ .../core/WorkflowSourceFilesCollection.scala | 43 +++++++++ core/src/test/scala/cromwell/util/SampleWdl.scala | 4 +- .../migration/src/main/resources/changelog.xml | 1 + .../changesets/workflow_store_imports_file.xml | 15 +++ .../slick/tables/WorkflowStoreEntryComponent.scala | 6 +- .../database/sql/tables/WorkflowStoreEntry.scala | 3 +- engine/src/main/resources/swagger/cromwell.yaml | 10 ++ .../workflow/SingleWorkflowRunnerActor.scala | 10 +- .../cromwell/engine/workflow/WorkflowActor.scala | 10 +- .../engine/workflow/WorkflowManagerActor.scala | 10 +- .../MaterializeWorkflowDescriptorActor.scala | 83 ++++++++++++++-- .../execution/SubWorkflowExecutionActor.scala | 7 +- .../execution/WorkflowExecutionActor.scala | 12 ++- .../execution/WorkflowExecutionActorData.scala | 16 ++-- .../workflow/workflowstore/SqlWorkflowStore.scala | 12 ++- .../workflowstore/WorkflowStoreActor.scala | 7 +- .../scala/cromwell/server/CromwellRootActor.scala | 3 +- .../scala/cromwell/server/CromwellServer.scala | 1 + .../cromwell/webservice/CromwellApiHandler.scala | 10 +- .../cromwell/webservice/CromwellApiService.scala | 104 +++++++++++++++------ .../cromwell/webservice/WorkflowJsonSupport.scala | 16 +++- .../test/scala/cromwell/ArrayWorkflowSpec.scala | 4 +- .../test/scala/cromwell/CromwellTestKitSpec.scala | 5 +- .../scala/cromwell/DeclarationWorkflowSpec.scala | 4 +- .../src/test/scala/cromwell/MapWorkflowSpec.scala | 4 +- .../test/scala/cromwell/RestartWorkflowSpec.scala | 2 +- .../scala/cromwell/SimpleWorkflowActorSpec.scala | 7 +- .../workflow/SingleWorkflowRunnerActorSpec.scala | 3 +- .../engine/workflow/WorkflowActorSpec.scala | 2 +- .../workflow/WorkflowDescriptorBuilder.scala | 2 +- .../MaterializeWorkflowDescriptorActorSpec.scala | 50 +++++----- .../subworkflowstore/SubWorkflowStoreSpec.scala | 4 +- .../webservice/CromwellApiServiceSpec.scala | 10 +- project/Dependencies.scala | 2 +- src/main/scala/cromwell/CromwellCommandLine.scala | 29 +----- .../scala/cromwell/CromwellCommandLineSpec.scala | 20 +--- .../jes/JesAsyncBackendJobExecutionActorSpec.scala | 24 ++--- .../SharedFileSystemJobExecutionActorSpec.scala | 2 +- .../impl/spark/SparkRuntimeAttributesSpec.scala | 2 +- 42 files changed, 378 insertions(+), 228 deletions(-) delete mode 100644 core/src/main/scala/cromwell/core/WorkflowSourceFiles.scala create mode 100644 core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala create mode 100644 database/migration/src/main/resources/changesets/workflow_store_imports_file.xml diff --git a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala index 897e782f2..816f43da9 100644 --- a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala @@ -7,7 +7,7 @@ import akka.event.LoggingReceive import cromwell.backend.BackendJobExecutionActor._ import cromwell.backend.BackendLifecycleActor._ import cromwell.backend.wdl.OutputEvaluator -import cromwell.core.{CallOutputs, ExecutionEvent} +import cromwell.core.{CallOutputs, ExecutionEvent, JobKey} import wdl4s.expression.WdlStandardLibraryFunctions import wdl4s.values.WdlValue @@ -24,11 +24,11 @@ object BackendJobExecutionActor { // Responses sealed trait BackendJobExecutionActorResponse extends BackendWorkflowLifecycleActorResponse - sealed trait BackendJobExecutionResponse extends BackendJobExecutionActorResponse { def jobKey: BackendJobDescriptorKey } + sealed trait BackendJobExecutionResponse extends BackendJobExecutionActorResponse { def jobKey: JobKey } case class JobSucceededResponse(jobKey: BackendJobDescriptorKey, returnCode: Option[Int], jobOutputs: CallOutputs, jobDetritusFiles: Option[Map[String, Path]], executionEvents: Seq[ExecutionEvent]) extends BackendJobExecutionResponse case class AbortedResponse(jobKey: BackendJobDescriptorKey) extends BackendJobExecutionResponse sealed trait BackendJobFailedResponse extends BackendJobExecutionResponse { def throwable: Throwable; def returnCode: Option[Int] } - case class JobFailedNonRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse + case class JobFailedNonRetryableResponse(jobKey: JobKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse case class JobFailedRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse } diff --git a/backend/src/test/scala/cromwell/backend/BackendSpec.scala b/backend/src/test/scala/cromwell/backend/BackendSpec.scala index cfd73f084..7e2354bd5 100644 --- a/backend/src/test/scala/cromwell/backend/BackendSpec.scala +++ b/backend/src/test/scala/cromwell/backend/BackendSpec.scala @@ -4,6 +4,7 @@ import com.typesafe.config.ConfigFactory import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobFailedNonRetryableResponse, JobFailedRetryableResponse, JobSucceededResponse} import cromwell.backend.io.TestWorkflows._ import cromwell.core.{WorkflowId, WorkflowOptions} +import wdl4s.util.AggregatedException import org.scalatest.Matchers import org.scalatest.concurrent.ScalaFutures import org.scalatest.time.{Millis, Seconds, Span} @@ -27,7 +28,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito { runtime: String = "") = { BackendWorkflowDescriptor( WorkflowId.randomId(), - WdlNamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime)).workflow, + WdlNamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime), Seq.empty[ImportResolver]).workflow, inputs, options ) @@ -93,7 +94,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito { } case (JobFailedNonRetryableResponse(_, failure, _), JobFailedNonRetryableResponse(_, expectedFailure, _)) => failure.getClass shouldBe expectedFailure.getClass - failure.getMessage should include(expectedFailure.getMessage) + concatenateCauseMessages(failure) should include(expectedFailure.getMessage) case (JobFailedRetryableResponse(_, failure, _), JobFailedRetryableResponse(_, expectedFailure, _)) => failure.getClass shouldBe expectedFailure.getClass case (response, expectation) => @@ -101,6 +102,13 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito { } } + private def concatenateCauseMessages(t: Throwable): String = t match { + case null => "" + case ae: AggregatedException => ae.getMessage + ae.exceptions.map(concatenateCauseMessages(_)).mkString + concatenateCauseMessages(ae.getCause) + case other: Throwable => other.getMessage + concatenateCauseMessages(t.getCause) + } + + def executeJobAndAssertOutputs(backend: BackendJobExecutionActor, expectedResponse: BackendJobExecutionResponse) = { whenReady(backend.execute) { executionResponse => assertResponse(executionResponse, expectedResponse) diff --git a/core/src/main/scala/cromwell/core/WorkflowSourceFiles.scala b/core/src/main/scala/cromwell/core/WorkflowSourceFiles.scala deleted file mode 100644 index 44bd23cc7..000000000 --- a/core/src/main/scala/cromwell/core/WorkflowSourceFiles.scala +++ /dev/null @@ -1,29 +0,0 @@ -package cromwell.core - -import better.files.File -import wdl4s.{WdlJson, WdlSource} - -/** - * Represents the collection of source files that a user submits to run a workflow - */ - -sealed trait WorkflowSourceFilesCollection { - def wdlSource: WdlSource - def inputsJson: WdlJson - def workflowOptionsJson: WorkflowOptionsJson - - def copyOptions(workflowOptions: WorkflowOptionsJson) = this match { - case w: WorkflowSourceFiles => WorkflowSourceFiles(w.wdlSource, w.inputsJson, workflowOptions) - case w: WorkflowSourceFilesWithImports => WorkflowSourceFilesWithImports(w.wdlSource, w.inputsJson, workflowOptions, w.importsFile) - } - -} - -final case class WorkflowSourceFiles(wdlSource: WdlSource, - inputsJson: WdlJson, - workflowOptionsJson: WorkflowOptionsJson) extends WorkflowSourceFilesCollection - -final case class WorkflowSourceFilesWithImports(wdlSource: WdlSource, - inputsJson: WdlJson, - workflowOptionsJson: WorkflowOptionsJson, - importsFile: File) extends WorkflowSourceFilesCollection diff --git a/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala b/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala new file mode 100644 index 000000000..ac7a7d6aa --- /dev/null +++ b/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala @@ -0,0 +1,43 @@ +package cromwell.core + +import wdl4s.{WdlJson, WdlSource} + +/** + * Represents the collection of source files that a user submits to run a workflow + */ + +sealed trait WorkflowSourceFilesCollection { + def wdlSource: WdlSource + def inputsJson: WdlJson + def workflowOptionsJson: WorkflowOptionsJson + def importsZipFileOption: Option[Array[Byte]] = this match { + case _: WorkflowSourceFilesWithoutImports => None + case WorkflowSourceFilesWithDependenciesZip(_, _, _, importsZip) => Option(importsZip) // i.e. Some(importsZip) if our wiring is correct + } + + def copyOptions(workflowOptions: WorkflowOptionsJson) = this match { + case w: WorkflowSourceFilesWithoutImports => WorkflowSourceFilesWithoutImports(w.wdlSource, w.inputsJson, workflowOptions) + case w: WorkflowSourceFilesWithDependenciesZip => WorkflowSourceFilesWithDependenciesZip(w.wdlSource, w.inputsJson, workflowOptions, w.importsZip) + } +} + +object WorkflowSourceFilesCollection { + def apply(wdlSource: WdlSource, + inputsJson: WdlJson, + workflowOptionsJson: WorkflowOptionsJson, + importsFile: Option[Array[Byte]]): WorkflowSourceFilesCollection = importsFile match { + case Some(imports) => WorkflowSourceFilesWithDependenciesZip(wdlSource, inputsJson, workflowOptionsJson, imports) + case None => WorkflowSourceFilesWithoutImports(wdlSource, inputsJson, workflowOptionsJson) + } +} + +final case class WorkflowSourceFilesWithoutImports(wdlSource: WdlSource, + inputsJson: WdlJson, + workflowOptionsJson: WorkflowOptionsJson) extends WorkflowSourceFilesCollection + +final case class WorkflowSourceFilesWithDependenciesZip(wdlSource: WdlSource, + inputsJson: WdlJson, + workflowOptionsJson: WorkflowOptionsJson, + importsZip: Array[Byte]) extends WorkflowSourceFilesCollection { + override def toString = s"WorkflowSourceFilesWithDependenciesZip($wdlSource, $inputsJson, $workflowOptionsJson, <>)" +} diff --git a/core/src/test/scala/cromwell/util/SampleWdl.scala b/core/src/test/scala/cromwell/util/SampleWdl.scala index 677dfebf5..2e4d77399 100644 --- a/core/src/test/scala/cromwell/util/SampleWdl.scala +++ b/core/src/test/scala/cromwell/util/SampleWdl.scala @@ -4,7 +4,7 @@ import java.nio.file.{Files, Path} import java.util.UUID import better.files._ -import cromwell.core.{WorkflowSourceFiles} +import cromwell.core.{WorkflowSourceFilesWithoutImports} import spray.json._ import wdl4s._ import wdl4s.types.{WdlArrayType, WdlStringType} @@ -15,7 +15,7 @@ import scala.language.postfixOps trait SampleWdl extends TestFileUtil { def wdlSource(runtime: String = ""): WdlSource def asWorkflowSources(runtime: String = "", workflowOptions: String = "{}") = - WorkflowSourceFiles(wdlSource(runtime), wdlJson, workflowOptions) + WorkflowSourceFilesWithoutImports(wdlSource(runtime), wdlJson, workflowOptions) val rawInputs: WorkflowRawInputs def name = getClass.getSimpleName.stripSuffix("$") diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml index b2751b7cc..c67137ae0 100644 --- a/database/migration/src/main/resources/changelog.xml +++ b/database/migration/src/main/resources/changelog.xml @@ -53,6 +53,7 @@ +