From e56c9e8a0b33f5f99c864c4de73283c4c64a0af7 Mon Sep 17 00:00:00 2001 From: "Francisco M. Casares" Date: Thu, 22 Sep 2016 16:24:46 -0700 Subject: [PATCH 01/23] Added configurable default working and output folders. (#1471) --- core/src/main/resources/reference.conf | 13 ++++++++----- .../impl/htcondor/HtCondorJobExecutionActor.scala | 7 +++++-- .../htcondor/HtCondorJobExecutionActorSpec.scala | 22 +++++++++++++--------- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 795103bad..02490fb28 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -228,13 +228,16 @@ backend { # # #Placeholders: # #1. Working directory. - # #2. Inputs volumes. - # #3. Output volume. - # #4. Docker image. - # #5. Job command. + # #2. Working directory volume. + # #3. Inputs volumes. + # #4. Output volume. + # #5. Docker image. + # #6. Job command. # docker { # #Allow soft links in dockerized jobs - # cmd = "docker run -w %s %s %s --rm %s %s" + # cmd = "docker run -w %s %s %s %s --rm %s %s" + # defaultWorkingDir = "/workingDir/" + # defaultOutputDir = "/output/" # } # # cache { diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala index f30111b3d..870b160cb 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala @@ -319,12 +319,15 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor log.debug("{} List of input volumes: {}", tag, dockerInputDataVol.mkString(",")) val dockerCmd = configurationDescriptor.backendConfig.getString("docker.cmd") + val defaultWorkingDir = configurationDescriptor.backendConfig.getString("docker.defaultWorkingDir") + val defaultOutputDir = configurationDescriptor.backendConfig.getString("docker.defaultOutputDir") val dockerVolume = "-v %s:%s" val dockerVolumeInputs = s"$dockerVolume:ro" // `v.get` is safe below since we filtered the list earlier with only defined elements val inputVolumes = dockerInputDataVol.distinct.map(v => dockerVolumeInputs.format(v, v)).mkString(" ") - val outputVolume = dockerVolume.format(executionDir.toAbsolutePath.toString, runtimeAttributes.dockerOutputDir.getOrElse(executionDir.toAbsolutePath.toString)) - val cmd = dockerCmd.format(runtimeAttributes.dockerWorkingDir.getOrElse(executionDir.toAbsolutePath.toString), inputVolumes, outputVolume, runtimeAttributes.dockerImage.get, jobCmd.get) + val outputVolume = dockerVolume.format(executionDir.toAbsolutePath.toString, runtimeAttributes.dockerOutputDir.getOrElse(defaultOutputDir)) + val workingDir = dockerVolume.format(executionDir.toAbsolutePath.toString, runtimeAttributes.dockerWorkingDir.getOrElse(defaultWorkingDir)) + val cmd = dockerCmd.format(runtimeAttributes.dockerWorkingDir.getOrElse(defaultWorkingDir), workingDir, inputVolumes, outputVolume, runtimeAttributes.dockerImage.get, jobCmd.get) log.debug("{} Docker command line to be used for task execution: {}.", tag, cmd) cmd } diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala index e44e66b4a..8a18ddf0d 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala @@ -102,7 +102,9 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc | root = "local-cromwell-executions" | | docker { - | cmd = "docker run -w %s %s %s --rm %s %s" + | cmd = "docker run -w %s %s %s %s --rm %s %s" + | defaultWorkingDir = "/workingDir/" + | defaultOutputDir = "/output/" | } | | filesystems { @@ -280,8 +282,8 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc """ |runtime { | docker: "ubuntu/latest" - | dockerWorkingDir: "/workingDir" - | dockerOutputDir: "/outputDir" + | dockerWorkingDir: "/workingDir/" + | dockerOutputDir: "/outputDir/" |} """.stripMargin val jsonInputFile = createCannedFile("testFile", "some content").pathAsString @@ -313,9 +315,10 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc val bashScript = Source.fromFile(jobPaths.script.toFile).getLines.mkString - assert(bashScript.contains("docker run -w /workingDir -v")) + assert(bashScript.contains("docker run -w /workingDir/ -v")) + assert(bashScript.contains(":/workingDir/")) assert(bashScript.contains(":ro")) - assert(bashScript.contains("/call-hello/execution:/outputDir --rm ubuntu/latest echo")) + assert(bashScript.contains("/call-hello/execution:/outputDir/ --rm ubuntu/latest echo")) cleanUpJob(jobPaths) } @@ -356,8 +359,8 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc """ |runtime { | docker: "ubuntu/latest" - | dockerWorkingDir: "/workingDir" - | dockerOutputDir: "/outputDir" + | dockerWorkingDir: "/workingDir/" + | dockerOutputDir: "/outputDir/" |} """.stripMargin @@ -396,10 +399,11 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc val bashScript = Source.fromFile(jobPaths.script.toFile).getLines.mkString - assert(bashScript.contains("docker run -w /workingDir -v")) + assert(bashScript.contains("docker run -w /workingDir/ -v")) + assert(bashScript.contains(":/workingDir/")) assert(bashScript.contains(tempDir1.toAbsolutePath.toString)) assert(bashScript.contains(tempDir2.toAbsolutePath.toString)) - assert(bashScript.contains("/call-hello/execution:/outputDir --rm ubuntu/latest echo")) + assert(bashScript.contains("/call-hello/execution:/outputDir/ --rm ubuntu/latest echo")) cleanUpJob(jobPaths) } From d6874976269d8295ece402e44112d80ae159287b Mon Sep 17 00:00:00 2001 From: Jeff Gentry Date: Fri, 23 Sep 2016 12:46:31 -0400 Subject: [PATCH 02/23] Fix incorrect URL --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3c5efc009..fd1477d50 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management * [GET /api/workflows/:version/:id/metadata](#get-apiworkflowsversionidmetadata) * [POST /api/workflows/:version/:id/abort](#post-apiworkflowsversionidabort) * [GET /api/workflows/:version/backends](#get-apiworkflowsversionbackends) - * [GET /api/workflows/:version/stats](#get-apiworkflowsversionstats) + * [GET /api/engine/:version/stats](#get-apiengineversionstats) * [Error handling](#error-handling) * [Developer](#developer) * [Generating table of contents on Markdown files](#generating-table-of-contents-on-markdown-files) @@ -2501,18 +2501,18 @@ Server: spray-can/1.3.3 } ``` -## GET /api/workflows/:version/stats +## GET /api/engine/:version/stats This endpoint returns some basic statistics on the current state of the engine. At the moment that includes the number of running workflows and the number of active jobs. cURL: ``` -$ curl http://localhost:8000/api/workflows/v1/stats +$ curl http://localhost:8000/api/engine/v1/stats ``` HTTPie: ``` -$ http http://localhost:8000/api/workflows/v1/stats +$ http http://localhost:8000/api/engine/v1/stats ``` Response: From fb258d8fee4366665e72016682d963f00ee0a351 Mon Sep 17 00:00:00 2001 From: Himanshu Jain Date: Fri, 23 Sep 2016 11:31:18 -0700 Subject: [PATCH 03/23] [HtCondor] pushing metadata events from htcondor backend (#1460) * pushing metadata events from htcondor backend * adding metadata event for command failed as well. --- .../impl/htcondor/HtCondorJobExecutionActor.scala | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala index 870b160cb..46678ae25 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala @@ -2,6 +2,7 @@ package cromwell.backend.impl.htcondor import java.nio.file.attribute.PosixFilePermission import java.nio.file.{FileSystems, Files, Path, Paths} +import java.util.UUID import akka.actor.{ActorRef, Props} import better.files.File @@ -13,6 +14,7 @@ import cromwell.backend.io.JobPaths import cromwell.backend.sfs.{SharedFileSystem, SharedFileSystemExpressionFunctions} import cromwell.core.{JobOutput, JobOutputs, LocallyQualifiedName} import cromwell.services.keyvalue.KeyValueServiceActor._ +import cromwell.services.metadata.CallMetadataKeys import org.apache.commons.codec.digest.DigestUtils import wdl4s._ import wdl4s.parser.MemoryUnit @@ -299,10 +301,26 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor } private def resolveJobCommand(localizedInputs: CallInputs): Try[String] = { - if (runtimeAttributes.dockerImage.isDefined) + val command = if (runtimeAttributes.dockerImage.isDefined) { modifyCommandForDocker(call.task.instantiateCommand(localizedInputs, callEngineFunction, identity), localizedInputs) - else + } else { call.task.instantiateCommand(localizedInputs, callEngineFunction, identity) + } + command match { + case Success(cmd) => tellMetadata(Map("command" -> cmd)) + case Failure(ex) => + log.error("{} failed to resolve command due to exception:{}", tag, ex) + tellMetadata(Map(s"${CallMetadataKeys.Failures}[${UUID.randomUUID().toString}]" -> ex.getMessage)) + } + command + } + + /** + * Fire and forget data to the metadata service + */ + private def tellMetadata(metadataKeyValues: Map[String, Any]): Unit = { + import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter + serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), metadataKeyValues) } private def modifyCommandForDocker(jobCmd: Try[String], localizedInputs: CallInputs): Try[String] = { From 32dcba9d104a48624c17a716e6ed0c66f2f73046 Mon Sep 17 00:00:00 2001 From: Kristian Cibulskis Date: Fri, 23 Sep 2016 12:12:08 -0400 Subject: [PATCH 04/23] added to entrypoint --- project/Settings.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Settings.scala b/project/Settings.scala index cfb0032d3..e3f949e32 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -57,7 +57,7 @@ object Settings { from("openjdk:8") expose(8000) add(artifact, artifactTargetPath) - entryPoint("java", "-jar", artifactTargetPath) + entryPoint("/bin/bash", "-c", "java -jar " + artifactTargetPath + " $CROMWELL_ARGS") } }, buildOptions in docker := BuildOptions( From b6f2a2b7abcde50e906c18da53d75f028dac85ab Mon Sep 17 00:00:00 2001 From: Kristian Cibulskis Date: Fri, 23 Sep 2016 13:44:21 -0400 Subject: [PATCH 05/23] fixed regression for docker run --- project/Settings.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Settings.scala b/project/Settings.scala index e3f949e32..785122cba 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -57,7 +57,7 @@ object Settings { from("openjdk:8") expose(8000) add(artifact, artifactTargetPath) - entryPoint("/bin/bash", "-c", "java -jar " + artifactTargetPath + " $CROMWELL_ARGS") + entryPoint("/bin/bash", "-c", "java -jar " + artifactTargetPath + " ${CROMWELL_ARGS} ${*}", "--") } }, buildOptions in docker := BuildOptions( From 33c3fd63bd2ac029f06b6f145e1db1cffe7bc129 Mon Sep 17 00:00:00 2001 From: Kristian Cibulskis Date: Mon, 26 Sep 2016 13:24:12 -0400 Subject: [PATCH 06/23] addressed PR comments --- project/Settings.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/project/Settings.scala b/project/Settings.scala index 785122cba..d9fd8bb36 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -57,6 +57,11 @@ object Settings { from("openjdk:8") expose(8000) add(artifact, artifactTargetPath) + + // If you use the 'exec' form for an entry point, shell processing is not performed and + // environment variable substitution does not occur. Thus we have to /bin/bash here + // and pass along any subsequent command line arguments + // See https://docs.docker.com/engine/reference/builder/#/entrypoint entryPoint("/bin/bash", "-c", "java -jar " + artifactTargetPath + " ${CROMWELL_ARGS} ${*}", "--") } }, From afc702d4c4baa62b2851367a64bc35b29fe45a55 Mon Sep 17 00:00:00 2001 From: Ruchi Date: Mon, 26 Sep 2016 15:55:51 -0400 Subject: [PATCH 07/23] enable caching for local centaur (#1478) * enable caching for local_centaur * test branch rm_convertToStandardSpec --- src/bin/travis/resources/local_centaur.conf | 18 ++++++++++++++++++ src/bin/travis/testCentaurLocal.sh | 3 ++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 src/bin/travis/resources/local_centaur.conf diff --git a/src/bin/travis/resources/local_centaur.conf b/src/bin/travis/resources/local_centaur.conf new file mode 100644 index 000000000..68ba866bf --- /dev/null +++ b/src/bin/travis/resources/local_centaur.conf @@ -0,0 +1,18 @@ +akka { + loggers = ["akka.event.slf4j.Slf4jLogger"] + logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" +} + +spray.can { + server { + request-timeout = 40s + } + client { + request-timeout = 40s + connecting-timeout = 40s + } +} + +call-caching { + enabled = true +} diff --git a/src/bin/travis/testCentaurLocal.sh b/src/bin/travis/testCentaurLocal.sh index 203d87c18..fe77a1347 100755 --- a/src/bin/travis/testCentaurLocal.sh +++ b/src/bin/travis/testCentaurLocal.sh @@ -31,6 +31,7 @@ set -e sbt assembly CROMWELL_JAR=$(find "$(pwd)/target/scala-2.11" -name "cromwell-*.jar") +LOCAL_CONF="$(pwd)/src/bin/travis/resources/local_centaur.conf" git clone https://github.com/broadinstitute/centaur.git cd centaur -./test_cromwell.sh -j"${CROMWELL_JAR}" +./test_cromwell.sh -j"${CROMWELL_JAR}" -c${LOCAL_CONF} From c239aad1712f3cb20e9d9c319b04f08710b12e40 Mon Sep 17 00:00:00 2001 From: Thib Date: Mon, 26 Sep 2016 16:03:51 -0400 Subject: [PATCH 08/23] Update develop version to 0.22 (#1486) --- project/Version.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Version.scala b/project/Version.scala index 99e00f9f6..e6de0a5cd 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -4,7 +4,7 @@ import sbt._ object Version { // Upcoming release, or current if we're on the master branch - val cromwellVersion = "0.20" + val cromwellVersion = "0.22" // Adapted from SbtGit.versionWithGit def cromwellVersionWithGit: Seq[Setting[_]] = From 4c8dd90902f1f913181ac377a04a19dd94003688 Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Fri, 2 Sep 2016 17:21:31 -0400 Subject: [PATCH 09/23] meow meow meow meow --- README.md | 4 +- .../main/scala/cromwell/backend/MemorySize.scala | 26 ++-- .../ContinueOnReturnCodeValidation.scala | 20 ++- .../backend/validation/CpuValidation.scala | 5 +- .../backend/validation/DockerValidation.scala | 5 +- .../validation/FailOnStderrValidation.scala | 6 +- .../backend/validation/MemoryValidation.scala | 15 +- .../validation/RuntimeAttributesDefault.scala | 8 +- .../validation/RuntimeAttributesValidation.scala | 23 +-- .../ValidatedRuntimeAttributesBuilder.scala | 35 ++--- .../validation/RuntimeAttributesDefaultSpec.scala | 12 +- .../RuntimeAttributesValidationSpec.scala | 169 ++++++++++----------- build.sbt | 1 + core/src/main/scala/cromwell/core/ConfigUtil.scala | 26 ++-- core/src/main/scala/cromwell/core/ErrorOr.scala | 22 +++ .../main/scala/cromwell/core/WorkflowState.scala | 7 +- core/src/main/scala/cromwell/core/package.scala | 7 +- .../database/slick/CallCachingSlickDatabase.scala | 2 +- .../database/slick/MetadataSlickDatabase.scala | 2 +- .../tables/CallCachingHashEntryComponent.scala | 4 +- .../slick/tables/MetadataEntryComponent.scala | 5 +- .../database/sql/CallCachingSqlDatabase.scala | 2 +- .../database/sql/MetadataSqlDatabase.scala | 2 +- .../scala/cromwell/engine/EngineFilesystems.scala | 7 +- .../engine/workflow/WorkflowManagerActor.scala | 7 +- .../MaterializeWorkflowDescriptorActor.scala | 75 +++++---- .../execution/WorkflowExecutionActor.scala | 11 +- .../execution/callcaching/CallCache.scala | 5 +- .../workflow/workflowstore/SqlWorkflowStore.scala | 4 +- .../workflow/workflowstore/WorkflowStore.scala | 2 +- .../workflowstore/WorkflowStoreActor.scala | 14 +- .../scala/cromwell/webservice/ApiDataModels.scala | 6 +- .../cromwell/webservice/CromwellApiHandler.scala | 4 +- .../cromwell/webservice/CromwellApiService.scala | 11 +- .../webservice/metadata/IndexedJsonValue.scala | 13 +- .../webservice/metadata/MetadataBuilderActor.scala | 11 +- .../cromwell/engine/WorkflowStoreActorSpec.scala | 26 ++-- .../workflow/lifecycle/CachingConfigSpec.scala | 8 +- .../workflowstore/InMemoryWorkflowStore.scala | 4 +- .../filesystems/gcs/GoogleConfiguration.scala | 29 ++-- .../scala/cromwell/filesystems/gcs/package.scala | 2 - project/Dependencies.scala | 11 +- .../cromwell/services/metadata/MetadataQuery.scala | 3 +- .../services/metadata/MetadataService.scala | 2 +- .../services/metadata/WorkflowQueryKey.scala | 50 +++--- .../metadata/WorkflowQueryParameters.scala | 29 ++-- .../metadata/impl/MetadataDatabaseAccess.scala | 13 +- .../metadata/WorkflowQueryParametersSpec.scala | 77 +++++----- src/main/scala/cromwell/CromwellCommandLine.scala | 28 ++-- .../impl/htcondor/HtCondorRuntimeAttributes.scala | 31 ++-- .../cromwell/backend/impl/jes/JesAttributes.scala | 22 +-- .../backend/impl/jes/JesRuntimeAttributes.scala | 45 +++--- .../backend/impl/jes/io/JesAttachedDisk.scala | 29 ++-- .../backend/impl/jes/JesAttributesSpec.scala | 2 +- .../PrimitiveRuntimeAttributesValidation.scala | 11 +- .../sfs/GcsWorkflowFileSystemProvider.scala | 5 +- .../impl/spark/SparkRuntimeAttributes.scala | 24 +-- 57 files changed, 535 insertions(+), 494 deletions(-) create mode 100644 core/src/main/scala/cromwell/core/ErrorOr.scala diff --git a/README.md b/README.md index fd1477d50..1800278e9 100644 --- a/README.md +++ b/README.md @@ -101,8 +101,8 @@ There is a [Cromwell gitter channel](https://gitter.im/broadinstitute/cromwell) The following is the toolchain used for development of Cromwell. Other versions may work, but these are recommended. -* [Scala 2.11.7](http://www.scala-lang.org/news/2.11.7/) -* [SBT 0.13.8](https://github.com/sbt/sbt/releases/tag/v0.13.8) +* [Scala 2.11.8](http://www.scala-lang.org/news/2.11.8/) +* [SBT 0.13.12](https://github.com/sbt/sbt/releases/tag/v0.13.12) * [Java 8](http://www.oracle.com/technetwork/java/javase/overview/java8-2100321.html) # Building diff --git a/backend/src/main/scala/cromwell/backend/MemorySize.scala b/backend/src/main/scala/cromwell/backend/MemorySize.scala index 52e6364a9..b207174b6 100644 --- a/backend/src/main/scala/cromwell/backend/MemorySize.scala +++ b/backend/src/main/scala/cromwell/backend/MemorySize.scala @@ -1,10 +1,16 @@ package cromwell.backend -import wdl4s.parser.MemoryUnit + +import cats.data.Validated._ +import cats.syntax.cartesian._ +import cats.syntax.validated._ +import cromwell.core.ErrorOr._ +import mouse.string._ import scala.language.postfixOps import scala.util.{Failure, Success, Try} -import scalaz.Scalaz._ +import wdl4s.parser.MemoryUnit + object MemorySize { val memoryPattern = """(\d+(?:\.\d+)?)\s*(\w+)""".r @@ -12,18 +18,18 @@ object MemorySize { def parse(unparsed: String): Try[MemorySize] = { unparsed match { case memoryPattern(amountString, unitString) => - val amount = amountString.parseDouble leftMap { + val amount: ErrorOr[Double] = amountString.parseDouble leftMap { _.getMessage - } toValidationNel - val unit = MemoryUnit.values find { + } toValidatedNel + val unit: ErrorOr[MemoryUnit] = MemoryUnit.values find { _.suffixes.contains(unitString) } match { - case Some(s) => s.successNel[String] - case None => s"$unitString is an invalid memory unit".failureNel + case Some(s) => s.validNel + case None => s"$unitString is an invalid memory unit".invalidNel } - (amount |@| unit) { (a, u) => new MemorySize(a, u) } match { - case scalaz.Success(memorySize) => Success(memorySize) - case scalaz.Failure(nel) => Failure(new UnsupportedOperationException(nel.list.toList.mkString("\n"))) + (amount |@| unit) map { (a, u) => new MemorySize(a, u) } match { + case Valid(memorySize) => Success(memorySize) + case Invalid(nel) => Failure(new UnsupportedOperationException(nel.toList.mkString("\n"))) } case _ => Failure(new UnsupportedOperationException(s"$unparsed should be of the form 'X Unit' where X is a number, e.g. 8 GB")) } diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala index 977c21618..81f9f1c89 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala @@ -1,12 +1,16 @@ package cromwell.backend.validation +import cats.data.Validated.{Invalid, Valid} +import cats.instances.list._ +import cats.syntax.traverse._ +import cats.syntax.validated._ import cromwell.backend.validation.RuntimeAttributesValidation._ import cromwell.core._ +import cromwell.core.ErrorOr._ import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType} import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString} import scala.util.Try -import scalaz.Scalaz._ /** * Validates the "continueOnReturnCode" runtime attribute a Boolean, a String 'true' or 'false', or an Array[Int], @@ -41,14 +45,14 @@ class ContinueOnReturnCodeValidation extends RuntimeAttributesValidation[Continu override def coercion = ContinueOnReturnCode.validWdlTypes override def validateValue = { - case WdlBoolean(value) => ContinueOnReturnCodeFlag(value).successNel - case WdlString(value) if Try(value.toBoolean).isSuccess => ContinueOnReturnCodeFlag(value.toBoolean).successNel - case WdlInteger(value) => ContinueOnReturnCodeSet(Set(value)).successNel + case WdlBoolean(value) => ContinueOnReturnCodeFlag(value).validNel + case WdlString(value) if Try(value.toBoolean).isSuccess => ContinueOnReturnCodeFlag(value.toBoolean).validNel + case WdlInteger(value) => ContinueOnReturnCodeSet(Set(value)).validNel case WdlArray(wdlType, seq) => val errorOrInts: ErrorOr[List[Int]] = (seq.toList map validateInt).sequence[ErrorOr, Int] errorOrInts match { - case scalaz.Success(ints) => ContinueOnReturnCodeSet(ints.toSet).successNel - case scalaz.Failure(_) => failureWithMessage + case Valid(ints) => ContinueOnReturnCodeSet(ints.toSet).validNel + case Invalid(_) => failureWithMessage } } @@ -56,8 +60,8 @@ class ContinueOnReturnCodeValidation extends RuntimeAttributesValidation[Continu case _: WdlBoolean => true case WdlString(value) if Try(value.toBoolean).isSuccess => true case _: WdlInteger => true - case WdlArray(WdlArrayType(WdlStringType), elements) => elements.forall(validateInt(_).isSuccess) - case WdlArray(WdlArrayType(WdlIntegerType), elements) => elements.forall(validateInt(_).isSuccess) + case WdlArray(WdlArrayType(WdlStringType), elements) => elements.forall(validateInt(_).isValid) + case WdlArray(WdlArrayType(WdlIntegerType), elements) => elements.forall(validateInt(_).isValid) } override protected def failureMessage = missingMessage diff --git a/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala b/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala index 051f63326..43303b6cb 100644 --- a/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala @@ -1,10 +1,9 @@ package cromwell.backend.validation +import cats.syntax.validated._ import wdl4s.types.WdlIntegerType import wdl4s.values.WdlInteger -import scalaz.Scalaz._ - /** * Validates the "cpu" runtime attribute an Integer greater than 0, returning the value as an `Int`. * @@ -39,7 +38,7 @@ class CpuValidation extends RuntimeAttributesValidation[Int] { override protected def validateValue = { case wdlValue if WdlIntegerType.coerceRawValue(wdlValue).isSuccess => WdlIntegerType.coerceRawValue(wdlValue).get match { - case WdlInteger(value) => if (value.toInt <= 0) wrongAmountMsg.failureNel else value.toInt.successNel + case WdlInteger(value) => if (value.toInt <= 0) wrongAmountMsg.invalidNel else value.toInt.validNel } } diff --git a/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala b/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala index 97129d391..12e090fcf 100644 --- a/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala @@ -1,10 +1,9 @@ package cromwell.backend.validation +import cats.syntax.validated._ import wdl4s.types.WdlStringType import wdl4s.values.WdlString -import scalaz.Scalaz._ - /** * Validates the "docker" runtime attribute as a String, returning it as `String`. * @@ -32,7 +31,7 @@ class DockerValidation extends RuntimeAttributesValidation[String] { override def coercion = Seq(WdlStringType) override protected def validateValue = { - case WdlString(value) => value.successNel + case WdlString(value) => value.validNel } override protected def failureMessage = missingMessage diff --git a/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala b/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala index 662499ddf..6c75a40cf 100644 --- a/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala @@ -1,10 +1,10 @@ package cromwell.backend.validation +import cats.syntax.validated._ import wdl4s.types.{WdlBooleanType, WdlStringType} import wdl4s.values.{WdlBoolean, WdlString} import scala.util.Try -import scalaz.Scalaz._ /** * Validates the "failOnStderr" runtime attribute as a Boolean or a String 'true' or 'false', returning the value as a @@ -36,8 +36,8 @@ class FailOnStderrValidation extends RuntimeAttributesValidation[Boolean] { override def coercion = Seq(WdlBooleanType, WdlStringType) override protected def validateValue = { - case WdlBoolean(value) => value.successNel - case WdlString(value) if Try(value.toBoolean).isSuccess => value.toBoolean.successNel + case WdlBoolean(value) => value.validNel + case WdlString(value) if Try(value.toBoolean).isSuccess => value.toBoolean.validNel } override def validateExpression = { diff --git a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala index d5a2c8367..81e8f3ea7 100644 --- a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala @@ -1,13 +1,12 @@ package cromwell.backend.validation +import cats.syntax.validated._ import cromwell.backend.MemorySize -import cromwell.core._ +import cromwell.core.ErrorOr._ import wdl4s.parser.MemoryUnit import wdl4s.types.{WdlIntegerType, WdlStringType} import wdl4s.values.{WdlInteger, WdlString} -import scalaz.Scalaz._ - /** * Validates the "memory" runtime attribute as an Integer or String with format '8 GB', returning the value as a * `MemorySize`. @@ -39,11 +38,11 @@ object MemoryValidation { private[validation] def validateMemoryString(value: String): ErrorOr[MemorySize] = { MemorySize.parse(value) match { case scala.util.Success(memorySize: MemorySize) if memorySize.amount > 0 => - memorySize.to(MemoryUnit.GB).successNel + memorySize.to(MemoryUnit.GB).validNel case scala.util.Success(memorySize: MemorySize) => - wrongAmountFormat.format(memorySize.amount).failureNel + wrongAmountFormat.format(memorySize.amount).invalidNel case scala.util.Failure(throwable) => - missingFormat.format(throwable.getMessage).failureNel + missingFormat.format(throwable.getMessage).invalidNel } } @@ -52,9 +51,9 @@ object MemoryValidation { private[validation] def validateMemoryInteger(value: Int): ErrorOr[MemorySize] = { if (value <= 0) - wrongAmountFormat.format(value).failureNel + wrongAmountFormat.format(value).invalidNel else - MemorySize(value, MemoryUnit.Bytes).to(MemoryUnit.GB).successNel + MemorySize(value, MemoryUnit.Bytes).to(MemoryUnit.GB).validNel } } diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala index 0a2ed3b5c..7a080170e 100644 --- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala +++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala @@ -1,13 +1,13 @@ package cromwell.backend.validation -import cromwell.core.{OptionNotFoundException, EvaluatedRuntimeAttributes, WorkflowOptions} +import cats.data.ValidatedNel +import cats.syntax.validated._ +import cromwell.core.{EvaluatedRuntimeAttributes, OptionNotFoundException, WorkflowOptions} import wdl4s.types.WdlType import wdl4s.util.TryUtil import wdl4s.values.WdlValue import scala.util.{Failure, Try} -import scalaz.Scalaz._ -import scalaz.ValidationNel object RuntimeAttributesDefault { @@ -35,5 +35,5 @@ object RuntimeAttributesDefault { }) } - def noValueFoundFor[A](attribute: String): ValidationNel[String, A] = s"Can't find an attribute value for key $attribute".failureNel + def noValueFoundFor[A](attribute: String): ValidatedNel[String, A] = s"Can't find an attribute value for key $attribute".invalidNel } diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala index 47e90edae..9e56c71be 100644 --- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala @@ -1,8 +1,10 @@ package cromwell.backend.validation +import cats.syntax.validated._ import cromwell.backend.wdl.OnlyPureFunctions import cromwell.backend.{MemorySize, RuntimeAttributeDefinition} import cromwell.core._ +import cromwell.core.ErrorOr._ import org.slf4j.Logger import wdl4s.WdlExpression import wdl4s.WdlExpression._ @@ -10,7 +12,6 @@ import wdl4s.types.{WdlBooleanType, WdlIntegerType, WdlType} import wdl4s.values._ import scala.util.{Failure, Success} -import scalaz.Scalaz._ object RuntimeAttributesValidation { @@ -48,22 +49,22 @@ object RuntimeAttributesValidation { missingValidationMessage: String): ErrorOr[T] = { valueOption match { case Some(value) => - validation.validateValue.applyOrElse(value, (_: Any) => missingValidationMessage.failureNel) + validation.validateValue.applyOrElse(value, (_: Any) => missingValidationMessage.invalidNel) case None => onMissingValue } } def validateInt(value: WdlValue): ErrorOr[Int] = { WdlIntegerType.coerceRawValue(value) match { - case scala.util.Success(WdlInteger(i)) => i.intValue.successNel - case _ => s"Could not coerce ${value.valueString} into an integer".failureNel + case scala.util.Success(WdlInteger(i)) => i.intValue.validNel + case _ => s"Could not coerce ${value.valueString} into an integer".invalidNel } } def validateBoolean(value: WdlValue): ErrorOr[Boolean] = { WdlBooleanType.coerceRawValue(value) match { - case scala.util.Success(WdlBoolean(b)) => b.booleanValue.successNel - case _ => s"Could not coerce ${value.valueString} into a boolean".failureNel + case scala.util.Success(WdlBoolean(b)) => b.booleanValue.validNel + case _ => s"Could not coerce ${value.valueString} into a boolean".invalidNel } } @@ -238,7 +239,7 @@ trait RuntimeAttributesValidation[ValidatedType] { /** * Validates the wdl value. * - * @return The validated value or an error, wrapped in a scalaz validation. + * @return The validated value or an error, wrapped in a cats validation. */ protected def validateValue: PartialFunction[WdlValue, ErrorOr[ValidatedType]] @@ -282,7 +283,7 @@ trait RuntimeAttributesValidation[ValidatedType] { * * @return Wrapped failureMessage. */ - protected final lazy val failureWithMessage: ErrorOr[ValidatedType] = failureMessage.failureNel + protected final lazy val failureWithMessage: ErrorOr[ValidatedType] = failureMessage.invalidNel /** * Runs this validation on the value matching key. @@ -313,7 +314,7 @@ trait RuntimeAttributesValidation[ValidatedType] { */ def validateOptionalExpression(wdlExpressionMaybe: Option[WdlValue]): Boolean = { wdlExpressionMaybe match { - case None => staticDefaultOption.isDefined || validateNone.isSuccess + case None => staticDefaultOption.isDefined || validateNone.isValid case Some(wdlExpression: WdlExpression) => /* TODO: BUG: @@ -384,7 +385,7 @@ trait OptionalRuntimeAttributesValidation[ValidatedType] extends RuntimeAttribut * This method is the same as `validateValue`, but allows the implementor to not have to wrap the response in an * `Option`. * - * @return The validated value or an error, wrapped in a scalaz validation. + * @return The validated value or an error, wrapped in a cats validation. */ protected def validateOption: PartialFunction[WdlValue, ErrorOr[ValidatedType]] @@ -394,5 +395,5 @@ trait OptionalRuntimeAttributesValidation[ValidatedType] extends RuntimeAttribut override def apply(wdlValue: WdlValue) = validateOption.apply(wdlValue).map(Option.apply) } - override final protected lazy val validateNone = None.successNel[String] + override final protected lazy val validateNone = None.validNel[String] } diff --git a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala index 22c632520..59e8cec26 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala @@ -1,14 +1,13 @@ package cromwell.backend.validation +import cats.data.Validated._ +import cats.instances.list._ import cromwell.backend.RuntimeAttributeDefinition -import cromwell.core._ import lenthall.exception.MessageAggregation +import cromwell.core.ErrorOr._ import org.slf4j.Logger -import wdl4s.types.WdlType import wdl4s.values.WdlValue -import scalaz.{Failure, Success} - final case class ValidatedRuntimeAttributes(attributes: Map[String, Any]) /** @@ -54,32 +53,26 @@ trait ValidatedRuntimeAttributesBuilder { val runtimeAttributesErrorOr: ErrorOr[ValidatedRuntimeAttributes] = validate(attrs) runtimeAttributesErrorOr match { - case Success(runtimeAttributes) => runtimeAttributes - case Failure(nel) => throw new RuntimeException with MessageAggregation { + case Valid(runtimeAttributes) => runtimeAttributes + case Invalid(nel) => throw new RuntimeException with MessageAggregation { override def exceptionContext: String = "Runtime attribute validation failed" - override def errorMessages: Traversable[String] = nel.list.toList + override def errorMessages: Traversable[String] = nel.toList } } } private def validate(values: Map[String, WdlValue]): ErrorOr[ValidatedRuntimeAttributes] = { - val validationsForValues: Seq[RuntimeAttributesValidation[_]] = validations ++ unsupportedExtraValidations - val errorsOrValuesMap: Seq[(String, ErrorOr[Any])] = - validationsForValues.map(validation => validation.key -> validation.validate(values)) - - import scalaz.Scalaz._ + val validationsForValues = validations ++ unsupportedExtraValidations + val listOfKeysToErrorOrAnys: List[(String, ErrorOr[Any])] = + validationsForValues.map(validation => validation.key -> validation.validate(values)).toList - val emptyResult: ErrorOr[List[(String, Any)]] = List.empty[(String, Any)].success - val validationResult = errorsOrValuesMap.foldLeft(emptyResult) { (agg, errorOrValue) => - agg +++ { - errorOrValue match { - case (key, Success(value)) => List(key -> value).success - case (key, Failure(nel)) => nel.failure - } - } + val listOfErrorOrKeysToAnys: List[ErrorOr[(String, Any)]] = listOfKeysToErrorOrAnys map { + case (key, errorOrAny) => errorOrAny map { any => (key, any) } } - validationResult.map(result => ValidatedRuntimeAttributes(result.toMap)) + import cats.syntax.traverse._ + val errorOrListOfKeysToAnys: ErrorOr[List[(String, Any)]] = listOfErrorOrKeysToAnys.sequence[ErrorOr, (String, Any)] + errorOrListOfKeysToAnys map { listOfKeysToAnys => ValidatedRuntimeAttributes(listOfKeysToAnys.toMap) } } } diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala index 14898d92d..d1df9d790 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala @@ -1,11 +1,11 @@ package cromwell.backend.validation +import cromwell.backend.validation.RuntimeAttributesDefault._ import cromwell.core.WorkflowOptions -import org.scalatest.{Matchers, FlatSpec} +import org.scalatest.{FlatSpec, Matchers} import spray.json._ -import cromwell.backend.validation.RuntimeAttributesDefault._ import wdl4s.types._ -import wdl4s.values.{WdlArray, WdlInteger, WdlBoolean, WdlString} +import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString} class RuntimeAttributesDefaultSpec extends FlatSpec with Matchers { @@ -103,8 +103,8 @@ class RuntimeAttributesDefaultSpec extends FlatSpec with Matchers { ) } - "noValueFoundFor" should "provide a failureNel for missing values" in { - import scalaz.Scalaz._ - noValueFoundFor("myKey") shouldBe "Can't find an attribute value for key myKey".failureNel + "noValueFoundFor" should "provide an invalidNel for missing values" in { + import cats.syntax.validated._ + noValueFoundFor("myKey") shouldBe "Can't find an attribute value for key myKey".invalidNel } } diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala index 6c914f161..22180bb77 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala @@ -1,150 +1,149 @@ package cromwell.backend.validation +import cats.data.Validated.{Invalid, Valid} +import cats.syntax.validated._ import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} -import wdl4s.parser.MemoryUnit -import wdl4s.types.{WdlIntegerType, WdlStringType, WdlArrayType} +import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType} import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString} -import scalaz.Scalaz._ - class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with BeforeAndAfterAll { "RuntimeAttributesValidation" should { "return success when tries to validate a valid Docker entry" in { val dockerValue = Some(WdlString("someImage")) val result = RuntimeAttributesValidation.validateDocker(dockerValue, - "Failed to get Docker mandatory key from runtime attributes".failureNel) + "Failed to get Docker mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x.get == "someImage") - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x.get == "someImage") + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return success (based on defined HoF) when tries to validate a docker entry but it does not contain a value" in { val dockerValue = None - val result = RuntimeAttributesValidation.validateDocker(dockerValue, None.successNel) + val result = RuntimeAttributesValidation.validateDocker(dockerValue, None.validNel) result match { - case scalaz.Success(x) => assert(!x.isDefined) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x.isEmpty) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return failure (based on defined HoF) when tries to validate a docker entry but it does not contain a value" in { val dockerValue = None val result = RuntimeAttributesValidation.validateDocker(dockerValue, - "Failed to get Docker mandatory key from runtime attributes".failureNel) + "Failed to get Docker mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Failed to get Docker mandatory key from runtime attributes") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Failed to get Docker mandatory key from runtime attributes") } } "return failure when there is an invalid docker runtime attribute defined" in { val dockerValue = Some(WdlInteger(1)) val result = RuntimeAttributesValidation.validateDocker(dockerValue, - "Failed to get Docker mandatory key from runtime attributes".failureNel) + "Failed to get Docker mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Expecting docker runtime attribute to be a String") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting docker runtime attribute to be a String") } } "return success when tries to validate a failOnStderr boolean entry" in { val failOnStderrValue = Some(WdlBoolean(true)) val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, - "Failed to get failOnStderr mandatory key from runtime attributes".failureNel) + "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return success when tries to validate a failOnStderr 'true' string entry" in { val failOnStderrValue = Some(WdlString("true")) val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, - "Failed to get failOnStderr mandatory key from runtime attributes".failureNel) + "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return success when tries to validate a failOnStderr 'false' string entry" in { val failOnStderrValue = Some(WdlString("false")) val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, - "Failed to get failOnStderr mandatory key from runtime attributes".failureNel) + "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(!x) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(!x) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return failure when there is an invalid failOnStderr runtime attribute defined" in { val failOnStderrValue = Some(WdlInteger(1)) val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, - "Failed to get failOnStderr mandatory key from runtime attributes".failureNel) + "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'") } } "return success (based on defined HoF) when tries to validate a failOnStderr entry but it does not contain a value" in { val failOnStderrValue = None - val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, true.successNel) + val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, true.validNel) result match { - case scalaz.Success(x) => assert(x) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return success when tries to validate a continueOnReturnCode boolean entry" in { val continueOnReturnCodeValue = Some(WdlBoolean(true)) val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".failureNel) + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x == ContinueOnReturnCodeFlag(true)) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x == ContinueOnReturnCodeFlag(true)) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return success when tries to validate a continueOnReturnCode 'true' string entry" in { val continueOnReturnCodeValue = Some(WdlString("true")) val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".failureNel) + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x == ContinueOnReturnCodeFlag(true)) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x == ContinueOnReturnCodeFlag(true)) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return success when tries to validate a continueOnReturnCode 'false' string entry" in { val continueOnReturnCodeValue = Some(WdlString("false")) val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".failureNel) + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x == ContinueOnReturnCodeFlag(false)) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x == ContinueOnReturnCodeFlag(false)) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return success when tries to validate a continueOnReturnCode int entry" in { val continueOnReturnCodeValue = Some(WdlInteger(12)) val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".failureNel) + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x == ContinueOnReturnCodeSet(Set(12))) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x == ContinueOnReturnCodeSet(Set(12))) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return failure when there is an invalid continueOnReturnCode runtime attribute defined" in { val continueOnReturnCodeValue = Some(WdlString("yes")) val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".failureNel) + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]") } } @@ -152,29 +151,29 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be "return success when there is a valid integer array in continueOnReturnCode runtime attribute" in { val continueOnReturnCodeValue = Some(WdlArray(WdlArrayType(WdlIntegerType), Seq(WdlInteger(1), WdlInteger(2)))) val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".failureNel) + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x == ContinueOnReturnCodeSet(Set(1, 2))) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x == ContinueOnReturnCodeSet(Set(1, 2))) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return failure when there is an invalid array in continueOnReturnCode runtime attribute" in { val continueOnReturnCodeValue = Some(WdlArray(WdlArrayType(WdlStringType), Seq(WdlString("one"), WdlString("two")))) val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".failureNel) + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]") } } "return success (based on defined HoF) when tries to validate a continueOnReturnCode entry but it does not contain a value" in { val continueOnReturnCodeValue = None - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, ContinueOnReturnCodeFlag(false).successNel) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, ContinueOnReturnCodeFlag(false).validNel) result match { - case scalaz.Success(x) => assert(x == ContinueOnReturnCodeFlag(false)) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x == ContinueOnReturnCodeFlag(false)) + case Invalid(e) => fail(e.toList.mkString(" ")) } } @@ -182,20 +181,20 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be val expectedGb = 1 val memoryValue = Some(WdlInteger(1000000000)) val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".failureNel) + "Failed to get memory mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x.amount == expectedGb) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x.amount == expectedGb) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return failure when tries to validate an invalid Integer memory entry" in { val memoryValue = Some(WdlInteger(-1)) val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".failureNel) + "Failed to get memory mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Expecting memory runtime attribute value greater than 0 but got -1") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting memory runtime attribute value greater than 0 but got -1") } } @@ -203,80 +202,80 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be val expectedGb = 2 val memoryValue = Some(WdlString("2 GB")) val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".failureNel) + "Failed to get memory mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x.amount == expectedGb) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x.amount == expectedGb) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return failure when tries to validate an invalid size in String memory entry" in { val memoryValue = Some(WdlString("0 GB")) val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".failureNel) + "Failed to get memory mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Expecting memory runtime attribute value greater than 0 but got 0.0") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting memory runtime attribute value greater than 0 but got 0.0") } } "return failure when tries to validate an invalid String memory entry" in { val memoryValue = Some(WdlString("value")) val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".failureNel) + "Failed to get memory mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: value should be of the form 'X Unit' where X is a number, e.g. 8 GB") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: value should be of the form 'X Unit' where X is a number, e.g. 8 GB") } } "return failure when tries to validate an invalid memory entry" in { val memoryValue = Some(WdlBoolean(true)) val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".failureNel) + "Failed to get memory mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: Not supported WDL type value") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: Not supported WDL type value") } } "return failure when tries to validate a non-provided memory entry" in { val memoryValue = None val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".failureNel) + "Failed to get memory mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Failed to get memory mandatory key from runtime attributes") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Failed to get memory mandatory key from runtime attributes") } } "return success when tries to validate a valid cpu entry" in { val cpuValue = Some(WdlInteger(1)) val result = RuntimeAttributesValidation.validateCpu(cpuValue, - "Failed to get cpu mandatory key from runtime attributes".failureNel) + "Failed to get cpu mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => assert(x == 1) - case scalaz.Failure(e) => fail(e.toList.mkString(" ")) + case Valid(x) => assert(x == 1) + case Invalid(e) => fail(e.toList.mkString(" ")) } } "return failure when tries to validate an invalid cpu entry" in { val cpuValue = Some(WdlInteger(-1)) val result = RuntimeAttributesValidation.validateCpu(cpuValue, - "Failed to get cpu mandatory key from runtime attributes".failureNel) + "Failed to get cpu mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Expecting cpu runtime attribute value greater than 0") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Expecting cpu runtime attribute value greater than 0") } } "return failure when tries to validate a non-provided cpu entry" in { val cpuValue = None val result = RuntimeAttributesValidation.validateMemory(cpuValue, - "Failed to get cpu mandatory key from runtime attributes".failureNel) + "Failed to get cpu mandatory key from runtime attributes".invalidNel) result match { - case scalaz.Success(x) => fail("A failure was expected.") - case scalaz.Failure(e) => assert(e.head == "Failed to get cpu mandatory key from runtime attributes") + case Valid(x) => fail("A failure was expected.") + case Invalid(e) => assert(e.head == "Failed to get cpu mandatory key from runtime attributes") } } } diff --git a/build.sbt b/build.sbt index 26934f881..8e666691f 100644 --- a/build.sbt +++ b/build.sbt @@ -8,6 +8,7 @@ lazy val core = (project in file("core")) lazy val gcsFileSystem = (project in file("filesystems/gcs")) .settings(gcsFileSystemSettings:_*) .withTestSettings + .dependsOn(core) lazy val databaseSql = (project in file("database/sql")) .settings(databaseSqlSettings:_*) diff --git a/core/src/main/scala/cromwell/core/ConfigUtil.scala b/core/src/main/scala/cromwell/core/ConfigUtil.scala index 0d90f3e2e..9f22c330e 100644 --- a/core/src/main/scala/cromwell/core/ConfigUtil.scala +++ b/core/src/main/scala/cromwell/core/ConfigUtil.scala @@ -2,13 +2,13 @@ package cromwell.core import java.net.URL +import cats.data.ValidatedNel +import cats.syntax.validated._ import com.typesafe.config.{Config, ConfigException, ConfigValue} import org.slf4j.LoggerFactory import scala.collection.JavaConversions._ import scala.reflect.{ClassTag, classTag} -import scalaz.Scalaz._ -import scalaz._ object ConfigUtil { @@ -30,21 +30,21 @@ object ConfigUtil { /** * Validates that the value for this key is a well formed URL. */ - def validateURL(key: String): ValidationNel[String, URL] = key.validateAny { url => + def validateURL(key: String): ValidatedNel[String, URL] = key.validateAny { url => new URL(config.getString(url)) } - def validateString(key: String): ValidationNel[String, String] = try { - config.getString(key).successNel + def validateString(key: String): ValidatedNel[String, String] = try { + config.getString(key).validNel } catch { - case e: ConfigException.Missing => s"Could not find key: $key".failureNel + case e: ConfigException.Missing => s"Could not find key: $key".invalidNel } - def validateConfig(key: String): ValidationNel[String, Config] = try { - config.getConfig(key).successNel + def validateConfig(key: String): ValidatedNel[String, Config] = try { + config.getConfig(key).validNel } catch { - case e: ConfigException.Missing => "Could not find key: $key".failureNel - case e: ConfigException.WrongType => s"key $key cannot be parsed to a Config".failureNel + case e: ConfigException.Missing => "Could not find key: $key".invalidNel + case e: ConfigException.WrongType => s"key $key cannot be parsed to a Config".invalidNel } } @@ -58,10 +58,10 @@ object ConfigUtil { * @tparam O return type of validationFunction * @tparam E Restricts the subtype of Exception that should be caught during validation */ - def validateAny[O, E <: Exception: ClassTag](validationFunction: I => O): ValidationNel[String, O] = try { - validationFunction(value).successNel + def validateAny[O, E <: Exception: ClassTag](validationFunction: I => O): ValidatedNel[String, O] = try { + validationFunction(value).validNel } catch { - case e if classTag[E].runtimeClass.isInstance(e) => e.getMessage.failureNel + case e if classTag[E].runtimeClass.isInstance(e) => e.getMessage.invalidNel } } diff --git a/core/src/main/scala/cromwell/core/ErrorOr.scala b/core/src/main/scala/cromwell/core/ErrorOr.scala new file mode 100644 index 000000000..cd344f8ac --- /dev/null +++ b/core/src/main/scala/cromwell/core/ErrorOr.scala @@ -0,0 +1,22 @@ +package cromwell.core + +import cats.data.Validated.{Invalid, Valid} +import cats.data.{NonEmptyList, Validated} + +object ErrorOr { + type ErrorOr[A] = Validated[NonEmptyList[String], A] + + implicit class ShortCircuitingFlatMap[A](val fa: ErrorOr[A]) extends AnyVal { + /** + * Not consistent with `Applicative#ap` but useful in for comprehensions. + * + * @see http://typelevel.org/cats/tut/validated.html#of-flatmaps-and-xors + */ + def flatMap[B](f: A => ErrorOr[B]): ErrorOr[B] = { + fa match { + case Valid(v) => f(v) + case i @ Invalid(_) => i + } + } + } +} diff --git a/core/src/main/scala/cromwell/core/WorkflowState.scala b/core/src/main/scala/cromwell/core/WorkflowState.scala index b5066d292..41ac2bb97 100644 --- a/core/src/main/scala/cromwell/core/WorkflowState.scala +++ b/core/src/main/scala/cromwell/core/WorkflowState.scala @@ -1,11 +1,12 @@ package cromwell.core -import scalaz.Semigroup +import cats.Semigroup + sealed trait WorkflowState { def isTerminal: Boolean protected def ordinal: Int - def append(that: WorkflowState): WorkflowState = if (this.ordinal > that.ordinal) this else that + def combine(that: WorkflowState): WorkflowState = if (this.ordinal > that.ordinal) this else that } object WorkflowState { @@ -15,7 +16,7 @@ object WorkflowState { throw new NoSuchElementException(s"No such WorkflowState: $str")) implicit val WorkflowStateSemigroup = new Semigroup[WorkflowState] { - override def append(f1: WorkflowState, f2: => WorkflowState): WorkflowState = f1.append(f2) + override def combine(f1: WorkflowState, f2: WorkflowState): WorkflowState = f1.combine(f2) } implicit val WorkflowStateOrdering = Ordering.by { self: WorkflowState => self.ordinal } diff --git a/core/src/main/scala/cromwell/core/package.scala b/core/src/main/scala/cromwell/core/package.scala index b218a2dfb..ee1448640 100644 --- a/core/src/main/scala/cromwell/core/package.scala +++ b/core/src/main/scala/cromwell/core/package.scala @@ -1,16 +1,13 @@ package cromwell -import lenthall.exception.ThrowableAggregation import java.nio.file.Path -import wdl4s.values.{SymbolHash, WdlValue} - -import scalaz._ +import lenthall.exception.ThrowableAggregation +import wdl4s.values.WdlValue package object core { case class CallContext(root: Path, stdout: String, stderr: String) - type ErrorOr[+A] = ValidationNel[String, A] type LocallyQualifiedName = String type FullyQualifiedName = String type WorkflowOutputs = Map[FullyQualifiedName, JobOutput] diff --git a/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala index 1193f4a21..51ea7c3a4 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala @@ -1,10 +1,10 @@ package cromwell.database.slick +import cats.data.NonEmptyList import cromwell.database.sql._ import cromwell.database.sql.joins.CallCachingJoin import scala.concurrent.{ExecutionContext, Future} -import scalaz.NonEmptyList trait CallCachingSlickDatabase extends CallCachingSqlDatabase { this: SlickDatabase => diff --git a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala index 2be8f0069..7a0725ffe 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala @@ -2,11 +2,11 @@ package cromwell.database.slick import java.sql.Timestamp +import cats.data.NonEmptyList import cromwell.database.sql.MetadataSqlDatabase import cromwell.database.sql.tables.{MetadataEntry, WorkflowMetadataSummaryEntry} import scala.concurrent.{ExecutionContext, Future} -import scalaz._ trait MetadataSlickDatabase extends MetadataSqlDatabase { this: SlickDatabase with SummaryStatusSlickDatabase => diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala index 21d2d4c9b..b82ec957c 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala @@ -1,7 +1,7 @@ package cromwell.database.slick.tables +import cats.data.NonEmptyList import cromwell.database.sql.tables.CallCachingHashEntry -import scalaz._ trait CallCachingHashEntryComponent { @@ -61,7 +61,7 @@ trait CallCachingHashEntryComponent { Rep[Boolean] = { hashKeyHashValues. map(existsCallCachingEntryIdHashKeyHashValue(callCachingEntryId)). - list.toList.reduce(_ && _) + toList.reduce(_ && _) } /** diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala index 42f79fdf2..90c1898fc 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala @@ -2,10 +2,9 @@ package cromwell.database.slick.tables import java.sql.Timestamp +import cats.data.NonEmptyList import cromwell.database.sql.tables.MetadataEntry -import scalaz._ - trait MetadataEntryComponent { this: DriverComponent => @@ -146,7 +145,7 @@ trait MetadataEntryComponent { private[this] def metadataEntryHasMetadataKeysLike(metadataEntry: MetadataEntries, metadataKeys: NonEmptyList[String]): Rep[Boolean] = { - metadataKeys.list.toList.map(metadataEntry.metadataKey like _).reduce(_ || _) + metadataKeys.toList.map(metadataEntry.metadataKey like _).reduce(_ || _) } private[this] def metadataEntryHasEmptyJobKey(metadataEntry: MetadataEntries, diff --git a/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala index 12f9b1c77..abf262ae3 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala @@ -1,9 +1,9 @@ package cromwell.database.sql +import cats.data.NonEmptyList import cromwell.database.sql.joins.CallCachingJoin import scala.concurrent.{ExecutionContext, Future} -import scalaz.NonEmptyList trait CallCachingSqlDatabase { def addCallCaching(callCachingJoin: CallCachingJoin)(implicit ec: ExecutionContext): Future[Unit] diff --git a/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala index 596d84749..3f8b83a14 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala @@ -2,10 +2,10 @@ package cromwell.database.sql import java.sql.Timestamp +import cats.data.NonEmptyList import cromwell.database.sql.tables.{MetadataEntry, WorkflowMetadataSummaryEntry} import scala.concurrent.{ExecutionContext, Future} -import scalaz.NonEmptyList trait MetadataSqlDatabase { this: SqlDatabase => diff --git a/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala b/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala index e05df2bb6..e18bd3dae 100644 --- a/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala +++ b/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala @@ -2,6 +2,7 @@ package cromwell.engine import java.nio.file.{FileSystem, FileSystems} +import cats.data.Validated.{Invalid, Valid} import com.typesafe.config.ConfigFactory import cromwell.core.WorkflowOptions import cromwell.engine.backend.EnhancedWorkflowOptions._ @@ -17,10 +18,10 @@ object EngineFilesystems { private val googleConf: GoogleConfiguration = GoogleConfiguration(config) private val googleAuthMode = config.getStringOption("engine.filesystems.gcs.auth") map { confMode => googleConf.auth(confMode) match { - case scalaz.Success(mode) => mode - case scalaz.Failure(errors) => throw new RuntimeException() with MessageAggregation { + case Valid(mode) => mode + case Invalid(errors) => throw new RuntimeException() with MessageAggregation { override def exceptionContext: String = s"Failed to create authentication mode for $confMode" - override def errorMessages: Traversable[String] = errors.list.toList + override def errorMessages: Traversable[String] = errors.toList } } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala index e7879c803..78d34439c 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala @@ -1,10 +1,10 @@ package cromwell.engine.workflow -import java.util.UUID import akka.actor.FSM.{CurrentState, SubscribeTransitionCallBack, Transition} import akka.actor._ import akka.event.Logging +import cats.data.NonEmptyList import com.typesafe.config.{Config, ConfigFactory} import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.{WorkflowAborted, WorkflowId} @@ -19,7 +19,6 @@ import lenthall.config.ScalaConfig.EnhancedScalaConfig import scala.concurrent.duration._ import scala.concurrent.{Await, Promise} import scala.language.postfixOps -import scalaz.NonEmptyList object WorkflowManagerActor { val DefaultMaxWorkflowsToRun = 5000 @@ -66,7 +65,7 @@ object WorkflowManagerActor { def withAddition(entries: NonEmptyList[WorkflowIdToActorRef]): WorkflowManagerData = { val entryTuples = entries map { e => e.workflowId -> e.workflowActor } - this.copy(workflows = workflows ++ entryTuples.list.toList) + this.copy(workflows = workflows ++ entryTuples.toList) } def without(id: WorkflowId): WorkflowManagerData = this.copy(workflows = workflows - id) @@ -144,7 +143,7 @@ class WorkflowManagerActor(config: Config, stay() case Event(WorkflowStoreActor.NewWorkflowsToStart(newWorkflows), stateData) => val newSubmissions = newWorkflows map submitWorkflow - log.info("Retrieved {} workflows from the WorkflowStoreActor", newSubmissions.size) + log.info("Retrieved {} workflows from the WorkflowStoreActor", newSubmissions.toList.size) scheduleNextNewWorkflowPoll() stay() using stateData.withAddition(newSubmissions) case Event(SubscribeToWorkflowCommand(id), data) => diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala index 833001588..d79d97998 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala @@ -3,10 +3,16 @@ package cromwell.engine.workflow.lifecycle import java.nio.file.FileSystem import akka.actor.{ActorRef, FSM, LoggingFSM, Props} +import cats.data.NonEmptyList +import cats.data.Validated._ +import cats.instances.list._ +import cats.syntax.cartesian._ +import cats.syntax.traverse._ +import cats.syntax.validated._ +import cromwell.core._ import com.typesafe.config.Config import com.typesafe.scalalogging.LazyLogging import cromwell.backend.BackendWorkflowDescriptor -import cromwell.core._ import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.WorkflowOptions.{ReadFromCache, WorkflowOption, WriteToCache} import cromwell.core.callcaching._ @@ -15,8 +21,9 @@ import cromwell.engine._ import cromwell.engine.backend.CromwellBackends import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.{MaterializeWorkflowDescriptorActorData, MaterializeWorkflowDescriptorActorState} import cromwell.services.metadata.MetadataService._ -import cromwell.services.metadata.{MetadataValue, MetadataKey, MetadataEvent} +import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} import lenthall.config.ScalaConfig.EnhancedScalaConfig +import cromwell.core.ErrorOr._ import spray.json.{JsObject, _} import wdl4s._ import wdl4s.expression.NoFunctions @@ -24,8 +31,6 @@ import wdl4s.values.{WdlString, WdlValue} import scala.language.postfixOps import scala.util.{Failure, Success, Try} -import scalaz.Scalaz._ -import scalaz.Validation.FlatMap._ object MaterializeWorkflowDescriptorActor { @@ -79,9 +84,9 @@ object MaterializeWorkflowDescriptorActor { def readOptionalOption(option: WorkflowOption): ErrorOr[Boolean] = { workflowOptions.getBoolean(option.name) match { - case Success(x) => x.successNel - case Failure(_: OptionNotFoundException) => true.successNel - case Failure(t) => t.getMessage.failureNel + case Success(x) => x.validNel + case Failure(_: OptionNotFoundException) => true.validNel + case Failure(t) => t.getMessage.invalidNel } } @@ -90,7 +95,7 @@ object MaterializeWorkflowDescriptorActor { val readFromCache = readOptionalOption(ReadFromCache) val writeToCache = readOptionalOption(WriteToCache) - (readFromCache |@| writeToCache) { + (readFromCache |@| writeToCache) map { case (false, false) => CallCachingOff case (true, false) => CallCachingActivity(ReadCache) case (false, true) => CallCachingActivity(WriteCache) @@ -98,7 +103,7 @@ object MaterializeWorkflowDescriptorActor { } } else { - CallCachingOff.successNel + CallCachingOff.validNel } } } @@ -116,10 +121,10 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor when(ReadyToMaterializeState) { case Event(MaterializeWorkflowDescriptorCommand(workflowSourceFiles, conf), _) => buildWorkflowDescriptor(workflowId, workflowSourceFiles, conf) match { - case scalaz.Success(descriptor) => + case Valid(descriptor) => sender() ! MaterializeWorkflowDescriptorSuccessResponse(descriptor) goto(MaterializationSuccessfulState) - case scalaz.Failure(error) => + case Invalid(error) => sender() ! MaterializeWorkflowDescriptorFailureResponse( new IllegalArgumentException with ExceptionWithErrors { val message = s"Workflow input processing failed." @@ -157,7 +162,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor conf: Config): ErrorOr[EngineWorkflowDescriptor] = { val namespaceValidation = validateNamespace(sourceFiles.wdlSource) val workflowOptionsValidation = validateWorkflowOptions(sourceFiles.workflowOptionsJson) - (namespaceValidation |@| workflowOptionsValidation) { + (namespaceValidation |@| workflowOptionsValidation) map { (_, _) } flatMap { case (namespace, workflowOptions) => pushWfNameMetadataService(namespace.workflow.unqualifiedName) @@ -185,7 +190,8 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor val failureModeValidation = validateWorkflowFailureMode(workflowOptions, conf) val backendAssignmentsValidation = validateBackendAssignments(namespace.workflow.calls, workflowOptions, defaultBackendName) val callCachingModeValidation = validateCallCachingMode(workflowOptions, conf) - (rawInputsValidation |@| failureModeValidation |@| backendAssignmentsValidation |@| callCachingModeValidation ) { + + (rawInputsValidation |@| failureModeValidation |@| backendAssignmentsValidation |@| callCachingModeValidation ) map { (_, _, _, _) } flatMap { case (rawInputs, failureMode, backendAssignments, callCachingMode) => buildWorkflowDescriptor(id, namespace, rawInputs, backendAssignments, workflowOptions, failureMode, engineFilesystems, callCachingMode) @@ -203,13 +209,16 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor def checkTypes(inputs: Map[FullyQualifiedName, WdlValue]): ErrorOr[Map[FullyQualifiedName, WdlValue]] = { val allDeclarations = namespace.workflow.scopedDeclarations ++ namespace.workflow.calls.flatMap(_.scopedDeclarations) - inputs.map({ case (k, v) => + val list: List[ErrorOr[(FullyQualifiedName, WdlValue)]] = inputs.map({ case (k, v) => allDeclarations.find(_.fullyQualifiedName == k) match { case Some(decl) if decl.wdlType.coerceRawValue(v).isFailure => - s"Invalid right-side type of '$k'. Expecting ${decl.wdlType.toWdlString}, got ${v.wdlType.toWdlString}".failureNel - case _ => (k, v).successNel[String] + s"Invalid right-side type of '$k'. Expecting ${decl.wdlType.toWdlString}, got ${v.wdlType.toWdlString}".invalidNel + case _ => (k, v).validNel[String] } - }).toList.sequence[ErrorOr, (FullyQualifiedName, WdlValue)].map(_.toMap) + }).toList + + val validatedInputs: ErrorOr[List[(FullyQualifiedName, WdlValue)]] = list.sequence[ErrorOr, (FullyQualifiedName, WdlValue)] + validatedInputs.map(_.toMap) } for { @@ -256,8 +265,8 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor case Success(backendMap) => val backendMapAsString = backendMap.map({case (k, v) => s"${k.fullyQualifiedName} -> $v"}).mkString(", ") workflowLogger.info(s"Call-to-Backend assignments: $backendMapAsString") - backendMap.successNel - case Failure(t) => t.getMessage.failureNel + backendMap.validNel + case Failure(t) => t.getMessage.invalidNel } } @@ -284,40 +293,40 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor coercedInputs: WorkflowCoercedInputs, engineFileSystems: List[FileSystem]): ErrorOr[WorkflowCoercedInputs] = { namespace.staticWorkflowDeclarationsRecursive(coercedInputs, new WdlFunctions(engineFileSystems)) match { - case Success(d) => d.successNel - case Failure(e) => s"Workflow has invalid declarations: ${e.getMessage}".failureNel + case Success(d) => d.validNel + case Failure(e) => s"Workflow has invalid declarations: ${e.getMessage}".invalidNel } } private def validateNamespace(source: WdlSource): ErrorOr[NamespaceWithWorkflow] = { try { - NamespaceWithWorkflow.load(source).successNel + NamespaceWithWorkflow.load(source).validNel } catch { - case e: Exception => s"Unable to load namespace from workflow: ${e.getMessage}".failureNel + case e: Exception => s"Unable to load namespace from workflow: ${e.getMessage}".invalidNel } } private def validateRawInputs(json: WdlJson): ErrorOr[Map[String, JsValue]] = { Try(json.parseJson) match { - case Success(JsObject(inputs)) => inputs.successNel - case Failure(reason: Throwable) => s"Workflow contains invalid inputs JSON: ${reason.getMessage}".failureNel - case _ => s"Workflow inputs JSON cannot be parsed to JsObject: $json".failureNel + case Success(JsObject(inputs)) => inputs.validNel + case Failure(reason: Throwable) => s"Workflow contains invalid inputs JSON: ${reason.getMessage}".invalidNel + case _ => s"Workflow inputs JSON cannot be parsed to JsObject: $json".invalidNel } } private def validateCoercedInputs(rawInputs: Map[String, JsValue], namespace: NamespaceWithWorkflow): ErrorOr[WorkflowCoercedInputs] = { namespace.coerceRawInputs(rawInputs) match { - case Success(r) => r.successNel - case Failure(e: ExceptionWithErrors) => scalaz.Failure(e.errors) - case Failure(e) => e.getMessage.failureNel + case Success(r) => r.validNel + case Failure(e: ExceptionWithErrors) => Invalid(e.errors) + case Failure(e) => e.getMessage.invalidNel } } private def validateWorkflowOptions(workflowOptions: WdlJson): ErrorOr[WorkflowOptions] = { WorkflowOptions.fromJsonString(workflowOptions) match { - case Success(opts) => opts.successNel - case Failure(e) => s"Workflow contains invalid options JSON: ${e.getMessage}".failureNel + case Success(opts) => opts.validNel + case Failure(e) => s"Workflow contains invalid options JSON: ${e.getMessage}".invalidNel } } @@ -329,8 +338,8 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor } modeString flatMap WorkflowFailureMode.tryParse match { - case Success(mode) => mode.successNel - case Failure(t) => t.getMessage.failureNel + case Success(mode) => mode.validNel + case Failure(t) => t.getMessage.invalidNel } } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index 750a215b5..139f5d6c5 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -4,8 +4,9 @@ import java.time.OffsetDateTime import akka.actor.SupervisorStrategy.{Escalate, Stop} import akka.actor._ +import cats.data.NonEmptyList import com.typesafe.config.ConfigFactory -import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, FailedRetryableResponse, FailedNonRetryableResponse, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, FailedNonRetryableResponse, FailedRetryableResponse, SucceededResponse} import cromwell.backend.BackendLifecycleActor.AbortJobCommand import cromwell.backend.{AllBackendInitializationData, BackendJobDescriptor, BackendJobDescriptorKey} import cromwell.core.Dispatcher.EngineDispatcher @@ -17,10 +18,10 @@ import cromwell.core.WorkflowOptions.WorkflowFailureMode import cromwell.core._ import cromwell.core.logging.WorkflowLogging import cromwell.engine.backend.CromwellBackends -import cromwell.engine.workflow.lifecycle.{EngineLifecycleActorAbortCommand, EngineLifecycleActorAbortedResponse} import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.JobRunning import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor.BackendJobPreparationFailed import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.WorkflowExecutionActorState +import cromwell.engine.workflow.lifecycle.{EngineLifecycleActorAbortCommand, EngineLifecycleActorAbortedResponse} import cromwell.engine.{ContinueWhilePossible, EngineWorkflowDescriptor} import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata._ @@ -34,8 +35,6 @@ import wdl4s.{Scope, _} import scala.annotation.tailrec import scala.language.postfixOps import scala.util.{Failure, Random, Success, Try} -import scalaz.NonEmptyList -import scalaz.Scalaz._ object WorkflowExecutionActor { @@ -130,7 +129,7 @@ object WorkflowExecutionActor { } case class WorkflowExecutionException[T <: Throwable](exceptions: NonEmptyList[T]) extends ThrowableAggregation { - override val throwables = exceptions.list.toList + override val throwables = exceptions.toList override val exceptionContext = s"WorkflowExecutionActor" } @@ -592,7 +591,7 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, ejeaRef ! EngineJobExecutionActor.Execute Success(WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Starting))) case None => - throw WorkflowExecutionException(new Exception(s"Could not get BackendLifecycleActor for backend $backendName").wrapNel) + throw WorkflowExecutionException(NonEmptyList.of(new Exception(s"Could not get BackendLifecycleActor for backend $backendName"))) } } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala index 99e0fbd44..d4c144d9a 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala @@ -1,5 +1,6 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching +import cats.data.NonEmptyList import cromwell.backend.BackendJobExecutionActor.SucceededResponse import cromwell.core.ExecutionIndex.IndexEnhancedIndex import cromwell.core.WorkflowId @@ -12,8 +13,6 @@ import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashing import scala.concurrent.{ExecutionContext, Future} import scala.language.postfixOps -import scalaz.Scalaz._ -import scalaz._ final case class MetaInfoId(id: Int) @@ -64,7 +63,7 @@ class CallCache(database: CallCachingSqlDatabase) { } def fetchMetaInfoIdsMatchingHashes(callCacheHashes: CallCacheHashes)(implicit ec: ExecutionContext): Future[Set[MetaInfoId]] = { - metaInfoIdsMatchingHashes(callCacheHashes.hashes.toList.toNel.get) + metaInfoIdsMatchingHashes(NonEmptyList.fromListUnsafe(callCacheHashes.hashes.toList)) } private def metaInfoIdsMatchingHashes(hashKeyValuePairs: NonEmptyList[HashResult]) diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala index 63a12a154..7056137c3 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala @@ -2,6 +2,7 @@ package cromwell.engine.workflow.workflowstore import java.time.OffsetDateTime +import cats.data.NonEmptyList import cromwell.core.{WorkflowId, WorkflowSourceFiles} import cromwell.database.sql.SqlConverters._ import cromwell.database.sql.WorkflowStoreSqlDatabase @@ -9,7 +10,6 @@ import cromwell.database.sql.tables.WorkflowStoreEntry import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState import scala.concurrent.{ExecutionContext, Future} -import scalaz.NonEmptyList case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase) extends WorkflowStore { override def initialize(implicit ec: ExecutionContext): Future[Unit] = { @@ -42,7 +42,7 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase) extends Workf val returnValue = asStoreEntries map { workflowStore => WorkflowId.fromString(workflowStore.workflowExecutionUuid) } // The results from the Future aren't useful, so on completion map it into the precalculated return value instead. Magic! - sqlDatabase.addWorkflowStoreEntries(asStoreEntries.list.toList) map { _ => returnValue } + sqlDatabase.addWorkflowStoreEntries(asStoreEntries.toList) map { _ => returnValue } } private def fromWorkflowStoreEntry(workflowStoreEntry: WorkflowStoreEntry): WorkflowToStart = { diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala index f24cd99cb..e3d7b44be 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStore.scala @@ -1,10 +1,10 @@ package cromwell.engine.workflow.workflowstore +import cats.data.NonEmptyList import cromwell.core.{WorkflowId, WorkflowSourceFiles} import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState import scala.concurrent.{ExecutionContext, Future} -import scalaz.NonEmptyList trait WorkflowStore { diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala index 3655b6938..6355b64c5 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala @@ -3,19 +3,19 @@ package cromwell.engine.workflow.workflowstore import java.time.OffsetDateTime import akka.actor.{ActorLogging, ActorRef, LoggingFSM, Props} +import cats.data.NonEmptyList import cromwell.core.{WorkflowId, WorkflowMetadataKeys, WorkflowSourceFiles} import cromwell.engine.workflow.WorkflowManagerActor import cromwell.engine.workflow.WorkflowManagerActor.WorkflowNotFoundException import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._ import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState -import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} import cromwell.services.metadata.MetadataService.{MetadataPutAcknowledgement, PutMetadataAction} +import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} import org.apache.commons.lang3.exception.ExceptionUtils import scala.concurrent.{ExecutionContext, Future} import scala.language.postfixOps import scala.util.{Failure, Success} -import scalaz.NonEmptyList case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor: ActorRef) extends LoggingFSM[WorkflowStoreActorState, WorkflowStoreActorData] with ActorLogging { @@ -70,7 +70,7 @@ case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor: ActorR private def startNewWork(command: WorkflowStoreActorCommand, sndr: ActorRef, nextData: WorkflowStoreActorData) = { val work: Future[Any] = command match { case cmd @ SubmitWorkflow(sourceFiles) => - store.add(NonEmptyList(sourceFiles)) map { ids => + store.add(NonEmptyList.of(sourceFiles)) map { ids => val id = ids.head registerSubmissionWithMetadataService(id, sourceFiles) sndr ! WorkflowSubmittedToStore(id) @@ -78,15 +78,15 @@ case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor: ActorR } case cmd @ BatchSubmitWorkflows(sources) => store.add(sources) map { ids => - val assignedSources = ids.zip(sources) + val assignedSources = ids.toList.zip(sources.toList) assignedSources foreach { case (id, sourceFiles) => registerSubmissionWithMetadataService(id, sourceFiles) } sndr ! WorkflowsBatchSubmittedToStore(ids) - log.info("Workflows {} submitted.", ids.list.toList.mkString(", ")) + log.info("Workflows {} submitted.", ids.toList.mkString(", ")) } case cmd @ FetchRunnableWorkflows(n) => newWorkflowMessage(n) map { nwm => nwm match { - case NewWorkflowsToStart(workflows) => log.info("{} new workflows fetched", workflows.size) + case NewWorkflowsToStart(workflows) => log.info("{} new workflows fetched", workflows.toList.size) case NoNewWorkflowsToStart => log.debug("No workflows fetched") case _ => log.error("Unexpected response from newWorkflowMessage({}): {}", n, nwm) } @@ -145,7 +145,7 @@ case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor: ActorR } yield restartableWorkflows ++ submittedWorkflows runnableWorkflows map { - case x :: xs => NewWorkflowsToStart(NonEmptyList.nels(x, xs: _*)) + case x :: xs => NewWorkflowsToStart(NonEmptyList.of(x, xs: _*)) case _ => NoNewWorkflowsToStart } } diff --git a/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala b/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala index c910ecce9..b39c2fc51 100644 --- a/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala +++ b/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala @@ -2,7 +2,7 @@ package cromwell.webservice import spray.json._ import wdl4s.values.WdlValue -import wdl4s.{FullyQualifiedName, ExceptionWithErrors} +import wdl4s.{ExceptionWithErrors, FullyQualifiedName} import scala.language.postfixOps @@ -19,14 +19,12 @@ case class CallOutputResponse(id: String, callFqn: String, outputs: Map[FullyQua case class WorkflowMetadataQueryParameters(outputs: Boolean = true, timings: Boolean = true) object APIResponse { - import WorkflowJsonSupport._ - import spray.httpx.SprayJsonSupport._ private def constructFailureResponse(status: String, ex: Throwable) ={ ex match { case exceptionWithErrors: ExceptionWithErrors => FailureResponse(status, exceptionWithErrors.message, - Option(JsArray(exceptionWithErrors.errors.list.toList.map(JsString(_)).toVector))) + Option(JsArray(exceptionWithErrors.errors.toList.map(JsString(_)).toVector))) case e: Throwable => FailureResponse(status, e.getMessage, None) } } diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala index d7afa5834..0aae04dac 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala @@ -2,6 +2,7 @@ package cromwell.webservice import akka.actor.{Actor, ActorRef, Props} import akka.event.Logging +import cats.data.NonEmptyList import com.typesafe.config.ConfigFactory import cromwell.core._ import cromwell.engine.workflow.WorkflowManagerActor @@ -13,7 +14,6 @@ import spray.http.{StatusCodes, Uri} import spray.httpx.SprayJsonSupport._ import scala.language.postfixOps -import scalaz.NonEmptyList object CromwellApiHandler { def props(requestHandlerActor: ActorRef): Props = { @@ -71,6 +71,6 @@ class CromwellApiHandler(requestHandlerActor: ActorRef) extends Actor with Workf case WorkflowStoreActor.WorkflowsBatchSubmittedToStore(ids) => val responses = ids map { id => WorkflowSubmitResponse(id.toString, WorkflowSubmitted.toString) } - context.parent ! RequestComplete(StatusCodes.OK, responses.list.toList) + context.parent ! RequestComplete(StatusCodes.OK, responses.toList) } } diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala index 170ed3020..8e7c55168 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala @@ -1,6 +1,7 @@ package cromwell.webservice import akka.actor._ +import cats.data.NonEmptyList import cromwell.core.{WorkflowId, WorkflowSourceFiles} import cromwell.engine.backend.BackendConfiguration import cromwell.services.metadata.MetadataService._ @@ -13,8 +14,6 @@ import spray.httpx.SprayJsonSupport._ import spray.json._ import spray.routing._ -import scalaz.NonEmptyList - trait SwaggerService extends SwaggerUiResourceHttpService { override def swaggerServiceName = "cromwell" @@ -127,7 +126,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator { import spray.json._ workflowInputs.parseJson match { case JsArray(Seq(x, xs@_*)) => - val nelInputses = NonEmptyList.nels(x, xs: _*) + val nelInputses = NonEmptyList.of(x, xs: _*) val sources = nelInputses.map(inputs => WorkflowSourceFiles(wdlSource, inputs.compactPrint, workflowOptions.getOrElse("{}"))) perRequest(requestContext, CromwellApiHandler.props(workflowStoreActor), CromwellApiHandler.ApiHandlerWorkflowSubmitBatch(sources)) case JsArray(_) => failBadRequest(new RuntimeException("Nothing was submitted")) @@ -158,10 +157,8 @@ trait CromwellApiService extends HttpService with PerRequestCreator { def metadataRoute = path("workflows" / Segment / Segment / "metadata") { (version, possibleWorkflowId) => parameterMultiMap { parameters => - // import scalaz_ & Scalaz._ add too many slow implicits, on top of the spray and json implicits - import scalaz.syntax.std.list._ - val includeKeysOption = parameters.getOrElse("includeKey", List.empty).toNel - val excludeKeysOption = parameters.getOrElse("excludeKey", List.empty).toNel + val includeKeysOption = NonEmptyList.fromList(parameters.getOrElse("includeKey", List.empty)) + val excludeKeysOption = NonEmptyList.fromList(parameters.getOrElse("excludeKey", List.empty)) (includeKeysOption, excludeKeysOption) match { case (Some(_), Some(_)) => failBadRequest(new IllegalArgumentException("includeKey and excludeKey may not be specified together")) diff --git a/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala b/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala index 4f1ba9bcc..c1969342c 100644 --- a/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala +++ b/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala @@ -2,30 +2,29 @@ package cromwell.webservice.metadata import java.time.OffsetDateTime +import cats.{Monoid, Semigroup} +import cats.instances.map._ import spray.json._ -import scalaz.{Monoid, Semigroup} -// This is useful, do not remove -import scalaz.Scalaz._ private object IndexedJsonValue { private implicit val dateTimeOrdering: Ordering[OffsetDateTime] = scala.Ordering.fromLessThan(_ isBefore _) private val timestampedJsValueOrdering: Ordering[TimestampedJsValue] = scala.Ordering.by(_.timestamp) implicit val TimestampedJsonMonoid: Monoid[TimestampedJsValue] = new Monoid[TimestampedJsValue] { - def append(f1: TimestampedJsValue, f2: => TimestampedJsValue): TimestampedJsValue = { + def combine(f1: TimestampedJsValue, f2: TimestampedJsValue): TimestampedJsValue = { (f1, f2) match { case (o1: TimestampedJsObject, o2: TimestampedJsObject) => val sg = implicitly[Semigroup[Map[String, TimestampedJsValue]]] - TimestampedJsObject(sg.append(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) + TimestampedJsObject(sg.combine(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) case (o1: TimestampedJsList, o2: TimestampedJsList) => val sg = implicitly[Semigroup[Map[Int, TimestampedJsValue]]] - TimestampedJsList(sg.append(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) + TimestampedJsList(sg.combine(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) case (o1, o2) => timestampedJsValueOrdering.max(o1, o2) } } - override def zero: TimestampedJsValue = TimestampedJsObject(Map.empty, OffsetDateTime.now) + override def empty: TimestampedJsValue = TimestampedJsObject(Map.empty, OffsetDateTime.now) } } diff --git a/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala b/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala index 35ea1aaa5..6d007dddc 100644 --- a/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala +++ b/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala @@ -3,6 +3,9 @@ package cromwell.webservice.metadata import java.time.OffsetDateTime import akka.actor.{ActorRef, LoggingFSM, Props} +import cromwell.webservice.metadata.IndexedJsonValue._ +import cats.instances.list._ +import cats.syntax.foldable._ import cromwell.core.Dispatcher.ApiDispatcher import cromwell.core.ExecutionIndex.ExecutionIndex import cromwell.core.{WorkflowId, WorkflowMetadataKeys, WorkflowState} @@ -10,7 +13,6 @@ import cromwell.services.ServiceRegistryActor.ServiceRegistryFailure import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata._ import cromwell.webservice.PerRequest.{RequestComplete, RequestCompleteWithHeaders} -import cromwell.webservice.metadata.IndexedJsonValue._ import cromwell.webservice.metadata.MetadataBuilderActor.{Idle, MetadataBuilderActorState, WaitingForMetadataService} import cromwell.webservice.{APIResponse, WorkflowJsonSupport} import org.slf4j.LoggerFactory @@ -21,8 +23,7 @@ import spray.json._ import scala.collection.immutable.TreeMap import scala.language.postfixOps import scala.util.{Failure, Success, Try} -import scalaz.std.list._ -import scalaz.syntax.foldable._ + object MetadataBuilderActor { sealed trait MetadataBuilderActorState @@ -133,8 +134,8 @@ object MetadataBuilderActor { /** Sort events by timestamp, transform them into TimestampedJsValues, and merge them together. */ private def eventsToIndexedJson(events: Seq[MetadataEvent]): TimestampedJsValue = { // The `List` has a `Foldable` instance defined in scope, and because the `List`'s elements have a `Monoid` instance - // defined in scope, `suml` can derive a sane `TimestampedJsValue` value even if the `List` of events is empty. - events.toList map { e => keyValueToIndexedJson(e.key.key, e.value, e.offsetDateTime) } suml + // defined in scope, `combineAll` can derive a sane `TimestampedJsValue` value even if the `List` of events is empty. + events.toList map { e => keyValueToIndexedJson(e.key.key, e.value, e.offsetDateTime) } combineAll } private def eventsToAttemptMetadata(attempt: Int, events: Seq[MetadataEvent]) = { diff --git a/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala b/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala index 43764e705..18460e765 100644 --- a/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala @@ -1,5 +1,6 @@ package cromwell.engine +import cats.data.NonEmptyList import cromwell.CromwellTestkitSpec import cromwell.core.WorkflowId import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._ @@ -9,7 +10,6 @@ import org.scalatest.Matchers import scala.concurrent.duration._ import scala.language.postfixOps -import scalaz.NonEmptyList class WorkflowStoreActorSpec extends CromwellTestkitSpec with Matchers { val helloWorldSourceFiles = HelloWorld.asWorkflowSources() @@ -42,25 +42,25 @@ class WorkflowStoreActorSpec extends CromwellTestkitSpec with Matchers { "return 3 IDs for a batch submission of 3" in { val store = new InMemoryWorkflowStore val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestkitSpec.ServiceRegistryActorInstance)) - storeActor ! BatchSubmitWorkflows(NonEmptyList(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles)) + storeActor ! BatchSubmitWorkflows(NonEmptyList.of(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles)) expectMsgPF(10 seconds) { - case WorkflowsBatchSubmittedToStore(ids) => ids.size shouldBe 3 + case WorkflowsBatchSubmittedToStore(ids) => ids.toList.size shouldBe 3 } } "fetch exactly N workflows" in { val store = new InMemoryWorkflowStore val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestkitSpec.ServiceRegistryActorInstance)) - storeActor ! BatchSubmitWorkflows(NonEmptyList(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles)) - val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.list.toList + storeActor ! BatchSubmitWorkflows(NonEmptyList.of(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles)) + val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.toList storeActor ! FetchRunnableWorkflows(2) expectMsgPF(10 seconds) { case NewWorkflowsToStart(workflowNel) => - workflowNel.size shouldBe 2 - checkDistinctIds(workflowNel.list.toList) shouldBe true - workflowNel.foreach { + workflowNel.toList.size shouldBe 2 + checkDistinctIds(workflowNel.toList) shouldBe true + workflowNel map { case WorkflowToStart(id, sources, state) => insertedIds.contains(id) shouldBe true sources shouldBe helloWorldSourceFiles @@ -72,16 +72,16 @@ class WorkflowStoreActorSpec extends CromwellTestkitSpec with Matchers { "return only the remaining workflows if N is larger than size" in { val store = new InMemoryWorkflowStore val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestkitSpec.ServiceRegistryActorInstance)) - storeActor ! BatchSubmitWorkflows(NonEmptyList(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles)) - val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.list.toList + storeActor ! BatchSubmitWorkflows(NonEmptyList.of(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles)) + val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.toList storeActor ! FetchRunnableWorkflows(100) expectMsgPF(10 seconds) { case NewWorkflowsToStart(workflowNel) => - workflowNel.size shouldBe 3 - checkDistinctIds(workflowNel.list.toList) shouldBe true - workflowNel.foreach { + workflowNel.toList.size shouldBe 3 + checkDistinctIds(workflowNel.toList) shouldBe true + workflowNel map { case WorkflowToStart(id, sources, state) => insertedIds.contains(id) shouldBe true sources shouldBe helloWorldSourceFiles diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/CachingConfigSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/CachingConfigSpec.scala index 50c0218e7..4015f01de 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/CachingConfigSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/CachingConfigSpec.scala @@ -1,12 +1,12 @@ package cromwell.engine.workflow.lifecycle +import cats.data.Validated.{Invalid, Valid} import com.typesafe.config.{Config, ConfigFactory} import cromwell.core.WorkflowOptions import cromwell.core.callcaching.CallCachingMode import org.scalatest.{FlatSpec, Matchers} import scala.collection.JavaConverters._ -import scalaz.{Failure => ScalazFailure, Success => ScalazSuccess} import scala.util.{Success, Try} class CachingConfigSpec extends FlatSpec with Matchers { @@ -62,9 +62,9 @@ class CachingConfigSpec extends FlatSpec with Matchers { combinations foreach { case (config, Success(wfOptions)) => MaterializeWorkflowDescriptorActor.validateCallCachingMode(wfOptions, config) match { - case ScalazSuccess(activity) => verificationFunction(activity) - case ScalazFailure(errors) => - val errorsList = errors.list.toList.mkString(", ") + case Valid(activity) => verificationFunction(activity) + case Invalid(errors) => + val errorsList = errors.toList.mkString(", ") fail(s"Failure generating Call Config Mode: $errorsList") } case x => fail(s"Unexpected test tuple: $x") diff --git a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala index 9ee29439d..a24be2d32 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/InMemoryWorkflowStore.scala @@ -1,10 +1,10 @@ package cromwell.engine.workflow.workflowstore +import cats.data.NonEmptyList import cromwell.core.{WorkflowId, WorkflowSourceFiles} import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState import scala.concurrent.{ExecutionContext, Future} -import scalaz.NonEmptyList class InMemoryWorkflowStore extends WorkflowStore { @@ -16,7 +16,7 @@ class InMemoryWorkflowStore extends WorkflowStore { */ override def add(sources: NonEmptyList[WorkflowSourceFiles])(implicit ec: ExecutionContext): Future[NonEmptyList[WorkflowId]] = { val submittedWorkflows = sources map { SubmittedWorkflow(WorkflowId.randomId(), _, WorkflowStoreState.Submitted) } - workflowStore = workflowStore ++ submittedWorkflows.list.toList + workflowStore = workflowStore ++ submittedWorkflows.toList Future.successful(submittedWorkflows map { _.id }) } diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala index 8c4e559ae..9315b51cc 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala @@ -1,16 +1,19 @@ package cromwell.filesystems.gcs +import cats.data.Validated._ +import cats.instances.list._ +import cats.syntax.cartesian._ +import cats.syntax.traverse._ +import cats.syntax.validated._ import com.google.api.services.storage.StorageScopes import com.typesafe.config.Config import lenthall.config.ConfigValidationException import lenthall.config.ValidatedConfig._ +import cromwell.core.ErrorOr._ import org.slf4j.LoggerFactory import scala.collection.JavaConverters._ import scala.language.postfixOps -import scalaz.Scalaz._ -import scalaz.Validation.FlatMap._ -import scalaz._ final case class GoogleConfiguration private (applicationName: String, authsByName: Map[String, GoogleAuthMode]) { @@ -19,8 +22,8 @@ final case class GoogleConfiguration private (applicationName: String, authsByNa authsByName.get(name) match { case None => val knownAuthNames = authsByName.keys.mkString(", ") - s"`google` configuration stanza does not contain an auth named '$name'. Known auth names: $knownAuthNames".failureNel - case Some(a) => a.successNel + s"`google` configuration stanza does not contain an auth named '$name'. Known auth names: $knownAuthNames".invalidNel + case Some(a) => a.validNel } } } @@ -56,7 +59,7 @@ object GoogleConfiguration { cfg => RefreshTokenMode(name, cfg.getString("client-id"), cfg.getString("client-secret")) } - def applicationDefaultAuth(name: String) = ApplicationDefaultMode(name, GoogleScopes).successNel[String] + def applicationDefaultAuth(name: String): ErrorOr[GoogleAuthMode] = ApplicationDefaultMode(name, GoogleScopes).validNel val name = authConfig.getString("name") val scheme = authConfig.getString("scheme") @@ -65,7 +68,7 @@ object GoogleConfiguration { case "user_account" => userAccountAuth(authConfig, name) case "refresh_token" => refreshTokenAuth(authConfig, name) case "application_default" => applicationDefaultAuth(name) - case wut => s"Unsupported authentication scheme: $wut".failureNel + case wut => s"Unsupported authentication scheme: $wut".invalidNel } } @@ -75,20 +78,20 @@ object GoogleConfiguration { def uniqueAuthNames(list: List[GoogleAuthMode]): ErrorOr[Unit] = { val duplicateAuthNames = list.groupBy(_.name) collect { case (n, as) if as.size > 1 => n } if (duplicateAuthNames.nonEmpty) { - ("Duplicate auth names: " + duplicateAuthNames.mkString(", ")).failureNel + ("Duplicate auth names: " + duplicateAuthNames.mkString(", ")).invalidNel } else { - ().successNel + ().validNel } } - (appName |@| errorOrAuthList) { (_, _) } flatMap { case (name, list) => + (appName |@| errorOrAuthList) map { (_, _) } flatMap { case (name, list) => uniqueAuthNames(list) map { _ => GoogleConfiguration(name, list map { a => a.name -> a } toMap) } } match { - case Success(r) => r - case Failure(f) => - val errorMessages = f.list.toList.mkString(", ") + case Valid(r) => r + case Invalid(f) => + val errorMessages = f.toList.mkString(", ") log.error(errorMessages) throw new ConfigValidationException("Google", errorMessages) } diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/package.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/package.scala index 19140a069..0ec2c0316 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/package.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/package.scala @@ -1,8 +1,6 @@ package cromwell.filesystems -import scalaz.ValidationNel package object gcs { - type ErrorOr[+A] = ValidationNel[String, A] type RefreshToken = String } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index cf6b04238..5b632d157 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,8 +1,8 @@ import sbt._ object Dependencies { - lazy val lenthallV = "0.18" - lazy val wdl4sV = "0.5" + lazy val lenthallV = "0.19-98b3f3a-SNAPSHOT" + lazy val wdl4sV = "0.6-2964173-SNAPSHOT" lazy val sprayV = "1.3.3" /* spray-json is an independent project from the "spray suite" @@ -16,13 +16,14 @@ object Dependencies { lazy val slickV = "3.1.1" lazy val googleClientApiV = "1.20.0" lazy val betterFilesV = "2.16.0" - lazy val scalazCoreV = "7.2.5" + lazy val catsV = "0.7.2" // Internal collections of dependencies private val baseDependencies = List( "org.broadinstitute" %% "lenthall" % lenthallV, - "org.scalaz" %% "scalaz-core" % scalazCoreV, + "org.typelevel" %% "cats" % catsV, + "com.github.benhutchison" %% "mouse" % "0.5", "org.scalatest" %% "scalatest" % "3.0.0" % Test, "org.specs2" %% "specs2" % "3.7" % Test ) @@ -120,7 +121,7 @@ object Dependencies { "org.webjars" % "swagger-ui" % "2.1.1", "commons-codec" % "commons-codec" % "1.10", "commons-io" % "commons-io" % "2.5", - "org.scalaz" %% "scalaz-core" % scalazCoreV, + "org.typelevel" %% "cats" % catsV, "com.github.pathikrit" %% "better-files" % betterFilesV, "io.swagger" % "swagger-parser" % "1.0.22" % Test, "org.yaml" % "snakeyaml" % "1.17" % Test diff --git a/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala b/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala index 979a6474c..374169d05 100644 --- a/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala +++ b/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala @@ -2,12 +2,11 @@ package cromwell.services.metadata import java.time.OffsetDateTime +import cats.data.NonEmptyList import cromwell.core.WorkflowId import org.slf4j.LoggerFactory import wdl4s.values.{WdlBoolean, WdlFloat, WdlInteger, WdlValue} -import scalaz.NonEmptyList - case class MetadataJobKey(callFqn: String, index: Option[Int], attempt: Int) case class MetadataKey(workflowId: WorkflowId, jobKey: Option[MetadataJobKey], key: String) diff --git a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala index cce4f6eb1..b440b9f9e 100644 --- a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala +++ b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala @@ -3,12 +3,12 @@ package cromwell.services.metadata import java.time.OffsetDateTime import akka.actor.{ActorRef, DeadLetterSuppression} +import cats.data.NonEmptyList import cromwell.core.{JobKey, WorkflowId, WorkflowState} import cromwell.services.ServiceRegistryActor.ServiceRegistryMessage import wdl4s.values._ import scala.language.postfixOps -import scalaz.NonEmptyList object MetadataService { diff --git a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala index 0cdba09bd..3bf9fce2b 100644 --- a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala +++ b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala @@ -2,12 +2,14 @@ package cromwell.services.metadata import java.time.OffsetDateTime -import cromwell.core.{ErrorOr, WorkflowId, WorkflowState} +import cats.instances.list._ +import cats.syntax.traverse._ +import cats.syntax.validated._ +import cromwell.core.{WorkflowId, WorkflowState} +import cromwell.core.ErrorOr._ import scala.language.postfixOps import scala.util.{Success, Try} -import scalaz.Scalaz._ -import scalaz.ValidationNel object WorkflowQueryKey { val ValidKeys = Set(StartDate, EndDate, Name, Id, Status, Page, PageSize) map { _.name } @@ -35,38 +37,38 @@ object WorkflowQueryKey { case object Name extends SeqStringWorkflowQueryKey { override val name = "Name" - override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[Seq[String]] = { + override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[List[String]] = { val values = valuesFromMap(grouped).toList val nels = values map { - case Patterns.WorkflowName(n) => n.successNel[String] - case v => v.failureNel + case Patterns.WorkflowName(n) => n.validNel[String] + case v => v.invalidNel[String] } - sequenceListOfValidationNels(s"Name values do not match allowed workflow naming pattern", nels) + sequenceListOfValidatedNels(s"Name values do not match allowed workflow naming pattern", nels) } } case object Id extends SeqStringWorkflowQueryKey { override val name = "Id" - override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[Seq[String]] = { + override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[List[String]] = { val values = valuesFromMap(grouped).toList val nels = values map { v => - if (Try(WorkflowId.fromString(v.toLowerCase.capitalize)).isSuccess) v.successNel[String] else v.failureNel + if (Try(WorkflowId.fromString(v.toLowerCase.capitalize)).isSuccess) v.validNel[String] else v.invalidNel[String] } - sequenceListOfValidationNels(s"Id values do match allowed workflow id pattern", nels) + sequenceListOfValidatedNels(s"Id values do match allowed workflow id pattern", nels) } } case object Status extends SeqStringWorkflowQueryKey { override val name = "Status" - override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[Seq[String]] = { + override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[List[String]] = { val values = valuesFromMap(grouped).toList val nels = values map { v => - if (Try(WorkflowState.fromString(v.toLowerCase.capitalize)).isSuccess) v.successNel[String] else v.failureNel + if (Try(WorkflowState.fromString(v.toLowerCase.capitalize)).isSuccess) v.validNel[String] else v.invalidNel[String] } - sequenceListOfValidationNels("Unrecognized status values", nels) + sequenceListOfValidatedNels("Unrecognized status values", nels) } } } @@ -83,12 +85,12 @@ sealed trait DateTimeWorkflowQueryKey extends WorkflowQueryKey[Option[OffsetDate override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[Option[OffsetDateTime]] = { valuesFromMap(grouped) match { case vs if vs.size > 1 => - s"Found ${vs.size} values for key '$name' but at most one is allowed.".failureNel - case Nil => None.successNel + s"Found ${vs.size} values for key '$name' but at most one is allowed.".invalidNel[Option[OffsetDateTime]] + case Nil => None.validNel[String] case v :: Nil => Try(OffsetDateTime.parse(v)) match { - case Success(dt) => Option(dt).successNel - case _ => s"Value given for $displayName does not parse as a datetime: $v".failureNel + case Success(dt) => Option(dt).validNel[String] + case _ => s"Value given for $displayName does not parse as a datetime: $v".invalidNel[Option[OffsetDateTime]] } } } @@ -97,11 +99,11 @@ sealed trait DateTimeWorkflowQueryKey extends WorkflowQueryKey[Option[OffsetDate sealed trait SeqStringWorkflowQueryKey extends WorkflowQueryKey[Seq[String]] { /** `sequence` the `List[ErrorOr[String]]` to a single `ErrorOr[List[String]]` */ - protected def sequenceListOfValidationNels(prefix: String, errorOrList: List[ErrorOr[String]]): ErrorOr[List[String]] = { + protected def sequenceListOfValidatedNels(prefix: String, errorOrList: List[ErrorOr[String]]): ErrorOr[List[String]] = { val errorOr = errorOrList.sequence[ErrorOr, String] // With a leftMap, prepend an error message to the concatenated error values if there are error values. - // This turns the ValidationNel into a Validation, force it back to a ValidationNel with toValidationNel. - errorOr.leftMap(prefix + ": " + _.list.toList.mkString(", ")).toValidationNel + // This turns the ValidatedNel into a Validated, force it back to a ValidatedNel with toValidationNel. + errorOr.leftMap(prefix + ": " + _.toList.mkString(", ")).toValidatedNel } } @@ -109,12 +111,12 @@ sealed trait IntWorkflowQueryKey extends WorkflowQueryKey[Option[Int]] { override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[Option[Int]] = { valuesFromMap(grouped) match { case vs if vs.size > 1 => - s"Found ${vs.size} values for key '$name' but at most one is allowed.".failureNel - case Nil => None.successNel + s"Found ${vs.size} values for key '$name' but at most one is allowed.".invalidNel[Option[Int]] + case Nil => None.validNel case v :: Nil => Try(v.toInt) match { - case Success(intVal) => if (intVal > 0) Option(intVal).successNel else s"Integer value not greater than 0".failureNel - case _ => s"Value given for $displayName does not parse as a integer: $v".failureNel + case Success(intVal) => if (intVal > 0) Option(intVal).validNel else s"Integer value not greater than 0".invalidNel[Option[Int]] + case _ => s"Value given for $displayName does not parse as a integer: $v".invalidNel[Option[Int]] } } } diff --git a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala index a7952d1b4..669bde613 100644 --- a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala +++ b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala @@ -2,13 +2,14 @@ package cromwell.services.metadata import java.time.OffsetDateTime +import cats.data.Validated._ +import cats.syntax.cartesian._ +import cats.syntax.validated._ import cromwell.core.WorkflowId import cromwell.services.metadata.WorkflowQueryKey._ +import cromwell.core.ErrorOr._ import scala.language.postfixOps -import scalaz.Scalaz._ -import scalaz.{Name => _, _} - case class WorkflowQueryParameters private(statuses: Set[String], names: Set[String], @@ -20,7 +21,7 @@ case class WorkflowQueryParameters private(statuses: Set[String], object WorkflowQueryParameters { - private def validateStartBeforeEnd(start: Option[OffsetDateTime], end: Option[OffsetDateTime]): ValidationNel[String, Unit] = { + private def validateStartBeforeEnd(start: Option[OffsetDateTime], end: Option[OffsetDateTime]): ErrorOr[Unit] = { // Invert the notion of success/failure here to only "successfully" generate an error message if // both start and end dates have been specified and start is after end. val startAfterEndError = for { @@ -30,10 +31,10 @@ object WorkflowQueryParameters { } yield s"Specified start date is after specified end date: start: $s, end: $e" // If the Option is defined this represents a failure, if it's empty this is a success. - startAfterEndError map { _.failureNel } getOrElse ().successNel + startAfterEndError map { _.invalidNel } getOrElse ().validNel } - private def validateOnlyRecognizedKeys(rawParameters: Seq[(String, String)]): ValidationNel[String, Unit] = { + private def validateOnlyRecognizedKeys(rawParameters: Seq[(String, String)]): ErrorOr[Unit] = { // Create a map of keys by canonical capitalization (capitalized first letter, lowercase everything else). // The values are the keys capitalized as actually given to the API, which is what will be used in any // error messages. @@ -43,8 +44,8 @@ object WorkflowQueryParameters { keysByCanonicalCapitalization.keys.toSet -- WorkflowQueryKey.ValidKeys match { case set if set.nonEmpty => val unrecognized = set flatMap keysByCanonicalCapitalization - ("Unrecognized query keys: " + unrecognized.mkString(", ")).failureNel - case _ => ().successNel + ("Unrecognized query keys: " + unrecognized.mkString(", ")).invalidNel + case _ => ().validNel } } @@ -52,7 +53,7 @@ object WorkflowQueryParameters { * Run the validation logic over the specified raw parameters, creating a `WorkflowQueryParameters` if all * validation succeeds, otherwise accumulate all validation messages within the `ValidationNel`. */ - private [metadata] def runValidation(rawParameters: Seq[(String, String)]): ValidationNel[String, WorkflowQueryParameters] = { + private [metadata] def runValidation(rawParameters: Seq[(String, String)]): ErrorOr[WorkflowQueryParameters] = { val onlyRecognizedKeys = validateOnlyRecognizedKeys(rawParameters) @@ -68,11 +69,11 @@ object WorkflowQueryParameters { // Only validate start before end if both of the individual date parsing validations have already succeeded. val startBeforeEnd = (startDate, endDate) match { - case (Success(s), Success(e)) => validateStartBeforeEnd(s, e) - case _ => ().successNel[String] + case (Valid(s), Valid(e)) => validateStartBeforeEnd(s, e) + case _ => ().validNel[String] } - (onlyRecognizedKeys |@| startBeforeEnd |@| statuses |@| names |@| ids |@| startDate |@| endDate |@| page |@| pageSize) { + (onlyRecognizedKeys |@| startBeforeEnd |@| statuses |@| names |@| ids |@| startDate |@| endDate |@| page |@| pageSize) map { case (_, _, status, name, uuid, start, end, _page, _pageSize) => val workflowId = uuid map WorkflowId.fromString WorkflowQueryParameters(status.toSet, name.toSet, workflowId.toSet, start, end, _page, _pageSize) @@ -81,8 +82,8 @@ object WorkflowQueryParameters { def apply(rawParameters: Seq[(String, String)]): WorkflowQueryParameters = { runValidation(rawParameters) match { - case Success(queryParameters) => queryParameters - case Failure(x) => throw new IllegalArgumentException(x.list.toList.mkString("\n")) + case Valid(queryParameters) => queryParameters + case Invalid(x) => throw new IllegalArgumentException(x.toList.mkString("\n")) } } } diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala index dfd14c94a..92ba34589 100644 --- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala +++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala @@ -2,6 +2,9 @@ package cromwell.services.metadata.impl import java.time.OffsetDateTime +import cats.Semigroup +import cats.data.NonEmptyList +import cats.syntax.semigroup._ import cromwell.core.{WorkflowId, WorkflowMetadataKeys, WorkflowState} import cromwell.database.sql.SqlConverters._ import cromwell.database.sql.tables.{MetadataEntry, WorkflowMetadataSummaryEntry} @@ -10,14 +13,12 @@ import cromwell.services.metadata.MetadataService.{QueryMetadata, WorkflowQueryR import cromwell.services.metadata._ import scala.concurrent.{ExecutionContext, Future} -import scalaz.Scalaz._ -import scalaz.{NonEmptyList, Semigroup} object MetadataDatabaseAccess { private lazy val WorkflowMetadataSummarySemigroup = new Semigroup[WorkflowMetadataSummaryEntry] { - override def append(summary1: WorkflowMetadataSummaryEntry, - summary2: => WorkflowMetadataSummaryEntry): WorkflowMetadataSummaryEntry = { + override def combine(summary1: WorkflowMetadataSummaryEntry, + summary2: WorkflowMetadataSummaryEntry): WorkflowMetadataSummaryEntry = { // Resolve the status if both `this` and `that` have defined statuses. This will evaluate to `None` // if one or both of the statuses is not defined. val resolvedStatus = for { @@ -131,7 +132,7 @@ trait MetadataDatabaseAccess { (implicit ec: ExecutionContext): Future[Seq[MetadataEvent]] = { val uuid = id.id.toString databaseInterface.queryMetadataEntriesLikeMetadataKeys( - uuid, s"${WorkflowMetadataKeys.Outputs}:%".wrapNel, requireEmptyJobKey = true). + uuid, NonEmptyList.of(s"${WorkflowMetadataKeys.Outputs}:%"), requireEmptyJobKey = true). map(metadataToMetadataEvents(id)) } @@ -139,7 +140,7 @@ trait MetadataDatabaseAccess { (implicit ec: ExecutionContext): Future[Seq[MetadataEvent]] = { import cromwell.services.metadata.CallMetadataKeys._ - val keys = NonEmptyList(Stdout, Stderr, BackendLogsPrefix + ":%") + val keys = NonEmptyList.of(Stdout, Stderr, BackendLogsPrefix + ":%") databaseInterface.queryMetadataEntriesLikeMetadataKeys(id.id.toString, keys, requireEmptyJobKey = false) map metadataToMetadataEvents(id) } diff --git a/services/src/test/scala/cromwell/services/metadata/WorkflowQueryParametersSpec.scala b/services/src/test/scala/cromwell/services/metadata/WorkflowQueryParametersSpec.scala index 50ae75fe0..161d5b198 100644 --- a/services/src/test/scala/cromwell/services/metadata/WorkflowQueryParametersSpec.scala +++ b/services/src/test/scala/cromwell/services/metadata/WorkflowQueryParametersSpec.scala @@ -2,10 +2,9 @@ package cromwell.services.metadata import java.time.OffsetDateTime +import cats.data.Validated._ import cromwell.services.metadata.WorkflowQueryKey._ -import org.scalatest.{WordSpec, Matchers} - -import scalaz.{Name => _, _} +import org.scalatest.{Matchers, WordSpec} class WorkflowQueryParametersSpec extends WordSpec with Matchers { @@ -17,13 +16,13 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { "be accepted if empty" in { val result = WorkflowQueryParameters.runValidation(Seq.empty) result match { - case Success(r) => + case Valid(r) => r.startDate should be('empty) r.endDate should be('empty) r.names should be('empty) r.statuses should be('empty) - case Failure(fs) => - throw new RuntimeException(fs.list.toList.mkString(", ")) + case Invalid(fs) => + throw new RuntimeException(fs.toList.mkString(", ")) } } @@ -38,13 +37,13 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { ) val result = WorkflowQueryParameters.runValidation(rawParameters) result match { - case Success(r) => + case Valid(r) => r.startDate.get.toInstant should equal(OffsetDateTime.parse(StartDateString).toInstant) r.endDate.get.toInstant should equal(OffsetDateTime.parse(EndDateString).toInstant) r.names should be(Set("my_workflow", "my_other_workflow")) r.statuses should be(Set("Succeeded", "Running")) - case Failure(fs) => - throw new RuntimeException(fs.list.toList.mkString(", ")) + case Invalid(fs) => + throw new RuntimeException(fs.toList.mkString(", ")) } } @@ -55,11 +54,11 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { ) val result = WorkflowQueryParameters.runValidation(rawParameters) result match { - case Success(r) => + case Valid(r) => throw new RuntimeException(s"Unexpected success: $r") - case Failure(fs) => - fs.list.toList should have size 1 - fs.list.toList.head should include("Unrecognized query keys: Bogosity") + case Invalid(fs) => + fs.toList should have size 1 + fs.toList.head should include("Unrecognized query keys: Bogosity") } } @@ -71,11 +70,11 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { ) val result = WorkflowQueryParameters.runValidation(rawParameters) result match { - case Success(r) => + case Valid(r) => throw new RuntimeException(s"Unexpected success: $r") - case Failure(fs) => - fs.list.toList should have size 1 - fs.list.toList.head should include("Specified start date is after specified end date") + case Invalid(fs) => + fs.toList should have size 1 + fs.toList.head should include("Specified start date is after specified end date") } } @@ -88,11 +87,11 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { ) val result = WorkflowQueryParameters.runValidation(rawParameters) result match { - case Success(r) => + case Valid(r) => throw new RuntimeException(s"Unexpected success: $r") - case Failure(fs) => - fs.list.toList should have size 1 - fs.list.toList.head should include("Name values do not match allowed workflow naming pattern") + case Invalid(fs) => + fs.toList should have size 1 + fs.toList.head should include("Name values do not match allowed workflow naming pattern") } } @@ -104,11 +103,11 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { ) val result = WorkflowQueryParameters.runValidation(rawParameters) result match { - case Success(r) => + case Valid(r) => throw new RuntimeException(s"Unexpected success: $r") - case Failure(fs) => - fs.list.toList should have size 1 - fs.list.toList.head should include("does not parse as a datetime") + case Invalid(fs) => + fs.toList should have size 1 + fs.toList.head should include("does not parse as a datetime") } } @@ -119,11 +118,11 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { ) val result = WorkflowQueryParameters.runValidation(rawParameters) result match { - case Success(r) => + case Valid(r) => throw new RuntimeException(s"Unexpected success: $r") - case Failure(fs) => - fs.list.toList should have size 1 - fs.list.toList.head should include("at most one is allowed") + case Invalid(fs) => + fs.toList should have size 1 + fs.toList.head should include("at most one is allowed") } } @@ -135,11 +134,11 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { ) val result = WorkflowQueryParameters.runValidation(rawParameters) result match { - case Success(r) => + case Valid(r) => throw new RuntimeException(s"Unexpected success: $r") - case Failure(fs) => - fs.list.toList should have size 1 - fs.list.toList.head should be("Unrecognized status values: Moseying") + case Invalid(fs) => + fs.toList should have size 1 + fs.toList.head should be("Unrecognized status values: Moseying") } } @@ -152,13 +151,13 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers { ) val result = WorkflowQueryParameters.runValidation(rawParameters) result match { - case Success(r) => + case Valid(r) => throw new RuntimeException(s"Unexpected success: $r") - case Failure(fs) => - fs.list.toList should have size 3 - fs.list.toList find { _ == "Unrecognized status values: Moseying" } getOrElse fail - fs.list.toList find { _ contains "does not parse as a datetime" } getOrElse fail - fs.list.toList find { _ contains "Name values do not match allowed workflow naming pattern" } getOrElse fail + case Invalid(fs) => + fs.toList should have size 3 + fs.toList find { _ == "Unrecognized status values: Moseying" } getOrElse fail + fs.toList find { _ contains "does not parse as a datetime" } getOrElse fail + fs.toList find { _ contains "Name values do not match allowed workflow naming pattern" } getOrElse fail } } } diff --git a/src/main/scala/cromwell/CromwellCommandLine.scala b/src/main/scala/cromwell/CromwellCommandLine.scala index 192242a5e..c52ebbd66 100644 --- a/src/main/scala/cromwell/CromwellCommandLine.scala +++ b/src/main/scala/cromwell/CromwellCommandLine.scala @@ -3,12 +3,15 @@ package cromwell import java.nio.file.{Files, Path, Paths} import better.files._ -import cromwell.core.{ErrorOr, WorkflowSourceFiles} +import cats.data.Validated._ +import cats.syntax.cartesian._ +import cats.syntax.validated._ +import cromwell.core.WorkflowSourceFiles import cromwell.util.FileUtil._ import lenthall.exception.MessageAggregation +import cromwell.core.ErrorOr._ import scala.util.{Failure, Success, Try} -import scalaz.Scalaz._ sealed abstract class CromwellCommandLine case object UsageAndExit extends CromwellCommandLine @@ -40,44 +43,43 @@ object RunSingle { val inputsJson = readJson("Inputs", inputsPath) val optionsJson = readJson("Workflow Options", optionsPath) - val sourceFiles = (wdl |@| inputsJson |@| optionsJson) { WorkflowSourceFiles.apply } + val sourceFiles = (wdl |@| inputsJson |@| optionsJson) map { WorkflowSourceFiles.apply } - import scalaz.Validation.FlatMap._ val runSingle = for { sources <- sourceFiles _ <- writeableMetadataPath(metadataPath) } yield RunSingle(wdlPath, sources, inputsPath, optionsPath, metadataPath) runSingle match { - case scalaz.Success(r) => r - case scalaz.Failure(nel) => throw new RuntimeException with MessageAggregation { + case Valid(r) => r + case Invalid(nel) => throw new RuntimeException with MessageAggregation { override def exceptionContext: String = "ERROR: Unable to run Cromwell:" - override def errorMessages: Traversable[String] = nel.list.toList + override def errorMessages: Traversable[String] = nel.toList } } } private def writeableMetadataPath(path: Option[Path]): ErrorOr[Unit] = { path match { - case Some(p) if !metadataPathIsWriteable(p) => s"Unable to write to metadata directory: $p".failureNel - case otherwise => ().successNel + case Some(p) if !metadataPathIsWriteable(p) => s"Unable to write to metadata directory: $p".invalidNel + case otherwise => ().validNel } } /** Read the path to a string. */ private def readContent(inputDescription: String, path: Path): ErrorOr[String] = { if (!Files.exists(path)) { - s"$inputDescription does not exist: $path".failureNel + s"$inputDescription does not exist: $path".invalidNel } else if (!Files.isReadable(path)) { - s"$inputDescription is not readable: $path".failureNel - } else File(path).contentAsString.successNel + s"$inputDescription is not readable: $path".invalidNel + } else File(path).contentAsString.validNel } /** Read the path to a string, unless the path is None, in which case returns "{}". */ private def readJson(inputDescription: String, pathOption: Option[Path]): ErrorOr[String] = { pathOption match { case Some(path) => readContent(inputDescription, path) - case None => "{}".successNel + case None => "{}".validNel } } diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala index 1ddb35c35..f8dd9a595 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala @@ -1,17 +1,20 @@ package cromwell.backend.impl.htcondor + +import cats.data.Validated.{Invalid, Valid} +import cats.syntax.cartesian._ +import cats.syntax.validated._ import cromwell.backend.MemorySize import cromwell.backend.validation.ContinueOnReturnCode import cromwell.backend.validation.RuntimeAttributesDefault._ import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.backend.validation.RuntimeAttributesValidation._ import cromwell.core._ +import cromwell.core.ErrorOr._ import lenthall.exception.MessageAggregation import wdl4s.types.{WdlIntegerType, WdlStringType, WdlBooleanType, WdlType} import wdl4s.values.{WdlString, WdlBoolean, WdlInteger, WdlValue} -import scalaz.Scalaz._ -import scalaz._ object HtCondorRuntimeAttributes { private val FailOnStderrDefaultValue = false @@ -48,39 +51,39 @@ object HtCondorRuntimeAttributes { val defaultFromOptions = workflowOptionsDefault(options, coercionMap).get val withDefaultValues = withDefaults(attrs, List(defaultFromOptions, staticDefaults)) - val docker = validateDocker(withDefaultValues.get(DockerKey), None.successNel) - val dockerWorkingDir = validateDockerWorkingDir(withDefaultValues.get(DockerWorkingDirKey), None.successNel) - val dockerOutputDir = validateDockerOutputDir(withDefaultValues.get(DockerOutputDirKey), None.successNel) + val docker = validateDocker(withDefaultValues.get(DockerKey), None.validNel) + val dockerWorkingDir = validateDockerWorkingDir(withDefaultValues.get(DockerWorkingDirKey), None.validNel) + val dockerOutputDir = validateDockerOutputDir(withDefaultValues.get(DockerOutputDirKey), None.validNel) val failOnStderr = validateFailOnStderr(withDefaultValues.get(FailOnStderrKey), noValueFoundFor(FailOnStderrKey)) val continueOnReturnCode = validateContinueOnReturnCode(withDefaultValues.get(ContinueOnReturnCodeKey), noValueFoundFor(ContinueOnReturnCodeKey)) val cpu = validateCpu(withDefaultValues.get(CpuKey), noValueFoundFor(CpuKey)) val memory = validateMemory(withDefaultValues.get(MemoryKey), noValueFoundFor(MemoryKey)) val disk = validateDisk(withDefaultValues.get(DiskKey), noValueFoundFor(DiskKey)) - (continueOnReturnCode |@| docker |@| dockerWorkingDir |@| dockerOutputDir |@| failOnStderr |@| cpu |@| memory |@| disk) { + (continueOnReturnCode |@| docker |@| dockerWorkingDir |@| dockerOutputDir |@| failOnStderr |@| cpu |@| memory |@| disk) map { new HtCondorRuntimeAttributes(_, _, _, _, _, _, _, _) } match { - case Success(x) => x - case Failure(nel) => throw new RuntimeException with MessageAggregation { + case Valid(x) => x + case Invalid(nel) => throw new RuntimeException with MessageAggregation { override def exceptionContext: String = "Runtime attribute validation failed" - override def errorMessages: Traversable[String] = nel.list.toList + override def errorMessages: Traversable[String] = nel.toList } } } private def validateDockerWorkingDir(dockerWorkingDir: Option[WdlValue], onMissingKey: => ErrorOr[Option[String]]): ErrorOr[Option[String]] = { dockerWorkingDir match { - case Some(WdlString(s)) => Some(s).successNel + case Some(WdlString(s)) => Some(s).validNel case None => onMissingKey - case _ => s"Expecting $DockerWorkingDirKey runtime attribute to be a String".failureNel + case _ => s"Expecting $DockerWorkingDirKey runtime attribute to be a String".invalidNel } } private def validateDockerOutputDir(dockerOutputDir: Option[WdlValue], onMissingKey: => ErrorOr[Option[String]]): ErrorOr[Option[String]] = { dockerOutputDir match { - case Some(WdlString(s)) => Some(s).successNel + case Some(WdlString(s)) => Some(s).validNel case None => onMissingKey - case _ => s"Expecting $DockerOutputDirKey runtime attribute to be a String".failureNel + case _ => s"Expecting $DockerOutputDirKey runtime attribute to be a String".invalidNel } } @@ -90,7 +93,7 @@ object HtCondorRuntimeAttributes { value match { case Some(i: WdlInteger) => parseMemoryInteger(i) case Some(s: WdlString) => parseMemoryString(s) - case Some(_) => String.format(diskWrongFormatMsg, "Not supported WDL type value").failureNel + case Some(_) => String.format(diskWrongFormatMsg, "Not supported WDL type value").invalidNel case None => onMissingKey } } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala index fc10cff9b..8fd05c348 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala @@ -2,18 +2,20 @@ package cromwell.backend.impl.jes import java.net.URL +import cats.data._ +import cats.data.Validated._ +import cats.syntax.cartesian._ +import cats.syntax.validated._ import com.typesafe.config.Config import cromwell.backend.impl.jes.JesImplicits.GoogleAuthWorkflowOptions -import cromwell.core.{ErrorOr, WorkflowOptions} +import cromwell.core.WorkflowOptions import cromwell.filesystems.gcs.{GoogleAuthMode, GoogleConfiguration} import lenthall.config.ScalaConfig._ import lenthall.config.ValidatedConfig._ +import cromwell.core.ErrorOr._ import wdl4s.ExceptionWithErrors import scala.language.postfixOps -import scalaz.Scalaz._ -import scalaz.Validation.FlatMap._ -import scalaz._ case class JesAttributes(project: String, genomicsAuth: GoogleAuthMode, @@ -44,22 +46,22 @@ object JesAttributes { def apply(googleConfig: GoogleConfiguration, backendConfig: Config): JesAttributes = { backendConfig.warnNotRecognized(jesKeys, context) - val project: ErrorOr[String] = backendConfig.validateString("project") - val executionBucket: ErrorOr[String] = backendConfig.validateString("root") + val project: ValidatedNel[String, String] = backendConfig.validateString("project") + val executionBucket: ValidatedNel[String, String] = backendConfig.validateString("root") val endpointUrl: ErrorOr[URL] = backendConfig.validateURL("genomics.endpoint-url") val maxPollingInterval: Int = backendConfig.getIntOption("maximum-polling-interval").getOrElse(600) val genomicsAuthName: ErrorOr[String] = backendConfig.validateString("genomics.auth") val gcsFilesystemAuthName: ErrorOr[String] = backendConfig.validateString("filesystems.gcs.auth") - (project |@| executionBucket |@| endpointUrl |@| genomicsAuthName |@| gcsFilesystemAuthName) { + (project |@| executionBucket |@| endpointUrl |@| genomicsAuthName |@| gcsFilesystemAuthName) map { (_, _, _, _, _) } flatMap { case (p, b, u, genomicsName, gcsName) => - (googleConfig.auth(genomicsName) |@| googleConfig.auth(gcsName)) { case (genomicsAuth, gcsAuth) => + (googleConfig.auth(genomicsName) |@| googleConfig.auth(gcsName)) map { case (genomicsAuth, gcsAuth) => JesAttributes(p, genomicsAuth, gcsAuth, b, u, maxPollingInterval) } } match { - case Success(r) => r - case Failure(f) => + case Valid(r) => r + case Invalid(f) => throw new IllegalArgumentException with ExceptionWithErrors { override val message = "Jes Configuration is not valid: Errors" override val errors = f diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala index 5e4a42ea5..19891ee61 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala @@ -1,18 +1,21 @@ package cromwell.backend.impl.jes +import cats.data.Validated._ +import cats.syntax.cartesian._ +import cats.syntax.validated._ import cromwell.backend.MemorySize -import cromwell.backend.impl.jes.io.{JesWorkingDisk, JesAttachedDisk} +import cromwell.backend.impl.jes.io.{JesAttachedDisk, JesWorkingDisk} +import cromwell.backend.validation.RuntimeAttributesDefault._ import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.backend.validation.RuntimeAttributesValidation._ import cromwell.backend.validation._ import cromwell.core._ import lenthall.exception.MessageAggregation +import cromwell.core.ErrorOr._ +import mouse.string._ import org.slf4j.Logger import wdl4s.types._ import wdl4s.values._ -import cromwell.backend.validation.RuntimeAttributesDefault._ -import scalaz._ -import Scalaz._ case class JesRuntimeAttributes(cpu: Int, zones: Vector[String], @@ -98,28 +101,28 @@ object JesRuntimeAttributes { val noAddress = validateNoAddress(attrs(NoAddressKey)) val bootDiskSize = validateBootDisk(attrs(BootDiskSizeKey)) val disks = validateLocalDisks(attrs(DisksKey)) - (cpu |@| zones |@| preemptible |@| bootDiskSize |@| memory |@| disks |@| docker |@| failOnStderr |@| continueOnReturnCode |@| noAddress) { + (cpu |@| zones |@| preemptible |@| bootDiskSize |@| memory |@| disks |@| docker |@| failOnStderr |@| continueOnReturnCode |@| noAddress) map { new JesRuntimeAttributes(_, _, _, _, _, _, _, _, _, _) } match { - case Success(x) => x - case Failure(nel) => throw new RuntimeException with MessageAggregation { + case Valid(x) => x + case Invalid(nel) => throw new RuntimeException with MessageAggregation { override def exceptionContext: String = "Runtime attribute validation failed" - override def errorMessages: Traversable[String] = nel.list.toList + override def errorMessages: Traversable[String] = nel.toList } } } private def validateZone(zoneValue: WdlValue): ErrorOr[Vector[String]] = { zoneValue match { - case WdlString(s) => s.split("\\s+").toVector.successNel + case WdlString(s) => s.split("\\s+").toVector.validNel case WdlArray(wdlType, value) if wdlType.memberType == WdlStringType => - value.map(_.valueString).toVector.successNel - case _ => s"Expecting $ZonesKey runtime attribute to be either a whitespace separated String or an Array[String]".failureNel + value.map(_.valueString).toVector.validNel + case _ => s"Expecting $ZonesKey runtime attribute to be either a whitespace separated String or an Array[String]".invalidNel } } private def contextualizeFailure[T](validation: ErrorOr[T], key: String): ErrorOr[T] = { - validation.leftMap[String](errors => s"Failed to validate $key runtime attribute: " + errors.toList.mkString(",")).toValidationNel + validation.leftMap[String](errors => s"Failed to validate $key runtime attribute: " + errors.toList.mkString(",")).toValidatedNel } private def validatePreemptible(preemptible: WdlValue): ErrorOr[Int] = { @@ -133,23 +136,23 @@ object JesRuntimeAttributes { private def validateBootDisk(diskSize: WdlValue): ErrorOr[Int] = diskSize match { case x if WdlIntegerType.isCoerceableFrom(x.wdlType) => WdlIntegerType.coerceRawValue(x) match { - case scala.util.Success(x: WdlInteger) => x.value.intValue.successNel - case scala.util.Success(unhandled) => s"Coercion was expected to create an Integer but instead got $unhandled".failureNel - case scala.util.Failure(t) => s"Expecting $BootDiskSizeKey runtime attribute to be an Integer".failureNel + case scala.util.Success(x: WdlInteger) => x.value.intValue.validNel + case scala.util.Success(unhandled) => s"Coercion was expected to create an Integer but instead got $unhandled".invalidNel + case scala.util.Failure(t) => s"Expecting $BootDiskSizeKey runtime attribute to be an Integer".invalidNel } } private def validateLocalDisks(value: WdlValue): ErrorOr[Seq[JesAttachedDisk]] = { - val nels = value match { + val nels: Seq[ErrorOr[JesAttachedDisk]] = value match { case WdlString(s) => s.split(",\\s*").toSeq.map(validateLocalDisk) case WdlArray(wdlType, seq) if wdlType.memberType == WdlStringType => seq.map(_.valueString).map(validateLocalDisk) case _ => - Seq(s"Expecting $DisksKey runtime attribute to be a comma separated String or Array[String]".failureNel[JesAttachedDisk]) + Seq(s"Expecting $DisksKey runtime attribute to be a comma separated String or Array[String]".invalidNel) } - val emptyDiskNel = Vector.empty[JesAttachedDisk].successNel[String] - val disksNel = nels.foldLeft(emptyDiskNel)((acc, v) => (acc |@| v) { (a, v) => a :+ v }) + val emptyDiskNel = Vector.empty[JesAttachedDisk].validNel[String] + val disksNel = nels.foldLeft(emptyDiskNel)((acc, v) => (acc |@| v) map { (a, v) => a :+ v }) disksNel map { case disks if disks.exists(_.name == JesWorkingDisk.Name) => disks @@ -159,8 +162,8 @@ object JesRuntimeAttributes { private def validateLocalDisk(disk: String): ErrorOr[JesAttachedDisk] = { JesAttachedDisk.parse(disk) match { - case scala.util.Success(localDisk) => localDisk.successNel - case scala.util.Failure(ex) => ex.getMessage.failureNel + case scala.util.Success(localDisk) => localDisk.validNel + case scala.util.Failure(ex) => ex.getMessage.invalidNel } } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala index 32b558b53..848b188c1 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala @@ -2,14 +2,17 @@ package cromwell.backend.impl.jes.io import java.nio.file.{Path, Paths} +import cats.data.Validated._ +import cats.syntax.cartesian._ +import cats.syntax.validated._ import com.google.api.services.genomics.model.Disk -import cromwell.core.ErrorOr +import cromwell.core.ErrorOr._ +import mouse.string._ import wdl4s.ExceptionWithErrors import wdl4s.values._ import scala.util.Try -import scalaz.Scalaz._ -import scalaz._ + object JesAttachedDisk { val Identifier = "[a-zA-Z0-9-_]+" @@ -19,21 +22,21 @@ object JesAttachedDisk { val MountedDiskPattern = s"""($Directory)\\s+($Integer)\\s+($Identifier)""".r def parse(s: String): Try[JesAttachedDisk] = { - val validation = s match { + val validation: ErrorOr[JesAttachedDisk] = s match { case WorkingDiskPattern(sizeGb, diskType) => - (validateLong(sizeGb) |@| validateDiskType(diskType)) { (s, dt) => + (validateLong(sizeGb) |@| validateDiskType(diskType)) map { (s, dt) => JesWorkingDisk(dt, s.toInt) } case MountedDiskPattern(mountPoint, sizeGb, diskType) => - (validateLong(sizeGb) |@| validateDiskType(diskType)) { (s, dt) => + (validateLong(sizeGb) |@| validateDiskType(diskType)) map { (s, dt) => JesEmptyMountedDisk(dt, s.toInt, Paths.get(mountPoint)) } - case _ => s"Disk strings should be of the format 'local-disk SIZE TYPE' or '/mount/point SIZE TYPE'".failureNel + case _ => s"Disk strings should be of the format 'local-disk SIZE TYPE' or '/mount/point SIZE TYPE'".invalidNel } Try(validation match { - case Success(localDisk) => localDisk - case Failure(nels) => + case Valid(localDisk) => localDisk + case Invalid(nels) => throw new UnsupportedOperationException with ExceptionWithErrors { val message = "" val errors = nels @@ -43,18 +46,18 @@ object JesAttachedDisk { private def validateDiskType(diskTypeName: String): ErrorOr[DiskType] = { DiskType.values().find(_.diskTypeName == diskTypeName) match { - case Some(diskType) => diskType.successNel[String] + case Some(diskType) => diskType.validNel case None => val diskTypeNames = DiskType.values.map(_.diskTypeName).mkString(", ") - s"Disk TYPE $diskTypeName should be one of $diskTypeNames".failureNel + s"Disk TYPE $diskTypeName should be one of $diskTypeNames".invalidNel } } private def validateLong(value: String): ErrorOr[Long] = { try { - value.toLong.successNel + value.toLong.validNel } catch { - case _: IllegalArgumentException => s"$value not convertible to a Long".failureNel[Long] + case _: IllegalArgumentException => s"$value not convertible to a Long".invalidNel } } } diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala index 020be850b..17c33d0de 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala @@ -52,7 +52,7 @@ class JesAttributesSpec extends FlatSpec with Matchers { val exception = intercept[IllegalArgumentException with ExceptionWithErrors] { JesAttributes(googleConfig, nakedConfig) } - val errorsList = exception.errors.list.toList + val errorsList = exception.errors.toList errorsList should contain("Could not find key: project") errorsList should contain("Could not find key: root") errorsList should contain("Could not find key: genomics.auth") diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/PrimitiveRuntimeAttributesValidation.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/PrimitiveRuntimeAttributesValidation.scala index deec1bcf9..16fed1f3d 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/PrimitiveRuntimeAttributesValidation.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/PrimitiveRuntimeAttributesValidation.scala @@ -1,11 +1,10 @@ package cromwell.backend.impl.sfs.config +import cats.syntax.validated._ import cromwell.backend.validation.RuntimeAttributesValidation import wdl4s.types._ import wdl4s.values.{WdlBoolean, WdlFloat, WdlInteger, WdlString} -import scalaz.Scalaz._ - /** * Validates one of the wdl primitive types: Boolean, Float, Integer, or String. WdlFile is not supported. * @@ -23,7 +22,7 @@ class BooleanRuntimeAttributesValidation(override val key: String) extends override val wdlType = WdlBooleanType override protected def validateValue = { - case WdlBoolean(value) => value.successNel + case WdlBoolean(value) => value.validNel } } @@ -31,7 +30,7 @@ class FloatRuntimeAttributesValidation(override val key: String) extends Primiti override val wdlType = WdlFloatType override protected def validateValue = { - case WdlFloat(value) => value.successNel + case WdlFloat(value) => value.validNel } } @@ -39,7 +38,7 @@ class IntRuntimeAttributesValidation(override val key: String) extends Primitive override val wdlType = WdlIntegerType override protected def validateValue = { - case WdlInteger(value) => value.toInt.successNel + case WdlInteger(value) => value.toInt.validNel } } @@ -47,6 +46,6 @@ class StringRuntimeAttributesValidation(override val key: String) extends Primit override val wdlType = WdlStringType override protected def validateValue = { - case WdlString(value) => value.successNel + case WdlString(value) => value.validNel } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala index 35cbd97d6..4cde3b7c9 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala @@ -1,5 +1,6 @@ package cromwell.backend.sfs +import cats.data.Validated.{Invalid, Valid} import cromwell.backend.wfs.{WorkflowFileSystemProvider, WorkflowFileSystemProviderParams} import cromwell.filesystems.gcs.GoogleAuthMode.GoogleAuthOptions import cromwell.filesystems.gcs.{GcsFileSystem, GcsFileSystemProvider, GoogleConfiguration} @@ -20,8 +21,8 @@ object GcsWorkflowFileSystemProvider extends WorkflowFileSystemProvider { val googleAuthModeValidation = googleConfig.auth(gcsAuthName) val gcsAuthMode = googleAuthModeValidation match { - case scalaz.Success(googleAuthMode) => googleAuthMode - case scalaz.Failure(errors) => + case Valid(googleAuthMode) => googleAuthMode + case Invalid(errors) => throw new ValidationException("Could not create gcs filesystem from configuration", errors) } diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkRuntimeAttributes.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkRuntimeAttributes.scala index 1a951f4b7..778b85a86 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkRuntimeAttributes.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkRuntimeAttributes.scala @@ -1,16 +1,18 @@ package cromwell.backend.impl.spark +import cats.data.Validated.{Invalid, Valid} +import cats.syntax.cartesian._ +import cats.syntax.validated._ import cromwell.backend.MemorySize import cromwell.backend.validation.RuntimeAttributesDefault._ import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.backend.validation.RuntimeAttributesValidation._ import cromwell.core._ +import cromwell.core.ErrorOr._ import lenthall.exception.MessageAggregation import wdl4s.types.{WdlBooleanType, WdlIntegerType, WdlStringType, WdlType} import wdl4s.values.{WdlBoolean, WdlInteger, WdlString, WdlValue} -import scalaz.Scalaz._ -import scalaz._ object SparkRuntimeAttributes { private val FailOnStderrDefaultValue = false @@ -46,33 +48,33 @@ object SparkRuntimeAttributes { val executorCores = validateCpu(withDefaultValues.get(ExecutorCoresKey), noValueFoundFor(ExecutorCoresKey)) val executorMemory = validateMemory(withDefaultValues.get(ExecutorMemoryKey), noValueFoundFor(ExecutorMemoryKey)) - val numberOfExecutors = validateNumberOfExecutors(withDefaultValues.get(NumberOfExecutorsKey), None.successNel) + val numberOfExecutors = validateNumberOfExecutors(withDefaultValues.get(NumberOfExecutorsKey), None.validNel) val appMainCLass = validateAppEntryPoint(withDefaultValues(AppMainClassKey)) - (executorCores |@| executorMemory |@| numberOfExecutors |@| appMainCLass |@| failOnStderr) { + (executorCores |@| executorMemory |@| numberOfExecutors |@| appMainCLass |@| failOnStderr) map { new SparkRuntimeAttributes(_, _, _, _, _) } match { - case Success(x) => x - case Failure(nel) => throw new RuntimeException with MessageAggregation { + case Valid(x) => x + case Invalid(nel) => throw new RuntimeException with MessageAggregation { override def exceptionContext: String = "Runtime attribute validation failed" - override def errorMessages: Traversable[String] = nel.list.toList + override def errorMessages: Traversable[String] = nel.toList } } } private def validateNumberOfExecutors(numOfExecutors: Option[WdlValue], onMissingKey: => ErrorOr[Option[Int]]): ErrorOr[Option[Int]] = { numOfExecutors match { - case Some(i: WdlInteger) => Option(i.value.intValue()).successNel + case Some(i: WdlInteger) => Option(i.value.intValue()).validNel case None => onMissingKey - case _ => s"Expecting $NumberOfExecutorsKey runtime attribute to be an Integer".failureNel + case _ => s"Expecting $NumberOfExecutorsKey runtime attribute to be an Integer".invalidNel } } private def validateAppEntryPoint(mainClass: WdlValue): ErrorOr[String] = { WdlStringType.coerceRawValue(mainClass) match { - case scala.util.Success(WdlString(s)) => s.successNel - case _ => s"Could not coerce $AppMainClassKey into a String".failureNel + case scala.util.Success(WdlString(s)) => s.validNel + case _ => s"Could not coerce $AppMainClassKey into a String".invalidNel } } } From 5198253cb6e1958cc498caa844c7b5bf16d4e52b Mon Sep 17 00:00:00 2001 From: Thib Date: Fri, 30 Sep 2016 14:03:37 -0400 Subject: [PATCH 10/23] Hashing strategies v2 - Closes #1483 (#1508) * follow symlinks for hashing and localization and introduce some hashing strategies --- core/src/main/resources/reference.conf | 16 +++ .../src/main/scala/cromwell/core/PathFactory.scala | 4 + core/src/main/scala/cromwell/util/FileUtil.scala | 13 ++ .../sfs/config/ConfigAsyncJobExecutionActor.scala | 4 +- .../ConfigBackendLifecycleActorFactory.scala | 12 +- .../impl/sfs/config/ConfigHashingStrategy.scala | 74 +++++++++++ .../sfs/BackgroundAsyncJobExecutionActor.scala | 10 +- .../cromwell/backend/sfs/SharedFileSystem.scala | 67 +++++----- .../SharedFileSystemAsyncJobExecutionActor.scala | 6 +- .../sfs/config/ConfigHashingStrategySpec.scala | 144 +++++++++++++++++++++ 10 files changed, 303 insertions(+), 47 deletions(-) create mode 100644 supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala create mode 100644 supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 02490fb28..9bf2807ea 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -161,6 +161,22 @@ backend { localization: [ "hard-link", "soft-link", "copy" ] + + caching { + duplication-strategy: [ + "hard-link", "soft-link", "copy" + ] + + # Possible values: file, path + # "file" will compute an md5 hash of the file content. + # "path" will compute an md5 hash of the file path. This strategy will only be effective if the duplication-strategy (above) is set to "soft-link", + # in order to allow for the original file path to be hashed. + hashing-strategy: "file" + + # When true, will check if a sibling file with the same name and the .md5 extension exists, and if it does, use the content of this file as a hash. + # If false or the md5 does not exist, will proceed with the above-defined hashing strategy. + check-sibling-md5: false + } } } } diff --git a/core/src/main/scala/cromwell/core/PathFactory.scala b/core/src/main/scala/cromwell/core/PathFactory.scala index a7889227b..499578107 100644 --- a/core/src/main/scala/cromwell/core/PathFactory.scala +++ b/core/src/main/scala/cromwell/core/PathFactory.scala @@ -3,6 +3,8 @@ package cromwell.core import java.io.Writer import java.nio.file.{FileSystem, Path} +import better.files.File + import scala.collection.immutable.Queue import scala.util.{Success, Failure, Try} @@ -30,6 +32,8 @@ trait PathFactory { }) } + def buildFile(rawString: String, fileSystems: List[FileSystem]): File = File(buildPath(rawString, fileSystems)) + private def hasWrongScheme(rawString: String, fileSystem: FileSystem): Boolean = { schemeMatcher.findFirstMatchIn(rawString) match { case Some(m) => m.group(1) != fileSystem.provider().getScheme diff --git a/core/src/main/scala/cromwell/util/FileUtil.scala b/core/src/main/scala/cromwell/util/FileUtil.scala index 3c0744755..28cd9d072 100644 --- a/core/src/main/scala/cromwell/util/FileUtil.scala +++ b/core/src/main/scala/cromwell/util/FileUtil.scala @@ -5,6 +5,7 @@ import java.nio.file.Path import better.files._ import wdl4s.values.Hashable +import scala.annotation.tailrec import scala.util.{Failure, Success, Try} object FileUtil { @@ -23,4 +24,16 @@ object FileUtil { implicit class EnhancedFile(val file: Path) extends AnyVal with Hashable { def md5Sum: String = File(file).md5.toLowerCase // toLowerCase for backwards compatibility } + + @tailrec + final private def followSymlinks(file: better.files.File): better.files.File = { + file.symbolicLink match { + case Some(target) => followSymlinks(target) + case None => file + } + } + + implicit class EnvenBetterFile(val file: better.files.File) extends AnyVal { + def followSymlinks = FileUtil.followSymlinks(file) + } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala index 014faa1b0..cf6fcea8c 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala @@ -48,12 +48,12 @@ sealed trait ConfigAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecut * @param taskName The name of the task to retrieve from the precomputed wdl namespace. * @param inputs The customized inputs to this task. */ - def writeTaskScript(script: Path, taskName: String, inputs: CallInputs): Unit = { + def writeTaskScript(script: File, taskName: String, inputs: CallInputs): Unit = { val task = configInitializationData.wdlNamespace.findTask(taskName). getOrElse(throw new RuntimeException(s"Unable to find task $taskName")) val command = task.instantiateCommand(inputs, NoFunctions).get jobLogger.info(s"executing: $command") - File(script).write( + script.write( s"""|#!/bin/bash |$command |""".stripMargin) diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala index e37d9416a..a92e7fdb5 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala @@ -5,6 +5,7 @@ import cromwell.backend.impl.sfs.config.ConfigConstants._ import cromwell.backend.sfs._ import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, RuntimeAttributeDefinition} import lenthall.config.ScalaConfig._ +import org.slf4j.LoggerFactory /** * Builds a backend by reading the job control from the config. @@ -14,6 +15,11 @@ import lenthall.config.ScalaConfig._ class ConfigBackendLifecycleActorFactory(val configurationDescriptor: BackendConfigurationDescriptor) extends SharedFileSystemBackendLifecycleActorFactory { + lazy val logger = LoggerFactory.getLogger(getClass) + lazy val hashingStrategy = { + configurationDescriptor.backendConfig.getConfigOption("filesystems.local.caching") map ConfigHashingStrategy.apply getOrElse ConfigHashingStrategy.defaultStrategy + } + override def initializationActorClass = classOf[ConfigInitializationActor] override def asyncJobExecutionActorClass: Class[_ <: ConfigAsyncJobExecutionActor] = { @@ -32,6 +38,10 @@ class ConfigBackendLifecycleActorFactory(val configurationDescriptor: BackendCon initializationData.runtimeAttributesBuilder.definitions.toSet } - override lazy val fileHashingFunction: Option[FileHashingFunction] = Option(FileHashingFunction(ConfigBackendFileHashing.getMd5Result)) + override lazy val fileHashingFunction: Option[FileHashingFunction] = { + logger.debug(hashingStrategy.toString) + Option(FileHashingFunction(hashingStrategy.getHash)) + } + override lazy val fileHashingActorCount: Int = 5 } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala new file mode 100644 index 000000000..21719550e --- /dev/null +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala @@ -0,0 +1,74 @@ +package cromwell.backend.impl.sfs.config + +import akka.event.LoggingAdapter +import better.files.File +import com.typesafe.config.Config +import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest +import cromwell.util.TryWithResource._ +import cromwell.util.FileUtil._ +import lenthall.config.ScalaConfig._ +import org.apache.commons.codec.digest.DigestUtils +import org.slf4j.LoggerFactory + +import scala.util.Try + +object ConfigHashingStrategy { + val logger = LoggerFactory.getLogger(getClass) + val defaultStrategy = HashFileStrategy(false) + + def apply(hashingConfig: Config): ConfigHashingStrategy = { + val checkSiblingMd5 = hashingConfig.getBooleanOr("check-sibling-md5", default = false) + + hashingConfig.getStringOr("hashing-strategy", "file") match { + case "path" => HashPathStrategy(checkSiblingMd5) + case "file" => HashFileStrategy(checkSiblingMd5) + case what => + logger.warn(s"Unrecognized hashing strategy $what.") + HashPathStrategy(checkSiblingMd5) + } + } +} + +abstract class ConfigHashingStrategy { + def checkSiblingMd5: Boolean + protected def hash(file: File): Try[String] + protected def description: String + + protected lazy val checkSiblingMessage = if (checkSiblingMd5) "Check first for sibling md5 and if not found " else "" + + def getHash(request: SingleFileHashRequest, log: LoggingAdapter): Try[String] = { + val file = File(request.file.valueString).followSymlinks + + if (checkSiblingMd5) { + precomputedMd5(file) match { + case Some(md5) => Try(md5.contentAsString) + case None => hash(file) + } + } else hash(file) + } + + private def precomputedMd5(file: File): Option[File] = { + val md5 = file.sibling(s"${file.name}.md5") + if (md5.exists) Option(md5) else None + } + + override def toString = { + s"Call caching hashing strategy: $checkSiblingMessage$description." + } +} + +final case class HashPathStrategy(checkSiblingMd5: Boolean) extends ConfigHashingStrategy { + override def hash(file: File): Try[String] = { + Try(DigestUtils.md5Hex(file.path.toAbsolutePath.toString)) + } + + override val description = "hash file path" +} + +final case class HashFileStrategy(checkSiblingMd5: Boolean) extends ConfigHashingStrategy { + override protected def hash(file: File): Try[String] = { + tryWithResource(() => file.newInputStream) { DigestUtils.md5Hex } + } + + override val description = "hash file content" +} diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala index 41173d811..c17ebe269 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala @@ -13,10 +13,10 @@ trait BackgroundAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecution val stdout = pathPlusSuffix(jobPaths.stdout, "background") val stderr = pathPlusSuffix(jobPaths.stderr, "background") val argv = Seq("/bin/bash", backgroundScript) - new ProcessRunner(argv, stdout, stderr) + new ProcessRunner(argv, stdout.path, stderr.path) } - private def writeBackgroundScript(backgroundScript: Path, backgroundCommand: String): Unit = { + private def writeBackgroundScript(backgroundScript: File, backgroundCommand: String): Unit = { /* Run the `backgroundCommand` in the background. Redirect the stdout and stderr to the appropriate files. While not necessary, mark the job as not receiving any stdin by pointing it at /dev/null. @@ -35,7 +35,7 @@ trait BackgroundAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecution & | send the entire compound command, including the || to the background $! | a variable containing the previous background command's process id (PID) */ - File(backgroundScript).write( + backgroundScript.write( s"""|#!/bin/bash |$backgroundCommand \\ | > ${jobPaths.stdout} \\ @@ -63,11 +63,11 @@ trait BackgroundAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecution SharedFileSystemCommand("/bin/bash", killScript) } - private def writeKillScript(killScript: Path, job: SharedFileSystemJob): Unit = { + private def writeKillScript(killScript: File, job: SharedFileSystemJob): Unit = { /* Use pgrep to find the children of a process, and recursively kill the children before killing the parent. */ - File(killScript).write( + killScript.write( s"""|#!/bin/bash |kill_children() { | local pid=$$1 diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala index 1ac20f284..3d81722c6 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala @@ -1,6 +1,6 @@ package cromwell.backend.sfs -import java.nio.file.{FileSystem, Files, Path, Paths} +import java.nio.file.{FileSystem, Path, Paths} import com.typesafe.config.Config import cromwell.backend.io.JobPaths @@ -29,51 +29,46 @@ object SharedFileSystem { } } - type PathsPair = (Path, Path) - type DuplicationStrategy = (Path, Path) => Try[Unit] + case class PairOfFiles(src: File, dst: File) + type DuplicationStrategy = (File, File) => Try[Unit] /** * Return a `Success` result if the file has already been localized, otherwise `Failure`. */ - private def localizePathAlreadyLocalized(originalPath: Path, executionPath: Path): Try[Unit] = { - if (File(executionPath).exists) Success(Unit) else Failure(new RuntimeException(s"$originalPath doesn't exists")) + private def localizePathAlreadyLocalized(originalPath: File, executionPath: File): Try[Unit] = { + if (executionPath.exists) Success(()) else Failure(new RuntimeException(s"$originalPath doesn't exists")) } - private def localizePathViaCopy(originalPath: Path, executionPath: Path): Try[Unit] = { - File(executionPath).parent.createDirectories() + private def localizePathViaCopy(originalPath: File, executionPath: File): Try[Unit] = { + executionPath.parent.createDirectories() val executionTmpPath = pathPlusSuffix(executionPath, ".tmp") - Try(File(originalPath).copyTo(executionTmpPath, overwrite = true).moveTo(executionPath, overwrite = true)) + Try(originalPath.copyTo(executionTmpPath, overwrite = true).moveTo(executionPath, overwrite = true)) } - private def localizePathViaHardLink(originalPath: Path, executionPath: Path): Try[Unit] = { - File(executionPath).parent.createDirectories() - Try(Files.createLink(executionPath, originalPath)) + private def localizePathViaHardLink(originalPath: File, executionPath: File): Try[Unit] = { + executionPath.parent.createDirectories() + // link.linkTo(target) returns target, + // however we want to return the link, not the target, so map the result back to executionPath + Try(executionPath.linkTo(originalPath, symbolic = false)) map { _ => executionPath } } - /** - * TODO: The 'call' parameter here represents the call statement in WDL that references this path. - * We're supposed to not use symbolic links if the call uses Docker. However, this is currently a - * bit incorrect because multiple calls can reference the same path if that path is in a declaration. - * - * The symbolic link will only fail in the Docker case if a Call uses the file directly and not - * indirectly through one of its input expressions - */ - - private def localizePathViaSymbolicLink(originalPath: Path, executionPath: Path): Try[Unit] = { - if (File(originalPath).isDirectory) Failure(new UnsupportedOperationException("Cannot localize directory with symbolic links")) + private def localizePathViaSymbolicLink(originalPath: File, executionPath: File): Try[Unit] = { + if (originalPath.isDirectory) Failure(new UnsupportedOperationException("Cannot localize directory with symbolic links")) else { - File(executionPath).parent.createDirectories() - Try(Files.createSymbolicLink(executionPath, originalPath.toAbsolutePath)) + executionPath.parent.createDirectories() + Try(executionPath.linkTo(originalPath, symbolic = true)) map { _ => executionPath } } } - private def duplicate(description: String, source: Path, dest: Path, strategies: Stream[DuplicationStrategy]) = { - strategies.map(_ (source, dest)).find(_.isSuccess) getOrElse { + private def duplicate(description: String, source: File, dest: File, strategies: Stream[DuplicationStrategy]) = { + import cromwell.util.FileUtil._ + + strategies.map(_ (source.followSymlinks, dest)).find(_.isSuccess) getOrElse { Failure(new UnsupportedOperationException(s"Could not $description $source -> $dest")) } } - def pathPlusSuffix(path: Path, suffix: String) = path.resolveSibling(s"${File(path).name}.$suffix") + def pathPlusSuffix(path: File, suffix: String) = path.sibling(s"${path.name}.$suffix") } trait SharedFileSystem extends PathFactory { @@ -88,7 +83,7 @@ trait SharedFileSystem extends PathFactory { lazy val Localizers = createStrategies(LocalizationStrategies, docker = false) lazy val DockerLocalizers = createStrategies(LocalizationStrategies, docker = true) - lazy val CachingStrategies = getConfigStrategies("caching") + lazy val CachingStrategies = getConfigStrategies("caching.duplication-strategy") lazy val Cachers = createStrategies(CachingStrategies, docker = false) private def getConfigStrategies(configPath: String): Seq[String] = { @@ -163,17 +158,17 @@ trait SharedFileSystem extends PathFactory { * Transform an original input path to a path in the call directory. * The new path matches the original path, it only "moves" the root to be the call directory. */ - def toCallPath(path: String): Try[PathsPair] = Try { - val src = buildPath(path, filesystems) + def toCallPath(path: String): Try[PairOfFiles] = Try { + val src = buildFile(path, filesystems) // Strip out potential prefix protocol - val localInputPath = stripProtocolScheme(src) - val dest = if (File(inputsRoot).isParentOf(localInputPath)) localInputPath + val localInputPath = stripProtocolScheme(src.path) + val dest = if (File(inputsRoot).isParentOf(localInputPath)) File(localInputPath) else { // Concatenate call directory with absolute input path - Paths.get(inputsRoot.toString, localInputPath.toString) + File(Paths.get(inputsRoot.toString, localInputPath.toString)) } - (src, dest) + PairOfFiles(src, dest) } // Optional function to adjust the path to "docker path" if the call runs in docker @@ -195,7 +190,7 @@ trait SharedFileSystem extends PathFactory { * @param wdlValue WdlValue to localize * @return localized wdlValue */ - private def localizeWdlValue(toDestPath: (String => Try[PathsPair]), strategies: Stream[DuplicationStrategy]) + private def localizeWdlValue(toDestPath: (String => Try[PairOfFiles]), strategies: Stream[DuplicationStrategy]) (wdlValue: WdlValue): Try[WdlValue] = { def adjustArray(t: WdlArrayType, inputArray: Seq[WdlValue]): Try[WdlArray] = { @@ -216,7 +211,7 @@ trait SharedFileSystem extends PathFactory { def adjustFile(path: String) = { toDestPath(path) flatMap { - case (src, dst) => duplicate("localize", src, dst, strategies) map { _ => WdlFile(dst.toString) } + case PairOfFiles(src, dst) => duplicate("localize", src, dst, strategies) map { _ => WdlFile(dst.toString) } } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala index 3a925aad9..6aa913658 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala @@ -286,12 +286,12 @@ trait SharedFileSystemAsyncJobExecutionActor val argv = checkAliveArgs(job).argv val stdout = pathPlusSuffix(jobPaths.stdout, "check") val stderr = pathPlusSuffix(jobPaths.stderr, "check") - val checkAlive = new ProcessRunner(argv, stdout, stderr) + val checkAlive = new ProcessRunner(argv, stdout.path, stderr.path) checkAlive.run() == 0 } def tryKill(job: SharedFileSystemJob): Unit = { - val returnCodeTmp = File(pathPlusSuffix(jobPaths.returnCode, "kill")) + val returnCodeTmp = pathPlusSuffix(jobPaths.returnCode, "kill") returnCodeTmp.write(s"$SIGTERM\n") try { returnCodeTmp.moveTo(jobPaths.returnCode) @@ -303,7 +303,7 @@ trait SharedFileSystemAsyncJobExecutionActor val argv = killArgs(job).argv val stdout = pathPlusSuffix(jobPaths.stdout, "kill") val stderr = pathPlusSuffix(jobPaths.stderr, "kill") - val killer = new ProcessRunner(argv, stdout, stderr) + val killer = new ProcessRunner(argv, stdout.path, stderr.path) killer.run() } diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala new file mode 100644 index 000000000..17ea69fc7 --- /dev/null +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala @@ -0,0 +1,144 @@ +package cromwell.backend.impl.sfs.config + +import java.util.UUID + +import akka.event.LoggingAdapter +import better.files._ +import com.typesafe.config.{ConfigFactory, ConfigValueFactory} +import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest +import org.apache.commons.codec.digest.DigestUtils +import org.scalatest.prop.TableDrivenPropertyChecks +import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers} +import org.specs2.mock.Mockito +import wdl4s.values.WdlFile + +import scala.util.Success + +class ConfigHashingStrategySpec extends FlatSpec with Matchers with TableDrivenPropertyChecks with Mockito with BeforeAndAfterAll { + + behavior of "ConfigHashingStrategy" + + val steak = "Steak" + val steakHash = DigestUtils.md5Hex(steak) + val file = File.newTemporaryFile() + val symLinksDir = File.newTemporaryDirectory("sym-dir") + val pathHash = DigestUtils.md5Hex(file.pathAsString) + val md5File = file.sibling(s"${file.name}.md5") + // Not the md5 value of "Steak". This is intentional so we can verify which hash is used depending on the strategy + val md5FileHash = "103508832bace55730c8ee8d89c1a45f" + + override def beforeAll() = { + file.write(steak) + } + + private def randomName(): String = UUID.randomUUID().toString + + def mockRequest(withSibling: Boolean, symlink: Boolean = false) = { + if (withSibling && md5File.notExists) md5File.write(md5FileHash) + val request = mock[SingleFileHashRequest] + val requestFile = if (symlink) { + val symLink : File = symLinksDir./(s"symlink-${randomName()}") + symLink.symbolicLinkTo(file) + symLink + } else file + + request.file returns WdlFile(requestFile.pathAsString) + + request + } + + def makeStrategy(strategy: String, checkSibling: Option[Boolean] = None) = { + val conf = ConfigFactory.parseString(s"""hashing-strategy: "$strategy"""") + ConfigHashingStrategy( + checkSibling map { check => conf.withValue("check-sibling-md5", ConfigValueFactory.fromAnyRef(check)) } getOrElse conf + ) + } + + it should "create a path hashing strategy from config" in { + val defaultSibling = makeStrategy("path") + defaultSibling.isInstanceOf[HashPathStrategy] shouldBe true + defaultSibling.checkSiblingMd5 shouldBe false + + val checkSibling = makeStrategy("path", Option(true)) + + checkSibling.isInstanceOf[HashPathStrategy] shouldBe true + checkSibling.checkSiblingMd5 shouldBe true + checkSibling.toString shouldBe "Call caching hashing strategy: Check first for sibling md5 and if not found hash file path." + + val dontCheckSibling = makeStrategy("path", Option(false)) + + dontCheckSibling.isInstanceOf[HashPathStrategy] shouldBe true + dontCheckSibling.checkSiblingMd5 shouldBe false + dontCheckSibling.toString shouldBe "Call caching hashing strategy: hash file path." + } + + it should "have a path hashing strategy and use md5 sibling file when appropriate" in { + val table = Table( + ("check", "withMd5", "expected"), + (true, true, md5FileHash), + (false, true, pathHash), + (true, false, pathHash), + (false, false, pathHash) + ) + + forAll(table) { (check, withMd5, expected) => + md5File.delete(swallowIOExceptions = true) + val checkSibling = makeStrategy("path", Option(check)) + + checkSibling.getHash(mockRequest(withMd5, symlink = false), mock[LoggingAdapter]) shouldBe Success(expected) + + val symLinkRequest: SingleFileHashRequest = mockRequest(withMd5, symlink = true) + val symlink = File(symLinkRequest.file.valueString) + + symlink.isSymbolicLink shouldBe true + DigestUtils.md5Hex(symlink.pathAsString) should not be expected + checkSibling.getHash(symLinkRequest, mock[LoggingAdapter]) shouldBe Success(expected) + } + } + + it should "create a file hashing strategy from config" in { + val defaultSibling = makeStrategy("file") + defaultSibling.isInstanceOf[HashFileStrategy] shouldBe true + defaultSibling.checkSiblingMd5 shouldBe false + + val checkSibling = makeStrategy("file", Option(true)) + + checkSibling.isInstanceOf[HashFileStrategy] shouldBe true + checkSibling.checkSiblingMd5 shouldBe true + checkSibling.toString shouldBe "Call caching hashing strategy: Check first for sibling md5 and if not found hash file content." + + val dontCheckSibling = makeStrategy("file", Option(false)) + + dontCheckSibling.isInstanceOf[HashFileStrategy] shouldBe true + dontCheckSibling.checkSiblingMd5 shouldBe false + dontCheckSibling.toString shouldBe "Call caching hashing strategy: hash file content." + } + + it should "have a file hashing strategy and use md5 sibling file when appropriate" in { + val table = Table( + ("check", "withMd5", "expected"), + (true, true, md5FileHash), + (false, true, steakHash), + (true, false, steakHash), + (false, false, steakHash) + ) + + forAll(table) { (check, withMd5, expected) => + md5File.delete(swallowIOExceptions = true) + val checkSibling = makeStrategy("file", Option(check)) + + checkSibling.getHash(mockRequest(withMd5, symlink = false), mock[LoggingAdapter]) shouldBe Success(expected) + + val symLinkRequest: SingleFileHashRequest = mockRequest(withMd5, symlink = true) + val symlink = File(symLinkRequest.file.valueString) + + symlink.isSymbolicLink shouldBe true + checkSibling.getHash(symLinkRequest, mock[LoggingAdapter]) shouldBe Success(expected) + } + } + + override def afterAll() = { + file.delete(true) + md5File.delete(true) + } +} From 108ff10affe38379683e355a73eef776718b8fdb Mon Sep 17 00:00:00 2001 From: Kristian Cibulskis Date: Thu, 29 Sep 2016 15:58:43 -0400 Subject: [PATCH 11/23] adding support for JAVA_OPTS env variable and symlink jar to /app/cromwell.jar --- project/Settings.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project/Settings.scala b/project/Settings.scala index d9fd8bb36..3038ffed3 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -57,12 +57,13 @@ object Settings { from("openjdk:8") expose(8000) add(artifact, artifactTargetPath) + runRaw(s"ln -s $artifactTargetPath /app/cromwell.jar") // If you use the 'exec' form for an entry point, shell processing is not performed and // environment variable substitution does not occur. Thus we have to /bin/bash here // and pass along any subsequent command line arguments // See https://docs.docker.com/engine/reference/builder/#/entrypoint - entryPoint("/bin/bash", "-c", "java -jar " + artifactTargetPath + " ${CROMWELL_ARGS} ${*}", "--") + entryPoint("/bin/bash", "-c", "java ${JAVA_OPTS} -jar /app/cromwell.jar ${CROMWELL_ARGS} ${*}", "--") } }, buildOptions in docker := BuildOptions( From 11bc336aa65050350f9e5e34730730833658d082 Mon Sep 17 00:00:00 2001 From: Thib Date: Tue, 4 Oct 2016 13:07:22 -0400 Subject: [PATCH 12/23] Clone pipeline resources and strip out mount point for runtime Closes #1501 (#1507) * clone pipeline resources and strip out mount point for runtime * PR comments --- .../jes/src/main/scala/cromwell/backend/impl/jes/Run.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala index caf87e024..b24f7affe 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala @@ -54,9 +54,15 @@ object Run { .setInputParameters(jesParameters.collect({ case i: JesInput => i.toGooglePipelineParameter }).toVector.asJava) .setOutputParameters(jesParameters.collect({ case i: JesFileOutput => i.toGooglePipelineParameter }).toVector.asJava) + // disks cannot have mount points at runtime, so set them null + val runtimePipelineResources = { + val resources = pipelineInfoBuilder.build(commandLine, runtimeAttributes).resources + val disksWithoutMountPoint = resources.getDisks.asScala map { _.setMountPoint(null) } + resources.setDisks(disksWithoutMountPoint.asJava) + } + def runPipeline: String = { - val runtimeResources = new PipelineResources().set(NoAddressFieldName, runtimeAttributes.noAddress) - val rpargs = new RunPipelineArgs().setProjectId(projectId).setServiceAccount(JesServiceAccount).setResources(runtimeResources) + val rpargs = new RunPipelineArgs().setProjectId(projectId).setServiceAccount(JesServiceAccount).setResources(runtimePipelineResources) rpargs.setInputs(jesParameters.collect({ case i: JesInput => i.name -> i.toGoogleRunParameter }).toMap.asJava) logger.debug(s"Inputs:\n${stringifyMap(rpargs.getInputs.asScala.toMap)}") From b34264e3957c03469a27e8081c13e12189143879 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 20 Sep 2016 15:46:46 -0400 Subject: [PATCH 13/23] Added an execution token dispenser to limit concurrent calls --- .../backend/BackendJobExecutionActor.scala | 5 +- .../backend/BackendLifecycleActorFactory.scala | 3 + .../backend/callcaching/CacheHitDuplicating.scala | 2 +- .../scala/cromwell/core/JobExecutionToken.scala | 11 + .../engine/backend/BackendConfiguration.scala | 5 +- .../cromwell/engine/workflow/WorkflowActor.scala | 9 +- .../engine/workflow/WorkflowManagerActor.scala | 16 +- .../execution/EngineJobExecutionActor.scala | 35 ++- .../execution/WorkflowExecutionActor.scala | 7 +- .../tokens/JobExecutionTokenDispenserActor.scala | 133 +++++++++ .../engine/workflow/tokens/TokenPool.scala | 50 ++++ .../scala/cromwell/server/CromwellRootActor.scala | 5 +- .../scala/cromwell/SimpleWorkflowActorSpec.scala | 4 +- .../mock/DefaultBackendJobExecutionActor.scala | 2 +- .../RetryableBackendLifecycleActorFactory.scala | 2 +- .../workflow/SingleWorkflowRunnerActorSpec.scala | 5 +- .../engine/workflow/WorkflowActorSpec.scala | 6 +- .../execution/WorkflowExecutionActorSpec.scala | 7 +- .../lifecycle/execution/ejea/EjeaPendingSpec.scala | 32 +-- .../ejea/EjeaRequestingExecutionTokenSpec.scala | 54 ++++ .../lifecycle/execution/ejea/PerTestHelper.scala | 12 +- .../JobExecutionTokenDispenserActorSpec.scala | 311 +++++++++++++++++++++ .../workflow/tokens/TokenGrabbingActor.scala | 38 +++ .../impl/htcondor/HtCondorBackendFactory.scala | 2 +- .../impl/jes/JesBackendLifecycleActorFactory.scala | 2 +- .../ConfigBackendLifecycleActorFactory.scala | 8 +- .../backend/impl/spark/SparkBackendFactory.scala | 2 +- 27 files changed, 710 insertions(+), 58 deletions(-) create mode 100644 core/src/main/scala/cromwell/core/JobExecutionToken.scala create mode 100644 engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala create mode 100644 engine/src/main/scala/cromwell/engine/workflow/tokens/TokenPool.scala create mode 100644 engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRequestingExecutionTokenSpec.scala create mode 100644 engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala create mode 100644 engine/src/test/scala/cromwell/engine/workflow/tokens/TokenGrabbingActor.scala diff --git a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala index c7f712eba..60cdc1e02 100644 --- a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala @@ -24,8 +24,9 @@ object BackendJobExecutionActor { sealed trait BackendJobExecutionResponse extends BackendJobExecutionActorResponse { def jobKey: BackendJobDescriptorKey } case class SucceededResponse(jobKey: BackendJobDescriptorKey, returnCode: Option[Int], jobOutputs: JobOutputs, jobDetritusFiles: Option[Map[String, String]], executionEvents: Seq[ExecutionEvent]) extends BackendJobExecutionResponse case class AbortedResponse(jobKey: BackendJobDescriptorKey) extends BackendJobExecutionResponse - case class FailedNonRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobExecutionResponse - case class FailedRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobExecutionResponse + sealed trait BackendJobFailedResponse extends BackendJobExecutionResponse { def throwable: Throwable; def returnCode: Option[Int] } + case class FailedNonRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse + case class FailedRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse } /** diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala index 6c48329ce..0101da9f0 100644 --- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala +++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala @@ -7,6 +7,7 @@ import com.typesafe.config.Config import cromwell.backend.callcaching.FileHashingActor import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction import cromwell.backend.io.WorkflowPaths +import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.core.{ExecutionStore, OutputStore} import wdl4s.Call import wdl4s.expression.WdlStandardLibraryFunctions @@ -52,4 +53,6 @@ trait BackendLifecycleActorFactory { lazy val fileHashingActorCount: Int = 50 def fileHashingActorProps: Props = FileHashingActor.props(fileHashingFunction) + + def jobExecutionTokenType: JobExecutionTokenType = JobExecutionTokenType("Default", None) } diff --git a/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala b/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala index addbe0163..c199de6d4 100644 --- a/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala +++ b/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala @@ -48,7 +48,7 @@ trait CacheHitDuplicating { private def lookupSourceCallRootPath(sourceJobDetritusFiles: Map[String, String]): Path = { sourceJobDetritusFiles.get(JobPaths.CallRootPathKey).map(getPath).getOrElse(throw new RuntimeException( - s"The call detritus files for source cache hit aren't found for call ${jobDescriptor.call.fullyQualifiedName}") + s"${JobPaths.CallRootPathKey} wasn't found for call ${jobDescriptor.call.fullyQualifiedName}") ) } diff --git a/core/src/main/scala/cromwell/core/JobExecutionToken.scala b/core/src/main/scala/cromwell/core/JobExecutionToken.scala new file mode 100644 index 000000000..460d36e6c --- /dev/null +++ b/core/src/main/scala/cromwell/core/JobExecutionToken.scala @@ -0,0 +1,11 @@ +package cromwell.core + +import java.util.UUID + +import cromwell.core.JobExecutionToken.JobExecutionTokenType + +case class JobExecutionToken(jobExecutionTokenType: JobExecutionTokenType, id: UUID) + +object JobExecutionToken { + case class JobExecutionTokenType(backend: String, maxPoolSize: Option[Int]) +} diff --git a/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala b/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala index 866e11062..fcd6da9ce 100644 --- a/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala +++ b/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala @@ -1,6 +1,5 @@ package cromwell.engine.backend -import akka.actor.ActorSystem import com.typesafe.config.{Config, ConfigFactory} import cromwell.backend.{BackendConfigurationDescriptor, BackendLifecycleActorFactory} import lenthall.config.ScalaConfig._ @@ -11,8 +10,8 @@ import scala.util.{Failure, Success, Try} case class BackendConfigurationEntry(name: String, lifecycleActorFactoryClass: String, config: Config) { def asBackendLifecycleActorFactory: BackendLifecycleActorFactory = { Class.forName(lifecycleActorFactoryClass) - .getConstructor(classOf[BackendConfigurationDescriptor]) - .newInstance(asBackendConfigurationDescriptor) + .getConstructor(classOf[String], classOf[BackendConfigurationDescriptor]) + .newInstance(name, asBackendConfigurationDescriptor) .asInstanceOf[BackendLifecycleActorFactory] } diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala index 3759136f8..f137883fd 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala @@ -140,9 +140,10 @@ object WorkflowActor { serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, - callCacheReadActor: ActorRef): Props = { + callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef): Props = { Props(new WorkflowActor(workflowId, startMode, wdlSource, conf, serviceRegistryActor, workflowLogCopyRouter, - jobStoreActor, callCacheReadActor)).withDispatcher(EngineDispatcher) + jobStoreActor, callCacheReadActor, jobTokenDispenserActor)).withDispatcher(EngineDispatcher) } } @@ -156,7 +157,8 @@ class WorkflowActor(val workflowId: WorkflowId, serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, - callCacheReadActor: ActorRef) + callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef) extends LoggingFSM[WorkflowActorState, WorkflowActorData] with WorkflowLogging with PathFactory { implicit val ec = context.dispatcher @@ -204,6 +206,7 @@ class WorkflowActor(val workflowId: WorkflowId, serviceRegistryActor, jobStoreActor, callCacheReadActor, + jobTokenDispenserActor, initializationData, restarting = restarting), name = s"WorkflowExecutionActor-$workflowId") diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala index 78d34439c..9c797bb76 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala @@ -43,9 +43,10 @@ object WorkflowManagerActor { serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, - callCacheReadActor: ActorRef): Props = { + callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef): Props = { Props(new WorkflowManagerActor( - workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor) + workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) ).withDispatcher(EngineDispatcher) } @@ -82,15 +83,17 @@ class WorkflowManagerActor(config: Config, val serviceRegistryActor: ActorRef, val workflowLogCopyRouter: ActorRef, val jobStoreActor: ActorRef, - val callCacheReadActor: ActorRef) + val callCacheReadActor: ActorRef, + val jobTokenDispenserActor: ActorRef) extends LoggingFSM[WorkflowManagerState, WorkflowManagerData] { def this(workflowStore: ActorRef, serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, - callCacheReadActor: ActorRef) = this( - ConfigFactory.load, workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor) + callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef) = this( + ConfigFactory.load, workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) private val maxWorkflowsRunning = config.getConfig("system").getIntOr("max-concurrent-workflows", default=DefaultMaxWorkflowsToRun) private val maxWorkflowsToLaunch = config.getConfig("system").getIntOr("max-workflow-launch-count", default=DefaultMaxWorkflowsToLaunch) @@ -259,7 +262,7 @@ class WorkflowManagerActor(config: Config, } val wfProps = WorkflowActor.props(workflowId, startMode, workflow.sources, config, serviceRegistryActor, - workflowLogCopyRouter, jobStoreActor, callCacheReadActor) + workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) val wfActor = context.actorOf(wfProps, name = s"WorkflowActor-$workflowId") wfActor ! SubscribeTransitionCallBack(self) @@ -272,4 +275,3 @@ class WorkflowManagerActor(config: Config, context.system.scheduler.scheduleOnce(newWorkflowPollRate, self, RetrieveNewWorkflows)(context.dispatcher) } } - diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala index 26aa93891..4ff275a35 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala @@ -18,6 +18,7 @@ import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor.{Backend import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CacheMiss, CallCacheHashes, HashError} import cromwell.engine.workflow.lifecycle.execution.callcaching.FetchCachedResultsActor.{CachedOutputLookupFailed, CachedOutputLookupSucceeded} import cromwell.engine.workflow.lifecycle.execution.callcaching._ +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed, JobExecutionTokenRequest, JobExecutionTokenReturn} import cromwell.jobstore.JobStoreActor._ import cromwell.jobstore.{Pending => _, _} import cromwell.services.SingletonServicesStore @@ -25,6 +26,7 @@ import cromwell.services.metadata.MetadataService.PutMetadataAction import cromwell.services.metadata.{MetadataEvent, MetadataJobKey, MetadataKey, MetadataValue} import wdl4s.TaskOutput +import scala.concurrent.ExecutionContext import scala.util.{Failure, Success, Try} class EngineJobExecutionActor(replyTo: ActorRef, @@ -36,6 +38,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, serviceRegistryActor: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef, backendName: String, callCachingMode: CallCachingMode) extends LoggingFSM[EngineJobExecutionActorState, EJEAData] with WorkflowLogging { @@ -51,8 +54,12 @@ class EngineJobExecutionActor(replyTo: ActorRef, // For tests: private[execution] def checkEffectiveCallCachingMode = effectiveCallCachingMode + private[execution] var executionToken: Option[JobExecutionToken] = None + private val effectiveCallCachingKey = "Effective call caching mode" + implicit val ec: ExecutionContext = context.dispatcher + log.debug(s"$tag: $effectiveCallCachingKey: $effectiveCallCachingMode") writeCallCachingModeToMetadata() @@ -62,6 +69,13 @@ class EngineJobExecutionActor(replyTo: ActorRef, // When Pending, the FSM always has NoData when(Pending) { case Event(Execute, NoData) => + requestExecutionToken() + goto(RequestingExecutionToken) + } + + when(RequestingExecutionToken) { + case Event(JobExecutionTokenDispensed(jobExecutionToken), NoData) => + executionToken = Option(jobExecutionToken) if (restarting) { val jobStoreKey = jobDescriptorKey.toJobStoreKey(workflowId) jobStoreActor ! QueryJobCompletion(jobStoreKey, jobDescriptorKey.call.task.outputs) @@ -69,6 +83,9 @@ class EngineJobExecutionActor(replyTo: ActorRef, } else { prepareJob() } + case Event(JobExecutionTokenDenied(positionInQueue), NoData) => + log.debug("Token denied so cannot start yet. Currently position {} in the queue", positionInQueue) + stay() } // When CheckingJobStore, the FSM always has NoData @@ -146,7 +163,10 @@ class EngineJobExecutionActor(replyTo: ActorRef, saveJobCompletionToJobStore(data.withSuccessResponse(response)) case Event(response: BackendJobExecutionResponse, data: ResponsePendingData) => // This matches all response types other than `SucceededResponse`. - log.error("{}: Failed copying cache results, falling back to running job.", jobDescriptorKey) + response match { + case f: BackendJobFailedResponse =>log.error("{}: Failed copying cache results, falling back to running job: {}", jobDescriptorKey, f.throwable) + case _ => // + } runJob(data) // Hashes arrive: @@ -217,8 +237,17 @@ class EngineJobExecutionActor(replyTo: ActorRef, stay } + private def requestExecutionToken(): Unit = { + jobTokenDispenserActor ! JobExecutionTokenRequest(factory.jobExecutionTokenType) + } + + private def returnExecutionToken(): Unit = { + executionToken foreach { jobTokenDispenserActor ! JobExecutionTokenReturn(_) } + } + private def forwardAndStop(response: Any): State = { replyTo forward response + returnExecutionToken() tellEventMetadata() context stop self stay() @@ -226,6 +255,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, private def respondAndStop(response: Any): State = { replyTo ! response + returnExecutionToken() tellEventMetadata() context stop self stay() @@ -393,6 +423,7 @@ object EngineJobExecutionActor { /** States */ sealed trait EngineJobExecutionActorState case object Pending extends EngineJobExecutionActorState + case object RequestingExecutionToken extends EngineJobExecutionActorState case object CheckingJobStore extends EngineJobExecutionActorState case object CheckingCallCache extends EngineJobExecutionActorState case object FetchingCachedOutputsFromDatabase extends EngineJobExecutionActorState @@ -417,6 +448,7 @@ object EngineJobExecutionActor { serviceRegistryActor: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef, backendName: String, callCachingMode: CallCachingMode) = { Props(new EngineJobExecutionActor( @@ -429,6 +461,7 @@ object EngineJobExecutionActor { serviceRegistryActor = serviceRegistryActor, jobStoreActor = jobStoreActor, callCacheReadActor = callCacheReadActor, + jobTokenDispenserActor = jobTokenDispenserActor, backendName = backendName: String, callCachingMode = callCachingMode)).withDispatcher(EngineDispatcher) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index 139f5d6c5..8e6c93e6c 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -138,10 +138,11 @@ object WorkflowExecutionActor { serviceRegistryActor: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef, initializationData: AllBackendInitializationData, restarting: Boolean): Props = { Props(WorkflowExecutionActor(workflowId, workflowDescriptor, serviceRegistryActor, jobStoreActor, - callCacheReadActor, initializationData, restarting)).withDispatcher(EngineDispatcher) + callCacheReadActor, jobTokenDispenserActor, initializationData, restarting)).withDispatcher(EngineDispatcher) } private implicit class EnhancedExecutionStore(val executionStore: ExecutionStore) extends AnyVal { @@ -227,7 +228,6 @@ object WorkflowExecutionActor { } toMap } } - } final case class WorkflowExecutionActor(workflowId: WorkflowId, @@ -235,6 +235,7 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, serviceRegistryActor: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef, initializationData: AllBackendInitializationData, restarting: Boolean) extends LoggingFSM[WorkflowExecutionActorState, WorkflowExecutionActorData] with WorkflowLogging { @@ -585,7 +586,7 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, val ejeaName = s"${workflowDescriptor.id}-EngineJobExecutionActor-${jobKey.tag}" val ejeaProps = EngineJobExecutionActor.props( self, jobKey, data, factory, initializationData.get(backendName), restarting, serviceRegistryActor, - jobStoreActor, callCacheReadActor, backendName, workflowDescriptor.callCachingMode) + jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendName, workflowDescriptor.callCachingMode) val ejeaRef = context.actorOf(ejeaProps, ejeaName) pushNewJobMetadata(jobKey, backendName) ejeaRef ! EngineJobExecutionActor.Execute diff --git a/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala b/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala new file mode 100644 index 000000000..4d5e460bb --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala @@ -0,0 +1,133 @@ +package cromwell.engine.workflow.tokens + +import akka.actor.{Actor, ActorLogging, ActorRef, Props, Terminated} +import cromwell.core.JobExecutionToken +import JobExecutionToken._ +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor._ +import cromwell.engine.workflow.tokens.TokenPool.TokenPoolPop + +import scala.collection.immutable.Queue +import scala.language.postfixOps + +class JobExecutionTokenDispenserActor extends Actor with ActorLogging { + + /** + * Lazily created token pool. We only create a pool for a token type when we need it + */ + var tokenPools: Map[JobExecutionTokenType, TokenPool] = Map.empty + var tokenAssignments: Map[ActorRef, JobExecutionToken] = Map.empty + + override def receive: Actor.Receive = { + case JobExecutionTokenRequest(tokenType) => sendTokenRequestResult(sender, tokenType) + case JobExecutionTokenReturn(token) => unassign(sender, token) + case Terminated(terminee) => onTerminate(terminee) + } + + private def sendTokenRequestResult(sndr: ActorRef, tokenType: JobExecutionTokenType): Unit = { + if (tokenAssignments.contains(sndr)) { + sndr ! JobExecutionTokenDispensed(tokenAssignments(sndr)) + } else { + context.watch(sndr) + val updatedTokenPool = getTokenPool(tokenType).pop() match { + case TokenPoolPop(newTokenPool, Some(token)) => + assignAndSendToken(sndr, token) + newTokenPool + case TokenPoolPop(sizedTokenPoolAndQueue: SizedTokenPoolAndActorQueue, None) => + val (poolWithActorEnqueued, positionInQueue) = sizedTokenPoolAndQueue.enqueue(sndr) + sndr ! JobExecutionTokenDenied(positionInQueue) + poolWithActorEnqueued + case TokenPoolPop(someOtherTokenPool, None) => + //If this has happened, somebody's been playing around in this class and not covered this case: + throw new RuntimeException(s"Unexpected token pool type didn't return a token: ${someOtherTokenPool.getClass.getSimpleName}") + } + + tokenPools += tokenType -> updatedTokenPool + } + } + + private def getTokenPool(tokenType: JobExecutionTokenType): TokenPool = tokenPools.getOrElse(tokenType, createNewPool(tokenType)) + + private def createNewPool(tokenType: JobExecutionTokenType): TokenPool = { + val newPool = TokenPool(tokenType) match { + case s: SizedTokenPool => SizedTokenPoolAndActorQueue(s, Queue.empty) + case anythingElse => anythingElse + } + tokenPools += tokenType -> newPool + newPool + } + + private def assignAndSendToken(actor: ActorRef, token: JobExecutionToken) = { + tokenAssignments += actor -> token + actor ! JobExecutionTokenDispensed(token) + } + + private def unassign(actor: ActorRef, token: JobExecutionToken) = { + if (tokenAssignments.contains(actor) && tokenAssignments(actor) == token) { + tokenAssignments -= actor + + val pool = getTokenPool(token.jobExecutionTokenType) match { + case SizedTokenPoolAndActorQueue(innerPool, queue) if queue.nonEmpty => + val (nextInLine, newQueue) = queue.dequeue + assignAndSendToken(nextInLine, token) + SizedTokenPoolAndActorQueue(innerPool, newQueue) + case other => + other.push(token) + } + + tokenPools += token.jobExecutionTokenType -> pool + context.unwatch(actor) + } else { + log.error("Job execution token returned from incorrect actor: {}", token) + } + } + + private def onTerminate(terminee: ActorRef) = { + tokenAssignments.get(terminee) match { + case Some(token) => + log.error("Actor {} stopped without returning its Job Execution Token. Reclaiming it!", terminee) + self.tell(msg = JobExecutionTokenReturn(token), sender = terminee) + case None => + log.debug("Actor {} stopped while we were still watching it... but it doesn't have a token. Removing it from any queues if necessary", terminee) + tokenPools = tokenPools map { + case (tokenType, SizedTokenPoolAndActorQueue(pool, queue)) => tokenType -> SizedTokenPoolAndActorQueue(pool, queue.filterNot(_ == terminee)) + case (tokenType, other) => tokenType -> other + } + } + context.unwatch(terminee) + } +} + +object JobExecutionTokenDispenserActor { + + def props = Props(new JobExecutionTokenDispenserActor) + + case class JobExecutionTokenRequest(jobExecutionTokenType: JobExecutionTokenType) + case class JobExecutionTokenReturn(jobExecutionToken: JobExecutionToken) + + sealed trait JobExecutionTokenRequestResult + case class JobExecutionTokenDispensed(jobExecutionToken: JobExecutionToken) extends JobExecutionTokenRequestResult + case class JobExecutionTokenDenied(positionInQueue: Integer) extends JobExecutionTokenRequestResult + + case class SizedTokenPoolAndActorQueue(sizedPool: SizedTokenPool, queue: Queue[ActorRef]) extends TokenPool { + override def currentLoans = sizedPool.currentLoans + override def push(jobExecutionToken: JobExecutionToken) = SizedTokenPoolAndActorQueue(sizedPool.push(jobExecutionToken), queue) + override def pop() = { + val underlyingPop = sizedPool.pop() + TokenPoolPop(SizedTokenPoolAndActorQueue(underlyingPop.newTokenPool.asInstanceOf[SizedTokenPool], queue), underlyingPop.poppedItem) + } + + /** + * Enqueues an actor (or just finds its current position) + * + * @return The actor's position in the queue + */ + def enqueue(actor: ActorRef): (SizedTokenPoolAndActorQueue, Int) = { + queue.indexOf(actor) match { + case -1 => + val newQueue = queue :+ actor + (SizedTokenPoolAndActorQueue(sizedPool, newQueue), newQueue.size - 1) // Convert from 1-indexed to 0-indexed + case index => (this, index) + } + } + } +} diff --git a/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenPool.scala b/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenPool.scala new file mode 100644 index 000000000..ee378856c --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/workflow/tokens/TokenPool.scala @@ -0,0 +1,50 @@ +package cromwell.engine.workflow.tokens + +import java.util.UUID + +import cromwell.core.JobExecutionToken +import JobExecutionToken.JobExecutionTokenType +import cromwell.engine.workflow.tokens.TokenPool.TokenPoolPop + +import scala.language.postfixOps + +trait TokenPool { + def currentLoans: Set[JobExecutionToken] + def pop(): TokenPoolPop + def push(jobExecutionToken: JobExecutionToken): TokenPool +} + +object TokenPool { + + case class TokenPoolPop(newTokenPool: TokenPool, poppedItem: Option[JobExecutionToken]) + + def apply(tokenType: JobExecutionTokenType): TokenPool = { + tokenType.maxPoolSize map { ps => + val pool = (1 to ps toList) map { _ => JobExecutionToken(tokenType, UUID.randomUUID()) } + SizedTokenPool(pool, Set.empty) + } getOrElse { + InfiniteTokenPool(tokenType, Set.empty) + } + } +} + +final case class SizedTokenPool(pool: List[JobExecutionToken], override val currentLoans: Set[JobExecutionToken]) extends TokenPool { + + override def pop(): TokenPoolPop = pool match { + case head :: tail => TokenPoolPop(SizedTokenPool(tail, currentLoans + head), Option(head)) + case Nil => TokenPoolPop(SizedTokenPool(List.empty, currentLoans), None) + } + + + override def push(token: JobExecutionToken): SizedTokenPool = { + if (currentLoans.contains(token)) { SizedTokenPool(pool :+ token, currentLoans - token) } else this + } +} + +final case class InfiniteTokenPool(tokenType: JobExecutionTokenType, override val currentLoans: Set[JobExecutionToken]) extends TokenPool { + override def pop() = { + val newToken = JobExecutionToken(tokenType, UUID.randomUUID()) + TokenPoolPop(InfiniteTokenPool(tokenType, currentLoans + newToken), Option(newToken)) + } + override def push(token: JobExecutionToken): InfiniteTokenPool = if (currentLoans.contains(token)) { InfiniteTokenPool(tokenType, currentLoans - token) } else this +} diff --git a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala index 3289e9df4..9e8c04c17 100644 --- a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala +++ b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala @@ -8,6 +8,7 @@ import com.typesafe.config.ConfigFactory import cromwell.engine.workflow.WorkflowManagerActor import cromwell.engine.workflow.lifecycle.CopyWorkflowLogsActor import cromwell.engine.workflow.lifecycle.execution.callcaching.{CallCache, CallCacheReadActor} +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor import cromwell.engine.workflow.workflowstore.{SqlWorkflowStore, WorkflowStore, WorkflowStoreActor} import cromwell.jobstore.{JobStore, JobStoreActor, SqlJobStore} import cromwell.services.{ServiceRegistryActor, SingletonServicesStore} @@ -48,9 +49,11 @@ import lenthall.config.ScalaConfig.EnhancedScalaConfig .props(CallCacheReadActor.props(callCache)), "CallCacheReadActor") + lazy val jobExecutionTokenDispenserActor = context.actorOf(JobExecutionTokenDispenserActor.props) + lazy val workflowManagerActor = context.actorOf( WorkflowManagerActor.props( - workflowStoreActor, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor), + workflowStoreActor, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobExecutionTokenDispenserActor), "WorkflowManagerActor") override def receive = { diff --git a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala index 8925f1142..f2dd4db13 100644 --- a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala @@ -10,6 +10,7 @@ import cromwell.SimpleWorkflowActorSpec._ import cromwell.core.{WorkflowId, WorkflowSourceFiles} import cromwell.engine.workflow.WorkflowActor import cromwell.engine.workflow.WorkflowActor._ +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor import cromwell.util.SampleWdl import cromwell.util.SampleWdl.HelloWorld.Addressee import org.scalatest.BeforeAndAfter @@ -42,7 +43,8 @@ class SimpleWorkflowActorSpec extends CromwellTestkitSpec with BeforeAndAfter { serviceRegistryActor = watchActor, workflowLogCopyRouter = system.actorOf(Props.empty, s"workflow-copy-log-router-$workflowId-${UUID.randomUUID()}"), jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props), - callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props)), + callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props), + jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props)), supervisor = supervisor.ref, name = s"workflow-actor-$workflowId" ) diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala b/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala index 4b6f55e4a..6893f6baf 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala @@ -22,7 +22,7 @@ case class DefaultBackendJobExecutionActor(override val jobDescriptor: BackendJo override def abort(): Unit = () } -class DefaultBackendLifecycleActorFactory(configurationDescriptor: BackendConfigurationDescriptor) +class DefaultBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala index ec527db26..74e900c6d 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala @@ -5,7 +5,7 @@ import cromwell.backend._ import wdl4s.Call import wdl4s.expression.{NoFunctions, WdlStandardLibraryFunctions} -class RetryableBackendLifecycleActorFactory(configurationDescriptor: BackendConfigurationDescriptor) +class RetryableBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala index 46e4c00a8..5c0a48253 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala @@ -12,6 +12,7 @@ import cromwell.CromwellTestkitSpec._ import cromwell.core.WorkflowSourceFiles import cromwell.engine.workflow.SingleWorkflowRunnerActor.RunWorkflow import cromwell.engine.workflow.SingleWorkflowRunnerActorSpec._ +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreActor} import cromwell.util.SampleWdl import cromwell.util.SampleWdl.{ExpressionsInInputs, GoodbyeWorld, ThreeStep} @@ -55,6 +56,7 @@ abstract class SingleWorkflowRunnerActorSpec extends CromwellTestkitSpec { private val workflowStore = system.actorOf(WorkflowStoreActor.props(new InMemoryWorkflowStore, dummyServiceRegistryActor)) private val jobStore = system.actorOf(AlwaysHappyJobStoreActor.props) private val callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props) + private val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props) def workflowManagerActor(): ActorRef = { @@ -63,7 +65,8 @@ abstract class SingleWorkflowRunnerActorSpec extends CromwellTestkitSpec { dummyServiceRegistryActor, dummyLogCopyRouter, jobStore, - callCacheReadActor)), "WorkflowManagerActor") + callCacheReadActor, + jobTokenDispenserActor)), "WorkflowManagerActor") } def createRunnerActor(sampleWdl: SampleWdl = ThreeStep, managerActor: => ActorRef = workflowManagerActor(), diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala index cc3581234..6a0245528 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala @@ -52,7 +52,8 @@ class WorkflowActorSpec extends CromwellTestkitSpec with WorkflowDescriptorBuild serviceRegistryActor = mockServiceRegistryActor, workflowLogCopyRouter = TestProbe().ref, jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props), - callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props) + callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props), + jobTokenDispenserActor = TestProbe().ref ), supervisor = supervisorProbe.ref) actor.setState(stateName = state, stateData = WorkflowActorData(Option(currentLifecycleActor.ref), Option(descriptor), @@ -155,7 +156,8 @@ class MockWorkflowActor(val finalizationProbe: TestProbe, serviceRegistryActor: ActorRef, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, - callCacheReadActor: ActorRef) extends WorkflowActor(workflowId, startMode, workflowSources, conf, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor) { + callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef) extends WorkflowActor(workflowId, startMode, workflowSources, conf, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) { override def makeFinalizationActor(workflowDescriptor: EngineWorkflowDescriptor, executionStore: ExecutionStore, outputStore: OutputStore) = finalizationProbe.ref } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala index 3d9ef1938..639d8f250 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala @@ -8,6 +8,7 @@ import cromwell.core.WorkflowId import cromwell.engine.backend.{BackendConfigurationEntry, CromwellBackends} import cromwell.engine.workflow.WorkflowDescriptorBuilder import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.ExecuteWorkflowCommand +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor import cromwell.services.ServiceRegistryActor import cromwell.services.metadata.MetadataService import cromwell.util.SampleWdl @@ -49,6 +50,7 @@ class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter val metadataWatcherProps = Props(MetadataWatchActor(metadataSuccessPromise, requiredMetadataMatchers: _*)) val serviceRegistryActor = system.actorOf(ServiceRegistryActor.props(ConfigFactory.load(), overrides = Map(MetadataService.MetadataServiceName -> metadataWatcherProps))) val jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props) + val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props) val MockBackendConfigEntry = BackendConfigurationEntry( name = "Mock", lifecycleActorFactoryClass = "cromwell.engine.backend.mock.RetryableBackendLifecycleActorFactory", @@ -62,7 +64,7 @@ class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter val workflowExecutionActor = system.actorOf( WorkflowExecutionActor.props(workflowId, engineWorkflowDescriptor, serviceRegistryActor, jobStoreActor, - callCacheReadActor.ref, AllBackendInitializationData.empty, restarting = false), + callCacheReadActor.ref, jobTokenDispenserActor, AllBackendInitializationData.empty, restarting = false), "WorkflowExecutionActor") EventFilter.info(pattern = ".*Final Outputs", occurrences = 1).intercept { @@ -82,6 +84,7 @@ class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter val serviceRegistry = mockServiceRegistryActor val jobStore = system.actorOf(AlwaysHappyJobStoreActor.props) val callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props) + val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props) val MockBackendConfigEntry = BackendConfigurationEntry( name = "Mock", @@ -94,7 +97,7 @@ class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter val engineWorkflowDescriptor = createMaterializedEngineWorkflowDescriptor(workflowId, SampleWdl.SimpleScatterWdl.asWorkflowSources(runtime = runtimeSection)) val workflowExecutionActor = system.actorOf( WorkflowExecutionActor.props(workflowId, engineWorkflowDescriptor, serviceRegistry, jobStore, - callCacheReadActor, AllBackendInitializationData.empty, restarting = false), + callCacheReadActor, jobTokenDispenserActor, AllBackendInitializationData.empty, restarting = false), "WorkflowExecutionActor") val scatterLog = "Starting calls: scatter0.inside_scatter:0:1, scatter0.inside_scatter:1:1, scatter0.inside_scatter:2:1, scatter0.inside_scatter:3:1, scatter0.inside_scatter:4:1" diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPendingSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPendingSpec.scala index 9fc78f1f7..2810753cc 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPendingSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPendingSpec.scala @@ -1,8 +1,7 @@ package cromwell.engine.workflow.lifecycle.execution.ejea -import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{CheckingJobStore, EngineJobExecutionActorState, Execute, Pending, PreparingJob} -import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor -import cromwell.jobstore.JobStoreActor.QueryJobCompletion +import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.JobExecutionTokenRequest import org.scalatest.concurrent.Eventually class EjeaPendingSpec extends EngineJobExecutionActorSpec with CanValidateJobStoreKey with Eventually { @@ -11,29 +10,16 @@ class EjeaPendingSpec extends EngineJobExecutionActorSpec with CanValidateJobSto "An EJEA in the Pending state" should { - CallCachingModes foreach { mode => - s"check against the Job Store if restarting is true ($mode)" in { - ejea = helper.buildEJEA(restarting = true) + List(false, true) foreach { restarting => + s"wait for the Execute signal then request an execution token (with restarting=$restarting)" in { + ejea = helper.buildEJEA(restarting = restarting) ejea ! Execute - helper.jobStoreProbe.expectMsgPF(max = awaitTimeout, hint = "Awaiting job store lookup") { - case QueryJobCompletion(jobKey, taskOutputs) => - validateJobStoreKey(jobKey) - taskOutputs should be(helper.task.outputs) - } - helper.bjeaProbe.expectNoMsg(awaitAlmostNothing) - helper.jobHashingInitializations shouldBe NothingYet - ejea.stateName should be(CheckingJobStore) - } - - - s"bypass the Job Store and start preparing the job for running or call caching ($mode)" in { - ejea = helper.buildEJEA(restarting = false) - ejea ! Execute + helper.jobTokenDispenserProbe.expectMsgClass(max = awaitTimeout, classOf[JobExecutionTokenRequest]) - helper.jobPreparationProbe.expectMsg(max = awaitTimeout, hint = "Awaiting job preparation", JobPreparationActor.Start) - helper.jobStoreProbe.expectNoMsg(awaitAlmostNothing) - ejea.stateName should be(PreparingJob) + helper.jobPreparationProbe.msgAvailable should be(false) + helper.jobStoreProbe.msgAvailable should be(false) + ejea.stateName should be(RequestingExecutionToken) } } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRequestingExecutionTokenSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRequestingExecutionTokenSpec.scala new file mode 100644 index 000000000..83ae08835 --- /dev/null +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRequestingExecutionTokenSpec.scala @@ -0,0 +1,54 @@ +package cromwell.engine.workflow.lifecycle.execution.ejea + +import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ +import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed} +import cromwell.jobstore.JobStoreActor.QueryJobCompletion +import org.scalatest.concurrent.Eventually + +class EjeaRequestingExecutionTokenSpec extends EngineJobExecutionActorSpec with CanValidateJobStoreKey with Eventually { + + override implicit val stateUnderTest: EngineJobExecutionActorState = RequestingExecutionToken + + "An EJEA in the RequestingExecutionToken state" should { + + List(true, false) foreach { restarting => + s"do nothing when denied a token (with restarting=$restarting)" in { + ejea = helper.buildEJEA(restarting = restarting) + ejea ! JobExecutionTokenDenied(1) // 1 is arbitrary. Doesn't matter what position in the queue we are. + + helper.jobTokenDispenserProbe.expectNoMsg(max = awaitAlmostNothing) + helper.jobPreparationProbe.msgAvailable should be(false) + helper.jobStoreProbe.msgAvailable should be(false) + + ejea.stateName should be(RequestingExecutionToken) + } + } + + CallCachingModes foreach { mode => + s"check against the Job Store if restarting is true ($mode)" in { + ejea = helper.buildEJEA(restarting = true) + ejea ! JobExecutionTokenDispensed(helper.executionToken) + + helper.jobStoreProbe.expectMsgPF(max = awaitTimeout, hint = "Awaiting job store lookup") { + case QueryJobCompletion(jobKey, taskOutputs) => + validateJobStoreKey(jobKey) + taskOutputs should be(helper.task.outputs) + } + helper.bjeaProbe.expectNoMsg(awaitAlmostNothing) + helper.jobHashingInitializations shouldBe NothingYet + ejea.stateName should be(CheckingJobStore) + } + + + s"bypass the Job Store and start preparing the job for running or call caching ($mode)" in { + ejea = helper.buildEJEA(restarting = false) + ejea ! JobExecutionTokenDispensed(helper.executionToken) + + helper.jobPreparationProbe.expectMsg(max = awaitTimeout, hint = "Awaiting job preparation", JobPreparationActor.Start) + helper.jobStoreProbe.expectNoMsg(awaitAlmostNothing) + ejea.stateName should be(PreparingJob) + } + } + } +} diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala index 6fd15b90a..80ae87fa9 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala @@ -1,11 +1,14 @@ package cromwell.engine.workflow.lifecycle.execution.ejea +import java.util.UUID + import akka.actor.{ActorRef, ActorSystem, Props} import akka.testkit.{TestFSMRef, TestProbe} import cromwell.backend.BackendJobExecutionActor.SucceededResponse import cromwell.backend.{BackendInitializationData, BackendJobDescriptor, BackendJobDescriptorKey, BackendLifecycleActorFactory, BackendWorkflowDescriptor} +import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.core.callcaching.{CallCachingActivity, CallCachingMode, CallCachingOff} -import cromwell.core.{ExecutionStore, OutputStore, WorkflowId} +import cromwell.core.{ExecutionStore, JobExecutionToken, OutputStore, WorkflowId} import cromwell.engine.EngineWorkflowDescriptor import cromwell.engine.workflow.lifecycle.execution.{EngineJobExecutionActor, WorkflowExecutionActorData} import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{EJEAData, EngineJobExecutionActorState} @@ -31,6 +34,8 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock val jobIndex = Some(1) val jobAttempt = 1 + val executionToken = JobExecutionToken(JobExecutionTokenType("test", None), UUID.randomUUID()) + val task = mock[Task] task.declarations returns Seq.empty task.runtimeAttributes returns RuntimeAttributes(Map.empty) @@ -70,6 +75,7 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock val callCacheReadActorProbe = TestProbe() val callCacheHitCopyingProbe = TestProbe() val jobPreparationProbe = TestProbe() + val jobTokenDispenserProbe = TestProbe() def buildFactory() = new BackendLifecycleActorFactory { @@ -113,6 +119,7 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock serviceRegistryActor = serviceRegistryProbe.ref, jobStoreActor = jobStoreProbe.ref, callCacheReadActor = callCacheReadActorProbe.ref, + jobTokenDispenserActor = jobTokenDispenserProbe.ref, backendName = "NOT USED", callCachingMode = callCachingMode )), parentProbe.ref, s"EngineJobExecutionActorSpec-$workflowId") @@ -133,8 +140,9 @@ private[ejea] class MockEjea(helper: PerTestHelper, serviceRegistryActor: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, + jobTokenDispenserActor: ActorRef, backendName: String, - callCachingMode: CallCachingMode) extends EngineJobExecutionActor(replyTo, jobDescriptorKey, executionData, factory, initializationData, restarting, serviceRegistryActor, jobStoreActor, callCacheReadActor, backendName, callCachingMode) { + callCachingMode: CallCachingMode) extends EngineJobExecutionActor(replyTo, jobDescriptorKey, executionData, factory, initializationData, restarting, serviceRegistryActor, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendName, callCachingMode) { override def makeFetchCachedResultsActor(cacheHit: CacheHit, taskOutputs: Seq[TaskOutput]) = helper.fetchCachedResultsActorCreations = helper.fetchCachedResultsActorCreations.foundOne((cacheHit, taskOutputs)) override def initializeJobHashing(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity) = helper.jobHashingInitializations = helper.jobHashingInitializations.foundOne((jobDescriptor, activity)) diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala new file mode 100644 index 000000000..543fc9473 --- /dev/null +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala @@ -0,0 +1,311 @@ +package cromwell.engine.workflow.tokens + +import java.util.UUID + +import akka.actor.{ ActorRef, ActorSystem, Kill, PoisonPill} +import org.scalatest._ +import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} +import cromwell.core.JobExecutionToken.JobExecutionTokenType +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed, JobExecutionTokenRequest, JobExecutionTokenReturn} +import JobExecutionTokenDispenserActorSpec._ +import cromwell.core.JobExecutionToken +import cromwell.engine.workflow.tokens.TokenGrabbingActor.StoppingSupervisor +import org.scalatest.concurrent.Eventually + +import scala.concurrent.duration._ + +class JobExecutionTokenDispenserActorSpec extends TestKit(ActorSystem("JETDASpec")) with ImplicitSender with FlatSpecLike with Matchers with BeforeAndAfter with BeforeAndAfterAll with Eventually { + + val MaxWaitTime = 100.milliseconds + implicit val pc: PatienceConfig = PatienceConfig(MaxWaitTime) + + behavior of "JobExecutionTokenDispenserActor" + + it should "dispense an infinite token correctly" in { + actorRefUnderTest ! JobExecutionTokenRequest(TestInfiniteTokenType) + expectMsgPF(max = MaxWaitTime, hint = "token dispensed message") { + case JobExecutionTokenDispensed(token) => + token.jobExecutionTokenType should be(TestInfiniteTokenType) + } + } + + it should "accept return of an infinite token correctly" in { + actorRefUnderTest ! JobExecutionTokenRequest(TestInfiniteTokenType) + expectMsgPF(max = MaxWaitTime, hint = "token dispensed message") { + case JobExecutionTokenDispensed(token) => + actorRefUnderTest ! JobExecutionTokenReturn(token) + } + } + + it should "dispense indefinitely for an infinite token type" in { + var currentSet: Set[JobExecutionToken] = Set.empty + 100 indexedTimes { i => + val sender = TestProbe() + actorRefUnderTest.tell(msg = JobExecutionTokenRequest(TestInfiniteTokenType), sender = sender.ref) + sender.expectMsgPF(max = MaxWaitTime, hint = "token dispensed message") { + case JobExecutionTokenDispensed(token) => + token.jobExecutionTokenType should be(TestInfiniteTokenType) + currentSet.contains(token) should be(false) + currentSet += token + } + } + } + + it should "dispense a limited token correctly" in { + + actorRefUnderTest ! JobExecutionTokenRequest(LimitedTo5Tokens) + expectMsgPF(max = MaxWaitTime, hint = "token dispensed message") { + case JobExecutionTokenDispensed(token) => token.jobExecutionTokenType should be(LimitedTo5Tokens) + } + } + + it should "accept return of a limited token type correctly" in { + actorRefUnderTest ! JobExecutionTokenRequest(LimitedTo5Tokens) + expectMsgPF(max = MaxWaitTime, hint = "token dispensed message") { + case JobExecutionTokenDispensed(token) => actorRefUnderTest ! JobExecutionTokenReturn(token) + } + } + + it should "limit the dispensing of a limited token type" in { + + var currentTokens: Map[TestProbe, JobExecutionToken] = Map.empty + val dummyActors = (0 until 100 map { i => i -> TestProbe("dummy_" + i) }).toMap + + // Dispense the first 5: + 5 indexedTimes { i => + val sndr = dummyActors(i) + actorRefUnderTest.tell(msg = JobExecutionTokenRequest(LimitedTo5Tokens), sender = sndr.ref) + sndr.expectMsgPF(max = MaxWaitTime, hint = "token dispensed message") { + case JobExecutionTokenDispensed(token) => + token.jobExecutionTokenType should be(LimitedTo5Tokens) + currentTokens.values.toList.contains(token) should be(false) // Check we didn't already get this token + currentTokens += sndr -> token + } + } + + // Queue the next 95: + 95 indexedTimes { i => + val sndr = dummyActors(5 + i) + actorRefUnderTest.tell(msg = JobExecutionTokenRequest(LimitedTo5Tokens), sender = sndr.ref) + sndr.expectMsgPF(max = MaxWaitTime, hint = "token denied message") { + case JobExecutionTokenDenied(positionInQueue) => + positionInQueue should be(i) + } + } + + // It should allow queued actors to check their position in the queue: + 95 indexedTimes { i => + val sndr = dummyActors(5 + i) + actorRefUnderTest.tell(msg = JobExecutionTokenRequest(LimitedTo5Tokens), sender = sndr.ref) + sndr.expectMsgPF(max = MaxWaitTime, hint = "token denied message") { + case JobExecutionTokenDenied(positionInQueue) => + positionInQueue should be(i) + } + } + + // It should release tokens as soon as they're available (while there's still a queue...): + 95 indexedTimes { i => + val returner = dummyActors(i) + val nextInLine = dummyActors(i + 5) + val tokenBeingReturned = currentTokens(returner) + actorRefUnderTest.tell(msg = JobExecutionTokenReturn(tokenBeingReturned), sender = returner.ref) + currentTokens -= returner + nextInLine.expectMsgPF(max = MaxWaitTime, hint = s"token dispensed message to the next in line actor (#${i + 5})") { + case JobExecutionTokenDispensed(token) => + token should be(tokenBeingReturned) // It just gets immediately passed out again! + currentTokens += nextInLine -> token + } + } + + // Double-check the queue state: when we request a token now, we should still be denied: + actorRefUnderTest ! JobExecutionTokenRequest(LimitedTo5Tokens) + expectMsgClass(classOf[JobExecutionTokenDenied]) + + //And finally, silently release the remaining tokens: + 5 indexedTimes { i => + val returner = dummyActors(i + 95) + val tokenBeingReturned = currentTokens(returner) + actorRefUnderTest.tell(msg = JobExecutionTokenReturn(tokenBeingReturned), sender = returner.ref) + currentTokens -= returner + } + + // And we should have gotten our own token by now: + expectMsgClass(classOf[JobExecutionTokenDispensed]) + + // Check we didn't get anything else in the meanwhile: + msgAvailable should be(false) + dummyActors.values foreach { testProbe => testProbe.msgAvailable should be(false) } + } + + it should "resend the same token to an actor which already has one" in { + actorRefUnderTest ! JobExecutionTokenRequest(LimitedTo5Tokens) + val firstResponse = expectMsgClass(classOf[JobExecutionTokenDispensed]) + + 5 indexedTimes { i => + actorRefUnderTest ! JobExecutionTokenRequest(LimitedTo5Tokens) + expectMsg(MaxWaitTime, s"same token again (attempt ${i + 1})", firstResponse) // Always the same + } + } + + + // Incidentally, also covers: it should "not be fooled if the wrong actor returns a token" + it should "not be fooled by a doubly-returned token" in { + var currentTokens: Map[TestProbe, JobExecutionToken] = Map.empty + val dummyActors = (0 until 7 map { i => i -> TestProbe("dummy_" + i) }).toMap + + // Set up by taking all 5 tokens out, and then adding 2 to the queue: + 5 indexedTimes { i => + val sndr = dummyActors(i) + actorRefUnderTest.tell(msg = JobExecutionTokenRequest(LimitedTo5Tokens), sender = sndr.ref) + currentTokens += dummyActors(i) -> sndr.expectMsgClass(classOf[JobExecutionTokenDispensed]).jobExecutionToken + } + 2 indexedTimes { i => + val sndr = dummyActors(5 + i) + actorRefUnderTest.tell(msg = JobExecutionTokenRequest(LimitedTo5Tokens), sender = sndr.ref) + sndr.expectMsgClass(classOf[JobExecutionTokenDenied]) + } + + // The first time we return a token, the next in line should be given it: + val returningActor = dummyActors(0) + val nextInLine1 = dummyActors(5) + val nextInLine2 = dummyActors(6) + val tokenBeingReturned = currentTokens(returningActor) + currentTokens -= returningActor + actorRefUnderTest.tell(msg = JobExecutionTokenReturn(tokenBeingReturned), sender = returningActor.ref) + val tokenPassedOn = nextInLine1.expectMsgClass(classOf[JobExecutionTokenDispensed]).jobExecutionToken + tokenPassedOn should be(tokenBeingReturned) + currentTokens += nextInLine1 -> tokenPassedOn + + // But the next time, nothing should happen because the wrong actor is returning the token: + actorRefUnderTest.tell(msg = JobExecutionTokenReturn(tokenBeingReturned), sender = returningActor.ref) + nextInLine2.expectNoMsg(MaxWaitTime) + } + + it should "not be fooled if an actor returns a token which doesn't exist" in { + var currentTokens: Map[TestProbe, JobExecutionToken] = Map.empty + val dummyActors = (0 until 6 map { i => i -> TestProbe("dummy_" + i) }).toMap + + // Set up by taking all 5 tokens out, and then adding 2 to the queue: + 5 indexedTimes { i => + val sndr = dummyActors(i) + actorRefUnderTest.tell(msg = JobExecutionTokenRequest(LimitedTo5Tokens), sender = sndr.ref) + currentTokens += dummyActors(i) -> sndr.expectMsgClass(classOf[JobExecutionTokenDispensed]).jobExecutionToken + } + 1 indexedTimes { i => + val sndr = dummyActors(5 + i) + actorRefUnderTest.tell(msg = JobExecutionTokenRequest(LimitedTo5Tokens), sender = sndr.ref) + sndr.expectMsgClass(classOf[JobExecutionTokenDenied]) + } + + actorRefUnderTest.tell(msg = JobExecutionTokenReturn(JobExecutionToken(LimitedTo5Tokens, UUID.randomUUID())), sender = dummyActors(0).ref) + dummyActors(5).expectNoMsg(MaxWaitTime) + } + + val actorDeathMethods: List[(String, ActorRef => Unit)] = List( + ("external_stop", (a: ActorRef) => system.stop(a)), + ("internal_stop", (a: ActorRef) => a ! TokenGrabbingActor.InternalStop), + ("poison_pill", (a: ActorRef) => a ! PoisonPill), + ("kill_message", (a: ActorRef) => a ! Kill), + ("throw_exception", (a: ActorRef) => a ! TokenGrabbingActor.ThrowException) + ) + + actorDeathMethods foreach { case (name, stopMethod) => + it should s"recover tokens lost to actors which are $name before they hand back their token" in { + var currentTokens: Map[TestActorRef[TokenGrabbingActor], JobExecutionToken] = Map.empty + var tokenGrabbingActors: Map[Int, TestActorRef[TokenGrabbingActor]] = Map.empty + val grabberSupervisor = TestActorRef(new StoppingSupervisor()) + + // Set up by taking all 5 tokens out, and then adding 2 to the queue: + 5 indexedTimes { i => + val newGrabbingActor = TestActorRef[TokenGrabbingActor](TokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"grabber_${name}_" + i) + tokenGrabbingActors += i -> newGrabbingActor + eventually { + newGrabbingActor.underlyingActor.token.isDefined should be(true) + } + currentTokens += newGrabbingActor -> newGrabbingActor.underlyingActor.token.get + } + + val unassignedActorIndex = 5 + val newGrabbingActor = TestActorRef(new TokenGrabbingActor(actorRefUnderTest, LimitedTo5Tokens), s"grabber_${name}_" + unassignedActorIndex) + tokenGrabbingActors += unassignedActorIndex -> newGrabbingActor + eventually { + newGrabbingActor.underlyingActor.rejections should be(1) + } + + val actorToStop = tokenGrabbingActors(0) + val actorToStopsToken = currentTokens(actorToStop) + val nextInLine = tokenGrabbingActors(unassignedActorIndex) + + val deathwatch = TestProbe() + deathwatch watch actorToStop + stopMethod(actorToStop) + deathwatch.expectTerminated(actorToStop) + eventually { nextInLine.underlyingActor.token should be(Some(actorToStopsToken)) } + } + } + + it should "skip over dead actors when assigning tokens to the actor queue" in { + var currentTokens: Map[TestActorRef[TokenGrabbingActor], JobExecutionToken] = Map.empty + var tokenGrabbingActors: Map[Int, TestActorRef[TokenGrabbingActor]] = Map.empty + val grabberSupervisor = TestActorRef(new StoppingSupervisor()) + + // Set up by taking all 5 tokens out, and then adding 2 to the queue: + 5 indexedTimes { i => + val newGrabbingActor = TestActorRef[TokenGrabbingActor](TokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"skip_test_" + i) + tokenGrabbingActors += i -> newGrabbingActor + eventually { + newGrabbingActor.underlyingActor.token.isDefined should be(true) + } + currentTokens += newGrabbingActor -> newGrabbingActor.underlyingActor.token.get + } + 2 indexedTimes { i => + val index = i + 5 + val newGrabbingActor = TestActorRef[TokenGrabbingActor](TokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"skip_test_" + index) + tokenGrabbingActors += index -> newGrabbingActor + eventually { + newGrabbingActor.underlyingActor.rejections should be(1) + } + } + + val returningActor = tokenGrabbingActors(0) + val returnedToken = currentTokens(returningActor) + val nextInLine1 = tokenGrabbingActors(5) + val nextInLine2 = tokenGrabbingActors(6) + + // First, kill off the actor which would otherwise be first in line: + val deathwatch = TestProbe() + deathwatch watch nextInLine1 + nextInLine1 ! PoisonPill + deathwatch.expectTerminated(nextInLine1) + + // Now, stop one of the workers unexpectedly and check that the released token goes to the right place: + actorRefUnderTest.tell(msg = JobExecutionTokenReturn(returnedToken), sender = returningActor) + eventually { nextInLine2.underlyingActor.token should be(Some(returnedToken)) } // Some is OK. This is the **expected** value! + } + + var actorRefUnderTest: TestActorRef[JobExecutionTokenDispenserActor] = _ + + before { + actorRefUnderTest = TestActorRef(new JobExecutionTokenDispenserActor()) + + } + after { + actorRefUnderTest = null + } + + override def afterAll = { + TestKit.shutdownActorSystem(system) + } +} + +object JobExecutionTokenDispenserActorSpec { + + implicit class intWithTimes(n: Int) { + def times(f: => Unit) = 1 to n foreach { _ => f } + def indexedTimes(f: Int => Unit) = 0 until n foreach { i => f(i) } + } + + val TestInfiniteTokenType = JobExecutionTokenType("infinite", maxPoolSize = None) + def limitedTokenType(limit: Int) = JobExecutionTokenType(s"$limit-limit", maxPoolSize = Option(limit)) + val LimitedTo5Tokens = limitedTokenType(5) +} diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/TokenGrabbingActor.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/TokenGrabbingActor.scala new file mode 100644 index 000000000..06eff8583 --- /dev/null +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/TokenGrabbingActor.scala @@ -0,0 +1,38 @@ +package cromwell.engine.workflow.tokens + +import akka.actor.{Actor, ActorRef, Props, SupervisorStrategy} +import cromwell.core.JobExecutionToken +import cromwell.core.JobExecutionToken.JobExecutionTokenType +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed, JobExecutionTokenRequest} +import cromwell.engine.workflow.tokens.TokenGrabbingActor.{InternalStop, ThrowException} + +/** + * Grabs a token and doesn't let it go! + */ +class TokenGrabbingActor(tokenDispenser: ActorRef, tokenType: JobExecutionTokenType) extends Actor { + + var token: Option[JobExecutionToken] = None + var rejections = 0 + + def receive = { + case JobExecutionTokenDispensed(dispensedToken) => token = Option(dispensedToken) + case JobExecutionTokenDenied(positionInQueue) => rejections += 1 + case ThrowException => throw new RuntimeException("Test exception (don't be scared by the stack trace, it's deliberate!)") + case InternalStop => context.stop(self) + } + + tokenDispenser ! JobExecutionTokenRequest(tokenType) +} + +object TokenGrabbingActor { + + def props(tokenDispenserActor: ActorRef, tokenType: JobExecutionTokenType) = Props(new TokenGrabbingActor(tokenDispenserActor, tokenType)) + + case object ThrowException + case object InternalStop + + class StoppingSupervisor extends Actor { + override val supervisorStrategy = SupervisorStrategy.stoppingStrategy + override def receive = Actor.emptyBehavior + } +} diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala index bd4ce3add..7faf6baed 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala @@ -13,7 +13,7 @@ import wdl4s.expression.WdlStandardLibraryFunctions import scala.util.{Failure, Success, Try} -case class HtCondorBackendFactory(configurationDescriptor: BackendConfigurationDescriptor) +case class HtCondorBackendFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory with StrictLogging { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala index 4028166cc..3f498ff71 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala @@ -15,7 +15,7 @@ import wdl4s.expression.WdlStandardLibraryFunctions import scala.language.postfixOps -case class JesBackendLifecycleActorFactory(configurationDescriptor: BackendConfigurationDescriptor) +case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory { import JesBackendLifecycleActorFactory._ diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala index a92e7fdb5..93ad1fe3a 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala @@ -4,6 +4,7 @@ import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction import cromwell.backend.impl.sfs.config.ConfigConstants._ import cromwell.backend.sfs._ import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, RuntimeAttributeDefinition} +import cromwell.core.JobExecutionToken.JobExecutionTokenType import lenthall.config.ScalaConfig._ import org.slf4j.LoggerFactory @@ -12,7 +13,7 @@ import org.slf4j.LoggerFactory * * @param configurationDescriptor The config information. */ -class ConfigBackendLifecycleActorFactory(val configurationDescriptor: BackendConfigurationDescriptor) +class ConfigBackendLifecycleActorFactory(name: String, val configurationDescriptor: BackendConfigurationDescriptor) extends SharedFileSystemBackendLifecycleActorFactory { lazy val logger = LoggerFactory.getLogger(getClass) @@ -44,4 +45,9 @@ class ConfigBackendLifecycleActorFactory(val configurationDescriptor: BackendCon } override lazy val fileHashingActorCount: Int = 5 + + override val jobExecutionTokenType: JobExecutionTokenType = { + val concurrentJobLimit = configurationDescriptor.backendConfig.getIntOption("concurrent-job-limit") + JobExecutionTokenType(name, concurrentJobLimit) + } } diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala index d69446519..ba09b6cb9 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala @@ -8,7 +8,7 @@ import cromwell.core.CallContext import wdl4s.Call import wdl4s.expression.WdlStandardLibraryFunctions -case class SparkBackendFactory(configurationDescriptor: BackendConfigurationDescriptor, actorSystem: ActorSystem) extends BackendLifecycleActorFactory { +case class SparkBackendFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor, actorSystem: ActorSystem) extends BackendLifecycleActorFactory { override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], serviceRegistryActor: ActorRef): Option[Props] = { Option(SparkInitializationActor.props(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor)) } From b5eeb30bbc33b296dca6ec8d9c02e047261c8eea Mon Sep 17 00:00:00 2001 From: Ruchi Date: Wed, 5 Oct 2016 10:09:06 -0400 Subject: [PATCH 14/23] allowing for multiple inputs (#1511) change submitRoute to accept multiple inputs + test overrides occur for conflict input keys --- engine/src/main/resources/swagger/cromwell.yaml | 22 +++++++++++++++++- .../cromwell/webservice/CromwellApiService.scala | 27 ++++++++++++++++++++-- .../webservice/CromwellApiServiceSpec.scala | 15 +++++++++++- 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/engine/src/main/resources/swagger/cromwell.yaml b/engine/src/main/resources/swagger/cromwell.yaml index cdbce1392..7960bb937 100644 --- a/engine/src/main/resources/swagger/cromwell.yaml +++ b/engine/src/main/resources/swagger/cromwell.yaml @@ -64,7 +64,27 @@ paths: type: file in: formData - name: workflowInputs - description: WDL Inputs JSON + description: WDL Inputs JSON, 1 + required: false + type: file + in: formData + - name: workflowInputs_2 + description: WDL Inputs JSON, 2 + required: false + type: file + in: formData + - name: workflowInputs_3 + description: WDL Inputs JSON, 3 + required: false + type: file + in: formData + - name: workflowInputs_4 + description: WDL Inputs JSON, 4 + required: false + type: file + in: formData + - name: workflowInputs_5 + description: WDL Inputs JSON, 5 required: false type: file in: formData diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala index 8e7c55168..6f66407fd 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala @@ -1,6 +1,7 @@ package cromwell.webservice import akka.actor._ +import java.lang.Throwable._ import cats.data.NonEmptyList import cromwell.core.{WorkflowId, WorkflowSourceFiles} import cromwell.engine.backend.BackendConfiguration @@ -25,6 +26,23 @@ trait CromwellApiService extends HttpService with PerRequestCreator { val workflowStoreActor: ActorRef val serviceRegistryActor: ActorRef + def toMap(someInput: Option[String]): Map[String, JsValue] = { + import spray.json._ + someInput match { + case Some(inputs: String) => inputs.parseJson match { + case JsObject(inputMap) => inputMap + case _ => + throw new RuntimeException(s"Submitted inputs couldn't be processed, please check for syntactical errors") + } + case None => Map.empty + } + } + + def mergeMaps(allInputs: Seq[Option[String]]): JsObject = { + val convertToMap = allInputs.map(x => toMap(x)) + JsObject(convertToMap reduce (_ ++ _)) + } + def metadataBuilderProps: Props = MetadataBuilderActor.props(serviceRegistryActor) def handleMetadataRequest(message: AnyRef): Route = { @@ -109,9 +127,14 @@ trait CromwellApiService extends HttpService with PerRequestCreator { def submitRoute = path("workflows" / Segment) { version => post { - formFields("wdlSource", "workflowInputs".?, "workflowOptions".?) { (wdlSource, workflowInputs, workflowOptions) => + formFields("wdlSource", "workflowInputs".?, "workflowInputs_2".?, "workflowInputs_3".?, + "workflowInputs_4".?, "workflowInputs_5".?, "workflowOptions".?) { + (wdlSource, workflowInputs, workflowInputs_2, workflowInputs_3, workflowInputs_4, workflowInputs_5, workflowOptions) => requestContext => - val workflowSourceFiles = WorkflowSourceFiles(wdlSource, workflowInputs.getOrElse("{}"), workflowOptions.getOrElse("{}")) + //The order of addition allows for the expected override of colliding keys. + val wfInputs = mergeMaps(Seq(workflowInputs, workflowInputs_2, workflowInputs_3, workflowInputs_4, workflowInputs_5)).toString + + val workflowSourceFiles = WorkflowSourceFiles(wdlSource, wfInputs, workflowOptions.getOrElse("{}")) perRequest(requestContext, CromwellApiHandler.props(workflowStoreActor), CromwellApiHandler.ApiHandlerWorkflowSubmit(workflowSourceFiles)) } } diff --git a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala index d6dc8919d..87db769c9 100644 --- a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala @@ -15,7 +15,8 @@ import cromwell.server.{CromwellServerActor, CromwellSystem} import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata._ import cromwell.services.metadata.impl.MetadataSummaryRefreshActor.MetadataSummarySuccess -import cromwell.util.SampleWdl.HelloWorld +import cromwell.util.SampleWdl.DeclarationsWorkflow._ +import cromwell.util.SampleWdl.{ExpressionsInInputs, DeclarationsWorkflow, ThreeStep, HelloWorld} import org.scalatest.concurrent.{PatienceConfiguration, ScalaFutures} import org.scalatest.{FlatSpec, Matchers} import org.specs2.mock.Mockito @@ -256,6 +257,18 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala } } } + it should "succesfully merge and override multiple input files" in { + + val input1 = Map("wf.a1" -> "hello", "wf.a2" -> "world").toJson.toString + val input2 = Map.empty.toJson.toString + val overrideInput1 = Map("wf.a2" -> "universe").toJson.toString + val allInputs = mergeMaps(Seq(Option(input1), Option(input2), Option(overrideInput1))) + + check { + allInputs.fields.keys should contain allOf("wf.a1", "wf.a2") + allInputs.fields("wf.a2") should be(JsString("universe")) + } + } behavior of "REST API batch submission endpoint" it should "return 200 for a successful workflow submission " in { From 72a3cfde8a1993970ef0362b63180ac6bddf0b61 Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Fri, 30 Sep 2016 17:12:56 -0400 Subject: [PATCH 15/23] Lots of cool compiler flags. Closes #1282. --- .../BackendWorkflowInitializationActor.scala | 2 +- .../backend/RuntimeAttributeDefinition.scala | 1 - .../async/AsyncBackendJobExecutionActor.scala | 3 + .../src/main/scala/cromwell/backend/backend.scala | 53 ++++++++ .../src/main/scala/cromwell/backend/package.scala | 54 +------- .../backend/validation/MemoryValidation.scala | 2 +- .../cromwell/backend/wdl/ReadLikeFunctions.scala | 1 - .../scala/cromwell/backend/io/TestWorkflows.scala | 14 +-- core/src/main/scala/cromwell/core/ConfigUtil.scala | 2 +- .../main/scala/cromwell/core/ExecutionStore.scala | 1 - core/src/main/scala/cromwell/core/PathCopier.scala | 1 + .../src/main/scala/cromwell/core/PathFactory.scala | 4 +- core/src/main/scala/cromwell/core/core.scala | 12 ++ .../cromwell/core/logging/LoggerWrapper.scala | 10 +- core/src/main/scala/cromwell/core/package.scala | 9 -- .../test/scala/cromwell/core/retry/RetrySpec.scala | 4 - core/src/test/scala/cromwell/util/SampleWdl.scala | 137 +++++++++++---------- .../scala/cromwell/util/TryWithResourceSpec.scala | 2 +- .../database/migration/WdlTransformation.scala | 1 - .../metadata/table/symbol/QueryPaginator.scala | 3 +- .../table/symbol/SymbolTableMigration.scala | 1 - .../restart/table/JobStoreSimpletonMigration.scala | 2 - .../database/slick/JobKeyValueSlickDatabase.scala | 6 +- .../database/slick/JobStoreSlickDatabase.scala | 7 +- .../database/slick/MetadataSlickDatabase.scala | 12 +- .../cromwell/database/slick/SlickDatabase.scala | 2 +- .../slick/SummaryStatusSlickDatabase.scala | 4 +- .../slick/WorkflowStoreSlickDatabase.scala | 9 +- .../scala/cromwell/database/sql/SqlDatabase.scala | 2 +- .../cromwell/engine/backend/CromwellBackends.scala | 1 - engine/src/main/scala/cromwell/engine/engine.scala | 25 ++++ .../src/main/scala/cromwell/engine/package.scala | 22 ---- .../workflow/SingleWorkflowRunnerActor.scala | 8 +- .../cromwell/engine/workflow/WorkflowActor.scala | 1 - .../engine/workflow/WorkflowManagerActor.scala | 10 +- .../workflow/lifecycle/CopyWorkflowLogsActor.scala | 6 +- .../MaterializeWorkflowDescriptorActor.scala | 1 - .../lifecycle/WorkflowFinalizationActor.scala | 1 - .../lifecycle/WorkflowInitializationActor.scala | 1 - .../execution/EngineJobExecutionActor.scala | 14 ++- .../execution/WorkflowExecutionActor.scala | 4 +- .../execution/WorkflowExecutionActorData.scala | 1 - .../execution/callcaching/CallCache.scala | 1 - .../execution/callcaching/CallCacheReadActor.scala | 11 +- .../engine/workflow/lifecycle/lifecycle.scala | 7 ++ .../engine/workflow/lifecycle/package.scala | 8 -- .../tokens/JobExecutionTokenDispenserActor.scala | 7 +- .../workflowstore/WorkflowStoreActor.scala | 1 - .../engine/workflow/workflowstore/package.scala | 18 --- .../workflow/workflowstore/workflowstore_.scala | 15 +++ .../cromwell/jobstore/JobStoreWriterActor.scala | 2 +- .../main/scala/cromwell/jobstore/jobstore_.scala | 10 ++ .../src/main/scala/cromwell/jobstore/package.scala | 8 +- .../scala/cromwell/server/CromwellServer.scala | 13 +- .../scala/cromwell/server/CromwellSystem.scala | 8 +- .../scala/cromwell/webservice/ApiDataModels.scala | 1 - .../cromwell/webservice/CromwellApiHandler.scala | 19 ++- .../cromwell/webservice/CromwellApiService.scala | 5 +- .../scala/cromwell/webservice/PerRequest.scala | 13 +- .../webservice/metadata/IndexedJsonValue.scala | 34 ++--- .../webservice/metadata/MetadataBuilderActor.scala | 17 +-- .../main/scala/cromwell/webservice/package.scala | 37 ------ .../scala/cromwell/webservice/webservice_.scala | 39 ++++++ .../scala/cromwell/ArrayOfArrayCoercionSpec.scala | 1 - .../test/scala/cromwell/ArrayWorkflowSpec.scala | 3 - .../scala/cromwell/CallCachingWorkflowSpec.scala | 16 +-- engine/src/test/scala/cromwell/CromwellSpec.scala | 1 - .../test/scala/cromwell/CromwellTestkitSpec.scala | 10 +- .../scala/cromwell/DeclarationWorkflowSpec.scala | 1 - .../scala/cromwell/FilePassingWorkflowSpec.scala | 4 +- .../src/test/scala/cromwell/MapWorkflowSpec.scala | 1 - .../test/scala/cromwell/MetadataWatchActor.scala | 1 + .../MultipleFilesWithSameNameWorkflowSpec.scala | 3 +- .../scala/cromwell/OptionalParamWorkflowSpec.scala | 5 +- .../cromwell/PostfixQuantifierWorkflowSpec.scala | 1 - .../test/scala/cromwell/RestartWorkflowSpec.scala | 4 +- .../test/scala/cromwell/ScatterWorkflowSpec.scala | 2 - .../scala/cromwell/SimpleWorkflowActorSpec.scala | 1 - .../cromwell/WdlFunctionsAtWorkflowLevelSpec.scala | 6 +- .../test/scala/cromwell/WorkflowOutputsSpec.scala | 1 - .../cromwell/engine/WorkflowManagerActorSpec.scala | 3 +- .../workflow/SingleWorkflowRunnerActorSpec.scala | 18 +-- .../engine/workflow/WorkflowActorSpec.scala | 1 - .../workflow/lifecycle/CachingConfigSpec.scala | 12 +- .../MaterializeWorkflowDescriptorActorSpec.scala | 7 +- .../execution/ejea/EjeaCheckingCallCacheSpec.scala | 1 + .../execution/ejea/EjeaCheckingJobStoreSpec.scala | 13 +- .../ejea/EngineJobExecutionActorSpec.scala | 4 +- .../ejea/EngineJobExecutionActorSpecUtil.scala | 3 +- .../lifecycle/execution/ejea/ExpectOne.scala | 6 +- .../JobExecutionTokenDispenserActorSpec.scala | 2 +- .../cromwell/jobstore/JobStoreServiceSpec.scala | 2 - .../cromwell/jobstore/JobStoreWriterSpec.scala | 3 + .../webservice/CromwellApiServiceSpec.scala | 7 +- .../webservice/MetadataBuilderActorSpec.scala | 15 ++- .../cromwell/filesystems/gcs/GcsFileSystem.scala | 1 - .../filesystems/gcs/GcsFileSystemProvider.scala | 30 +++-- .../cromwell/filesystems/gcs/GoogleAuthMode.scala | 2 +- .../filesystems/gcs/GoogleConfiguration.scala | 1 - .../cromwell/filesystems/gcs/NioGcsPath.scala | 2 +- .../filesystems/gcs/GoogleConfigurationSpec.scala | 1 - project/Settings.scala | 26 +++- .../services/metadata/MetadataService.scala | 1 - .../services/metadata/WorkflowQueryKey.scala | 1 - .../metadata/WorkflowQueryParameters.scala | 1 - .../metadata/impl/MetadataDatabaseAccess.scala | 3 +- .../metadata/impl/MetadataServiceActor.scala | 10 +- .../impl/MetadataSummaryRefreshActor.scala | 1 - .../cromwell/services/metadata/metadata.scala | 39 ++++++ .../scala/cromwell/services/metadata/package.scala | 37 ------ src/main/scala/cromwell/Main.scala | 2 - .../scala/cromwell/CromwellCommandLineSpec.scala | 1 - .../impl/htcondor/HtCondorJobExecutionActor.scala | 28 +++-- .../backend/impl/htcondor/HtCondorWrapper.scala | 4 +- .../htcondor/HtCondorInitializationActorSpec.scala | 4 +- .../htcondor/HtCondorJobExecutionActorSpec.scala | 13 +- .../htcondor/HtCondorRuntimeAttributesSpec.scala | 12 +- .../backend/impl/jes/GenomicsFactory.scala | 1 - .../jes/JesAsyncBackendJobExecutionActor.scala | 8 +- .../cromwell/backend/impl/jes/JesAttributes.scala | 3 - .../impl/jes/JesBackendLifecycleActorFactory.scala | 1 - .../backend/impl/jes/JesFinalizationActor.scala | 5 +- .../backend/impl/jes/JesInitializationActor.scala | 6 +- .../impl/jes/JesJobCachingActorHelper.scala | 2 +- .../backend/impl/jes/JesJobExecutionActor.scala | 1 - .../backend/impl/jes/JesRuntimeAttributes.scala | 1 - .../main/scala/cromwell/backend/impl/jes/Run.scala | 7 +- .../backend/impl/jes/io/JesAttachedDisk.scala | 1 - .../cromwell/backend/impl/jes/io/package.scala | 2 - .../jes/JesAsyncBackendJobExecutionActorSpec.scala | 19 +-- .../impl/jes/JesInitializationActorSpec.scala | 4 +- .../impl/jes/JesRuntimeAttributesSpec.scala | 5 +- .../sfs/config/ConfigAsyncJobExecutionActor.scala | 1 + .../sfs/BackgroundAsyncJobExecutionActor.scala | 2 + .../scala/cromwell/backend/sfs/ProcessRunner.scala | 4 +- .../cromwell/backend/sfs/SharedFileSystem.scala | 13 +- .../SharedFileSystemAsyncJobExecutionActor.scala | 1 + .../sfs/SharedFileSystemCacheHitCopyingActor.scala | 6 +- .../sfs/SharedFileSystemInitializationActor.scala | 2 +- .../sfs/config/ConfigHashingStrategySpec.scala | 2 + .../SharedFileSystemInitializationActorSpec.scala | 4 +- ...stemValidatedRuntimeAttributesBuilderSpec.scala | 9 +- .../backend/impl/spark/SparkClusterProcess.scala | 5 +- .../impl/spark/SparkInitializationActor.scala | 4 +- .../impl/spark/SparkJobExecutionActor.scala | 10 +- .../cromwell/backend/impl/spark/SparkProcess.scala | 1 - .../impl/spark/SparkClusterProcessSpec.scala | 2 - .../impl/spark/SparkJobExecutionActorSpec.scala | 10 +- .../impl/spark/SparkRuntimeAttributesSpec.scala | 12 +- 149 files changed, 648 insertions(+), 614 deletions(-) create mode 100644 backend/src/main/scala/cromwell/backend/backend.scala create mode 100644 core/src/main/scala/cromwell/core/core.scala create mode 100644 engine/src/main/scala/cromwell/engine/engine.scala create mode 100644 engine/src/main/scala/cromwell/engine/workflow/lifecycle/lifecycle.scala delete mode 100644 engine/src/main/scala/cromwell/engine/workflow/lifecycle/package.scala delete mode 100644 engine/src/main/scala/cromwell/engine/workflow/workflowstore/package.scala create mode 100644 engine/src/main/scala/cromwell/engine/workflow/workflowstore/workflowstore_.scala create mode 100644 engine/src/main/scala/cromwell/jobstore/jobstore_.scala create mode 100644 engine/src/main/scala/cromwell/webservice/webservice_.scala create mode 100644 services/src/main/scala/cromwell/services/metadata/metadata.scala diff --git a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala index 7f0a273db..f98234ce5 100644 --- a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala @@ -142,7 +142,7 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w def badRuntimeAttrsForTask(task: Task) = { runtimeAttributeValidators map { case (attributeName, validator) => val value = task.runtimeAttributes.attrs.get(attributeName) orElse defaultRuntimeAttribute(attributeName) - attributeName -> (value, validator(value)) + attributeName -> ((value, validator(value))) } collect { case (name, (value, false)) => s"Task ${task.name} has an invalid runtime attribute $name = ${value map { _.valueString} getOrElse "!! NOT FOUND !!"}" } diff --git a/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala b/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala index a3b4d9255..238309143 100644 --- a/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala +++ b/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala @@ -39,7 +39,6 @@ object RuntimeAttributeDefinition { def addDefaultsToAttributes(runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition], workflowOptions: WorkflowOptions) (specifiedAttributes: Map[LocallyQualifiedName, WdlValue]): Map[LocallyQualifiedName, WdlValue] = { import WdlValueJsonFormatter._ - import spray.json._ // IGNORE INTELLIJ - this *is* required (unless it isn't any more, who will ever know...!) def isUnspecifiedAttribute(name: String) = !specifiedAttributes.contains(name) diff --git a/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala index 40747c346..bbbfbf82b 100644 --- a/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala @@ -70,7 +70,10 @@ trait AsyncBackendJobExecutionActor { this: Actor with ActorLogging => case IssuePollRequest(handle) => robustPoll(handle) case PollResponseReceived(handle) if handle.isDone => self ! Finish(handle) case PollResponseReceived(handle) => + // This should stash the Cancellable someplace so it can be cancelled once polling is complete. + // -Ywarn-value-discard context.system.scheduler.scheduleOnce(pollBackOff.backoffMillis.millis, self, IssuePollRequest(handle)) + () case Finish(SuccessfulExecutionHandle(outputs, returnCode, jobDetritusFiles, executionEvents, resultsClonedFrom)) => completionPromise.success(SucceededResponse(jobDescriptor.key, Some(returnCode), outputs, Option(jobDetritusFiles), executionEvents)) context.stop(self) diff --git a/backend/src/main/scala/cromwell/backend/backend.scala b/backend/src/main/scala/cromwell/backend/backend.scala new file mode 100644 index 000000000..8ac55a347 --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/backend.scala @@ -0,0 +1,53 @@ +package cromwell.backend + +import com.typesafe.config.Config +import cromwell.core.WorkflowOptions.WorkflowOption +import cromwell.core.{JobKey, WorkflowId, WorkflowOptions} +import wdl4s.values.WdlValue +import wdl4s.{Call, NamespaceWithWorkflow, _} + +import scala.util.Try + +/** + * For uniquely identifying a job which has been or will be sent to the backend. + */ +case class BackendJobDescriptorKey(call: Call, index: Option[Int], attempt: Int) extends JobKey { + def scope = call + private val indexString = index map { _.toString } getOrElse "NA" + val tag = s"${call.fullyQualifiedName}:$indexString:$attempt" + val isShard = index.isDefined + def mkTag(workflowId: WorkflowId) = s"$workflowId:$this" +} + +/** + * For passing to a BackendWorkflowActor for job execution or recovery + */ +case class BackendJobDescriptor(workflowDescriptor: BackendWorkflowDescriptor, + key: BackendJobDescriptorKey, + runtimeAttributes: Map[LocallyQualifiedName, WdlValue], + inputs: Map[LocallyQualifiedName, WdlValue]) { + val call = key.call + override val toString = s"${key.mkTag(workflowDescriptor.id)}" +} + +/** + * For passing to a BackendActor construction time + */ +case class BackendWorkflowDescriptor(id: WorkflowId, + workflowNamespace: NamespaceWithWorkflow, + inputs: Map[FullyQualifiedName, WdlValue], + workflowOptions: WorkflowOptions) { + override def toString: String = s"[BackendWorkflowDescriptor id=${id.shortString} workflowName=${workflowNamespace.workflow.unqualifiedName}]" + def getWorkflowOption(key: WorkflowOption) = workflowOptions.get(key).toOption +} + +/** + * For passing to a BackendActor construction time + */ +case class BackendConfigurationDescriptor(backendConfig: Config, globalConfig: Config) + +final case class AttemptedLookupResult(name: String, value: Try[WdlValue]) { + def toPair = name -> value +} + +case class PreemptedException(msg: String) extends Exception(msg) diff --git a/backend/src/main/scala/cromwell/backend/package.scala b/backend/src/main/scala/cromwell/backend/package.scala index e5bb64474..3bad6f61f 100644 --- a/backend/src/main/scala/cromwell/backend/package.scala +++ b/backend/src/main/scala/cromwell/backend/package.scala @@ -1,66 +1,14 @@ package cromwell -import com.typesafe.config.Config -import cromwell.core.WorkflowOptions.WorkflowOption -import cromwell.core.{JobKey, WorkflowId, WorkflowOptions} -import cromwell.util.JsonFormatting.WdlValueJsonFormatter -import wdl4s._ -import wdl4s.expression.WdlStandardLibraryFunctions -import wdl4s.util.TryUtil import wdl4s.values.WdlValue import scala.language.postfixOps -import scala.util.{Success, Try} +import scala.util.Success package object backend { - - /** - * For uniquely identifying a job which has been or will be sent to the backend. - */ - case class BackendJobDescriptorKey(call: Call, index: Option[Int], attempt: Int) extends JobKey { - def scope = call - private val indexString = index map { _.toString } getOrElse "NA" - val tag = s"${call.fullyQualifiedName}:$indexString:$attempt" - val isShard = index.isDefined - def mkTag(workflowId: WorkflowId) = s"$workflowId:$this" - } - - /** - * For passing to a BackendWorkflowActor for job execution or recovery - */ - case class BackendJobDescriptor(workflowDescriptor: BackendWorkflowDescriptor, - key: BackendJobDescriptorKey, - runtimeAttributes: Map[LocallyQualifiedName, WdlValue], - inputs: Map[LocallyQualifiedName, WdlValue]) { - val call = key.call - override val toString = s"${key.mkTag(workflowDescriptor.id)}" - } - - /** - * For passing to a BackendActor construction time - */ - case class BackendWorkflowDescriptor(id: WorkflowId, - workflowNamespace: NamespaceWithWorkflow, - inputs: Map[FullyQualifiedName, WdlValue], - workflowOptions: WorkflowOptions) { - override def toString: String = s"[BackendWorkflowDescriptor id=${id.shortString} workflowName=${workflowNamespace.workflow.unqualifiedName}]" - def getWorkflowOption(key: WorkflowOption) = workflowOptions.get(key).toOption - } - - /** - * For passing to a BackendActor construction time - */ - case class BackendConfigurationDescriptor(backendConfig: Config, globalConfig: Config) - - final case class AttemptedLookupResult(name: String, value: Try[WdlValue]) { - def toPair = name -> value - } - implicit class AugmentedAttemptedLookupSequence(s: Seq[AttemptedLookupResult]) { def toLookupMap: Map[String, WdlValue] = s collect { case AttemptedLookupResult(name, Success(value)) => (name, value) } toMap } - - case class PreemptedException(msg: String) extends Exception(msg) } diff --git a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala index 81e8f3ea7..17ba9fb66 100644 --- a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala @@ -53,7 +53,7 @@ object MemoryValidation { if (value <= 0) wrongAmountFormat.format(value).invalidNel else - MemorySize(value, MemoryUnit.Bytes).to(MemoryUnit.GB).validNel + MemorySize(value.toDouble, MemoryUnit.Bytes).to(MemoryUnit.GB).validNel } } diff --git a/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala b/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala index fd2aae6ea..1f06e10d8 100644 --- a/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala +++ b/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala @@ -6,7 +6,6 @@ import wdl4s.parser.MemoryUnit import wdl4s.types.{WdlArrayType, WdlFileType, WdlObjectType, WdlStringType} import wdl4s.values._ -import scala.language.postfixOps import scala.util.{Failure, Success, Try} trait ReadLikeFunctions extends FileSystems { this: WdlStandardLibraryFunctions => diff --git a/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala b/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala index 6905fa0f1..34497e2ef 100644 --- a/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala +++ b/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala @@ -10,11 +10,11 @@ object TestWorkflows { expectedResponse: BackendJobExecutionResponse) val HelloWorld = - """ + s""" |task hello { | String addressee = "you " | command { - | echo "Hello ${addressee}!" + | echo "Hello $${addressee}!" | } | output { | String salutation = read_string(stdout()) @@ -45,14 +45,14 @@ object TestWorkflows { """.stripMargin val InputFiles = - """ + s""" |task localize { | File inputFileFromJson | File inputFileFromCallInputs | command { - | cat ${inputFileFromJson} + | cat $${inputFileFromJson} | echo "" - | cat ${inputFileFromCallInputs} + | cat $${inputFileFromCallInputs} | } | output { | Array[String] out = read_lines(stdout()) @@ -82,11 +82,11 @@ object TestWorkflows { """.stripMargin val Scatter = - """ + s""" |task scattering { | Int intNumber | command { - | echo ${intNumber} + | echo $${intNumber} | } | output { | Int out = read_string(stdout()) diff --git a/core/src/main/scala/cromwell/core/ConfigUtil.scala b/core/src/main/scala/cromwell/core/ConfigUtil.scala index 9f22c330e..881fec686 100644 --- a/core/src/main/scala/cromwell/core/ConfigUtil.scala +++ b/core/src/main/scala/cromwell/core/ConfigUtil.scala @@ -43,7 +43,7 @@ object ConfigUtil { def validateConfig(key: String): ValidatedNel[String, Config] = try { config.getConfig(key).validNel } catch { - case e: ConfigException.Missing => "Could not find key: $key".invalidNel + case e: ConfigException.Missing => s"Could not find key: $key".invalidNel case e: ConfigException.WrongType => s"key $key cannot be parsed to a Config".invalidNel } diff --git a/core/src/main/scala/cromwell/core/ExecutionStore.scala b/core/src/main/scala/cromwell/core/ExecutionStore.scala index 3512e68ef..1632061ce 100644 --- a/core/src/main/scala/cromwell/core/ExecutionStore.scala +++ b/core/src/main/scala/cromwell/core/ExecutionStore.scala @@ -2,7 +2,6 @@ package cromwell.core import cromwell.core.ExecutionStatus._ -import scala.language.postfixOps object ExecutionStore { def empty = ExecutionStore(Map.empty) diff --git a/core/src/main/scala/cromwell/core/PathCopier.scala b/core/src/main/scala/cromwell/core/PathCopier.scala index f7ec545ce..c941ea890 100644 --- a/core/src/main/scala/cromwell/core/PathCopier.scala +++ b/core/src/main/scala/cromwell/core/PathCopier.scala @@ -24,5 +24,6 @@ object PathCopier { def copy(sourceFilePath: Path, destinationFilePath: Path): Unit = { Option(File(destinationFilePath).parent).foreach(_.createDirectories()) File(sourceFilePath).copyTo(destinationFilePath, overwrite = true) + () } } diff --git a/core/src/main/scala/cromwell/core/PathFactory.scala b/core/src/main/scala/cromwell/core/PathFactory.scala index 499578107..f85a05c95 100644 --- a/core/src/main/scala/cromwell/core/PathFactory.scala +++ b/core/src/main/scala/cromwell/core/PathFactory.scala @@ -80,7 +80,7 @@ trait PathWriter { * * @param string Line to add to the logs. */ - def writeWithNewline(string: String) { + def writeWithNewline(string: String): Unit = { writer.write(string) writer.write("\n") } @@ -108,7 +108,7 @@ case class TailedWriter(path: Path, tailedSize: Int) extends PathWriter { * * @param string Line to add to the logs. */ - override def writeWithNewline(string: String) { + override def writeWithNewline(string: String): Unit = { tailedLines :+= string while (tailedLines.size > tailedSize) { tailedLines = tailedLines.takeRight(tailedSize) diff --git a/core/src/main/scala/cromwell/core/core.scala b/core/src/main/scala/cromwell/core/core.scala new file mode 100644 index 000000000..ec1a3babe --- /dev/null +++ b/core/src/main/scala/cromwell/core/core.scala @@ -0,0 +1,12 @@ +package cromwell.core + +import java.nio.file.Path + +import lenthall.exception.ThrowableAggregation +import wdl4s.values.WdlValue + + +case class CallContext(root: Path, stdout: String, stderr: String) +case class JobOutput(wdlValue: WdlValue) +class CromwellFatalException(exception: Throwable) extends Exception(exception) +case class CromwellAggregatedException(throwables: Seq[Throwable], exceptionContext: String = "") extends ThrowableAggregation diff --git a/core/src/main/scala/cromwell/core/logging/LoggerWrapper.scala b/core/src/main/scala/cromwell/core/logging/LoggerWrapper.scala index ee04700e5..196762aa1 100644 --- a/core/src/main/scala/cromwell/core/logging/LoggerWrapper.scala +++ b/core/src/main/scala/cromwell/core/logging/LoggerWrapper.scala @@ -89,7 +89,7 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { slf4jLoggers.foreach(_.error(formatted, t)) } - override def error(pattern: String, arguments: AnyRef*) { + override def error(pattern: String, arguments: AnyRef*): Unit = { lazy val formatted: String = format(pattern) varargsAkkaLog(pattern, arguments) @@ -110,7 +110,7 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { slf4jLoggers.foreach(_.error(formatted, arg1, arg2: Any)) } - def error(t: Throwable, pattern: String, arguments: Any*) { + def error(t: Throwable, pattern: String, arguments: Any*): Unit = { lazy val formatted: String = format(pattern) akkaLogger.foreach(_.error(t, formatted, arguments)) @@ -131,7 +131,7 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { slf4jLoggers.foreach(_.debug(formatted, t)) } - override def debug(pattern: String, arguments: AnyRef*) { + override def debug(pattern: String, arguments: AnyRef*): Unit = { lazy val formatted: String = format(pattern) varargsAkkaLog(pattern, arguments) @@ -160,7 +160,7 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { slf4jLoggers.foreach(_.trace(format(msg), t)) } - override def trace(pattern: String, arguments: AnyRef*) { + override def trace(pattern: String, arguments: AnyRef*): Unit = { slf4jLoggers.foreach(_.trace(format(pattern), arguments:_*)) } @@ -186,7 +186,7 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { slf4jLoggers.foreach(_.info(formatted, t)) } - override def info(pattern: String, arguments: AnyRef*) { + override def info(pattern: String, arguments: AnyRef*): Unit = { lazy val formatted: String = format(pattern) varargsAkkaLog(pattern, arguments) diff --git a/core/src/main/scala/cromwell/core/package.scala b/core/src/main/scala/cromwell/core/package.scala index ee1448640..3334bfa24 100644 --- a/core/src/main/scala/cromwell/core/package.scala +++ b/core/src/main/scala/cromwell/core/package.scala @@ -1,22 +1,13 @@ package cromwell -import java.nio.file.Path - -import lenthall.exception.ThrowableAggregation import wdl4s.values.WdlValue package object core { - case class CallContext(root: Path, stdout: String, stderr: String) - type LocallyQualifiedName = String type FullyQualifiedName = String type WorkflowOutputs = Map[FullyQualifiedName, JobOutput] type WorkflowOptionsJson = String - case class JobOutput(wdlValue: WdlValue) type JobOutputs = Map[LocallyQualifiedName, JobOutput] type HostInputs = Map[String, WdlValue] type EvaluatedRuntimeAttributes = Map[String, WdlValue] - - class CromwellFatalException(exception: Throwable) extends Exception(exception) - case class CromwellAggregatedException(throwables: Seq[Throwable], exceptionContext: String = "") extends ThrowableAggregation } diff --git a/core/src/test/scala/cromwell/core/retry/RetrySpec.scala b/core/src/test/scala/cromwell/core/retry/RetrySpec.scala index 3e5c03886..27f24076c 100644 --- a/core/src/test/scala/cromwell/core/retry/RetrySpec.scala +++ b/core/src/test/scala/cromwell/core/retry/RetrySpec.scala @@ -7,7 +7,6 @@ import org.scalatest.time.{Millis, Seconds, Span} import org.scalatest.{FlatSpecLike, Matchers} import scala.concurrent.Future -import scala.concurrent.duration._ class RetrySpec extends TestKitSuite("retry-spec") with FlatSpecLike with Matchers with ScalaFutures { class TransientException extends Exception @@ -33,9 +32,6 @@ class RetrySpec extends TestKitSuite("retry-spec") with FlatSpecLike with Matche work: MockWork, isTransient: Throwable => Boolean = Retry.throwableToFalse, isFatal: Throwable => Boolean = Retry.throwableToFalse): Future[Int] = { - implicit val ec = system.dispatcher - - val backoff = SimpleExponentialBackoff(1.millis, 2.millis, 1) withRetry( f = work.doIt, diff --git a/core/src/test/scala/cromwell/util/SampleWdl.scala b/core/src/test/scala/cromwell/util/SampleWdl.scala index cdc3fde5f..dc2598cf1 100644 --- a/core/src/test/scala/cromwell/util/SampleWdl.scala +++ b/core/src/test/scala/cromwell/util/SampleWdl.scala @@ -24,6 +24,7 @@ trait SampleWdl extends TestFileUtil { createFile("f1", base, "line1\nline2\n") createFile("f2", base, "line3\nline4\n") createFile("f3", base, "line5\n") + () } def cleanupFileArray(base: Path) = { @@ -61,11 +62,11 @@ object SampleWdl { object HelloWorld extends SampleWdl { override def wdlSource(runtime: String = "") = - """ + s""" |task hello { | String addressee | command { - | echo "Hello ${addressee}!" + | echo "Hello $${addressee}!" | } | output { | String salutation = read_string(stdout()) @@ -86,11 +87,11 @@ object SampleWdl { object HelloWorldWithoutWorkflow extends SampleWdl { override def wdlSource(runtime: String = "") = - """ + s""" |task hello { | String addressee | command { - | echo "Hello ${addressee}!" + | echo "Hello $${addressee}!" | } | output { | String salutation = read_string(stdout()) @@ -127,7 +128,7 @@ object SampleWdl { object EmptyString extends SampleWdl { override def wdlSource(runtime: String = "") = - """ + s""" |task hello { | command { | echo "Hello!" @@ -141,7 +142,7 @@ object SampleWdl { |task goodbye { | String emptyInputString | command { - | echo "${emptyInputString}" + | echo "$${emptyInputString}" | } | output { | String empty = read_string(stdout()) @@ -174,11 +175,11 @@ object SampleWdl { object CoercionNotDefined extends SampleWdl { override def wdlSource(runtime: String = "") = { - """ + s""" |task summary { | String bfile | command { - | ~/plink --bfile ${bfile} --missing --hardy --out foo --allow-no-sex + | ~/plink --bfile $${bfile} --missing --hardy --out foo --allow-no-sex | } | output { | File hwe = "foo.hwe" @@ -208,7 +209,7 @@ object SampleWdl { private val outputSectionPlaceholder = "OUTPUTSECTIONPLACEHOLDER" def sourceString(outputsSection: String = "") = { val withPlaceholders = - """ + s""" |task ps { | command { | ps @@ -224,7 +225,7 @@ object SampleWdl { | File in_file | | command { - | grep '${pattern}' ${in_file} | wc -l + | grep '$${pattern}' $${in_file} | wc -l | } | output { | Int count = read_int(stdout()) @@ -235,7 +236,7 @@ object SampleWdl { |task wc { | File in_file | command { - | cat ${in_file} | wc -l + | cat $${in_file} | wc -l | } | output { | Int count = read_int(stdout()) @@ -385,12 +386,12 @@ object SampleWdl { object DeclarationsWorkflow extends SampleWdl { override def wdlSource(runtime: String): WdlSource = - """ + s""" |task cat { | File file | String? flags | command { - | cat ${flags} ${file} + | cat $${flags} $${file} | } | output { | File procs = stdout() @@ -402,7 +403,7 @@ object SampleWdl { | String pattern | File in_file | command { - | grep '${pattern}' ${in_file} | wc -l + | grep '$${pattern}' $${in_file} | wc -l | } | output { | Int count = read_int(stdout()) @@ -439,11 +440,11 @@ object SampleWdl { trait ZeroOrMorePostfixQuantifier extends SampleWdl { override def wdlSource(runtime: String): WdlSource = - """ + s""" |task hello { | Array[String] person | command { - | echo "hello ${sep = "," person}" + | echo "hello $${sep = "," person}" | } | output { | String greeting = read_string(stdout()) @@ -470,11 +471,11 @@ object SampleWdl { trait OneOrMorePostfixQuantifier extends SampleWdl { override def wdlSource(runtime: String): WdlSource = - """ + s""" |task hello { | Array[String]+ person | command { - | echo "hello ${sep = "," person}" + | echo "hello $${sep = "," person}" | } | output { | String greeting = read_string(stdout()) @@ -518,11 +519,11 @@ object SampleWdl { object ArrayIO extends SampleWdl { override def wdlSource(runtime: String = "") = - """task concat_files { + s"""task concat_files { | String? flags | Array[File]+ files | command { - | cat ${default = "-s" flags} ${sep = " " files} + | cat $${default = "-s" flags} $${sep = " " files} | } | output { | File concatenated = stdout() @@ -534,7 +535,7 @@ object SampleWdl { | String pattern | File root | command { - | find ${root} ${"-name " + pattern} + | find $${root} $${"-name " + pattern} | } | output { | Array[String] results = read_lines(stdout()) @@ -545,7 +546,7 @@ object SampleWdl { |task count_lines { | Array[File]+ files | command { - | cat ${sep = ' ' files} | wc -l + | cat $${sep = ' ' files} | wc -l | } | output { | Int count = read_int(stdout()) @@ -556,7 +557,7 @@ object SampleWdl { |task serialize { | Array[String] strs | command { - | cat ${write_lines(strs)} + | cat $${write_lines(strs)} | } | output { | String contents = read_string(stdout()) @@ -599,11 +600,11 @@ object SampleWdl { def cleanup() = cleanupFileArray(catRootDir) override def wdlSource(runtime: String = "") = - """ + s""" |task cat { | Array[File]+ files | command { - | cat -s ${sep = ' ' files} + | cat -s $${sep = ' ' files} | } | output { | Array[String] lines = read_lines(stdout()) @@ -624,11 +625,11 @@ object SampleWdl { def cleanup() = cleanupFileArray(catRootDir) override def wdlSource(runtime: String = "") = - """ + s""" |task write_map { | Map[File, String] file_to_name | command { - | cat ${write_map(file_to_name)} + | cat $${write_map(file_to_name)} | } | output { | String contents = read_string(stdout()) @@ -639,7 +640,7 @@ object SampleWdl { | command <<< | python <>> | output { @@ -658,7 +659,7 @@ object SampleWdl { } class ScatterWdl extends SampleWdl { - val tasks = """task A { + val tasks = s"""task A { | command { | echo -n -e "jeff\nchris\nmiguel\nthibault\nkhalid\nscott" | } @@ -671,7 +672,7 @@ object SampleWdl { |task B { | String B_in | command { - | python -c "print(len('${B_in}'))" + | python -c "print(len('$${B_in}'))" | } | RUNTIME | output { @@ -682,7 +683,7 @@ object SampleWdl { |task C { | Int C_in | command { - | python -c "print(${C_in}*100)" + | python -c "print($${C_in}*100)" | } | RUNTIME | output { @@ -693,7 +694,7 @@ object SampleWdl { |task D { | Array[Int] D_in | command { - | python -c "print(${sep = '+' D_in})" + | python -c "print($${sep = '+' D_in})" | } | RUNTIME | output { @@ -752,9 +753,9 @@ object SampleWdl { object SimpleScatterWdl extends SampleWdl { override def wdlSource(runtime: String = "") = - """task echo_int { + s"""task echo_int { | Int int - | command {echo ${int}} + | command {echo $${int}} | output {Int out = read_int(stdout())} | RUNTIME_PLACEHOLDER |} @@ -775,9 +776,9 @@ object SampleWdl { object SimpleScatterWdlWithOutputs extends SampleWdl { override def wdlSource(runtime: String = "") = - """task echo_int { + s"""task echo_int { | Int int - | command {echo ${int}} + | command {echo $${int}} | output {Int out = read_int(stdout())} |} | @@ -800,7 +801,7 @@ object SampleWdl { case class PrepareScatterGatherWdl(salt: String = UUID.randomUUID().toString) extends SampleWdl { override def wdlSource(runtime: String = "") = { - """ + s""" |# |# Goal here is to split up the input file into files of 1 line each (in the prepare) then in parallel call wc -w on each newly created file and count the words into another file then in the gather, sum the results of each parallel call to come up with |# the word-count for the fil @@ -809,7 +810,7 @@ object SampleWdl { |task do_prepare { | File input_file | command { - | split -l 1 ${input_file} temp_ && ls -1 temp_?? > files.list + | split -l 1 $${input_file} temp_ && ls -1 temp_?? > files.list | } | output { | Array[File] split_files = read_lines("files.list") @@ -821,8 +822,8 @@ object SampleWdl { | String salt | File input_file | command { - | # ${salt} - | wc -w ${input_file} > output.txt + | # $${salt} + | wc -w $${input_file} > output.txt | } | output { | File count_file = "output.txt" @@ -833,7 +834,7 @@ object SampleWdl { |task do_gather { | Array[File] input_files | command <<< - | cat ${sep = ' ' input_files} | awk '{s+=$1} END {print s}' + | cat $${sep = ' ' input_files} | awk '{s+=$$1} END {print s}' | >>> | output { | Int sum = read_int(stdout()) @@ -867,9 +868,9 @@ object SampleWdl { object FileClobber extends SampleWdl { override def wdlSource(runtime: String = "") = - """task read_line { + s"""task read_line { | File in - | command { cat ${in} } + | command { cat $${in} } | output { String out = read_string(stdout()) } |} | @@ -892,18 +893,18 @@ object SampleWdl { object FilePassingWorkflow extends SampleWdl { override def wdlSource(runtime: String): WdlSource = - """task a { + s"""task a { | File in | String out_name = "out" | | command { - | cat ${in} > ${out_name} + | cat $${in} > $${out_name} | } | RUNTIME | output { | File out = "out" - | File out_interpolation = "${out_name}" - | String contents = read_string("${out_name}") + | File out_interpolation = "$${out_name}" + | String contents = read_string("$${out_name}") | } |} | @@ -932,21 +933,21 @@ object SampleWdl { */ case class CallCachingWorkflow(salt: String) extends SampleWdl { override def wdlSource(runtime: String): WdlSource = - """task a { + s"""task a { | File in | String out_name = "out" | String salt | | command { - | # ${salt} + | # $${salt} | echo "Something" - | cat ${in} > ${out_name} + | cat $${in} > $${out_name} | } | RUNTIME | output { | File out = "out" - | File out_interpolation = "${out_name}" - | String contents = read_string("${out_name}") + | File out_interpolation = "$${out_name}" + | String contents = read_string("$${out_name}") | Array[String] stdoutContent = read_lines(stdout()) | } |} @@ -984,13 +985,13 @@ object SampleWdl { """.stripMargin.trim override def wdlSource(runtime: String): WdlSource = - """ + s""" |task a { | Array[String] array | Map[String, String] map | | command { - | echo ${sep = ' ' array} > concat + | echo $${sep = ' ' array} > concat | } | output { | String x = read_string("concat") @@ -1020,10 +1021,10 @@ object SampleWdl { object ArrayOfArrays extends SampleWdl { override def wdlSource(runtime: String = "") = - """task subtask { + s"""task subtask { | Array[File] a | command { - | cat ${sep = " " a} + | cat $${sep = " " a} | } | output { | String concatenated = read_string(stdout()) @@ -1060,17 +1061,17 @@ object SampleWdl { object CallCachingHashingWdl extends SampleWdl { override def wdlSource(runtime: String): WdlSource = - """task t { + s"""task t { | Int a | Float b | String c | File d | | command { - | echo "${a}" > a - | echo "${b}" > b - | echo "${c}" > c - | cat ${d} > d + | echo "$${a}" > a + | echo "$${b}" > b + | echo "$${c}" > c + | cat $${d} > d | } | output { | Int w = read_int("a") + 2 @@ -1098,10 +1099,10 @@ object SampleWdl { object ExpressionsInInputs extends SampleWdl { override def wdlSource(runtime: String = "") = - """task echo { + s"""task echo { | String inString | command { - | echo ${inString} + | echo $${inString} | } | | output { @@ -1128,11 +1129,11 @@ object SampleWdl { object WorkflowFailSlow extends SampleWdl { override def wdlSource(runtime: String = "") = - """ + s""" task shouldCompleteFast { | Int a | command { - | echo "The number was: ${a}" + | echo "The number was: $${a}" | } | output { | Int echo = a @@ -1142,7 +1143,7 @@ task shouldCompleteFast { |task shouldCompleteSlow { | Int a | command { - | echo "The number was: ${a}" + | echo "The number was: $${a}" | # More than 1 so this should finish second | sleep 2 | } @@ -1154,7 +1155,7 @@ task shouldCompleteFast { |task failMeSlowly { | Int a | command { - | echo "The number was: ${a}" + | echo "The number was: $${a}" | # Less than 2 so this should finish first | sleep 1 | ./NOOOOOO @@ -1168,7 +1169,7 @@ task shouldCompleteFast { | Int a | Int b | command { - | echo "You can't fight in here - this is the war room ${a + b}" + | echo "You can't fight in here - this is the war room $${a + b}" | } | output { | Int echo = a diff --git a/core/src/test/scala/cromwell/util/TryWithResourceSpec.scala b/core/src/test/scala/cromwell/util/TryWithResourceSpec.scala index 8aa73d339..77046fc27 100644 --- a/core/src/test/scala/cromwell/util/TryWithResourceSpec.scala +++ b/core/src/test/scala/cromwell/util/TryWithResourceSpec.scala @@ -9,7 +9,7 @@ class TryWithResourceSpec extends FlatSpec with Matchers { behavior of "tryWithResource" it should "catch instantiation errors" in { - val triedMyBest = tryWithResource(() => throw InstantiationException) { _ => 5 } + val triedMyBest = tryWithResource(() => if (1 == 1) throw InstantiationException else null) { _ => 5 } triedMyBest should be(Failure(InstantiationException)) } diff --git a/database/migration/src/main/scala/cromwell/database/migration/WdlTransformation.scala b/database/migration/src/main/scala/cromwell/database/migration/WdlTransformation.scala index 22766d9c7..84f38a7db 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/WdlTransformation.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/WdlTransformation.scala @@ -8,7 +8,6 @@ import org.apache.commons.codec.binary.Base64 import org.apache.commons.io.IOUtils import wdl4s.types.{WdlPrimitiveType, WdlType} -import scala.language.postfixOps import scala.util.Try private [migration] object WdlTransformation { diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/QueryPaginator.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/QueryPaginator.scala index e0d90d3d4..f7929cea4 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/QueryPaginator.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/QueryPaginator.scala @@ -2,7 +2,6 @@ package cromwell.database.migration.metadata.table.symbol import java.sql.{PreparedStatement, ResultSet} -import liquibase.database.jvm.JdbcConnection class QueryPaginator(statement: PreparedStatement, batchSize: Int, @@ -17,5 +16,5 @@ class QueryPaginator(statement: PreparedStatement, statement.executeQuery() } - def hasNext(): Boolean = cursor <= count + def hasNext: Boolean = cursor <= count } diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/SymbolTableMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/SymbolTableMigration.scala index 04a7bbaac..a400866a9 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/SymbolTableMigration.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/SymbolTableMigration.scala @@ -15,7 +15,6 @@ import wdl4s.WdlExpression import wdl4s.types.WdlType import wdl4s.values.WdlValue -import scala.language.postfixOps import scala.util.{Failure, Success, Try} object SymbolTableMigration { diff --git a/database/migration/src/main/scala/cromwell/database/migration/restart/table/JobStoreSimpletonMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/restart/table/JobStoreSimpletonMigration.scala index c48e76943..9287add50 100644 --- a/database/migration/src/main/scala/cromwell/database/migration/restart/table/JobStoreSimpletonMigration.scala +++ b/database/migration/src/main/scala/cromwell/database/migration/restart/table/JobStoreSimpletonMigration.scala @@ -5,8 +5,6 @@ import cromwell.database.migration.WdlTransformation._ import liquibase.database.jvm.JdbcConnection import wdl4s.types.WdlType -import scala.language.postfixOps - class JobStoreSimpletonMigration extends AbstractRestartMigration { override val description = "WORKFLOW_EXECUTION + EXECUTION + SYMBOL + JOB_STORE -> JOB_STORE_RESULT_SIMPLETON" diff --git a/database/sql/src/main/scala/cromwell/database/slick/JobKeyValueSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/JobKeyValueSlickDatabase.scala index c9f40e436..41d9c2237 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/JobKeyValueSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/JobKeyValueSlickDatabase.scala @@ -19,12 +19,12 @@ trait JobKeyValueSlickDatabase extends JobKeyValueSqlDatabase { } else { for { updateCount <- dataAccess. - storeValuesForJobKeyAndStoreKey( + storeValuesForJobKeyAndStoreKey(( jobKeyValueEntry.workflowExecutionUuid, jobKeyValueEntry.callFullyQualifiedName, jobKeyValueEntry.jobIndex, jobKeyValueEntry.jobAttempt, - jobKeyValueEntry.storeKey). + jobKeyValueEntry.storeKey)). update(jobKeyValueEntry.storeValue) _ <- updateCount match { case 0 => dataAccess.jobKeyValueEntryIdsAutoInc += jobKeyValueEntry @@ -39,7 +39,7 @@ trait JobKeyValueSlickDatabase extends JobKeyValueSqlDatabase { jobRetryAttempt: Int, storeKey: String) (implicit ec: ExecutionContext): Future[Option[String]] = { val action = dataAccess. - storeValuesForJobKeyAndStoreKey(workflowExecutionUuid, callFqn, jobScatterIndex, jobRetryAttempt, storeKey). + storeValuesForJobKeyAndStoreKey((workflowExecutionUuid, callFqn, jobScatterIndex, jobRetryAttempt, storeKey)). result.headOption runTransaction(action) } diff --git a/database/sql/src/main/scala/cromwell/database/slick/JobStoreSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/JobStoreSlickDatabase.scala index 8794ca8bd..829ddd86a 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/JobStoreSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/JobStoreSlickDatabase.scala @@ -1,9 +1,12 @@ package cromwell.database.slick +import cats.instances.future._ +import cats.syntax.functor._ import cromwell.database.sql.JobStoreSqlDatabase import cromwell.database.sql.joins.JobStoreJoin import scala.concurrent.{ExecutionContext, Future} +import scala.language.postfixOps trait JobStoreSlickDatabase extends JobStoreSqlDatabase { this: SlickDatabase => @@ -21,7 +24,7 @@ trait JobStoreSlickDatabase extends JobStoreSqlDatabase { override def addJobStores(jobStoreJoins: Seq[JobStoreJoin]) (implicit ec: ExecutionContext): Future[Unit] = { val action = DBIO.sequence(jobStoreJoins map addJobStore) - runTransaction(action) map { _ => () } + runTransaction(action) void } override def queryJobStores(workflowExecutionUuid: String, callFqn: String, jobScatterIndex: Int, @@ -30,7 +33,7 @@ trait JobStoreSlickDatabase extends JobStoreSqlDatabase { val action = for { jobStoreEntryOption <- dataAccess. - jobStoreEntriesForJobKey(workflowExecutionUuid, callFqn, jobScatterIndex, jobScatterAttempt).result.headOption + jobStoreEntriesForJobKey((workflowExecutionUuid, callFqn, jobScatterIndex, jobScatterAttempt)).result.headOption jobStoreSimpletonEntries <- jobStoreEntryOption match { case Some(jobStoreEntry) => dataAccess.jobStoreSimpletonEntriesForJobStoreEntryId(jobStoreEntry.jobStoreEntryId.get).result diff --git a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala index 7a0725ffe..efc27b1f6 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala @@ -34,7 +34,7 @@ trait MetadataSlickDatabase extends MetadataSqlDatabase { metadataKey: String) (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = { val action = - dataAccess.metadataEntriesForWorkflowExecutionUuidAndMetadataKey(workflowExecutionUuid, metadataKey).result + dataAccess.metadataEntriesForWorkflowExecutionUuidAndMetadataKey((workflowExecutionUuid, metadataKey)).result runTransaction(action) } @@ -44,7 +44,7 @@ trait MetadataSlickDatabase extends MetadataSqlDatabase { jobAttempt: Int) (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = { val action = dataAccess. - metadataEntriesForJobKey(workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt).result + metadataEntriesForJobKey((workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt)).result runTransaction(action) } @@ -54,8 +54,8 @@ trait MetadataSlickDatabase extends MetadataSqlDatabase { jobIndex: Option[Int], jobAttempt: Int) (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = { - val action = dataAccess.metadataEntriesForJobKeyAndMetadataKey( - workflowUuid, metadataKey, callFullyQualifiedName, jobIndex, jobAttempt).result + val action = dataAccess.metadataEntriesForJobKeyAndMetadataKey(( + workflowUuid, metadataKey, callFullyQualifiedName, jobIndex, jobAttempt)).result runTransaction(action) } @@ -121,8 +121,8 @@ trait MetadataSlickDatabase extends MetadataSqlDatabase { previousMetadataEntryIdOption <- getSummaryStatusEntryMaximumId( "WORKFLOW_METADATA_SUMMARY_ENTRY", "METADATA_ENTRY") previousMetadataEntryId = previousMetadataEntryIdOption.getOrElse(0L) - metadataEntries <- dataAccess.metadataEntriesForIdGreaterThanOrEqual( - previousMetadataEntryId + 1L, metadataKey1, metadataKey2, metadataKey3, metadataKey4).result + metadataEntries <- dataAccess.metadataEntriesForIdGreaterThanOrEqual(( + previousMetadataEntryId + 1L, metadataKey1, metadataKey2, metadataKey3, metadataKey4)).result metadataByWorkflowUuid = metadataEntries.groupBy(_.workflowExecutionUuid) _ <- DBIO.sequence(metadataByWorkflowUuid map updateWorkflowMetadataSummaryEntry(buildUpdatedSummary)) maximumMetadataEntryId = previousOrMaximum(previousMetadataEntryId, metadataEntries.map(_.metadataEntryId.get)) diff --git a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala index 752248cb1..fbfbece8a 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala @@ -104,7 +104,7 @@ class SlickDatabase(override val originalDatabaseConfig: Config) extends SqlData protected[this] def assertUpdateCount(description: String, updates: Int, expected: Int): DBIO[Unit] = { if (updates == expected) { - DBIO.successful(Unit) + DBIO.successful(()) } else { DBIO.failed(new RuntimeException(s"$description expected update count $expected, got $updates")) } diff --git a/database/sql/src/main/scala/cromwell/database/slick/SummaryStatusSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SummaryStatusSlickDatabase.scala index 1bd388b66..6903a29e6 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/SummaryStatusSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/SummaryStatusSlickDatabase.scala @@ -12,7 +12,7 @@ trait SummaryStatusSlickDatabase { private[slick] def getSummaryStatusEntryMaximumId(summaryTableName: String, summarizedTableName: String) (implicit ec: ExecutionContext): DBIO[Option[Long]] = { dataAccess. - maximumIdForSummaryTableNameSummarizedTableName(summaryTableName, summarizedTableName). + maximumIdForSummaryTableNameSummarizedTableName((summaryTableName, summarizedTableName)). result.headOption } @@ -28,7 +28,7 @@ trait SummaryStatusSlickDatabase { } else { for { updateCount <- dataAccess. - maximumIdForSummaryTableNameSummarizedTableName(summaryTableName, summarizedTableName). + maximumIdForSummaryTableNameSummarizedTableName((summaryTableName, summarizedTableName)). update(maximumId) _ <- updateCount match { case 0 => diff --git a/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala index e514822c2..fa1a9a5fc 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/WorkflowStoreSlickDatabase.scala @@ -1,9 +1,12 @@ package cromwell.database.slick +import cats.instances.future._ +import cats.syntax.functor._ import cromwell.database.sql.WorkflowStoreSqlDatabase import cromwell.database.sql.tables.WorkflowStoreEntry import scala.concurrent.{ExecutionContext, Future} +import scala.language.postfixOps trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase { this: SlickDatabase => @@ -13,19 +16,19 @@ trait WorkflowStoreSlickDatabase extends WorkflowStoreSqlDatabase { override def updateWorkflowState(queryWorkflowState: String, updateWorkflowState: String) (implicit ec: ExecutionContext): Future[Unit] = { val action = dataAccess.workflowStateForWorkflowState(queryWorkflowState).update(updateWorkflowState) - runTransaction(action) map { _ => () } + runTransaction(action) void } override def addWorkflowStoreEntries(workflowStoreEntries: Iterable[WorkflowStoreEntry]) (implicit ec: ExecutionContext): Future[Unit] = { val action = dataAccess.workflowStoreEntryIdsAutoInc ++= workflowStoreEntries - runTransaction(action) map { _ => () } + runTransaction(action) void } override def queryWorkflowStoreEntries(limit: Int, queryWorkflowState: String, updateWorkflowState: String) (implicit ec: ExecutionContext): Future[Seq[WorkflowStoreEntry]] = { val action = for { - workflowStoreEntries <- dataAccess.workflowStoreEntriesForWorkflowState(queryWorkflowState, limit).result + workflowStoreEntries <- dataAccess.workflowStoreEntriesForWorkflowState((queryWorkflowState, limit.toLong)).result _ <- DBIO.sequence(workflowStoreEntries map updateWorkflowStateForWorkflowExecutionUuid(updateWorkflowState)) } yield workflowStoreEntries runTransaction(action) diff --git a/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala index c90e76e3b..c6c29479b 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala @@ -30,7 +30,7 @@ object SqlDatabase { */ def withUniqueSchema(config: Config, urlKey: String): Config = { val urlValue = config.getString(urlKey) - if (urlValue.contains("${uniqueSchema}")) { + if (urlValue.contains(s"$${uniqueSchema}")) { // Config wasn't updating with a simple withValue/withFallback. // So instead, do a bit of extra work to insert the generated schema name in the url. val schema = UUID.randomUUID().toString diff --git a/engine/src/main/scala/cromwell/engine/backend/CromwellBackends.scala b/engine/src/main/scala/cromwell/engine/backend/CromwellBackends.scala index b9d86ee65..9b90e6277 100644 --- a/engine/src/main/scala/cromwell/engine/backend/CromwellBackends.scala +++ b/engine/src/main/scala/cromwell/engine/backend/CromwellBackends.scala @@ -2,7 +2,6 @@ package cromwell.engine.backend import cromwell.backend.BackendLifecycleActorFactory -import scala.language.postfixOps import scala.util.{Failure, Success, Try} /** diff --git a/engine/src/main/scala/cromwell/engine/engine.scala b/engine/src/main/scala/cromwell/engine/engine.scala new file mode 100644 index 000000000..7a65770b1 --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/engine.scala @@ -0,0 +1,25 @@ +package cromwell.engine + +import java.time.OffsetDateTime + +import wdl4s._ + +import scala.util.{Failure, Success, Try} + +final case class AbortFunction(function: () => Unit) +final case class AbortRegistrationFunction(register: AbortFunction => Unit) + +final case class FailureEventEntry(failure: String, timestamp: OffsetDateTime) +final case class CallAttempt(fqn: FullyQualifiedName, attempt: Int) + +object WorkflowFailureMode { + def tryParse(mode: String): Try[WorkflowFailureMode] = { + val modes = Seq(ContinueWhilePossible, NoNewCalls) + modes find { _.toString.equalsIgnoreCase(mode) } map { Success(_) } getOrElse Failure(new Exception(s"Invalid workflow failure mode: $mode")) + } +} +sealed trait WorkflowFailureMode { + def allowNewCallsAfterFailure: Boolean +} +case object ContinueWhilePossible extends WorkflowFailureMode { override val allowNewCallsAfterFailure = true } +case object NoNewCalls extends WorkflowFailureMode { override val allowNewCallsAfterFailure = false } \ No newline at end of file diff --git a/engine/src/main/scala/cromwell/engine/package.scala b/engine/src/main/scala/cromwell/engine/package.scala index 4ea1c964b..6b2df1cc7 100644 --- a/engine/src/main/scala/cromwell/engine/package.scala +++ b/engine/src/main/scala/cromwell/engine/package.scala @@ -1,22 +1,11 @@ package cromwell -import java.time.OffsetDateTime - import cromwell.core.JobOutput import wdl4s._ import wdl4s.values.WdlValue -import scala.language.implicitConversions -import scala.util.{Failure, Success, Try} - package object engine { - final case class AbortFunction(function: () => Unit) - final case class AbortRegistrationFunction(register: AbortFunction => Unit) - - final case class FailureEventEntry(failure: String, timestamp: OffsetDateTime) - final case class CallAttempt(fqn: FullyQualifiedName, attempt: Int) - implicit class EnhancedFullyQualifiedName(val fqn: FullyQualifiedName) extends AnyVal { def scopeAndVariableName: (String, String) = { val array = fqn.split("\\.(?=[^\\.]+$)") @@ -30,15 +19,4 @@ package object engine { } } - object WorkflowFailureMode { - def tryParse(mode: String): Try[WorkflowFailureMode] = { - val modes = Seq(ContinueWhilePossible, NoNewCalls) - modes find { _.toString.equalsIgnoreCase(mode) } map { Success(_) } getOrElse Failure(new Exception(s"Invalid workflow failure mode: $mode")) - } - } - sealed trait WorkflowFailureMode { - def allowNewCallsAfterFailure: Boolean - } - case object ContinueWhilePossible extends WorkflowFailureMode { override val allowNewCallsAfterFailure = true } - case object NoNewCalls extends WorkflowFailureMode { override val allowNewCallsAfterFailure = false } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala index d7605a2b9..8abb0874c 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala @@ -6,6 +6,8 @@ import java.util.UUID import akka.actor.FSM.{CurrentState, Transition} import akka.actor._ import better.files._ +import cats.instances.try_._ +import cats.syntax.functor._ import cromwell.core.retry.SimpleExponentialBackoff import cromwell.core.{ExecutionStore => _, _} import cromwell.engine.workflow.SingleWorkflowRunnerActor._ @@ -80,7 +82,9 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: } private def schedulePollRequest(): Unit = { + // -Ywarn-value-discard should stash Cancellable to cancel context.system.scheduler.scheduleOnce(backoff.backoffMillis.millis, self, IssuePollRequest) + () } private def requestStatus(): Unit = { @@ -148,7 +152,7 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: data.terminalState foreach { state => log.info(s"$Tag workflow finished with status '$state'.") } data.failures foreach { e => log.error(e, e.getMessage) } - val message = data.terminalState collect { case WorkflowSucceeded => () } getOrElse Status.Failure(data.failures.head) + val message: Any = data.terminalState collect { case WorkflowSucceeded => () } getOrElse Status.Failure(data.failures.head) data.replyTo foreach { _ ! message } stay() } @@ -192,6 +196,6 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFiles, metadataOutputPath: log.info(s"$Tag writing metadata to $path") path.createIfNotExists(asDirectory = false, createParents = true).write(metadata.prettyPrint) } - } + } void } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala index f137883fd..684c81cce 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala @@ -22,7 +22,6 @@ import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} import cromwell.webservice.EngineStatsActor -import scala.language.postfixOps import scala.util.Random object WorkflowActor { diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala index 9c797bb76..1fa7fdb0e 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala @@ -18,7 +18,6 @@ import lenthall.config.ScalaConfig.EnhancedScalaConfig import scala.concurrent.duration._ import scala.concurrent.{Await, Promise} -import scala.language.postfixOps object WorkflowManagerActor { val DefaultMaxWorkflowsToRun = 5000 @@ -106,13 +105,13 @@ class WorkflowManagerActor(config: Config, private var abortingWorkflowToReplyTo = Map.empty[WorkflowId, ActorRef] - override def preStart() { + override def preStart(): Unit = { addShutdownHook() // Starts the workflow polling cycle self ! RetrieveNewWorkflows } - private def addShutdownHook(): Unit = { + private def addShutdownHook() = { // Only abort jobs on SIGINT if the config explicitly sets backend.abortJobsOnTerminate = true. val abortJobsOnTerminate = config.getConfig("system").getBooleanOr("abort-jobs-on-terminate", default = false) @@ -121,7 +120,7 @@ class WorkflowManagerActor(config: Config, sys.addShutdownHook { logger.info(s"$tag: Received shutdown signal. Aborting all running workflows...") self ! AbortAllWorkflowsCommand - Await.ready(donePromise.future, Duration.Inf) + Await.result(donePromise.future, Duration.Inf) } } } @@ -242,6 +241,7 @@ class WorkflowManagerActor(config: Config, case _ -> Done => logger.info(s"$tag All workflows finished. Stopping self.") donePromise.trySuccess(()) + () case fromState -> toState => logger.debug(s"$tag transitioning from $fromState to $toState") } @@ -271,7 +271,7 @@ class WorkflowManagerActor(config: Config, WorkflowIdToActorRef(workflowId, wfActor) } - private def scheduleNextNewWorkflowPoll(): Unit = { + private def scheduleNextNewWorkflowPoll() = { context.system.scheduler.scheduleOnce(newWorkflowPollRate, self, RetrieveNewWorkflows)(context.dispatcher) } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala index ff3df1b20..aa25fdfb8 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala @@ -30,11 +30,13 @@ class CopyWorkflowLogsActor(serviceRegistryActor: ActorRef) with ActorLogging with PathFactory { - def copyAndClean(src: Path, dest: Path): Unit = { + def copyAndClean(src: Path, dest: Path) = { File(dest).parent.createDirectories() File(src).copyTo(dest, overwrite = true) - if (WorkflowLogger.isTemporary) File(src).delete() + if (WorkflowLogger.isTemporary) { + File(src).delete() + } } override def receive = { diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala index d79d97998..67e93faf2 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala @@ -3,7 +3,6 @@ package cromwell.engine.workflow.lifecycle import java.nio.file.FileSystem import akka.actor.{ActorRef, FSM, LoggingFSM, Props} -import cats.data.NonEmptyList import cats.data.Validated._ import cats.instances.list._ import cats.syntax.cartesian._ diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala index 7c8d3748c..9614696e2 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala @@ -10,7 +10,6 @@ import cromwell.engine.backend.CromwellBackends import cromwell.engine.workflow.lifecycle.WorkflowFinalizationActor._ import cromwell.engine.workflow.lifecycle.WorkflowLifecycleActor._ -import scala.language.postfixOps import scala.util.{Failure, Success, Try} object WorkflowFinalizationActor { diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala index fc18939f6..2fd5d75aa 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala @@ -11,7 +11,6 @@ import cromwell.engine.backend.CromwellBackends import cromwell.engine.workflow.lifecycle.WorkflowInitializationActor._ import cromwell.engine.workflow.lifecycle.WorkflowLifecycleActor._ -import scala.language.postfixOps import scala.util.{Failure, Success, Try} object WorkflowInitializationActor { diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala index 4ff275a35..64f3eec7a 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala @@ -298,15 +298,20 @@ class EngineJobExecutionActor(replyTo: ActorRef, callCacheReadActor, factory.runtimeAttributeDefinitions(initializationData), backendName, activity) context.actorOf(props, s"ejha_for_$jobDescriptor") + () } - def makeFetchCachedResultsActor(cacheHit: CacheHit, taskOutputs: Seq[TaskOutput]): Unit = context.actorOf(FetchCachedResultsActor.props(cacheHit, self, new CallCache(SingletonServicesStore.databaseInterface))) - def fetchCachedResults(data: ResponsePendingData, taskOutputs: Seq[TaskOutput], cacheHit: CacheHit) = { + def makeFetchCachedResultsActor(cacheHit: CacheHit, taskOutputs: Seq[TaskOutput]): Unit = { + context.actorOf(FetchCachedResultsActor.props(cacheHit, self, new CallCache(SingletonServicesStore.databaseInterface))) + () + } + + private def fetchCachedResults(data: ResponsePendingData, taskOutputs: Seq[TaskOutput], cacheHit: CacheHit) = { makeFetchCachedResultsActor(cacheHit, taskOutputs) goto(FetchingCachedOutputsFromDatabase) } - def makeBackendCopyCacheHit(cacheHit: CacheHit, wdlValueSimpletons: Seq[WdlValueSimpleton], jobDetritusFiles: Map[String,String], returnCode: Option[Int], data: ResponsePendingData) = { + private def makeBackendCopyCacheHit(cacheHit: CacheHit, wdlValueSimpletons: Seq[WdlValueSimpleton], jobDetritusFiles: Map[String,String], returnCode: Option[Int], data: ResponsePendingData) = { factory.cacheHitCopyingActorProps match { case Some(propsMaker) => val backendCacheHitCopyingActorProps = propsMaker(data.jobDescriptor, initializationData, serviceRegistryActor) @@ -323,7 +328,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, } } - def runJob(data: ResponsePendingData) = { + private def runJob(data: ResponsePendingData) = { val backendJobExecutionActor = context.actorOf(data.bjeaProps, buildJobExecutionActorName(data.jobDescriptor)) val message = if (restarting) RecoverJobCommand else ExecuteJobCommand backendJobExecutionActor ! message @@ -342,6 +347,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, protected def createSaveCacheResultsActor(hashes: CallCacheHashes, success: SucceededResponse): Unit = { val callCache = new CallCache(SingletonServicesStore.databaseInterface) context.actorOf(CallCacheWriteActor.props(callCache, workflowId, hashes, success), s"CallCacheWriteActor-$tag") + () } private def saveCacheResults(hashes: CallCacheHashes, data: SucceededResponseData) = { diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index 8e6c93e6c..a3ef978ec 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -145,7 +145,7 @@ object WorkflowExecutionActor { callCacheReadActor, jobTokenDispenserActor, initializationData, restarting)).withDispatcher(EngineDispatcher) } - private implicit class EnhancedExecutionStore(val executionStore: ExecutionStore) extends AnyVal { + implicit class EnhancedExecutionStore(val executionStore: ExecutionStore) extends AnyVal { // Convert the store to a `List` before `collect`ing to sidestep expensive and pointless hashing of `Scope`s when // assembling the result. def runnableScopes = executionStore.store.toList collect { case entry if isRunnable(entry) => entry._1 } @@ -208,7 +208,7 @@ object WorkflowExecutionActor { } } - private implicit class EnhancedOutputStore(val outputStore: OutputStore) extends AnyVal { + implicit class EnhancedOutputStore(val outputStore: OutputStore) extends AnyVal { /** * Try to generate output for a collector call, by collecting outputs for all of its shards. * It's fail-fast on shard output retrieval diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala index 985d9c812..599c8f1b4 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala @@ -8,7 +8,6 @@ import cromwell.engine.{EngineWorkflowDescriptor, WdlFunctions} import cromwell.util.JsonFormatting.WdlValueJsonFormatter import wdl4s.Scope -import scala.language.postfixOps object WorkflowExecutionDiff { def empty = WorkflowExecutionDiff(Map.empty) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala index d4c144d9a..16ecc89f0 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala @@ -12,7 +12,6 @@ import cromwell.database.sql.tables.{CallCachingDetritusEntry, CallCachingEntry, import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes import scala.concurrent.{ExecutionContext, Future} -import scala.language.postfixOps final case class MetaInfoId(id: Int) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala index 2080ae8e8..229fc47a2 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala @@ -15,10 +15,10 @@ import scala.concurrent.ExecutionContext */ class CallCacheReadActor(cache: CallCache) extends Actor with ActorLogging { - implicit val ec: ExecutionContext = context.dispatcher + private implicit val ec: ExecutionContext = context.dispatcher - var requestQueue: List[RequestTuple] = List.empty - var currentRequester: Option[ActorRef] = None + private var requestQueue: List[RequestTuple] = List.empty + private var currentRequester: Option[ActorRef] = None override def receive: Receive = { case CacheLookupRequest(callCacheHashes) => @@ -30,7 +30,7 @@ class CallCacheReadActor(cache: CallCache) extends Actor with ActorLogging { log.error("Unexpected message type to CallCacheReadActor: " + other.getClass.getSimpleName) } - private def runRequest(callCacheHashes: CallCacheHashes) = { + private def runRequest(callCacheHashes: CallCacheHashes): Unit = { val response = cache.fetchMetaInfoIdsMatchingHashes(callCacheHashes) map { CacheResultMatchesForHashes(callCacheHashes.hashes, _) } recover { @@ -38,6 +38,7 @@ class CallCacheReadActor(cache: CallCache) extends Actor with ActorLogging { } response.pipeTo(self) + () } private def cycleRequestQueue() = requestQueue match { @@ -49,7 +50,7 @@ class CallCacheReadActor(cache: CallCache) extends Actor with ActorLogging { currentRequester = None } - private def receiveNewRequest(callCacheHashes: CallCacheHashes) = currentRequester match { + private def receiveNewRequest(callCacheHashes: CallCacheHashes): Unit = currentRequester match { case Some(x) => requestQueue :+= RequestTuple(sender, callCacheHashes) case None => currentRequester = Option(sender) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/lifecycle.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/lifecycle.scala new file mode 100644 index 000000000..bb0d8fa80 --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/lifecycle.scala @@ -0,0 +1,7 @@ +package cromwell.engine.workflow.lifecycle + +case object EngineLifecycleActorAbortCommand + +trait EngineLifecycleStateCompleteResponse + +trait EngineLifecycleActorAbortedResponse extends EngineLifecycleStateCompleteResponse diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/package.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/package.scala deleted file mode 100644 index 1f240dc7b..000000000 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/package.scala +++ /dev/null @@ -1,8 +0,0 @@ -package cromwell.engine.workflow - -package object lifecycle { - case object EngineLifecycleActorAbortCommand - - trait EngineLifecycleStateCompleteResponse - trait EngineLifecycleActorAbortedResponse extends EngineLifecycleStateCompleteResponse -} diff --git a/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala b/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala index 4d5e460bb..b2afd6b5e 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala @@ -7,7 +7,6 @@ import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor._ import cromwell.engine.workflow.tokens.TokenPool.TokenPoolPop import scala.collection.immutable.Queue -import scala.language.postfixOps class JobExecutionTokenDispenserActor extends Actor with ActorLogging { @@ -61,7 +60,7 @@ class JobExecutionTokenDispenserActor extends Actor with ActorLogging { actor ! JobExecutionTokenDispensed(token) } - private def unassign(actor: ActorRef, token: JobExecutionToken) = { + private def unassign(actor: ActorRef, token: JobExecutionToken): Unit = { if (tokenAssignments.contains(actor) && tokenAssignments(actor) == token) { tokenAssignments -= actor @@ -76,12 +75,13 @@ class JobExecutionTokenDispenserActor extends Actor with ActorLogging { tokenPools += token.jobExecutionTokenType -> pool context.unwatch(actor) + () } else { log.error("Job execution token returned from incorrect actor: {}", token) } } - private def onTerminate(terminee: ActorRef) = { + private def onTerminate(terminee: ActorRef): Unit = { tokenAssignments.get(terminee) match { case Some(token) => log.error("Actor {} stopped without returning its Job Execution Token. Reclaiming it!", terminee) @@ -94,6 +94,7 @@ class JobExecutionTokenDispenserActor extends Actor with ActorLogging { } } context.unwatch(terminee) + () } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala index 6355b64c5..24cb3a6a7 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala @@ -14,7 +14,6 @@ import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} import org.apache.commons.lang3.exception.ExceptionUtils import scala.concurrent.{ExecutionContext, Future} -import scala.language.postfixOps import scala.util.{Failure, Success} case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor: ActorRef) diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/package.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/package.scala deleted file mode 100644 index e8f90bd74..000000000 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/package.scala +++ /dev/null @@ -1,18 +0,0 @@ -package cromwell.engine.workflow - -import cromwell.core.{WorkflowId, WorkflowSourceFiles} -import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState - -package object workflowstore { - - sealed trait WorkflowStoreState {def isStartable: Boolean} - - object WorkflowStoreState { - case object Running extends WorkflowStoreState { override def isStartable = false } - sealed trait StartableState extends WorkflowStoreState { override def isStartable = true } - case object Submitted extends StartableState - case object Restartable extends StartableState - } - - final case class WorkflowToStart(id: WorkflowId, sources: WorkflowSourceFiles, state: StartableState) -} diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/workflowstore_.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/workflowstore_.scala new file mode 100644 index 000000000..0d9481c47 --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/workflowstore_.scala @@ -0,0 +1,15 @@ +package cromwell.engine.workflow.workflowstore + +import cromwell.core.{WorkflowId, WorkflowSourceFiles} +import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState + +sealed trait WorkflowStoreState {def isStartable: Boolean} + +object WorkflowStoreState { + case object Running extends WorkflowStoreState { override def isStartable = false } + sealed trait StartableState extends WorkflowStoreState { override def isStartable = true } + case object Submitted extends StartableState + case object Restartable extends StartableState +} + +final case class WorkflowToStart(id: WorkflowId, sources: WorkflowSourceFiles, state: StartableState) diff --git a/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala b/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala index cf7b37acb..9bb680b0c 100644 --- a/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala +++ b/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala @@ -83,7 +83,7 @@ object JobStoreWriterData { case class JobStoreWriterData(currentOperation: List[(ActorRef, JobStoreWriterCommand)], nextOperation: List[(ActorRef, JobStoreWriterCommand)]) { def isEmpty = nextOperation.isEmpty && currentOperation.isEmpty - def withNewOperation(sender: ActorRef, command: JobStoreWriterCommand) = this.copy(nextOperation = this.nextOperation :+ (sender, command)) + def withNewOperation(sender: ActorRef, command: JobStoreWriterCommand) = this.copy(nextOperation = this.nextOperation :+ ((sender, command))) def rolledOver = JobStoreWriterData(this.nextOperation, List.empty) } diff --git a/engine/src/main/scala/cromwell/jobstore/jobstore_.scala b/engine/src/main/scala/cromwell/jobstore/jobstore_.scala new file mode 100644 index 000000000..7fbdd0107 --- /dev/null +++ b/engine/src/main/scala/cromwell/jobstore/jobstore_.scala @@ -0,0 +1,10 @@ +package cromwell.jobstore + +import cromwell.core.{WorkflowId, _} + +case class JobStoreKey(workflowId: WorkflowId, callFqn: String, index: Option[Int], attempt: Int) + +sealed trait JobResult +case class JobResultSuccess(returnCode: Option[Int], jobOutputs: JobOutputs) extends JobResult +case class JobResultFailure(returnCode: Option[Int], reason: Throwable, retryable: Boolean) extends JobResult + diff --git a/engine/src/main/scala/cromwell/jobstore/package.scala b/engine/src/main/scala/cromwell/jobstore/package.scala index 2e82109e0..f96ca4041 100644 --- a/engine/src/main/scala/cromwell/jobstore/package.scala +++ b/engine/src/main/scala/cromwell/jobstore/package.scala @@ -1,14 +1,8 @@ package cromwell -import cromwell.core.{JobKey, JobOutputs, WorkflowId} +import cromwell.core.{JobKey, WorkflowId} package object jobstore { - case class JobStoreKey(workflowId: WorkflowId, callFqn: String, index: Option[Int], attempt: Int) - - sealed trait JobResult - case class JobResultSuccess(returnCode: Option[Int], jobOutputs: JobOutputs) extends JobResult - case class JobResultFailure(returnCode: Option[Int], reason: Throwable, retryable: Boolean) extends JobResult - implicit class EnhancedJobKey(val jobKey: JobKey) extends AnyVal { def toJobStoreKey(workflowId: WorkflowId): JobStoreKey = JobStoreKey(workflowId, jobKey.scope.fullyQualifiedName, jobKey.index, jobKey.attempt) } diff --git a/engine/src/main/scala/cromwell/server/CromwellServer.scala b/engine/src/main/scala/cromwell/server/CromwellServer.scala index c3772f3f7..0dd8ac59f 100644 --- a/engine/src/main/scala/cromwell/server/CromwellServer.scala +++ b/engine/src/main/scala/cromwell/server/CromwellServer.scala @@ -4,20 +4,17 @@ import java.util.concurrent.TimeoutException import akka.actor.Props import akka.util.Timeout -import com.typesafe.config.{Config, ConfigFactory} -import cromwell.services.ServiceRegistryActor +import com.typesafe.config.Config +import cromwell.webservice.WorkflowJsonSupport._ import cromwell.webservice.{APIResponse, CromwellApiService, SwaggerService} +import lenthall.config.ScalaConfig._ import lenthall.spray.SprayCanHttpService._ -import spray.http.HttpHeaders.`Content-Type` -import spray.http.MediaTypes._ -import spray.http.{ContentType, MediaTypes, _} import lenthall.spray.WrappedRoute._ -import lenthall.config.ScalaConfig._ -import cromwell.webservice.WorkflowJsonSupport._ +import spray.http.{ContentType, MediaTypes, _} import spray.json._ -import scala.concurrent.{Await, Future} import scala.concurrent.duration._ +import scala.concurrent.{Await, Future} import scala.util.{Failure, Success} // Note that as per the language specification, this is instantiated lazily and only used when necessary (i.e. server mode) diff --git a/engine/src/main/scala/cromwell/server/CromwellSystem.scala b/engine/src/main/scala/cromwell/server/CromwellSystem.scala index d125b8c9c..648850f71 100644 --- a/engine/src/main/scala/cromwell/server/CromwellSystem.scala +++ b/engine/src/main/scala/cromwell/server/CromwellSystem.scala @@ -1,10 +1,12 @@ package cromwell.server -import akka.actor.ActorSystem +import akka.actor.{ActorSystem, Terminated} import com.typesafe.config.ConfigFactory import cromwell.engine.backend.{BackendConfiguration, CromwellBackends} import org.slf4j.LoggerFactory +import scala.concurrent.Future + trait CromwellSystem { protected def systemName = "cromwell-system" protected def newActorSystem(): ActorSystem = ActorSystem(systemName) @@ -12,7 +14,9 @@ trait CromwellSystem { val logger = LoggerFactory.getLogger(getClass.getName) implicit final lazy val actorSystem = newActorSystem() - def shutdownActorSystem(): Unit = actorSystem.terminate() + def shutdownActorSystem(): Future[Terminated] = { + actorSystem.terminate() + } CromwellBackends.initBackends(BackendConfiguration.AllBackendEntries) } diff --git a/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala b/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala index b39c2fc51..dee173ca3 100644 --- a/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala +++ b/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala @@ -4,7 +4,6 @@ import spray.json._ import wdl4s.values.WdlValue import wdl4s.{ExceptionWithErrors, FullyQualifiedName} -import scala.language.postfixOps case class WorkflowStatusResponse(id: String, status: String) diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala index 0aae04dac..a2441abb4 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala @@ -13,7 +13,6 @@ import cromwell.webservice.metadata.WorkflowQueryPagination import spray.http.{StatusCodes, Uri} import spray.httpx.SprayJsonSupport._ -import scala.language.postfixOps object CromwellApiHandler { def props(requestHandlerActor: ActorRef): Props = { @@ -43,34 +42,34 @@ class CromwellApiHandler(requestHandlerActor: ActorRef) extends Actor with Workf val conf = ConfigFactory.load() def callNotFound(callFqn: String, id: WorkflowId) = { - RequestComplete(StatusCodes.NotFound, APIResponse.error( - new RuntimeException(s"Call $callFqn not found for workflow '$id'."))) + RequestComplete((StatusCodes.NotFound, APIResponse.error( + new RuntimeException(s"Call $callFqn not found for workflow '$id'.")))) } private def error(t: Throwable)(f: Throwable => RequestComplete[_]): Unit = context.parent ! f(t) override def receive = { case ApiHandlerEngineStats => requestHandlerActor ! WorkflowManagerActor.EngineStatsCommand - case stats: EngineStatsActor.EngineStats => context.parent ! RequestComplete(StatusCodes.OK, stats) + case stats: EngineStatsActor.EngineStats => context.parent ! RequestComplete((StatusCodes.OK, stats)) case ApiHandlerWorkflowAbort(id, manager) => requestHandlerActor ! WorkflowStoreActor.AbortWorkflow(id, manager) case WorkflowStoreActor.WorkflowAborted(id) => - context.parent ! RequestComplete(StatusCodes.OK, WorkflowAbortResponse(id.toString, WorkflowAborted.toString)) + context.parent ! RequestComplete((StatusCodes.OK, WorkflowAbortResponse(id.toString, WorkflowAborted.toString))) case WorkflowStoreActor.WorkflowAbortFailed(_, e) => error(e) { - case _: IllegalStateException => RequestComplete(StatusCodes.Forbidden, APIResponse.error(e)) - case _: WorkflowNotFoundException => RequestComplete(StatusCodes.NotFound, APIResponse.error(e)) - case _ => RequestComplete(StatusCodes.InternalServerError, APIResponse.error(e)) + case _: IllegalStateException => RequestComplete((StatusCodes.Forbidden, APIResponse.error(e))) + case _: WorkflowNotFoundException => RequestComplete((StatusCodes.NotFound, APIResponse.error(e))) + case _ => RequestComplete((StatusCodes.InternalServerError, APIResponse.error(e))) } case ApiHandlerWorkflowSubmit(source) => requestHandlerActor ! WorkflowStoreActor.SubmitWorkflow(source) case WorkflowStoreActor.WorkflowSubmittedToStore(id) => - context.parent ! RequestComplete(StatusCodes.Created, WorkflowSubmitResponse(id.toString, WorkflowSubmitted.toString)) + context.parent ! RequestComplete((StatusCodes.Created, WorkflowSubmitResponse(id.toString, WorkflowSubmitted.toString))) case ApiHandlerWorkflowSubmitBatch(sources) => requestHandlerActor ! WorkflowStoreActor.BatchSubmitWorkflows(sources) case WorkflowStoreActor.WorkflowsBatchSubmittedToStore(ids) => val responses = ids map { id => WorkflowSubmitResponse(id.toString, WorkflowSubmitted.toString) } - context.parent ! RequestComplete(StatusCodes.OK, responses.toList) + context.parent ! RequestComplete((StatusCodes.OK, responses.toList)) } } diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala index 6f66407fd..54ddf2786 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala @@ -1,7 +1,7 @@ package cromwell.webservice import akka.actor._ -import java.lang.Throwable._ +import java.lang.Throwable import cats.data.NonEmptyList import cromwell.core.{WorkflowId, WorkflowSourceFiles} import cromwell.engine.backend.BackendConfiguration @@ -51,7 +51,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator { } private def failBadRequest(exception: Exception, statusCode: StatusCode = StatusCodes.BadRequest) = respondWithMediaType(`application/json`) { - complete(statusCode, APIResponse.fail(exception).toJson.prettyPrint) + complete((statusCode, APIResponse.fail(exception).toJson.prettyPrint)) } val workflowRoutes = queryRoute ~ queryPostRoute ~ workflowOutputsRoute ~ submitRoute ~ submitBatchRoute ~ @@ -155,6 +155,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator { case JsArray(_) => failBadRequest(new RuntimeException("Nothing was submitted")) case _ => reject } + () } } } diff --git a/engine/src/main/scala/cromwell/webservice/PerRequest.scala b/engine/src/main/scala/cromwell/webservice/PerRequest.scala index 0ae86447a..f4fb9e876 100644 --- a/engine/src/main/scala/cromwell/webservice/PerRequest.scala +++ b/engine/src/main/scala/cromwell/webservice/PerRequest.scala @@ -63,7 +63,7 @@ trait PerRequest extends Actor { OneForOneStrategy() { case e => system.log.error(e, "error processing request: " + r.request.uri) - r.complete(InternalServerError, e.getMessage) + r.complete((InternalServerError, e.getMessage)) Stop } } @@ -85,13 +85,13 @@ object PerRequest { case class RequestCompleteWithHeaders[T](response: T, headers: HttpHeader*)(implicit val marshaller: ToResponseMarshaller[T]) extends PerRequestMessage /** allows for pattern matching with extraction of marshaller */ - private object RequestComplete_ { - def unapply[T](requestComplete: RequestComplete[T]) = Some((requestComplete.response, requestComplete.marshaller)) + object RequestComplete_ { + def unapply[T](requestComplete: RequestComplete[T]) = Option((requestComplete.response, requestComplete.marshaller)) } /** allows for pattern matching with extraction of marshaller */ - private object RequestCompleteWithHeaders_ { - def unapply[T](requestComplete: RequestCompleteWithHeaders[T]) = Some((requestComplete.response, requestComplete.headers, requestComplete.marshaller)) + object RequestCompleteWithHeaders_ { + def unapply[T](requestComplete: RequestCompleteWithHeaders[T]) = Option((requestComplete.response, requestComplete.headers, requestComplete.marshaller)) } case class WithProps(r: RequestContext, props: Props, message: AnyRef, timeout: Duration, name: String) extends PerRequest { @@ -108,8 +108,9 @@ trait PerRequestCreator { def perRequest(r: RequestContext, props: Props, message: AnyRef, timeout: Duration = 1 minutes, - name: String = PerRequestCreator.endpointActorName) = { + name: String = PerRequestCreator.endpointActorName): Unit = { actorRefFactory.actorOf(Props(WithProps(r, props, message, timeout, name)).withDispatcher(ApiDispatcher), name) + () } } diff --git a/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala b/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala index c1969342c..d9ed77438 100644 --- a/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala +++ b/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala @@ -7,29 +7,29 @@ import cats.instances.map._ import spray.json._ -private object IndexedJsonValue { +object IndexedJsonValue { private implicit val dateTimeOrdering: Ordering[OffsetDateTime] = scala.Ordering.fromLessThan(_ isBefore _) private val timestampedJsValueOrdering: Ordering[TimestampedJsValue] = scala.Ordering.by(_.timestamp) - implicit val TimestampedJsonMonoid: Monoid[TimestampedJsValue] = new Monoid[TimestampedJsValue] { - def combine(f1: TimestampedJsValue, f2: TimestampedJsValue): TimestampedJsValue = { - (f1, f2) match { - case (o1: TimestampedJsObject, o2: TimestampedJsObject) => - val sg = implicitly[Semigroup[Map[String, TimestampedJsValue]]] - TimestampedJsObject(sg.combine(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) - case (o1: TimestampedJsList, o2: TimestampedJsList) => - val sg = implicitly[Semigroup[Map[Int, TimestampedJsValue]]] - TimestampedJsList(sg.combine(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) - case (o1, o2) => timestampedJsValueOrdering.max(o1, o2) - } - } - - override def empty: TimestampedJsValue = TimestampedJsObject(Map.empty, OffsetDateTime.now) - } + implicit val TimestampedJsonMonoid: Monoid[TimestampedJsValue] = new Monoid[TimestampedJsValue] { + def combine(f1: TimestampedJsValue, f2: TimestampedJsValue): TimestampedJsValue = { + (f1, f2) match { + case (o1: TimestampedJsObject, o2: TimestampedJsObject) => + val sg = implicitly[Semigroup[Map[String, TimestampedJsValue]]] + TimestampedJsObject(sg.combine(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) + case (o1: TimestampedJsList, o2: TimestampedJsList) => + val sg = implicitly[Semigroup[Map[Int, TimestampedJsValue]]] + TimestampedJsList(sg.combine(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) + case (o1, o2) => timestampedJsValueOrdering.max(o1, o2) + } + } + + override def empty: TimestampedJsValue = TimestampedJsObject(Map.empty, OffsetDateTime.now) + } } /** Customized version of Json data structure, to account for timestamped values and lazy array creation */ -private sealed trait TimestampedJsValue { +sealed trait TimestampedJsValue { def toJson: JsValue def timestamp: OffsetDateTime } diff --git a/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala b/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala index 6d007dddc..0653be425 100644 --- a/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala +++ b/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala @@ -215,36 +215,37 @@ class MetadataBuilderActor(serviceRegistryActor: ActorRef) extends LoggingFSM[Me when(WaitingForMetadataService) { case Event(MetadataLookupResponse(query, metadata), _) => - context.parent ! RequestComplete(StatusCodes.OK, processMetadataResponse(query, metadata)) + context.parent ! RequestComplete((StatusCodes.OK, processMetadataResponse(query, metadata))) allDone case Event(StatusLookupResponse(w, status), _) => - context.parent ! RequestComplete(StatusCodes.OK, processStatusResponse(w, status)) + context.parent ! RequestComplete((StatusCodes.OK, processStatusResponse(w, status))) allDone case Event(failure: ServiceRegistryFailure, _) => val response = APIResponse.fail(new RuntimeException("Can't find metadata service")) - context.parent ! RequestComplete(StatusCodes.InternalServerError, response) + context.parent ! RequestComplete((StatusCodes.InternalServerError, response)) allDone case Event(WorkflowQuerySuccess(uri: Uri, response, metadata), _) => + import WorkflowJsonSupport._ context.parent ! RequestCompleteWithHeaders(response, generateLinkHeaders(uri, metadata):_*) allDone case Event(failure: WorkflowQueryFailure, _) => - context.parent ! RequestComplete(StatusCodes.BadRequest, APIResponse.fail(failure.reason)) + context.parent ! RequestComplete((StatusCodes.BadRequest, APIResponse.fail(failure.reason))) allDone case Event(WorkflowOutputsResponse(id, events), _) => // Add in an empty output event if there aren't already any output events. val hasOutputs = events exists { _.key.key.startsWith(WorkflowMetadataKeys.Outputs + ":") } val updatedEvents = if (hasOutputs) events else MetadataEvent.empty(MetadataKey(id, None, WorkflowMetadataKeys.Outputs)) +: events - context.parent ! RequestComplete(StatusCodes.OK, workflowMetadataResponse(id, updatedEvents, includeCallsIfEmpty = false)) + context.parent ! RequestComplete((StatusCodes.OK, workflowMetadataResponse(id, updatedEvents, includeCallsIfEmpty = false))) allDone case Event(LogsResponse(w, l), _) => - context.parent ! RequestComplete(StatusCodes.OK, workflowMetadataResponse(w, l, includeCallsIfEmpty = false)) + context.parent ! RequestComplete((StatusCodes.OK, workflowMetadataResponse(w, l, includeCallsIfEmpty = false))) allDone case Event(failure: MetadataServiceFailure, _) => - context.parent ! RequestComplete(StatusCodes.InternalServerError, APIResponse.error(failure.reason)) + context.parent ! RequestComplete((StatusCodes.InternalServerError, APIResponse.error(failure.reason))) allDone case Event(unexpectedMessage, stateData) => val response = APIResponse.fail(new RuntimeException(s"MetadataBuilderActor $tag(WaitingForMetadataService, $stateData) got an unexpected message: $unexpectedMessage")) - context.parent ! RequestComplete(StatusCodes.InternalServerError, response) + context.parent ! RequestComplete((StatusCodes.InternalServerError, response)) context stop self stay() } diff --git a/engine/src/main/scala/cromwell/webservice/package.scala b/engine/src/main/scala/cromwell/webservice/package.scala index 1fd8ec1d4..5b22ab108 100644 --- a/engine/src/main/scala/cromwell/webservice/package.scala +++ b/engine/src/main/scala/cromwell/webservice/package.scala @@ -1,42 +1,5 @@ package cromwell package object webservice { - case class QueryParameter(key: String, value: String) type QueryParameters = Seq[QueryParameter] - - object Patterns { - val WorkflowName = """ - (?x) # Turn on comments and whitespace insensitivity. - - ( # Begin capture. - - [a-zA-Z][a-zA-Z0-9_]* # WDL identifier naming pattern of an initial alpha character followed by zero - # or more alphanumeric or underscore characters. - - ) # End capture. - """.trim.r - - val CallFullyQualifiedName = """ - (?x) # Turn on comments and whitespace insensitivity. - - ( # Begin outer capturing group for FQN. - - (?:[a-zA-Z][a-zA-Z0-9_]*) # Inner noncapturing group for top-level workflow name. This is the WDL - # identifier naming pattern of an initial alpha character followed by zero - # or more alphanumeric or underscore characters. - - (?:\.[a-zA-Z][a-zA-Z0-9_]*){1} # Inner noncapturing group for call name, a literal dot followed by a WDL - # identifier. Currently this is quantified to {1} since the call name is - # mandatory and nested workflows are not supported. This could be changed - # to + or a different quantifier if these assumptions change. - - ) # End outer capturing group for FQN. - - - (?: # Begin outer noncapturing group for shard. - \. # Literal dot. - (\d+) # Captured shard digits. - )? # End outer optional noncapturing group for shard. - """.trim.r // The trim is necessary as (?x) must be at the beginning of the regex. - } } diff --git a/engine/src/main/scala/cromwell/webservice/webservice_.scala b/engine/src/main/scala/cromwell/webservice/webservice_.scala new file mode 100644 index 000000000..d68ba0bdb --- /dev/null +++ b/engine/src/main/scala/cromwell/webservice/webservice_.scala @@ -0,0 +1,39 @@ +package cromwell.webservice + +case class QueryParameter(key: String, value: String) + +object Patterns { + val WorkflowName = """ + (?x) # Turn on comments and whitespace insensitivity. + + ( # Begin capture. + + [a-zA-Z][a-zA-Z0-9_]* # WDL identifier naming pattern of an initial alpha character followed by zero + # or more alphanumeric or underscore characters. + + ) # End capture. + """.trim.r + + val CallFullyQualifiedName = """ + (?x) # Turn on comments and whitespace insensitivity. + + ( # Begin outer capturing group for FQN. + + (?:[a-zA-Z][a-zA-Z0-9_]*) # Inner noncapturing group for top-level workflow name. This is the WDL + # identifier naming pattern of an initial alpha character followed by zero + # or more alphanumeric or underscore characters. + + (?:\.[a-zA-Z][a-zA-Z0-9_]*){1} # Inner noncapturing group for call name, a literal dot followed by a WDL + # identifier. Currently this is quantified to {1} since the call name is + # mandatory and nested workflows are not supported. This could be changed + # to + or a different quantifier if these assumptions change. + + ) # End outer capturing group for FQN. + + + (?: # Begin outer noncapturing group for shard. + \. # Literal dot. + (\d+) # Captured shard digits. + )? # End outer optional noncapturing group for shard. + """.trim.r // The trim is necessary as (?x) must be at the beginning of the regex. +} diff --git a/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala b/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala index 06f437de6..00a530f31 100644 --- a/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala +++ b/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala @@ -5,7 +5,6 @@ import wdl4s.types.{WdlArrayType, WdlStringType} import wdl4s.values.{WdlArray, WdlString} import cromwell.util.SampleWdl -import scala.language.postfixOps class ArrayOfArrayCoercionSpec extends CromwellTestkitSpec { "A workflow that has an Array[Array[File]] input " should { diff --git a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala index dada6157f..843796c1a 100644 --- a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala @@ -1,7 +1,6 @@ package cromwell import java.nio.file.Files -import java.util.UUID import akka.testkit._ import better.files._ @@ -11,7 +10,6 @@ import wdl4s.expression.NoFunctions import wdl4s.types.{WdlArrayType, WdlFileType, WdlStringType} import wdl4s.values.{WdlArray, WdlFile, WdlInteger, WdlString} -import scala.language.postfixOps class ArrayWorkflowSpec extends CromwellTestkitSpec { val tmpDir = Files.createTempDirectory("ArrayWorkflowSpec") @@ -65,7 +63,6 @@ class ArrayWorkflowSpec extends CromwellTestkitSpec { ) ) ) - val uuid = UUID.randomUUID() val pwd = File(".") val sampleWdl = SampleWdl.ArrayLiteral(pwd.path) runWdlAndAssertOutputs( diff --git a/engine/src/test/scala/cromwell/CallCachingWorkflowSpec.scala b/engine/src/test/scala/cromwell/CallCachingWorkflowSpec.scala index 5718ad337..cea47fe8d 100644 --- a/engine/src/test/scala/cromwell/CallCachingWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/CallCachingWorkflowSpec.scala @@ -5,15 +5,11 @@ import java.util.UUID import akka.testkit._ import com.typesafe.config.ConfigFactory import cromwell.CallCachingWorkflowSpec._ -import cromwell.core.Tags.DockerTest -import cromwell.core.Tags._ -import cromwell.engine.workflow.WorkflowManagerActor -import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreActor} +import cromwell.core.Tags.{DockerTest, _} import cromwell.util.SampleWdl import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType} import wdl4s.values.{WdlArray, WdlFile, WdlInteger, WdlString} -import scala.language.postfixOps class CallCachingWorkflowSpec extends CromwellTestkitSpec { def cacheHitMessageForCall(name: String) = s"Call Caching: Cache hit. Using UUID\\(.{8}\\):$name\\.*" @@ -146,11 +142,11 @@ class CallCachingWorkflowSpec extends CromwellTestkitSpec { FIXME: This test had been constructing a custom WorkflowManagerActor. I don't believe this is still necessary but this test is being ignored so I'm not sure */ - val workflowId = runWdlAndAssertOutputs( - sampleWdl = SampleWdl.CallCachingWorkflow(UUID.randomUUID().toString), - eventFilter = EventFilter.info(pattern = cacheHitMessageForCall("a"), occurrences = 1), - expectedOutputs = expectedOutputs, - config = CallCachingWorkflowSpec.callCachingConfig) +// val workflowId = runWdlAndAssertOutputs( +// sampleWdl = SampleWdl.CallCachingWorkflow(UUID.randomUUID().toString), +// eventFilter = EventFilter.info(pattern = cacheHitMessageForCall("a"), occurrences = 1), +// expectedOutputs = expectedOutputs, +// config = CallCachingWorkflowSpec.callCachingConfig) // val status = messageAndWait[WorkflowManagerStatusSuccess](WorkflowStatus(workflowId)).state // status shouldEqual WorkflowSucceeded diff --git a/engine/src/test/scala/cromwell/CromwellSpec.scala b/engine/src/test/scala/cromwell/CromwellSpec.scala index f1ff598d7..da1720f63 100644 --- a/engine/src/test/scala/cromwell/CromwellSpec.scala +++ b/engine/src/test/scala/cromwell/CromwellSpec.scala @@ -1,7 +1,6 @@ package cromwell import com.typesafe.config.ConfigFactory -import org.scalatest.Tag object CromwellSpec { val BackendConfText = diff --git a/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala b/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala index c6f6b3045..c4aec5879 100644 --- a/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala +++ b/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala @@ -3,7 +3,7 @@ package cromwell import java.nio.file.Paths import java.util.UUID -import akka.actor.{Actor, ActorRef, ActorSystem, Props} +import akka.actor.{Actor, ActorRef, ActorSystem, Props, Terminated} import akka.pattern.ask import akka.testkit._ import com.typesafe.config.{Config, ConfigFactory} @@ -36,7 +36,7 @@ import wdl4s.types._ import wdl4s.values._ import scala.concurrent.duration._ -import scala.concurrent.{Await, ExecutionContext} +import scala.concurrent.{Await, ExecutionContext, Future} import scala.language.postfixOps import scala.util.matching.Regex @@ -127,7 +127,8 @@ object CromwellTestkitSpec { * Do NOT shut down the test actor system inside the normal flow. * The actor system will be externally shutdown outside the block. */ - override def shutdownActorSystem() = {} + // -Ywarn-value-discard + override def shutdownActorSystem(): Future[Terminated] = { Future.successful(null) } def shutdownTestActorSystem() = super.shutdownActorSystem() } @@ -276,7 +277,7 @@ object CromwellTestkitSpec { abstract class CromwellTestkitSpec(val twms: TestWorkflowManagerSystem = new CromwellTestkitSpec.TestWorkflowManagerSystem()) extends TestKit(twms.actorSystem) with DefaultTimeout with ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll with ScalaFutures with OneInstancePerTest with Eventually { - override protected def afterAll() = { twms.shutdownTestActorSystem() } + override protected def afterAll() = { twms.shutdownTestActorSystem(); () } implicit val defaultPatience = PatienceConfig(timeout = Span(30, Seconds), interval = Span(100, Millis)) implicit val ec = system.dispatcher @@ -407,6 +408,7 @@ abstract class CromwellTestkitSpec(val twms: TestWorkflowManagerSystem = new Cro } getWorkflowState(workflowId, serviceRegistryActor) should equal (expectedState) + () } private def getWorkflowOutputsFromMetadata(id: WorkflowId, serviceRegistryActor: ActorRef): Map[FullyQualifiedName, WdlValue] = { diff --git a/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala b/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala index 9671706ef..d829b84eb 100644 --- a/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala @@ -5,7 +5,6 @@ import wdl4s.{NamespaceWithWorkflow, WorkflowInput} import cromwell.util.SampleWdl import org.scalatest.{Matchers, WordSpecLike} -import scala.language.postfixOps class DeclarationWorkflowSpec extends Matchers with WordSpecLike { "A workflow with declarations in it" should { diff --git a/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala b/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala index 3ddffcba7..1aaafaf2f 100644 --- a/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala @@ -1,12 +1,10 @@ package cromwell import akka.testkit._ -import wdl4s.values.{WdlFile, WdlString} import cromwell.util.SampleWdl +import wdl4s.values.{WdlFile, WdlString} -import scala.concurrent.ExecutionContext import scala.concurrent.duration._ -import scala.language.postfixOps class FilePassingWorkflowSpec extends CromwellTestkitSpec { "A workflow that passes files between tasks" should { diff --git a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala index 1176a4f99..9a00c115e 100644 --- a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala @@ -8,7 +8,6 @@ import wdl4s.expression.{NoFunctions, WdlFunctions} import wdl4s.types.{WdlFileType, WdlIntegerType, WdlMapType, WdlStringType} import wdl4s.values._ -import scala.language.postfixOps import scala.util.{Success, Try} class MapWorkflowSpec extends CromwellTestkitSpec { diff --git a/engine/src/test/scala/cromwell/MetadataWatchActor.scala b/engine/src/test/scala/cromwell/MetadataWatchActor.scala index cc334b05f..691c4efc5 100644 --- a/engine/src/test/scala/cromwell/MetadataWatchActor.scala +++ b/engine/src/test/scala/cromwell/MetadataWatchActor.scala @@ -19,6 +19,7 @@ final case class MetadataWatchActor(promise: Promise[Unit], matchers: Matcher*) unsatisfiedMatchers = unsatisfiedMatchers.filterNot { m => m.matches(events) } if (unsatisfiedMatchers.isEmpty) { promise.trySuccess(()) + () } case PutMetadataAction(_) => // Superfluous message. Ignore case _ => throw new Exception("Invalid message to MetadataWatchActor") diff --git a/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala b/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala index ca7e2720b..f0b7a70af 100644 --- a/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala @@ -1,10 +1,9 @@ package cromwell import akka.testkit._ -import wdl4s.values.{WdlString, WdlFile} import cromwell.util.SampleWdl +import wdl4s.values.WdlString -import scala.language.postfixOps class MultipleFilesWithSameNameWorkflowSpec extends CromwellTestkitSpec { "A workflow with two file inputs that have the same name" should { diff --git a/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala b/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala index af3d68c29..919008315 100644 --- a/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala @@ -5,17 +5,16 @@ import wdl4s.WdlNamespace import wdl4s.expression.NoFunctions import wdl4s.values.{WdlFile, WdlString} -import scala.language.postfixOps class OptionalParamWorkflowSpec extends Matchers with WordSpecLike { "A workflow with an optional parameter that has a prefix inside the tag" should { "not include that prefix if no value is specified" in { - val wf = """ + val wf = s""" |task find { | String? pattern | File root | command { - | find ${root} ${"-name " + pattern} + | find $${root} $${"-name " + pattern} | } |} | diff --git a/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala b/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala index 8daf9be69..8530dd6d2 100644 --- a/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala @@ -4,7 +4,6 @@ import akka.testkit._ import wdl4s.values.WdlString import cromwell.util.SampleWdl -import scala.language.postfixOps class PostfixQuantifierWorkflowSpec extends CromwellTestkitSpec { "A task which contains a parameter with a zero-or-more postfix quantifier" should { diff --git a/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala b/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala index b999e3623..6b706fbc6 100644 --- a/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala @@ -23,8 +23,8 @@ class RestartWorkflowSpec extends CromwellTestkitSpec with WorkflowDescriptorBui "RestartWorkflowSpec" should { "restart a call in Running state" taggedAs PostMVP ignore { - val id = WorkflowId.randomId() - val descriptor = createMaterializedEngineWorkflowDescriptor(id, sources) +// val id = WorkflowId.randomId() +// val descriptor = createMaterializedEngineWorkflowDescriptor(id, sources) // val a = ExecutionDatabaseKey("w.a", Option(-1), 1) // val b = ExecutionDatabaseKey("w.b", Option(-1), 1) // diff --git a/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala b/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala index 035004669..0d8847a27 100644 --- a/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala +++ b/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala @@ -6,8 +6,6 @@ import wdl4s.types.{WdlArrayType, WdlFileType, WdlIntegerType, WdlStringType} import wdl4s.values.{WdlArray, WdlFile, WdlInteger, WdlString} import cromwell.util.SampleWdl -import scala.language.postfixOps - class ScatterWorkflowSpec extends CromwellTestkitSpec { "A workflow with a stand-alone scatter block in it" should { "run properly" in { diff --git a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala index f2dd4db13..34ec73895 100644 --- a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala @@ -17,7 +17,6 @@ import org.scalatest.BeforeAndAfter import scala.concurrent.duration._ import scala.concurrent.{Await, Promise} -import scala.language.postfixOps object SimpleWorkflowActorSpec { diff --git a/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala b/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala index 4a13be95d..72c618fca 100644 --- a/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala +++ b/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala @@ -1,12 +1,10 @@ package cromwell import akka.testkit._ -import wdl4s.types.{WdlMapType, WdlStringType, WdlArrayType} -import wdl4s.values.{WdlMap, WdlArray, WdlString} -import cromwell.core.Tags.DockerTest import cromwell.util.SampleWdl +import wdl4s.types.{WdlMapType, WdlStringType} +import wdl4s.values.{WdlMap, WdlString} -import scala.language.postfixOps class WdlFunctionsAtWorkflowLevelSpec extends CromwellTestkitSpec { val outputMap = WdlMap(WdlMapType(WdlStringType, WdlStringType), Map( diff --git a/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala b/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala index bbdd67a6e..18df31795 100644 --- a/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala +++ b/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala @@ -4,7 +4,6 @@ import akka.testkit._ import cromwell.util.SampleWdl import cromwell.CromwellTestkitSpec.AnyValueIsFine -import scala.language.postfixOps class WorkflowOutputsSpec extends CromwellTestkitSpec { "Workflow outputs" should { diff --git a/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala b/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala index 9ecff4ba5..55faea29f 100644 --- a/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala @@ -4,7 +4,6 @@ import cromwell.CromwellTestkitSpec import cromwell.engine.workflow.WorkflowDescriptorBuilder import cromwell.util.SampleWdl -import scala.language.postfixOps class WorkflowManagerActorSpec extends CromwellTestkitSpec with WorkflowDescriptorBuilder { override implicit val actorSystem = system @@ -15,7 +14,7 @@ class WorkflowManagerActorSpec extends CromwellTestkitSpec with WorkflowDescript val outputs = runWdl(sampleWdl = SampleWdl.CurrentDirectory) val outputName = "whereami.whereami.pwd" - val salutation = outputs.get(outputName).get + val salutation = outputs(outputName) val actualOutput = salutation.valueString.trim actualOutput should endWith("/call-whereami/execution") } diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala index 5c0a48253..73e8bfa7b 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala @@ -22,7 +22,6 @@ import spray.json._ import scala.concurrent.Await import scala.concurrent.duration.Duration -import scala.language.postfixOps import scala.util._ /** @@ -79,6 +78,7 @@ abstract class SingleWorkflowRunnerActorSpec extends CromwellTestkitSpec { val actorRef = createRunnerActor(sampleWdl, managerActor, outputFile) val futureResult = actorRef ? RunWorkflow Await.ready(futureResult, Duration.Inf) + () } } @@ -103,7 +103,7 @@ class SingleWorkflowRunnerActorWithMetadataSpec extends SingleWorkflowRunnerActo super.afterAll() } - private def doTheTest(wdlFile: SampleWdl, expectedCalls: TableFor3[String, Int, Int], workflowInputs: Int, workflowOutputs: Int) = { + private def doTheTest(wdlFile: SampleWdl, expectedCalls: TableFor3[String, Long, Long], workflowInputs: Long, workflowOutputs: Long) = { val testStart = OffsetDateTime.now within(TimeoutDuration) { singleWorkflowActor( @@ -153,18 +153,18 @@ class SingleWorkflowRunnerActorWithMetadataSpec extends SingleWorkflowRunnerActo "successfully run a workflow outputting metadata" in { val expectedCalls = Table( ("callName", "numInputs", "numOutputs"), - ("three_step.wc", 1, 1), - ("three_step.ps", 0, 1), - ("three_step.cgrep", 2, 1)) + ("three_step.wc", 1L, 1L), + ("three_step.ps", 0L, 1L), + ("three_step.cgrep", 2L, 1L)) - doTheTest(ThreeStep, expectedCalls, 1, 3) + doTheTest(ThreeStep, expectedCalls, 1L, 3L) } "run a workflow outputting metadata with no remaining input expressions" in { val expectedCalls = Table( ("callName", "numInputs", "numOutputs"), - ("wf.echo", 1, 1), - ("wf.echo2", 1, 1)) - doTheTest(ExpressionsInInputs, expectedCalls, 2, 2) + ("wf.echo", 1L, 1L), + ("wf.echo2", 1L, 1L)) + doTheTest(ExpressionsInInputs, expectedCalls, 2L, 2L) } } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala index 6a0245528..860ac6dab 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala @@ -66,7 +66,6 @@ class WorkflowActorSpec extends CromwellTestkitSpec with WorkflowDescriptorBuild "WorkflowActor" should { "run Finalization actor if Initialization fails" in { - val workflowId = WorkflowId.randomId() val actor = createWorkflowActor(InitializingWorkflowState) deathwatch watch actor actor ! WorkflowInitializationFailedResponse(Seq(new Exception("Materialization Failed"))) diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/CachingConfigSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/CachingConfigSpec.scala index 4015f01de..499e6f0c2 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/CachingConfigSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/CachingConfigSpec.scala @@ -4,7 +4,7 @@ import cats.data.Validated.{Invalid, Valid} import com.typesafe.config.{Config, ConfigFactory} import cromwell.core.WorkflowOptions import cromwell.core.callcaching.CallCachingMode -import org.scalatest.{FlatSpec, Matchers} +import org.scalatest.{Assertion, FlatSpec, Matchers} import scala.collection.JavaConverters._ import scala.util.{Success, Try} @@ -51,13 +51,13 @@ class CachingConfigSpec extends FlatSpec with Matchers { val writeCacheOffCombinations = allCombinations -- writeCacheOnCombinations val readCacheOffCombinations = allCombinations -- readCacheOnCombinations - validateCallCachingMode("write cache on options", writeCacheOnCombinations) { mode => mode.writeToCache should be(true) } - validateCallCachingMode("read cache on options", readCacheOnCombinations) { mode => mode.readFromCache should be(true) } - validateCallCachingMode("write cache off options", writeCacheOffCombinations) { mode => mode.writeToCache should be(false) } - validateCallCachingMode("read cache off options", readCacheOffCombinations) { mode => mode.readFromCache should be(false) } + validateCallCachingMode("write cache on options", writeCacheOnCombinations) { _.writeToCache should be(true) } + validateCallCachingMode("read cache on options", readCacheOnCombinations) { _.readFromCache should be(true) } + validateCallCachingMode("write cache off options", writeCacheOffCombinations) { _.writeToCache should be(false) } + validateCallCachingMode("read cache off options", readCacheOffCombinations) { _.readFromCache should be(false) } - private def validateCallCachingMode(testName: String, combinations: Set[(Config, Try[WorkflowOptions])])(verificationFunction: CallCachingMode => Unit) = { + private def validateCallCachingMode(testName: String, combinations: Set[(Config, Try[WorkflowOptions])])(verificationFunction: CallCachingMode => Assertion) = { it should s"correctly identify $testName" in { combinations foreach { case (config, Success(wfOptions)) => diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala index 9414647bd..970bc7156 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala @@ -15,7 +15,6 @@ import spray.json._ import wdl4s.values.{WdlInteger, WdlString} import scala.concurrent.duration._ -import scala.language.postfixOps class MaterializeWorkflowDescriptorActorSpec extends CromwellTestkitSpec with BeforeAndAfter with MockitoSugar { @@ -65,9 +64,9 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestkitSpec with Be wfDesc.id shouldBe workflowId wfDesc.name shouldBe "hello" wfDesc.namespace.tasks.size shouldBe 1 - wfDesc.workflowInputs.head shouldBe ("hello.hello.addressee", WdlString("world")) - wfDesc.backendDescriptor.inputs.head shouldBe ("hello.hello.addressee", WdlString("world")) - wfDesc.getWorkflowOption(WorkflowOptions.WriteToCache) shouldBe Some("true") + wfDesc.workflowInputs.head shouldBe (("hello.hello.addressee", WdlString("world"))) + wfDesc.backendDescriptor.inputs.head shouldBe (("hello.hello.addressee", WdlString("world"))) + wfDesc.getWorkflowOption(WorkflowOptions.WriteToCache) shouldBe Option("true") wfDesc.getWorkflowOption(WorkflowOptions.ReadFromCache) shouldBe None // Default backend assignment is "Local": wfDesc.backendAssignments foreach { diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala index 498180780..d9de4432b 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala @@ -46,5 +46,6 @@ class EjeaCheckingCallCacheSpec extends EngineJobExecutionActorSpec with Eventua private def createCheckingCallCacheEjea(restarting: Boolean = false): Unit = { ejea = helper.buildEJEA(restarting = restarting, callCachingMode = CallCachingActivity(ReadCache)) ejea.setStateInline(state = CheckingCallCache, data = ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, None)) + () } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala index e5a8ff36e..a8f2bdf46 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala @@ -1,15 +1,12 @@ package cromwell.engine.workflow.lifecycle.execution.ejea -import akka.testkit.TestProbe -import cromwell.backend.BackendJobDescriptorKey -import cromwell.backend.BackendJobExecutionActor.{FailedNonRetryableResponse, FailedRetryableResponse, RecoverJobCommand, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{FailedNonRetryableResponse, FailedRetryableResponse, SucceededResponse} import cromwell.core._ -import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{CheckingJobStore, JobRunning, NoData, PreparingJob} -import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor.BackendJobPreparationFailed -import cromwell.jobstore.{JobResultFailure, JobResultSuccess} -import cromwell.jobstore.JobStoreActor.{JobComplete, JobNotComplete} -import EngineJobExecutionActorSpec.EnhancedTestEJEA +import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{CheckingJobStore, NoData, PreparingJob} import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor +import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec.EnhancedTestEJEA +import cromwell.jobstore.JobStoreActor.{JobComplete, JobNotComplete} +import cromwell.jobstore.{JobResultFailure, JobResultSuccess} class EjeaCheckingJobStoreSpec extends EngineJobExecutionActorSpec { diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala index 87f2c5131..e8eaeec1a 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala @@ -29,8 +29,8 @@ trait EngineJobExecutionActorSpec extends CromwellTestkitSpec // The default values for these are "null". The helper is created in "before", the ejea is up to the test cases - var helper: PerTestHelper = _ - var ejea: TestFSMRef[EngineJobExecutionActorState, EJEAData, MockEjea] = _ + private[ejea] var helper: PerTestHelper = _ + private[ejea] var ejea: TestFSMRef[EngineJobExecutionActorState, EJEAData, MockEjea] = _ implicit def stateUnderTest: EngineJobExecutionActorState before { diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala index 5a2e7bff1..60cc183a9 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala @@ -31,7 +31,7 @@ private[ejea] trait CanExpectCacheWrites extends Eventually { self: EngineJobExe creation._2 should be(expectedResponse) case _ => fail("Expected exactly one cache write actor creation.") } - + () } } @@ -46,6 +46,7 @@ private[ejea] trait CanExpectJobStoreWrites extends CanValidateJobStoreKey { sel ejea.stateName should be(UpdatingJobStore) ejea.stateData should be(expectedData) } + () } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/ExpectOne.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/ExpectOne.scala index bf8049098..7ddfd340f 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/ExpectOne.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/ExpectOne.scala @@ -1,10 +1,10 @@ package cromwell.engine.workflow.lifecycle.execution.ejea private[ejea] sealed trait ExpectOne[+A] { - def checkIt(block: A => Unit): Unit = throw new IllegalStateException("An ExpectOne must have exactly one element for checkIt to work") + def checkIt(block: A => Any): Unit = throw new IllegalStateException("An ExpectOne must have exactly one element for checkIt to work") def hasExactlyOne: Boolean def foundOne[B >: A](theFoundOne: B) = this match { - case NothingYet => new GotOne(theFoundOne) + case NothingYet => GotOne(theFoundOne) case GotOne(theOriginalOne) => GotTooMany(List(theOriginalOne, theFoundOne)) case GotTooMany(theOnes) => GotTooMany(theOnes :+ theFoundOne) } @@ -15,7 +15,7 @@ private[ejea] case object NothingYet extends ExpectOne[scala.Nothing] { } private[ejea] case class GotOne[+A](theOne: A) extends ExpectOne[A] { - override def checkIt(block: A => Unit): Unit = block(theOne) + override def checkIt(block: A => Any): Unit = { block(theOne); () } override def hasExactlyOne = true } diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala index 543fc9473..b71c6a48f 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala @@ -302,7 +302,7 @@ object JobExecutionTokenDispenserActorSpec { implicit class intWithTimes(n: Int) { def times(f: => Unit) = 1 to n foreach { _ => f } - def indexedTimes(f: Int => Unit) = 0 until n foreach { i => f(i) } + def indexedTimes(f: Int => Any) = 0 until n foreach { i => f(i) } } val TestInfiniteTokenType = JobExecutionTokenType("infinite", maxPoolSize = None) diff --git a/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala b/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala index e1853e7e1..fa1cc5067 100644 --- a/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala +++ b/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala @@ -1,6 +1,5 @@ package cromwell.jobstore -import com.typesafe.config.ConfigFactory import cromwell.CromwellTestkitSpec import cromwell.backend.BackendJobDescriptorKey import cromwell.core.{JobOutput, WorkflowId} @@ -25,7 +24,6 @@ class JobStoreServiceSpec extends CromwellTestkitSpec with Matchers with Mockito "JobStoreService" should { "work" in { - val config = ConfigFactory.parseString("{}") lazy val jobStore: JobStore = new SqlJobStore(SingletonServicesStore.databaseInterface) val jobStoreService = system.actorOf(JobStoreActor.props(jobStore)) diff --git a/engine/src/test/scala/cromwell/jobstore/JobStoreWriterSpec.scala b/engine/src/test/scala/cromwell/jobstore/JobStoreWriterSpec.scala index 5699f61ed..a6e59679a 100644 --- a/engine/src/test/scala/cromwell/jobstore/JobStoreWriterSpec.scala +++ b/engine/src/test/scala/cromwell/jobstore/JobStoreWriterSpec.scala @@ -35,12 +35,14 @@ class JobStoreWriterSpec extends CromwellTestkitSpec with Matchers with BeforeAn key.callFqn shouldBe "call.fqn" key.index shouldBe None result shouldBe successResult + () } private def assertDb(totalWritesCalled: Int, jobCompletionsRecorded: Int, workflowCompletionsRecorded: Int): Unit = { database.totalWritesCalled shouldBe totalWritesCalled database.jobCompletionsRecorded shouldBe jobCompletionsRecorded database.workflowCompletionsRecorded shouldBe workflowCompletionsRecorded + () } private def assertReceived(expectedJobStoreWriteAcks: Int): Unit = { @@ -51,6 +53,7 @@ class JobStoreWriterSpec extends CromwellTestkitSpec with Matchers with BeforeAn case message => fail(s"Unexpected response message: $message") } jobStoreWriter.underlyingActor.stateName shouldBe Pending + () } "JobStoreWriter" should { diff --git a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala index 87db769c9..9904835dc 100644 --- a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala @@ -16,7 +16,7 @@ import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata._ import cromwell.services.metadata.impl.MetadataSummaryRefreshActor.MetadataSummarySuccess import cromwell.util.SampleWdl.DeclarationsWorkflow._ -import cromwell.util.SampleWdl.{ExpressionsInInputs, DeclarationsWorkflow, ThreeStep, HelloWorld} +import cromwell.util.SampleWdl.HelloWorld import org.scalatest.concurrent.{PatienceConfiguration, ScalaFutures} import org.scalatest.{FlatSpec, Matchers} import org.specs2.mock.Mockito @@ -80,6 +80,7 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala import akka.pattern.ask val putResult = serviceRegistryActor.ask(PutMetadataAction(events))(timeout) putResult.futureValue(PatienceConfiguration.Timeout(timeout.duration)) shouldBe a[MetadataPutAcknowledgement] + () } def forceSummary(): Unit = { @@ -242,7 +243,7 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala behavior of "REST API submission endpoint" it should "return 201 for a successful workflow submission " in { - Post("/workflows/$version", FormData(Seq("wdlSource" -> HelloWorld.wdlSource(), "workflowInputs" -> HelloWorld.rawInputs.toJson.toString()))) ~> + Post(s"/workflows/$version", FormData(Seq("wdlSource" -> HelloWorld.wdlSource(), "workflowInputs" -> HelloWorld.rawInputs.toJson.toString()))) ~> submitRoute ~> check { assertResult( @@ -274,7 +275,7 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala it should "return 200 for a successful workflow submission " in { val inputs = HelloWorld.rawInputs.toJson - Post("/workflows/$version/batch", + Post(s"/workflows/$version/batch", FormData(Seq("wdlSource" -> HelloWorld.wdlSource(), "workflowInputs" -> s"[$inputs, $inputs]"))) ~> submitBatchRoute ~> check { diff --git a/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala b/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala index 08c1e5b16..5ae82221b 100644 --- a/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala @@ -5,9 +5,8 @@ import java.util.UUID import akka.testkit._ import cromwell.core.{TestKitSuite, WorkflowId} +import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata._ -import MetadataService._ -import cromwell.services._ import cromwell.webservice.PerRequest.RequestComplete import cromwell.webservice.metadata.MetadataBuilderActor import org.scalatest.prop.TableDrivenPropertyChecks @@ -52,12 +51,12 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik val workflowA = WorkflowId.randomId() val workflowACalls = List( - Option(new MetadataJobKey("callB", Some(1), 3)), - Option(new MetadataJobKey("callB", None, 1)), - Option(new MetadataJobKey("callB", Some(1), 2)), - Option(new MetadataJobKey("callA", None, 1)), - Option(new MetadataJobKey("callB", Some(1), 1)), - Option(new MetadataJobKey("callB", Some(0), 1)), + Option(MetadataJobKey("callB", Some(1), 3)), + Option(MetadataJobKey("callB", None, 1)), + Option(MetadataJobKey("callB", Some(1), 2)), + Option(MetadataJobKey("callA", None, 1)), + Option(MetadataJobKey("callB", Some(1), 1)), + Option(MetadataJobKey("callB", Some(0), 1)), None ) val workflowAEvents = workflowACalls map { makeEvent(workflowA, _) } diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystem.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystem.scala index 2aad4b57b..215b18935 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystem.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystem.scala @@ -6,7 +6,6 @@ import java.nio.file.attribute.UserPrincipalLookupService import java.nio.file.spi.FileSystemProvider import java.util.{Collections, Set => JSet} -import scala.language.postfixOps case class NotAGcsPathException(path: String) extends IllegalArgumentException(s"$path is not a valid GCS path.") diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala index 7199d58ec..fd40a5403 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala @@ -11,6 +11,8 @@ import java.util import java.util.Collections import java.util.concurrent.{AbstractExecutorService, TimeUnit} +import cats.instances.try_._ +import cats.syntax.functor._ import com.google.api.client.googleapis.json.GoogleJsonResponseException import com.google.api.client.googleapis.media.MediaHttpUploader import com.google.api.services.storage.Storage @@ -47,7 +49,7 @@ object GcsFileSystemProvider { if retries > 0 && (ex.getStatusCode == 404 || ex.getStatusCode == 500) => // FIXME remove this sleep - Thread.sleep(retryInterval.toMillis.toInt) + Thread.sleep(retryInterval.toMillis) withRetry(f, retries - 1) case Failure(ex) => throw ex } @@ -102,12 +104,13 @@ class GcsFileSystemProvider private[gcs](storageClient: Try[Storage], val execut lazy val defaultFileSystem: GcsFileSystem = GcsFileSystem(this) - private def exists(path: Path) = path match { + private def exists(path: Path): Unit = path match { case gcsPath: NioGcsPath => - Try(withRetry(client.objects.get(gcsPath.bucket, gcsPath.objectName).execute)) recover { + val attempt: Try[Any] = Try(withRetry(client.objects.get(gcsPath.bucket, gcsPath.objectName).execute)) recover { case ex: GoogleJsonResponseException if ex.getStatusCode == 404 => if (!gcsPath.isDirectory) throw new FileNotFoundException(path.toString) - } get + } + attempt.void.get case _ => throw new FileNotFoundException(path.toString) } @@ -173,12 +176,13 @@ class GcsFileSystemProvider private[gcs](storageClient: Try[Storage], val execut override def copy(source: Path, target: Path, options: CopyOption*): Unit = { (source, target) match { case (s: NioGcsPath, d: NioGcsPath) => - def innerCopy = { + def innerCopy(): Unit = { val storageObject = client.objects.get(s.bucket, s.objectName).execute client.objects.copy(s.bucket, s.objectName, d.bucket, d.objectName, storageObject).execute + () } - withRetry(innerCopy) + withRetry(innerCopy()) case _ => throw new UnsupportedOperationException(s"Can only copy from GCS to GCS: $source or $target is not a GCS path") } } @@ -186,7 +190,10 @@ class GcsFileSystemProvider private[gcs](storageClient: Try[Storage], val execut override def delete(path: Path): Unit = { path match { case gcs: NioGcsPath => try { - withRetry(client.objects.delete(gcs.bucket, gcs.objectName).execute()) + withRetry { + client.objects.delete(gcs.bucket, gcs.objectName).execute() + () + } } catch { case ex: GoogleJsonResponseException if ex.getStatusCode == 404 => throw new NoSuchFileException(path.toString) } @@ -204,12 +211,13 @@ class GcsFileSystemProvider private[gcs](storageClient: Try[Storage], val execut override def move(source: Path, target: Path, options: CopyOption*): Unit = { (source, target) match { case (s: NioGcsPath, d: NioGcsPath) => - def moveInner = { + def moveInner(): Unit = { val storageObject = client.objects.get(s.bucket, s.objectName).execute client.objects.rewrite(s.bucket, s.objectName, d.bucket, d.objectName, storageObject).execute + () } - withRetry(moveInner) + withRetry(moveInner()) case _ => throw new UnsupportedOperationException(s"Can only move from GCS to GCS: $source or $target is not a GCS path") } } @@ -219,7 +227,7 @@ class GcsFileSystemProvider private[gcs](storageClient: Try[Storage], val execut case _ => notAGcsPath(path) } - override def checkAccess(path: Path, modes: AccessMode*): Unit = exists(path) + override def checkAccess(path: Path, modes: AccessMode*): Unit = { exists(path); () } override def createDirectory(dir: Path, attrs: FileAttribute[_]*): Unit = {} override def getFileSystem(uri: URI): FileSystem = defaultFileSystem @@ -239,7 +247,7 @@ class GcsFileSystemProvider private[gcs](storageClient: Try[Storage], val execut // Contains a Seq corresponding to the current page of objects, plus a token for the next page of objects, if any. case class ListPageResult(objects: Seq[StorageObject], nextPageToken: Option[String]) - def requestListPage(pageToken: Option[String] = None): ListPageResult = { + def requestListPage(pageToken: Option[String]): ListPageResult = { val objects = withRetry(listRequest.setPageToken(pageToken.orNull).execute()) ListPageResult(objects.getItems.asScala, Option(objects.getNextPageToken)) } diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleAuthMode.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleAuthMode.scala index c18a85a25..2930cc911 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleAuthMode.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleAuthMode.scala @@ -161,7 +161,7 @@ final case class RefreshTokenMode(name: String, clientId: String, clientSecret: /** * Throws if the refresh token is not specified. */ - override def assertWorkflowOptions(options: GoogleAuthOptions) = getToken(options) + override def assertWorkflowOptions(options: GoogleAuthOptions): Unit = { getToken(options); () } private def getToken(options: GoogleAuthOptions): String = { options.get(RefreshTokenOptionKey).getOrElse(throw new IllegalArgumentException(s"Missing parameters in workflow options: $RefreshTokenOptionKey")) diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala index 9315b51cc..9c5579839 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala @@ -13,7 +13,6 @@ import cromwell.core.ErrorOr._ import org.slf4j.LoggerFactory import scala.collection.JavaConverters._ -import scala.language.postfixOps final case class GoogleConfiguration private (applicationName: String, authsByName: Map[String, GoogleAuthMode]) { diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/NioGcsPath.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/NioGcsPath.scala index 672b64cf0..65e148f77 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/NioGcsPath.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/NioGcsPath.scala @@ -7,7 +7,7 @@ import java.nio.file._ import java.util import scala.collection.JavaConverters._ -import scala.language.{implicitConversions, postfixOps} +import scala.language.postfixOps import scala.util.Try object NioGcsPath { diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala index b1d44feaa..3eeeaf568 100644 --- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala +++ b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala @@ -4,7 +4,6 @@ import com.typesafe.config.{ConfigException, ConfigFactory} import lenthall.config.ConfigValidationException import org.scalatest.{FlatSpec, Matchers} -import scala.language.postfixOps class GoogleConfigurationSpec extends FlatSpec with Matchers { diff --git a/project/Settings.scala b/project/Settings.scala index 3038ffed3..627ac080c 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -23,13 +23,31 @@ object Settings { https://github.com/sbt/sbt-assembly/issues/69 https://github.com/scala/pickling/issues/10 + + Other fancy flags from + + http://blog.threatstack.com/useful-scalac-options-for-better-scala-development-part-1 + + and + + https://tpolecat.github.io/2014/04/11/scalac-flags.html + */ val compilerSettings = List( - "-deprecation", - "-unchecked", + "-Xlint", "-feature", - "-Xmax-classfile-name", - "200" + "-Xmax-classfile-name", "200", + "-target:jvm-1.8", + "-encoding", "UTF-8", + "-unchecked", + "-deprecation", + "-Xfuture", + "-Yno-adapted-args", + "-Ywarn-dead-code", + "-Ywarn-numeric-widen", + "-Ywarn-value-discard", + "-Ywarn-unused", + "-Ywarn-unused-import" ) lazy val assemblySettings = Seq( diff --git a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala index b440b9f9e..672f68580 100644 --- a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala +++ b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala @@ -8,7 +8,6 @@ import cromwell.core.{JobKey, WorkflowId, WorkflowState} import cromwell.services.ServiceRegistryActor.ServiceRegistryMessage import wdl4s.values._ -import scala.language.postfixOps object MetadataService { diff --git a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala index 3bf9fce2b..435c14df2 100644 --- a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala +++ b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala @@ -8,7 +8,6 @@ import cats.syntax.validated._ import cromwell.core.{WorkflowId, WorkflowState} import cromwell.core.ErrorOr._ -import scala.language.postfixOps import scala.util.{Success, Try} object WorkflowQueryKey { diff --git a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala index 669bde613..f66173657 100644 --- a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala +++ b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala @@ -9,7 +9,6 @@ import cromwell.core.WorkflowId import cromwell.services.metadata.WorkflowQueryKey._ import cromwell.core.ErrorOr._ -import scala.language.postfixOps case class WorkflowQueryParameters private(statuses: Set[String], names: Set[String], diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala index 92ba34589..76476d3fd 100644 --- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala +++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala @@ -38,7 +38,8 @@ object MetadataDatabaseAccess { def baseSummary(workflowUuid: String) = WorkflowMetadataSummaryEntry(workflowUuid, None, None, None, None, None) - private implicit class MetadatumEnhancer(val metadatum: MetadataEntry) extends AnyVal { + // If visibility is made `private`, there's a bogus warning about this being unused. + implicit class MetadatumEnhancer(val metadatum: MetadataEntry) extends AnyVal { def toSummary: WorkflowMetadataSummaryEntry = { val base = baseSummary(metadatum.workflowExecutionUuid) metadatum.metadataKey match { diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala index 48a85b71e..ab9e20f29 100644 --- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala +++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala @@ -1,6 +1,5 @@ package cromwell.services.metadata.impl -import java.time.OffsetDateTime import java.util.UUID import akka.actor.{Actor, ActorLogging, ActorRef, Props} @@ -13,7 +12,6 @@ import cromwell.services.metadata.impl.MetadataSummaryRefreshActor.{MetadataSumm import lenthall.config.ScalaConfig._ import scala.concurrent.duration.{Duration, FiniteDuration} -import scala.language.postfixOps import scala.util.{Failure, Success, Try} object MetadataServiceActor { @@ -37,8 +35,8 @@ case class MetadataServiceActor(serviceConfig: Config, globalConfig: Config) summaryActor foreach { _ => self ! RefreshSummary } - private def scheduleSummary = { - MetadataSummaryRefreshInterval map { context.system.scheduler.scheduleOnce(_, self, RefreshSummary)(context.dispatcher, self) } + private def scheduleSummary(): Unit = { + MetadataSummaryRefreshInterval foreach { context.system.scheduler.scheduleOnce(_, self, RefreshSummary)(context.dispatcher, self) } } private def buildSummaryActor: Option[ActorRef] = { @@ -76,9 +74,9 @@ case class MetadataServiceActor(serviceConfig: Config, globalConfig: Config) case v: ValidateWorkflowIdAndExecute => validateWorkflowId(v) case action: ReadAction => readActor forward action case RefreshSummary => summaryActor foreach { _ ! SummarizeMetadata(sender()) } - case MetadataSummarySuccess => scheduleSummary + case MetadataSummarySuccess => scheduleSummary() case MetadataSummaryFailure(t) => log.error(t, "Error summarizing metadata") - scheduleSummary + scheduleSummary() } } diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala index ad176567e..007755fa6 100644 --- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala +++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala @@ -1,6 +1,5 @@ package cromwell.services.metadata.impl -import java.time.OffsetDateTime import akka.actor.{ActorRef, LoggingFSM, Props} import com.typesafe.config.ConfigFactory diff --git a/services/src/main/scala/cromwell/services/metadata/metadata.scala b/services/src/main/scala/cromwell/services/metadata/metadata.scala new file mode 100644 index 000000000..9e621cfad --- /dev/null +++ b/services/src/main/scala/cromwell/services/metadata/metadata.scala @@ -0,0 +1,39 @@ +package cromwell.services.metadata + +case class QueryParameter(key: String, value: String) + +object Patterns { + val WorkflowName = """ + (?x) # Turn on comments and whitespace insensitivity. + + ( # Begin capture. + + [a-zA-Z][a-zA-Z0-9_]* # WDL identifier naming pattern of an initial alpha character followed by zero + # or more alphanumeric or underscore characters. + + ) # End capture. + """.trim.r + + val CallFullyQualifiedName = """ + (?x) # Turn on comments and whitespace insensitivity. + + ( # Begin outer capturing group for FQN. + + (?:[a-zA-Z][a-zA-Z0-9_]*) # Inner noncapturing group for top-level workflow name. This is the WDL + # identifier naming pattern of an initial alpha character followed by zero + # or more alphanumeric or underscore characters. + + (?:\.[a-zA-Z][a-zA-Z0-9_]*){1} # Inner noncapturing group for call name, a literal dot followed by a WDL + # identifier. Currently this is quantified to {1} since the call name is + # mandatory and nested workflows are not supported. This could be changed + # to + or a different quantifier if these assumptions change. + + ) # End outer capturing group for FQN. + + + (?: # Begin outer noncapturing group for shard. + \. # Literal dot. + (\d+) # Captured shard digits. + )? # End outer optional noncapturing group for shard. + """.trim.r // The trim is necessary as (?x) must be at the beginning of the regex. +} diff --git a/services/src/main/scala/cromwell/services/metadata/package.scala b/services/src/main/scala/cromwell/services/metadata/package.scala index a6ed193a2..f35408296 100644 --- a/services/src/main/scala/cromwell/services/metadata/package.scala +++ b/services/src/main/scala/cromwell/services/metadata/package.scala @@ -1,42 +1,5 @@ package cromwell.services package object metadata { - case class QueryParameter(key: String, value: String) type QueryParameters = Seq[QueryParameter] - - object Patterns { - val WorkflowName = """ - (?x) # Turn on comments and whitespace insensitivity. - - ( # Begin capture. - - [a-zA-Z][a-zA-Z0-9_]* # WDL identifier naming pattern of an initial alpha character followed by zero - # or more alphanumeric or underscore characters. - - ) # End capture. - """.trim.r - - val CallFullyQualifiedName = """ - (?x) # Turn on comments and whitespace insensitivity. - - ( # Begin outer capturing group for FQN. - - (?:[a-zA-Z][a-zA-Z0-9_]*) # Inner noncapturing group for top-level workflow name. This is the WDL - # identifier naming pattern of an initial alpha character followed by zero - # or more alphanumeric or underscore characters. - - (?:\.[a-zA-Z][a-zA-Z0-9_]*){1} # Inner noncapturing group for call name, a literal dot followed by a WDL - # identifier. Currently this is quantified to {1} since the call name is - # mandatory and nested workflows are not supported. This could be changed - # to + or a different quantifier if these assumptions change. - - ) # End outer capturing group for FQN. - - - (?: # Begin outer noncapturing group for shard. - \. # Literal dot. - (\d+) # Captured shard digits. - )? # End outer optional noncapturing group for shard. - """.trim.r // The trim is necessary as (?x) must be at the beginning of the regex. - } } diff --git a/src/main/scala/cromwell/Main.scala b/src/main/scala/cromwell/Main.scala index a54a982de..74a5405c1 100644 --- a/src/main/scala/cromwell/Main.scala +++ b/src/main/scala/cromwell/Main.scala @@ -10,7 +10,6 @@ import org.slf4j.LoggerFactory import scala.collection.JavaConverters._ import scala.concurrent.duration._ import scala.concurrent.{Await, Future} -import scala.language.postfixOps import scala.util.{Failure, Success} object Main extends App { @@ -74,7 +73,6 @@ object Main extends App { val runner = CromwellSystem.actorSystem.actorOf(runnerProps, "SingleWorkflowRunnerActor") import PromiseActor.EnhancedActorRef - import scala.concurrent.ExecutionContext.Implicits.global val promise = runner.askNoTimeout(RunWorkflow) waitAndExit(promise, CromwellSystem) diff --git a/src/test/scala/cromwell/CromwellCommandLineSpec.scala b/src/test/scala/cromwell/CromwellCommandLineSpec.scala index ec85f3ab7..42f03fdfc 100644 --- a/src/test/scala/cromwell/CromwellCommandLineSpec.scala +++ b/src/test/scala/cromwell/CromwellCommandLineSpec.scala @@ -6,7 +6,6 @@ import cromwell.util.SampleWdl import cromwell.util.SampleWdl.ThreeStep import org.scalatest.{FlatSpec, Matchers} -import scala.language.postfixOps import scala.util.Try class CromwellCommandLineSpec extends FlatSpec with Matchers { diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala index 46678ae25..400c6a55f 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala @@ -1,18 +1,16 @@ package cromwell.backend.impl.htcondor +import java.nio.file.FileSystems import java.nio.file.attribute.PosixFilePermission -import java.nio.file.{FileSystems, Files, Path, Paths} import java.util.UUID import akka.actor.{ActorRef, Props} -import better.files.File -import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse, FailedNonRetryableResponse, SucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, FailedNonRetryableResponse, SucceededResponse} import cromwell.backend._ import cromwell.backend.impl.htcondor.caching.CacheActor._ import cromwell.backend.impl.htcondor.caching.localization.CachedResultLocalization import cromwell.backend.io.JobPaths import cromwell.backend.sfs.{SharedFileSystem, SharedFileSystemExpressionFunctions} -import cromwell.core.{JobOutput, JobOutputs, LocallyQualifiedName} import cromwell.services.keyvalue.KeyValueServiceActor._ import cromwell.services.metadata.CallMetadataKeys import org.apache.commons.codec.digest.DigestUtils @@ -20,12 +18,11 @@ import wdl4s._ import wdl4s.parser.MemoryUnit import wdl4s.types.{WdlArrayType, WdlFileType} import wdl4s.util.TryUtil -import wdl4s.values.{WdlArray, WdlFile, WdlSingleFile, WdlValue} +import wdl4s.values.WdlArray import scala.concurrent.{Future, Promise} import scala.sys.process.ProcessLogger import scala.util.{Failure, Success, Try} -import scala.language.postfixOps object HtCondorJobExecutionActor { val HtCondorJobIdKey = "htCondor_job_id" @@ -110,12 +107,15 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor case JobExecutionResponse(resp) => log.debug("{}: Completing job [{}] with response: [{}]", tag, jobDescriptor.key, resp) executionResponse trySuccess resp + () case TrackTaskStatus(id) => // Avoid the redundant status check if the response is already completed (e.g. in case of abort) if (!executionResponse.isCompleted) trackTask(id) // Messages received from Caching actor - case ExecutionResultFound(succeededResponse) => executionResponse trySuccess localizeCachedResponse(succeededResponse) + case ExecutionResultFound(succeededResponse) => + executionResponse trySuccess localizeCachedResponse(succeededResponse) + () case ExecutionResultNotFound => prepareAndExecute() case ExecutionResultStored(hash) => log.debug("{} Cache entry was stored for Job with hash {}.", tag, hash) case ExecutionResultAlreadyExist => log.warning("{} Cache entry for hash {} already exist.", tag, jobHash) @@ -127,9 +127,13 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor case KvKeyLookupFailed(_) => log.debug("{} Job id not found. Falling back to execute.", tag) execute + // -Ywarn-value-discard + () case KvFailure(_, e) => log.error("{} Failure attempting to look up HtCondor job id. Exception message: {}. Falling back to execute.", tag, e.getMessage) execute + // -Ywarn-value-discard + () } /** @@ -223,6 +227,7 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor import scala.concurrent.duration._ // Job is still running in HtCondor. Check back again after `pollingInterval` seconds context.system.scheduler.scheduleOnce(pollingInterval.seconds, self, TrackTaskStatus(jobIdentifier)) + () case Success(Some(rc)) if runtimeAttributes.continueOnReturnCode.continueFor(rc) => self ! JobExecutionResponse(processSuccess(rc)) case Success(Some(rc)) => self ! JobExecutionResponse(FailedNonRetryableResponse(jobDescriptor.key, new IllegalStateException("Job exited with invalid return code: " + rc), Option(rc))) @@ -254,7 +259,7 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor log.error(ex.getCause, errMsg) throw new IllegalStateException(errMsg, ex.getCause) } - val str = Seq(cmd, + val str = Seq[Any](cmd, runtimeAttributes.failOnStderr, runtimeAttributes.dockerImage.getOrElse(""), runtimeAttributes.dockerWorkingDir.getOrElse(""), @@ -292,6 +297,7 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor ) cmds.generateSubmitFile(submitFilePath, attributes) // This writes the condor submit file + () } catch { case ex: Exception => @@ -352,11 +358,11 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor } private def prepareAndExecute(): Unit = { - Try { + try { createExecutionFolderAndScript() executeTask() - } recover { - case exception => self ! JobExecutionResponse(FailedNonRetryableResponse(jobDescriptor.key, exception, None)) + } catch { + case e: Exception => self ! JobExecutionResponse(FailedNonRetryableResponse(jobDescriptor.key, e, None)) } } diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala index 27364b8f2..76a81c08a 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala @@ -8,7 +8,6 @@ import cromwell.backend.impl.htcondor import cromwell.core.PathFactory.{EnhancedPath, FlushingAndClosingWriter} import cromwell.core.{TailedWriter, UntailedWriter} -import scala.language.postfixOps import scala.sys.process._ object JobStatus { @@ -62,6 +61,7 @@ class HtCondorCommands extends StrictLogging { |$instantiatedCommand |echo $$? > rc |""".stripMargin) + () } def generateSubmitFile(path: Path, attributes: Map[String, Any]): String = { @@ -84,7 +84,7 @@ class HtCondorProcess extends StrictLogging { private val stdout = new StringBuilder private val stderr = new StringBuilder - def processLogger: ProcessLogger = ProcessLogger(stdout append _, stderr append _) + def processLogger: ProcessLogger = ProcessLogger(s => { stdout append s; () }, s => { stderr append s; () }) def processStdout: String = stdout.toString().trim def processStderr: String = stderr.toString().trim def commandList(command: String): Seq[String] = Seq("/bin/bash",command) diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala index 45da26644..9e71d1a14 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala @@ -16,11 +16,11 @@ class HtCondorInitializationActorSpec extends TestKitSuite("HtCondorInitializati import BackendSpec._ val HelloWorld = - """ + s""" |task hello { | String addressee = "you" | command { - | echo "Hello ${addressee}!" + | echo "Hello $${addressee}!" | } | output { | String salutation = read_string(stdout()) diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala index 8a18ddf0d..589554e94 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala @@ -60,12 +60,12 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc """.stripMargin private val helloWorldWdlWithFileInput = - """ + s""" |task hello { | File inputFile | | command { - | echo ${inputFile} + | echo $${inputFile} | } | output { | String salutation = read_string(stdout()) @@ -79,12 +79,12 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc """.stripMargin private val helloWorldWdlWithFileArrayInput = - """ + s""" |task hello { | Array[File] inputFiles | | command { - | echo ${sep=' ' inputFiles} + | echo $${sep=' ' inputFiles} | } | output { | String salutation = read_string(stdout()) @@ -408,7 +408,10 @@ class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionAc cleanUpJob(jobPaths) } - private def cleanUpJob(jobPaths: JobPaths): Unit = File(jobPaths.workflowRoot).delete(true) + private def cleanUpJob(jobPaths: JobPaths): Unit = { + File(jobPaths.workflowRoot).delete(true) + () + } private def createCannedFile(prefix: String, contents: String, dir: Option[Path] = None): File = { val suffix = ".out" diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala index a411a7aef..db95a999e 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala @@ -17,11 +17,11 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { import BackendSpec._ val HelloWorld = - """ + s""" |task hello { | String addressee = "you" | command { - | echo "Hello ${addressee}!" + | echo "Hello $${addressee}!" | } | output { | String salutation = read_string(stdout()) @@ -61,7 +61,7 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { "return an instance of itself when tries to validate a valid Docker entry based on input" in { val expectedRuntimeAttributes = staticDefaults.copy(dockerImage = Option("you")) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { docker: "\${addressee}" }""").head + val runtimeAttributes = createRuntimeAttributes(HelloWorld, s"""runtime { docker: "\\$${addressee}" }""").head assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes) } @@ -84,7 +84,7 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { "return an instance of itself when tries to validate a valid docker working directory entry based on input" in { val expectedRuntimeAttributes = staticDefaults.copy(dockerWorkingDir = Option("you")) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { dockerWorkingDir: "\${addressee}" }""").head + val runtimeAttributes = createRuntimeAttributes(HelloWorld, s"""runtime { dockerWorkingDir: "\\$${addressee}" }""").head assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes) } @@ -107,7 +107,7 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { "return an instance of itself when tries to validate a valid docker output directory entry based on input" in { val expectedRuntimeAttributes = staticDefaults.copy(dockerOutputDir = Option("you")) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { dockerOutputDir: "\${addressee}" }""").head + val runtimeAttributes = createRuntimeAttributes(HelloWorld, s"""runtime { dockerOutputDir: "\\$${addressee}" }""").head assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes) } @@ -245,6 +245,7 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { } catch { case ex: RuntimeException => fail(s"Exception was not expected but received: ${ex.getMessage}") } + () } private def assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue], exMsg: String): Unit = { @@ -254,6 +255,7 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { } catch { case ex: RuntimeException => assert(ex.getMessage.contains(exMsg)) } + () } private def createRuntimeAttributes(wdlSource: WdlSource, runtimeAttributes: String): Seq[Map[String, WdlValue]] = { diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GenomicsFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GenomicsFactory.scala index 3427f3f09..7285171b5 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GenomicsFactory.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GenomicsFactory.scala @@ -6,7 +6,6 @@ import com.google.api.client.auth.oauth2.Credential import com.google.api.client.http.HttpTransport import com.google.api.client.json.JsonFactory import com.google.api.services.genomics.Genomics -import cromwell.filesystems.gcs.GoogleConfiguration object GenomicsFactory { diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala index 3b2a4bc11..b2e5974e0 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala @@ -6,6 +6,8 @@ import java.nio.file.{Path, Paths} import akka.actor.{Actor, ActorLogging, ActorRef, Props} import akka.event.LoggingReceive import better.files._ +import cats.instances.future._ +import cats.syntax.functor._ import com.google.api.client.googleapis.json.GoogleJsonResponseException import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse} import cromwell.backend.BackendLifecycleActor.AbortJobCommand @@ -277,7 +279,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes |echo $$? > $rcPath """.stripMargin.trim - def writeScript(): Future[Unit] = Future(File(jesCallPaths.gcsExecPath).write(fileContent)) + def writeScript(): Future[Unit] = Future { File(jesCallPaths.gcsExecPath).write(fileContent) } void implicit val system = context.system Retry.withRetry( @@ -291,7 +293,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes descriptor.workflowOptions.getOrElse(WorkflowOptionKeys.GoogleProject, jesAttributes.project) } - private def createJesRun(jesParameters: Seq[JesParameter], runIdForResumption: Option[String] = None): Future[Run] = { + private def createJesRun(jesParameters: Seq[JesParameter], runIdForResumption: Option[String]): Future[Run] = { def createRun() = Future(Run( runIdForResumption, @@ -423,7 +425,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes fileMetadata += JesMetadataKeys.MonitoringLog -> monitoringOutput.get.gcs } - val otherMetadata = Map( + val otherMetadata: Map[String, Any] = Map( JesMetadataKeys.GoogleProject -> jesAttributes.project, JesMetadataKeys.ExecutionBucket -> jesAttributes.executionBucket, JesMetadataKeys.EndpointUrl -> jesAttributes.endpointUrl, diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala index 8fd05c348..3c72ca7fc 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala @@ -5,7 +5,6 @@ import java.net.URL import cats.data._ import cats.data.Validated._ import cats.syntax.cartesian._ -import cats.syntax.validated._ import com.typesafe.config.Config import cromwell.backend.impl.jes.JesImplicits.GoogleAuthWorkflowOptions import cromwell.core.WorkflowOptions @@ -15,8 +14,6 @@ import lenthall.config.ValidatedConfig._ import cromwell.core.ErrorOr._ import wdl4s.ExceptionWithErrors -import scala.language.postfixOps - case class JesAttributes(project: String, genomicsAuth: GoogleAuthMode, gcsFilesystemAuth: GoogleAuthMode, diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala index 3f498ff71..1e87b4d1f 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala @@ -13,7 +13,6 @@ import cromwell.core.{ExecutionStore, OutputStore} import wdl4s.Call import wdl4s.expression.WdlStandardLibraryFunctions -import scala.language.postfixOps case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory { diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala index 9b50bb8e2..038c615dd 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala @@ -4,12 +4,15 @@ import java.nio.file.Path import akka.actor.Props import better.files._ +import cats.instances.future._ +import cats.syntax.functor._ import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor, BackendWorkflowFinalizationActor} import cromwell.core.Dispatcher.IoDispatcher import cromwell.core.{ExecutionStore, OutputStore, PathCopier} import wdl4s.Call import scala.concurrent.Future +import scala.language.postfixOps object JesFinalizationActor { def props(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], jesConfiguration: JesConfiguration, @@ -40,7 +43,7 @@ class JesFinalizationActor (override val workflowDescriptor: BackendWorkflowDesc private def deleteAuthenticationFile(): Future[Unit] = { (jesConfiguration.needAuthFileUpload, workflowPaths) match { - case (true, Some(paths)) => Future(File(paths.gcsAuthFilePath).delete(false)) map { _ => () } + case (true, Some(paths)) => Future { File(paths.gcsAuthFilePath).delete(false) } void case _ => Future.successful(()) } } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala index 04ac8f70d..e76a62e9d 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala @@ -3,6 +3,8 @@ package cromwell.backend.impl.jes import java.io.IOException import akka.actor.{ActorRef, Props} +import cats.instances.future._ +import cats.syntax.functor._ import com.google.api.services.genomics.Genomics import cromwell.backend.impl.jes.JesInitializationActor._ import cromwell.backend.impl.jes.authentication.{GcsLocalizing, JesAuthInformation, JesCredentials} @@ -15,9 +17,9 @@ import cromwell.core.WorkflowOptions import cromwell.core.retry.Retry import cromwell.filesystems.gcs.{ClientSecrets, GoogleAuthMode} import spray.json.JsObject +import wdl4s.Call import wdl4s.types.{WdlBooleanType, WdlFloatType, WdlIntegerType, WdlStringType} import wdl4s.values.WdlValue -import wdl4s.{Call, WdlExpression} import scala.concurrent.Future import scala.util.Try @@ -96,7 +98,7 @@ class JesInitializationActor(override val workflowDescriptor: BackendWorkflowDes val upload = () => Future(path.writeAsJson(content)) workflowLogger.info(s"Creating authentication file for workflow ${workflowDescriptor.id} at \n ${path.toString}") - Retry.withRetry(upload, isFatal = isFatalJesException, isTransient = isTransientJesException)(context.system) map { _ => () } recoverWith { + Retry.withRetry(upload, isFatal = isFatalJesException, isTransient = isTransientJesException)(context.system).void.recoverWith { case failure => Future.failed(new IOException("Failed to upload authentication file", failure)) } } getOrElse Future.successful(()) diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala index acfa8c6b2..4ded3e9d1 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala @@ -71,7 +71,7 @@ trait JesJobCachingActorHelper extends JobCachingActorHelper { fileMetadata += JesMetadataKeys.MonitoringLog -> monitoringOutput.get.gcs } - val otherMetadata = Map( + val otherMetadata: Map[String, Any] = Map( JesMetadataKeys.GoogleProject -> jesAttributes.project, JesMetadataKeys.ExecutionBucket -> jesAttributes.executionBucket, JesMetadataKeys.EndpointUrl -> jesAttributes.endpointUrl, diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala index 594b7f0c2..eed3268f5 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala @@ -12,7 +12,6 @@ import cromwell.services.keyvalue.KeyValueServiceActor._ import org.slf4j.LoggerFactory import scala.concurrent.{Future, Promise} -import scala.language.postfixOps object JesJobExecutionActor { val logger = LoggerFactory.getLogger("JesBackend") diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala index 19891ee61..593d1e6df 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala @@ -12,7 +12,6 @@ import cromwell.backend.validation._ import cromwell.core._ import lenthall.exception.MessageAggregation import cromwell.core.ErrorOr._ -import mouse.string._ import org.slf4j.Logger import wdl4s.types._ import wdl4s.values._ diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala index b24f7affe..0f948344c 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala @@ -6,7 +6,6 @@ import java.util.{ArrayList => JArrayList} import com.google.api.client.util.{ArrayMap => GArrayMap} import com.google.api.services.genomics.Genomics import com.google.api.services.genomics.model._ -import com.typesafe.config.ConfigFactory import cromwell.backend.BackendJobDescriptor import cromwell.backend.impl.jes.RunStatus.{Failed, Initializing, Running, Success} import cromwell.core.ExecutionEvent @@ -14,7 +13,6 @@ import cromwell.core.logging.JobLogger import org.slf4j.LoggerFactory import scala.collection.JavaConverters._ -import scala.concurrent.duration._ import scala.language.postfixOps object Run { @@ -156,13 +154,12 @@ case class Run(runId: String, genomicsInterface: Genomics) { } private def eventIfExists(name: String, metadata: Map[String, AnyRef], eventName: String): Option[ExecutionEvent] = { - metadata.get(name) map { - case time => ExecutionEvent(eventName, OffsetDateTime.parse(time.toString)) - } + metadata.get(name) map { time => ExecutionEvent(eventName, OffsetDateTime.parse(time.toString)) } } def abort(): Unit = { val cancellationRequest: CancelOperationRequest = new CancelOperationRequest() genomicsInterface.operations().cancel(runId, cancellationRequest).execute + () } } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala index 848b188c1..c1ce80b4f 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala @@ -7,7 +7,6 @@ import cats.syntax.cartesian._ import cats.syntax.validated._ import com.google.api.services.genomics.model.Disk import cromwell.core.ErrorOr._ -import mouse.string._ import wdl4s.ExceptionWithErrors import wdl4s.values._ diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala index a4a2b0ba8..24d417c99 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala @@ -3,8 +3,6 @@ package cromwell.backend.impl.jes import java.nio.file.{Files, Path} import com.google.api.client.http.HttpResponseException -import cromwell.backend.BackendWorkflowDescriptor -import cromwell.backend.impl.jes.JesImplicits.GoogleAuthWorkflowOptions import cromwell.filesystems.gcs._ package object io { diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala index d2527df6e..2a2ae3ec9 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala @@ -10,16 +10,15 @@ import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse import cromwell.backend.async.AsyncBackendJobExecutionActor.{Execute, ExecutionMode} import cromwell.backend.async.{AbortedExecutionHandle, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle} import cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.JesPendingExecutionHandle +import cromwell.backend.impl.jes.MockObjects._ import cromwell.backend.impl.jes.RunStatus.Failed import cromwell.backend.impl.jes.io.{DiskType, JesWorkingDisk} import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobDescriptorKey, BackendWorkflowDescriptor, PreemptedException, RuntimeAttributeDefinition} -import cromwell.core._ import cromwell.core.logging.LoggerWrapper +import cromwell.core.{WorkflowId, WorkflowOptions, _} import cromwell.filesystems.gcs._ import cromwell.util.SampleWdl import org.scalatest._ -import cromwell.core.{WorkflowId, WorkflowOptions} -import cromwell.filesystems.gcs.GoogleAuthMode.GoogleAuthOptions import org.scalatest.prop.Tables.Table import org.slf4j.Logger import org.specs2.mock.Mockito @@ -31,7 +30,6 @@ import wdl4s.{Call, LocallyQualifiedName, NamespaceWithWorkflow} import scala.concurrent.duration._ import scala.concurrent.{Await, ExecutionContext, Future, Promise} import scala.util.{Success, Try} -import cromwell.backend.impl.jes.MockObjects._ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackendJobExecutionActorSpec") with FlatSpecLike with Matchers with ImplicitSender with Mockito { @@ -41,11 +39,11 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend implicit val Timeout = 5.seconds.dilated val YoSup = - """ + s""" |task sup { | String addressee | command { - | echo "yo sup ${addressee}!" + | echo "yo sup $${addressee}!" | } | output { | String salutation = read_string(stdout()) @@ -72,15 +70,6 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend } private def buildInitializationData(jobDescriptor: BackendJobDescriptor, configuration: JesConfiguration) = { - def gcsFileSystem = { - val authOptions = new GoogleAuthOptions { - override def get(key: String): Try[String] = Try(throw new RuntimeException(s"key '$key' not found")) - } - - val storage = jesConfiguration.jesAttributes.gcsFilesystemAuth.buildStorage(authOptions, "appName") - GcsFileSystem(GcsFileSystemProvider(storage)(scala.concurrent.ExecutionContext.global)) - } - val workflowPaths = JesWorkflowPaths(jobDescriptor.workflowDescriptor, configuration, mockCredentials)(scala.concurrent.ExecutionContext.global) diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala index 2669552b9..4699402bc 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala @@ -25,11 +25,11 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe import BackendSpec._ val HelloWorld = - """ + s""" |task hello { | String addressee = "you" | command { - | echo "Hello ${addressee}!" + | echo "Hello $${addressee}!" | } | output { | String salutation = read_string(stdout()) diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala index efbcb53e6..204907110 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala @@ -2,7 +2,7 @@ package cromwell.backend.impl.jes import cromwell.backend.impl.jes.io.{DiskType, JesAttachedDisk, JesWorkingDisk} import cromwell.backend.validation.ContinueOnReturnCodeSet -import cromwell.backend.{BackendSpec, MemorySize, RuntimeAttributeDefinition} +import cromwell.backend.{MemorySize, RuntimeAttributeDefinition} import cromwell.core.WorkflowOptions import org.scalatest.{Matchers, WordSpecLike} import org.slf4j.helpers.NOPLogger @@ -190,11 +190,11 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito { private def assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes: Map[String, WdlValue], expectedRuntimeAttributes: JesRuntimeAttributes, workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = { val withDefaults = RuntimeAttributeDefinition.addDefaultsToAttributes(JesBackendLifecycleActorFactory.staticRuntimeAttributeDefinitions, workflowOptions) _ try { - assert(JesRuntimeAttributes(withDefaults(runtimeAttributes), NOPLogger.NOP_LOGGER) == expectedRuntimeAttributes) } catch { case ex: RuntimeException => fail(s"Exception was not expected but received: ${ex.getMessage}") } + () } private def assertJesRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue], exMsg: String, workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = { @@ -205,6 +205,7 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito { } catch { case ex: RuntimeException => assert(ex.getMessage.contains(exMsg)) } + () } private val emptyWorkflowOptions = WorkflowOptions.fromMap(Map.empty).get diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala index cf6fcea8c..ad9761bca 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala @@ -57,6 +57,7 @@ sealed trait ConfigAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecut s"""|#!/bin/bash |$command |""".stripMargin) + () } /** diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala index c17ebe269..8ac1e2c21 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala @@ -46,6 +46,7 @@ trait BackgroundAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecution | & |echo $$! |""".stripMargin) + () } override def getJob(exitValue: Int, stdout: Path, stderr: Path) = { @@ -80,5 +81,6 @@ trait BackgroundAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecution | |kill_children ${job.jobId} |""".stripMargin) + () } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/ProcessRunner.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/ProcessRunner.scala index 509ec01ba..bf6bebb28 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/ProcessRunner.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/ProcessRunner.scala @@ -17,7 +17,7 @@ class ProcessRunner(val argv: Seq[Any], val stdoutPath: Path, val stderrPath: Pa processBuilder.command(argv.map(_.toString): _*) processBuilder.redirectOutput(stdoutPath.toFile) processBuilder.redirectError(stderrPath.toFile) - val proccess = processBuilder.start() - proccess.waitFor() + val process = processBuilder.start() + process.waitFor() } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala index 3d81722c6..20c8c3d39 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala @@ -2,6 +2,8 @@ package cromwell.backend.sfs import java.nio.file.{FileSystem, Path, Paths} +import cats.instances.try_._ +import cats.syntax.functor._ import com.typesafe.config.Config import cromwell.backend.io.JobPaths import cromwell.core._ @@ -42,21 +44,26 @@ object SharedFileSystem { private def localizePathViaCopy(originalPath: File, executionPath: File): Try[Unit] = { executionPath.parent.createDirectories() val executionTmpPath = pathPlusSuffix(executionPath, ".tmp") - Try(originalPath.copyTo(executionTmpPath, overwrite = true).moveTo(executionPath, overwrite = true)) + Try(originalPath.copyTo(executionTmpPath, overwrite = true).moveTo(executionPath, overwrite = true)).void } private def localizePathViaHardLink(originalPath: File, executionPath: File): Try[Unit] = { executionPath.parent.createDirectories() // link.linkTo(target) returns target, // however we want to return the link, not the target, so map the result back to executionPath - Try(executionPath.linkTo(originalPath, symbolic = false)) map { _ => executionPath } + + // -Ywarn-value-discard + // Try(executionPath.linkTo(originalPath, symbolic = false)) map { _ => executionPath } + Try { executionPath.linkTo(originalPath, symbolic = false) } void } private def localizePathViaSymbolicLink(originalPath: File, executionPath: File): Try[Unit] = { if (originalPath.isDirectory) Failure(new UnsupportedOperationException("Cannot localize directory with symbolic links")) else { executionPath.parent.createDirectories() - Try(executionPath.linkTo(originalPath, symbolic = true)) map { _ => executionPath } + // -Ywarn-value-discard + // Try(executionPath.linkTo(originalPath, symbolic = true)) map { _ => executionPath } + Try { executionPath.linkTo(originalPath, symbolic = true) } void } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala index 6aa913658..6b5107b69 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala @@ -305,6 +305,7 @@ trait SharedFileSystemAsyncJobExecutionActor val stderr = pathPlusSuffix(jobPaths.stderr, "kill") val killer = new ProcessRunner(argv, stdout.path, stderr.path) killer.run() + () } def processReturnCode()(implicit ec: ExecutionContext): Future[ExecutionHandle] = { diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala index be4ab0ef9..62dc98b07 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala @@ -19,5 +19,9 @@ class SharedFileSystemCacheHitCopyingActor(override val jobDescriptor: BackendJo override protected def getPath(file: String) = Paths.get(file) - override protected def duplicate(source: Path, destination: Path) = sharedFileSystem.cacheCopy(source, destination) + override protected def duplicate(source: Path, destination: Path) = { + // -Ywarn-value-discard + sharedFileSystem.cacheCopy(source, destination) + () + } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala index 36b010547..54a370b39 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala @@ -7,8 +7,8 @@ import cromwell.backend.validation.RuntimeAttributesDefault import cromwell.backend.wfs.{DefaultWorkflowFileSystemProvider, WorkflowFileSystemProvider} import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor} import cromwell.core.{Dispatcher, WorkflowOptions} +import wdl4s.Call import wdl4s.values.WdlValue -import wdl4s.{Call, WdlExpression} import scala.concurrent.Future import scala.util.Try diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala index 17ea69fc7..81edb6f60 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala @@ -29,6 +29,7 @@ class ConfigHashingStrategySpec extends FlatSpec with Matchers with TableDrivenP override def beforeAll() = { file.write(steak) + () } private def randomName(): String = UUID.randomUUID().toString @@ -140,5 +141,6 @@ class ConfigHashingStrategySpec extends FlatSpec with Matchers with TableDrivenP override def afterAll() = { file.delete(true) md5File.delete(true) + () } } diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala index ac04a0bd9..64dbfa9d7 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala @@ -17,11 +17,11 @@ class SharedFileSystemInitializationActorSpec extends TestKitSuite("SharedFileSy val Timeout = 5.second.dilated val HelloWorld = - """ + s""" |task hello { | String addressee = "you" | command { - | echo "Hello ${addressee}!" + | echo "Hello $${addressee}!" | } | output { | String salutation = read_string(stdout()) diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemValidatedRuntimeAttributesBuilderSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemValidatedRuntimeAttributesBuilderSpec.scala index 2934eb040..c257ffa4d 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemValidatedRuntimeAttributesBuilderSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemValidatedRuntimeAttributesBuilderSpec.scala @@ -1,6 +1,5 @@ package cromwell.backend.sfs -import cromwell.backend.BackendSpec._ import cromwell.backend.RuntimeAttributeDefinition import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.backend.validation._ @@ -14,11 +13,11 @@ import wdl4s.values.{WdlBoolean, WdlInteger, WdlString, WdlValue} class SharedFileSystemValidatedRuntimeAttributesBuilderSpec extends WordSpecLike with Matchers with Mockito { val HelloWorld = - """ + s""" |task hello { | String addressee = "you" | command { - | echo "Hello ${addressee}!" + | echo "Hello $${addressee}!" | } | output { | String salutation = read_string(stdout()) @@ -33,7 +32,7 @@ class SharedFileSystemValidatedRuntimeAttributesBuilderSpec extends WordSpecLike """.stripMargin - val defaultRuntimeAttributes = Map( + val defaultRuntimeAttributes: Map[String, Any] = Map( DockerKey -> None, FailOnStderrKey -> false, ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(0))) @@ -170,6 +169,7 @@ class SharedFileSystemValidatedRuntimeAttributesBuilderSpec extends WordSpecLike failOnStderr should be(expectedRuntimeAttributes(FailOnStderrKey).asInstanceOf[Boolean]) continueOnReturnCode should be( expectedRuntimeAttributes(ContinueOnReturnCodeKey).asInstanceOf[ContinueOnReturnCode]) + () } private def assertRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue], exMsg: String, @@ -188,5 +188,6 @@ class SharedFileSystemValidatedRuntimeAttributesBuilderSpec extends WordSpecLike builder.build(addDefaultsToAttributes(runtimeAttributes), logger) } thrown.getMessage should include(exMsg) + () } } diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkClusterProcess.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkClusterProcess.scala index aa9575fd4..f731c990f 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkClusterProcess.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkClusterProcess.scala @@ -12,7 +12,6 @@ import scala.concurrent.{ExecutionContext, Future, Promise} import better.files._ import com.typesafe.scalalogging.Logger import org.slf4j.LoggerFactory -import spray.httpx.unmarshalling._ import scala.concurrent.duration._ import scala.util.{Failure, Success, Try} @@ -119,7 +118,9 @@ class SparkClusterProcess(implicit system: ActorSystem) extends SparkProcess override def completeMonitoringProcess(rcPath: Path, status: String, promise: Promise[Unit]) = { File(rcPath) write status - promise success Unit + val unitValue = () + promise success unitValue + () } def pollForJobStatus(subId: String): Future[SparkDriverStateQueryResponse] = { diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala index c03a2975d..2c4b5f94f 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala @@ -1,14 +1,14 @@ package cromwell.backend.impl.spark import akka.actor.{ActorRef, Props} +import cromwell.backend.impl.spark.SparkInitializationActor._ import cromwell.backend.validation.RuntimeAttributesDefault import cromwell.backend.validation.RuntimeAttributesKeys._ -import cromwell.backend.impl.spark.SparkInitializationActor._ import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor} import cromwell.core.WorkflowOptions +import wdl4s.Call import wdl4s.types.{WdlBooleanType, WdlIntegerType, WdlStringType} import wdl4s.values.WdlValue -import wdl4s.{Call, WdlExpression} import scala.concurrent.Future import scala.util.Try diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala index 3d34dda47..c793338eb 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala @@ -16,7 +16,6 @@ import wdl4s.util.TryUtil import scala.concurrent.{Future, Promise} import scala.sys.process.ProcessLogger import scala.util.{Failure, Success, Try} -import scala.language.postfixOps object SparkJobExecutionActor { val DefaultFileSystems = List(FileSystems.getDefault) @@ -145,7 +144,9 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, /** * Abort a running job. */ - override def abort(): Unit = Future.failed(new UnsupportedOperationException("SparkBackend currently doesn't support aborting jobs.")) + // -Ywarn-value-discard + // override def abort(): Unit = Future.failed(new UnsupportedOperationException("SparkBackend currently doesn't support aborting jobs.")) + override def abort(): Unit = throw new UnsupportedOperationException("SparkBackend currently doesn't support aborting jobs.") private def createExecutionFolderAndScript(): Unit = { @@ -178,11 +179,14 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, cmds.writeScript(sparkCommand, scriptPath, executionDir) File(scriptPath).addPermission(PosixFilePermission.OWNER_EXECUTE) + () } catch { case ex: Exception => log.error(ex, "Failed to prepare task: " + ex.getMessage) - executionResponse success FailedNonRetryableResponse(jobDescriptor.key, ex, None) + // -Ywarn-value-discard + // executionResponse success FailedNonRetryableResponse(jobDescriptor.key, ex, None) + () } } diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala index d04041f20..60f399218 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala @@ -8,7 +8,6 @@ import cromwell.core.PathFactory.EnhancedPath import scala.sys.process._ import better.files._ -import scala.language.postfixOps import scala.util.{Failure, Success, Try} object SparkCommands { diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkClusterProcessSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkClusterProcessSpec.scala index c2b003a53..f4dbbe1d4 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkClusterProcessSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkClusterProcessSpec.scala @@ -21,8 +21,6 @@ import cromwell.backend.impl.spark.SparkClusterProcess.{Failed, _} import org.scalatest.concurrent.ScalaFutures import spray.http._ import SparkClusterJsonProtocol._ -import spray.httpx.unmarshalling._ -import spray.httpx.SprayJsonSupport._ class SparkClusterProcessSpec extends TestKitSuite("SparkClusterProcess") with WordSpecLike diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala index fac5f389b..aa4400ac8 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala @@ -144,7 +144,10 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") Mockito.reset(sparkClusterProcess) } - override def afterAll(): Unit = system.terminate() + override def afterAll(): Unit = { + system.terminate() + () + } "executeTask method in cluster deploy mode " should { "return succeed response when the spark cluster process monitor method returns finished status" in { @@ -435,7 +438,10 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") } - private def cleanUpJob(jobPaths: JobPaths): Unit = File(jobPaths.workflowRoot).delete(true) + private def cleanUpJob(jobPaths: JobPaths): Unit = { + File(jobPaths.workflowRoot).delete(true) + () + } private def prepareJob(wdlSource: WdlSource = helloWorldWdl, runtimeString: String = passOnStderr, inputFiles: Option[Map[String, WdlValue]] = None, isCluster: Boolean = false): TestJobDescriptor = { val backendWorkflowDescriptor = buildWorkflowDescriptor(wdl = wdlSource, inputs = inputFiles.getOrElse(Map.empty), runtime = runtimeString) diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala index d166dca11..33724cb6d 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala @@ -1,15 +1,15 @@ package cromwell.backend.impl.spark -import cromwell.backend.{MemorySize, BackendWorkflowDescriptor} import cromwell.backend.validation.RuntimeAttributesKeys._ +import cromwell.backend.{BackendWorkflowDescriptor, MemorySize} import cromwell.core.{WorkflowId, WorkflowOptions} import org.scalatest.{Matchers, WordSpecLike} -import spray.json.{JsBoolean, JsNumber, JsObject, JsString, JsValue} +import spray.json.{JsBoolean, JsNumber, JsObject, JsValue} import wdl4s.WdlExpression._ import wdl4s.expression.NoFunctions import wdl4s.util.TryUtil -import wdl4s.{Call, WdlExpression, _} import wdl4s.values.WdlValue +import wdl4s.{Call, WdlExpression, _} class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { @@ -33,7 +33,7 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { val emptyWorkflowOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue])) - val staticDefaults = SparkRuntimeAttributes(1, MemorySize.parse("1 GB").get, None, "com.test.spark" , false) + val staticDefaults = SparkRuntimeAttributes(1, MemorySize.parse("1 GB").get, None, "com.test.spark" , failOnStderr = false) def workflowOptionsWithDefaultRA(defaults: Map[String, JsValue]) = { WorkflowOptions(JsObject(Map( @@ -88,7 +88,7 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { inputs: Map[String, WdlValue] = Map.empty, options: WorkflowOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue])), runtime: String) = { - new BackendWorkflowDescriptor( + BackendWorkflowDescriptor( WorkflowId.randomId(), NamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime.format("appMainClass", "com.test.spark"))), inputs, @@ -118,6 +118,7 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { } catch { case ex: RuntimeException => fail(s"Exception was not expected but received: ${ex.getMessage}") } + () } private def assertSparkRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue], exMsg: String): Unit = { @@ -127,5 +128,6 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { } catch { case ex: RuntimeException => assert(ex.getMessage.contains(exMsg)) } + () } } From 7b5274338f4bf238f28a6d48b82e0a2f451f0135 Mon Sep 17 00:00:00 2001 From: Ruchi Date: Fri, 7 Oct 2016 08:17:10 -0400 Subject: [PATCH 16/23] Print PID to stdout.background, SGE (#1520) * fix pid being added to stdout for sge + change file extension from background -> submit for SGE stdout/stderr --- .../cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala index 6b5107b69..3013b12af 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala @@ -231,7 +231,9 @@ trait SharedFileSystemAsyncJobExecutionActor * @return A process runner that will relatively quickly submit the script asynchronously. */ def makeProcessRunner(): ProcessRunner = { - new ProcessRunner(processArgs.argv, jobPaths.stdout, jobPaths.stderr) + val stdout = pathPlusSuffix(jobPaths.stdout, "submit") + val stderr = pathPlusSuffix(jobPaths.stderr, "submit") + new ProcessRunner(processArgs.argv, stdout.path, stderr.path) } /** From 89faa14ef2e9e94ad6e290a833c18b719014f92a Mon Sep 17 00:00:00 2001 From: Miguel Covarrubias Date: Fri, 7 Oct 2016 16:08:29 -0400 Subject: [PATCH 17/23] Firstly, you must find... another shrubbery! Closes #1211 --- .../main/scala/cromwell/backend/io/WorkflowPaths.scala | 4 ++-- .../backend/wfs/WorkflowFileSystemProvider.scala | 6 +++--- .../scala/cromwell/backend/io/WorkflowPathsSpec.scala | 1 + .../main/scala/cromwell/core/DockerCredentials.scala | 14 +++++++------- .../scala/cromwell/core/logging/WorkflowLogger.scala | 10 +++++----- .../scala/cromwell/database/slick/SlickDatabase.scala | 10 +++++----- .../main/scala/cromwell/engine/EngineFilesystems.scala | 4 ++-- .../cromwell/engine/backend/BackendConfiguration.scala | 5 ++--- .../engine/workflow/WorkflowManagerActor.scala | 11 +++++------ .../lifecycle/MaterializeWorkflowDescriptorActor.scala | 8 ++++---- .../lifecycle/execution/WorkflowExecutionActor.scala | 4 ++-- .../main/scala/cromwell/server/CromwellRootActor.scala | 5 ++--- .../main/scala/cromwell/server/CromwellServer.scala | 4 ++-- .../filesystems/gcs/GcsFileSystemProvider.scala | 18 +++++++++++++----- project/Dependencies.scala | 3 ++- project/Settings.scala | 1 + .../scala/cromwell/services/ServiceRegistryActor.scala | 8 +++----- .../main/scala/cromwell/services/ServicesStore.scala | 4 ++-- .../services/metadata/impl/MetadataServiceActor.scala | 5 ++--- .../cromwell/backend/impl/jes/JesAttributes.scala | 4 ++-- .../config/ConfigBackendLifecycleActorFactory.scala | 9 +++++---- .../impl/sfs/config/ConfigHashingStrategy.scala | 6 +++--- .../backend/impl/sfs/config/ConfigWdlNamespace.scala | 12 ++++++------ .../backend/sfs/GcsWorkflowFileSystemProvider.scala | 4 ++-- .../sfs/SharedFileSystemJobCachingActorHelper.scala | 5 +++-- 25 files changed, 86 insertions(+), 79 deletions(-) diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala index d05797307..23bdae992 100644 --- a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala +++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala @@ -5,14 +5,14 @@ import java.nio.file.{FileSystem, FileSystems, Path, Paths} import com.typesafe.config.Config import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} import cromwell.core.PathFactory -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ object WorkflowPaths{ val DockerRoot = Paths.get("/root") } class WorkflowPaths(workflowDescriptor: BackendWorkflowDescriptor, config: Config, val fileSystems: List[FileSystem] = List(FileSystems.getDefault)) extends PathFactory { - val executionRoot = Paths.get(config.getStringOr("root", "cromwell-executions")).toAbsolutePath + val executionRoot = Paths.get(config.as[Option[String]]("root").getOrElse("cromwell-executions")).toAbsolutePath private def workflowPathBuilder(root: Path) = { root.resolve(workflowDescriptor.workflowNamespace.workflow.unqualifiedName) diff --git a/backend/src/main/scala/cromwell/backend/wfs/WorkflowFileSystemProvider.scala b/backend/src/main/scala/cromwell/backend/wfs/WorkflowFileSystemProvider.scala index 720605482..de8272473 100644 --- a/backend/src/main/scala/cromwell/backend/wfs/WorkflowFileSystemProvider.scala +++ b/backend/src/main/scala/cromwell/backend/wfs/WorkflowFileSystemProvider.scala @@ -2,11 +2,11 @@ package cromwell.backend.wfs import java.nio.file.FileSystem -import com.typesafe.config.Config +import com.typesafe.config.{Config, ConfigFactory} import cromwell.backend.io.WorkflowPaths import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor} import cromwell.core.WorkflowOptions -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ import scala.concurrent.ExecutionContext @@ -16,7 +16,7 @@ object WorkflowFileSystemProvider { providers: Traversable[WorkflowFileSystemProvider], fileSystemExecutionContext: ExecutionContext): WorkflowPaths = { val backendConfig = configurationDescriptor.backendConfig - val fileSystemConfig = backendConfig.getConfigOr("filesystems") + val fileSystemConfig = backendConfig.as[Option[Config]]("filesystems").getOrElse(ConfigFactory.empty()) val globalConfig = configurationDescriptor.globalConfig val params = WorkflowFileSystemProviderParams(fileSystemConfig, globalConfig, workflowDescriptor.workflowOptions, fileSystemExecutionContext) diff --git a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala index ff56ebaeb..bfae5930d 100644 --- a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala +++ b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala @@ -13,6 +13,7 @@ class WorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec with Moc val backendConfig = mock[Config] "WorkflowPaths" should "provide correct paths for a workflow" in { + when(backendConfig.hasPath(any[String])).thenReturn(true) when(backendConfig.getString(any[String])).thenReturn("local-cromwell-executions") // This is the folder defined in the config as the execution root dir val wd = buildWorkflowDescriptor(TestWorkflows.HelloWorld) val workflowPaths = new WorkflowPaths(wd, backendConfig) diff --git a/core/src/main/scala/cromwell/core/DockerCredentials.scala b/core/src/main/scala/cromwell/core/DockerCredentials.scala index a63076e2d..cfe7be01f 100644 --- a/core/src/main/scala/cromwell/core/DockerCredentials.scala +++ b/core/src/main/scala/cromwell/core/DockerCredentials.scala @@ -16,23 +16,23 @@ case class DockerConfiguration(dockerCredentials: Option[DockerCredentials], doc * Singleton encapsulating a DockerConf instance. */ object DockerConfiguration { - import lenthall.config.ScalaConfig._ private val dockerKeys = Set("account", "token") def build(config: Config) = { + import net.ceedubs.ficus.Ficus._ val dockerConf: Option[DockerCredentials] = for { - dockerConf <- config.getConfigOption("dockerhub") + dockerConf <- config.as[Option[Config]]("dockerhub") _ = dockerConf.warnNotRecognized(dockerKeys, "dockerhub") account <- dockerConf.validateString("account").toOption token <- dockerConf.validateString("token").toOption - } yield new DockerCredentials(account, token) + } yield DockerCredentials(account, token) val dockerHubConf = { - new DockerHubConfiguration( - namespace = config.getStringOr("docker.hub.namespace", "docker.io"), - v1Registry = config.getStringOr("docker.hub.v1Registry", "index.docker.io"), - v2Registry = config.getStringOr("docker.hub.v2Registry", "registry-1.docker.io") + DockerHubConfiguration( + namespace = config.as[Option[String]]("docker.hub.namespace").getOrElse("docker.io"), + v1Registry = config.as[Option[String]]("docker.hub.v1Registry").getOrElse("index.docker.io"), + v2Registry = config.as[Option[String]]("docker.hub.v2Registry").getOrElse("registry-1.docker.io") ) } new DockerConfiguration(dockerConf, dockerHubConf) diff --git a/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala b/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala index b36dd58a9..b79e9f7b6 100644 --- a/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala +++ b/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala @@ -8,9 +8,9 @@ import ch.qos.logback.classic.encoder.PatternLayoutEncoder import ch.qos.logback.classic.spi.ILoggingEvent import ch.qos.logback.classic.{Level, LoggerContext} import ch.qos.logback.core.FileAppender -import com.typesafe.config.ConfigFactory +import com.typesafe.config.{Config, ConfigFactory} import cromwell.core.WorkflowId -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ import org.slf4j.helpers.NOPLogger import org.slf4j.{Logger, LoggerFactory} @@ -71,9 +71,9 @@ object WorkflowLogger { val workflowLogConfiguration: Option[WorkflowLogConfiguration] = { for { - workflowConfig <- conf.getConfigOption("workflow-options") - dir <- workflowConfig.getStringOption("workflow-log-dir") if !dir.isEmpty - temporary <- workflowConfig.getBooleanOption("workflow-log-temporary") orElse Option(true) + workflowConfig <- conf.as[Option[Config]]("workflow-options") + dir <- workflowConfig.as[Option[String]]("workflow-log-dir") if !dir.isEmpty + temporary <- workflowConfig.as[Option[Boolean]]("workflow-log-temporary") orElse Option(true) } yield WorkflowLogConfiguration(Paths.get(dir).toAbsolutePath, temporary) } diff --git a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala index fbfbece8a..80a4413fd 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala @@ -6,7 +6,7 @@ import java.util.concurrent.{ExecutorService, Executors} import com.typesafe.config.Config import cromwell.database.slick.tables.DataAccessComponent import cromwell.database.sql.SqlDatabase -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ import org.slf4j.LoggerFactory import slick.backend.DatabaseConfig import slick.driver.JdbcProfile @@ -38,7 +38,7 @@ object SlickDatabase { // generate unique schema instances that don't conflict. // // Otherwise, create one DataAccess and hold on to the reference. - if (slickDatabase.databaseConfig.getBooleanOr("slick.createSchema", default = true)) { + if (slickDatabase.databaseConfig.as[Option[Boolean]]("slick.createSchema").getOrElse(true)) { import slickDatabase.dataAccess.driver.api._ Await.result(slickDatabase.database.run(slickDatabase.dataAccess.schema.create), Duration.Inf) } @@ -89,9 +89,9 @@ class SlickDatabase(override val originalDatabaseConfig: Config) extends SqlData * Reuses the error reporter from the database's executionContext. */ private val actionThreadPool: ExecutorService = { - val dbNumThreads = databaseConfig.getIntOr("db.numThreads", 20) - val dbMaximumPoolSize = databaseConfig.getIntOr("db.maxConnections", dbNumThreads * 5) - val actionThreadPoolSize = databaseConfig.getIntOr("actionThreadPoolSize", dbNumThreads) min dbMaximumPoolSize + val dbNumThreads = databaseConfig.as[Option[Int]]("db.numThreads").getOrElse(20) + val dbMaximumPoolSize = databaseConfig.as[Option[Int]]("db.maxConnections").getOrElse(dbNumThreads * 5) + val actionThreadPoolSize = databaseConfig.as[Option[Int]]("actionThreadPoolSize").getOrElse(dbNumThreads) min dbMaximumPoolSize Executors.newFixedThreadPool(actionThreadPoolSize) } diff --git a/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala b/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala index e18bd3dae..ab9cbceac 100644 --- a/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala +++ b/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala @@ -7,8 +7,8 @@ import com.typesafe.config.ConfigFactory import cromwell.core.WorkflowOptions import cromwell.engine.backend.EnhancedWorkflowOptions._ import cromwell.filesystems.gcs.{GcsFileSystem, GcsFileSystemProvider, GoogleConfiguration} -import lenthall.config.ScalaConfig._ import lenthall.exception.MessageAggregation +import net.ceedubs.ficus.Ficus._ import scala.concurrent.ExecutionContext @@ -16,7 +16,7 @@ object EngineFilesystems { private val config = ConfigFactory.load private val googleConf: GoogleConfiguration = GoogleConfiguration(config) - private val googleAuthMode = config.getStringOption("engine.filesystems.gcs.auth") map { confMode => + private val googleAuthMode = config.as[Option[String]]("engine.filesystems.gcs.auth") map { confMode => googleConf.auth(confMode) match { case Valid(mode) => mode case Invalid(errors) => throw new RuntimeException() with MessageAggregation { diff --git a/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala b/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala index fcd6da9ce..49248312e 100644 --- a/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala +++ b/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala @@ -2,8 +2,7 @@ package cromwell.engine.backend import com.typesafe.config.{Config, ConfigFactory} import cromwell.backend.{BackendConfigurationDescriptor, BackendLifecycleActorFactory} -import lenthall.config.ScalaConfig._ - +import net.ceedubs.ficus.Ficus._ import scala.collection.JavaConverters._ import scala.util.{Failure, Success, Try} @@ -29,7 +28,7 @@ object BackendConfiguration { BackendConfigurationEntry( backendName, entry.getString("actor-factory"), - entry.getConfigOr("config") + entry.as[Option[Config]]("config").getOrElse(ConfigFactory.empty("empty")) ) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala index 1fa7fdb0e..59b105dd3 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala @@ -14,8 +14,7 @@ import cromwell.engine.workflow.workflowstore.{WorkflowStoreActor, WorkflowStore import cromwell.jobstore.JobStoreActor.{JobStoreWriteFailure, JobStoreWriteSuccess, RegisterWorkflowCompleted} import cromwell.services.metadata.MetadataService._ import cromwell.webservice.EngineStatsActor -import lenthall.config.ScalaConfig.EnhancedScalaConfig - +import net.ceedubs.ficus.Ficus._ import scala.concurrent.duration._ import scala.concurrent.{Await, Promise} @@ -94,9 +93,9 @@ class WorkflowManagerActor(config: Config, jobTokenDispenserActor: ActorRef) = this( ConfigFactory.load, workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) - private val maxWorkflowsRunning = config.getConfig("system").getIntOr("max-concurrent-workflows", default=DefaultMaxWorkflowsToRun) - private val maxWorkflowsToLaunch = config.getConfig("system").getIntOr("max-workflow-launch-count", default=DefaultMaxWorkflowsToLaunch) - private val newWorkflowPollRate = config.getConfig("system").getIntOr("new-workflow-poll-rate", default=DefaultNewWorkflowPollRate).seconds + private val maxWorkflowsRunning = config.getConfig("system").as[Option[Int]]("max-concurrent-workflows").getOrElse(DefaultMaxWorkflowsToRun) + private val maxWorkflowsToLaunch = config.getConfig("system").as[Option[Int]]("max-workflow-launch-count").getOrElse(DefaultMaxWorkflowsToLaunch) + private val newWorkflowPollRate = config.getConfig("system").as[Option[Int]]("new-workflow-poll-rate").getOrElse(DefaultNewWorkflowPollRate).seconds private val logger = Logging(context.system, this) private val tag = self.path.name @@ -114,7 +113,7 @@ class WorkflowManagerActor(config: Config, private def addShutdownHook() = { // Only abort jobs on SIGINT if the config explicitly sets backend.abortJobsOnTerminate = true. val abortJobsOnTerminate = - config.getConfig("system").getBooleanOr("abort-jobs-on-terminate", default = false) + config.getConfig("system").as[Option[Boolean]]("abort-jobs-on-terminate").getOrElse(false) if (abortJobsOnTerminate) { sys.addShutdownHook { diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala index 67e93faf2..1a58b849a 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala @@ -21,8 +21,8 @@ import cromwell.engine.backend.CromwellBackends import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.{MaterializeWorkflowDescriptorActorData, MaterializeWorkflowDescriptorActorState} import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} -import lenthall.config.ScalaConfig.EnhancedScalaConfig import cromwell.core.ErrorOr._ +import net.ceedubs.ficus.Ficus._ import spray.json.{JsObject, _} import wdl4s._ import wdl4s.expression.NoFunctions @@ -89,7 +89,7 @@ object MaterializeWorkflowDescriptorActor { } } - val enabled = conf.getBooleanOption("call-caching.enabled").getOrElse(false) + val enabled = conf.as[Option[Boolean]]("call-caching.enabled").getOrElse(false) if (enabled) { val readFromCache = readOptionalOption(ReadFromCache) val writeToCache = readOptionalOption(WriteToCache) @@ -184,7 +184,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor workflowOptions: WorkflowOptions, conf: Config, engineFilesystems: List[FileSystem]): ErrorOr[EngineWorkflowDescriptor] = { - val defaultBackendName = conf.getStringOption("backend.default") + val defaultBackendName = conf.as[Option[String]]("backend.default") val rawInputsValidation = validateRawInputs(sourceFiles.inputsJson) val failureModeValidation = validateWorkflowFailureMode(workflowOptions, conf) val backendAssignmentsValidation = validateBackendAssignments(namespace.workflow.calls, workflowOptions, defaultBackendName) @@ -332,7 +332,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, val wor private def validateWorkflowFailureMode(workflowOptions: WorkflowOptions, conf: Config): ErrorOr[WorkflowFailureMode] = { val modeString: Try[String] = workflowOptions.get(WorkflowOptions.WorkflowFailureMode) match { case Success(x) => Success(x) - case Failure(_: OptionNotFoundException) => Success(conf.getStringOption("workflow-options.workflow-failure-mode") getOrElse DefaultWorkflowFailureMode) + case Failure(_: OptionNotFoundException) => Success(conf.as[Option[String]]("workflow-options.workflow-failure-mode") getOrElse DefaultWorkflowFailureMode) case Failure(t) => Failure(t) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index a3ef978ec..c40815057 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -27,6 +27,7 @@ import cromwell.services.metadata.MetadataService._ import cromwell.services.metadata._ import cromwell.webservice.EngineStatsActor import lenthall.exception.ThrowableAggregation +import net.ceedubs.ficus.Ficus._ import wdl4s.types.WdlArrayType import wdl4s.util.TryUtil import wdl4s.values.{WdlArray, WdlValue} @@ -241,7 +242,6 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, extends LoggingFSM[WorkflowExecutionActorState, WorkflowExecutionActorData] with WorkflowLogging { import WorkflowExecutionActor._ - import lenthall.config.ScalaConfig._ override def supervisorStrategy = AllForOneStrategy() { case ex: ActorInitializationException => @@ -256,7 +256,7 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, implicit val ec = context.dispatcher - val MaxRetries = ConfigFactory.load().getIntOption("system.max-retries") match { + val MaxRetries = ConfigFactory.load().as[Option[Int]]("system.max-retries") match { case Some(value) => value case None => workflowLogger.warn(s"Failed to load the max-retries value from the configuration. Defaulting back to a value of '$DefaultMaxRetriesFallbackValue'.") diff --git a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala index 9e8c04c17..b3c64b67f 100644 --- a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala +++ b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala @@ -12,8 +12,7 @@ import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor import cromwell.engine.workflow.workflowstore.{SqlWorkflowStore, WorkflowStore, WorkflowStoreActor} import cromwell.jobstore.{JobStore, JobStoreActor, SqlJobStore} import cromwell.services.{ServiceRegistryActor, SingletonServicesStore} -import lenthall.config.ScalaConfig.EnhancedScalaConfig - +import net.ceedubs.ficus.Ficus._ /** * An actor which serves as the lord protector for the rest of Cromwell, allowing us to have more fine grain * control on top level supervision, etc. @@ -31,7 +30,7 @@ import lenthall.config.ScalaConfig.EnhancedScalaConfig private val config = ConfigFactory.load() lazy val serviceRegistryActor: ActorRef = context.actorOf(ServiceRegistryActor.props(config), "ServiceRegistryActor") - lazy val numberOfWorkflowLogCopyWorkers = config.getConfig("system").getIntOr("number-of-workflow-log-copy-workers", default=DefaultNumberOfWorkflowLogCopyWorkers) + lazy val numberOfWorkflowLogCopyWorkers = config.getConfig("system").as[Option[Int]]("number-of-workflow-log-copy-workers").getOrElse(DefaultNumberOfWorkflowLogCopyWorkers) lazy val workflowLogCopyRouter: ActorRef = context.actorOf(RoundRobinPool(numberOfWorkflowLogCopyWorkers) .withSupervisorStrategy(CopyWorkflowLogsActor.strategy) diff --git a/engine/src/main/scala/cromwell/server/CromwellServer.scala b/engine/src/main/scala/cromwell/server/CromwellServer.scala index 0dd8ac59f..dca31a625 100644 --- a/engine/src/main/scala/cromwell/server/CromwellServer.scala +++ b/engine/src/main/scala/cromwell/server/CromwellServer.scala @@ -7,9 +7,9 @@ import akka.util.Timeout import com.typesafe.config.Config import cromwell.webservice.WorkflowJsonSupport._ import cromwell.webservice.{APIResponse, CromwellApiService, SwaggerService} -import lenthall.config.ScalaConfig._ import lenthall.spray.SprayCanHttpService._ import lenthall.spray.WrappedRoute._ +import net.ceedubs.ficus.Ficus._ import spray.http.{ContentType, MediaTypes, _} import spray.json._ @@ -56,7 +56,7 @@ class CromwellServerActor(config: Config) extends CromwellRootActor with Cromwel override def actorRefFactory = context override def receive = handleTimeouts orElse runRoute(possibleRoutes) - val possibleRoutes = workflowRoutes.wrapped("api", config.getBooleanOr("api.routeUnwrapped")) ~ swaggerUiResourceRoute + val possibleRoutes = workflowRoutes.wrapped("api", config.as[Option[Boolean]]("api.routeUnwrapped").getOrElse(false)) ~ swaggerUiResourceRoute val timeoutError = APIResponse.error(new TimeoutException("The server was not able to produce a timely response to your request.")).toJson.prettyPrint def handleTimeouts: Receive = { diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala index fd40a5403..845ec29ef 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsFileSystemProvider.scala @@ -19,8 +19,9 @@ import com.google.api.services.storage.Storage import com.google.api.services.storage.model.StorageObject import com.google.cloud.hadoop.gcsio.{GoogleCloudStorageReadChannel, GoogleCloudStorageWriteChannel, ObjectWriteConditions} import com.google.cloud.hadoop.util.{ApiErrorExtractor, AsyncWriteChannelOptions, ClientRequestHelper} -import com.typesafe.config.ConfigFactory -import lenthall.config.ScalaConfig.EnhancedScalaConfig +import com.typesafe.config.{Config, ConfigFactory, ConfigMemorySize} +import net.ceedubs.ficus.Ficus._ +import net.ceedubs.ficus.readers.ValueReader import scala.annotation.tailrec import scala.collection.JavaConverters._ @@ -53,6 +54,11 @@ object GcsFileSystemProvider { withRetry(f, retries - 1) case Failure(ex) => throw ex } + + // TODO refactor as part of Ficus and submit a PR + implicit val configMemorySizeValueReader: ValueReader[ConfigMemorySize] = new ValueReader[ConfigMemorySize] { + override def read(config: Config, path: String): ConfigMemorySize = config.getMemorySize(path) + } } /** @@ -140,8 +146,10 @@ class GcsFileSystemProvider private[gcs](storageClient: Try[Storage], val execut - com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.setUploadBufferSize - com.google.api.client.googleapis.media.MediaHttpUploader.setContentAndHeadersOnCurrentRequest */ - private[this] lazy val uploadBufferBytes = config.getBytesOr("google.upload-buffer-bytes", - MediaHttpUploader.MINIMUM_CHUNK_SIZE).toInt + private[this] lazy val uploadBufferBytes = { + val configBytes = config.as[Option[ConfigMemorySize]]("google.upload-buffer-bytes").map(_.toBytes.toInt) + configBytes.getOrElse(MediaHttpUploader.MINIMUM_CHUNK_SIZE) + } /** * Overrides the default implementation to provide a writable channel (which newByteChannel doesn't). @@ -234,7 +242,7 @@ class GcsFileSystemProvider private[gcs](storageClient: Try[Storage], val execut override def isHidden(path: Path): Boolean = throw new NotImplementedError() - private[this] lazy val maxResults = config.getIntOr("google.list-max-results", 1000).toLong + private[this] lazy val maxResults = config.as[Option[Int]]("google.list-max-results").getOrElse(1000).toLong private def list(gcsDir: NioGcsPath) = { val listRequest = client.objects().list(gcsDir.bucket).setMaxResults(maxResults) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 5b632d157..ca5420d5e 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,7 +1,7 @@ import sbt._ object Dependencies { - lazy val lenthallV = "0.19-98b3f3a-SNAPSHOT" + lazy val lenthallV = "0.19-882a763-SNAPSHOT" lazy val wdl4sV = "0.6-2964173-SNAPSHOT" lazy val sprayV = "1.3.3" /* @@ -24,6 +24,7 @@ object Dependencies { "org.broadinstitute" %% "lenthall" % lenthallV, "org.typelevel" %% "cats" % catsV, "com.github.benhutchison" %% "mouse" % "0.5", + "com.iheart" %% "ficus" % "1.3.0", "org.scalatest" %% "scalatest" % "3.0.0" % Test, "org.specs2" %% "specs2" % "3.7" % Test ) diff --git a/project/Settings.scala b/project/Settings.scala index 627ac080c..4a092ce68 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -12,6 +12,7 @@ import sbtdocker.DockerPlugin.autoImport._ object Settings { val commonResolvers = List( + Resolver.jcenterRepo, "Broad Artifactory Releases" at "https://artifactory.broadinstitute.org/artifactory/libs-release/", "Broad Artifactory Snapshots" at "https://artifactory.broadinstitute.org/artifactory/libs-snapshot/" ) diff --git a/services/src/main/scala/cromwell/services/ServiceRegistryActor.scala b/services/src/main/scala/cromwell/services/ServiceRegistryActor.scala index 357f5d33f..e58cd5c05 100644 --- a/services/src/main/scala/cromwell/services/ServiceRegistryActor.scala +++ b/services/src/main/scala/cromwell/services/ServiceRegistryActor.scala @@ -3,8 +3,7 @@ package cromwell.services import akka.actor.SupervisorStrategy.Escalate import akka.actor.{Actor, ActorInitializationException, ActorLogging, ActorRef, OneForOneStrategy, Props} import com.typesafe.config.{Config, ConfigFactory, ConfigObject} -import lenthall.config.ScalaConfig._ - +import net.ceedubs.ficus.Ficus._ import scala.collection.JavaConverters._ object ServiceRegistryActor { @@ -31,9 +30,8 @@ object ServiceRegistryActor { } private def serviceProps(serviceName: String, globalConfig: Config, serviceStanza: Config): Props = { - val serviceConfigStanza = serviceStanza.getConfigOr("config", ConfigFactory.parseString("")) - val className = serviceStanza.getStringOr( - "class", + val serviceConfigStanza = serviceStanza.as[Option[Config]]("config").getOrElse(ConfigFactory.parseString("")) + val className = serviceStanza.as[Option[String]]("class").getOrElse( throw new IllegalArgumentException(s"Invalid configuration for service $serviceName: missing 'class' definition") ) diff --git a/services/src/main/scala/cromwell/services/ServicesStore.scala b/services/src/main/scala/cromwell/services/ServicesStore.scala index a73c3f3ec..c00aad09c 100644 --- a/services/src/main/scala/cromwell/services/ServicesStore.scala +++ b/services/src/main/scala/cromwell/services/ServicesStore.scala @@ -4,7 +4,7 @@ import com.typesafe.config.ConfigFactory import cromwell.database.migration.liquibase.LiquibaseUtils import cromwell.database.slick.SlickDatabase import cromwell.database.sql.SqlDatabase -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ import org.slf4j.LoggerFactory trait ServicesStore { @@ -15,7 +15,7 @@ object ServicesStore { implicit class EnhancedSqlDatabase[A <: SqlDatabase](val sqlDatabase: A) extends AnyVal { def initialized: A = { - if (sqlDatabase.databaseConfig.getBooleanOr("liquibase.updateSchema", default = true)) { + if (sqlDatabase.databaseConfig.as[Option[Boolean]]("liquibase.updateSchema").getOrElse(true)) { sqlDatabase withConnection LiquibaseUtils.updateSchema } sqlDatabase diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala index ab9e20f29..a92b92fbd 100644 --- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala +++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala @@ -9,15 +9,14 @@ import cromwell.services.SingletonServicesStore import cromwell.services.metadata.MetadataService.{PutMetadataAction, ReadAction, RefreshSummary, ValidateWorkflowIdAndExecute} import cromwell.services.metadata.impl.MetadataServiceActor._ import cromwell.services.metadata.impl.MetadataSummaryRefreshActor.{MetadataSummaryFailure, MetadataSummarySuccess, SummarizeMetadata} -import lenthall.config.ScalaConfig._ - +import net.ceedubs.ficus.Ficus._ import scala.concurrent.duration.{Duration, FiniteDuration} import scala.util.{Failure, Success, Try} object MetadataServiceActor { val MetadataSummaryRefreshInterval: Option[FiniteDuration] = { - val duration = Duration(ConfigFactory.load().getStringOr("services.MetadataService.metadata-summary-refresh-interval", "2 seconds")) + val duration = Duration(ConfigFactory.load().as[Option[String]]("services.MetadataService.metadata-summary-refresh-interval").getOrElse("2 seconds")) if (duration.isFinite()) Option(duration.asInstanceOf[FiniteDuration]) else None } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala index 3c72ca7fc..2e1d00b0e 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala @@ -9,8 +9,8 @@ import com.typesafe.config.Config import cromwell.backend.impl.jes.JesImplicits.GoogleAuthWorkflowOptions import cromwell.core.WorkflowOptions import cromwell.filesystems.gcs.{GoogleAuthMode, GoogleConfiguration} -import lenthall.config.ScalaConfig._ import lenthall.config.ValidatedConfig._ +import net.ceedubs.ficus.Ficus._ import cromwell.core.ErrorOr._ import wdl4s.ExceptionWithErrors @@ -46,7 +46,7 @@ object JesAttributes { val project: ValidatedNel[String, String] = backendConfig.validateString("project") val executionBucket: ValidatedNel[String, String] = backendConfig.validateString("root") val endpointUrl: ErrorOr[URL] = backendConfig.validateURL("genomics.endpoint-url") - val maxPollingInterval: Int = backendConfig.getIntOption("maximum-polling-interval").getOrElse(600) + val maxPollingInterval: Int = backendConfig.as[Option[Int]]("maximum-polling-interval").getOrElse(600) val genomicsAuthName: ErrorOr[String] = backendConfig.validateString("genomics.auth") val gcsFilesystemAuthName: ErrorOr[String] = backendConfig.validateString("filesystems.gcs.auth") diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala index 93ad1fe3a..0debe3aee 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala @@ -1,11 +1,12 @@ package cromwell.backend.impl.sfs.config +import com.typesafe.config.Config import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction import cromwell.backend.impl.sfs.config.ConfigConstants._ import cromwell.backend.sfs._ import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, RuntimeAttributeDefinition} import cromwell.core.JobExecutionToken.JobExecutionTokenType -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ import org.slf4j.LoggerFactory /** @@ -18,13 +19,13 @@ class ConfigBackendLifecycleActorFactory(name: String, val configurationDescript lazy val logger = LoggerFactory.getLogger(getClass) lazy val hashingStrategy = { - configurationDescriptor.backendConfig.getConfigOption("filesystems.local.caching") map ConfigHashingStrategy.apply getOrElse ConfigHashingStrategy.defaultStrategy + configurationDescriptor.backendConfig.as[Option[Config]]("filesystems.local.caching") map ConfigHashingStrategy.apply getOrElse ConfigHashingStrategy.defaultStrategy } override def initializationActorClass = classOf[ConfigInitializationActor] override def asyncJobExecutionActorClass: Class[_ <: ConfigAsyncJobExecutionActor] = { - val runInBackground = configurationDescriptor.backendConfig.getBooleanOr(RunInBackgroundConfig, default = false) + val runInBackground = configurationDescriptor.backendConfig.as[Option[Boolean]](RunInBackgroundConfig).getOrElse(false) if (runInBackground) classOf[BackgroundConfigAsyncJobExecutionActor] else @@ -47,7 +48,7 @@ class ConfigBackendLifecycleActorFactory(name: String, val configurationDescript override lazy val fileHashingActorCount: Int = 5 override val jobExecutionTokenType: JobExecutionTokenType = { - val concurrentJobLimit = configurationDescriptor.backendConfig.getIntOption("concurrent-job-limit") + val concurrentJobLimit = configurationDescriptor.backendConfig.as[Option[Int]]("concurrent-job-limit") JobExecutionTokenType(name, concurrentJobLimit) } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala index 21719550e..6261e65c9 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala @@ -6,7 +6,7 @@ import com.typesafe.config.Config import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest import cromwell.util.TryWithResource._ import cromwell.util.FileUtil._ -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ import org.apache.commons.codec.digest.DigestUtils import org.slf4j.LoggerFactory @@ -17,9 +17,9 @@ object ConfigHashingStrategy { val defaultStrategy = HashFileStrategy(false) def apply(hashingConfig: Config): ConfigHashingStrategy = { - val checkSiblingMd5 = hashingConfig.getBooleanOr("check-sibling-md5", default = false) + val checkSiblingMd5 = hashingConfig.as[Option[Boolean]]("check-sibling-md5").getOrElse(false) - hashingConfig.getStringOr("hashing-strategy", "file") match { + hashingConfig.as[Option[String]]("hashing-strategy").getOrElse("file") match { case "path" => HashPathStrategy(checkSiblingMd5) case "file" => HashFileStrategy(checkSiblingMd5) case what => diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala index 7b6909edb..cb56e35a0 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala @@ -2,7 +2,7 @@ package cromwell.backend.impl.sfs.config import com.typesafe.config.Config import cromwell.backend.impl.sfs.config.ConfigConstants._ -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ import wdl4s._ /** @@ -14,20 +14,20 @@ class ConfigWdlNamespace(backendConfig: Config) { import ConfigWdlNamespace._ - private val configRuntimeAttributes = backendConfig.getStringOr(RuntimeAttributesConfig) + private val configRuntimeAttributes = backendConfig.as[Option[String]](RuntimeAttributesConfig).getOrElse("") - private val submitCommandOption = backendConfig.getStringOption(SubmitConfig) + private val submitCommandOption = backendConfig.as[Option[String]](SubmitConfig) private val submitSourceOption = submitCommandOption.map(makeWdlSource( SubmitTask, _, submitRuntimeAttributes + configRuntimeAttributes)) - private val submitDockerCommandOption = backendConfig.getStringOption(SubmitDockerConfig) + private val submitDockerCommandOption = backendConfig.as[Option[String]](SubmitDockerConfig) private val submitDockerSourceOption = submitDockerCommandOption.map(makeWdlSource( SubmitDockerTask, _, submitRuntimeAttributes + submitDockerRuntimeAttributes + configRuntimeAttributes)) - private val killCommandOption = backendConfig.getStringOption(KillConfig) + private val killCommandOption = backendConfig.as[Option[String]](KillConfig) private val killSourceOption = killCommandOption.map(makeWdlSource(KillTask, _, jobIdRuntimeAttributes)) - private val checkAliveCommandOption = backendConfig.getStringOption(CheckAliveConfig) + private val checkAliveCommandOption = backendConfig.as[Option[String]](CheckAliveConfig) private val checkAliveSourceOption = checkAliveCommandOption.map(makeWdlSource( CheckAliveTask, _, jobIdRuntimeAttributes)) diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala index 4cde3b7c9..d96140014 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/GcsWorkflowFileSystemProvider.scala @@ -4,14 +4,14 @@ import cats.data.Validated.{Invalid, Valid} import cromwell.backend.wfs.{WorkflowFileSystemProvider, WorkflowFileSystemProviderParams} import cromwell.filesystems.gcs.GoogleAuthMode.GoogleAuthOptions import cromwell.filesystems.gcs.{GcsFileSystem, GcsFileSystemProvider, GoogleConfiguration} -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ import wdl4s.ValidationException import scala.util.Try object GcsWorkflowFileSystemProvider extends WorkflowFileSystemProvider { override def fileSystemOption(params: WorkflowFileSystemProviderParams): Option[GcsFileSystem] = { - params.fileSystemConfig.getStringOption("gcs.auth") map gcsFileSystem(params) + params.fileSystemConfig.as[Option[String]]("gcs.auth") map gcsFileSystem(params) } private def gcsFileSystem(params: WorkflowFileSystemProviderParams)(gcsAuthName: String): GcsFileSystem = { diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala index a423d6cb7..d9d4f4213 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala @@ -1,12 +1,13 @@ package cromwell.backend.sfs import akka.actor.{Actor, ActorRef} +import com.typesafe.config.{Config, ConfigFactory} import cromwell.backend.BackendInitializationData import cromwell.backend.callcaching.JobCachingActorHelper import cromwell.backend.io.JobPaths import cromwell.backend.validation.{RuntimeAttributesValidation, ValidatedRuntimeAttributes} import cromwell.core.logging.JobLogging -import lenthall.config.ScalaConfig._ +import net.ceedubs.ficus.Ficus._ trait SharedFileSystemJobCachingActorHelper extends JobCachingActorHelper { this: Actor with JobLogging => @@ -37,7 +38,7 @@ trait SharedFileSystemJobCachingActorHelper extends JobCachingActorHelper { lazy val sharedFileSystem = new SharedFileSystem { override lazy val sharedFileSystemConfig = { - configurationDescriptor.backendConfig.getConfigOr("filesystems.local") + configurationDescriptor.backendConfig.as[Option[Config]]("filesystems.local").getOrElse(ConfigFactory.empty()) } } } From 350772909e11ea4bbf977854fa84c55b4a757b37 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 11 Oct 2016 13:37:22 -0400 Subject: [PATCH 18/23] Fixed .pullapprove.yml --- .pullapprove.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pullapprove.yml b/.pullapprove.yml index a1b902b1c..8e5e37ffa 100644 --- a/.pullapprove.yml +++ b/.pullapprove.yml @@ -7,7 +7,6 @@ reviewers: members: - cjllanwarne - Horneth - - scottfrazer - mcovarr - geoffjentry - kshakir From 5eb428585a914915180b2d3717882dab714cc350 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Thu, 29 Sep 2016 10:01:05 -0400 Subject: [PATCH 19/23] Updated the JES status polling to be batched --- .../backend/BackendLifecycleActorFactory.scala | 5 +- .../test/scala/cromwell/util/AkkaTestUtil.scala | 26 +++- .../backend/BackendSingletonCollection.scala | 5 + .../cromwell/engine/workflow/WorkflowActor.scala | 10 +- .../engine/workflow/WorkflowManagerActor.scala | 18 ++- .../execution/EngineJobExecutionActor.scala | 5 +- .../lifecycle/execution/JobPreparationActor.scala | 16 ++- .../execution/WorkflowExecutionActor.scala | 9 +- .../scala/cromwell/server/CromwellRootActor.scala | 8 +- .../test/scala/cromwell/CromwellTestkitSpec.scala | 3 +- .../scala/cromwell/SimpleWorkflowActorSpec.scala | 4 +- .../mock/DefaultBackendJobExecutionActor.scala | 3 +- .../RetryableBackendLifecycleActorFactory.scala | 3 +- .../workflow/SingleWorkflowRunnerActorSpec.scala | 4 +- .../engine/workflow/WorkflowActorSpec.scala | 3 +- .../execution/WorkflowExecutionActorSpec.scala | 11 +- .../lifecycle/execution/ejea/PerTestHelper.scala | 6 +- .../JobExecutionTokenDispenserActorSpec.scala | 31 ++--- ...ingActor.scala => TestTokenGrabbingActor.scala} | 15 +- .../impl/htcondor/HtCondorBackendFactory.scala | 3 +- .../jes/JesAsyncBackendJobExecutionActor.scala | 107 +++++++------- .../impl/jes/JesBackendLifecycleActorFactory.scala | 7 +- .../impl/jes/JesBackendSingletonActor.scala | 20 +++ .../backend/impl/jes/JesJobExecutionActor.scala | 13 +- .../main/scala/cromwell/backend/impl/jes/Run.scala | 24 ++-- .../cromwell/backend/impl/jes/RunStatus.scala | 2 +- .../jes/statuspolling/JesApiQueryManager.scala | 108 +++++++++++++++ .../impl/jes/statuspolling/JesPollingActor.scala | 122 ++++++++++++++++ .../jes/statuspolling/JesPollingActorClient.scala | 51 +++++++ .../jes/JesAsyncBackendJobExecutionActorSpec.scala | 56 ++++---- .../scala/cromwell/backend/impl/jes/RunSpec.scala | 3 +- .../jes/statuspolling/JesApiQueryManagerSpec.scala | 153 +++++++++++++++++++++ .../jes/statuspolling/JesPollingActorSpec.scala | 131 ++++++++++++++++++ ...redFileSystemBackendLifecycleActorFactory.scala | 3 +- .../backend/impl/spark/SparkBackendFactory.scala | 6 +- 35 files changed, 832 insertions(+), 162 deletions(-) create mode 100644 engine/src/main/scala/cromwell/engine/backend/BackendSingletonCollection.scala rename engine/src/test/scala/cromwell/engine/workflow/tokens/{TokenGrabbingActor.scala => TestTokenGrabbingActor.scala} (64%) create mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala create mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala create mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala create mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorClient.scala create mode 100644 supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala create mode 100644 supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala index 0101da9f0..5a6c5d268 100644 --- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala +++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala @@ -20,7 +20,8 @@ trait BackendLifecycleActorFactory { def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef): Props + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props /** * Providing this method to generate Props for a cache hit copying actor is optional. @@ -33,6 +34,8 @@ trait BackendLifecycleActorFactory { */ def cacheHitCopyingActorProps: Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef) => Props] = None + def backendSingletonActorProps: Option[Props] = None + def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Seq[Call], executionStore: ExecutionStore, diff --git a/core/src/test/scala/cromwell/util/AkkaTestUtil.scala b/core/src/test/scala/cromwell/util/AkkaTestUtil.scala index 1a1ebd618..10b05dc2b 100644 --- a/core/src/test/scala/cromwell/util/AkkaTestUtil.scala +++ b/core/src/test/scala/cromwell/util/AkkaTestUtil.scala @@ -1,6 +1,6 @@ package cromwell.util -import akka.actor.{Actor, ActorLogging, Props} +import akka.actor.{Actor, ActorLogging, ActorRef, ActorSystem, Kill, PoisonPill, Props, SupervisorStrategy} import akka.testkit.TestProbe object AkkaTestUtil { @@ -16,4 +16,28 @@ object AkkaTestUtil { } }) } + + def actorDeathMethods(system: ActorSystem): List[(String, ActorRef => Unit)] = List( + ("external_stop", (a: ActorRef) => system.stop(a)), + ("internal_stop", (a: ActorRef) => a ! InternalStop), + ("poison_pill", (a: ActorRef) => a ! PoisonPill), + ("kill_message", (a: ActorRef) => a ! Kill), + ("throw_exception", (a: ActorRef) => a ! ThrowException) + ) + + case object InternalStop + case object ThrowException + + class StoppingSupervisor extends Actor { + override val supervisorStrategy = SupervisorStrategy.stoppingStrategy + def receive = Actor.emptyBehavior + } + + class DeathTestActor extends Actor { + private def stoppingReceive: Actor.Receive = { + case InternalStop => context.stop(self) + case ThrowException => throw new Exception("Don't panic, dear debugger! This was a deliberate exception for the test case.") + } + override def receive = stoppingReceive orElse Actor.ignoringBehavior + } } diff --git a/engine/src/main/scala/cromwell/engine/backend/BackendSingletonCollection.scala b/engine/src/main/scala/cromwell/engine/backend/BackendSingletonCollection.scala new file mode 100644 index 000000000..ecb1a6753 --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/backend/BackendSingletonCollection.scala @@ -0,0 +1,5 @@ +package cromwell.engine.backend + +import akka.actor.ActorRef + +final case class BackendSingletonCollection(backendSingletonActors: Map[String, Option[ActorRef]]) diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala index 684c81cce..1035872d3 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala @@ -11,6 +11,7 @@ import cromwell.core.WorkflowOptions.FinalWorkflowLogDir import cromwell.core._ import cromwell.core.logging.{WorkflowLogger, WorkflowLogging} import cromwell.engine._ +import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.WorkflowActor._ import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.{MaterializeWorkflowDescriptorCommand, MaterializeWorkflowDescriptorFailureResponse, MaterializeWorkflowDescriptorSuccessResponse} import cromwell.engine.workflow.lifecycle.WorkflowFinalizationActor.{StartFinalizationCommand, WorkflowFinalizationFailedResponse, WorkflowFinalizationSucceededResponse} @@ -140,9 +141,10 @@ object WorkflowActor { workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, - jobTokenDispenserActor: ActorRef): Props = { + jobTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection): Props = { Props(new WorkflowActor(workflowId, startMode, wdlSource, conf, serviceRegistryActor, workflowLogCopyRouter, - jobStoreActor, callCacheReadActor, jobTokenDispenserActor)).withDispatcher(EngineDispatcher) + jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection)).withDispatcher(EngineDispatcher) } } @@ -157,7 +159,8 @@ class WorkflowActor(val workflowId: WorkflowId, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, - jobTokenDispenserActor: ActorRef) + jobTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection) extends LoggingFSM[WorkflowActorState, WorkflowActorData] with WorkflowLogging with PathFactory { implicit val ec = context.dispatcher @@ -206,6 +209,7 @@ class WorkflowActor(val workflowId: WorkflowId, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, + backendSingletonCollection, initializationData, restarting = restarting), name = s"WorkflowExecutionActor-$workflowId") diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala index 59b105dd3..08bade654 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala @@ -8,6 +8,7 @@ import cats.data.NonEmptyList import com.typesafe.config.{Config, ConfigFactory} import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.{WorkflowAborted, WorkflowId} +import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.WorkflowActor._ import cromwell.engine.workflow.WorkflowManagerActor._ import cromwell.engine.workflow.workflowstore.{WorkflowStoreActor, WorkflowStoreState} @@ -42,9 +43,10 @@ object WorkflowManagerActor { workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, - jobTokenDispenserActor: ActorRef): Props = { + jobTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection): Props = { Props(new WorkflowManagerActor( - workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) + workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection) ).withDispatcher(EngineDispatcher) } @@ -82,7 +84,8 @@ class WorkflowManagerActor(config: Config, val workflowLogCopyRouter: ActorRef, val jobStoreActor: ActorRef, val callCacheReadActor: ActorRef, - val jobTokenDispenserActor: ActorRef) + val jobTokenDispenserActor: ActorRef, + val backendSingletonCollection: BackendSingletonCollection) extends LoggingFSM[WorkflowManagerState, WorkflowManagerData] { def this(workflowStore: ActorRef, @@ -90,8 +93,9 @@ class WorkflowManagerActor(config: Config, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, - jobTokenDispenserActor: ActorRef) = this( - ConfigFactory.load, workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) + jobTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection) = this( + ConfigFactory.load, workflowStore, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection) private val maxWorkflowsRunning = config.getConfig("system").as[Option[Int]]("max-concurrent-workflows").getOrElse(DefaultMaxWorkflowsToRun) private val maxWorkflowsToLaunch = config.getConfig("system").as[Option[Int]]("max-workflow-launch-count").getOrElse(DefaultMaxWorkflowsToLaunch) @@ -111,7 +115,7 @@ class WorkflowManagerActor(config: Config, } private def addShutdownHook() = { - // Only abort jobs on SIGINT if the config explicitly sets backend.abortJobsOnTerminate = true. + // Only abort jobs on SIGINT if the config explicitly sets system.abortJobsOnTerminate = true. val abortJobsOnTerminate = config.getConfig("system").as[Option[Boolean]]("abort-jobs-on-terminate").getOrElse(false) @@ -261,7 +265,7 @@ class WorkflowManagerActor(config: Config, } val wfProps = WorkflowActor.props(workflowId, startMode, workflow.sources, config, serviceRegistryActor, - workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) + workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection) val wfActor = context.actorOf(wfProps, name = s"WorkflowActor-$workflowId") wfActor ! SubscribeTransitionCallBack(self) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala index 64f3eec7a..07114e7e8 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala @@ -39,6 +39,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, jobTokenDispenserActor: ActorRef, + backendSingletonActor: Option[ActorRef], backendName: String, callCachingMode: CallCachingMode) extends LoggingFSM[EngineJobExecutionActorState, EJEAData] with WorkflowLogging { @@ -282,7 +283,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, def createJobPreparationActor(jobPrepProps: Props, name: String): ActorRef = context.actorOf(jobPrepProps, name) def prepareJob() = { val jobPreparationActorName = s"BackendPreparationActor_for_$jobTag" - val jobPrepProps = JobPreparationActor.props(executionData, jobDescriptorKey, factory, initializationData, serviceRegistryActor) + val jobPrepProps = JobPreparationActor.props(executionData, jobDescriptorKey, factory, initializationData, serviceRegistryActor, backendSingletonActor) val jobPreparationActor = createJobPreparationActor(jobPrepProps, jobPreparationActorName) jobPreparationActor ! JobPreparationActor.Start goto(PreparingJob) @@ -455,6 +456,7 @@ object EngineJobExecutionActor { jobStoreActor: ActorRef, callCacheReadActor: ActorRef, jobTokenDispenserActor: ActorRef, + backendSingletonActor: Option[ActorRef], backendName: String, callCachingMode: CallCachingMode) = { Props(new EngineJobExecutionActor( @@ -468,6 +470,7 @@ object EngineJobExecutionActor { jobStoreActor = jobStoreActor, callCacheReadActor = callCacheReadActor, jobTokenDispenserActor = jobTokenDispenserActor, + backendSingletonActor = backendSingletonActor, backendName = backendName: String, callCachingMode = callCachingMode)).withDispatcher(EngineDispatcher) } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala index cc7f76f92..12c719994 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala @@ -14,10 +14,11 @@ import wdl4s.values.WdlValue import scala.util.{Failure, Success, Try} final case class JobPreparationActor(executionData: WorkflowExecutionActorData, - jobKey: BackendJobDescriptorKey, - factory: BackendLifecycleActorFactory, - initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef) + jobKey: BackendJobDescriptorKey, + factory: BackendLifecycleActorFactory, + initializationData: Option[BackendInitializationData], + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]) extends Actor with WdlLookup with WorkflowLogging { override lazy val workflowDescriptor: EngineWorkflowDescriptor = executionData.workflowDescriptor @@ -92,7 +93,7 @@ final case class JobPreparationActor(executionData: WorkflowExecutionActorData, evaluatedRuntimeAttributes <- evaluateRuntimeAttributes(unevaluatedRuntimeAttributes, expressionLanguageFunctions, inputEvaluation) attributesWithDefault = curriedAddDefaultsToAttributes(evaluatedRuntimeAttributes) jobDescriptor = BackendJobDescriptor(workflowDescriptor.backendDescriptor, jobKey, attributesWithDefault, inputEvaluation) - } yield BackendJobPreparationSucceeded(jobDescriptor, factory.jobExecutionActorProps(jobDescriptor, initializationData, serviceRegistryActor))) match { + } yield BackendJobPreparationSucceeded(jobDescriptor, factory.jobExecutionActorProps(jobDescriptor, initializationData, serviceRegistryActor, backendSingletonActor))) match { case Success(s) => s case Failure(f) => BackendJobPreparationFailed(jobKey, f) } @@ -111,9 +112,10 @@ object JobPreparationActor { jobKey: BackendJobDescriptorKey, factory: BackendLifecycleActorFactory, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef) = { + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]) = { // Note that JobPreparationActor doesn't run on the engine dispatcher as it mostly executes backend-side code // (WDL expression evaluation using Backend's expressionLanguageFunctions) - Props(new JobPreparationActor(executionData, jobKey, factory, initializationData, serviceRegistryActor)) + Props(new JobPreparationActor(executionData, jobKey, factory, initializationData, serviceRegistryActor, backendSingletonActor)) } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index c40815057..b7358296b 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -17,7 +17,7 @@ import cromwell.core.OutputStore.OutputEntry import cromwell.core.WorkflowOptions.WorkflowFailureMode import cromwell.core._ import cromwell.core.logging.WorkflowLogging -import cromwell.engine.backend.CromwellBackends +import cromwell.engine.backend.{BackendSingletonCollection, CromwellBackends} import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.JobRunning import cromwell.engine.workflow.lifecycle.execution.JobPreparationActor.BackendJobPreparationFailed import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.WorkflowExecutionActorState @@ -140,10 +140,11 @@ object WorkflowExecutionActor { jobStoreActor: ActorRef, callCacheReadActor: ActorRef, jobTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection, initializationData: AllBackendInitializationData, restarting: Boolean): Props = { Props(WorkflowExecutionActor(workflowId, workflowDescriptor, serviceRegistryActor, jobStoreActor, - callCacheReadActor, jobTokenDispenserActor, initializationData, restarting)).withDispatcher(EngineDispatcher) + callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection, initializationData, restarting)).withDispatcher(EngineDispatcher) } implicit class EnhancedExecutionStore(val executionStore: ExecutionStore) extends AnyVal { @@ -237,6 +238,7 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, jobTokenDispenserActor: ActorRef, + backendSingletonCollection: BackendSingletonCollection, initializationData: AllBackendInitializationData, restarting: Boolean) extends LoggingFSM[WorkflowExecutionActorState, WorkflowExecutionActorData] with WorkflowLogging { @@ -584,9 +586,10 @@ final case class WorkflowExecutionActor(workflowId: WorkflowId, factories.get(backendName) match { case Some(factory) => val ejeaName = s"${workflowDescriptor.id}-EngineJobExecutionActor-${jobKey.tag}" + val backendSingleton = backendSingletonCollection.backendSingletonActors(backendName) val ejeaProps = EngineJobExecutionActor.props( self, jobKey, data, factory, initializationData.get(backendName), restarting, serviceRegistryActor, - jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendName, workflowDescriptor.callCachingMode) + jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingleton, backendName, workflowDescriptor.callCachingMode) val ejeaRef = context.actorOf(ejeaProps, ejeaName) pushNewJobMetadata(jobKey, backendName) ejeaRef ! EngineJobExecutionActor.Execute diff --git a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala index b3c64b67f..cc0f5f4aa 100644 --- a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala +++ b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala @@ -5,6 +5,7 @@ import akka.actor.{Actor, ActorInitializationException, ActorRef, OneForOneStrat import akka.event.Logging import akka.routing.RoundRobinPool import com.typesafe.config.ConfigFactory +import cromwell.engine.backend.{BackendSingletonCollection, CromwellBackends} import cromwell.engine.workflow.WorkflowManagerActor import cromwell.engine.workflow.lifecycle.CopyWorkflowLogsActor import cromwell.engine.workflow.lifecycle.execution.callcaching.{CallCache, CallCacheReadActor} @@ -48,11 +49,16 @@ import net.ceedubs.ficus.Ficus._ .props(CallCacheReadActor.props(callCache)), "CallCacheReadActor") + lazy val backendSingletons = CromwellBackends.instance.get.backendLifecycleActorFactories map { + case (name, factory) => name -> (factory.backendSingletonActorProps map context.actorOf) + } + lazy val backendSingletonCollection = BackendSingletonCollection(backendSingletons) + lazy val jobExecutionTokenDispenserActor = context.actorOf(JobExecutionTokenDispenserActor.props) lazy val workflowManagerActor = context.actorOf( WorkflowManagerActor.props( - workflowStoreActor, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobExecutionTokenDispenserActor), + workflowStoreActor, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobExecutionTokenDispenserActor, backendSingletonCollection), "WorkflowManagerActor") override def receive = { diff --git a/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala b/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala index c4aec5879..090d94fd1 100644 --- a/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala +++ b/engine/src/test/scala/cromwell/CromwellTestkitSpec.scala @@ -48,7 +48,8 @@ case class TestBackendLifecycleActorFactory(configurationDescriptor: BackendConf override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef): Props = { + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props = { throw new NotImplementedError("this is not implemented") } diff --git a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala index 34ec73895..f327c6da4 100644 --- a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala @@ -8,6 +8,7 @@ import com.typesafe.config.ConfigFactory import cromwell.MetadataWatchActor.{FailureMatcher, Matcher} import cromwell.SimpleWorkflowActorSpec._ import cromwell.core.{WorkflowId, WorkflowSourceFiles} +import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.WorkflowActor import cromwell.engine.workflow.WorkflowActor._ import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor @@ -43,7 +44,8 @@ class SimpleWorkflowActorSpec extends CromwellTestkitSpec with BeforeAndAfter { workflowLogCopyRouter = system.actorOf(Props.empty, s"workflow-copy-log-router-$workflowId-${UUID.randomUUID()}"), jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props), callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props), - jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props)), + jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props), + backendSingletonCollection = BackendSingletonCollection(Map("Local" -> None))), supervisor = supervisor.ref, name = s"workflow-actor-$workflowId" ) diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala b/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala index 6893f6baf..f98fa17bd 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala @@ -30,7 +30,8 @@ class DefaultBackendLifecycleActorFactory(name: String, configurationDescriptor: override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef): Props = { + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props = { DefaultBackendJobExecutionActor.props(jobDescriptor, configurationDescriptor) } diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala index 74e900c6d..46f28f447 100644 --- a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala +++ b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala @@ -13,7 +13,8 @@ class RetryableBackendLifecycleActorFactory(name: String, configurationDescripto override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef): Props = { + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props = { RetryableBackendJobExecutionActor.props(jobDescriptor, configurationDescriptor) } diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala index 73e8bfa7b..540da9863 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala @@ -10,6 +10,7 @@ import better.files._ import com.typesafe.config.ConfigFactory import cromwell.CromwellTestkitSpec._ import cromwell.core.WorkflowSourceFiles +import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.SingleWorkflowRunnerActor.RunWorkflow import cromwell.engine.workflow.SingleWorkflowRunnerActorSpec._ import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor @@ -65,7 +66,8 @@ abstract class SingleWorkflowRunnerActorSpec extends CromwellTestkitSpec { dummyLogCopyRouter, jobStore, callCacheReadActor, - jobTokenDispenserActor)), "WorkflowManagerActor") + jobTokenDispenserActor, + BackendSingletonCollection(Map.empty))), "WorkflowManagerActor") } def createRunnerActor(sampleWdl: SampleWdl = ThreeStep, managerActor: => ActorRef = workflowManagerActor(), diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala index 860ac6dab..b98c5657f 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala @@ -6,6 +6,7 @@ import com.typesafe.config.{Config, ConfigFactory} import cromwell.backend.AllBackendInitializationData import cromwell.core.{ExecutionStore, OutputStore, WorkflowId, WorkflowSourceFiles} import cromwell.engine.EngineWorkflowDescriptor +import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.WorkflowActor._ import cromwell.engine.workflow.lifecycle.EngineLifecycleActorAbortCommand import cromwell.engine.workflow.lifecycle.WorkflowFinalizationActor.{StartFinalizationCommand, WorkflowFinalizationSucceededResponse} @@ -156,7 +157,7 @@ class MockWorkflowActor(val finalizationProbe: TestProbe, workflowLogCopyRouter: ActorRef, jobStoreActor: ActorRef, callCacheReadActor: ActorRef, - jobTokenDispenserActor: ActorRef) extends WorkflowActor(workflowId, startMode, workflowSources, conf, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor) { + jobTokenDispenserActor: ActorRef) extends WorkflowActor(workflowId, startMode, workflowSources, conf, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, BackendSingletonCollection(Map.empty)) { override def makeFinalizationActor(workflowDescriptor: EngineWorkflowDescriptor, executionStore: ExecutionStore, outputStore: OutputStore) = finalizationProbe.ref } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala index 639d8f250..01af65d1a 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala @@ -5,7 +5,7 @@ import akka.testkit.{EventFilter, TestActorRef, TestDuration, TestProbe} import com.typesafe.config.ConfigFactory import cromwell.backend.AllBackendInitializationData import cromwell.core.WorkflowId -import cromwell.engine.backend.{BackendConfigurationEntry, CromwellBackends} +import cromwell.engine.backend.{BackendConfigurationEntry, BackendSingletonCollection, CromwellBackends} import cromwell.engine.workflow.WorkflowDescriptorBuilder import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.ExecuteWorkflowCommand import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor @@ -29,6 +29,9 @@ class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter } }) + val MockBackendName = "Mock" + val MockBackendSingletonCollection = BackendSingletonCollection(Map(MockBackendName -> None)) + val stubbedConfig = ConfigFactory.load().getConfig("backend.providers.Mock").getConfig("config") val runtimeSection = @@ -64,7 +67,7 @@ class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter val workflowExecutionActor = system.actorOf( WorkflowExecutionActor.props(workflowId, engineWorkflowDescriptor, serviceRegistryActor, jobStoreActor, - callCacheReadActor.ref, jobTokenDispenserActor, AllBackendInitializationData.empty, restarting = false), + callCacheReadActor.ref, jobTokenDispenserActor, MockBackendSingletonCollection, AllBackendInitializationData.empty, restarting = false), "WorkflowExecutionActor") EventFilter.info(pattern = ".*Final Outputs", occurrences = 1).intercept { @@ -87,7 +90,7 @@ class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props) val MockBackendConfigEntry = BackendConfigurationEntry( - name = "Mock", + name = MockBackendName, lifecycleActorFactoryClass = "cromwell.engine.backend.mock.DefaultBackendLifecycleActorFactory", stubbedConfig ) @@ -97,7 +100,7 @@ class WorkflowExecutionActorSpec extends CromwellTestkitSpec with BeforeAndAfter val engineWorkflowDescriptor = createMaterializedEngineWorkflowDescriptor(workflowId, SampleWdl.SimpleScatterWdl.asWorkflowSources(runtime = runtimeSection)) val workflowExecutionActor = system.actorOf( WorkflowExecutionActor.props(workflowId, engineWorkflowDescriptor, serviceRegistry, jobStore, - callCacheReadActor, jobTokenDispenserActor, AllBackendInitializationData.empty, restarting = false), + callCacheReadActor, jobTokenDispenserActor, MockBackendSingletonCollection, AllBackendInitializationData.empty, restarting = false), "WorkflowExecutionActor") val scatterLog = "Starting calls: scatter0.inside_scatter:0:1, scatter0.inside_scatter:1:1, scatter0.inside_scatter:2:1, scatter0.inside_scatter:3:1, scatter0.inside_scatter:4:1" diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala index 80ae87fa9..aa3f5a28c 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala @@ -10,6 +10,7 @@ import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.core.callcaching.{CallCachingActivity, CallCachingMode, CallCachingOff} import cromwell.core.{ExecutionStore, JobExecutionToken, OutputStore, WorkflowId} import cromwell.engine.EngineWorkflowDescriptor +import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.lifecycle.execution.{EngineJobExecutionActor, WorkflowExecutionActorData} import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{EJEAData, EngineJobExecutionActorState} import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CallCacheHashes} @@ -81,7 +82,8 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef): Props = bjeaProps + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props = bjeaProps override def cacheHitCopyingActorProps: Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef) => Props] = Option((_, _, _) => callCacheHitCopyingProbe.props) @@ -142,7 +144,7 @@ private[ejea] class MockEjea(helper: PerTestHelper, callCacheReadActor: ActorRef, jobTokenDispenserActor: ActorRef, backendName: String, - callCachingMode: CallCachingMode) extends EngineJobExecutionActor(replyTo, jobDescriptorKey, executionData, factory, initializationData, restarting, serviceRegistryActor, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendName, callCachingMode) { + callCachingMode: CallCachingMode) extends EngineJobExecutionActor(replyTo, jobDescriptorKey, executionData, factory, initializationData, restarting, serviceRegistryActor, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, None, backendName, callCachingMode) { override def makeFetchCachedResultsActor(cacheHit: CacheHit, taskOutputs: Seq[TaskOutput]) = helper.fetchCachedResultsActorCreations = helper.fetchCachedResultsActorCreations.foundOne((cacheHit, taskOutputs)) override def initializeJobHashing(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity) = helper.jobHashingInitializations = helper.jobHashingInitializations.foundOne((jobDescriptor, activity)) diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala index b71c6a48f..0575b7b33 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala @@ -2,14 +2,15 @@ package cromwell.engine.workflow.tokens import java.util.UUID -import akka.actor.{ ActorRef, ActorSystem, Kill, PoisonPill} +import akka.actor.{ActorRef, ActorSystem, Kill, PoisonPill} import org.scalatest._ import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed, JobExecutionTokenRequest, JobExecutionTokenReturn} import JobExecutionTokenDispenserActorSpec._ import cromwell.core.JobExecutionToken -import cromwell.engine.workflow.tokens.TokenGrabbingActor.StoppingSupervisor +import cromwell.engine.workflow.tokens.TestTokenGrabbingActor.StoppingSupervisor +import cromwell.util.AkkaTestUtil import org.scalatest.concurrent.Eventually import scala.concurrent.duration._ @@ -201,23 +202,15 @@ class JobExecutionTokenDispenserActorSpec extends TestKit(ActorSystem("JETDASpec dummyActors(5).expectNoMsg(MaxWaitTime) } - val actorDeathMethods: List[(String, ActorRef => Unit)] = List( - ("external_stop", (a: ActorRef) => system.stop(a)), - ("internal_stop", (a: ActorRef) => a ! TokenGrabbingActor.InternalStop), - ("poison_pill", (a: ActorRef) => a ! PoisonPill), - ("kill_message", (a: ActorRef) => a ! Kill), - ("throw_exception", (a: ActorRef) => a ! TokenGrabbingActor.ThrowException) - ) - - actorDeathMethods foreach { case (name, stopMethod) => + AkkaTestUtil.actorDeathMethods(system) foreach { case (name, stopMethod) => it should s"recover tokens lost to actors which are $name before they hand back their token" in { - var currentTokens: Map[TestActorRef[TokenGrabbingActor], JobExecutionToken] = Map.empty - var tokenGrabbingActors: Map[Int, TestActorRef[TokenGrabbingActor]] = Map.empty + var currentTokens: Map[TestActorRef[TestTokenGrabbingActor], JobExecutionToken] = Map.empty + var tokenGrabbingActors: Map[Int, TestActorRef[TestTokenGrabbingActor]] = Map.empty val grabberSupervisor = TestActorRef(new StoppingSupervisor()) // Set up by taking all 5 tokens out, and then adding 2 to the queue: 5 indexedTimes { i => - val newGrabbingActor = TestActorRef[TokenGrabbingActor](TokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"grabber_${name}_" + i) + val newGrabbingActor = TestActorRef[TestTokenGrabbingActor](TestTokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"grabber_${name}_" + i) tokenGrabbingActors += i -> newGrabbingActor eventually { newGrabbingActor.underlyingActor.token.isDefined should be(true) @@ -226,7 +219,7 @@ class JobExecutionTokenDispenserActorSpec extends TestKit(ActorSystem("JETDASpec } val unassignedActorIndex = 5 - val newGrabbingActor = TestActorRef(new TokenGrabbingActor(actorRefUnderTest, LimitedTo5Tokens), s"grabber_${name}_" + unassignedActorIndex) + val newGrabbingActor = TestActorRef(new TestTokenGrabbingActor(actorRefUnderTest, LimitedTo5Tokens), s"grabber_${name}_" + unassignedActorIndex) tokenGrabbingActors += unassignedActorIndex -> newGrabbingActor eventually { newGrabbingActor.underlyingActor.rejections should be(1) @@ -245,13 +238,13 @@ class JobExecutionTokenDispenserActorSpec extends TestKit(ActorSystem("JETDASpec } it should "skip over dead actors when assigning tokens to the actor queue" in { - var currentTokens: Map[TestActorRef[TokenGrabbingActor], JobExecutionToken] = Map.empty - var tokenGrabbingActors: Map[Int, TestActorRef[TokenGrabbingActor]] = Map.empty + var currentTokens: Map[TestActorRef[TestTokenGrabbingActor], JobExecutionToken] = Map.empty + var tokenGrabbingActors: Map[Int, TestActorRef[TestTokenGrabbingActor]] = Map.empty val grabberSupervisor = TestActorRef(new StoppingSupervisor()) // Set up by taking all 5 tokens out, and then adding 2 to the queue: 5 indexedTimes { i => - val newGrabbingActor = TestActorRef[TokenGrabbingActor](TokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"skip_test_" + i) + val newGrabbingActor = TestActorRef[TestTokenGrabbingActor](TestTokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"skip_test_" + i) tokenGrabbingActors += i -> newGrabbingActor eventually { newGrabbingActor.underlyingActor.token.isDefined should be(true) @@ -260,7 +253,7 @@ class JobExecutionTokenDispenserActorSpec extends TestKit(ActorSystem("JETDASpec } 2 indexedTimes { i => val index = i + 5 - val newGrabbingActor = TestActorRef[TokenGrabbingActor](TokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"skip_test_" + index) + val newGrabbingActor = TestActorRef[TestTokenGrabbingActor](TestTokenGrabbingActor.props(actorRefUnderTest, LimitedTo5Tokens), grabberSupervisor, s"skip_test_" + index) tokenGrabbingActors += index -> newGrabbingActor eventually { newGrabbingActor.underlyingActor.rejections should be(1) diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/TokenGrabbingActor.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/TestTokenGrabbingActor.scala similarity index 64% rename from engine/src/test/scala/cromwell/engine/workflow/tokens/TokenGrabbingActor.scala rename to engine/src/test/scala/cromwell/engine/workflow/tokens/TestTokenGrabbingActor.scala index 06eff8583..669e6783f 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/tokens/TokenGrabbingActor.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/TestTokenGrabbingActor.scala @@ -4,12 +4,12 @@ import akka.actor.{Actor, ActorRef, Props, SupervisorStrategy} import cromwell.core.JobExecutionToken import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed, JobExecutionTokenRequest} -import cromwell.engine.workflow.tokens.TokenGrabbingActor.{InternalStop, ThrowException} +import cromwell.util.AkkaTestUtil /** * Grabs a token and doesn't let it go! */ -class TokenGrabbingActor(tokenDispenser: ActorRef, tokenType: JobExecutionTokenType) extends Actor { +class TestTokenGrabbingActor(tokenDispenser: ActorRef, tokenType: JobExecutionTokenType) extends Actor { var token: Option[JobExecutionToken] = None var rejections = 0 @@ -17,19 +17,16 @@ class TokenGrabbingActor(tokenDispenser: ActorRef, tokenType: JobExecutionTokenT def receive = { case JobExecutionTokenDispensed(dispensedToken) => token = Option(dispensedToken) case JobExecutionTokenDenied(positionInQueue) => rejections += 1 - case ThrowException => throw new RuntimeException("Test exception (don't be scared by the stack trace, it's deliberate!)") - case InternalStop => context.stop(self) + case AkkaTestUtil.ThrowException => throw new RuntimeException("Test exception (don't be scared by the stack trace, it's deliberate!)") + case AkkaTestUtil.InternalStop => context.stop(self) } tokenDispenser ! JobExecutionTokenRequest(tokenType) } -object TokenGrabbingActor { +object TestTokenGrabbingActor { - def props(tokenDispenserActor: ActorRef, tokenType: JobExecutionTokenType) = Props(new TokenGrabbingActor(tokenDispenserActor, tokenType)) - - case object ThrowException - case object InternalStop + def props(tokenDispenserActor: ActorRef, tokenType: JobExecutionTokenType) = Props(new TestTokenGrabbingActor(tokenDispenserActor, tokenType)) class StoppingSupervisor extends Actor { override val supervisorStrategy = SupervisorStrategy.stoppingStrategy diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala index 7faf6baed..75f2f779a 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala @@ -24,7 +24,8 @@ case class HtCondorBackendFactory(name: String, configurationDescriptor: Backend override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef): Props = { + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props = { HtCondorJobExecutionActor.props(jobDescriptor, configurationDescriptor, serviceRegistryActor, resolveCacheProviderProps(jobDescriptor.workflowDescriptor.workflowOptions)) } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala index b2e5974e0..a08be8df9 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala @@ -17,6 +17,7 @@ import cromwell.backend.impl.jes.JesImplicits.PathString import cromwell.backend.impl.jes.JesJobExecutionActor.JesOperationIdKey import cromwell.backend.impl.jes.RunStatus.TerminalRunStatus import cromwell.backend.impl.jes.io._ +import cromwell.backend.impl.jes.statuspolling.JesPollingActorClient import cromwell.backend.{AttemptedLookupResult, BackendJobDescriptor, BackendWorkflowDescriptor, PreemptedException} import cromwell.core.Dispatcher.BackendDispatcher import cromwell.core._ @@ -44,12 +45,14 @@ object JesAsyncBackendJobExecutionActor { completionPromise: Promise[BackendJobExecutionResponse], jesWorkflowInfo: JesConfiguration, initializationData: JesBackendInitializationData, - serviceRegistryActor: ActorRef): Props = { + serviceRegistryActor: ActorRef, + jesBackendSingletonActor: ActorRef): Props = { Props(new JesAsyncBackendJobExecutionActor(jobDescriptor, completionPromise, jesWorkflowInfo, initializationData, - serviceRegistryActor)).withDispatcher(BackendDispatcher) + serviceRegistryActor, + jesBackendSingletonActor)).withDispatcher(BackendDispatcher) } object WorkflowOptionKeys { @@ -80,11 +83,14 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes override val completionPromise: Promise[BackendJobExecutionResponse], override val jesConfiguration: JesConfiguration, override val initializationData: JesBackendInitializationData, - override val serviceRegistryActor: ActorRef) - extends Actor with ActorLogging with AsyncBackendJobExecutionActor with JesJobCachingActorHelper with JobLogging { + override val serviceRegistryActor: ActorRef, + val jesBackendSingletonActor: ActorRef) + extends Actor with ActorLogging with AsyncBackendJobExecutionActor with JesJobCachingActorHelper with JobLogging with JesPollingActorClient { import JesAsyncBackendJobExecutionActor._ + override val pollingActor = jesBackendSingletonActor + override lazy val pollBackOff = SimpleExponentialBackoff( initialInterval = 30 seconds, maxInterval = 10 minutes, multiplier = 1.1) @@ -121,7 +127,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes case KvPutSuccess(_) => // expected after the KvPut for the operation ID } - override def receive: Receive = jesReceiveBehavior orElse super.receive + override def receive: Receive = pollingActorClientReceive orElse jesReceiveBehavior orElse super.receive private def globOutputPath(glob: String) = callRootPath.resolve(s"glob-${glob.md5Sum}/") @@ -366,50 +372,55 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes /** * Update the ExecutionHandle */ - override def poll(previous: ExecutionHandle)(implicit ec: ExecutionContext): Future[ExecutionHandle] = Future { + override def poll(previous: ExecutionHandle)(implicit ec: ExecutionContext): Future[ExecutionHandle] = { previous match { case handle: JesPendingExecutionHandle => - val runId = handle.run.runId - jobLogger.debug(s"$tag Polling JES Job $runId") - val previousStatus = handle.previousStatus - val status = Try(handle.run.status()) - status foreach { currentStatus => - if (!(handle.previousStatus contains currentStatus)) { - // If this is the first time checking the status, we log the transition as '-' to 'currentStatus'. Otherwise - // just use the state names. - val prevStateName = previousStatus map { _.toString } getOrElse "-" - jobLogger.info(s"$tag Status change from $prevStateName to $currentStatus") - tellMetadata(Map("backendStatus" -> currentStatus)) - } - } - status match { - case Success(s: TerminalRunStatus) => - val metadata = Map( - JesMetadataKeys.MachineType -> s.machineType.getOrElse("unknown"), - JesMetadataKeys.InstanceName -> s.instanceName.getOrElse("unknown"), - JesMetadataKeys.Zone -> s.zone.getOrElse("unknown") - ) - - tellMetadata(metadata) - executionResult(s, handle) - case Success(s) => handle.copy(previousStatus = Option(s)).future // Copy the current handle with updated previous status. - case Failure(e: GoogleJsonResponseException) if e.getStatusCode == 404 => - jobLogger.error(s"$tag JES Job ID ${handle.run.runId} has not been found, failing call") - FailedNonRetryableExecutionHandle(e).future - case Failure(e: Exception) => - // Log exceptions and return the original handle to try again. - jobLogger.warn(s"Caught exception, retrying", e) - handle.future - case Failure(e: Error) => Future.failed(e) // JVM-ending calamity. - case Failure(throwable) => - // Someone has subclassed Throwable directly? - FailedNonRetryableExecutionHandle(throwable).future - } + jobLogger.debug(s"$tag Polling JES Job ${handle.run.runId}") + pollStatus(handle.run) map updateExecutionHandleSuccess(handle) recover updateExecutionHandleFailure(handle) flatten case f: FailedNonRetryableExecutionHandle => f.future case s: SuccessfulExecutionHandle => s.future case badHandle => Future.failed(new IllegalArgumentException(s"Unexpected execution handle: $badHandle")) } - } flatten + } + + private def updateExecutionHandleFailure(oldHandle: JesPendingExecutionHandle): PartialFunction[Throwable, Future[ExecutionHandle]] = { + case e: GoogleJsonResponseException if e.getStatusCode == 404 => + jobLogger.error(s"$tag JES Job ID ${oldHandle.run.runId} has not been found, failing call") + FailedNonRetryableExecutionHandle(e).future + case e: Exception => + // Log exceptions and return the original handle to try again. + jobLogger.warn(s"Caught exception, retrying", e) + oldHandle.future + case e: Error => Future.failed(e) // JVM-ending calamity. + case throwable => + // Someone has subclassed Throwable directly? + FailedNonRetryableExecutionHandle(throwable).future + } + + private def updateExecutionHandleSuccess(oldHandle: JesPendingExecutionHandle)(status: RunStatus): Future[ExecutionHandle] = { + val previousStatus = oldHandle.previousStatus + if (!(previousStatus contains status)) { + // If this is the first time checking the status, we log the transition as '-' to 'currentStatus'. Otherwise + // just use the state names. + val prevStateName = previousStatus map { _.toString } getOrElse "-" + jobLogger.info(s"$tag Status change from $prevStateName to $status") + tellMetadata(Map("backendStatus" -> status)) + } + + status match { + case s: TerminalRunStatus => + val metadata = Map( + JesMetadataKeys.MachineType -> s.machineType.getOrElse("unknown"), + JesMetadataKeys.InstanceName -> s.instanceName.getOrElse("unknown"), + JesMetadataKeys.Zone -> s.zone.getOrElse("unknown") + ) + + tellMetadata(metadata) + executionResult(s, oldHandle) + case s => oldHandle.copy(previousStatus = Option(s)).future // Copy the current handle with updated previous status. + + } + } /** * Fire and forget start info to the metadata service @@ -512,19 +523,19 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes errorMessage.substring(0, errorMessage.indexOf(':')).toInt } - private def preempted(errorCode: Int, errorMessage: Option[String]): Boolean = { + private def preempted(errorCode: Int, errorMessage: List[String]): Boolean = { def isPreemptionCode(code: Int) = code == 13 || code == 14 try { - errorCode == 10 && errorMessage.isDefined && isPreemptionCode(extractErrorCodeFromErrorMessage(errorMessage.get)) && preemptible + errorCode == 10 && errorMessage.exists(e => isPreemptionCode(extractErrorCodeFromErrorMessage(e))) && preemptible } catch { case _: NumberFormatException | _: StringIndexOutOfBoundsException => - jobLogger.warn(s"Unable to parse JES error code from error message: {}, assuming this was not a preempted VM.", errorMessage.get) + jobLogger.warn(s"Unable to parse JES error code from error messages: [{}], assuming this was not a preempted VM.", errorMessage.mkString(", ")) false } } - private def handleFailure(errorCode: Int, errorMessage: Option[String]) = { + private def handleFailure(errorCode: Int, errorMessage: List[String]) = { import lenthall.numeric.IntegerUtil._ val taskName = s"${workflowDescriptor.id}:${call.unqualifiedName}" @@ -550,7 +561,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes } else { val id = workflowDescriptor.id val name = jobDescriptor.call.unqualifiedName - val message = errorMessage.getOrElse("null") + val message = if (errorMessage.isEmpty) "null" else errorMessage.mkString(", ") val exception = new RuntimeException(s"Task $id:$name failed: error code $errorCode. Message: $message") FailedNonRetryableExecutionHandle(exception, None).future } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala index 1e87b4d1f..92f653c10 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala @@ -28,10 +28,11 @@ case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef): Props = { + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props = { // The `JesInitializationActor` will only return a non-`Empty` `JesBackendInitializationData` from a successful `beforeAll` // invocation, so the `get` here is safe. - JesJobExecutionActor.props(jobDescriptor, jesConfiguration, initializationData.toJes.get, serviceRegistryActor).withDispatcher(BackendDispatcher) + JesJobExecutionActor.props(jobDescriptor, jesConfiguration, initializationData.toJes.get, serviceRegistryActor, backendSingletonActor).withDispatcher(BackendDispatcher) } override def cacheHitCopyingActorProps = Option(cacheHitCopyingActorInner _) @@ -71,6 +72,8 @@ case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor initializationData.toJes.get.workflowPaths.rootPath } + override def backendSingletonActorProps = Option(JesBackendSingletonActor.props()) + override lazy val fileHashingFunction: Option[FileHashingFunction] = Option(FileHashingFunction(JesBackendFileHashing.getCrc32c)) } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala new file mode 100644 index 000000000..3b830107a --- /dev/null +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala @@ -0,0 +1,20 @@ +package cromwell.backend.impl.jes + +import akka.actor.{Actor, ActorLogging, Props} +import cromwell.backend.impl.jes.statuspolling.{JesApiQueryManager} +import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll + +class JesBackendSingletonActor extends Actor with ActorLogging { + + val pollingActor = context.actorOf(JesApiQueryManager.props) + + override def receive = { + case poll: DoPoll => + log.debug("Forwarding status poll to JES polling actor") + pollingActor.forward(poll) + } +} + +object JesBackendSingletonActor { + def props(): Props = Props(new JesBackendSingletonActor()) +} diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala index eed3268f5..b1a8ba8d6 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala @@ -19,8 +19,9 @@ object JesJobExecutionActor { def props(jobDescriptor: BackendJobDescriptor, jesWorkflowInfo: JesConfiguration, initializationData: JesBackendInitializationData, - serviceRegistryActor: ActorRef): Props = { - Props(new JesJobExecutionActor(jobDescriptor, jesWorkflowInfo, initializationData, serviceRegistryActor)) + serviceRegistryActor: ActorRef, + jesBackendSingletonActor: Option[ActorRef]): Props = { + Props(new JesJobExecutionActor(jobDescriptor, jesWorkflowInfo, initializationData, serviceRegistryActor, jesBackendSingletonActor)) } val JesOperationIdKey = "__jes_operation_id" @@ -29,9 +30,12 @@ object JesJobExecutionActor { case class JesJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, jesConfiguration: JesConfiguration, initializationData: JesBackendInitializationData, - serviceRegistryActor: ActorRef) + serviceRegistryActor: ActorRef, + jesBackendSingletonActorOption: Option[ActorRef]) extends BackendJobExecutionActor { + val jesBackendSingletonActor = jesBackendSingletonActorOption.getOrElse(throw new RuntimeException("JES Backend actor cannot exist without the JES backend singleton actor")) + private def jesReceiveBehavior: Receive = LoggingReceive { case AbortJobCommand => executor.foreach(_ ! AbortJobCommand) @@ -63,7 +67,8 @@ case class JesJobExecutionActor(override val jobDescriptor: BackendJobDescriptor completionPromise, jesConfiguration, initializationData, - serviceRegistryActor) + serviceRegistryActor, + jesBackendSingletonActor) val executorRef = context.actorOf(executionProps, "JesAsyncBackendJobExecutionActor") executor = Option(executorRef) () diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala index 0f948344c..b5a8b5f9f 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala @@ -90,24 +90,19 @@ object Run { implicit class RunOperationExtension(val operation: Operation) extends AnyVal { def hasStarted = operation.getMetadata.asScala.get("startTime") isDefined } -} - -case class Run(runId: String, genomicsInterface: Genomics) { - import Run._ - def status(): RunStatus = { - val op = genomicsInterface.operations().get(runId).execute + def interpretOperationStatus(op: Operation): RunStatus = { if (op.getDone) { - val eventList = getEventList(op) - val ceInfo = op.getMetadata.get ("runtimeMetadata").asInstanceOf[GArrayMap[String,Object]].get("computeEngine").asInstanceOf[GArrayMap[String, String]] - val machineType = Option(ceInfo.get("machineType")) - val instanceName = Option(ceInfo.get("instanceName")) - val zone = Option(ceInfo.get("zone")) + lazy val eventList = getEventList(op) + lazy val ceInfo = op.getMetadata.get ("runtimeMetadata").asInstanceOf[GArrayMap[String,Object]].get("computeEngine").asInstanceOf[GArrayMap[String, String]] + lazy val machineType = Option(ceInfo.get("machineType")) + lazy val instanceName = Option(ceInfo.get("instanceName")) + lazy val zone = Option(ceInfo.get("zone")) // If there's an error, generate a Failed status. Otherwise, we were successful! Option(op.getError) match { case None => Success(eventList, machineType, zone, instanceName) - case Some(error) => Failed(error.getCode, Option(error.getMessage), eventList, machineType, zone, instanceName) + case Some(error) => Failed(error.getCode, Option(error.getMessage).toList, eventList, machineType, zone, instanceName) } } else if (op.hasStarted) { Running @@ -156,6 +151,11 @@ case class Run(runId: String, genomicsInterface: Genomics) { private def eventIfExists(name: String, metadata: Map[String, AnyRef], eventName: String): Option[ExecutionEvent] = { metadata.get(name) map { time => ExecutionEvent(eventName, OffsetDateTime.parse(time.toString)) } } +} + +case class Run(runId: String, genomicsInterface: Genomics) { + + def getOperationCommand = genomicsInterface.operations().get(runId) def abort(): Unit = { val cancellationRequest: CancelOperationRequest = new CancelOperationRequest() diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala index 7a2e31b77..d0609573e 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala @@ -27,7 +27,7 @@ object RunStatus { override def toString = "Success" } - final case class Failed(errorCode: Int, errorMessage: Option[String], eventList: Seq[ExecutionEvent], machineType: Option[String], zone: Option[String], instanceName: Option[String]) + final case class Failed(errorCode: Int, errorMessage: List[String], eventList: Seq[ExecutionEvent], machineType: Option[String], zone: Option[String], instanceName: Option[String]) extends TerminalRunStatus { // Don't want to include errorMessage or code in the snappy status toString: override def toString = "Failed" diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala new file mode 100644 index 000000000..f3a7739ac --- /dev/null +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala @@ -0,0 +1,108 @@ +package cromwell.backend.impl.jes.statuspolling + +import akka.actor.{Actor, ActorLogging, ActorRef, Props, SupervisorStrategy, Terminated} +import cats.data.NonEmptyList +import cromwell.backend.impl.jes.Run +import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager._ + +import scala.collection.immutable.Queue + +/** + * Currently, just holds a set of JES status poll requests until a PollingActor pulls the work. + * TODO: Could eventually move all of the JES queries into a single work-pulling model. + */ +class JesApiQueryManager extends Actor with ActorLogging { + + private var workQueue: Queue[JesStatusPollQuery] = Queue.empty + private var workInProgress: Map[ActorRef, JesPollingWorkBatch] = Map.empty + + // If the statusPoller dies, we want to stop it and handle the termination ourselves. + override val supervisorStrategy = SupervisorStrategy.stoppingStrategy + private def statusPollerProps = JesPollingActor.props(self) + private var statusPoller: ActorRef = _ + resetStatusPoller() + + override def receive = { + case DoPoll(run) => workQueue :+= JesStatusPollQuery(sender, run) + case RequestJesPollingWork(maxBatchSize) => + log.debug(s"Request for JES Polling Work received (max batch: $maxBatchSize, current queue size is ${workQueue.size})") + handleJesPollingRequest(sender, maxBatchSize) + case Terminated(actorRef) => handleTerminated(actorRef) + case other => log.error(s"Unexpected message to JesPollingManager: $other") + } + + /** + * !! Function Impurity Warning: Modifies Actor Data !! + */ + private def handleJesPollingRequest(workPullingJesPollingActor: ActorRef, maxBatchSize: Int) = { + workInProgress -= workPullingJesPollingActor + val beheaded = beheadWorkQueue(maxBatchSize) + beheaded.workToDo match { + case Some(work) => + sendWork(workPullingJesPollingActor, JesPollingWorkBatch(work)) + case None => + log.debug(s"No work for JES poller. Sad.") + workPullingJesPollingActor ! NoWorkToDo + } + + workQueue = beheaded.newWorkQueue + } + + private def sendWork(workPullingJesPollingActor: ActorRef, work: JesPollingWorkBatch) = { + log.debug(s"Sending work to JES poller.") + workPullingJesPollingActor ! work + workInProgress += (workPullingJesPollingActor -> work) + } + + private final case class BeheadedWorkQueue(workToDo: Option[NonEmptyList[JesStatusPollQuery]], newWorkQueue: Queue[JesStatusPollQuery]) + private def beheadWorkQueue(maxBatchSize: Int): BeheadedWorkQueue = { + + val head = workQueue.take(maxBatchSize).toList + val tail = workQueue.drop(maxBatchSize) + + head match { + case h :: t => BeheadedWorkQueue(Option(NonEmptyList(h, t)), tail) + case Nil => BeheadedWorkQueue(None, Queue.empty) + } + } + + private def handleTerminated(terminee: ActorRef) = { + // Currently we can assume this is a polling actor. Might change in a future update: + workInProgress.get(terminee) match { + case Some(work) => + // Ouch. We should tell all of its clients that it fell over. And then start a new one. + log.error(s"The JES polling actor $terminee unexpectedly terminated while conducting ${work.workBatch.tail.size + 1} polls. Making a new one...") + work.workBatch.toList foreach { _.requester ! JesPollingActor.JesPollError } + case None => + // It managed to die while doing absolutely nothing...!? + // Maybe it deserves an entry in https://en.wikipedia.org/wiki/List_of_unusual_deaths + // Oh well, in the mean time don't do anything, just start a new one + log.error(s"The JES polling actor $terminee managed to unexpectedly terminate whilst doing absolutely nothing. This is probably a programming error. Making a new one...") + } + resetStatusPoller() + } + + private def resetStatusPoller() = { + statusPoller = makeStatusPoller() + context.watch(statusPoller) + log.info(s"watching $statusPoller") + } + + private[statuspolling] def makeStatusPoller(): ActorRef = context.actorOf(statusPollerProps) +} + +object JesApiQueryManager { + + def props: Props = Props(new JesApiQueryManager) + + /** + * Poll the job represented by the Run. + */ + final case class DoPoll(run: Run) + + private[statuspolling] final case class JesStatusPollQuery(requester: ActorRef, run: Run) + private[statuspolling] final case class JesPollingWorkBatch(workBatch: NonEmptyList[JesStatusPollQuery]) + private[statuspolling] case object NoWorkToDo + + private[statuspolling] final case class RequestJesPollingWork(maxBatchSize: Int) +} diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala new file mode 100644 index 000000000..4152d3933 --- /dev/null +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala @@ -0,0 +1,122 @@ +package cromwell.backend.impl.jes.statuspolling + +import akka.actor.{Actor, ActorLogging, ActorRef, Props} +import cats.data.NonEmptyList +import com.google.api.client.googleapis.batch.BatchRequest +import com.google.api.client.googleapis.batch.json.JsonBatchCallback +import com.google.api.client.googleapis.json.GoogleJsonError +import com.google.api.client.http.HttpHeaders +import com.google.api.services.genomics.model.Operation +import cromwell.backend.impl.jes.Run +import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.{JesPollingWorkBatch, JesStatusPollQuery, NoWorkToDo} +import cromwell.backend.impl.jes.statuspolling.JesPollingActor._ + +import scala.collection.JavaConversions._ +import scala.concurrent.{ExecutionContext, Future, Promise} +import scala.util.{Failure, Success, Try} +import scala.concurrent.duration._ + +/** + * Polls JES for status. Pipes the results back (so expect either a RunStatus or a akka.actor.Status.Failure). + */ +class JesPollingActor(pollingManager: ActorRef) extends Actor with ActorLogging { + + // We want to query at just under our fixed JES QPS limit of 20 per second. That should hopefully allow some room at the edges + // for things like new calls, etc. + val MaxBatchSize = 100 + val BatchInterval = 5.5.seconds + self ! NoWorkToDo // Starts the check-for-work cycle + + implicit val ec: ExecutionContext = context.dispatcher + + override def receive = { + + case JesPollingWorkBatch(workBatch) => + log.debug(s"Got a polling batch with ${workBatch.tail.size + 1} requests.") + val batchResultFutures = handleBatch(workBatch) + val overallFuture = Future.sequence(batchResultFutures.toList) + overallFuture.andThen(interstitialRecombobulation) + () + case NoWorkToDo => + scheduleCheckForWork() + } + + private def handleBatch(workBatch: NonEmptyList[JesStatusPollQuery]): NonEmptyList[Future[Try[Unit]]] = { + + // Assume that the auth for the first element is also able to query the remaining Runs + val batch: BatchRequest = createBatch(workBatch.head.run) + + // Create the batch: + val batchFutures = workBatch map { pollingRequest => + val completionPromise = Promise[Try[Unit]]() + val resultHandler = batchResultHandler(pollingRequest.requester, completionPromise) + enqueueStatusPollInBatch(pollingRequest.run, batch, resultHandler) + completionPromise.future + } + + // Execute the batch and return the map: + runBatch(batch) + batchFutures + } + + // These are separate functions so that the tests can hook in and replace the JES-side stuff + private[statuspolling] def createBatch(run: Run): BatchRequest = run.genomicsInterface.batch() + private[statuspolling] def enqueueStatusPollInBatch(run: Run, batch: BatchRequest, resultHandler: JsonBatchCallback[Operation]) = { + run.getOperationCommand.queue(batch, resultHandler) + } + private[statuspolling] def runBatch(batch: BatchRequest) = batch.execute() + + private def batchResultHandler(originalRequester: ActorRef, completionPromise: Promise[Try[Unit]]) = new JsonBatchCallback[Operation] { + override def onSuccess(operation: Operation, responseHeaders: HttpHeaders): Unit = { + log.debug(s"Batch result onSuccess callback triggered!") + originalRequester ! interpretOperationStatus(operation) + completionPromise.trySuccess(Success(())) + () + } + + override def onFailure(e: GoogleJsonError, responseHeaders: HttpHeaders): Unit = { + log.debug(s"Batch request onFailure callback triggered!") + originalRequester ! JesPollFailed(e, responseHeaders) + completionPromise.trySuccess(Failure(new Exception(mkErrorString(e)))) + () + } + } + + private[statuspolling] def mkErrorString(e: GoogleJsonError) = e.getErrors.toList.mkString(", ") + private[statuspolling] def interpretOperationStatus(operation: Operation) = Run.interpretOperationStatus(operation) + + // TODO: FSMify this actor? + private def interstitialRecombobulation: PartialFunction[Try[List[Try[Unit]]], Unit] = { + case Success(allSuccesses) if allSuccesses.forall(_.isSuccess) => + log.debug(s"All status polls completed successfully.") + scheduleCheckForWork() + case Success(someFailures) => + val errors = someFailures collect { case Failure(t) => t.getMessage } + if (log.isDebugEnabled) { + log.warning("{} failures fetching JES statuses", errors.size) + } else { + log.warning("{} failures fetching JES statuses: {}", errors.size, errors.mkString(", ")) + } + scheduleCheckForWork() + case Failure(t) => + // NB: Should be impossible since we only ever do completionPromise.trySuccess() + log.error("Completion promise unexpectedly set to Failure: {}", t.getMessage) + scheduleCheckForWork() + } + + /** + * Schedules a check for work. + * Warning: Only use this from inside a receive method. + */ + private def scheduleCheckForWork(): Unit = { + context.system.scheduler.scheduleOnce(BatchInterval) { pollingManager ! JesApiQueryManager.RequestJesPollingWork(MaxBatchSize) } + () + } +} + +object JesPollingActor { + def props(pollingManager: ActorRef) = Props(new JesPollingActor(pollingManager)) + + final case class JesPollFailed(e: GoogleJsonError, responseHeaders: HttpHeaders) + case object JesPollError +} diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorClient.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorClient.scala new file mode 100644 index 000000000..1bf7328f8 --- /dev/null +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorClient.scala @@ -0,0 +1,51 @@ +package cromwell.backend.impl.jes.statuspolling + +import java.io.IOException + +import akka.actor.{Actor, ActorLogging, ActorRef} +import cromwell.backend.impl.jes.statuspolling.JesPollingActor.{JesPollError, JesPollFailed} +import cromwell.backend.impl.jes.{Run, RunStatus} + +import scala.concurrent.{Future, Promise} +import scala.util.{Failure, Success, Try} + +/** + * I'm putting this stuff in a mixin to avoid polluting the main class. + * + * Be sure to make the main class's receive look like: + * override def receive = pollingActorClientReceive orElse { ... } + */ +trait JesPollingActorClient { this: Actor with ActorLogging => + + private var pollingActorClientPromise: Option[Promise[RunStatus]] = None + + val pollingActor: ActorRef + + def pollingActorClientReceive: Actor.Receive = { + case r: RunStatus => + log.debug(s"Polled status received: $r") + completePromise(Success(r)) + case JesPollFailed(e, responseHeaders) => + log.debug("JES poll failed! Sad.") + completePromise(Failure(new IOException(s"Google request failed: ${e.toPrettyString}"))) + case JesPollError => + log.debug("JES poll failed when polling actor died unexpectedly! Sad.") + completePromise(Failure(new RuntimeException("Unexpected actor death!"))) + } + + private def completePromise(runStatus: Try[RunStatus]) = { + pollingActorClientPromise foreach { _.complete(runStatus) } + pollingActorClientPromise = None + } + + def pollStatus(run: Run): Future[RunStatus] = { + pollingActorClientPromise match { + case Some(p) => p.future + case None => + pollingActor ! JesApiQueryManager.DoPoll(run) + val newPromise = Promise[RunStatus]() + pollingActorClientPromise = Option(newPromise) + newPromise.future + } + } +} diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala index 2a2ae3ec9..5601e1ebb 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala @@ -5,7 +5,7 @@ import java.util.UUID import akka.actor.{ActorRef, Props} import akka.event.LoggingAdapter -import akka.testkit.{ImplicitSender, TestActorRef, TestDuration} +import akka.testkit.{ImplicitSender, TestActorRef, TestDuration, TestProbe} import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse import cromwell.backend.async.AsyncBackendJobExecutionActor.{Execute, ExecutionMode} import cromwell.backend.async.{AbortedExecutionHandle, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle} @@ -30,6 +30,8 @@ import wdl4s.{Call, LocallyQualifiedName, NamespaceWithWorkflow} import scala.concurrent.duration._ import scala.concurrent.{Await, ExecutionContext, Future, Promise} import scala.util.{Success, Try} +import cromwell.backend.impl.jes.MockObjects._ +import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackendJobExecutionActorSpec") with FlatSpecLike with Matchers with ImplicitSender with Mockito { @@ -79,8 +81,9 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend class TestableJesJobExecutionActor(jobDescriptor: BackendJobDescriptor, promise: Promise[BackendJobExecutionResponse], jesConfiguration: JesConfiguration, - functions: JesExpressionFunctions = TestableJesExpressionFunctions) - extends JesAsyncBackendJobExecutionActor(jobDescriptor, promise, jesConfiguration, buildInitializationData(jobDescriptor, jesConfiguration), emptyActor) { + functions: JesExpressionFunctions = TestableJesExpressionFunctions, + jesSingletonActor: ActorRef = emptyActor) + extends JesAsyncBackendJobExecutionActor(jobDescriptor, promise, jesConfiguration, buildInitializationData(jobDescriptor, jesConfiguration), emptyActor, jesSingletonActor) { override lazy val jobLogger = new LoggerWrapper { override def akkaLogger: Option[LoggingAdapter] = Option(log) @@ -121,30 +124,33 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend private def executionActor(jobDescriptor: BackendJobDescriptor, configurationDescriptor: BackendConfigurationDescriptor, promise: Promise[BackendJobExecutionResponse], - errorCode: Int, - innerErrorCode: Int): ActorRef = { + jesSingletonActor: ActorRef): ActorRef = { // Mock/stub out the bits that would reach out to JES. val run = mock[Run] - run.status() returns Failed(errorCode, Option(s"$innerErrorCode: I seen some things man"), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")) - val handle = JesPendingExecutionHandle(jobDescriptor, Seq.empty, run, None) - class ExecuteOrRecoverActor extends TestableJesJobExecutionActor(jobDescriptor, promise, jesConfiguration) { + class ExecuteOrRecoverActor extends TestableJesJobExecutionActor(jobDescriptor, promise, jesConfiguration, jesSingletonActor = jesSingletonActor) { override def executeOrRecover(mode: ExecutionMode)(implicit ec: ExecutionContext): Future[ExecutionHandle] = Future.successful(handle) } system.actorOf(Props(new ExecuteOrRecoverActor), "ExecuteOrRecoverActor-" + UUID.randomUUID) } - private def run(attempt: Int, preemptible: Int, errorCode: Int, innerErrorCode: Int): BackendJobExecutionResponse = { - within(Timeout) { - val promise = Promise[BackendJobExecutionResponse]() - val jobDescriptor = buildPreemptibleJobDescriptor(attempt, preemptible) - val backend = executionActor(jobDescriptor, JesBackendConfigurationDescriptor, promise, errorCode, innerErrorCode) - backend ! Execute - Await.result(promise.future, Timeout) + private def runAndFail(attempt: Int, preemptible: Int, errorCode: Int, innerErrorCode: Int): BackendJobExecutionResponse = { + + val runStatus = Failed(errorCode, List(s"$innerErrorCode: I seen some things man"), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")) + val statusPoller = TestProbe() + + val promise = Promise[BackendJobExecutionResponse]() + val jobDescriptor = buildPreemptibleJobDescriptor(attempt, preemptible) + val backend = executionActor(jobDescriptor, JesBackendConfigurationDescriptor, promise, statusPoller.ref) + backend ! Execute + statusPoller.expectMsgPF(max = Timeout, hint = "awaiting status poll") { + case DoPoll(_) => backend ! runStatus } + + Await.result(promise.future, Timeout) } def buildPreemptibleTestActorRef(attempt: Int, preemptible: Int): TestActorRef[TestableJesJobExecutionActor] = { @@ -180,7 +186,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend expectations foreach { case (attempt, preemptible, errorCode, innerErrorCode, shouldRetry) => it should s"handle call failures appropriately with respect to preemption (attempt=$attempt, preemptible=$preemptible, errorCode=$errorCode, innerErrorCode=$innerErrorCode)" in { - run(attempt, preemptible, errorCode, innerErrorCode).getClass.getSimpleName match { + runAndFail(attempt, preemptible, errorCode, innerErrorCode).getClass.getSimpleName match { case "FailedNonRetryableResponse" => false shouldBe shouldRetry case "FailedRetryableResponse" => true shouldBe shouldRetry case huh => fail(s"Unexpected response class name: '$huh'") @@ -195,7 +201,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val handle = mock[JesPendingExecutionHandle] implicit val ec = system.dispatcher - val failedStatus = Failed(10, Some("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")) + val failedStatus = Failed(10, List("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")) val executionResult = Await.result(jesBackend.executionResult(failedStatus, handle), 2.seconds) executionResult.isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true val failedHandle = executionResult.asInstanceOf[FailedNonRetryableExecutionHandle] @@ -208,7 +214,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val handle = mock[JesPendingExecutionHandle] implicit val ec = system.dispatcher - val failedStatus = Failed(10, Some("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")) + val failedStatus = Failed(10, List("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")) val executionResult = Await.result(jesBackend.executionResult(failedStatus, handle), 2.seconds) executionResult.isInstanceOf[FailedRetryableExecutionHandle] shouldBe true val retryableHandle = executionResult.asInstanceOf[FailedRetryableExecutionHandle] @@ -224,7 +230,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val handle = mock[JesPendingExecutionHandle] implicit val ec = system.dispatcher - val failedStatus = Failed(10, Some("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")) + val failedStatus = Failed(10, List("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")) val executionResult = Await.result(jesBackend.executionResult(failedStatus, handle), 2.seconds) executionResult.isInstanceOf[FailedRetryableExecutionHandle] shouldBe true val retryableHandle = executionResult.asInstanceOf[FailedRetryableExecutionHandle] @@ -241,22 +247,22 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend implicit val ec = system.dispatcher Await.result(jesBackend.executionResult( - Failed(10, Some("15: Other type of error."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds + Failed(10, List("15: Other type of error."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true Await.result(jesBackend.executionResult( - Failed(11, Some("14: Wrong errorCode."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds + Failed(11, List("14: Wrong errorCode."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true Await.result(jesBackend.executionResult( - Failed(10, Some("Weird error message."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds + Failed(10, List("Weird error message."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true Await.result(jesBackend.executionResult( - Failed(10, Some("UnparsableInt: Even weirder error message."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds + Failed(10, List("UnparsableInt: Even weirder error message."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true Await.result(jesBackend.executionResult( - Failed(10, None, Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds + Failed(10, List.empty, Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true Await.result(jesBackend.executionResult( - Failed(10, Some("Operation canceled at"), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds + Failed(10, List("Operation canceled at"), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds ) shouldBe AbortedExecutionHandle actorRef.stop() diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala index 0486d9ec5..39430abb2 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala @@ -38,8 +38,7 @@ class RunSpec extends FlatSpec with Matchers with MockitoTrait { val mockedCredentials = new MockGoogleCredential.Builder().build() val genomics = new Genomics(mockedCredentials.getTransport, mockedCredentials.getJsonFactory, mockedCredentials) - val run = new Run("runId", genomics) - val list = run.getEventList(op) + val list = Run.getEventList(op) list should contain theSameElementsAs List( ExecutionEvent("waiting for quota", OffsetDateTime.parse("2015-12-05T00:00:00+00:00")), ExecutionEvent("initializing VM", OffsetDateTime.parse("2015-12-05T00:00:01+00:00")), diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala new file mode 100644 index 000000000..1eaa42297 --- /dev/null +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala @@ -0,0 +1,153 @@ +package cromwell.backend.impl.jes.statuspolling + +import akka.actor.{ActorRef, Props} +import akka.testkit.{TestActorRef, TestProbe} +import cromwell.backend.impl.jes.Run +import cromwell.core.TestKitSuite +import org.scalatest.{FlatSpecLike, Matchers} + +import scala.concurrent.duration._ +import akka.testkit._ +import JesApiQueryManagerSpec._ +import cromwell.util.AkkaTestUtil +import org.scalatest.concurrent.Eventually + +import scala.collection.immutable.Queue + +class JesApiQueryManagerSpec extends TestKitSuite("JesApiQueryManagerSpec") with FlatSpecLike with Matchers with Eventually { + + behavior of "JesApiQueryManagerSpec" + + implicit val TestExecutionTimeout = 10.seconds.dilated + implicit val DefaultPatienceConfig = PatienceConfig(TestExecutionTimeout) + val AwaitAlmostNothing = 30.milliseconds.dilated + val BatchSize = 5 + + it should "queue up and dispense status poll requests, in order" in { + val statusPoller = TestProbe(name = "StatusPoller") + val jaqmActor: TestActorRef[TestJesApiQueryManager] = TestActorRef(TestJesApiQueryManager.props(statusPoller.ref)) + + var statusRequesters = ((0 until BatchSize * 2) map { i => i -> TestProbe(name = s"StatusRequester_$i") }).toMap + + // Initially, we should have no work: + jaqmActor.tell(msg = JesApiQueryManager.RequestJesPollingWork(BatchSize), sender = statusPoller.ref) + statusPoller.expectMsg(max = TestExecutionTimeout, obj = JesApiQueryManager.NoWorkToDo) + + // Send a few status poll requests: + statusRequesters foreach { case (index, probe) => + jaqmActor.tell(msg = JesApiQueryManager.DoPoll(Run(index.toString, null)), sender = probe.ref) + } + + // Should have no messages to the actual statusPoller yet: + statusPoller.expectNoMsg(max = AwaitAlmostNothing) + + // Verify batches: + 2 times { + jaqmActor.tell(msg = JesApiQueryManager.RequestJesPollingWork(BatchSize), sender = statusPoller.ref) + statusPoller.expectMsgPF(max = TestExecutionTimeout) { + case JesApiQueryManager.JesPollingWorkBatch(workBatch) => + val requesters = statusRequesters.take(BatchSize) + statusRequesters = statusRequesters.drop(BatchSize) + + val zippedWithRequesters = workBatch.toList.zip(requesters) + zippedWithRequesters foreach { case (pollQuery, (index, testProbe)) => + pollQuery.requester should be(testProbe.ref) + pollQuery.run.runId should be(index.toString) + } + } + } + + // Finally, we should have no work: + jaqmActor.tell(msg = JesApiQueryManager.RequestJesPollingWork(BatchSize), sender = statusPoller.ref) + statusPoller.expectMsg(max = TestExecutionTimeout, obj = JesApiQueryManager.NoWorkToDo) + + jaqmActor.underlyingActor.testPollerCreations should be(1) + } + + AkkaTestUtil.actorDeathMethods(system) foreach { case (name, stopMethod) => + it should s"catch polling actors if they $name and then recreate them" in { + + val statusPoller1 = TestActorRef(Props(new AkkaTestUtil.DeathTestActor()), TestActorRef(new AkkaTestUtil.StoppingSupervisor())) + val statusPoller2 = TestActorRef(Props(new AkkaTestUtil.DeathTestActor())) + val jaqmActor: TestActorRef[TestJesApiQueryManager] = TestActorRef(TestJesApiQueryManager.props(statusPoller1, statusPoller2)) + + val statusRequesters = ((0 until BatchSize * 2) map { i => i -> TestProbe(name = s"StatusRequester_$i") }).toMap + + // Send a few status poll requests: + BatchSize indexedTimes { index => + val probe = statusRequesters(index) + jaqmActor.tell(msg = JesApiQueryManager.DoPoll(Run(index.toString, null)), sender = probe.ref) + } + BatchSize indexedTimes { i => + val index = i + BatchSize // For the second half of the statusRequester set + val probe = statusRequesters(index) + jaqmActor.tell(msg = JesApiQueryManager.DoPoll(Run(index.toString, null)), sender = probe.ref) + } + + // Request a set of work from the middle of the queue: + val batchOffset = 2 + jaqmActor.tell(msg = JesApiQueryManager.RequestJesPollingWork(batchOffset), sender = statusPoller1) + jaqmActor.tell(msg = JesApiQueryManager.RequestJesPollingWork(BatchSize), sender = statusPoller1) + + // Kill the original status poller: + stopMethod(statusPoller1) + + // Only the appropriate requesters get an error: + (0 until batchOffset) foreach { index => + val probe = statusRequesters(index) + probe.expectNoMsg(max = AwaitAlmostNothing) + } + (batchOffset until batchOffset + BatchSize) foreach { index => + val probe = statusRequesters(index) + probe.expectMsg(max = TestExecutionTimeout, hint = s"Polling error to requester #$index", obj = JesPollingActor.JesPollError) + } + (batchOffset + BatchSize until 2 * BatchSize) foreach { index => + val probe = statusRequesters(index) + probe.expectNoMsg(max = AwaitAlmostNothing) + } + + // Check the next status poller gets created: + eventually { jaqmActor.underlyingActor.testPollerCreations should be(2) } + } + } +} + +object JesApiQueryManagerSpec { + implicit class intWithTimes(n: Int) { + def times(f: => Unit) = 1 to n foreach { _ => f } + def indexedTimes(f: Int => Unit) = 0 until n foreach { i => f(i) } + } +} + +/** + * This test class allows us to hook into the JesApiQueryManager's makeStatusPoller and provide our own TestProbes instead + */ +class TestJesApiQueryManager(statusPollerProbes: ActorRef*) extends JesApiQueryManager { + + var testProbes: Queue[ActorRef] = _ + var testPollerCreations: Int = _ + + private def init() = { + testProbes = Queue(statusPollerProbes: _*) + testPollerCreations = 0 + } + + override private[statuspolling] def makeStatusPoller(): ActorRef = { + // Initialise the queue, if necessary: + if (testProbes == null) { + init() + } + + // Register that the creation was requested: + testPollerCreations += 1 + + // Pop the queue to get the next test probe: + val (probe, newQueue) = testProbes.dequeue + testProbes = newQueue + probe + } +} + +object TestJesApiQueryManager { + def props(statusPollers: ActorRef*): Props = Props(new TestJesApiQueryManager(statusPollers: _*)) +} diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala new file mode 100644 index 000000000..b861cbf0f --- /dev/null +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala @@ -0,0 +1,131 @@ +package cromwell.backend.impl.jes.statuspolling + +import akka.actor.{ActorRef, Props} +import akka.testkit.{TestActorRef, TestProbe} +import cromwell.core.{ExecutionEvent, TestKitSuite} +import org.scalatest.{BeforeAndAfter, FlatSpecLike, Matchers} +import org.scalatest.concurrent.Eventually + +import scala.concurrent.duration._ +import akka.testkit._ +import cats.data.NonEmptyList +import com.google.api.client.googleapis.batch.BatchRequest +import com.google.api.client.googleapis.batch.json.JsonBatchCallback +import com.google.api.client.googleapis.json.GoogleJsonError +import com.google.api.services.genomics.model.Operation +import cromwell.backend.impl.jes.{Run, RunStatus} +import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.JesStatusPollQuery +import cromwell.backend.impl.jes.statuspolling.JesPollingActor.JesPollFailed +import cromwell.backend.impl.jes.statuspolling.TestJesPollingActor.{CallbackFailure, CallbackSuccess, JesBatchCallbackResponse} +import org.specs2.mock.Mockito + +import scala.collection.immutable.Queue + +class JesPollingActorSpec extends TestKitSuite("JesPollingActor") with FlatSpecLike with Matchers with Eventually with BeforeAndAfter with Mockito { + + behavior of "JesPollingActor" + + implicit val TestExecutionTimeout = 10.seconds.dilated + implicit val DefaultPatienceConfig = PatienceConfig(TestExecutionTimeout) + val AwaitAlmostNothing = 30.milliseconds.dilated + + var managerProbe: TestProbe = _ + var jpActor: TestActorRef[TestJesPollingActor] = _ + + it should "query for work and wait for a reply" in { + managerProbe.expectMsgClass(max = TestExecutionTimeout, c = classOf[JesApiQueryManager.RequestJesPollingWork]) + managerProbe.expectNoMsg(max = AwaitAlmostNothing) + } + + it should "respond directly to requesters with various run statuses" in { + managerProbe.expectMsgClass(max = TestExecutionTimeout, c = classOf[JesApiQueryManager.RequestJesPollingWork]) + + val requester1 = TestProbe() + val query1 = JesStatusPollQuery(requester1.ref, mock[Run]) + val requester2 = TestProbe() + val query2 = JesStatusPollQuery(requester2.ref, mock[Run]) + val requester3 = TestProbe() + val query3 = JesStatusPollQuery(requester3.ref, mock[Run]) + + // For two requests the callback succeeds (first with RunStatus.Success, then RunStatus.Failed). The third callback fails: + jpActor.underlyingActor.callbackResponses :+= CallbackSuccess + jpActor.underlyingActor.callbackResponses :+= CallbackSuccess + jpActor.underlyingActor.callbackResponses :+= CallbackFailure + + val successStatus = RunStatus.Success(Seq.empty[ExecutionEvent], None, None, None) + val failureStatus = RunStatus.Failed(-1, List.empty[String], Seq.empty[ExecutionEvent], None, None, None) + jpActor.underlyingActor.operationStatusResponses :+= successStatus + jpActor.underlyingActor.operationStatusResponses :+= failureStatus + + jpActor.tell(msg = JesApiQueryManager.JesPollingWorkBatch(NonEmptyList(query1, List(query2, query3))), sender = managerProbe.ref) + eventually { jpActor.underlyingActor.resultHandlers.size should be(3) } + eventually { jpActor.underlyingActor.runBatchRequested should be(true) } + + // The manager shouldn't have been asked for more work yet: + managerProbe.expectNoMsg(max = AwaitAlmostNothing) + + // Ok, let's trigger the callbacks: + jpActor.underlyingActor.executeBatch() + + requester1.expectMsg(successStatus) + requester2.expectMsg(failureStatus) + requester3.expectMsgClass(classOf[JesPollFailed]) + + // And the poller is done! Now the manager should now have (only one) request for more work: + managerProbe.expectMsgClass(max = TestExecutionTimeout, c = classOf[JesApiQueryManager.RequestJesPollingWork]) + } + + before { + managerProbe = TestProbe() + jpActor = TestActorRef(TestJesPollingActor.props(managerProbe.ref), managerProbe.ref) + } +} + +object JesPollingActorSpec extends Mockito { + def mockRun(runId: String): Run = { + val run = mock[Run] + run.runId returns runId + run + } +} + +/** + * Testable JES polling actor. + * - Mocks out the methods which actually call out to JES, and allows the callbacks to be triggered in a testable way + * - Also waits a **lot** less time before polls! + */ +class TestJesPollingActor(manager: ActorRef) extends JesPollingActor(manager) with Mockito { + override val BatchInterval = 10.milliseconds + + var operationStatusResponses: Queue[RunStatus] = Queue.empty + var resultHandlers: Queue[JsonBatchCallback[Operation]] = Queue.empty + var callbackResponses: Queue[JesBatchCallbackResponse] = Queue.empty + var runBatchRequested: Boolean = false + + override private[statuspolling] def createBatch(run: Run): BatchRequest = null + override private[statuspolling] def runBatch(batch: BatchRequest): Unit = runBatchRequested = true + + def executeBatch(): Unit = { + resultHandlers.zip(callbackResponses) foreach { case (handler, response) => response match { + case CallbackSuccess => handler.onSuccess(null, null) + case CallbackFailure => + val error: GoogleJsonError = null + handler.onFailure(error, null) + }} + } + override private[statuspolling] def enqueueStatusPollInBatch(run: Run, batch: BatchRequest, resultHandler: JsonBatchCallback[Operation]): Unit = resultHandlers :+= resultHandler + override private[statuspolling] def interpretOperationStatus(operation: Operation): RunStatus = { + val (status, newQueue) = operationStatusResponses.dequeue + operationStatusResponses = newQueue + status + } + override private[statuspolling] def mkErrorString(e: GoogleJsonError) = "NA" +} + +object TestJesPollingActor { + def props(manager: ActorRef) = Props(new TestJesPollingActor(manager)) + + sealed trait JesBatchCallbackResponse + case object CallbackSuccess extends JesBatchCallbackResponse + case object CallbackFailure extends JesBatchCallbackResponse +} diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala index d812b1c8e..edacd5303 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala @@ -50,7 +50,8 @@ trait SharedFileSystemBackendLifecycleActorFactory extends BackendLifecycleActor override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationDataOption: Option[BackendInitializationData], - serviceRegistryActor: ActorRef) = { + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]) = { def propsCreator(completionPromise: Promise[BackendJobExecutionResponse]): Props = { val params = SharedFileSystemAsyncJobExecutionActorParams(serviceRegistryActor, jobDescriptor, configurationDescriptor, completionPromise, initializationDataOption) diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala index ba09b6cb9..ef01c9985 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala @@ -13,8 +13,10 @@ case class SparkBackendFactory(name: String, configurationDescriptor: BackendCon Option(SparkInitializationActor.props(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor)) } - override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationData: Option[BackendInitializationData], - serviceRegistryActor: ActorRef): Props = { + override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, + initializationData: Option[BackendInitializationData], + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props = { SparkJobExecutionActor.props(jobDescriptor, configurationDescriptor) } From 5ceeee5c63b9f85b44102a86fcd3d26ea4a95d5f Mon Sep 17 00:00:00 2001 From: Ruchi Date: Wed, 12 Oct 2016 11:14:44 -0400 Subject: [PATCH 20/23] fix gotc build (#1557) --- docker/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/install.sh b/docker/install.sh index 2afc67f9e..b9fb8e91a 100755 --- a/docker/install.sh +++ b/docker/install.sh @@ -4,7 +4,7 @@ set -e CROMWELL_DIR=$1 cd $CROMWELL_DIR -sbt 'set test in assembly := {}' notests:assembly +sbt assembly CROMWELL_JAR=$(find target | grep 'cromwell.*\.jar') mv $CROMWELL_JAR ./cromwell.jar sbt clean From 86c80a890941e0cc7b2549ce0aa5f41316194ad4 Mon Sep 17 00:00:00 2001 From: Thib Date: Wed, 12 Oct 2016 17:28:58 -0400 Subject: [PATCH 21/23] Call caching output copy failures handling. Closes #1510 (#1559) * smarter handling of cache hit copying failures --- core/src/main/scala/cromwell/core/PathCopier.scala | 16 +++- .../database/slick/CallCachingSlickDatabase.scala | 8 ++ .../slick/tables/CallCachingEntryComponent.scala | 7 ++ .../database/sql/CallCachingSqlDatabase.scala | 5 +- .../execution/EngineJobExecutionActor.scala | 90 +++++++++++++++++----- .../execution/callcaching/CallCache.scala | 24 +++--- .../callcaching/CallCacheInvalidateActor.scala | 36 +++++++++ .../execution/callcaching/CallCacheReadActor.scala | 4 +- .../callcaching/EngineJobHashingActor.scala | 11 ++- .../callcaching/FetchCachedResultsActor.scala | 14 ++-- .../execution/callcaching/EJHADataSpec.scala | 18 ++--- .../callcaching/EngineJobHashingActorSpec.scala | 35 +++++---- .../PredictableCallCacheReadActor.scala | 6 +- .../EjeaBackendIsCopyingCachedOutputsSpec.scala | 34 ++++---- .../execution/ejea/EjeaCheckingCallCacheSpec.scala | 13 ++-- ...EjeaFetchingCachedOutputsFromDatabaseSpec.scala | 14 ++-- .../ejea/EjeaInvalidatingCacheEntrySpec.scala | 53 +++++++++++++ .../ejea/EngineJobExecutionActorSpec.scala | 1 + .../ejea/EngineJobExecutionActorSpecUtil.scala | 20 +++++ .../lifecycle/execution/ejea/PerTestHelper.scala | 13 ++-- .../JobExecutionTokenDispenserActorSpec.scala | 8 +- .../backend/impl/jes/JesCacheHitCopyingActor.scala | 2 +- 22 files changed, 312 insertions(+), 120 deletions(-) create mode 100644 engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala create mode 100644 engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaInvalidatingCacheEntrySpec.scala diff --git a/core/src/main/scala/cromwell/core/PathCopier.scala b/core/src/main/scala/cromwell/core/PathCopier.scala index c941ea890..f90dad604 100644 --- a/core/src/main/scala/cromwell/core/PathCopier.scala +++ b/core/src/main/scala/cromwell/core/PathCopier.scala @@ -1,9 +1,12 @@ package cromwell.core +import java.io.IOException import java.nio.file.Path import better.files._ +import scala.util.{Failure, Try} + object PathCopier { def getDestinationFilePath(sourceContextPath: Path, sourceFilePath: Path, destinationDirPath: Path): Path = { val relativeFileString = sourceContextPath.toAbsolutePath.relativize(sourceFilePath.toAbsolutePath).toString @@ -13,7 +16,7 @@ object PathCopier { /** * Copies from a relative source to destination dir. NOTE: Copies are not atomic, and may create a partial copy. */ - def copy(sourceContextPath: Path, sourceFilePath: Path, destinationDirPath: Path): Unit = { + def copy(sourceContextPath: Path, sourceFilePath: Path, destinationDirPath: Path): Try[Unit] = { val destinationFilePath = getDestinationFilePath(sourceContextPath, sourceFilePath, destinationDirPath) copy(sourceFilePath, destinationFilePath) } @@ -21,9 +24,14 @@ object PathCopier { /** * Copies from source to destination. NOTE: Copies are not atomic, and may create a partial copy. */ - def copy(sourceFilePath: Path, destinationFilePath: Path): Unit = { + def copy(sourceFilePath: Path, destinationFilePath: Path): Try[Unit] = { Option(File(destinationFilePath).parent).foreach(_.createDirectories()) - File(sourceFilePath).copyTo(destinationFilePath, overwrite = true) - () + Try { + File(sourceFilePath).copyTo(destinationFilePath, overwrite = true) + + () + } recoverWith { + case ex => Failure(new IOException(s"Failed to copy ${sourceFilePath.toUri} to ${destinationFilePath.toUri}", ex)) + } } } diff --git a/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala index 51ea7c3a4..ce40e4e6a 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala @@ -46,4 +46,12 @@ trait CallCachingSlickDatabase extends CallCachingSqlDatabase { runTransaction(action) } + + override def invalidateCall(callCachingEntryId: Int) + (implicit ec: ExecutionContext): Future[Unit] = { + import cats.syntax.functor._ + import cats.instances.future._ + val action = dataAccess.allowResultReuseForCallCachingEntryId(callCachingEntryId).update(false) + runTransaction(action) void + } } diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala index 8cf97e2a5..189402814 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala @@ -40,4 +40,11 @@ trait CallCachingEntryComponent { if callCachingEntry.callCachingEntryId === callCachingEntryId } yield callCachingEntry ) + + val allowResultReuseForCallCachingEntryId = Compiled( + (callCachingEntryId: Rep[Int]) => for { + callCachingEntry <- callCachingEntries + if callCachingEntry.callCachingEntryId === callCachingEntryId + } yield callCachingEntry.allowResultReuse + ) } diff --git a/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala index abf262ae3..b2b3d221a 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala @@ -11,6 +11,9 @@ trait CallCachingSqlDatabase { def queryCallCachingEntryIds(hashKeyHashValues: NonEmptyList[(String, String)]) (implicit ec: ExecutionContext): Future[Seq[Int]] - def queryCallCaching(callCachingResultMetainfoId: Int) + def queryCallCaching(callCachingEntryId: Int) (implicit ec: ExecutionContext): Future[Option[CallCachingJoin]] + + def invalidateCall(callCachingEntryId: Int) + (implicit ec: ExecutionContext): Future[Unit] } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala index 07114e7e8..25c1852dc 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala @@ -4,6 +4,7 @@ import java.time.OffsetDateTime import akka.actor.{ActorRef, ActorRefFactory, LoggingFSM, Props} import akka.routing.RoundRobinPool +import cats.data.NonEmptyList import cromwell.backend.BackendCacheHitCopyingActor.CopyOutputsCommand import cromwell.backend.BackendJobExecutionActor._ import cromwell.backend.{BackendInitializationData, BackendJobDescriptor, BackendJobDescriptorKey, BackendLifecycleActorFactory} @@ -129,8 +130,8 @@ class EngineJobExecutionActor(replyTo: ActorRef, writeToMetadata(Map(callCachingReadResultMetadataKey -> "Cache Miss")) log.debug("Cache miss for job {}", jobTag) runJob(data) - case Event(hit @ CacheHit(cacheResultId), data: ResponsePendingData) => - fetchCachedResults(data, jobDescriptorKey.call.task.outputs, hit) + case Event(hit: CacheHit, data: ResponsePendingData) => + fetchCachedResults(jobDescriptorKey.call.task.outputs, hit.cacheResultIds.head, data) case Event(HashError(t), data: ResponsePendingData) => writeToMetadata(Map(callCachingReadResultMetadataKey -> s"Hashing Error: ${t.getMessage}")) disableCallCaching(t) @@ -141,8 +142,8 @@ class EngineJobExecutionActor(replyTo: ActorRef, case Event(CachedOutputLookupSucceeded(wdlValueSimpletons, jobDetritus, returnCode, cacheResultId, cacheHitDetails), data: ResponsePendingData) => writeToMetadata(Map(callCachingReadResultMetadataKey -> s"Cache Hit: $cacheHitDetails")) log.debug("Cache hit for {}! Fetching cached result {}", jobTag, cacheResultId) - makeBackendCopyCacheHit(cacheResultId, wdlValueSimpletons, jobDetritus, returnCode, data) - case Event(CachedOutputLookupFailed(metaInfoId, error), data: ResponsePendingData) => + makeBackendCopyCacheHit(wdlValueSimpletons, jobDetritus, returnCode, data) + case Event(CachedOutputLookupFailed(callCachingEntryId, error), data: ResponsePendingData) => log.warning("Can't make a copy of the cached job outputs for {} due to {}. Running job.", jobTag, error) runJob(data) case Event(hashes: CallCacheHashes, data: ResponsePendingData) => @@ -155,20 +156,21 @@ class EngineJobExecutionActor(replyTo: ActorRef, when(BackendIsCopyingCachedOutputs) { // Backend copying response: - case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, Some(Success(hashes)))) => + case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, Some(Success(hashes)), _)) => saveCacheResults(hashes, data.withSuccessResponse(response)) - case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, None)) if effectiveCallCachingMode.writeToCache => + case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, None, _)) if effectiveCallCachingMode.writeToCache => // Wait for the CallCacheHashes stay using data.withSuccessResponse(response) case Event(response: SucceededResponse, data: ResponsePendingData) => // bad hashes or cache write off saveJobCompletionToJobStore(data.withSuccessResponse(response)) - case Event(response: BackendJobExecutionResponse, data: ResponsePendingData) => - // This matches all response types other than `SucceededResponse`. + case Event(response: BackendJobExecutionResponse, data @ ResponsePendingData(_, _, _, Some(cacheHit))) => response match { - case f: BackendJobFailedResponse =>log.error("{}: Failed copying cache results, falling back to running job: {}", jobDescriptorKey, f.throwable) - case _ => // + case f: BackendJobFailedResponse => + invalidateCacheHit(cacheHit.cacheResultIds.head) + log.error(f.throwable, "Failed copying cache results for job {}, invalidating cache entry.", jobDescriptorKey) + goto(InvalidatingCacheEntry) + case _ => runJob(data) } - runJob(data) // Hashes arrive: case Event(hashes: CallCacheHashes, data: SucceededResponseData) => @@ -186,6 +188,21 @@ class EngineJobExecutionActor(replyTo: ActorRef, stay using data.copy(hashes = Option(Failure(t))) } + when(InvalidatingCacheEntry) { + case Event(response: CallCacheInvalidatedResponse, data: ResponsePendingData) => + handleCacheInvalidatedResponse(response, data) + + // Hashes arrive: + case Event(hashes: CallCacheHashes, data: ResponsePendingData) => + addHashesAndStay(data, hashes) + + // Hash error occurs: + case Event(HashError(t), data: ResponsePendingData) => + disableCacheWrite(t) + // Can't write hashes for this job, but continue to wait for the copy response. + stay using data.copy(hashes = Option(Failure(t))) + } + when(RunningJob) { case Event(hashes: CallCacheHashes, data: SucceededResponseData) => saveCacheResults(hashes, data) @@ -199,10 +216,10 @@ class EngineJobExecutionActor(replyTo: ActorRef, disableCallCaching(t) stay using data.copy(hashes = Option(Failure(t))) - case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, Some(Success(hashes)))) if effectiveCallCachingMode.writeToCache => + case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, Some(Success(hashes)), _)) if effectiveCallCachingMode.writeToCache => eventList ++= response.executionEvents saveCacheResults(hashes, data.withSuccessResponse(response)) - case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, None)) if effectiveCallCachingMode.writeToCache => + case Event(response: SucceededResponse, data @ ResponsePendingData(_, _, None, _)) if effectiveCallCachingMode.writeToCache => log.debug(s"Got job result for {}, awaiting hashes", jobTag) stay using data.withSuccessResponse(response) case Event(response: BackendJobExecutionResponse, data: ResponsePendingData) => @@ -302,24 +319,24 @@ class EngineJobExecutionActor(replyTo: ActorRef, () } - def makeFetchCachedResultsActor(cacheHit: CacheHit, taskOutputs: Seq[TaskOutput]): Unit = { - context.actorOf(FetchCachedResultsActor.props(cacheHit, self, new CallCache(SingletonServicesStore.databaseInterface))) + def makeFetchCachedResultsActor(callCachingEntryId: CallCachingEntryId, taskOutputs: Seq[TaskOutput]): Unit = { + context.actorOf(FetchCachedResultsActor.props(callCachingEntryId, self, new CallCache(SingletonServicesStore.databaseInterface))) () } - private def fetchCachedResults(data: ResponsePendingData, taskOutputs: Seq[TaskOutput], cacheHit: CacheHit) = { - makeFetchCachedResultsActor(cacheHit, taskOutputs) - goto(FetchingCachedOutputsFromDatabase) + private def fetchCachedResults(taskOutputs: Seq[TaskOutput], callCachingEntryId: CallCachingEntryId, data: ResponsePendingData) = { + makeFetchCachedResultsActor(callCachingEntryId, taskOutputs) + goto(FetchingCachedOutputsFromDatabase) using data } - private def makeBackendCopyCacheHit(cacheHit: CacheHit, wdlValueSimpletons: Seq[WdlValueSimpleton], jobDetritusFiles: Map[String,String], returnCode: Option[Int], data: ResponsePendingData) = { + private def makeBackendCopyCacheHit(wdlValueSimpletons: Seq[WdlValueSimpleton], jobDetritusFiles: Map[String,String], returnCode: Option[Int], data: ResponsePendingData) = { factory.cacheHitCopyingActorProps match { case Some(propsMaker) => val backendCacheHitCopyingActorProps = propsMaker(data.jobDescriptor, initializationData, serviceRegistryActor) val cacheHitCopyActor = context.actorOf(backendCacheHitCopyingActorProps, buildCacheHitCopyingActorName(data.jobDescriptor)) cacheHitCopyActor ! CopyOutputsCommand(wdlValueSimpletons, jobDetritusFiles, returnCode) replyTo ! JobRunning(data.jobDescriptor, None) - goto(BackendIsCopyingCachedOutputs) using data + goto(BackendIsCopyingCachedOutputs) case None => // This should be impossible with the FSM, but luckily, we CAN recover if some foolish future programmer makes this happen: val errorMessage = "Call caching copying should never have even been attempted with no copy actor props! (Programmer error!)" @@ -337,6 +354,22 @@ class EngineJobExecutionActor(replyTo: ActorRef, goto(RunningJob) using data } + private def handleCacheInvalidatedResponse(response: CallCacheInvalidatedResponse, data: ResponsePendingData) = { + response match { + case CallCacheInvalidatedFailure(failure) => log.error(failure, "Failed to invalidate cache entry for job: {}", jobDescriptorKey) + case _ => + } + + data.popCacheHitId match { + case newData @ ResponsePendingData(_, _, _, Some(cacheHit)) => + log.info("Trying to use another cache hit for job: {}", jobDescriptorKey) + fetchCachedResults(jobDescriptorKey.call.task.outputs, cacheHit.cacheResultIds.head, newData) + case newData => + log.info("Could not find another cache hit, falling back to running job: {}", jobDescriptorKey) + runJob(newData) + } + } + private def buildJobExecutionActorName(jobDescriptor: BackendJobDescriptor) = { s"$workflowId-BackendJobExecutionActor-$jobTag" } @@ -351,6 +384,12 @@ class EngineJobExecutionActor(replyTo: ActorRef, () } + protected def invalidateCacheHit(cacheId: CallCachingEntryId): Unit = { + val callCache = new CallCache(SingletonServicesStore.databaseInterface) + context.actorOf(CallCacheInvalidateActor.props(callCache, cacheId), s"CallCacheInvalidateActor${cacheId.id}-$tag") + () + } + private def saveCacheResults(hashes: CallCacheHashes, data: SucceededResponseData) = { createSaveCacheResultsActor(hashes, data.successResponse) val updatedData = data.copy(hashes = Option(Success(hashes))) @@ -439,6 +478,7 @@ object EngineJobExecutionActor { case object RunningJob extends EngineJobExecutionActorState case object UpdatingCallCache extends EngineJobExecutionActorState case object UpdatingJobStore extends EngineJobExecutionActorState + case object InvalidatingCacheEntry extends EngineJobExecutionActorState /** Commands */ sealed trait EngineJobExecutionActorCommand @@ -483,7 +523,8 @@ object EngineJobExecutionActor { private[execution] case class ResponsePendingData(jobDescriptor: BackendJobDescriptor, bjeaProps: Props, - hashes: Option[Try[CallCacheHashes]] = None) extends EJEAData { + hashes: Option[Try[CallCacheHashes]] = None, + cacheHit: Option[CacheHit] = None) extends EJEAData { def withSuccessResponse(success: SucceededResponse) = SucceededResponseData(success, hashes) @@ -491,6 +532,13 @@ object EngineJobExecutionActor { case success: SucceededResponse => SucceededResponseData(success, hashes) case failure => NotSucceededResponseData(failure, hashes) } + + def withCacheHit(cacheHit: Option[CacheHit]) = this.copy(cacheHit = cacheHit) + + def popCacheHitId: ResponsePendingData = cacheHit match { + case Some(hit) => this.copy(cacheHit = NonEmptyList.fromList(hit.cacheResultIds.tail) map CacheHit.apply) + case None => this + } } private[execution] trait ResponseData extends EJEAData { diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala index 16ecc89f0..8c5331c42 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala @@ -13,7 +13,7 @@ import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashing import scala.concurrent.{ExecutionContext, Future} -final case class MetaInfoId(id: Int) +final case class CallCachingEntryId(id: Int) /** * Given a database-layer CallCacheStore, this accessor can access the database with engine-friendly data types. @@ -34,7 +34,7 @@ class CallCache(database: CallCachingSqlDatabase) { addToCache(metaInfo, hashes, result, jobDetritus) } - private def addToCache(metaInfo: CallCachingEntry, hashes: Set[HashResult], + private def addToCache(callCachingEntry: CallCachingEntry, hashes: Set[HashResult], result: Iterable[WdlValueSimpleton], jobDetritus: Map[String, String]) (implicit ec: ExecutionContext): Future[Unit] = { @@ -56,25 +56,29 @@ class CallCache(database: CallCachingSqlDatabase) { } val callCachingJoin = - CallCachingJoin(metaInfo, hashesToInsert.toSeq, resultToInsert.toSeq, jobDetritusToInsert.toSeq) + CallCachingJoin(callCachingEntry, hashesToInsert.toSeq, resultToInsert.toSeq, jobDetritusToInsert.toSeq) database.addCallCaching(callCachingJoin) } - def fetchMetaInfoIdsMatchingHashes(callCacheHashes: CallCacheHashes)(implicit ec: ExecutionContext): Future[Set[MetaInfoId]] = { - metaInfoIdsMatchingHashes(NonEmptyList.fromListUnsafe(callCacheHashes.hashes.toList)) + def callCachingEntryIdsMatchingHashes(callCacheHashes: CallCacheHashes)(implicit ec: ExecutionContext): Future[Set[CallCachingEntryId]] = { + callCachingEntryIdsMatchingHashes(NonEmptyList.fromListUnsafe(callCacheHashes.hashes.toList)) } - private def metaInfoIdsMatchingHashes(hashKeyValuePairs: NonEmptyList[HashResult]) - (implicit ec: ExecutionContext): Future[Set[MetaInfoId]] = { + private def callCachingEntryIdsMatchingHashes(hashKeyValuePairs: NonEmptyList[HashResult]) + (implicit ec: ExecutionContext): Future[Set[CallCachingEntryId]] = { val result = database.queryCallCachingEntryIds(hashKeyValuePairs map { case HashResult(hashKey, hashValue) => (hashKey.key, hashValue.value) }) - result.map(_.toSet.map(MetaInfoId)) + result.map(_.toSet.map(CallCachingEntryId)) } - def fetchCachedResult(metaInfoId: MetaInfoId)(implicit ec: ExecutionContext): Future[Option[CallCachingJoin]] = { - database.queryCallCaching(metaInfoId.id) + def fetchCachedResult(callCachingEntryId: CallCachingEntryId)(implicit ec: ExecutionContext): Future[Option[CallCachingJoin]] = { + database.queryCallCaching(callCachingEntryId.id) + } + + def invalidate(callCachingEntryId: CallCachingEntryId)(implicit ec: ExecutionContext) = { + database.invalidateCall(callCachingEntryId.id) } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala new file mode 100644 index 000000000..ef09d32e9 --- /dev/null +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala @@ -0,0 +1,36 @@ +package cromwell.engine.workflow.lifecycle.execution.callcaching + +import akka.actor.{Actor, ActorLogging, Props} + +import scala.concurrent.ExecutionContext +import scala.util.{Failure, Success} + +class CallCacheInvalidateActor(callCache: CallCache, cacheId: CallCachingEntryId) extends Actor with ActorLogging { + + implicit val ec: ExecutionContext = context.dispatcher + + def receiver = context.parent + + callCache.invalidate(cacheId) onComplete { + case Success(_) => + receiver ! CallCacheInvalidatedSuccess + context.stop(self) + case Failure(t) => + receiver ! CallCacheInvalidatedFailure(t) + context.stop(self) + } + + override def receive: Receive = { + case any => log.error("Unexpected message to InvalidateCallCacheActor: " + any) + } +} + +object CallCacheInvalidateActor { + def props(callCache: CallCache, cacheId: CallCachingEntryId) = { + Props(new CallCacheInvalidateActor(callCache: CallCache, cacheId: CallCachingEntryId)) + } +} + +sealed trait CallCacheInvalidatedResponse +case object CallCacheInvalidatedSuccess extends CallCacheInvalidatedResponse +case class CallCacheInvalidatedFailure(t: Throwable) extends CallCacheInvalidatedResponse \ No newline at end of file diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala index 229fc47a2..8aa1f6d6b 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala @@ -31,7 +31,7 @@ class CallCacheReadActor(cache: CallCache) extends Actor with ActorLogging { } private def runRequest(callCacheHashes: CallCacheHashes): Unit = { - val response = cache.fetchMetaInfoIdsMatchingHashes(callCacheHashes) map { + val response = cache.callCachingEntryIdsMatchingHashes(callCacheHashes) map { CacheResultMatchesForHashes(callCacheHashes.hashes, _) } recover { case t => CacheResultLookupFailure(t) @@ -66,6 +66,6 @@ object CallCacheReadActor { case class CacheLookupRequest(callCacheHashes: CallCacheHashes) sealed trait CallCacheReadActorResponse - case class CacheResultMatchesForHashes(hashResults: Set[HashResult], cacheResultIds: Set[MetaInfoId]) extends CallCacheReadActorResponse + case class CacheResultMatchesForHashes(hashResults: Set[HashResult], cacheResultIds: Set[CallCachingEntryId]) extends CallCacheReadActorResponse case class CacheResultLookupFailure(reason: Throwable) extends CallCacheReadActorResponse } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala index 0f03a5840..b4ad358f5 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala @@ -1,6 +1,7 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching import akka.actor.{ActorLogging, ActorRef, LoggingFSM, Props} +import cats.data.NonEmptyList import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest import cromwell.backend.{BackendInitializationData, BackendJobDescriptor, RuntimeAttributeDefinition} import cromwell.core.callcaching._ @@ -123,7 +124,8 @@ case class EngineJobHashingActor(receiver: ActorRef, } private def respondWithHitOrMissThenTransition(newData: EJHAData) = { - val hitOrMissResponse: EJHAResponse = newData.cacheHit map CacheHit getOrElse CacheMiss + import cats.data.NonEmptyList + val hitOrMissResponse: EJHAResponse = newData.cacheHits map { _.toList } flatMap NonEmptyList.fromList map CacheHit.apply getOrElse CacheMiss receiver ! hitOrMissResponse if (!activity.writeToCache) { @@ -199,7 +201,7 @@ object EngineJobHashingActor { case object GeneratingAllHashes extends EJHAState sealed trait EJHAResponse - case class CacheHit(cacheResultId: MetaInfoId) extends EJHAResponse + case class CacheHit(cacheResultIds: NonEmptyList[CallCachingEntryId]) extends EJHAResponse case object CacheMiss extends EJHAResponse case class HashError(t: Throwable) extends EJHAResponse { override def toString = s"HashError(${t.getMessage})" @@ -223,7 +225,7 @@ object EngineJobHashingActor { * @param hashesKnown The set of all hashes calculated so far (including initial hashes) * @param remainingHashesNeeded The set of hashes which are still needed for writing to the database */ -private[callcaching] case class EJHAData(possibleCacheResults: Option[Set[MetaInfoId]], +private[callcaching] case class EJHAData(possibleCacheResults: Option[Set[CallCachingEntryId]], remainingCacheChecks: Set[HashKey], hashesKnown: Set[HashResult], remainingHashesNeeded: Set[HashKey]) { @@ -249,7 +251,8 @@ private[callcaching] case class EJHAData(possibleCacheResults: Option[Set[MetaIn def allHashesKnown = remainingHashesNeeded.isEmpty def allCacheResultsIntersected = remainingCacheChecks.isEmpty def cacheHit = if (allCacheResultsIntersected) possibleCacheResults flatMap { _.headOption } else None - def isDefinitelyCacheHit = cacheHit.isDefined + def cacheHits = if (allCacheResultsIntersected) possibleCacheResults else None + def isDefinitelyCacheHit = cacheHits.isDefined def isDefinitelyCacheMiss = possibleCacheResults.exists(_.isEmpty) def isDefinitelyCacheHitOrMiss = isDefinitelyCacheHit || isDefinitelyCacheMiss } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala index c8f6be4a2..b0900cb16 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala @@ -3,29 +3,27 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching import akka.actor.{Actor, ActorLogging, ActorRef, Props} import cromwell.Simpletons._ import cromwell.core.simpleton.WdlValueSimpleton -import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CacheHit import cromwell.engine.workflow.lifecycle.execution.callcaching.FetchCachedResultsActor.{CachedOutputLookupFailed, CachedOutputLookupSucceeded} import scala.concurrent.ExecutionContext import scala.util.{Failure, Success} object FetchCachedResultsActor { - def props(cacheHit: CacheHit, replyTo: ActorRef, callCache: CallCache): Props = - Props(new FetchCachedResultsActor(cacheHit, replyTo, callCache)) + def props(callCachingEntryId: CallCachingEntryId, replyTo: ActorRef, callCache: CallCache): Props = + Props(new FetchCachedResultsActor(callCachingEntryId, replyTo, callCache)) sealed trait CachedResultResponse - case class CachedOutputLookupFailed(metaInfoId: MetaInfoId, failure: Throwable) extends CachedResultResponse + case class CachedOutputLookupFailed(callCachingEntryId: CallCachingEntryId, failure: Throwable) extends CachedResultResponse case class CachedOutputLookupSucceeded(simpletons: Seq[WdlValueSimpleton], callOutputFiles: Map[String,String], - returnCode: Option[Int], cacheHit: CacheHit, cacheHitDetails: String) extends CachedResultResponse + returnCode: Option[Int], cacheHit: CallCachingEntryId, cacheHitDetails: String) extends CachedResultResponse } -class FetchCachedResultsActor(cacheHit: CacheHit, replyTo: ActorRef, callCache: CallCache) +class FetchCachedResultsActor(cacheResultId: CallCachingEntryId, replyTo: ActorRef, callCache: CallCache) extends Actor with ActorLogging { { implicit val ec: ExecutionContext = context.dispatcher - val cacheResultId = cacheHit.cacheResultId callCache.fetchCachedResult(cacheResultId) onComplete { case Success(Some(result)) => @@ -39,7 +37,7 @@ class FetchCachedResultsActor(cacheHit: CacheHit, replyTo: ActorRef, callCache: replyTo ! CachedOutputLookupSucceeded(simpletons, jobDetritusFiles.toMap, result.callCachingEntry.returnCode, - cacheHit, sourceCacheDetails) + cacheResultId, sourceCacheDetails) case Success(None) => val reason = new RuntimeException(s"Cache hit vanished between discovery and retrieval: $cacheResultId") replyTo ! CachedOutputLookupFailed(cacheResultId, reason) diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EJHADataSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EJHADataSpec.scala index 45b996018..9c3d8d547 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EJHADataSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EJHADataSpec.scala @@ -59,14 +59,14 @@ class EJHADataSpec extends FlatSpec with Matchers { // To save you time I'll just tell you: the intersection of all these sets is Set(5) val cacheLookupResults: List[CacheResultMatchesForHashes] = List( - CacheResultMatchesForHashes(Set(makeHashResult(hashKey1)), Set(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) map MetaInfoId), - CacheResultMatchesForHashes(Set(makeHashResult(hashKey2)), Set(1, 2, 3, 4, 5, 6) map MetaInfoId), - CacheResultMatchesForHashes(Set(makeHashResult(hashKey3)), Set(1, 2, 3, 5, 7, 8, 9, 10) map MetaInfoId), - CacheResultMatchesForHashes(Set(makeHashResult(hashKey4)), Set(4, 5, 6, 7, 8, 9, 10) map MetaInfoId), - CacheResultMatchesForHashes(Set(makeHashResult(hashKey5)), Set(1, 2, 5, 6, 7, 10) map MetaInfoId)) + CacheResultMatchesForHashes(Set(makeHashResult(hashKey1)), Set(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) map CallCachingEntryId), + CacheResultMatchesForHashes(Set(makeHashResult(hashKey2)), Set(1, 2, 3, 4, 5, 6) map CallCachingEntryId), + CacheResultMatchesForHashes(Set(makeHashResult(hashKey3)), Set(1, 2, 3, 5, 7, 8, 9, 10) map CallCachingEntryId), + CacheResultMatchesForHashes(Set(makeHashResult(hashKey4)), Set(4, 5, 6, 7, 8, 9, 10) map CallCachingEntryId), + CacheResultMatchesForHashes(Set(makeHashResult(hashKey5)), Set(1, 2, 5, 6, 7, 10) map CallCachingEntryId)) val newData = cacheLookupResults.foldLeft(data)( (d, c) => d.intersectCacheResults(c) ) newData.possibleCacheResults match{ - case Some(set) => set should be(Set(MetaInfoId(5))) + case Some(set) => set should be(Set(CallCachingEntryId(5))) case None => fail("There should be a cache result set") } newData.allCacheResultsIntersected should be(true) @@ -79,9 +79,9 @@ class EJHADataSpec extends FlatSpec with Matchers { // To save you time I'll just tell you: the intersection of all these sets is empty Set() val cacheLookupResults: List[CacheResultMatchesForHashes] = List( - CacheResultMatchesForHashes(Set(makeHashResult(hashKey1)), Set(1, 2, 3, 4, 5, 6) map MetaInfoId), - CacheResultMatchesForHashes(Set(makeHashResult(hashKey2)), Set(1, 2, 3, 7, 8, 9) map MetaInfoId), - CacheResultMatchesForHashes(Set(makeHashResult(hashKey3)), Set(5, 7, 8, 9, 10) map MetaInfoId)) + CacheResultMatchesForHashes(Set(makeHashResult(hashKey1)), Set(1, 2, 3, 4, 5, 6) map CallCachingEntryId), + CacheResultMatchesForHashes(Set(makeHashResult(hashKey2)), Set(1, 2, 3, 7, 8, 9) map CallCachingEntryId), + CacheResultMatchesForHashes(Set(makeHashResult(hashKey3)), Set(5, 7, 8, 9, 10) map CallCachingEntryId)) val newData = cacheLookupResults.foldLeft(data)( (d, c) => d.intersectCacheResults(c) ) newData.possibleCacheResults match{ case Some(set) => set should be(Set.empty) diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala index 25ceffe3c..d79a383fb 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala @@ -2,6 +2,7 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching import akka.actor.{ActorRef, ActorSystem, Props} import akka.testkit.{ImplicitSender, TestKit, TestProbe} +import cats.data.NonEmptyList import cromwell.CromwellTestkitSpec import cromwell.backend.callcaching.FileHashingActor.{FileHashResponse, SingleFileHashRequest} import cromwell.backend.{BackendInitializationData, BackendJobDescriptor, BackendJobDescriptorKey, BackendWorkflowDescriptor, RuntimeAttributeDefinition} @@ -35,11 +36,11 @@ class EngineJobHashingActorSpec extends TestKit(new CromwellTestkitSpec.TestWork } s"Respect the CallCachingMode and report back $expectation for the ${activity.readWriteMode} activity" in { - val singleMetaInfoIdSet = Set(MetaInfoId(1)) + val singleCallCachingEntryIdSet = Set(CallCachingEntryId(1)) val replyTo = TestProbe() val deathWatch = TestProbe() - val cacheLookupResponses: Map[String, Set[MetaInfoId]] = if (activity.readFromCache) standardCacheLookupResponses(singleMetaInfoIdSet, singleMetaInfoIdSet, singleMetaInfoIdSet, singleMetaInfoIdSet) else Map.empty + val cacheLookupResponses: Map[String, Set[CallCachingEntryId]] = if (activity.readFromCache) standardCacheLookupResponses(singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet) else Map.empty val ejha = createEngineJobHashingActor( replyTo = replyTo.ref, activity = activity, @@ -47,7 +48,7 @@ class EngineJobHashingActorSpec extends TestKit(new CromwellTestkitSpec.TestWork deathWatch watch ejha - if (activity.readFromCache) replyTo.expectMsg(CacheHit(MetaInfoId(1))) + if (activity.readFromCache) replyTo.expectMsg(CacheHit(NonEmptyList.of(CallCachingEntryId(1)))) if (activity.writeToCache) replyTo.expectMsgPF(max = 5 seconds, hint = "awaiting cache hit message") { case CallCacheHashes(hashes) => hashes.size should be(4) case x => fail(s"Cache hit anticipated! Instead got a ${x.getClass.getSimpleName}") @@ -57,13 +58,13 @@ class EngineJobHashingActorSpec extends TestKit(new CromwellTestkitSpec.TestWork } s"Wait for requests to the FileHashingActor for the ${activity.readWriteMode} activity" in { - val singleMetaInfoIdSet = Set(MetaInfoId(1)) + val singleCallCachingEntryIdSet = Set(CallCachingEntryId(1)) val replyTo = TestProbe() val fileHashingActor = TestProbe() val deathWatch = TestProbe() - val initialCacheLookupResponses: Map[String, Set[MetaInfoId]] = if (activity.readFromCache) standardCacheLookupResponses(singleMetaInfoIdSet, singleMetaInfoIdSet, singleMetaInfoIdSet, singleMetaInfoIdSet) else Map.empty - val fileCacheLookupResponses = Map("input: File inputFile1" -> singleMetaInfoIdSet, "input: File inputFile2" -> singleMetaInfoIdSet) + val initialCacheLookupResponses: Map[String, Set[CallCachingEntryId]] = if (activity.readFromCache) standardCacheLookupResponses(singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet) else Map.empty + val fileCacheLookupResponses = Map("input: File inputFile1" -> singleCallCachingEntryIdSet, "input: File inputFile2" -> singleCallCachingEntryIdSet) val jobDescriptor = templateJobDescriptor(inputs = Map( "inputFile1" -> WdlFile("path"), @@ -86,7 +87,7 @@ class EngineJobHashingActorSpec extends TestKit(new CromwellTestkitSpec.TestWork } } - if (activity.readFromCache) replyTo.expectMsg(CacheHit(MetaInfoId(1))) + if (activity.readFromCache) replyTo.expectMsg(CacheHit(NonEmptyList.of(CallCachingEntryId(1)))) if (activity.writeToCache) replyTo.expectMsgPF(max = 5 seconds, hint = "awaiting cache hit message") { case CallCacheHashes(hashes) => hashes.size should be(6) case x => fail(s"Cache hit anticipated! Instead got a ${x.getClass.getSimpleName}") @@ -96,13 +97,13 @@ class EngineJobHashingActorSpec extends TestKit(new CromwellTestkitSpec.TestWork } s"Cache miss for bad FileHashingActor results but still return hashes in the ${activity.readWriteMode} activity" in { - val singleMetaInfoIdSet = Set(MetaInfoId(1)) + val singleCallCachingEntryIdSet = Set(CallCachingEntryId(1)) val replyTo = TestProbe() val fileHashingActor = TestProbe() val deathWatch = TestProbe() - val initialCacheLookupResponses: Map[String, Set[MetaInfoId]] = if (activity.readFromCache) standardCacheLookupResponses(singleMetaInfoIdSet, singleMetaInfoIdSet, singleMetaInfoIdSet, singleMetaInfoIdSet) else Map.empty - val fileCacheLookupResponses = Map("input: File inputFile1" -> Set(MetaInfoId(2)), "input: File inputFile2" -> singleMetaInfoIdSet) + val initialCacheLookupResponses: Map[String, Set[CallCachingEntryId]] = if (activity.readFromCache) standardCacheLookupResponses(singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet) else Map.empty + val fileCacheLookupResponses = Map("input: File inputFile1" -> Set(CallCachingEntryId(2)), "input: File inputFile2" -> singleCallCachingEntryIdSet) val jobDescriptor = templateJobDescriptor(inputs = Map( "inputFile1" -> WdlFile("path"), @@ -139,11 +140,11 @@ class EngineJobHashingActorSpec extends TestKit(new CromwellTestkitSpec.TestWork } s"Detect call cache misses for the ${activity.readWriteMode} activity" in { - val singleMetaInfoIdSet = Set(MetaInfoId(1)) + val singleCallCachingEntryIdSet = Set(CallCachingEntryId(1)) val replyTo = TestProbe() val deathWatch = TestProbe() - val cacheLookupResponses: Map[String, Set[MetaInfoId]] = if (activity.readFromCache) standardCacheLookupResponses(singleMetaInfoIdSet, singleMetaInfoIdSet, Set(MetaInfoId(2)), singleMetaInfoIdSet) else Map.empty + val cacheLookupResponses: Map[String, Set[CallCachingEntryId]] = if (activity.readFromCache) standardCacheLookupResponses(singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, Set(CallCachingEntryId(2)), singleCallCachingEntryIdSet) else Map.empty val ejha = createEngineJobHashingActor( replyTo = replyTo.ref, activity = activity, @@ -177,7 +178,7 @@ object EngineJobHashingActorSpec extends MockitoSugar { jobDescriptor: BackendJobDescriptor = templateJobDescriptor(), initializationData: Option[BackendInitializationData] = None, fileHashingActor: Option[ActorRef] = None, - cacheLookupResponses: Map[String, Set[MetaInfoId]] = Map.empty, + cacheLookupResponses: Map[String, Set[CallCachingEntryId]] = Map.empty, runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition] = Set.empty, backendName: String = "whatever" )(implicit system: ActorSystem) = { @@ -206,10 +207,10 @@ object EngineJobHashingActorSpec extends MockitoSugar { jobDescriptor } - def standardCacheLookupResponses(commandTemplate: Set[MetaInfoId], - inputCount: Set[MetaInfoId], - backendName: Set[MetaInfoId], - outputCount: Set[MetaInfoId]) = Map( + def standardCacheLookupResponses(commandTemplate: Set[CallCachingEntryId], + inputCount: Set[CallCachingEntryId], + backendName: Set[CallCachingEntryId], + outputCount: Set[CallCachingEntryId]) = Map( "command template" -> commandTemplate, "input count" -> inputCount, "backend name" -> backendName, diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/PredictableCallCacheReadActor.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/PredictableCallCacheReadActor.scala index 4182095c6..c98587840 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/PredictableCallCacheReadActor.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/PredictableCallCacheReadActor.scala @@ -10,7 +10,7 @@ import scala.util.{Failure, Success, Try} /** * Has a set of responses which it will respond with. If it gets a request for anything that it's not expecting to respond to will generate a failure. */ -class PredictableCallCacheReadActor(responses: Map[String, Set[MetaInfoId]]) extends Actor with ActorLogging { +class PredictableCallCacheReadActor(responses: Map[String, Set[CallCachingEntryId]]) extends Actor with ActorLogging { var responsesRemaining = responses @@ -25,7 +25,7 @@ class PredictableCallCacheReadActor(responses: Map[String, Set[MetaInfoId]]) ext } } - private def respond(sndr: ActorRef, hashes: Set[HashResult], result: Try[Set[MetaInfoId]]) = result match { + private def respond(sndr: ActorRef, hashes: Set[HashResult], result: Try[Set[CallCachingEntryId]]) = result match { case Success(cacheMatches) => sndr ! CacheResultMatchesForHashes(hashes, cacheMatches) case Failure(t) => sndr ! CacheResultLookupFailure(t) } @@ -35,7 +35,7 @@ class PredictableCallCacheReadActor(responses: Map[String, Set[MetaInfoId]]) ext case None => Failure(new Exception(s"Error looking up response $name!")) } - private def resultLookupFolder(current: Try[Set[MetaInfoId]], next: HashResult): Try[Set[MetaInfoId]] = current flatMap { c => + private def resultLookupFolder(current: Try[Set[CallCachingEntryId]], next: HashResult): Try[Set[CallCachingEntryId]] = current flatMap { c => val lookedUp = toTry(next.hashKey.key, responses.get(next.hashKey.key)) lookedUp map { l => c.intersect(l) } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaBackendIsCopyingCachedOutputsSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaBackendIsCopyingCachedOutputsSpec.scala index 51de21e6b..7aa4ccfda 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaBackendIsCopyingCachedOutputsSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaBackendIsCopyingCachedOutputsSpec.scala @@ -1,17 +1,19 @@ package cromwell.engine.workflow.lifecycle.execution.ejea +import cats.data.NonEmptyList import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ import EngineJobExecutionActorSpec._ import cromwell.core.callcaching.CallCachingMode -import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CallCacheHashes, EJHAResponse, HashError} +import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CallCacheHashes, EJHAResponse, HashError} +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId import scala.util.{Failure, Success, Try} import cromwell.engine.workflow.lifecycle.execution.ejea.HasJobSuccessResponse.SuccessfulCallCacheHashes -class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec with HasJobSuccessResponse with HasJobFailureResponses with CanExpectJobStoreWrites with CanExpectCacheWrites { +class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec with HasJobSuccessResponse with HasJobFailureResponses with CanExpectJobStoreWrites with CanExpectCacheWrites with CanExpectCacheInvalidation { override implicit val stateUnderTest = BackendIsCopyingCachedOutputs - "An EJEA in FetchingCachedOutputsFromDatabase state" should { + "An EJEA in BackendIsCopyingCachedOutputs state" should { val hashErrorCause = new Exception("blah") val hashResultsDataValue = Some(Success(SuccessfulCallCacheHashes)) @@ -95,19 +97,18 @@ class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec } } - RestartOrExecuteCommandTuples foreach { case RestartOrExecuteCommandTuple(operationName, restarting, expectedMessage) => - s"$operationName the job immediately when it gets a failure result, and it was going to receive $hashComboName, if call caching is $mode" in { - ejea = ejeaInBackendIsCopyingCachedOutputsState(initialHashData, mode, restarting = restarting) + s"invalidate a call for caching if backend coping failed when it was going to receive $hashComboName, if call caching is $mode" in { + ejea = ejeaInBackendIsCopyingCachedOutputsState(initialHashData, mode) // Send the response from the copying actor ejea ! failureNonRetryableResponse - helper.bjeaProbe.expectMsg(awaitTimeout, expectedMessage) - ejea.stateName should be(RunningJob) - ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper. bjeaProps, initialHashData)) + expectInvalidateCallCacheActor(cacheId) + eventually { ejea.stateName should be(InvalidatingCacheEntry) } + ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper. bjeaProps, initialHashData, cacheHit)) } - s"$operationName the job (preserving and received hashes) when call caching is $mode, the EJEA has $hashComboName and then gets a success result" in { - ejea = ejeaInBackendIsCopyingCachedOutputsState(initialHashData, mode, restarting = restarting) + s"invalidate a call for caching if backend coping failed (preserving and received hashes) when call caching is $mode, the EJEA has $hashComboName and then gets a success result" in { + ejea = ejeaInBackendIsCopyingCachedOutputsState(initialHashData, mode) // Send the response from the EJHA (if there was one!): ejhaResponse foreach { ejea ! _ } @@ -118,15 +119,16 @@ class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec // Send the response from the copying actor ejea ! failureNonRetryableResponse - helper.bjeaProbe.expectMsg(awaitTimeout, expectedMessage) - ejea.stateName should be(RunningJob) - ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper. bjeaProps, finalHashData)) + expectInvalidateCallCacheActor(cacheId) + eventually { ejea.stateName should be(InvalidatingCacheEntry) } + ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper. bjeaProps, finalHashData, cacheHit)) } - } } } } - def standardResponsePendingData(hashes: Option[Try[CallCacheHashes]]) = ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, hashes) + private val cacheId: CallCachingEntryId = CallCachingEntryId(74) + private val cacheHit = Option(CacheHit(NonEmptyList.of(cacheId))) + def standardResponsePendingData(hashes: Option[Try[CallCacheHashes]]) = ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, hashes, cacheHit) def ejeaInBackendIsCopyingCachedOutputsState(initialHashes: Option[Try[CallCacheHashes]], callCachingMode: CallCachingMode, restarting: Boolean = false) = helper.buildEJEA(restarting = restarting, callCachingMode = callCachingMode).setStateInline(data = standardResponsePendingData(initialHashes)) } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala index d9de4432b..930133d10 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala @@ -1,25 +1,22 @@ package cromwell.engine.workflow.lifecycle.execution.ejea +import cats.data.NonEmptyList import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{CheckingCallCache, FetchingCachedOutputsFromDatabase, ResponsePendingData, RunningJob} import EngineJobExecutionActorSpec.EnhancedTestEJEA import cromwell.core.callcaching.{CallCachingActivity, CallCachingOff, ReadCache} import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CacheMiss, HashError} -import cromwell.engine.workflow.lifecycle.execution.callcaching.MetaInfoId +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId import org.scalatest.concurrent.Eventually -class EjeaCheckingCallCacheSpec extends EngineJobExecutionActorSpec with Eventually { +class EjeaCheckingCallCacheSpec extends EngineJobExecutionActorSpec with Eventually with CanExpectFetchCachedResults { override implicit val stateUnderTest = CheckingCallCache "An EJEA in CheckingCallCache mode" should { "Try to fetch the call cache outputs if it gets a CacheHit" in { createCheckingCallCacheEjea() - ejea ! CacheHit(MetaInfoId(75)) - eventually { helper.fetchCachedResultsActorCreations.hasExactlyOne should be(true) } - helper.fetchCachedResultsActorCreations checkIt { - case (CacheHit(metainfoId), _) => metainfoId should be(MetaInfoId(75)) - case _ => fail("Incorrect creation of the fetchCachedResultsActor") - } + ejea ! CacheHit(NonEmptyList.of(CallCachingEntryId(75))) + expectFetchCachedResultsActor(CallCachingEntryId(75)) ejea.stateName should be(FetchingCachedOutputsFromDatabase) } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaFetchingCachedOutputsFromDatabaseSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaFetchingCachedOutputsFromDatabaseSpec.scala index cc6c83b41..4acd2efab 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaFetchingCachedOutputsFromDatabaseSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaFetchingCachedOutputsFromDatabaseSpec.scala @@ -1,14 +1,14 @@ package cromwell.engine.workflow.lifecycle.execution.ejea -import cromwell.core.WorkflowId -import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ -import EngineJobExecutionActorSpec._ import cromwell.backend.BackendCacheHitCopyingActor.CopyOutputsCommand +import cromwell.core.WorkflowId import cromwell.core.callcaching.{CallCachingActivity, ReadAndWriteCache} import cromwell.core.simpleton.WdlValueSimpleton -import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, HashError} +import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ +import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.HashError import cromwell.engine.workflow.lifecycle.execution.callcaching.FetchCachedResultsActor.{CachedOutputLookupFailed, CachedOutputLookupSucceeded} -import cromwell.engine.workflow.lifecycle.execution.callcaching.MetaInfoId +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId +import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec._ import cromwell.engine.workflow.lifecycle.execution.ejea.HasJobSuccessResponse.SuccessfulCallCacheHashes import wdl4s.values.WdlString @@ -36,7 +36,7 @@ class EjeaFetchingCachedOutputsFromDatabaseSpec extends EngineJobExecutionActorS val detritusMap = Map("stdout" -> "//somePath") val cachedReturnCode = Some(17) val sourceCacheDetails = s"${WorkflowId.randomId}:call-someTask:1" - ejea ! CachedOutputLookupSucceeded(cachedSimpletons, detritusMap, cachedReturnCode, CacheHit(MetaInfoId(75)), sourceCacheDetails) + ejea ! CachedOutputLookupSucceeded(cachedSimpletons, detritusMap, cachedReturnCode, CallCachingEntryId(75), sourceCacheDetails) helper.callCacheHitCopyingProbe.expectMsg(CopyOutputsCommand(cachedSimpletons, detritusMap, cachedReturnCode)) // Check we end up in the right state: @@ -61,7 +61,7 @@ class EjeaFetchingCachedOutputsFromDatabaseSpec extends EngineJobExecutionActorS // Send the response from the "Fetch" actor val failureReason = new Exception("You can't handle the truth!") - ejea ! CachedOutputLookupFailed(MetaInfoId(90210), failureReason) + ejea ! CachedOutputLookupFailed(CallCachingEntryId(90210), failureReason) helper.bjeaProbe.expectMsg(awaitTimeout, expectedMessage) // Check we end up in the right state: diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaInvalidatingCacheEntrySpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaInvalidatingCacheEntrySpec.scala new file mode 100644 index 000000000..40e43d77b --- /dev/null +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaInvalidatingCacheEntrySpec.scala @@ -0,0 +1,53 @@ +package cromwell.engine.workflow.lifecycle.execution.ejea + +import cats.data.NonEmptyList +import cromwell.core.callcaching.{CallCachingActivity, ReadCache} +import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._ +import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CacheHit +import cromwell.engine.workflow.lifecycle.execution.callcaching.{CallCacheInvalidatedFailure, CallCacheInvalidatedSuccess, CallCachingEntryId} +import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec._ + +class EjeaInvalidatingCacheEntrySpec extends EngineJobExecutionActorSpec with CanExpectFetchCachedResults { + + override implicit val stateUnderTest = InvalidatingCacheEntry + + "An EJEA in InvalidatingCacheEntry state" should { + + val invalidationErrorCause = new Exception("blah") + val invalidateSuccess = CallCacheInvalidatedSuccess + val invalidateFailure = CallCacheInvalidatedFailure(invalidationErrorCause) + + val metaInfo31: CallCachingEntryId = CallCachingEntryId(31) + val metaInfo32: CallCachingEntryId = CallCachingEntryId(32) + val cacheHitWithTwoIds = CacheHit(NonEmptyList.of(metaInfo32, metaInfo31)) + val cacheHitWithSingleId = CacheHit(NonEmptyList.of(metaInfo31)) + + List(invalidateSuccess, invalidateFailure) foreach { invalidateActorResponse => + s"try the next available hit when response is $invalidateActorResponse" in { + ejea = ejeaInvalidatingCacheEntryState(Option(cacheHitWithTwoIds), restarting = false) + // Send the response from the invalidate actor + ejea ! invalidateActorResponse + + helper.bjeaProbe.expectNoMsg(awaitAlmostNothing) + expectFetchCachedResultsActor(cacheHitWithSingleId.cacheResultIds.head) + eventually { ejea.stateName should be(FetchingCachedOutputsFromDatabase) } + ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, None, Option(cacheHitWithSingleId))) + } + + RestartOrExecuteCommandTuples foreach { case RestartOrExecuteCommandTuple(operationName, restarting, expectedMessage) => + s"$operationName a job if cache invalidation succeeds and there are no other cache hits to try when invalidate response is $invalidateActorResponse" in { + ejea = ejeaInvalidatingCacheEntryState(Option(cacheHitWithSingleId), restarting = restarting) + // Send the response from the invalidate actor + ejea ! invalidateActorResponse + + helper.bjeaProbe.expectMsg(awaitTimeout, expectedMessage) + eventually { ejea.stateName should be(RunningJob) } + ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper. bjeaProps, None, None)) + } + } + } + } + + def standardResponsePendingData(hit: Option[CacheHit]) = ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, None, hit) + def ejeaInvalidatingCacheEntryState(hit: Option[CacheHit], restarting: Boolean = false) = helper.buildEJEA(restarting = restarting, callCachingMode = CallCachingActivity(ReadCache)).setStateInline(data = standardResponsePendingData(hit)) +} diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala index e8eaeec1a..cbba49811 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala @@ -40,6 +40,7 @@ trait EngineJobExecutionActorSpec extends CromwellTestkitSpec List( ("FetchCachedResultsActor", helper.fetchCachedResultsActorCreations), ("JobHashingActor", helper.jobHashingInitializations), + ("CallCacheInvalidateActor", helper.invalidateCacheActorCreations), ("CallCacheWriteActor", helper.callCacheWriteActorCreations)) foreach { case (name, GotTooMany(list)) => fail(s"Too many $name creations (${list.size})") case _ => // Fine. diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala index 60cc183a9..8ef607c5b 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala @@ -5,6 +5,7 @@ import cromwell.core.JobOutput import cromwell.core.callcaching._ import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{EJEAData, SucceededResponseData, UpdatingCallCache, UpdatingJobStore} import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId import cromwell.jobstore.JobStoreActor.RegisterJobCompleted import cromwell.jobstore.{JobResultSuccess, JobStoreKey} import org.scalatest.concurrent.Eventually @@ -60,6 +61,25 @@ private[ejea] trait CanExpectHashingInitialization extends Eventually { self: En } } +private[ejea] trait CanExpectFetchCachedResults extends Eventually { self: EngineJobExecutionActorSpec => + def expectFetchCachedResultsActor(expectedCallCachingEntryId: CallCachingEntryId): Unit = { + eventually { helper.fetchCachedResultsActorCreations.hasExactlyOne should be(true) } + helper.fetchCachedResultsActorCreations.checkIt { + case (callCachingEntryId, _) => callCachingEntryId should be(expectedCallCachingEntryId) + case _ => fail("Incorrect creation of the fetchCachedResultsActor") + } + } +} + +private[ejea] trait CanExpectCacheInvalidation extends Eventually { self: EngineJobExecutionActorSpec => + def expectInvalidateCallCacheActor(expectedCacheId: CallCachingEntryId): Unit = { + eventually { helper.invalidateCacheActorCreations.hasExactlyOne should be(true) } + helper.invalidateCacheActorCreations.checkIt { cacheId => + cacheId shouldBe expectedCacheId + } + } +} + private[ejea] trait HasJobSuccessResponse { self: EngineJobExecutionActorSpec => val successRc = Option(171) val successOutputs = Map("a" -> JobOutput(WdlInteger(3)), "b" -> JobOutput(WdlString("bee"))) diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala index aa3f5a28c..16274a275 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala @@ -10,10 +10,10 @@ import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.core.callcaching.{CallCachingActivity, CallCachingMode, CallCachingOff} import cromwell.core.{ExecutionStore, JobExecutionToken, OutputStore, WorkflowId} import cromwell.engine.EngineWorkflowDescriptor -import cromwell.engine.backend.BackendSingletonCollection +import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId import cromwell.engine.workflow.lifecycle.execution.{EngineJobExecutionActor, WorkflowExecutionActorData} import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{EJEAData, EngineJobExecutionActorState} -import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CallCacheHashes} +import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes import org.specs2.mock.Mockito import wdl4s.WdlExpression.ScopedLookupFunction import wdl4s.expression.{NoFunctions, WdlFunctions, WdlStandardLibraryFunctions} @@ -62,9 +62,10 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock val backendWorkflowDescriptor = BackendWorkflowDescriptor(workflowId, null, null, null) val backendJobDescriptor = BackendJobDescriptor(backendWorkflowDescriptor, jobDescriptorKey, runtimeAttributes = Map.empty, inputs = Map.empty) - var fetchCachedResultsActorCreations: ExpectOne[(CacheHit, Seq[TaskOutput])] = NothingYet + var fetchCachedResultsActorCreations: ExpectOne[(CallCachingEntryId, Seq[TaskOutput])] = NothingYet var jobHashingInitializations: ExpectOne[(BackendJobDescriptor, CallCachingActivity)] = NothingYet var callCacheWriteActorCreations: ExpectOne[(CallCacheHashes, SucceededResponse)] = NothingYet + var invalidateCacheActorCreations: ExpectOne[CallCachingEntryId] = NothingYet val deathwatch = TestProbe() val bjeaProbe = TestProbe() @@ -146,9 +147,11 @@ private[ejea] class MockEjea(helper: PerTestHelper, backendName: String, callCachingMode: CallCachingMode) extends EngineJobExecutionActor(replyTo, jobDescriptorKey, executionData, factory, initializationData, restarting, serviceRegistryActor, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, None, backendName, callCachingMode) { - override def makeFetchCachedResultsActor(cacheHit: CacheHit, taskOutputs: Seq[TaskOutput]) = helper.fetchCachedResultsActorCreations = helper.fetchCachedResultsActorCreations.foundOne((cacheHit, taskOutputs)) + override def makeFetchCachedResultsActor(cacheId: CallCachingEntryId, taskOutputs: Seq[TaskOutput]) = helper.fetchCachedResultsActorCreations = helper.fetchCachedResultsActorCreations.foundOne((cacheId, taskOutputs)) override def initializeJobHashing(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity) = helper.jobHashingInitializations = helper.jobHashingInitializations.foundOne((jobDescriptor, activity)) override def createSaveCacheResultsActor(hashes: CallCacheHashes, success: SucceededResponse) = helper.callCacheWriteActorCreations = helper.callCacheWriteActorCreations.foundOne((hashes, success)) - + override def invalidateCacheHit(cacheId: CallCachingEntryId): Unit = { + helper.invalidateCacheActorCreations = helper.invalidateCacheActorCreations.foundOne(cacheId) + } override def createJobPreparationActor(jobPrepProps: Props, name: String) = jobPreparationProbe.ref } diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala index 0575b7b33..444948c0e 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActorSpec.scala @@ -2,15 +2,15 @@ package cromwell.engine.workflow.tokens import java.util.UUID -import akka.actor.{ActorRef, ActorSystem, Kill, PoisonPill} -import org.scalatest._ +import akka.actor.{ActorSystem, PoisonPill} import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe} +import cromwell.core.JobExecutionToken import cromwell.core.JobExecutionToken.JobExecutionTokenType import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed, JobExecutionTokenRequest, JobExecutionTokenReturn} -import JobExecutionTokenDispenserActorSpec._ -import cromwell.core.JobExecutionToken +import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActorSpec._ import cromwell.engine.workflow.tokens.TestTokenGrabbingActor.StoppingSupervisor import cromwell.util.AkkaTestUtil +import org.scalatest._ import org.scalatest.concurrent.Eventually import scala.concurrent.duration._ diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala index e24d8627d..078e2e081 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala @@ -13,7 +13,7 @@ case class JesCacheHitCopyingActor(override val jobDescriptor: BackendJobDescrip initializationData: JesBackendInitializationData, serviceRegistryActor: ActorRef) extends BackendCacheHitCopyingActor with CacheHitDuplicating with JesJobCachingActorHelper with JobLogging { - override protected def duplicate(source: Path, destination: Path) = PathCopier.copy(source, destination) + override protected def duplicate(source: Path, destination: Path) = PathCopier.copy(source, destination).get override protected def destinationCallRootPath = jesCallPaths.callRootPath From 88ff89f0ea378cea46aaaca7d206cb55acec482b Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Thu, 13 Oct 2016 10:13:07 -0400 Subject: [PATCH 22/23] release_notes --- CHANGELOG.md | 63 ++++++++++++++++++++++++++++++++++++---------- project/Dependencies.scala | 4 +-- 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ff214c55..eabc031c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,22 +1,44 @@ # Cromwell Change Log -## 0.20 +## 0.22 -* The default per-upload bytes size for GCS is now the minumum 256K -instead of 64M. There is also an undocumented config key -`google.upload-buffer-bytes` that allows adjusting this internal value. +* Improved retries for Call Caching and general bug fixes. +* Now there are configurable caching strategies for the SharedFileSystem backends (i.e. Local). This new "caching" + stanza will be nested inside of: backend.{your_SFS_backend}.config.filesystems.local stanza. + See below for detailed descriptions of each configurable key. -* Updated Docker Hub hash retriever to parse json with [custom media -types](https://github.com/docker/distribution/blob/05b0ab0/docs/spec/manifest-v2-1.md). +``` +caching { + duplication-strategy: [ + "hard-link", "soft-link", "copy" + ] + + # Possible values: file, path + # "file" will compute an md5 hash of the file content. + # "path" will compute an md5 hash of the file path. This strategy will only be effective if the duplication-strategy (above) is set to "soft-link", + # in order to allow for the original file path to be hashed. + hashing-strategy: "file" + + # When true, will check if a sibling file with the same name and the .md5 extension exists, and if it does, use the content of this file as a hash. + # If false or the md5 does not exist, will proceed with the above-defined hashing strategy. + check-sibling-md5: false +} +``` +* Mulitple Input JSON files can now be submitted in server mode through the existing submission endpoint: /api/workflows/:version. + This endpoint accepts a POST request with a multipart/form-data encoded body. You can now include multiple keys for workflow inputs. -* Added a `/batch` submit endpoint that accepts a single wdl with -multiple input files. + The keys below can contain optional JSON file(s) of the workflow inputs. A skeleton file can be generated from wdltool using the "inputs" subcommand. + NOTE: Each prcoceeding workflowInput file will override any JSON key conflicts. -* The `/query` endpoint now supports querying by `id`, and submitting -parameters as a HTTP POST. + workflowInputs + workflowInputs_2 + workflowInputs_3 + workflowInputs_4 + workflowInputs_5 -## 0.21 +* Batched status polling of Google Jes for running jobs. +## 0.21 * Warning: Significant database updates when you switch from version 0.19 to 0.21 of Cromwell. There may be a long wait period for the migration to finish for large databases. @@ -71,7 +93,7 @@ task { command { echo "I'm private !" } - + runtime { docker: "ubuntu:latest" noAddress: true @@ -94,7 +116,7 @@ passed absolute paths for input `File`s. * Override the default database configuration by setting the keys `database.driver`, `database.db.driver`, `database.db.url`, etc. * Override the default database configuration by setting the keys -`database.driver`, `database.db.driver`, `database.db.url`, etc. +`database.driver`, `database.db.driver`, `database.db.url`, etc. For example: ``` @@ -111,3 +133,18 @@ database { } ``` +## 0.20 + +* The default per-upload bytes size for GCS is now the minumum 256K +instead of 64M. There is also an undocumented config key +`google.upload-buffer-bytes` that allows adjusting this internal value. + +* Updated Docker Hub hash retriever to parse json with [custom media +types](https://github.com/docker/distribution/blob/05b0ab0/docs/spec/manifest-v2-1.md). + +* Added a `/batch` submit endpoint that accepts a single wdl with +multiple input files. + +* The `/query` endpoint now supports querying by `id`, and submitting +parameters as a HTTP POST. + diff --git a/project/Dependencies.scala b/project/Dependencies.scala index ca5420d5e..a902e1b2a 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,8 +1,8 @@ import sbt._ object Dependencies { - lazy val lenthallV = "0.19-882a763-SNAPSHOT" - lazy val wdl4sV = "0.6-2964173-SNAPSHOT" + lazy val lenthallV = "0.19" + lazy val wdl4sV = "0.6" lazy val sprayV = "1.3.3" /* spray-json is an independent project from the "spray suite" From 06d7f33f2bcd7c47fa714f69ac8daadfce2c77ee Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Thu, 13 Oct 2016 11:41:17 -0400 Subject: [PATCH 23/23] typos and reword --- CHANGELOG.md | 67 ++++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eabc031c9..cf48b60be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,32 +3,45 @@ ## 0.22 * Improved retries for Call Caching and general bug fixes. -* Now there are configurable caching strategies for the SharedFileSystem backends (i.e. Local). This new "caching" - stanza will be nested inside of: backend.{your_SFS_backend}.config.filesystems.local stanza. +* Users will experience better scalability of status polling for Google JES. +* Now there are configurable caching strategies for a SharedFileSystem backend (i.e. Local, SFS) in the backend's stanza: See below for detailed descriptions of each configurable key. ``` -caching { - duplication-strategy: [ - "hard-link", "soft-link", "copy" - ] - - # Possible values: file, path - # "file" will compute an md5 hash of the file content. - # "path" will compute an md5 hash of the file path. This strategy will only be effective if the duplication-strategy (above) is set to "soft-link", - # in order to allow for the original file path to be hashed. - hashing-strategy: "file" - - # When true, will check if a sibling file with the same name and the .md5 extension exists, and if it does, use the content of this file as a hash. - # If false or the md5 does not exist, will proceed with the above-defined hashing strategy. - check-sibling-md5: false -} +backend { + ... + providers { + SFS_BackendName { + actor-factory = ... + config { + ... + filesystems { + local { + localization: [ + ... + ] + caching { + duplication-strategy: [ + "hard-link", "soft-link", "copy" + ] + # Possible values: file, path + # "file" will compute an md5 hash of the file content. + # "path" will compute an md5 hash of the file path. This strategy will only be effective if the duplication-strategy (above) is set to "soft-link", + # in order to allow for the original file path to be hashed. + hashing-strategy: "file" + + # When true, will check if a sibling file with the same name and the .md5 extension exists, and if it does, use the content of this file as a hash. + # If false or the md5 does not exist, will proceed with the above-defined hashing strategy. + check-sibling-md5: false + } ``` -* Mulitple Input JSON files can now be submitted in server mode through the existing submission endpoint: /api/workflows/:version. +* Multiple Input JSON files can now be submitted in server mode through the existing submission endpoint: /api/workflows/:version. This endpoint accepts a POST request with a multipart/form-data encoded body. You can now include multiple keys for workflow inputs. - The keys below can contain optional JSON file(s) of the workflow inputs. A skeleton file can be generated from wdltool using the "inputs" subcommand. - NOTE: Each prcoceeding workflowInput file will override any JSON key conflicts. + Each key below can contain an optional JSON file of the workflow inputs. A skeleton file can be generated from wdltool using the "inputs" subcommand. + NOTE: In case of key conflicts between multiple JSON files, higher values of x in workflowInputs_x override lower values. For example, an input + specified in workflowInputs_3 will override an input with the same name that was given in workflowInputs or workflowInputs_2. Similarly, an input + specified in workflowInputs_5 will override an input with the same name in any other input file. workflowInputs workflowInputs_2 @@ -36,7 +49,17 @@ caching { workflowInputs_4 workflowInputs_5 -* Batched status polling of Google Jes for running jobs. +* You can now limit the number of concurrent jobs for a backend by specifying the following option in the backend's config stanza: +``` +backend { + ... + providers { + BackendName { + actor-factory = ... + config { + concurrent-job-limit = 5 +``` + ## 0.21 @@ -135,7 +158,7 @@ database { ## 0.20 -* The default per-upload bytes size for GCS is now the minumum 256K +* The default per-upload bytes size for GCS is now the minimum 256K instead of 64M. There is also an undocumented config key `google.upload-buffer-bytes` that allows adjusting this internal value.