diff --git a/.travis.yml b/.travis.yml index f4d0d9b2e..cfd59b202 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,16 @@ scala: - 2.11.8 jdk: - oraclejdk8 +cache: + # md5deep - https://github.com/travis-ci/travis-ci/issues/3122 + branch: md5deep + directories: + - $HOME/.ivy2/cache + - $HOME/.sbt/boot/ +before_cache: + # Tricks to avoid unnecessary cache updates + - find $HOME/.ivy2 -name "ivydata-*.properties" -delete + - find $HOME/.sbt -name "*.lock" -delete env: global: - CENTAUR_BRANCH=develop diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f7b649ee..06b720f39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Cromwell Change Log +## 24 + +* When emitting workflow outputs to the Cromwell log only the first 1000 characters per output will be printed +* Added support for conditional (`if`) statements. +* Globs for Shared File System (SFS) backends, such as local or SGE, now use bash globbing instead of Java globbing, consistent with the JES backend. + ## 23 * The `meta` and `parameter_meta` blocks are now valid within `workflow` blocks, not just `task` diff --git a/LICENSE.txt b/LICENSE.txt index cf0814020..6acd68571 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2015-2016, Broad Institute, Inc. +Copyright (c) 2015, Broad Institute, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -24,4 +24,4 @@ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE \ No newline at end of file diff --git a/README.md b/README.md index ca5c14576..49c09df42 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,9 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management * [Command Line Usage](#command-line-usage) * [run](#run) * [server](#server) + * [version](#version) * [Getting Started with WDL](#getting-started-with-wdl) + * [WDL Support](#wdl-support) * [Configuring Cromwell](#configuring-cromwell) * [Workflow Submission](#workflow-submission) * [Database](#database) @@ -85,6 +87,7 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management * [POST /api/workflows/:version/:id/abort](#post-apiworkflowsversionidabort) * [GET /api/workflows/:version/backends](#get-apiworkflowsversionbackends) * [GET /api/engine/:version/stats](#get-apiengineversionstats) + * [GET /api/engine/:version/version](#get-apiengineversionversion) * [Error handling](#error-handling) * [Developer](#developer) * [Generating table of contents on Markdown files](#generating-table-of-contents-on-markdown-files) @@ -158,6 +161,11 @@ run [] [] Starts a web server on port 8000. See the web server documentation for more details about the API endpoints. + + -version + + Returns the version of the Cromwell engine. + ``` ## run @@ -283,10 +291,117 @@ $ java -jar cromwell.jar run threestep.wdl - - - /path/to/my_WDLs.zip Start a server on port 8000, the API for the server is described in the [REST API](#rest-api) section. +## version + +Returns the version of Cromwell engine. + # Getting Started with WDL For many examples on how to use WDL see [the WDL site](https://github.com/broadinstitute/wdl#getting-started-with-wdl) +## WDL Support + +:pig2: Cromwell supports the following subset of WDL features: + +* [Language Specification](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#language-specification) + * [Whitespace, Strings, Identifiers, Constants](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#whitespace-strings-identifiers-constants) + * [Types](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#types) + * [Fully Qualified Names & Namespaced Identifiers](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#fully-qualified-names--namespaced-identifiers) + * [Declarations](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#declarations) + * [Expressions](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#expressions) + * [Operator Precedence Table](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#operator-precedence-table) + * [Member Access](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#member-access) + * [Map and Array Indexing](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#map-and-array-indexing) + * [Function Calls](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#function-calls) + * [Array Literals](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#array-literals) + * [Map Literals](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#map-literals) +* [Document](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#document) +* [Task Definition](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#task-definition) + * [Sections](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#sections) + * [Command Section](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#command-section) + * [Command Parts](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#command-parts) + * [Command Part Options](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#command-part-options) + * [sep](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#sep) + * [true and false](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#true-and-false) + * [default](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#default) + * [Alternative heredoc syntax](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#alternative-heredoc-syntax) + * [Stripping Leading Whitespace](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#stripping-leading-whitespace) + * [Outputs Section](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#outputs-section) + * [String Interpolation](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#string-interpolation) + * [Runtime Section](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#runtime-section) + * [docker](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#docker) + * [memory](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#memory) + * [Parameter Metadata Section](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#parameter-metadata-section) + * [Metadata Section](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#metadata-section) +* [Workflow Definition](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#workflow-definition) + * [Call Statement](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#call-statement) + * [Sub Workflows](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#sub-workflows) + * [Scatter](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#scatter) + * [Parameter Metadata](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#parameter-metadata) + * [Metadata](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#metadata) + * [Outputs](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#outputs) +* [Namespaces](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#namespaces) +* [Scope](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#scope) +* [Optional Parameters & Type Constraints](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#optional-parameters--type-constraints) + * [Prepending a String to an Optional Parameter](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#prepending-a-string-to-an-optional-parameter) +* [Scatter / Gather](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#scatter--gather) +* [Variable Resolution](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#variable-resolution) + * [Task-Level Resolution](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#task-level-resolution) + * [Workflow-Level Resolution](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#workflow-level-resolution) +* [Computing Inputs](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#computing-inputs) + * [Task Inputs](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#task-inputs) + * [Workflow Inputs](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#workflow-inputs) + * [Specifying Workflow Inputs in JSON](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#specifying-workflow-inputs-in-json) +* [Type Coercion](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#type-coercion) +* [Standard Library](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#standard-library) + * [File stdout()](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#file-stdout) + * [File stderr()](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#file-stderr) + * [Array\[String\] read_lines(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arraystring-read_linesstringfile) + * [Array\[Array\[String\]\] read_tsv(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arrayarraystring-read_tsvstringfile) + * [Map\[String, String\] read_map(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#mapstring-string-read_mapstringfile) + * [Object read_object(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#object-read_objectstringfile) + * [Array\[Object\] read_objects(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arrayobject-read_objectsstringfile) + * [Int read_int(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#int-read_intstringfile) + * [String read_string(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#string-read_stringstringfile) + * [Float read_float(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#float-read_floatstringfile) + * [Boolean read_boolean(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#boolean-read_booleanstringfile) + * [File write_lines(Array\[String\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#file-write_linesarraystring) + * [File write_tsv(Array\[Array\[String\]\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#file-write_tsvarrayarraystring) + * [File write_map(Map\[String, String\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#file-write_mapmapstring-string) + * [File write_object(Object)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#file-write_objectobject) + * [File write_objects(Array\[Object\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#file-write_objectsarrayobject) + * [Float size(File, \[String\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#float-sizefile-string) + * [String sub(String, String, String)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#string-substring-string-string) + * [Array\[Int\] range(Int)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arrayint-rangeint) + * [Array\[Array\[X\]\] transpose(Array\[Array\[X\]\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arrayarrayx-transposearrayarrayx) + * [Pair(X,Y) zip(X,Y)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#pairxy-zipxy) + * [Pair(X,Y) cross(X,Y)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#pairxy-crossxy) +* [Data Types & Serialization](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#data-types--serialization) + * [Serialization of Task Inputs](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#serialization-of-task-inputs) + * [Primitive Types](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#primitive-types) + * [Compound Types](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#compound-types) + * [Array serialization](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#array-serialization) + * [Array serialization by expansion](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#array-serialization-by-expansion) + * [Array serialization using write_lines()](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#array-serialization-using-write_lines) + * [Map serialization](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#map-serialization) + * [Map serialization using write_map()](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#map-serialization-using-write_map) + * [Object serialization](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#object-serialization) + * [Object serialization using write_object()](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#object-serialization-using-write_object) + * [Array\[Object\] serialization](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arrayobject-serialization) + * [Array\[Object\] serialization using write_objects()](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arrayobject-serialization-using-write_objects) + * [De-serialization of Task Outputs](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#de-serialization-of-task-outputs) + * [Primitive Types](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#primitive-types) + * [Compound Types](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#compound-types) + * [Array deserialization](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#array-deserialization) + * [Array deserialization using read_lines()](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#array-deserialization-using-read_lines) + * [Map deserialization](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#map-deserialization) + * [Map deserialization using read_map()](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#map-deserialization-using-read_map) + * [Object deserialization](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#object-deserialization) + * [Object deserialization using read_object()](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#object-deserialization-using-read_object) + * [Array\[Object\] deserialization](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arrayobject-deserialization) + * [Object deserialization using read_objects()](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#object-deserialization-using-read_objects) + + # Configuring Cromwell Cromwell's default configuration file is located at `src/main/resources/application.conf`. @@ -2313,6 +2428,7 @@ contain one workflow submission response for each input, respectively. * `workflowOptions` - *Optional* JSON file containing options for this workflow execution. See the [run](#run) CLI sub-command for some more information about this. +* `wdlDependencies` - *Optional* ZIP file containing WDL files that are used to resolve import statements. Applied equally to all workflowInput sets. cURL: @@ -3115,6 +3231,32 @@ Response: } ``` +## GET /api/engine/:version/version + +This endpoint returns the version of the Cromwell engine. + +cURL: +``` +$ curl http://localhost:8000/api/engine/v1/version +``` + +HTTPie: +``` +$ http http://localhost:8000/api/engine/v1/version +``` + +Response: +``` +"date": "Sun, 18 Sep 2016 14:38:11 GMT", +"server": "spray-can/1.3.3", +"content-length": "33", +"content-type": "application/json; charset=UTF-8" + +{ + "cromwell": 23-8be799a-SNAP +} +``` + ## Error handling diff --git a/backend/src/main/scala/cromwell/backend/MemorySize.scala b/backend/src/main/scala/cromwell/backend/MemorySize.scala index b207174b6..bf5438764 100644 --- a/backend/src/main/scala/cromwell/backend/MemorySize.scala +++ b/backend/src/main/scala/cromwell/backend/MemorySize.scala @@ -4,7 +4,7 @@ package cromwell.backend import cats.data.Validated._ import cats.syntax.cartesian._ import cats.syntax.validated._ -import cromwell.core.ErrorOr._ +import lenthall.validation.ErrorOr._ import mouse.string._ import scala.language.postfixOps diff --git a/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala b/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala index ae9d4b9bf..0c9bf8727 100644 --- a/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala +++ b/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala @@ -2,9 +2,9 @@ package cromwell.backend import cromwell.core.WorkflowOptions import cromwell.util.JsonFormatting.WdlValueJsonFormatter +import lenthall.util.TryUtil import wdl4s.{WdlExpressionException, _} import wdl4s.expression.WdlStandardLibraryFunctions -import wdl4s.util.TryUtil import wdl4s.values.WdlValue import scala.util.{Success, Try} diff --git a/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala index 759127d67..3aaadfd55 100644 --- a/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala @@ -4,7 +4,7 @@ import akka.actor.{Actor, ActorLogging, ActorRef} import cromwell.backend.BackendJobDescriptor import cromwell.backend.BackendJobExecutionActor._ import cromwell.backend.async.AsyncBackendJobExecutionActor._ -import cromwell.core.CromwellFatalException +import cromwell.core.CromwellFatalExceptionMarker import cromwell.core.retry.{Retry, SimpleExponentialBackoff} import cromwell.services.metadata.MetadataService.MetadataServiceResponse @@ -35,10 +35,12 @@ trait AsyncBackendJobExecutionActor { this: Actor with ActorLogging => def retryable: Boolean - private def withRetry[A](work: () => Future[A], backoff: SimpleExponentialBackoff): Future[A] = { - def isFatal(t: Throwable) = t.isInstanceOf[CromwellFatalException] + def isFatal(throwable: Throwable): Boolean = throwable.isInstanceOf[CromwellFatalExceptionMarker] - Retry.withRetry(work, isTransient = !isFatal(_), isFatal = isFatal, backoff = backoff)(context.system) + def isTransient(throwable: Throwable): Boolean = !isFatal(throwable) + + private def withRetry[A](work: () => Future[A], backOff: SimpleExponentialBackoff): Future[A] = { + Retry.withRetry(work, isTransient = isTransient, isFatal = isFatal, backoff = backOff)(context.system) } private def robustExecuteOrRecover(mode: ExecutionMode) = { @@ -74,7 +76,7 @@ trait AsyncBackendJobExecutionActor { this: Actor with ActorLogging => // -Ywarn-value-discard context.system.scheduler.scheduleOnce(pollBackOff.backoffMillis.millis, self, IssuePollRequest(handle)) () - case Finish(SuccessfulExecutionHandle(outputs, returnCode, jobDetritusFiles, executionEvents, resultsClonedFrom)) => + case Finish(SuccessfulExecutionHandle(outputs, returnCode, jobDetritusFiles, executionEvents, _)) => completionPromise.success(JobSucceededResponse(jobDescriptor.key, Some(returnCode), outputs, Option(jobDetritusFiles), executionEvents)) context.stop(self) case Finish(FailedNonRetryableExecutionHandle(throwable, returnCode)) => diff --git a/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala b/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala index 1e4238014..62618190c 100644 --- a/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala +++ b/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala @@ -3,6 +3,7 @@ package cromwell.backend.async import java.nio.file.Path import cromwell.backend.BackendJobDescriptor +import cromwell.backend.async.AsyncBackendJobExecutionActor.JobId import cromwell.core.{ExecutionEvent, CallOutputs} /** @@ -14,6 +15,17 @@ trait ExecutionHandle { def result: ExecutionResult } +case class PendingExecutionHandle[BackendJobId <: JobId, BackendRunInfo, BackendRunStatus] +( + jobDescriptor: BackendJobDescriptor, + pendingJob: BackendJobId, + runInfo: Option[BackendRunInfo], + previousStatus: Option[BackendRunStatus] +) extends ExecutionHandle { + override val isDone = false + override val result = NonRetryableExecution(new IllegalStateException("PendingExecutionHandle cannot yield a result")) +} + final case class SuccessfulExecutionHandle(outputs: CallOutputs, returnCode: Int, jobDetritusFiles: Map[String, Path], executionEvents: Seq[ExecutionEvent], resultsClonedFrom: Option[BackendJobDescriptor] = None) extends ExecutionHandle { override val isDone = true override val result = SuccessfulExecution(outputs, returnCode, jobDetritusFiles, executionEvents, resultsClonedFrom) diff --git a/backend/src/main/scala/cromwell/backend/backend.scala b/backend/src/main/scala/cromwell/backend/backend.scala index e1addfe30..82e863ad0 100644 --- a/backend/src/main/scala/cromwell/backend/backend.scala +++ b/backend/src/main/scala/cromwell/backend/backend.scala @@ -33,9 +33,9 @@ case class BackendJobDescriptor(workflowDescriptor: BackendWorkflowDescriptor, object BackendWorkflowDescriptor { def apply(id: WorkflowId, workflow: Workflow, - inputs: Map[FullyQualifiedName, WdlValue], + knownValues: Map[FullyQualifiedName, WdlValue], workflowOptions: WorkflowOptions) = { - new BackendWorkflowDescriptor(id, workflow, inputs, workflowOptions, List.empty) + new BackendWorkflowDescriptor(id, workflow, knownValues, workflowOptions, List.empty) } } @@ -44,7 +44,7 @@ object BackendWorkflowDescriptor { */ case class BackendWorkflowDescriptor(id: WorkflowId, workflow: Workflow, - inputs: Map[FullyQualifiedName, WdlValue], + knownValues: Map[FullyQualifiedName, WdlValue], workflowOptions: WorkflowOptions, breadCrumbs: List[BackendJobBreadCrumb]) { diff --git a/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala b/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala index 48a6d590e..bfde60986 100644 --- a/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala +++ b/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala @@ -7,6 +7,7 @@ import cromwell.backend.BackendCacheHitCopyingActor import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobSucceededResponse} import cromwell.backend.io.JobPaths import cromwell.core.path.PathCopier +import cromwell.core.path.PathImplicits._ import cromwell.core.simpleton.{WdlValueBuilder, WdlValueSimpleton} import wdl4s.values.WdlFile @@ -47,7 +48,7 @@ trait CacheHitDuplicating { protected def destinationJobDetritusPaths: Map[String, Path] // Usually implemented by a subclass of JobCachingActorHelper - protected def metadataKeyValues: Map[String, Any] + protected def startMetadataKeyValues: Map[String, Any] private def lookupSourceCallRootPath(sourceJobDetritusFiles: Map[String, String]): Path = { sourceJobDetritusFiles.get(JobPaths.CallRootPathKey).map(getPath).get recover { @@ -66,7 +67,7 @@ trait CacheHitDuplicating { val sourcePath = getPath(wdlFile.value).get val destinationPath = PathCopier.getDestinationFilePath(sourceCallRootPath, sourcePath, destinationCallRootPath) duplicate(sourcePath, destinationPath) - WdlValueSimpleton(key, WdlFile(destinationPath.toUri.toString)) + WdlValueSimpleton(key, WdlFile(destinationPath.toRealString)) case wdlValueSimpleton => wdlValueSimpleton } } @@ -97,7 +98,8 @@ trait CacheHitDuplicating { val destinationJobOutputs = WdlValueBuilder.toJobOutputs(jobDescriptor.call.task.outputs, destinationSimpletons) import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter - serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), metadataKeyValues) + serviceRegistryActor.putMetadata( + jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), startMetadataKeyValues) JobSucceededResponse(jobDescriptor.key, returnCodeOption, destinationJobOutputs, Option(destinationJobDetritusFiles), Seq.empty) } diff --git a/backend/src/main/scala/cromwell/backend/callcaching/FileHashingActor.scala b/backend/src/main/scala/cromwell/backend/callcaching/FileHashingActor.scala index c2785d0e6..ee1ecea4c 100644 --- a/backend/src/main/scala/cromwell/backend/callcaching/FileHashingActor.scala +++ b/backend/src/main/scala/cromwell/backend/callcaching/FileHashingActor.scala @@ -3,6 +3,7 @@ package cromwell.backend.callcaching import akka.actor.{Actor, ActorLogging, Props} import akka.event.LoggingAdapter import cromwell.backend.BackendInitializationData +import cromwell.core.Dispatcher.BackendDispatcher import cromwell.core.JobKey import cromwell.core.callcaching._ import wdl4s.values.WdlFile @@ -28,7 +29,7 @@ class FileHashingActor(workerFunction: Option[FileHashingFunction]) extends Acto } object FileHashingActor { - def props(workerFunction: Option[FileHashingFunction]): Props = Props(new FileHashingActor(workerFunction)) + def props(workerFunction: Option[FileHashingFunction]): Props = Props(new FileHashingActor(workerFunction)).withDispatcher(BackendDispatcher) case class FileHashingFunction(work: (SingleFileHashRequest, LoggingAdapter) => Try[String]) diff --git a/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala b/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala new file mode 100644 index 000000000..c4fce1353 --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala @@ -0,0 +1,60 @@ +package cromwell.backend.io + +import java.nio.file.Files + +import cromwell.backend.BackendJobDescriptor +import cromwell.core.CallContext +import wdl4s.TaskCall +import wdl4s.expression.{NoFunctions, PureStandardLibraryFunctionsLike} +import wdl4s.values._ +import scala.collection.JavaConverters._ +import cromwell.core.path.PathImplicits._ + +trait GlobFunctions extends PureStandardLibraryFunctionsLike { + + def callContext: CallContext + + def findGlobOutputs(call: TaskCall, jobDescriptor: BackendJobDescriptor): Set[WdlGlobFile] = { + val globOutputs = call.task.findOutputFiles(jobDescriptor.fullyQualifiedInputs, NoFunctions) collect { + case glob: WdlGlobFile => glob + } + + globOutputs.distinct.toSet + } + + def globDirectory(glob: String): String = globName(glob) + "/" + def globName(glob: String) = s"glob-${glob.md5Sum}" + + /** + * Returns a path to the glob using toRealString. + * + * NOTE: Due to use of toRealString, returned paths must be passed to PathBuilders.buildPath, and will not work with + * Paths.get. + * + * This path is usually passed back into the glob() method below. + * + * @param glob The glob. This is the same "pattern" passed to glob() below. + * @return The path converted using .toRealString. + */ + override def globPath(glob: String): String = callContext.root.resolve(globDirectory(glob)).toRealString + + /** + * Returns a list of path from the glob, each path converted to a string using toRealString. + * + * The paths are currently read from a list file based on the pattern, and the path parameter is not used. + * + * NOTE: Due to use of toRealString, returned paths must be passed to PathBuilders.buildPath, and will not work with + * Paths.get. + * + * @param path The path string returned by globPath. This isn't currently used. + * @param pattern The pattern of the glob. This is the same "glob" passed to globPath(). + * @return The paths that match the pattern, each path converted using .toRealString. + */ + override def glob(path: String, pattern: String): Seq[String] = { + val name = globName(pattern) + val listFile = callContext.root.resolve(s"$name.list").toRealPath() + Files.readAllLines(listFile).asScala map { fileName => + callContext.root.resolve(name).resolve(fileName).toRealString + } + } +} diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala index be959aec5..a342d443e 100644 --- a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala +++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala @@ -18,20 +18,21 @@ trait WorkflowPaths extends PathFactory { def workflowDescriptor: BackendWorkflowDescriptor def config: Config - protected lazy val executionRootString = config.as[Option[String]]("root").getOrElse("cromwell-executions") + protected lazy val executionRootString: String = config.as[Option[String]]("root").getOrElse("cromwell-executions") def getPath(url: String): Try[Path] = Try(PathFactory.buildPath(url, pathBuilders)) // Rebuild potential intermediate call directories in case of a sub workflow - protected def workflowPathBuilder(root: Path) = { + protected def workflowPathBuilder(root: Path): Path = { workflowDescriptor.breadCrumbs.foldLeft(root)((acc, breadCrumb) => { breadCrumb.toPath(acc) }).resolve(workflowDescriptor.workflow.unqualifiedName).resolve(workflowDescriptor.id.toString + "/") } - lazy val executionRoot = PathFactory.buildPath(executionRootString, pathBuilders).toAbsolutePath - lazy val workflowRoot = workflowPathBuilder(executionRoot) - lazy val finalCallLogsPath = workflowDescriptor.getWorkflowOption(FinalCallLogsDir) map getPath map { _.get } + lazy val executionRoot: Path = PathFactory.buildPath(executionRootString, pathBuilders).toAbsolutePath + lazy val workflowRoot: Path = workflowPathBuilder(executionRoot) + lazy val finalCallLogsPath: Option[Path] = + workflowDescriptor.getWorkflowOption(FinalCallLogsDir) map getPath map { _.get } def toJobPaths(jobKey: BackendJobDescriptorKey): JobPaths } diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala new file mode 100644 index 000000000..daabf40a1 --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala @@ -0,0 +1,481 @@ +package cromwell.backend.standard + +import java.nio.file.Path + +import akka.actor.{Actor, ActorLogging, ActorRef} +import akka.event.LoggingReceive +import better.files.File +import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse} +import cromwell.backend.BackendLifecycleActor.AbortJobCommand +import cromwell.backend.async.AsyncBackendJobExecutionActor.{ExecutionMode, JobId, Recover} +import cromwell.backend.async.{AbortedExecutionHandle, AsyncBackendJobExecutionActor, ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle, SuccessfulExecutionHandle} +import cromwell.backend.validation.{ContinueOnReturnCode, ContinueOnReturnCodeFlag} +import cromwell.backend.wdl.Command +import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, BackendJobLifecycleActor} +import cromwell.services.keyvalue.KeyValueServiceActor._ +import cromwell.services.metadata.CallMetadataKeys +import wdl4s._ +import wdl4s.expression.{PureStandardLibraryFunctions, WdlFunctions} +import wdl4s.values.WdlValue + +import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future, Promise} +import scala.util.{Failure, Success, Try} + +/** + * An extension of the generic AsyncBackendJobExecutionActor providing a standard abstract implementation of an + * asynchronous polling backend. + * + * Backends supported by this trait will all have common behavior. If a backend implementor wishes to provide a custom + * behavior, one should instead implement the various methods in `AsyncBackendJobExecutionActor`. + * + * NOTE: Unlike the parent trait `AsyncBackendJobExecutionActor`, this trait is subject to even more frequent updates + * as the common behavior among the backends adjusts in unison. + */ +trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor { + this: Actor with ActorLogging with BackendJobLifecycleActor => + + val SIGTERM = 143 + val SIGINT = 130 + + /** The type of the run info when a job is started. */ + type StandardAsyncRunInfo + + /** The type of the run status returned during each poll. */ + type StandardAsyncRunStatus + + /** The pending execution handle for each poll. */ + type StandardAsyncPendingExecutionHandle = + PendingExecutionHandle[StandardAsyncJob, StandardAsyncRunInfo, StandardAsyncRunStatus] + + /** Standard set of parameters passed to the backend. */ + val standardParams: StandardAsyncExecutionActorParams + + override lazy val jobDescriptor: BackendJobDescriptor = standardParams.jobDescriptor + + override lazy val configurationDescriptor: BackendConfigurationDescriptor = standardParams.configurationDescriptor + + override lazy val completionPromise: Promise[BackendJobExecutionResponse] = standardParams.completionPromise + + /** Backend initialization data created by the a factory initializer. */ + lazy val backendInitializationDataOption: Option[BackendInitializationData] = + standardParams.backendInitializationDataOption + + /** Typed backend initialization. */ + def backendInitializationDataAs[A <: BackendInitializationData]: A = + BackendInitializationData.as[A](backendInitializationDataOption) + + /** @see [[StandardJobExecutionActorParams.serviceRegistryActor]] */ + lazy val serviceRegistryActor: ActorRef = standardParams.serviceRegistryActor + + /** @see [[StandardJobExecutionActorParams.jobIdKey]] */ + def jobIdKey: String = standardParams.jobIdKey + + /** @see [[Command.instantiate]] */ + def commandLineFunctions: WdlFunctions[WdlValue] = PureStandardLibraryFunctions + + /** @see [[Command.instantiate]] */ + def commandLinePreProcessor: EvaluatedTaskInputs => Try[EvaluatedTaskInputs] = Success.apply + + /** @see [[Command.instantiate]] */ + def commandLineValueMapper: WdlValue => WdlValue = identity + + /** The instantiated command. */ + lazy val instantiatedCommand: String = Command.instantiate( + jobDescriptor, commandLineFunctions, commandLinePreProcessor, commandLineValueMapper).get + + /** A tag that may be used for logging. */ + lazy val tag = s"${this.getClass.getSimpleName} [UUID(${workflowId.shortString}):${jobDescriptor.key.tag}]" + + /** + * When returns true, the `remoteStdErrPath` will be read. If contents of that path are non-empty, the job will fail. + * + * @return True if a non-empty `remoteStdErrPath` should fail the job. + */ + def failOnStdErr: Boolean = false + + /** + * Returns the path to the standard error output of the job. Only needs to be implemented if `failOnStdErr` is + * returning `true`. + * + * @return The path to the standard error output. + */ + def remoteStdErrPath: Path = { + throw new NotImplementedError(s"failOnStdErr returned true but remote path not implemented by $getClass") + } + + /** + * Returns the path to the return code output of the job. Must be implemented unless `returnCodeContents` is + * overridden not to use this method. + * + * @return The path to the return code output. + */ + def remoteReturnCodePath: Path = { + throw new NotImplementedError(s"remoteReturnCodePath returned true but remote path not implemented by $getClass") + } + + /** + * Returns the contents of the return code file. + * + * @return The contents of the return code file. + */ + def returnCodeContents: String = File(remoteReturnCodePath).contentAsString + + /** + * Returns the behavior for continuing on the return code, obtained by converting `returnCodeContents` to an Int. + * + * @return the behavior for continuing on the return code. + */ + def continueOnReturnCode: ContinueOnReturnCode = ContinueOnReturnCodeFlag(false) + + /** + * Returns the metadata key values to store before executing a job. + * + * @return the metadata key values to store before executing a job. + */ + def startMetadataKeyValues: Map[String, Any] = Map.empty + + /** + * Execute the job specified in the params. Should return a `StandardAsyncPendingExecutionHandle`, or a + * `FailedExecutionHandle`. + * + * @return the execution handle for the job. + */ + def execute(): ExecutionHandle + + /** + * Recovers the specified job id, or starts a new job. The default implementation simply calls execute(). + * + * @param jobId The previously recorded job id. + * @return the execution handle for the job. + */ + def recover(jobId: StandardAsyncJob): ExecutionHandle = execute() + + /** + * Returns the run status for the job. + * + * @param handle The handle of the running job. + * @return The status of the job. + */ + def pollStatus(handle: StandardAsyncPendingExecutionHandle): StandardAsyncRunStatus = { + throw new NotImplementedError(s"pollStatus nor pollStatusAsync not implemented by $getClass") + } + + /** + * Returns the async run status for the job. + * + * @param handle The handle of the running job. + * @return The status of the job. + */ + def pollStatusAsync(handle: StandardAsyncPendingExecutionHandle) + (implicit ec: ExecutionContext): Future[StandardAsyncRunStatus] = { + Future.fromTry(Try(pollStatus(handle))) + } + + /** + * Adds custom behavior invoked when polling fails due to some exception. By default adds nothing. + * + * Examples may be when certain error codes are detected during polling, and a specific handle should be returned. + * + * For example, say a custom JobNotFoundException should be mapped to a FailedNonRetryableExecutionHandle. + * + * @return A partial function handler for exceptions after polling. + */ + def customPollStatusFailure: PartialFunction[(ExecutionHandle, Exception), ExecutionHandle] = { + PartialFunction.empty + } + + /** + * Maps the status to a status string that should be stored in the metadata. + * + * @param runStatus The run status. + * @return Some() string that should be stored, or None if nothing should be stored in the metadata. + */ + def statusString(runStatus: StandardAsyncRunStatus): Option[String] = None + + /** + * Returns true when a job is complete, either successfully or unsuccessfully. + * + * @param runStatus The run status. + * @return True if the job has completed. + */ + def isTerminal(runStatus: StandardAsyncRunStatus): Boolean + + /** + * Returns true if the status represents a success. + * + * @param runStatus The run status. + * @return True if the job is a success. + */ + def isSuccess(runStatus: StandardAsyncRunStatus): Boolean = true + + /** + * Returns any custom metadata from the polled status. + * + * @param runStatus The run status. + * @return The job metadata. + */ + def getTerminalMetadata(runStatus: StandardAsyncRunStatus): Map[String, Any] = Map.empty + + /** + * Attempts to abort a job when an abort signal is retrieved. + * + * If `abortAndDieImmediately` is true, then the actor will die immediately after this method returns. + * + * @param jobId The job to abort. + */ + def tryAbort(jobId: StandardAsyncJob): Unit = {} + + /** + * Returns true if when an abort signal is retrieved, the actor makes an attempt to abort and then immediately stops + * itself _without_ polling for an aborted status. + * + * The default is false. + * + * @return true if actor should request an abort and then die immediately. + */ + def requestsAbortAndDiesImmediately: Boolean = false + + /** + * Return true if the return code is an abort code. + * + * By default, return codes `SIGINT` and `SIGTERM` return true. + * + * @param returnCode The return code. + * @return True if the return code is for an abort. + */ + def isAbort(returnCode: Int): Boolean = returnCode == SIGINT || returnCode == SIGTERM + + /** + * Custom behavior to run after an abort signal is processed. + * + * By default handles the behavior of `abortAndDieImmediately`. + */ + def postAbort(): Unit = { + if (requestsAbortAndDiesImmediately) { + context.parent ! AbortedResponse(jobDescriptor.key) + context.stop(self) + } + } + + /** + * Process a successful run, as defined by `isSuccess`. + * + * @param runStatus The run status. + * @param handle The execution handle. + * @param returnCode The return code. + * @return The execution handle. + */ + def handleExecutionSuccess(runStatus: StandardAsyncRunStatus, handle: StandardAsyncPendingExecutionHandle, + returnCode: Int): ExecutionHandle + + /** + * Process an unsuccessful run, as defined by `isSuccess`. + * + * @param runStatus The run status. + * @param handle The execution handle. + * @return The execution handle. + */ + def handleExecutionFailure(runStatus: StandardAsyncRunStatus, + handle: StandardAsyncPendingExecutionHandle): ExecutionHandle = { + FailedNonRetryableExecutionHandle(new Exception(s"Task failed for unknown reason: $runStatus"), None) + } + + context.become(standardReceiveBehavior(None) orElse receive) + + def standardReceiveBehavior(jobIdOption: Option[StandardAsyncJob]): Receive = LoggingReceive { + case AbortJobCommand => + jobIdOption foreach { jobId => + Try(tryAbort(jobId)) match { + case Success(_) => jobLogger.info("{} Aborted {}", tag: Any, jobId) + case Failure(ex) => jobLogger.warn("{} Failed to abort {}: {}", tag, jobId, ex.getMessage) + } + } + postAbort() + case KvPutSuccess(_) => // expected after the KvPut for the operation ID + } + + override def retryable = false + + override def executeOrRecover(mode: ExecutionMode)(implicit ec: ExecutionContext): Future[ExecutionHandle] = { + Future.fromTry(Try { + val result = mode match { + case Recover(jobId: StandardAsyncJob@unchecked) => + recover(jobId) + case _ => + tellMetadata(startMetadataKeyValues) + execute() + } + result match { + case handle: PendingExecutionHandle[ + StandardAsyncJob@unchecked, StandardAsyncRunInfo@unchecked, StandardAsyncRunStatus@unchecked] => + tellKvJobId(handle.pendingJob) + jobLogger.info(s"job id: ${handle.pendingJob.jobId}") + tellMetadata(Map(CallMetadataKeys.JobId -> handle.pendingJob.jobId)) + context.become(standardReceiveBehavior(Option(handle.pendingJob)) orElse receive) + case _ => /* ignore */ + } + result + } recoverWith { + case exception: Exception => + jobLogger.error(s"Error attempting to $mode", exception) + Failure(exception) + }) + } + + override def poll(previous: ExecutionHandle)(implicit ec: ExecutionContext): Future[ExecutionHandle] = { + previous match { + case handle: PendingExecutionHandle[ + StandardAsyncJob@unchecked, StandardAsyncRunInfo@unchecked, StandardAsyncRunStatus@unchecked] => + + jobLogger.debug(s"$tag Polling Job ${handle.pendingJob}") + pollStatusAsync(handle) map { + backendRunStatus => + handlePollSuccess(handle, backendRunStatus) + } recover { + case throwable => + handlePollFailure(handle, throwable) + } + case successful: SuccessfulExecutionHandle => Future.successful(successful) + case failed: FailedNonRetryableExecutionHandle => Future.successful(failed) + case badHandle => Future.failed(new IllegalArgumentException(s"Unexpected execution handle: $badHandle")) + } + } + + /** + * Process a poll success. + * + * @param oldHandle The previous execution status. + * @param status The updated status. + * @return The updated execution handle. + */ + def handlePollSuccess(oldHandle: StandardAsyncPendingExecutionHandle, + status: StandardAsyncRunStatus): ExecutionHandle = { + val previousStatus = oldHandle.previousStatus + if (!(previousStatus contains status)) { + /* + If this is the first time checking the status, we log the transition as '-' to 'currentStatus'. Otherwise just use + the state names. + */ + val prevStateName = previousStatus.map(_.toString).getOrElse("-") + jobLogger.info(s"$tag Status change from $prevStateName to $status") + statusString(status) foreach { statusMetadata => + tellMetadata(Map(CallMetadataKeys.BackendStatus -> statusMetadata)) + } + } + + status match { + case _ if isTerminal(status) => + val metadata = getTerminalMetadata(status) + tellMetadata(metadata) + handleExecutionResult(status, oldHandle) + case s => oldHandle.copy(previousStatus = Option(s)) // Copy the current handle with updated previous status. + } + } + + /** + * Process a poll failure. + * + * @param oldHandle The previous execution handle. + * @param throwable The cause of the polling failure. + * @return The updated execution handle. + */ + def handlePollFailure(oldHandle: StandardAsyncPendingExecutionHandle, + throwable: Throwable): ExecutionHandle = { + throwable match { + case exception: Exception => + val handler: PartialFunction[(ExecutionHandle, Exception), ExecutionHandle] = + customPollStatusFailure orElse { + case (handle: ExecutionHandle, exception: Exception) => + // Log exceptions and return the original handle to try again. + jobLogger.warn(s"Caught exception, retrying", exception) + handle + } + handler((oldHandle, exception)) + case error: Error => throw error // JVM-ending calamity. + case _: Throwable => + // Someone has subclassed or instantiated Throwable directly. Kill the job. They should be using an Exception. + FailedNonRetryableExecutionHandle(throwable) + } + } + + /** + * Process an execution result. + * + * @param status The execution status. + * @param oldHandle The previous execution handle. + * @return The updated execution handle. + */ + def handleExecutionResult(status: StandardAsyncRunStatus, + oldHandle: StandardAsyncPendingExecutionHandle): ExecutionHandle = { + try { + if (isSuccess(status)) { + + lazy val stderrLength: Long = File(remoteStdErrPath).size + lazy val returnCode: Try[Int] = Try(returnCodeContents).map(_.trim.toInt) + status match { + case _ if failOnStdErr && stderrLength.intValue > 0 => + // returnCode will be None if it couldn't be downloaded/parsed, which will yield a null in the DB + FailedNonRetryableExecutionHandle(new RuntimeException( + s"execution failed: stderr has length $stderrLength"), returnCode.toOption) + case _ if returnCode.isFailure => + val exception = returnCode.failed.get + jobLogger.warn(s"could not download return code file, retrying", exception) + // Return handle to try again. + oldHandle + case _ if returnCode.isFailure => + FailedNonRetryableExecutionHandle(new RuntimeException( + s"execution failed: could not parse return code as integer", returnCode.failed.get)) + case _ if isAbort(returnCode.get) => + AbortedExecutionHandle + case _ if !continueOnReturnCode.continueFor(returnCode.get) => + val message = s"Call ${jobDescriptor.key.tag}: return code was ${returnCode.get}" + FailedNonRetryableExecutionHandle(new RuntimeException(message), returnCode.toOption) + case _ => + handleExecutionSuccess(status, oldHandle, returnCode.get) + } + + } else { + handleExecutionFailure(status, oldHandle) + } + } catch { + case e: Exception => + jobLogger.warn("Caught exception processing job result, retrying", e) + // Return the original handle to try again. + oldHandle + } + } + + /** + * Send the job id of the running job to the key value store. + * + * @param runningJob The running job. + */ + def tellKvJobId(runningJob: StandardAsyncJob): Unit = { + val kvJobKey = + KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt) + val scopedKey = ScopedKey(jobDescriptor.workflowDescriptor.id, kvJobKey, jobIdKey) + val kvValue = Option(runningJob.jobId) + val kvPair = KvPair(scopedKey, kvValue) + val kvPut = KvPut(kvPair) + serviceRegistryActor ! kvPut + } + + /** + * Sends metadata to the metadata store. + * + * @param metadataKeyValues Key/Values to store. + */ + def tellMetadata(metadataKeyValues: Map[String, Any]): Unit = { + import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter + serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), metadataKeyValues) + } + + override protected implicit def ec: ExecutionContextExecutor = context.dispatcher +} + +/** + * Implements the marker trait for a job id using a String. + * + * @param jobId The job id. + */ +case class StandardAsyncJob(jobId: String) extends JobId diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala b/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala new file mode 100644 index 000000000..9242aa601 --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala @@ -0,0 +1,70 @@ +package cromwell.backend.standard + +import akka.actor.ActorRef +import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse +import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor} + +import scala.concurrent.Promise +import scala.language.existentials + +/** + * Base trait for params passed to both the sync and async backend actors. + */ +trait StandardJobExecutionActorParams { + /** The service registry actor for key/value and metadata. */ + val serviceRegistryActor: ActorRef + + /** The descriptor of this job. */ + val jobDescriptor: BackendJobDescriptor + + /** The global and backend configuration. */ + val configurationDescriptor: BackendConfigurationDescriptor + + /** Any backend initialization data. */ + val backendInitializationDataOption: Option[BackendInitializationData] + + /** The key for this job. */ + val jobIdKey: String +} + +/** + * Extended trait for params passed to synchronous backend actors. + */ +trait StandardSyncExecutionActorParams extends StandardJobExecutionActorParams { + /** + * The class for creating an async backend. + * + * @see [[StandardSyncExecutionActor]] + */ + val asyncJobExecutionActorClass: Class[_ <: StandardAsyncExecutionActor] +} + +/** A default implementation of the sync params. */ +case class DefaultStandardSyncExecutionActorParams +( + override val jobIdKey: String, + override val serviceRegistryActor: ActorRef, + override val jobDescriptor: BackendJobDescriptor, + override val configurationDescriptor: BackendConfigurationDescriptor, + override val backendInitializationDataOption: Option[BackendInitializationData], + override val asyncJobExecutionActorClass: Class[_ <: StandardAsyncExecutionActor] +) extends StandardSyncExecutionActorParams + +/** + * Extended trait for params passed to asynchronous backend actors. + */ +trait StandardAsyncExecutionActorParams extends StandardJobExecutionActorParams { + /** The promise that will be completed when the async run is complete. */ + val completionPromise: Promise[BackendJobExecutionResponse] +} + +/** A default implementation of the async params. */ +case class DefaultStandardAsyncExecutionActorParams +( + override val jobIdKey: String, + override val serviceRegistryActor: ActorRef, + override val jobDescriptor: BackendJobDescriptor, + override val configurationDescriptor: BackendConfigurationDescriptor, + override val backendInitializationDataOption: Option[BackendInitializationData], + override val completionPromise: Promise[BackendJobExecutionResponse] +) extends StandardAsyncExecutionActorParams diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala new file mode 100644 index 000000000..6a0f9324b --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala @@ -0,0 +1,42 @@ +package cromwell.backend.standard + +import akka.actor.{ActorRef, Props} +import cromwell.backend._ +import cromwell.core.Dispatcher + +/** + * May be extended for using the standard sync/async backend pattern. + */ +trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory { + /** + * Config values for the backend, and a pointer to the global config. + * + * This is the single parameter passed into each factory during creation. + * + * @return The backend configuration. + */ + def configurationDescriptor: BackendConfigurationDescriptor + + /** + * Returns the main engine for async execution. + * + * @return the main engine for async execution. + */ + def asyncJobExecutionActorClass: Class[_ <: StandardAsyncExecutionActor] + + /** + * Returns the key to use for storing and looking up the job id. + * + * @return the key to use for storing and looking up the job id. + */ + def jobIdKey: String + + override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, + initializationDataOption: Option[BackendInitializationData], + serviceRegistryActor: ActorRef, + backendSingletonActor: Option[ActorRef]): Props = { + val params = DefaultStandardSyncExecutionActorParams(jobIdKey, serviceRegistryActor, + jobDescriptor, configurationDescriptor, initializationDataOption, asyncJobExecutionActorClass) + Props(new StandardSyncExecutionActor(params)).withDispatcher(Dispatcher.BackendDispatcher) + } +} diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala new file mode 100644 index 000000000..52bba0153 --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala @@ -0,0 +1,132 @@ +package cromwell.backend.standard + +import akka.actor.SupervisorStrategy.{Decider, Stop} +import akka.actor.{ActorRef, OneForOneStrategy, Props} +import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse} +import cromwell.backend.BackendLifecycleActor.AbortJobCommand +import cromwell.backend.async.AsyncBackendJobExecutionActor.{Execute, Recover} +import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobExecutionActor} +import cromwell.core.Dispatcher +import cromwell.services.keyvalue.KeyValueServiceActor._ + +import scala.concurrent.{Future, Promise} + +/** + * Facade to the asynchronous execution actor. + * + * Creates the asynchronous execution actor, then relays messages to that actor. + * + * NOTE: Although some methods return futures due to the (current) contract in BJEA/ABJEA, this actor only executes + * during the receive, and does not launch new runnables/futures from inside "receive". + * + * Thus there are no vars, and the context switches during "receive", once the asynchronous actor has been created. + * + * The backend synchronous actor is a proxy for an asynchronous actor that does the actual heavy lifting. + * + * - Synchronous params contain the class of the asynchronous actor. + * - Synchronous actor creates `Promise` that it will wait for. + * - Synchronous actor passes the `Promise` into a `StandardAsyncExecutionActorParams`. + * - Synchronous actor creates a `Props` using the asynchronous class plus asynchronous params. + * - Synchronous actor creates an asynchronous actor using the `Props`. + * - Synchronous actor waits for the `Promise` to complete. + * - Asynchronous actor runs. + * - Asynchronous actor completes the promise with a success or failure. + */ +class StandardSyncExecutionActor(val standardParams: StandardSyncExecutionActorParams) + extends BackendJobExecutionActor { + + override val jobDescriptor: BackendJobDescriptor = standardParams.jobDescriptor + override val configurationDescriptor: BackendConfigurationDescriptor = standardParams.configurationDescriptor + val jobIdKey: String = standardParams.jobIdKey + val serviceRegistryActor: ActorRef = standardParams.serviceRegistryActor + + context.become(startup orElse super.receive) + + private def startup: Receive = { + case AbortJobCommand => + context.parent ! AbortedResponse(jobDescriptor.key) + context.stop(self) + } + + private def running(executor: ActorRef): Receive = { + case AbortJobCommand => + executor ! AbortJobCommand + case abortResponse: AbortedResponse => + context.parent ! abortResponse + context.stop(self) + case KvPair(key, Some(jobId)) if key.key == jobIdKey => + // Successful operation ID lookup during recover. + executor ! Recover(StandardAsyncJob(jobId)) + case KvKeyLookupFailed(_) => + // Missed operation ID lookup during recover, fall back to execute. + executor ! Execute + case KvFailure(_, e) => + // Failed operation ID lookup during recover, crash and let the supervisor deal with it. + completionPromise.tryFailure(e) + throw new RuntimeException(s"Failure attempting to look up job id for key ${jobDescriptor.key}", e) + } + + /** + * This "synchronous" actor isn't finished until this promise finishes over in the asynchronous version. + * + * Still not sure why the AsyncBackendJobExecutionActor doesn't wait for an Akka message instead of using Scala promises. + */ + lazy val completionPromise: Promise[BackendJobExecutionResponse] = Promise[BackendJobExecutionResponse]() + + override def execute: Future[BackendJobExecutionResponse] = { + val executorRef = createAsyncRef() + context.become(running(executorRef) orElse super.receive) + executorRef ! Execute + completionPromise.future + } + + override def recover: Future[BackendJobExecutionResponse] = { + val executorRef = createAsyncRef() + context.become(running(executorRef) orElse super.receive) + val kvJobKey = + KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt) + val kvGet = KvGet(ScopedKey(jobDescriptor.workflowDescriptor.id, kvJobKey, jobIdKey)) + serviceRegistryActor ! kvGet + completionPromise.future + } + + def createAsyncParams(): StandardAsyncExecutionActorParams = { + DefaultStandardAsyncExecutionActorParams( + standardParams.jobIdKey, + standardParams.serviceRegistryActor, + standardParams.jobDescriptor, + standardParams.configurationDescriptor, + standardParams.backendInitializationDataOption, + completionPromise + ) + } + + def createAsyncProps(): Props = { + val asyncParams = createAsyncParams() + Props(standardParams.asyncJobExecutionActorClass, asyncParams) + } + + def createAsyncRefName(): String = { + standardParams.asyncJobExecutionActorClass.getSimpleName + } + + def createAsyncRef(): ActorRef = { + val props = createAsyncProps().withDispatcher(Dispatcher.BackendDispatcher) + val name = createAsyncRefName() + context.actorOf(props, name) + } + + override def abort(): Unit = { + throw new NotImplementedError("Abort is implemented via a custom receive of the message AbortJobCommand.") + } + + // Supervision strategy: if the async actor throws an exception, stop the actor and fail the job. + def jobFailingDecider: Decider = { + case exception: Exception => + completionPromise.tryFailure( + new RuntimeException(s"${createAsyncRefName()} failed and didn't catch its exception.", exception)) + Stop + } + + override val supervisorStrategy: OneForOneStrategy = OneForOneStrategy()(jobFailingDecider) +} diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala index 81f9f1c89..db0cacaf0 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala @@ -5,8 +5,7 @@ import cats.instances.list._ import cats.syntax.traverse._ import cats.syntax.validated._ import cromwell.backend.validation.RuntimeAttributesValidation._ -import cromwell.core._ -import cromwell.core.ErrorOr._ +import lenthall.validation.ErrorOr._ import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType} import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString} diff --git a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala index 17ba9fb66..1e1c807cf 100644 --- a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala @@ -2,7 +2,7 @@ package cromwell.backend.validation import cats.syntax.validated._ import cromwell.backend.MemorySize -import cromwell.core.ErrorOr._ +import lenthall.validation.ErrorOr._ import wdl4s.parser.MemoryUnit import wdl4s.types.{WdlIntegerType, WdlStringType} import wdl4s.values.{WdlInteger, WdlString} diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala index 7a080170e..30ffe1043 100644 --- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala +++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala @@ -3,8 +3,8 @@ package cromwell.backend.validation import cats.data.ValidatedNel import cats.syntax.validated._ import cromwell.core.{EvaluatedRuntimeAttributes, OptionNotFoundException, WorkflowOptions} +import lenthall.util.TryUtil import wdl4s.types.WdlType -import wdl4s.util.TryUtil import wdl4s.values.WdlValue import scala.util.{Failure, Try} @@ -20,7 +20,7 @@ object RuntimeAttributesDefault { Failure(new RuntimeException(s"Could not parse JsonValue $v to valid WdlValue for runtime attribute $k")) } k -> maybeTriedValue - }) + }, "Failed to coerce default runtime options") } recover { case _: OptionNotFoundException => Map.empty[String, WdlValue] } diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala index 1ba92527e..1adbc254f 100644 --- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala @@ -3,7 +3,7 @@ package cromwell.backend.validation import cats.syntax.validated._ import wdl4s.expression.PureStandardLibraryFunctions import cromwell.backend.{MemorySize, RuntimeAttributeDefinition} -import cromwell.core.ErrorOr._ +import lenthall.validation.ErrorOr._ import org.slf4j.Logger import wdl4s.WdlExpression import wdl4s.WdlExpression._ diff --git a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala index 59e8cec26..ad3c2cb57 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala @@ -4,7 +4,7 @@ import cats.data.Validated._ import cats.instances.list._ import cromwell.backend.RuntimeAttributeDefinition import lenthall.exception.MessageAggregation -import cromwell.core.ErrorOr._ +import lenthall.validation.ErrorOr._ import org.slf4j.Logger import wdl4s.values.WdlValue diff --git a/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala b/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala index 0f3e4679e..89f3ea00d 100644 --- a/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala +++ b/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala @@ -16,15 +16,15 @@ trait ReadLikeFunctions extends PathFactory { this: WdlStandardLibraryFunctions * Asserts that the parameter list contains a single parameter which will be interpreted * as a File and attempts to read the contents of that file */ - private def readContentsFromSingleFileParameter(params: Seq[Try[WdlValue]]): Try[String] = { + private def readContentsFromSingleFileParameter(functionName: String, params: Seq[Try[WdlValue]]): Try[String] = { for { - singleArgument <- extractSingleArgument(params) + singleArgument <- extractSingleArgument(functionName, params) string = fileContentsToString(singleArgument.valueString) } yield string } - private def extractObjects(params: Seq[Try[WdlValue]]): Try[Array[WdlObject]] = for { - contents <- readContentsFromSingleFileParameter(params) + private def extractObjects(functionName: String, params: Seq[Try[WdlValue]]): Try[Array[WdlObject]] = for { + contents <- readContentsFromSingleFileParameter(functionName, params) wdlObjects <- WdlObject.fromTsv(contents) } yield wdlObjects @@ -35,38 +35,38 @@ trait ReadLikeFunctions extends PathFactory { this: WdlStandardLibraryFunctions */ override def read_lines(params: Seq[Try[WdlValue]]): Try[WdlArray] = { for { - contents <- readContentsFromSingleFileParameter(params) + contents <- readContentsFromSingleFileParameter("read_lines", params) lines = contents.split("\n") } yield WdlArray(WdlArrayType(WdlStringType), lines map WdlString) } override def read_map(params: Seq[Try[WdlValue]]): Try[WdlMap] = { for { - contents <- readContentsFromSingleFileParameter(params) + contents <- readContentsFromSingleFileParameter("read_map", params) wdlMap <- WdlMap.fromTsv(contents) } yield wdlMap } override def read_object(params: Seq[Try[WdlValue]]): Try[WdlObject] = { - extractObjects(params) map { + extractObjects("read_object", params) map { case array if array.length == 1 => array.head case _ => throw new IllegalArgumentException("read_object yields an Object and thus can only read 2-rows TSV files. Try using read_objects instead.") } } - override def read_objects(params: Seq[Try[WdlValue]]): Try[WdlArray] = extractObjects(params) map { WdlArray(WdlArrayType(WdlObjectType), _) } + override def read_objects(params: Seq[Try[WdlValue]]): Try[WdlArray] = extractObjects("read_objects", params) map { WdlArray(WdlArrayType(WdlObjectType), _) } /** * Try to read a string from the file referenced by the specified `WdlValue`. */ - override def read_string(params: Seq[Try[WdlValue]]): Try[WdlString] = readContentsFromSingleFileParameter(params).map(s => WdlString(s.trim)) + override def read_string(params: Seq[Try[WdlValue]]): Try[WdlString] = readContentsFromSingleFileParameter("read_string", params).map(s => WdlString(s.trim)) /** * Read a file in TSV format into an Array[Array[String]] */ override def read_tsv(params: Seq[Try[WdlValue]]): Try[WdlArray] = { for { - contents <- readContentsFromSingleFileParameter(params) + contents <- readContentsFromSingleFileParameter("read_tsv", params) wdlArray = WdlArray.fromTsv(contents) } yield wdlArray } @@ -106,7 +106,7 @@ trait ReadLikeFunctions extends PathFactory { this: WdlStandardLibraryFunctions override def glob(params: Seq[Try[WdlValue]]): Try[WdlArray] = { for { - singleArgument <- extractSingleArgument(params) + singleArgument <- extractSingleArgument("glob", params) globVal = singleArgument.valueString files = glob(globPath(globVal), globVal) wdlFiles = files map { WdlFile(_, isGlob = false) } diff --git a/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala b/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala index 0f602c76e..4726c6947 100644 --- a/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala +++ b/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala @@ -30,19 +30,19 @@ trait WriteFunctions { this: WdlStandardLibraryFunctions => } } - private def writeToTsv(params: Seq[Try[WdlValue]], wdlClass: Class[_ <: WdlValue with TsvSerializable]) = { + private def writeToTsv(functionName: String, params: Seq[Try[WdlValue]], wdlClass: Class[_ <: WdlValue with TsvSerializable]) = { for { - singleArgument <- extractSingleArgument(params) + singleArgument <- extractSingleArgument(functionName, params) downcast <- Try(wdlClass.cast(singleArgument)) tsvSerialized <- downcast.tsvSerialize file <- writeContent(wdlClass.getSimpleName.toLowerCase, tsvSerialized) } yield file } - override def write_lines(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv(params, classOf[WdlArray]) - override def write_map(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv(params, classOf[WdlMap]) - override def write_object(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv(params, classOf[WdlObject]) - override def write_objects(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv(params, classOf[WdlArray]) - override def write_tsv(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv(params, classOf[WdlArray]) + override def write_lines(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv("write_lines", params, classOf[WdlArray]) + override def write_map(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv("write_map", params, classOf[WdlMap]) + override def write_object(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv("write_object", params, classOf[WdlObject]) + override def write_objects(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv("write_objects", params, classOf[WdlArray]) + override def write_tsv(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv("write_tsv", params, classOf[WdlArray]) override def write_json(params: Seq[Try[WdlValue]]): Try[WdlFile] = Failure(new NotImplementedError(s"write_json() not implemented yet")) } diff --git a/backend/src/test/scala/cromwell/backend/BackendSpec.scala b/backend/src/test/scala/cromwell/backend/BackendSpec.scala index 7e2354bd5..30bcc4a79 100644 --- a/backend/src/test/scala/cromwell/backend/BackendSpec.scala +++ b/backend/src/test/scala/cromwell/backend/BackendSpec.scala @@ -4,7 +4,7 @@ import com.typesafe.config.ConfigFactory import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobFailedNonRetryableResponse, JobFailedRetryableResponse, JobSucceededResponse} import cromwell.backend.io.TestWorkflows._ import cromwell.core.{WorkflowId, WorkflowOptions} -import wdl4s.util.AggregatedException +import lenthall.exception.AggregatedException import org.scalatest.Matchers import org.scalatest.concurrent.ScalaFutures import org.scalatest.time.{Millis, Seconds, Span} @@ -62,7 +62,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito { val workflowDescriptor = buildWorkflowDescriptor(wdl) val call = workflowDescriptor.workflow.taskCalls.head val jobKey = BackendJobDescriptorKey(call, None, 1) - val inputDeclarations = fqnMapToDeclarationMap(workflowDescriptor.inputs) + val inputDeclarations = fqnMapToDeclarationMap(workflowDescriptor.knownValues) val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, inputDeclarations).get // .get is OK here because this is a test val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes) BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations) @@ -76,7 +76,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito { val workflowDescriptor = buildWorkflowDescriptor(wdl, runtime = runtime) val call = workflowDescriptor.workflow.taskCalls.head val jobKey = BackendJobDescriptorKey(call, None, attempt) - val inputDeclarations = fqnMapToDeclarationMap(workflowDescriptor.inputs) + val inputDeclarations = fqnMapToDeclarationMap(workflowDescriptor.knownValues) val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, inputDeclarations).get // .get is OK here because this is a test val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes) BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations) @@ -104,7 +104,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito { private def concatenateCauseMessages(t: Throwable): String = t match { case null => "" - case ae: AggregatedException => ae.getMessage + ae.exceptions.map(concatenateCauseMessages(_)).mkString + concatenateCauseMessages(ae.getCause) + case ae: AggregatedException => ae.getMessage + " " + ae.throwables.map(innerT => concatenateCauseMessages(innerT.getCause)).mkString("\n") case other: Throwable => other.getMessage + concatenateCauseMessages(t.getCause) } diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala index 325f874b8..7420c2d89 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala @@ -80,7 +80,7 @@ class RuntimeAttributesDefaultSpec extends FlatSpec with Matchers { val defaults = workflowOptionsDefault(workflowOptions, coercionMap) defaults.isFailure shouldBe true - defaults.failed.get.getMessage shouldBe s": RuntimeException: Could not parse JsonValue ${map("str")} to valid WdlValue for runtime attribute str" + defaults.failed.get.getMessage shouldBe s"Failed to coerce default runtime options:\nCould not parse JsonValue ${map("str")} to valid WdlValue for runtime attribute str" } it should "fold default values" in { diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 52df9e771..0e57508b7 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -20,6 +20,8 @@ akka { #parallelism-max = 64 } + actor.guardian-supervisor-strategy = "cromwell.core.CromwellUserGuardianStrategy" + dispatchers { # A dispatcher for actors performing blocking io operations # Prevents the whole system from being slowed down when waiting for responses from external resources for instance @@ -50,6 +52,12 @@ akka { executor = "fork-join-executor" } + # A dispatcher used for the service registry + service-dispatcher { + type = Dispatcher + executor = "fork-join-executor" + } + # Note that without further configuration, all other actors run on the default dispatcher } } diff --git a/core/src/main/scala/cromwell/core/CromwellUserGuardianStrategy.scala b/core/src/main/scala/cromwell/core/CromwellUserGuardianStrategy.scala new file mode 100644 index 000000000..52af92e5c --- /dev/null +++ b/core/src/main/scala/cromwell/core/CromwellUserGuardianStrategy.scala @@ -0,0 +1,11 @@ +package cromwell.core + +import akka.actor.SupervisorStrategy.Escalate +import akka.actor.{ActorInitializationException, OneForOneStrategy, SupervisorStrategy, SupervisorStrategyConfigurator} + +class CromwellUserGuardianStrategy extends SupervisorStrategyConfigurator { + override def create(): SupervisorStrategy = OneForOneStrategy() { + case aie: ActorInitializationException => Escalate + case t => akka.actor.SupervisorStrategy.defaultDecider.applyOrElse(t, (_: Any) => Escalate) + } +} diff --git a/core/src/main/scala/cromwell/core/Dispatcher.scala b/core/src/main/scala/cromwell/core/Dispatcher.scala index 9c221e831..d617d2210 100644 --- a/core/src/main/scala/cromwell/core/Dispatcher.scala +++ b/core/src/main/scala/cromwell/core/Dispatcher.scala @@ -5,4 +5,5 @@ object Dispatcher { val IoDispatcher = "akka.dispatchers.io-dispatcher" val ApiDispatcher = "akka.dispatchers.api-dispatcher" val BackendDispatcher = "akka.dispatchers.backend-dispatcher" + val ServiceDispatcher = "akka.dispatchers.service-dispatcher" } diff --git a/core/src/main/scala/cromwell/core/ErrorOr.scala b/core/src/main/scala/cromwell/core/ErrorOr.scala deleted file mode 100644 index cd344f8ac..000000000 --- a/core/src/main/scala/cromwell/core/ErrorOr.scala +++ /dev/null @@ -1,22 +0,0 @@ -package cromwell.core - -import cats.data.Validated.{Invalid, Valid} -import cats.data.{NonEmptyList, Validated} - -object ErrorOr { - type ErrorOr[A] = Validated[NonEmptyList[String], A] - - implicit class ShortCircuitingFlatMap[A](val fa: ErrorOr[A]) extends AnyVal { - /** - * Not consistent with `Applicative#ap` but useful in for comprehensions. - * - * @see http://typelevel.org/cats/tut/validated.html#of-flatmaps-and-xors - */ - def flatMap[B](f: A => ErrorOr[B]): ErrorOr[B] = { - fa match { - case Valid(v) => f(v) - case i @ Invalid(_) => i - } - } - } -} diff --git a/core/src/main/scala/cromwell/core/ExecutionStatus.scala b/core/src/main/scala/cromwell/core/ExecutionStatus.scala index 353b44d65..2ef66629c 100644 --- a/core/src/main/scala/cromwell/core/ExecutionStatus.scala +++ b/core/src/main/scala/cromwell/core/ExecutionStatus.scala @@ -2,13 +2,15 @@ package cromwell.core object ExecutionStatus extends Enumeration { type ExecutionStatus = Value - val NotStarted, QueuedInCromwell, Starting, Running, Failed, Preempted, Done, Aborted = Value - val TerminalStatuses = Set(Failed, Done, Aborted, Preempted) + val NotStarted, QueuedInCromwell, Starting, Running, Failed, Preempted, Done, Bypassed, Aborted = Value + val TerminalStatuses = Set(Failed, Done, Aborted, Preempted, Bypassed) implicit class EnhancedExecutionStatus(val status: ExecutionStatus) extends AnyVal { def isTerminal: Boolean = { TerminalStatuses contains status } + + def isDoneOrBypassed: Boolean = status == Done || status == Bypassed } implicit class EnhancedString(val string: String) extends AnyVal { diff --git a/core/src/main/scala/cromwell/core/WorkflowOptions.scala b/core/src/main/scala/cromwell/core/WorkflowOptions.scala index 11f6e4115..709e667e1 100644 --- a/core/src/main/scala/cromwell/core/WorkflowOptions.scala +++ b/core/src/main/scala/cromwell/core/WorkflowOptions.scala @@ -1,8 +1,8 @@ package cromwell.core import com.typesafe.config.ConfigFactory +import lenthall.util.TryUtil import spray.json._ -import wdl4s.util.TryUtil import scala.collection.JavaConverters._ import scala.util.{Failure, Success, Try} diff --git a/core/src/main/scala/cromwell/core/core.scala b/core/src/main/scala/cromwell/core/core.scala index ec1a3babe..8f47182c0 100644 --- a/core/src/main/scala/cromwell/core/core.scala +++ b/core/src/main/scala/cromwell/core/core.scala @@ -8,5 +8,17 @@ import wdl4s.values.WdlValue case class CallContext(root: Path, stdout: String, stderr: String) case class JobOutput(wdlValue: WdlValue) -class CromwellFatalException(exception: Throwable) extends Exception(exception) +/** Marker trait for Cromwell exceptions that are to be treated as fatal (non-retryable) */ +trait CromwellFatalExceptionMarker { this: Throwable => } + +object CromwellFatalException { + // Don't wrap if it's already a fatal exception + def apply(throwable: Throwable) = throwable match { + case e: CromwellFatalExceptionMarker => e + case e => new CromwellFatalException(e) + } + def unapply(e: CromwellFatalException): Option[Throwable] = Option(e.exception) +} + +class CromwellFatalException(val exception: Throwable) extends Exception(exception) with CromwellFatalExceptionMarker case class CromwellAggregatedException(throwables: Seq[Throwable], exceptionContext: String = "") extends ThrowableAggregation diff --git a/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala b/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala index ba339f011..905362c2d 100644 --- a/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala +++ b/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala @@ -14,6 +14,8 @@ import net.ceedubs.ficus.Ficus._ import org.slf4j.helpers.NOPLogger import org.slf4j.{Logger, LoggerFactory} +import scala.util.Try + trait WorkflowLogging extends ActorLogging { this: Actor => def workflowIdForLogging: WorkflowId @@ -109,7 +111,7 @@ class WorkflowLogger(loggerName: String, override def getName = loggerName - def deleteLogFile() = workflowLogPath foreach { File(_).delete(swallowIOExceptions = false) } + def deleteLogFile() = Try { workflowLogPath foreach { File(_).delete(swallowIOExceptions = false) } } import WorkflowLogger._ diff --git a/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala b/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala index 7dc60c1e8..1db86bb35 100644 --- a/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala +++ b/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala @@ -3,6 +3,8 @@ package cromwell.core.path import java.net.URI import java.nio.file.{FileSystems, Path} +import com.google.common.net.UrlEscapers + import scala.util.Try /** @@ -10,12 +12,13 @@ import scala.util.Try */ case object DefaultPathBuilder extends PathBuilder { override def name = "Default" - override def build(pathAsString: String): Try[Path] = Try { - val uri = URI.create(pathAsString) + val uri = URI.create(UrlEscapers.urlFragmentEscaper().escape(pathAsString)) val host = Option(uri.getHost) getOrElse "" val path = Option(uri.getPath) getOrElse "" - - FileSystems.getDefault.getPath(host, path) + Option(uri.getScheme) match { + case Some("file") | None => FileSystems.getDefault.getPath(host, path) + case _ => throw new RuntimeException(s"Cannot build a local path from $pathAsString") + } } } diff --git a/core/src/main/scala/cromwell/core/path/PathFactory.scala b/core/src/main/scala/cromwell/core/path/PathFactory.scala index ff050b559..92af7a3b0 100644 --- a/core/src/main/scala/cromwell/core/path/PathFactory.scala +++ b/core/src/main/scala/cromwell/core/path/PathFactory.scala @@ -46,7 +46,7 @@ object PathFactory { postMapping: Path => Path = identity[Path]): Path = { pathBuilders.toStream map { _.build(preMapping(string)) } collectFirst { case Success(p) => postMapping(p) } getOrElse { val pathBuilderNames: String = pathBuilders map { _.name } mkString ", " - throw new PathParsingException(s"Could not find suitable filesystem among $pathBuilderNames to parse $string.") + throw PathParsingException(s"Could not find suitable filesystem among $pathBuilderNames to parse $string.") } } @@ -54,4 +54,6 @@ object PathFactory { pathBuilders: List[PathBuilder], preMapping: String => String = identity[String], postMapping: Path => Path = identity[Path]): File = File(buildPath(string, pathBuilders, preMapping, postMapping)) + + def pathPlusSuffix(path: File, suffix: String): File = path.sibling(s"${path.name}.$suffix") } diff --git a/core/src/main/scala/cromwell/core/path/PathImplicits.scala b/core/src/main/scala/cromwell/core/path/PathImplicits.scala index 83d4bff7c..fa5533ef3 100644 --- a/core/src/main/scala/cromwell/core/path/PathImplicits.scala +++ b/core/src/main/scala/cromwell/core/path/PathImplicits.scala @@ -11,5 +11,7 @@ object PathImplicits { def untailed = UntailedWriter(path) def tailed(tailedSize: Int) = TailedWriter(path, tailedSize) + + def toRealString: String = java.net.URLDecoder.decode(path.toUri.toString, "UTF-8") } } diff --git a/core/src/main/scala/cromwell/core/path/proxy/RetryableFileSystemProviderProxy.scala b/core/src/main/scala/cromwell/core/path/proxy/RetryableFileSystemProviderProxy.scala index db3975292..4d2a481c3 100644 --- a/core/src/main/scala/cromwell/core/path/proxy/RetryableFileSystemProviderProxy.scala +++ b/core/src/main/scala/cromwell/core/path/proxy/RetryableFileSystemProviderProxy.scala @@ -37,7 +37,9 @@ class RetryableFileSystemProviderProxy[T <: FileSystemProvider](delegate: T, ret new FileSystemProxy(delegate.newFileSystem(uri, env), this) } override def getScheme: String = delegate.getScheme - override def getFileSystem(uri: URI): FileSystem = delegate.getFileSystem(uri) + override def getFileSystem(uri: URI): FileSystem = { + new FileSystemProxy(delegate.getFileSystem(uri), this) + } override def getFileStore(path: Path): FileStore = delegate.getFileStore(path) /* retried operations */ diff --git a/core/src/main/scala/cromwell/core/retry/Retry.scala b/core/src/main/scala/cromwell/core/retry/Retry.scala index 002a8d6e5..a2788cbb8 100644 --- a/core/src/main/scala/cromwell/core/retry/Retry.scala +++ b/core/src/main/scala/cromwell/core/retry/Retry.scala @@ -35,7 +35,7 @@ object Retry { if (maxRetries.forall(_ > 0)) { f() recoverWith { - case throwable if isFatal(throwable) => Future.failed(new CromwellFatalException(throwable)) + case throwable if isFatal(throwable) => Future.failed(CromwellFatalException(throwable)) case throwable if !isFatal(throwable) => val retriesLeft = if (isTransient(throwable)) maxRetries else maxRetries map { _ - 1 } after(delay, actorSystem.scheduler)(withRetry(f, backoff = backoff, maxRetries = retriesLeft, isTransient = isTransient, isFatal = isFatal)) diff --git a/core/src/main/scala/cromwell/util/JsonFormatting/WdlValueJsonFormatter.scala b/core/src/main/scala/cromwell/util/JsonFormatting/WdlValueJsonFormatter.scala index 8a997efb0..dc7f55fe5 100644 --- a/core/src/main/scala/cromwell/util/JsonFormatting/WdlValueJsonFormatter.scala +++ b/core/src/main/scala/cromwell/util/JsonFormatting/WdlValueJsonFormatter.scala @@ -7,7 +7,7 @@ import wdl4s.values._ object WdlValueJsonFormatter extends DefaultJsonProtocol { implicit object WdlValueJsonFormat extends RootJsonFormat[WdlValue] { - def write(value: WdlValue) = value match { + def write(value: WdlValue): JsValue = value match { case s: WdlString => JsString(s.value) case i: WdlInteger => JsNumber(i.value) case f: WdlFloat => JsNumber(f.value) @@ -18,6 +18,8 @@ object WdlValueJsonFormatter extends DefaultJsonProtocol { case m: WdlMap => new JsObject(m.value map {case(k,v) => k.valueString -> write(v)}) case e: WdlExpression => JsString(e.toWdlString) case q: WdlPair => new JsObject(Map("left" -> write(q.left), "right" -> write(q.right))) + case WdlOptionalValue(_, Some(innerValue)) => write(innerValue) + case WdlOptionalValue(_, None) => JsNull } // NOTE: This assumes a map's keys are strings. Since we're coming from JSON this is fine. diff --git a/core/src/main/scala/cromwell/util/TerminalUtil.scala b/core/src/main/scala/cromwell/util/TerminalUtil.scala deleted file mode 100644 index 56269c9ca..000000000 --- a/core/src/main/scala/cromwell/util/TerminalUtil.scala +++ /dev/null @@ -1,17 +0,0 @@ -package cromwell.util - -object TerminalUtil { - def highlight(colorCode:Int, string:String) = s"\033[38;5;${colorCode}m$string\033[0m" - def mdTable(rows: Seq[Seq[String]], header: Seq[String]): String = { - def maxWidth(lengths: Seq[Seq[Int]], column: Int) = lengths.map { length => length(column) }.max - val widths = (rows :+ header).map { row => row.map { s => s.length } } - val maxWidths = widths.head.indices.map { column => maxWidth(widths, column) } - val tableHeader = header.indices.map { i => header(i).padTo(maxWidths(i), " ").mkString("") }.mkString("|") - val tableDivider = header.indices.map { i => "-" * maxWidths(i) }.mkString("|") - val tableRows = rows.map { row => - val mdRow = row.indices.map { i => row(i).padTo(maxWidths(i), " ").mkString("") }.mkString("|") - s"|$mdRow|" - } - s"|$tableHeader|\n|$tableDivider|\n${tableRows.mkString("\n")}\n" - } -} diff --git a/core/src/main/scala/cromwell/util/TryUtil.scala b/core/src/main/scala/cromwell/util/TryUtil.scala deleted file mode 100644 index 18f7ea58a..000000000 --- a/core/src/main/scala/cromwell/util/TryUtil.scala +++ /dev/null @@ -1,45 +0,0 @@ -package cromwell.util - -import java.io.{PrintWriter, StringWriter} - -import lenthall.exception.ThrowableAggregation - -import scala.util.{Success, Failure, Try} - -case class AggregatedException(exceptions: Seq[Throwable], prefixError: String = "") extends ThrowableAggregation { - override def throwables: Traversable[Throwable] = exceptions - override def exceptionContext: String = prefixError -} - -object TryUtil { - private def stringifyFailure(failure: Try[Any]): String = { - val stringWriter = new StringWriter() - val writer = new PrintWriter(stringWriter) - failure recover { case e => e.printStackTrace(writer) } - writer.flush() - writer.close() - stringWriter.toString - } - - def stringifyFailures[T](possibleFailures: Traversable[Try[T]]): Traversable[String] = - possibleFailures.collect { case failure: Failure[T] => stringifyFailure(failure) } - - private def sequenceIterable[T](tries: Iterable[Try[_]], unbox: () => T, prefixErrorMessage: String) = { - tries collect { case f: Failure[_] => f } match { - case failures if failures.nonEmpty => - val exceptions = failures.toSeq.map(_.exception) - Failure(AggregatedException(exceptions, prefixErrorMessage)) - case _ => Success(unbox()) - } - } - - def sequence[T](tries: Seq[Try[T]], prefixErrorMessage: String = ""): Try[Seq[T]] = { - def unbox = tries map { _.get } - sequenceIterable(tries, unbox _, prefixErrorMessage) - } - - def sequenceMap[T, U](tries: Map[T, Try[U]], prefixErrorMessage: String = ""): Try[Map[T, U]] = { - def unbox = tries mapValues { _.get } - sequenceIterable(tries.values, unbox _, prefixErrorMessage) - } -} diff --git a/core/src/test/scala/cromwell/core/TestKitSuite.scala b/core/src/test/scala/cromwell/core/TestKitSuite.scala index 094095a60..751949a7f 100644 --- a/core/src/test/scala/cromwell/core/TestKitSuite.scala +++ b/core/src/test/scala/cromwell/core/TestKitSuite.scala @@ -34,6 +34,7 @@ object TestKitSuite { | debug { | receive = on | } + | guardian-supervisor-strategy = "akka.actor.DefaultSupervisorStrategy" | } | dispatchers { | # A dispatcher for actors performing blocking io operations diff --git a/core/src/test/scala/cromwell/util/TryUtilSpec.scala b/core/src/test/scala/cromwell/util/TryUtilSpec.scala deleted file mode 100644 index ee71d489d..000000000 --- a/core/src/test/scala/cromwell/util/TryUtilSpec.scala +++ /dev/null @@ -1,36 +0,0 @@ -package cromwell.util - -import cromwell.core.retry.SimpleExponentialBackoff -import org.scalatest.mockito.MockitoSugar -import org.scalatest.{FlatSpec, Matchers} - -import scala.concurrent.duration._ -import scala.language.postfixOps - -class TryUtilSpec extends FlatSpec with Matchers with MockitoSugar { - - class TransientException extends Exception - class MockWork { - var counter: Int = _ - - /** - * @param n number of times an exception is thrown before succeeding - * @param transients how many transient exceptions to raise (must be <= n) - */ - def failNTimes(n: Int, transients: Int = 0): Option[Int] => Int = { - counter = n - def func(prior: Option[Int]): Int = { - if (counter > 0) { - counter -= 1 - if (counter <= transients) throw new TransientException - else throw new IllegalArgumentException("Failed") - } - 9 - } - func - } - } - - //val logger = mock[WorkflowLogger] - val backoff = SimpleExponentialBackoff(50 milliseconds, 10 seconds, 1D) -} diff --git a/engine/src/main/resources/swagger/cromwell.yaml b/engine/src/main/resources/swagger/cromwell.yaml index e2e825f8f..613ce5e7e 100644 --- a/engine/src/main/resources/swagger/cromwell.yaml +++ b/engine/src/main/resources/swagger/cromwell.yaml @@ -586,6 +586,26 @@ paths: security: - google_oauth: - openid + '/engine/{version}/version': + get: + summary: Returns the version of the Cromwell Engine + parameters: + - name: version + description: API Version + required: true + type: string + in: path + default: v1 + tags: + - Engine + responses: + '200': + description: Successful Request + schema: + $ref: '#/definitions/VersionResponse' + security: + - google_oauth: + - openid securityDefinitions: google_oauth: type: oauth2 @@ -595,7 +615,6 @@ securityDefinitions: openid: open id authorization definitions: WorkflowSubmitResponse: - description: '' required: - id - status @@ -607,7 +626,7 @@ definitions: type: string description: The status of the workflow WorkflowAbortResponse: - description: '' + required: - id - status @@ -619,7 +638,6 @@ definitions: type: string description: The status of the workflow WorkflowStatusResponse: - description: '' required: - id - status @@ -631,7 +649,7 @@ definitions: type: string description: The status of the workflow WorkflowMetadataResponse: - description: 'Workflow and call level metadata' + description: Workflow and call level metadata required: - id - status @@ -666,7 +684,7 @@ definitions: failures: $ref: '#/definitions/FailureMessage' CallMetadata: - description: 'Call level metadata' + description: Call level metadata required: - inputs - executionStatus @@ -709,7 +727,7 @@ definitions: type: object description: Paths to backend specific logs for this call FailureMessage: - description: 'Failure messages' + description: Failure messages required: - failure - timestamp @@ -797,7 +815,6 @@ definitions: format: date-time description: Workflow end datetime BackendResponse: - description: '' required: - supportedBackends - defaultBackend @@ -811,7 +828,7 @@ definitions: type: string description: The default backend of this server StatsResponse: - description: 'Provides engine level statistics for things running inside the system' + description: Provides engine level statistics for things running inside the system required: - workflows - jobs @@ -822,3 +839,9 @@ definitions: jobs: type: integer description: The number of currently running jobs + VersionResponse: + description: Returns the version of Cromwell + properties: + cromwell: + type: string + description: The version of the Cromwell Engine diff --git a/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala b/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala index d8aa2a44d..071b4d70d 100644 --- a/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala +++ b/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala @@ -8,7 +8,6 @@ import wdl4s._ final case class EngineWorkflowDescriptor(namespace: WdlNamespaceWithWorkflow, backendDescriptor: BackendWorkflowDescriptor, - workflowInputs: WorkflowCoercedInputs, backendAssignments: Map[TaskCall, String], failureMode: WorkflowFailureMode, pathBuilders: List[PathBuilder], @@ -20,9 +19,10 @@ final case class EngineWorkflowDescriptor(namespace: WdlNamespaceWithWorkflow, case None => this } - val id = backendDescriptor.id + lazy val id = backendDescriptor.id lazy val workflow = backendDescriptor.workflow lazy val name = workflow.unqualifiedName - val inputs = backendDescriptor.inputs + lazy val knownValues = backendDescriptor.knownValues + def getWorkflowOption(key: WorkflowOption) = backendDescriptor.getWorkflowOption(key) } diff --git a/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala b/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala index 49248312e..1dff0efa3 100644 --- a/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala +++ b/engine/src/main/scala/cromwell/engine/backend/BackendConfiguration.scala @@ -7,7 +7,7 @@ import scala.collection.JavaConverters._ import scala.util.{Failure, Success, Try} case class BackendConfigurationEntry(name: String, lifecycleActorFactoryClass: String, config: Config) { - def asBackendLifecycleActorFactory: BackendLifecycleActorFactory = { + def asBackendLifecycleActorFactory: Try[BackendLifecycleActorFactory] = Try { Class.forName(lifecycleActorFactoryClass) .getConstructor(classOf[String], classOf[BackendConfigurationDescriptor]) .newInstance(name, asBackendConfigurationDescriptor) diff --git a/engine/src/main/scala/cromwell/engine/backend/CromwellBackends.scala b/engine/src/main/scala/cromwell/engine/backend/CromwellBackends.scala index 9b90e6277..5a586edad 100644 --- a/engine/src/main/scala/cromwell/engine/backend/CromwellBackends.scala +++ b/engine/src/main/scala/cromwell/engine/backend/CromwellBackends.scala @@ -1,6 +1,7 @@ package cromwell.engine.backend import cromwell.backend.BackendLifecycleActorFactory +import lenthall.util.TryUtil import scala.util.{Failure, Success, Try} @@ -9,7 +10,8 @@ import scala.util.{Failure, Success, Try} */ case class CromwellBackends(backendEntries: List[BackendConfigurationEntry]) { - val backendLifecycleActorFactories = backendEntries.map(e => e.name -> e.asBackendLifecycleActorFactory).toMap + // Raise the exception here if some backend factories failed to instantiate + val backendLifecycleActorFactories = TryUtil.sequenceMap(backendEntries.map(e => e.name -> e.asBackendLifecycleActorFactory).toMap).get def backendLifecycleActorFactoryByName(backendName: String): Try[BackendLifecycleActorFactory] = { backendLifecycleActorFactories.get(backendName) match { diff --git a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala index 8a72bc414..98b51c50a 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala @@ -10,6 +10,7 @@ import cats.instances.try_._ import cats.syntax.functor._ import cromwell.core.retry.SimpleExponentialBackoff import cromwell.core._ +import cromwell.core.Dispatcher.EngineDispatcher import cromwell.engine.workflow.SingleWorkflowRunnerActor._ import cromwell.engine.workflow.WorkflowManagerActor.RetrieveNewWorkflows import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreActor} @@ -195,7 +196,7 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, metadataO object SingleWorkflowRunnerActor { def props(source: WorkflowSourceFilesCollection, metadataOutputFile: Option[Path]): Props = { - Props(new SingleWorkflowRunnerActor(source, metadataOutputFile)) + Props(new SingleWorkflowRunnerActor(source, metadataOutputFile)).withDispatcher(EngineDispatcher) } sealed trait RunnerMessage diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala index 6d2a2ff40..0fb49191d 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala @@ -8,7 +8,7 @@ import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.WorkflowOptions.FinalWorkflowLogDir import cromwell.core._ import cromwell.core.logging.{WorkflowLogger, WorkflowLogging} -import cromwell.core.path.PathFactory +import cromwell.core.path.{PathBuilder, PathFactory} import cromwell.engine._ import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.WorkflowActor._ @@ -23,6 +23,8 @@ import cromwell.subworkflowstore.SubWorkflowStoreActor.WorkflowComplete import cromwell.webservice.EngineStatsActor import wdl4s.{LocallyQualifiedName => _} +import scala.util.Failure + object WorkflowActor { /** @@ -179,7 +181,6 @@ class WorkflowActor(val workflowId: WorkflowId, val actor = context.actorOf(MaterializeWorkflowDescriptorActor.props(serviceRegistryActor, workflowId, importLocalFilesystem = !serverMode), "MaterializeWorkflowDescriptorActor") pushWorkflowStart(workflowId) - actor ! MaterializeWorkflowDescriptorCommand(workflowSources, conf) goto(MaterializingWorkflowDescriptorState) using stateData.copy(currentLifecycleStateActor = Option(actor)) case Event(AbortWorkflowCommand, stateData) => goto(WorkflowAbortedState) @@ -300,16 +301,29 @@ class WorkflowActor(val workflowId: WorkflowId, // Copy/Delete workflow logs if (WorkflowLogger.isEnabled) { - stateData.workflowDescriptor foreach { wd => - wd.getWorkflowOption(FinalWorkflowLogDir) match { + /* + * The submitted workflow options have been previously validated by the CromwellApiHandler. These are + * being recreated so that in case MaterializeWorkflowDescriptor fails, the workflow logs can still + * be copied by accessing the workflow options outside of the EngineWorkflowDescriptor. + */ + def bruteForceWorkflowOptions: WorkflowOptions = WorkflowOptions.fromJsonString(workflowSources.workflowOptionsJson).getOrElse(WorkflowOptions.fromJsonString("{}").get) + def bruteForcePathBuilders: List[PathBuilder] = EngineFilesystems(context.system).pathBuildersForWorkflow(bruteForceWorkflowOptions) + + val (workflowOptions, pathBuilders) = stateData.workflowDescriptor match { + case Some(wd) => (wd.backendDescriptor.workflowOptions, wd.pathBuilders) + case None => (bruteForceWorkflowOptions, bruteForcePathBuilders) + } + + workflowOptions.get(FinalWorkflowLogDir).toOption match { case Some(destinationDir) => - workflowLogCopyRouter ! CopyWorkflowLogsActor.Copy(wd.id, PathFactory.buildPath(destinationDir, wd.pathBuilders)) - case None if WorkflowLogger.isTemporary => workflowLogger.deleteLogFile() + workflowLogCopyRouter ! CopyWorkflowLogsActor.Copy(workflowId, PathFactory.buildPath(destinationDir, pathBuilders)) + case None if WorkflowLogger.isTemporary => workflowLogger.deleteLogFile() match { + case Failure(f) => log.error(f, "Failed to delete workflow log") + case _ => + } case _ => } } - } - context stop self } @@ -334,6 +348,7 @@ class WorkflowActor(val workflowId: WorkflowId, failures: Option[List[Throwable]]) = { val finalizationActor = makeFinalizationActor(workflowDescriptor, jobExecutionMap, workflowOutputs) finalizationActor ! StartFinalizationCommand - goto(FinalizingWorkflowState) using data.copy(lastStateReached = StateCheckpoint(stateName, failures)) + goto(FinalizingWorkflowState) using data.copy(lastStateReached = StateCheckpoint (stateName, failures)) } + } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala index 74a08a8fe..d8b2a6a00 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala @@ -4,6 +4,7 @@ import java.nio.file.Files import akka.actor.{ActorRef, FSM, LoggingFSM, Props} import better.files.File +import cats.data.NonEmptyList import cats.data.Validated._ import cats.instances.list._ import cats.syntax.cartesian._ @@ -24,7 +25,8 @@ import cromwell.engine.backend.CromwellBackends import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.{MaterializeWorkflowDescriptorActorData, MaterializeWorkflowDescriptorActorState} import cromwell.services.metadata.MetadataService.{PutMetadataAction, _} import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} -import cromwell.core.ErrorOr._ +import lenthall.exception.MessageAggregation +import lenthall.validation.ErrorOr._ import net.ceedubs.ficus.Ficus._ import spray.json.{JsObject, _} import wdl4s._ @@ -133,9 +135,9 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, goto(MaterializationSuccessfulState) case Invalid(error) => sender() ! MaterializeWorkflowDescriptorFailureResponse( - new IllegalArgumentException with ExceptionWithErrors { - val message = s"Workflow input processing failed." - val errors = error + new IllegalArgumentException with MessageAggregation { + val exceptionContext = s"Workflow input processing failed" + val errorMessages = error.toList }) goto(MaterializationFailedState) } @@ -234,7 +236,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, evaluatedWorkflowsDeclarations <- validateDeclarations(namespace, workflowOptions, coercedInputs, pathBuilders) declarationsAndInputs <- checkTypes(evaluatedWorkflowsDeclarations ++ coercedInputs) backendDescriptor = BackendWorkflowDescriptor(id, namespace.workflow, declarationsAndInputs, workflowOptions) - } yield EngineWorkflowDescriptor(namespace, backendDescriptor, coercedInputs, backendAssignments, failureMode, pathBuilders, callCachingMode) + } yield EngineWorkflowDescriptor(namespace, backendDescriptor, backendAssignments, failureMode, pathBuilders, callCachingMode) } private def pushWfInputsToMetadataService(workflowInputs: WorkflowCoercedInputs): Unit = { @@ -393,7 +395,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef, namespace: WdlNamespaceWithWorkflow): ErrorOr[WorkflowCoercedInputs] = { namespace.coerceRawInputs(rawInputs) match { case Success(r) => r.validNel - case Failure(e: ExceptionWithErrors) => Invalid(e.errors) + case Failure(e: MessageAggregation) if e.errorMessages.nonEmpty => Invalid(NonEmptyList.fromListUnsafe(e.errorMessages.toList)) case Failure(e) => e.getMessage.invalidNel } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala index 80748f6dd..67db2d926 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala @@ -103,9 +103,9 @@ trait CallMetadataHelper { val now = OffsetDateTime.now.withOffsetSameInstant(offset) val lastEvent = ExecutionEvent("!!Bring Back the Monarchy!!", now) val tailedEventList = eventList :+ lastEvent - val events = tailedEventList.sliding(2).zipWithIndex flatMap { - case (Seq(eventCurrent, eventNext), index) => - val eventKey = s"executionEvents[$index]" + val events = tailedEventList.sliding(2) flatMap { + case Seq(eventCurrent, eventNext) => + val eventKey = s"executionEvents[$randomNumberString]" List( metadataEvent(s"$eventKey:description", eventCurrent.name), metadataEvent(s"$eventKey:startTime", eventCurrent.offsetDateTime), diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala index afa48474d..e68ea38a2 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala @@ -215,6 +215,7 @@ class EngineJobExecutionActor(replyTo: ActorRef, eventList ++= response.executionEvents saveCacheResults(hashes, data.withSuccessResponse(response)) case Event(response: JobSucceededResponse, data @ ResponsePendingData(_, _, None, _)) if effectiveCallCachingMode.writeToCache => + eventList ++= response.executionEvents log.debug(s"Got job result for {}, awaiting hashes", jobTag) stay using data.withSuccessResponse(response) case Event(response: BackendJobExecutionResponse, data: ResponsePendingData) => @@ -407,7 +408,9 @@ class EngineJobExecutionActor(replyTo: ActorRef, private def saveJobCompletionToJobStore(updatedData: ResponseData) = { updatedData.response match { - case JobSucceededResponse(jobKey: BackendJobDescriptorKey, returnCode: Option[Int], jobOutputs: CallOutputs, _, _) => saveSuccessfulJobResults(jobKey, returnCode, jobOutputs) + case JobSucceededResponse(jobKey: BackendJobDescriptorKey, returnCode: Option[Int], jobOutputs: CallOutputs, _, executionEvents) => + eventList ++= executionEvents + saveSuccessfulJobResults(jobKey, returnCode, jobOutputs) case AbortedResponse(jobKey: BackendJobDescriptorKey) => log.debug("{}: Won't save aborted job response to JobStore", jobTag) forwardAndStop(updatedData.response) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala index 3b9d1af1f..e35f0e004 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala @@ -17,6 +17,7 @@ object ExecutionStore { case call: TaskCall => Option(BackendJobDescriptorKey(call, None, 1)) case call: WorkflowCall => Option(SubWorkflowKey(call, None, 1)) case scatter: Scatter => Option(ScatterKey(scatter)) + case conditional: If => Option(ConditionalKey(conditional, None)) case declaration: Declaration => Option(DeclarationKey(declaration, None, workflowCoercedInputs)) case _ => None // Ifs will need to be added here when supported } @@ -26,6 +27,9 @@ object ExecutionStore { } case class ExecutionStore(store: Map[JobKey, ExecutionStatus]) { + + override def toString = store.map { case (j, s) => s"$j -> $s" } mkString(System.lineSeparator()) + def add(values: Map[JobKey, ExecutionStatus]) = this.copy(store = store ++ values) // Convert the store to a `List` before `collect`ing to sidestep expensive and pointless hashing of `Scope`s when @@ -47,6 +51,28 @@ case class ExecutionStore(store: Map[JobKey, ExecutionStatus]) { case _ => false } + // Just used to decide whether a collector can be run. In case the shard entries haven't been populated into the + // execution store yet. + case class TempJobKey(scope: Scope with GraphNode, index: Option[Int]) extends JobKey { + // If these are ever used, we've done something wrong... + override def attempt: Int = ??? + override def tag: String = ??? + } + def emulateShardEntries(key: CollectorKey): List[JobKey] = { + (0 until key.scatterWidth).toList flatMap { i => key.scope match { + case c: Call => List(TempJobKey(c, Option(i))) + case d: Declaration => List(TempJobKey(d, Option(i))) + case _ => throw new RuntimeException("Don't collect that.") + }} + } + + +// store.toList filter { +// case (k: CallKey, v) => k.scope == key.scope && k.isShard +// case (k: DeclarationKey, v) => k.scope == key.scope && k.isShard +// case _ => false +// } + private def arePrerequisitesDone(key: JobKey): Boolean = { val upstream = key.scope.upstream collect { case n: Call => upstreamEntry(key, n) @@ -54,8 +80,8 @@ case class ExecutionStore(store: Map[JobKey, ExecutionStatus]) { case n: Declaration => upstreamEntry(key, n) } - val downstream: List[(JobKey, ExecutionStatus)] = key match { - case collector: CollectorKey => findShardEntries(collector) + val downstream: List[JobKey] = key match { + case collector: CollectorKey => emulateShardEntries(collector) case _ => Nil } @@ -65,11 +91,11 @@ case class ExecutionStore(store: Map[JobKey, ExecutionStatus]) { * of the scatter block. */ def isDone(e: JobKey): Boolean = store exists { - case (k, s) => k.scope.fullyQualifiedName == e.scope.fullyQualifiedName && k.index == e.index && s == ExecutionStatus.Done + case (k, s) => k.scope.fullyQualifiedName == e.scope.fullyQualifiedName && k.index == e.index && s.isDoneOrBypassed } - val dependencies = upstream.flatten ++ downstream - val dependenciesResolved = dependencies forall { case (k, _) => isDone(k) } + val dependencies = upstream.flatten.map(_._1) ++ downstream + val dependenciesResolved = dependencies forall { isDone } /* * We need to make sure that all prerequisiteScopes have been resolved to some entry before going forward. diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala index 467439622..a636a8d10 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala @@ -2,12 +2,14 @@ package cromwell.engine.workflow.lifecycle.execution import akka.actor.{Actor, ActorRef, Props} import cromwell.backend._ +import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.logging.WorkflowLogging import cromwell.core.{CallKey, JobKey, WorkflowId} import cromwell.engine.EngineWorkflowDescriptor import cromwell.engine.workflow.lifecycle.execution.CallPreparationActor._ import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.SubWorkflowKey import wdl4s._ +import wdl4s.exception.VariableLookupException import wdl4s.expression.WdlStandardLibraryFunctions import wdl4s.values.WdlValue @@ -34,15 +36,17 @@ abstract class CallPreparationActor(val workflowDescriptor: EngineWorkflowDescri val call = callKey.scope val scatterMap = callKey.index flatMap { i => // Will need update for nested scatters - call.upstream collectFirst { case s: Scatter => Map(s -> i) } + call.ancestry collectFirst { case s: Scatter => Map(s -> i) } } getOrElse Map.empty[Scatter, Int] call.evaluateTaskInputs( - workflowDescriptor.backendDescriptor.inputs, + workflowDescriptor.backendDescriptor.knownValues, expressionLanguageFunctions, outputStore.fetchNodeOutputEntries, scatterMap ) + } recoverWith { + case t: Throwable => Failure(new VariableLookupException(s"Couldn't resolve all inputs for ${callKey.scope.fullyQualifiedName} at index ${callKey.index}.", List(t))) } } } @@ -86,7 +90,7 @@ final case class SubWorkflowPreparationActor(executionData: WorkflowExecutionAct val newBackendDescriptor = oldBackendDescriptor.copy( id = subWorkflowId, workflow = key.scope.calledWorkflow, - inputs = workflowDescriptor.inputs ++ (inputEvaluation map { case (k, v) => k.fullyQualifiedName -> v }), + knownValues = workflowDescriptor.knownValues ++ (inputEvaluation map { case (k, v) => k.fullyQualifiedName -> v }), breadCrumbs = oldBackendDescriptor.breadCrumbs :+ BackendJobBreadCrumb(workflowDescriptor.workflow, workflowDescriptor.id, key) ) val engineDescriptor = workflowDescriptor.copy(backendDescriptor = newBackendDescriptor, parentWorkflow = Option(workflowDescriptor)) @@ -115,7 +119,12 @@ object JobPreparationActor { backendSingletonActor: Option[ActorRef]) = { // Note that JobPreparationActor doesn't run on the engine dispatcher as it mostly executes backend-side code // (WDL expression evaluation using Backend's expressionLanguageFunctions) - Props(new JobPreparationActor(executionData, jobKey, factory, initializationData, serviceRegistryActor, backendSingletonActor)) + Props(new JobPreparationActor(executionData, + jobKey, + factory, + initializationData, + serviceRegistryActor, + backendSingletonActor)).withDispatcher(EngineDispatcher) } } @@ -125,6 +134,6 @@ object SubWorkflowPreparationActor { subWorkflowId: WorkflowId) = { // Note that JobPreparationActor doesn't run on the engine dispatcher as it mostly executes backend-side code // (WDL expression evaluation using Backend's expressionLanguageFunctions) - Props(new SubWorkflowPreparationActor(executionData, key, subWorkflowId)) + Props(new SubWorkflowPreparationActor(executionData, key, subWorkflowId)).withDispatcher(EngineDispatcher) } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala index 02c0bc113..4a40a538f 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala @@ -4,10 +4,10 @@ import cromwell.core.ExecutionIndex._ import cromwell.core._ import cromwell.engine.workflow.lifecycle.execution.OutputStore.{OutputCallKey, OutputEntry} import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.CollectorKey +import lenthall.util.TryUtil import wdl4s.types.{WdlArrayType, WdlType} -import wdl4s.util.TryUtil import wdl4s.values.{WdlArray, WdlCallOutputsObject, WdlValue} -import wdl4s.{Call, Declaration, GraphNode, Scope} +import wdl4s._ import scala.language.postfixOps import scala.util.{Failure, Success, Try} @@ -19,6 +19,9 @@ object OutputStore { } case class OutputStore(store: Map[OutputCallKey, List[OutputEntry]]) { + + override def toString = store.map { case (k, l) => s"$k -> ${l.mkString(" ")}" } mkString System.lineSeparator + def add(values: Map[OutputCallKey, List[OutputEntry]]) = this.copy(store = store ++ values) def fetchNodeOutputEntries(node: GraphNode, index: ExecutionIndex): Try[WdlValue] = { @@ -55,7 +58,7 @@ case class OutputStore(store: Map[OutputCallKey, List[OutputEntry]]) { } } - def collectCall(call: Call, sortedShards: Seq[JobKey]) = Try { + def collectCall(call: Call, scatter: Scatter, sortedShards: Seq[JobKey]) = Try { val shardsOutputs = sortedShards map { e => fetchNodeOutputEntries(call, e.index) map { case callOutputs: WdlCallOutputsObject => callOutputs.outputs @@ -66,19 +69,20 @@ case class OutputStore(store: Map[OutputCallKey, List[OutputEntry]]) { call.outputs map { taskOutput => val wdlValues = shardsOutputs.map( _.getOrElse(taskOutput.unqualifiedName, throw new RuntimeException(s"Could not retrieve output ${taskOutput.unqualifiedName}"))) - val arrayOfValues = new WdlArray(WdlArrayType(taskOutput.wdlType), wdlValues) + val arrayType = taskOutput.relativeWdlType(scatter).asInstanceOf[WdlArrayType] + val arrayOfValues = new WdlArray(arrayType, wdlValues) taskOutput.unqualifiedName -> JobOutput(arrayOfValues) } toMap } - def collectDeclaration(declaration: Declaration, sortedShards: Seq[JobKey]) = Try { + def collectDeclaration(declaration: Declaration, scatter: Scatter, sortedShards: Seq[JobKey]) = Try { val shardsOutputs = sortedShards map { e => fetchNodeOutputEntries(declaration, e.index) getOrElse { throw new RuntimeException(s"Could not retrieve output for shard ${e.scope} #${e.index}") } } - - Map(declaration.unqualifiedName -> JobOutput(WdlArray(WdlArrayType(declaration.wdlType), shardsOutputs))) + val arrayType = declaration.relativeWdlType(scatter).asInstanceOf[WdlArrayType] + Map(declaration.unqualifiedName -> JobOutput(WdlArray(arrayType, shardsOutputs))) } /** @@ -90,8 +94,8 @@ case class OutputStore(store: Map[OutputCallKey, List[OutputEntry]]) { lazy val sortedShards = shards.toSeq sortBy { _.index.fromIndex } collector.scope match { - case call: Call => collectCall(call, sortedShards) - case declaration: Declaration => collectDeclaration(declaration, sortedShards) + case call: Call => collectCall(call, collector.scatter, sortedShards) + case declaration: Declaration => collectDeclaration(declaration, collector.scatter, sortedShards) case other => Failure(new RuntimeException(s"Cannot retrieve outputs for ${other.fullyQualifiedName}")) } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala index ffac77610..e3813dd61 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala @@ -4,6 +4,7 @@ import akka.actor.SupervisorStrategy.Escalate import akka.actor.{ActorRef, FSM, LoggingFSM, OneForOneStrategy, Props, SupervisorStrategy} import cromwell.backend.{AllBackendInitializationData, BackendLifecycleActorFactory, BackendWorkflowDescriptor} import cromwell.core._ +import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.logging.JobLogging import cromwell.engine.EngineWorkflowDescriptor import cromwell.engine.backend.{BackendConfiguration, BackendSingletonCollection} @@ -270,6 +271,6 @@ object SubWorkflowExecutionActor { backendSingletonCollection, initializationData, restarting) - ) + ).withDispatcher(EngineDispatcher) } } \ No newline at end of file diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index 10f0bf377..f4a4e4143 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -17,11 +17,13 @@ import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{appl import cromwell.engine.workflow.lifecycle.{EngineLifecycleActorAbortCommand, EngineLifecycleActorAbortedResponse} import cromwell.engine.{ContinueWhilePossible, EngineWorkflowDescriptor} import cromwell.services.metadata.MetadataService.{MetadataPutAcknowledgement, MetadataPutFailed} -import cromwell.util.{StopAndLogSupervisor, TryUtil} +import cromwell.util.StopAndLogSupervisor import cromwell.webservice.EngineStatsActor import lenthall.exception.ThrowableAggregation +import lenthall.util.TryUtil import net.ceedubs.ficus.Ficus._ -import wdl4s.values.{WdlArray, WdlValue} +import wdl4s.values.{WdlArray, WdlBoolean, WdlOptionalValue, WdlValue, WdlString} +import org.apache.commons.lang3.StringUtils import wdl4s.{Scope, _} import scala.annotation.tailrec @@ -62,7 +64,7 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, WorkflowExecutionPendingState, WorkflowExecutionActorData( workflowDescriptor, - executionStore = ExecutionStore(workflowDescriptor.backendDescriptor.workflow, workflowDescriptor.inputs), + executionStore = ExecutionStore(workflowDescriptor.backendDescriptor.workflow, workflowDescriptor.knownValues), backendJobExecutionActors = Map.empty, engineCallExecutionActors = Map.empty, subWorkflowExecutionActors = Map.empty, @@ -103,6 +105,11 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, // Declaration case Event(DeclarationEvaluationSucceededResponse(jobKey, callOutputs), stateData) => handleDeclarationEvaluationSuccessful(jobKey, callOutputs, stateData) + // Conditional + case Event(BypassedCallResults(callOutputs), stateData) => + handleCallBypassed(callOutputs, stateData) + case Event(BypassedDeclaration(declKey), stateData) => + handleDeclarationEvaluationSuccessful(declKey, WdlOptionalValue.none(declKey.scope.wdlType), stateData) // Failure // Initialization @@ -253,17 +260,18 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, } private def handleWorkflowSuccessful(data: WorkflowExecutionActorData) = { + import WorkflowExecutionActor.EnhancedWorkflowOutputs import cromwell.util.JsonFormatting.WdlValueJsonFormatter._ import spray.json._ - + val (response, finalState) = workflowDescriptor.workflow.evaluateOutputs( - workflowDescriptor.inputs, + workflowDescriptor.knownValues, data.expressionLanguageFunctions, data.outputStore.fetchNodeOutputEntries ) map { workflowOutputs => workflowLogger.info( s"""Workflow ${workflowDescriptor.workflow.unqualifiedName} complete. Final Outputs: - |${workflowOutputs.toJson.prettyPrint}""".stripMargin + |${workflowOutputs.stripLarge.toJson.prettyPrint}""".stripMargin ) pushWorkflowOutputMetadata(workflowOutputs) (WorkflowExecutionSucceededResponse(data.jobExecutionMap, workflowOutputs mapValues JobOutput.apply), WorkflowExecutionSuccessfulState) @@ -299,7 +307,14 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, private def handleDeclarationEvaluationSuccessful(key: DeclarationKey, value: WdlValue, data: WorkflowExecutionActorData) = { handleExecutionSuccess(data.declarationEvaluationSuccess(key, value)) } - + + private def handleCallBypassed(callOutputs: Map[CallKey, CallOutputs], data: WorkflowExecutionActorData) = { + def foldFunc(d: WorkflowExecutionActorData, output: (CallKey, CallOutputs)) = d.callExecutionSuccess(output._1, output._2) + + val updatedData = callOutputs.foldLeft(data)(foldFunc) + handleExecutionSuccess(updatedData) + } + private def handleExecutionSuccess(data: WorkflowExecutionActorData) = { data.workflowCompletionStatus match { case Some(ExecutionStatus.Done) => @@ -336,9 +351,12 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, // Each process returns a Try[WorkflowExecutionDiff], which, upon success, contains potential changes to be made to the execution store. val executionDiffs = runnableScopes map { + case k: CallKey if isInBypassedScope(k, data) => processBypassedScope(k, data) + case k: DeclarationKey if isInBypassedScope(k, data) => processBypassedScope(k, data) case k: BackendJobDescriptorKey => processRunnableJob(k, data) - case k: ScatterKey => processRunnableScatter(k, data) - case k: CollectorKey => processRunnableCollector(k, data) + case k: ScatterKey => processRunnableScatter(k, data, isInBypassedScope(k, data)) + case k: ConditionalKey => processRunnableConditional(k, data) + case k: CollectorKey => processRunnableCollector(k, data, isInBypassedScope(k, data)) case k: SubWorkflowKey => processRunnableSubWorkflow(k, data) case k: StaticDeclarationKey => processRunnableStaticDeclaration(k) case k: DynamicDeclarationKey => processRunnableDynamicDeclaration(k, data) @@ -353,14 +371,47 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, // Update the metadata for the jobs we just sent to EJEAs (they'll start off queued up waiting for tokens): pushQueuedCallMetadata(diffs) if (diffs.exists(_.containsNewEntry)) { - startRunnableScopes(data.mergeExecutionDiffs(diffs)) + val newData = data.mergeExecutionDiffs(diffs) + startRunnableScopes(newData) } else { - data.mergeExecutionDiffs(diffs) + val result = data.mergeExecutionDiffs(diffs) + result } case Failure(e) => throw new RuntimeException("Unexpected engine failure", e) } } + private def isInBypassedScope(jobKey: JobKey, data: WorkflowExecutionActorData) = { + // This is hugely ripe for optimization, if it becomes a bottleneck + def isBypassedConditional(conditional: If, executionStore: ExecutionStore): Boolean = { + executionStore.store.exists { case (jobKeyUnderExamination, executionStatus) => + if (jobKeyUnderExamination.isInstanceOf[ConditionalKey]) { + if (jobKeyUnderExamination.scope.fullyQualifiedName.equals(conditional.fullyQualifiedName) && jobKeyUnderExamination.index.equals(jobKey.index)) { + executionStatus.equals(ExecutionStatus.Bypassed) + } else false + } else false + } + } + + val result = jobKey.scope.ancestry.exists { + case i: If => isBypassedConditional(i, data.executionStore) + case _ => false + } + result + } + + def processBypassedScope(jobKey: JobKey, data: WorkflowExecutionActorData): Try[WorkflowExecutionDiff] = { + self ! bypassedScopeResults(jobKey) + Success(WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Running))) + } + + def bypassedScopeResults(jobKey: JobKey): BypassedScopeResults = jobKey match { + case callKey: CallKey => BypassedCallResults( + Map(callKey -> (callKey.scope.outputs map { callOutput => callOutput.unqualifiedName -> JobOutput(WdlOptionalValue.none(callOutput.wdlType)) } toMap))) + case declKey: DeclarationKey => BypassedDeclaration(declKey) + case _ => throw new RuntimeException("Only calls and declarations might generate results when Bypassed") + } + def processRunnableStaticDeclaration(declaration: StaticDeclarationKey) = { self ! DeclarationEvaluationSucceededResponse(declaration, declaration.value) Success(WorkflowExecutionDiff(Map(declaration -> ExecutionStatus.Running))) @@ -373,12 +424,12 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, } getOrElse Map.empty[Scatter, Int] val lookup = declaration.scope.lookupFunction( - workflowDescriptor.workflowInputs, + workflowDescriptor.knownValues, data.expressionLanguageFunctions, data.outputStore.fetchNodeOutputEntries, scatterMap ) - + declaration.requiredExpression.evaluate(lookup, data.expressionLanguageFunctions) match { case Success(result) => self ! DeclarationEvaluationSucceededResponse(declaration, result) case Failure(ex) => self ! DeclarationEvaluationFailedResponse(declaration, ex) @@ -430,27 +481,58 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor, engineJobExecutionActorAdditions = Map(sweaRef -> key))) } - private def processRunnableScatter(scatterKey: ScatterKey, data: WorkflowExecutionActorData): Try[WorkflowExecutionDiff] = { + private def processRunnableConditional(conditionalKey: ConditionalKey, data: WorkflowExecutionActorData): Try[WorkflowExecutionDiff] = { + val scatterMap = conditionalKey.index flatMap { i => + // Will need update for nested scatters + conditionalKey.scope.ancestry collectFirst { case s: Scatter => Map(s -> i) } + } getOrElse Map.empty[Scatter, Int] + + val lookup = conditionalKey.scope.lookupFunction( + workflowDescriptor.knownValues, + data.expressionLanguageFunctions, + data.outputStore.fetchNodeOutputEntries, + scatterMap + ) + + conditionalKey.scope.condition.evaluate(lookup, data.expressionLanguageFunctions) map { + case b: WdlBoolean => + val conditionalStatus = if (b.value) ExecutionStatus.Done else ExecutionStatus.Bypassed + val result = WorkflowExecutionDiff(conditionalKey.populate(workflowDescriptor.knownValues) + (conditionalKey -> conditionalStatus)) + result + case v: WdlValue => throw new RuntimeException("'if' condition must evaluate to a boolean") + } + } + + private def processRunnableScatter(scatterKey: ScatterKey, data: WorkflowExecutionActorData, bypassed: Boolean): Try[WorkflowExecutionDiff] = { val lookup = scatterKey.scope.lookupFunction( - workflowDescriptor.workflowInputs, + workflowDescriptor.knownValues, data.expressionLanguageFunctions, data.outputStore.fetchNodeOutputEntries ) - scatterKey.scope.collection.evaluate(lookup, data.expressionLanguageFunctions) map { - case a: WdlArray => WorkflowExecutionDiff(scatterKey.populate(a.value.size, workflowDescriptor.inputs) + (scatterKey -> ExecutionStatus.Done)) - case v: WdlValue => throw new RuntimeException("Scatter collection must evaluate to an array") + if (bypassed) { + Success(WorkflowExecutionDiff(scatterKey.populate(0, Map.empty) + (scatterKey -> ExecutionStatus.Bypassed))) + } else { + scatterKey.scope.collection.evaluate(lookup, data.expressionLanguageFunctions) map { + case a: WdlArray => + WorkflowExecutionDiff(scatterKey.populate(a.value.size, workflowDescriptor.knownValues) + (scatterKey -> ExecutionStatus.Done)) + case v: WdlValue => throw new RuntimeException("Scatter collection must evaluate to an array") + } } } - private def processRunnableCollector(collector: CollectorKey, data: WorkflowExecutionActorData): Try[WorkflowExecutionDiff] = { - val shards = data.executionStore.findShardEntries(collector) collect { - case (k: CallKey, v) if v == ExecutionStatus.Done => k - case (k: DynamicDeclarationKey, v) if v == ExecutionStatus.Done => k + private def processRunnableCollector(collector: CollectorKey, data: WorkflowExecutionActorData, isInBypassed: Boolean): Try[WorkflowExecutionDiff] = { + def shards(collectorKey: CollectorKey) = data.executionStore.findShardEntries(collectorKey) collect { + case (k: CallKey, v) if v.isDoneOrBypassed => k + case (k: DynamicDeclarationKey, v) if v.isDoneOrBypassed => k } - data.outputStore.generateCollectorOutput(collector, shards) match { - case Failure(e) => Failure(new RuntimeException(s"Failed to collect output shards for call ${collector.tag}")) - case Success(outputs) => self ! ScatterCollectionSucceededResponse(collector, outputs) + data.outputStore.generateCollectorOutput(collector, shards(collector)) match { + case Failure(e) => Failure(new RuntimeException(s"Failed to collect output shards for call ${collector.tag}", e)) + case Success(outputs) => + val adjustedOutputs: CallOutputs = if (isInBypassed) { + outputs map { output => (output._1, JobOutput(WdlOptionalValue.none(output._2.wdlValue.wdlType) )) } + } else outputs + self ! ScatterCollectionSucceededResponse(collector, adjustedOutputs) Success(WorkflowExecutionDiff(Map(collector -> ExecutionStatus.Starting))) } } @@ -519,7 +601,12 @@ object WorkflowExecutionActor { private case class ScatterCollectionSucceededResponse(collectorKey: CollectorKey, outputs: CallOutputs) private case class DeclarationEvaluationSucceededResponse(declarationKey: DeclarationKey, value: WdlValue) - + + private[execution] sealed trait BypassedScopeResults + + private case class BypassedCallResults(callOutputs: Map[CallKey, CallOutputs]) extends BypassedScopeResults + private case class BypassedDeclaration(declaration: DeclarationKey) extends BypassedScopeResults + private case class DeclarationEvaluationFailedResponse(declarationKey: DeclarationKey, reason: Throwable) case class SubWorkflowSucceededResponse(key: SubWorkflowKey, jobExecutionMap: JobExecutionMap, outputs: CallOutputs) @@ -531,7 +618,8 @@ object WorkflowExecutionActor { /** * Internal ADTs */ - case class ScatterKey(scope: Scatter) extends JobKey { + case class ScatterKey(scatter: Scatter) extends JobKey { + override val scope = scatter override val index = None // When scatters are nested, this might become Some(_) override val attempt = 1 @@ -553,26 +641,27 @@ object WorkflowExecutionActor { } private def explode(scope: Scope, count: Int, workflowCoercedInputs: WorkflowCoercedInputs): Seq[JobKey] = { - scope match { - case call: TaskCall => - val shards = (0 until count) map { i => BackendJobDescriptorKey(call, Option(i), 1) } - shards :+ CollectorKey(call) - case call: WorkflowCall => - val shards = (0 until count) map { i => SubWorkflowKey(call, Option(i), 1) } - shards :+ CollectorKey(call) - case declaration: Declaration => - val shards = (0 until count) map { i => DeclarationKey(declaration, Option(i), workflowCoercedInputs) } - shards :+ CollectorKey(declaration) + def makeCollectors(scope: Scope): Seq[CollectorKey] = scope match { + case call: Call => List(CollectorKey(call, scatter, count)) + case decl: Declaration => List(CollectorKey(decl, scatter, count)) + case i: If => i.children.flatMap(makeCollectors(_)) + } + + (scope match { + case call: TaskCall => (0 until count) map { i => BackendJobDescriptorKey(call, Option(i), 1) } + case call: WorkflowCall => (0 until count) map { i => SubWorkflowKey(call, Option(i), 1) } + case declaration: Declaration => (0 until count) map { i => DeclarationKey(declaration, Option(i), workflowCoercedInputs) } + case conditional: If => (0 until count) map { i => ConditionalKey(conditional, Option(i)) } case scatter: Scatter => throw new UnsupportedOperationException("Nested Scatters are not supported (yet) ... but you might try a sub workflow to achieve the same effect!") case e => throw new UnsupportedOperationException(s"Scope ${e.getClass.getName} is not supported.") - } + }) ++ makeCollectors(scope) } } // Represents a scatter collection for a call in the execution store - case class CollectorKey(scope: Scope with GraphNode) extends JobKey { + case class CollectorKey(scope: Scope with GraphNode, scatter: Scatter, scatterWidth: Int) extends JobKey { override val index = None override val attempt = 1 override val tag = s"Collector-${scope.unqualifiedName}" @@ -582,6 +671,40 @@ object WorkflowExecutionActor { override val tag = s"SubWorkflow-${scope.unqualifiedName}:${index.fromIndex}:$attempt" } + case class ConditionalKey(scope: If, index: ExecutionIndex) extends JobKey { + + override val tag = scope.unqualifiedName + override val attempt = 1 + + /** + * Creates a sub-ExecutionStore with entries for each of the scoped children. + * + * @return ExecutionStore of scattered children. + */ + def populate(workflowCoercedInputs: WorkflowCoercedInputs): Map[JobKey, ExecutionStatus.Value] = { + scope.children map { + keyify(_, workflowCoercedInputs) -> ExecutionStatus.NotStarted + } toMap + } + + /** + * Make a JobKey for all of the contained scopes. + */ + private def keyify(scope: Scope, workflowCoercedInputs: WorkflowCoercedInputs): JobKey = { + scope match { + case call: TaskCall => BackendJobDescriptorKey(call, index, 1) + case call: WorkflowCall => SubWorkflowKey(call, index, 1) + case declaration: Declaration => DeclarationKey(declaration, index, workflowCoercedInputs) + case i: If => ConditionalKey(i, index) + case scatter: Scatter if index.isEmpty => ScatterKey(scatter) + case _: Scatter => + throw new UnsupportedOperationException("Nested Scatters are not supported (yet) ... but you might try a sub workflow to achieve the same effect!") + case e => + throw new UnsupportedOperationException(s"Scope ${e.getClass.getName} is not supported in an If block.") + } + } + } + object DeclarationKey { def apply(declaration: Declaration, index: ExecutionIndex, inputs: WorkflowCoercedInputs): DeclarationKey = { inputs.find(_._1 == declaration.fullyQualifiedName) match { @@ -596,6 +719,7 @@ object WorkflowExecutionActor { } sealed trait DeclarationKey extends JobKey { + override val scope: Declaration override val attempt = 1 override val tag = s"Declaration-${scope.unqualifiedName}:${index.fromIndex}:$attempt" } @@ -623,4 +747,15 @@ object WorkflowExecutionActor { Props(WorkflowExecutionActor(workflowDescriptor, serviceRegistryActor, jobStoreActor, subWorkflowStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection, initializationData, restarting)).withDispatcher(EngineDispatcher) } -} \ No newline at end of file + + implicit class EnhancedWorkflowOutputs(val outputs: Map[LocallyQualifiedName, WdlValue]) extends AnyVal { + def maxStringLength = 1000 + + def stripLarge = outputs map { case (k, v) => + val wdlString = v.toWdlString + + if (wdlString.length > maxStringLength) (k, WdlString(StringUtils.abbreviate(wdlString, maxStringLength))) + else (k, v) + } + } +} diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala index 4bf2f213c..8cf1c7abc 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala @@ -7,7 +7,6 @@ import cromwell.core._ import cromwell.engine.workflow.lifecycle.execution.OutputStore.{OutputCallKey, OutputEntry} import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{DeclarationKey, SubWorkflowKey} import cromwell.engine.{EngineWorkflowDescriptor, WdlFunctions} -import cromwell.util.JsonFormatting.WdlValueJsonFormatter import wdl4s.values.WdlValue import wdl4s.{GraphNode, Scope} @@ -17,7 +16,7 @@ object WorkflowExecutionDiff { /** Data differential between current execution data, and updates performed in a method that needs to be merged. */ final case class WorkflowExecutionDiff(executionStoreChanges: Map[JobKey, ExecutionStatus], engineJobExecutionActorAdditions: Map[ActorRef, JobKey] = Map.empty) { - def containsNewEntry = executionStoreChanges.exists(_._2 == NotStarted) + def containsNewEntry = executionStoreChanges.exists(esc => esc._2 == NotStarted) } object WorkflowExecutionActorData { @@ -83,12 +82,9 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript * If complete, this will return Some(finalStatus). Otherwise, returns None */ def workflowCompletionStatus: Option[ExecutionStatus] = { // `List`ify the `prerequisiteScopes` to avoid expensive hashing of `Scope`s when assembling the result. - def upstream(scope: GraphNode): List[Scope] = { - val directUpstream: List[Scope with GraphNode] = scope.upstream.toList - directUpstream ++ directUpstream.flatMap(upstream) - } - def upstreamFailed(scope: Scope) = scope match { - case node: GraphNode => upstream(node) filter { s => + + def upstreamFailed(scope: Scope): List[GraphNode] = scope match { + case node: GraphNode => node.upstreamAncestry.toList filter { s => executionStore.store.exists({ case (key, status) => status == Failed && key.scope == s }) } } @@ -134,19 +130,6 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript this.copy(downstreamExecutionMap = downstreamExecutionMap ++ jobExecutionMap) } - def outputsJson(): String = { - // Printing the final outputs, temporarily here until SingleWorkflowManagerActor is made in-sync with the shadow mode - import WdlValueJsonFormatter._ - import spray.json._ - val workflowOutputs = outputStore.store collect { - case (key, outputs) if key.index.isEmpty => outputs map { output => - s"${key.call.fullyQualifiedName}.${output.name}" -> (output.wdlValue map { _.valueString } getOrElse "N/A") - } - } - - "Workflow complete. Final Outputs: \n" + workflowOutputs.flatten.toMap.toJson.prettyPrint - } - def mergeExecutionDiff(diff: WorkflowExecutionDiff): WorkflowExecutionActorData = { this.copy( executionStore = executionStore.add(diff.executionStoreChanges), diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala index 674b1ee88..53448d397 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala @@ -8,6 +8,7 @@ import cromwell.core.ExecutionIndex.IndexEnhancedIndex import cromwell.core.WorkflowId import cromwell.core.callcaching.HashResult import cromwell.core.simpleton.WdlValueSimpleton +import cromwell.core.path.PathImplicits._ import cromwell.database.sql._ import cromwell.database.sql.joins.CallCachingJoin import cromwell.database.sql.tables.{CallCachingDetritusEntry, CallCachingEntry, CallCachingHashEntry, CallCachingSimpletonEntry} @@ -53,7 +54,7 @@ class CallCache(database: CallCachingSqlDatabase) { val jobDetritusToInsert: Iterable[CallCachingDetritusEntry] = { jobDetritus map { - case (fileName, filePath) => CallCachingDetritusEntry(fileName, filePath.toUri.toString) + case (fileName, filePath) => CallCachingDetritusEntry(fileName, filePath.toRealString) } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala index a0ec75fc8..db2465f9f 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala @@ -1,6 +1,7 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching import akka.actor.{Actor, ActorLogging, Props} +import cromwell.core.Dispatcher.EngineDispatcher import scala.concurrent.ExecutionContext import scala.util.{Failure, Success} @@ -27,7 +28,7 @@ class CallCacheInvalidateActor(callCache: CallCache, cacheId: CallCachingEntryId object CallCacheInvalidateActor { def props(callCache: CallCache, cacheId: CallCachingEntryId) = { - Props(new CallCacheInvalidateActor(callCache: CallCache, cacheId: CallCachingEntryId)) + Props(new CallCacheInvalidateActor(callCache: CallCache, cacheId: CallCachingEntryId)).withDispatcher(EngineDispatcher) } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala index 8aa1f6d6b..739030c5f 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala @@ -2,6 +2,7 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching import akka.actor.{Actor, ActorLogging, ActorRef, Props} import akka.pattern.pipe +import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.callcaching.HashResult import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor._ import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes @@ -59,7 +60,7 @@ class CallCacheReadActor(cache: CallCache) extends Actor with ActorLogging { } object CallCacheReadActor { - def props(callCache: CallCache): Props = Props(new CallCacheReadActor(callCache)) + def props(callCache: CallCache): Props = Props(new CallCacheReadActor(callCache)).withDispatcher(EngineDispatcher) private[CallCacheReadActor] case class RequestTuple(requester: ActorRef, hashes: CallCacheHashes) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheWriteActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheWriteActor.scala index c6e42b5cc..b77bda043 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheWriteActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheWriteActor.scala @@ -3,6 +3,7 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching import akka.actor.{Actor, ActorLogging, Props} import cromwell.backend.BackendJobExecutionActor import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse +import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.WorkflowId import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes @@ -31,7 +32,7 @@ case class CallCacheWriteActor(callCache: CallCache, workflowId: WorkflowId, cal object CallCacheWriteActor { def props(callCache: CallCache, workflowId: WorkflowId, callCacheHashes: CallCacheHashes, succeededResponse: JobSucceededResponse): Props = - Props(CallCacheWriteActor(callCache, workflowId, callCacheHashes, succeededResponse)) + Props(CallCacheWriteActor(callCache, workflowId, callCacheHashes, succeededResponse)).withDispatcher(EngineDispatcher) } case object CallCacheWriteSuccess diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala index 053e8a14e..667e5b513 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala @@ -5,6 +5,7 @@ import cats.data.NonEmptyList import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest import cromwell.backend.{BackendInitializationData, BackendJobDescriptor, RuntimeAttributeDefinition} import cromwell.core.callcaching._ +import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.simpleton.WdlValueSimpleton import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor.{CacheLookupRequest, CacheResultLookupFailure, CacheResultMatchesForHashes} import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor._ @@ -191,7 +192,7 @@ object EngineJobHashingActor { callCacheReadActor = callCacheReadActor, runtimeAttributeDefinitions = runtimeAttributeDefinitions, backendName = backendName, - activity = activity)) + activity = activity)).withDispatcher(EngineDispatcher) private[callcaching] case class EJHAInitialHashingResults(hashes: Set[HashResult]) extends SuccessfulHashResultMessage private[callcaching] case object CheckWhetherAllHashesAreKnown diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala index b0900cb16..ab00d5856 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala @@ -2,6 +2,7 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching import akka.actor.{Actor, ActorLogging, ActorRef, Props} import cromwell.Simpletons._ +import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.simpleton.WdlValueSimpleton import cromwell.engine.workflow.lifecycle.execution.callcaching.FetchCachedResultsActor.{CachedOutputLookupFailed, CachedOutputLookupSucceeded} @@ -10,7 +11,7 @@ import scala.util.{Failure, Success} object FetchCachedResultsActor { def props(callCachingEntryId: CallCachingEntryId, replyTo: ActorRef, callCache: CallCache): Props = - Props(new FetchCachedResultsActor(callCachingEntryId, replyTo, callCache)) + Props(new FetchCachedResultsActor(callCachingEntryId, replyTo, callCache)).withDispatcher(EngineDispatcher) sealed trait CachedResultResponse case class CachedOutputLookupFailed(callCachingEntryId: CallCachingEntryId, failure: Throwable) extends CachedResultResponse diff --git a/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala b/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala index 99d10221d..8d0850c54 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/tokens/JobExecutionTokenDispenserActor.scala @@ -3,6 +3,7 @@ package cromwell.engine.workflow.tokens import akka.actor.{Actor, ActorLogging, ActorRef, Props, Terminated} import cromwell.core.JobExecutionToken import JobExecutionToken._ +import cromwell.core.Dispatcher.EngineDispatcher import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor._ import cromwell.engine.workflow.tokens.TokenPool.TokenPoolPop @@ -100,7 +101,7 @@ class JobExecutionTokenDispenserActor extends Actor with ActorLogging { object JobExecutionTokenDispenserActor { - def props = Props(new JobExecutionTokenDispenserActor) + def props = Props(new JobExecutionTokenDispenserActor).withDispatcher(EngineDispatcher) case class JobExecutionTokenRequest(jobExecutionTokenType: JobExecutionTokenType) case class JobExecutionTokenReturn(jobExecutionToken: JobExecutionToken) diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala index 2ecccbce2..27b9f2b6c 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala @@ -5,14 +5,15 @@ import java.time.OffsetDateTime import akka.actor.{ActorLogging, ActorRef, LoggingFSM, Props} import cats.data.NonEmptyList import cromwell.core._ +import cromwell.core.Dispatcher.EngineDispatcher import cromwell.engine.workflow.WorkflowManagerActor import cromwell.engine.workflow.WorkflowManagerActor.WorkflowNotFoundException import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._ import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState import cromwell.services.metadata.MetadataService.{MetadataPutAcknowledgement, PutMetadataAction} import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} +import lenthall.util.TryUtil import org.apache.commons.lang3.exception.ExceptionUtils -import wdl4s.util.TryUtil import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success, Try} @@ -238,6 +239,6 @@ object WorkflowStoreActor { final case class WorkflowAbortFailed(workflowId: WorkflowId, reason: Throwable) extends WorkflowStoreActorResponse def props(workflowStoreDatabase: WorkflowStore, serviceRegistryActor: ActorRef) = { - Props(WorkflowStoreActor(workflowStoreDatabase, serviceRegistryActor)) + Props(WorkflowStoreActor(workflowStoreDatabase, serviceRegistryActor)).withDispatcher(EngineDispatcher) } } diff --git a/engine/src/main/scala/cromwell/jobstore/JobStoreActor.scala b/engine/src/main/scala/cromwell/jobstore/JobStoreActor.scala index c5910c2be..e1e55ac19 100644 --- a/engine/src/main/scala/cromwell/jobstore/JobStoreActor.scala +++ b/engine/src/main/scala/cromwell/jobstore/JobStoreActor.scala @@ -1,6 +1,7 @@ package cromwell.jobstore import akka.actor.{Actor, Props} +import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.WorkflowId import cromwell.jobstore.JobStoreActor.{JobStoreReaderCommand, JobStoreWriterCommand} import wdl4s.TaskOutput @@ -50,5 +51,5 @@ object JobStoreActor { case class JobStoreReadFailure(reason: Throwable) extends JobStoreReaderResponse - def props(database: JobStore) = Props(new JobStoreActor(database)) + def props(database: JobStore) = Props(new JobStoreActor(database)).withDispatcher(EngineDispatcher) } diff --git a/engine/src/main/scala/cromwell/jobstore/JobStoreReaderActor.scala b/engine/src/main/scala/cromwell/jobstore/JobStoreReaderActor.scala index 151b0a711..9065a8f59 100644 --- a/engine/src/main/scala/cromwell/jobstore/JobStoreReaderActor.scala +++ b/engine/src/main/scala/cromwell/jobstore/JobStoreReaderActor.scala @@ -2,12 +2,13 @@ package cromwell.jobstore import akka.actor.{Actor, ActorLogging, Props} import akka.event.LoggingReceive +import cromwell.core.Dispatcher.EngineDispatcher import cromwell.jobstore.JobStoreActor.{JobComplete, JobNotComplete, JobStoreReadFailure, QueryJobCompletion} import scala.util.{Failure, Success} object JobStoreReaderActor { - def props(database: JobStore) = Props(new JobStoreReaderActor(database)) + def props(database: JobStore) = Props(new JobStoreReaderActor(database)).withDispatcher(EngineDispatcher) } class JobStoreReaderActor(database: JobStore) extends Actor with ActorLogging { diff --git a/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala b/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala index 9bb680b0c..70d0329d3 100644 --- a/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala +++ b/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala @@ -2,6 +2,7 @@ package cromwell.jobstore import akka.actor.{ActorRef, LoggingFSM, Props} import cromwell.jobstore.JobStoreActor._ +import cromwell.core.Dispatcher.EngineDispatcher import scala.util.{Failure, Success} @@ -74,7 +75,7 @@ case class JobStoreWriterActor(jsd: JobStore) extends LoggingFSM[JobStoreWriterS } object JobStoreWriterActor { - def props(jobStoreDatabase: JobStore): Props = Props(new JobStoreWriterActor(jobStoreDatabase)) + def props(jobStoreDatabase: JobStore): Props = Props(new JobStoreWriterActor(jobStoreDatabase)).withDispatcher(EngineDispatcher) } object JobStoreWriterData { diff --git a/engine/src/main/scala/cromwell/logging/TerminalLayout.scala b/engine/src/main/scala/cromwell/logging/TerminalLayout.scala index d18545d09..b157563dd 100644 --- a/engine/src/main/scala/cromwell/logging/TerminalLayout.scala +++ b/engine/src/main/scala/cromwell/logging/TerminalLayout.scala @@ -7,7 +7,7 @@ import ch.qos.logback.classic.Level import ch.qos.logback.classic.pattern.ThrowableProxyConverter import ch.qos.logback.classic.spi.ILoggingEvent import ch.qos.logback.core.LayoutBase -import cromwell.util.TerminalUtil +import lenthall.util.TerminalUtil object TerminalLayout { val Converter = new ThrowableProxyConverter diff --git a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala index 37dea8133..780b7c7ad 100644 --- a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala +++ b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala @@ -78,9 +78,7 @@ import net.ceedubs.ficus.Ficus._ * of Cromwell by passing a Throwable to the guardian. */ override val supervisorStrategy = OneForOneStrategy() { - case actorInitializationException: ActorInitializationException => throw new RuntimeException( - s"Unable to create actor for ActorRef ${actorInitializationException.getActor}", - actorInitializationException.getCause) + case actorInitializationException: ActorInitializationException => Escalate case t => super.supervisorStrategy.decider.applyOrElse(t, (_: Any) => Escalate) } } diff --git a/engine/src/main/scala/cromwell/server/CromwellServer.scala b/engine/src/main/scala/cromwell/server/CromwellServer.scala index 36010fb62..882150776 100644 --- a/engine/src/main/scala/cromwell/server/CromwellServer.scala +++ b/engine/src/main/scala/cromwell/server/CromwellServer.scala @@ -5,6 +5,7 @@ import java.util.concurrent.TimeoutException import akka.actor.Props import akka.util.Timeout import com.typesafe.config.Config +import cromwell.core.Dispatcher.EngineDispatcher import cromwell.webservice.WorkflowJsonSupport._ import cromwell.webservice.{APIResponse, CromwellApiService, SwaggerService} import lenthall.spray.SprayCanHttpService._ @@ -76,6 +77,6 @@ class CromwellServerActor(config: Config) extends CromwellRootActor with Cromwel object CromwellServerActor { def props(config: Config): Props = { - Props(new CromwellServerActor(config)) + Props(new CromwellServerActor(config)).withDispatcher(EngineDispatcher) } } diff --git a/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStoreActor.scala b/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStoreActor.scala index cf7624087..25fb62ce0 100644 --- a/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStoreActor.scala +++ b/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStoreActor.scala @@ -1,6 +1,7 @@ package cromwell.subworkflowstore import akka.actor.{Actor, ActorLogging, ActorRef, Props} +import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.ExecutionIndex._ import cromwell.core.{JobKey, WorkflowId} import cromwell.database.sql.tables.SubWorkflowStoreEntry @@ -68,5 +69,5 @@ object SubWorkflowStoreActor { def props(database: SubWorkflowStore) = Props( new SubWorkflowStoreActor(database) - ) + ).withDispatcher(EngineDispatcher) } diff --git a/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala b/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala index dee173ca3..45e7e6643 100644 --- a/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala +++ b/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala @@ -1,8 +1,9 @@ package cromwell.webservice +import lenthall.exception.MessageAggregation import spray.json._ +import wdl4s.FullyQualifiedName import wdl4s.values.WdlValue -import wdl4s.{ExceptionWithErrors, FullyQualifiedName} case class WorkflowStatusResponse(id: String, status: String) @@ -21,9 +22,9 @@ object APIResponse { private def constructFailureResponse(status: String, ex: Throwable) ={ ex match { - case exceptionWithErrors: ExceptionWithErrors => - FailureResponse(status, exceptionWithErrors.message, - Option(JsArray(exceptionWithErrors.errors.toList.map(JsString(_)).toVector))) + case exceptionWithErrors: MessageAggregation => + FailureResponse(status, exceptionWithErrors.getMessage, + Option(JsArray(exceptionWithErrors.errorMessages.toList.map(JsString(_)).toVector))) case e: Throwable => FailureResponse(status, e.getMessage, None) } } diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala index 76ec94d77..8afd3fafa 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala @@ -5,6 +5,7 @@ import akka.event.Logging import cats.data.NonEmptyList import com.typesafe.config.ConfigFactory import cromwell.core._ +import cromwell.core.Dispatcher.ApiDispatcher import cromwell.engine.workflow.WorkflowManagerActor import cromwell.engine.workflow.WorkflowManagerActor.WorkflowNotFoundException import cromwell.engine.workflow.workflowstore.WorkflowStoreActor @@ -16,7 +17,7 @@ import spray.httpx.SprayJsonSupport._ object CromwellApiHandler { def props(requestHandlerActor: ActorRef): Props = { - Props(new CromwellApiHandler(requestHandlerActor)) + Props(new CromwellApiHandler(requestHandlerActor)).withDispatcher(ApiDispatcher) } sealed trait ApiHandlerMessage @@ -67,7 +68,7 @@ class CromwellApiHandler(requestHandlerActor: ActorRef) extends Actor with Workf context.parent ! RequestComplete((StatusCodes.Created, WorkflowSubmitResponse(id.toString, WorkflowSubmitted.toString))) case ApiHandlerWorkflowSubmitBatch(sources) => requestHandlerActor ! - WorkflowStoreActor.BatchSubmitWorkflows(sources.map(x => WorkflowSourceFilesWithoutImports(x.wdlSource,x.inputsJson,x.workflowOptionsJson))) + WorkflowStoreActor.BatchSubmitWorkflows(sources.map(x => WorkflowSourceFilesCollection(x.wdlSource,x.inputsJson,x.workflowOptionsJson,x.importsZipFileOption))) case WorkflowStoreActor.WorkflowsBatchSubmittedToStore(ids) => diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala index 34fff945b..34113fd55 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala @@ -1,8 +1,8 @@ package cromwell.webservice import akka.actor._ - import cats.data.NonEmptyList +import com.typesafe.config.{Config, ConfigFactory} import cromwell.core.{WorkflowId, WorkflowOptionsJson, WorkflowSourceFilesCollection} import cromwell.engine.backend.BackendConfiguration import cromwell.services.metadata.MetadataService._ @@ -58,7 +58,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator { } val workflowRoutes = queryRoute ~ queryPostRoute ~ workflowOutputsRoute ~ submitRoute ~ submitBatchRoute ~ - workflowLogsRoute ~ abortRoute ~ metadataRoute ~ timingRoute ~ statusRoute ~ backendRoute ~ statsRoute + workflowLogsRoute ~ abortRoute ~ metadataRoute ~ timingRoute ~ statusRoute ~ backendRoute ~ statsRoute ~ versionRoute private def withRecognizedWorkflowId(possibleWorkflowId: String)(recognizedWorkflowId: WorkflowId => Route): Route = { def callback(requestContext: RequestContext) = new ValidationCallback { @@ -178,18 +178,15 @@ trait CromwellApiService extends HttpService with PerRequestCreator { path("workflows" / Segment) { version => post { entity(as[MultipartFormData]) { formData => - requestContext => { - PartialWorkflowSources.fromSubmitRoute(formData, allowNoInputs = true) match { - case Success(workflowSourceFiles) if workflowSourceFiles.size == 1 => + PartialWorkflowSources.fromSubmitRoute(formData, allowNoInputs = true) match { + case Success(workflowSourceFiles) if workflowSourceFiles.size == 1 => + requestContext => { perRequest(requestContext, CromwellApiHandler.props(workflowStoreActor), CromwellApiHandler.ApiHandlerWorkflowSubmit(workflowSourceFiles.head)) - case Success(workflowSourceFiles) => - failBadRequest(new IllegalArgumentException("To submit more than one workflow at a time, use the batch endpoint.")) - case Failure(t) => - System.err.println(t) - t.printStackTrace(System.err) - failBadRequest(t) - } - () + } + case Success(workflowSourceFiles) => + failBadRequest(new IllegalArgumentException("To submit more than one workflow at a time, use the batch endpoint.")) + case Failure(t) => + failBadRequest(t) } } } @@ -199,16 +196,13 @@ trait CromwellApiService extends HttpService with PerRequestCreator { path("workflows" / Segment / "batch") { version => post { entity(as[MultipartFormData]) { formData => - requestContext => { - PartialWorkflowSources.fromSubmitRoute(formData, allowNoInputs = false) match { - case Success(workflowSourceFiles) => + PartialWorkflowSources.fromSubmitRoute(formData, allowNoInputs = false) match { + case Success(workflowSourceFiles) => + requestContext => { perRequest(requestContext, CromwellApiHandler.props(workflowStoreActor), CromwellApiHandler.ApiHandlerWorkflowSubmitBatch(NonEmptyList.fromListUnsafe(workflowSourceFiles.toList))) - case Failure(t) => - System.err.println(t) - t.printStackTrace(System.err) - failBadRequest(t) - } - () + } + case Failure(t) => + failBadRequest(t) } } } @@ -271,6 +265,20 @@ trait CromwellApiService extends HttpService with PerRequestCreator { } } + def versionRoute = + path("engine" / Segment / "version") { version => + get { + complete { + lazy val versionConf = ConfigFactory.load("cromwell-version.conf").getConfig("version") + versionResponse(versionConf) + } + } + } + + def versionResponse(versionConf: Config) = JsObject(Map( + "cromwell" -> versionConf.getString("cromwell").toJson + )) + def backendRoute = path("workflows" / Segment / "backends") { version => get { diff --git a/engine/src/test/scala/cromwell/MetadataWatchActor.scala b/engine/src/test/scala/cromwell/MetadataWatchActor.scala index c0c294442..cb8802e22 100644 --- a/engine/src/test/scala/cromwell/MetadataWatchActor.scala +++ b/engine/src/test/scala/cromwell/MetadataWatchActor.scala @@ -1,6 +1,7 @@ package cromwell import akka.actor.{Actor, Props} +import cromwell.core.Dispatcher.EngineDispatcher import cromwell.services.metadata.{MetadataEvent, MetadataJobKey, MetadataString, MetadataValue} import cromwell.services.metadata.MetadataService.PutMetadataAction import MetadataWatchActor._ @@ -28,13 +29,19 @@ final case class MetadataWatchActor(promise: Promise[Unit], matchers: Matcher*) object MetadataWatchActor { - def props(promise: Promise[Unit], matchers: Matcher*): Props = Props(MetadataWatchActor(promise, matchers: _*)) + def props(promise: Promise[Unit], matchers: Matcher*): Props = Props(MetadataWatchActor(promise, matchers: _*)).withDispatcher(EngineDispatcher) trait Matcher { - def matches(events: Traversable[MetadataEvent]): Boolean + private var _fullEventList: List[MetadataEvent] = List.empty + final def matches(events: Traversable[MetadataEvent]): Boolean = { + _fullEventList ++= events + _matches(events) + } + def _matches(events: Traversable[MetadataEvent]): Boolean private var _nearMisses: List[String] = List.empty - protected def addNearMissInfo(miss: String) = _nearMisses :+= miss + private def addNearMissInfo(miss: String) = _nearMisses :+= miss def nearMissInformation = _nearMisses + def fullEventList = _fullEventList def checkMetadataValueContains(key: String, actual: MetadataValue, expected: String): Boolean = { val result = actual.value.contains(expected) @@ -49,14 +56,14 @@ object MetadataWatchActor { } final case class JobKeyMetadataKeyAndValueContainStringMatcher(jobKeyCheck: Option[MetadataJobKey] => Boolean, key: String, value: String) extends Matcher { - def matches(events: Traversable[MetadataEvent]): Boolean = { + def _matches(events: Traversable[MetadataEvent]): Boolean = { events.exists(e => e.key.key.contains(key) && jobKeyCheck(e.key.jobKey) && e.value.exists { v => v.valueType == MetadataString && checkMetadataValueContains(e.key.key, v, value) }) } } abstract class KeyMatchesRegexAndValueContainsStringMatcher(keyTemplate: String, value: String) extends Matcher { val templateRegex = keyTemplate.r - def matches(events: Traversable[MetadataEvent]): Boolean = { + def _matches(events: Traversable[MetadataEvent]): Boolean = { events.exists(e => templateRegex.findFirstIn(e.key.key).isDefined && e.value.exists { v => checkMetadataValueContains(e.key.key, v, value) }) } diff --git a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala index 3859368d7..893f56bfa 100644 --- a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala @@ -138,13 +138,22 @@ class SimpleWorkflowActorSpec extends CromwellTestKitSpec with BeforeAndAfter { } "gracefully handle malformed WDL" in { - val expectedError = "Input evaluation for Call test1.summary failed.\nVariable 'bfile' not found" + val expectedError = "Variable 'bfile' not found" val failureMatcher = FailureMatcher(expectedError) val TestableWorkflowActorAndMetadataPromise(workflowActor, supervisor, promise) = buildWorkflowActor(SampleWdl.CoercionNotDefined, SampleWdl.CoercionNotDefined.wdlJson, workflowId, failureMatcher) val probe = TestProbe() probe watch workflowActor workflowActor ! StartWorkflowCommand - Await.result(promise.future, TestExecutionTimeout) + try { + Await.result(promise.future, TestExecutionTimeout) + } catch { + case e: Throwable => + val info = failureMatcher.nearMissInformation + val errorString = + if (info.nonEmpty) "We had a near miss: " + info.mkString(", ") + else s"The expected key was never seen. We saw: [\n ${failureMatcher.fullEventList.map(e => s"${e.key} -> ${e.value}").mkString("\n ")}\n]." + fail(s"We didn't see the expected error message '$expectedError' within $TestExecutionTimeout. $errorString}") + } probe.expectTerminated(workflowActor, AwaitAlmostNothing) supervisor.expectMsgPF(AwaitAlmostNothing, "parent should get a failed response") { case x: WorkflowFailedResponse => diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala index 1301f0e9b..646487e1f 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala @@ -1,5 +1,7 @@ package cromwell.engine.workflow +import java.nio.file.Paths + import akka.actor.{Actor, ActorRef} import akka.testkit.{TestActorRef, TestFSMRef, TestProbe} import com.typesafe.config.{Config, ConfigFactory} @@ -8,12 +10,13 @@ import cromwell.core._ import cromwell.engine.EngineWorkflowDescriptor import cromwell.engine.backend.BackendSingletonCollection import cromwell.engine.workflow.WorkflowActor._ -import cromwell.engine.workflow.lifecycle.EngineLifecycleActorAbortCommand +import cromwell.engine.workflow.lifecycle.{CopyWorkflowLogsActor, EngineLifecycleActorAbortCommand} +import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.MaterializeWorkflowDescriptorFailureResponse import cromwell.engine.workflow.lifecycle.WorkflowFinalizationActor.{StartFinalizationCommand, WorkflowFinalizationSucceededResponse} import cromwell.engine.workflow.lifecycle.WorkflowInitializationActor.{WorkflowInitializationAbortedResponse, WorkflowInitializationFailedResponse} import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{WorkflowExecutionAbortedResponse, WorkflowExecutionFailedResponse, WorkflowExecutionSucceededResponse} import cromwell.util.SampleWdl.ThreeStep -import cromwell.{AlwaysHappyJobStoreActor, AlwaysHappySubWorkflowStoreActor, CromwellTestKitSpec, EmptyCallCacheReadActor} +import cromwell._ import org.scalatest.BeforeAndAfter import org.scalatest.concurrent.Eventually @@ -28,14 +31,17 @@ class WorkflowActorSpec extends CromwellTestKitSpec with WorkflowDescriptorBuild } }) + val mockDir = Paths.get("/where/to/copy/wf/logs") + val mockWorkflowOptions = s"""{ "final_workflow_log_dir" : "$mockDir" }""" + var currentWorkflowId: WorkflowId = _ val currentLifecycleActor = TestProbe() - val wdlSources = ThreeStep.asWorkflowSources() + val wdlSources = ThreeStep.asWorkflowSources(workflowOptions = mockWorkflowOptions) val descriptor = createMaterializedEngineWorkflowDescriptor(WorkflowId.randomId(), workflowSources = wdlSources) val supervisorProbe = TestProbe() val deathwatch = TestProbe() val finalizationProbe = TestProbe() - + val copyWorkflowLogsProbe = TestProbe() val AwaitAlmostNothing = 100.milliseconds before { @@ -51,7 +57,7 @@ class WorkflowActorSpec extends CromwellTestKitSpec with WorkflowDescriptorBuild workflowSources = wdlSources, conf = ConfigFactory.load, serviceRegistryActor = mockServiceRegistryActor, - workflowLogCopyRouter = TestProbe().ref, + workflowLogCopyRouter = copyWorkflowLogsProbe.ref, jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props), subWorkflowStoreActor = system.actorOf(AlwaysHappySubWorkflowStoreActor.props), callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props), @@ -146,6 +152,15 @@ class WorkflowActorSpec extends CromwellTestKitSpec with WorkflowDescriptorBuild finalizationProbe.expectNoMsg(AwaitAlmostNothing) deathwatch.expectTerminated(actor) } + + "copy workflow logs in the event of MaterializeWorkflowDescriptorFailureResponse" in { + val actor = createWorkflowActor(MaterializingWorkflowDescriptorState) + deathwatch watch actor + actor ! MaterializeWorkflowDescriptorFailureResponse(new Exception("Intentionally failing workflow materialization to test log copying")) + finalizationProbe.expectNoMsg(AwaitAlmostNothing) + copyWorkflowLogsProbe.expectMsg(CopyWorkflowLogsActor.Copy(currentWorkflowId, mockDir)) + deathwatch.expectTerminated(actor) + } } } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala index 0715ebeed..06f3be008 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala @@ -12,7 +12,7 @@ import org.scalatest.BeforeAndAfter import org.scalatest.mockito.MockitoSugar import spray.json.DefaultJsonProtocol._ import spray.json._ -import wdl4s.values.{WdlInteger, WdlString} +import wdl4s.values.WdlString import scala.concurrent.duration._ @@ -64,8 +64,8 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be wfDesc.id shouldBe workflowId wfDesc.name shouldBe "wf_hello" wfDesc.namespace.tasks.size shouldBe 1 - wfDesc.workflowInputs.head shouldBe (("wf_hello.hello.addressee", WdlString("world"))) - wfDesc.backendDescriptor.inputs.head shouldBe (("wf_hello.hello.addressee", WdlString("world"))) + wfDesc.knownValues.head shouldBe (("wf_hello.hello.addressee", WdlString("world"))) + wfDesc.backendDescriptor.knownValues.head shouldBe (("wf_hello.hello.addressee", WdlString("world"))) wfDesc.getWorkflowOption(WorkflowOptions.WriteToCache) shouldBe Option("true") wfDesc.getWorkflowOption(WorkflowOptions.ReadFromCache) shouldBe None // Default backend assignment is "Local": @@ -83,50 +83,6 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be system.stop(materializeWfActor) } - // Note to whoever comes next: I don't really know why this distinction exists. I've added this test but would - // not be at all upset if the whole thing gets removed. - "differently construct engine workflow inputs and backend inputs" in { - val wdl = - """ - |task bar { command { echo foobar } } - |workflow foo { - | Int i - | Int j = 5 - |} - """.stripMargin - val inputs = - """ - |{ "foo.i": "17" } - """.stripMargin - - val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false)) - val sources = WorkflowSourceFilesWithoutImports(wdl, inputs, validOptionsFile) - materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf) - - within(Timeout) { - expectMsgPF() { - case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => - - - wfDesc.workflowInputs foreach { - case ("foo.i", wdlValue) => wdlValue shouldBe WdlInteger(17) - case ("foo.j", wdlValue) => fail("Workflow declarations should not appear as workflow inputs") - case (x, y) => fail(s"Unexpected input $x -> $y") - } - - wfDesc.backendDescriptor.inputs foreach { - case ("foo.i", wdlValue) => wdlValue shouldBe WdlInteger(17) - case ("foo.j", wdlValue) => wdlValue shouldBe WdlInteger(5) - case (x, y) => fail(s"Unexpected input $x -> $y") - } - case MaterializeWorkflowDescriptorFailureResponse(reason) => fail(s"Unexpected materialization failure: $reason") - case unknown => fail(s"Unexpected materialization response: $unknown") - } - } - - system.stop(materializeWfActor) - } - "assign default runtime attributes" ignore { val wdl = """ @@ -257,7 +213,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be within(Timeout) { expectMsgPF() { case MaterializeWorkflowDescriptorFailureResponse(reason) => - reason.getMessage should startWith("Workflow input processing failed.\nUnable to load namespace from workflow: ERROR: Finished parsing without consuming all tokens.") + reason.getMessage should startWith("Workflow input processing failed:\nUnable to load namespace from workflow: ERROR: Finished parsing without consuming all tokens.") case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!") case unknown => fail(s"Unexpected materialization response: $unknown") @@ -281,7 +237,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be within(Timeout) { expectMsgPF() { case MaterializeWorkflowDescriptorFailureResponse(reason) => - reason.getMessage should startWith("Workflow input processing failed.\nUnable to load namespace from workflow: Namespace does not have a local workflow to run") + reason.getMessage should startWith("Workflow input processing failed:\nUnable to load namespace from workflow: Namespace does not have a local workflow to run") case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!") case unknown => fail(s"Unexpected materialization response: $unknown") @@ -306,7 +262,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be within(Timeout) { expectMsgPF() { case MaterializeWorkflowDescriptorFailureResponse(reason) => - reason.getMessage should startWith("Workflow input processing failed.\nUnable to load namespace from workflow: Namespace does not have a local workflow to run") + reason.getMessage should startWith("Workflow input processing failed:\nUnable to load namespace from workflow: Namespace does not have a local workflow to run") case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!") case unknown => fail(s"Unexpected materialization response: $unknown") @@ -325,7 +281,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be within(Timeout) { expectMsgPF() { case MaterializeWorkflowDescriptorFailureResponse(reason) => - reason.getMessage should startWith("Workflow input processing failed.\nWorkflow contains invalid options JSON") + reason.getMessage should startWith("Workflow input processing failed:\nWorkflow contains invalid options JSON") case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!") case unknown => fail(s"Unexpected materialization response: $unknown") @@ -343,7 +299,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be within(Timeout) { expectMsgPF() { case MaterializeWorkflowDescriptorFailureResponse(reason) => - reason.getMessage should startWith("Workflow input processing failed.\nWorkflow contains invalid inputs JSON") + reason.getMessage should startWith("Workflow input processing failed:\nWorkflow contains invalid inputs JSON") case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!") case unknown => fail(s"Unexpected materialization response: $unknown") @@ -362,7 +318,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be within(Timeout) { expectMsgPF() { case MaterializeWorkflowDescriptorFailureResponse(reason) => - reason.getMessage should startWith("Workflow input processing failed.\nRequired workflow input 'wf_hello.hello.addressee' not specified") + reason.getMessage should startWith("Workflow input processing failed:\nRequired workflow input 'wf_hello.hello.addressee' not specified") case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!") case unknown => fail(s"Unexpected materialization response: $unknown") @@ -388,7 +344,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be within(Timeout) { expectMsgPF() { case MaterializeWorkflowDescriptorFailureResponse(reason) => - reason.getMessage should startWith("Workflow input processing failed.\nUnable to load namespace from workflow: ERROR: Value for j is not coerceable into a Int") + reason.getMessage should startWith("Workflow input processing failed:\nUnable to load namespace from workflow: ERROR: Value for j is not coerceable into a Int") case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!") case unknown => fail(s"Unexpected materialization response: $unknown") } diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala index 1f61772e6..32a75e57f 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala @@ -42,7 +42,6 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with FlatSpecLike with mock[WdlNamespaceWithWorkflow], parentBackendDescriptor, Map.empty, - Map.empty, ContinueWhilePossible, List.empty, CallCachingOff @@ -141,7 +140,6 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with FlatSpecLike with mock[WdlNamespaceWithWorkflow], subBackendDescriptor, Map.empty, - Map.empty, ContinueWhilePossible, List.empty, CallCachingOff diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala index 9389362ad..2add49473 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala @@ -101,7 +101,7 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock (implicit startingState: EngineJobExecutionActorState): TestFSMRef[EngineJobExecutionActorState, EJEAData, MockEjea] = { val factory: BackendLifecycleActorFactory = buildFactory() - val descriptor = EngineWorkflowDescriptor(mock[WdlNamespaceWithWorkflow], backendWorkflowDescriptor, Map.empty, null, null, null, callCachingMode) + val descriptor = EngineWorkflowDescriptor(mock[WdlNamespaceWithWorkflow], backendWorkflowDescriptor, null, null, null, callCachingMode) val myBrandNewEjea = new TestFSMRef[EngineJobExecutionActorState, EJEAData, MockEjea](system, Props(new MockEjea( helper = this, diff --git a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala index b3fbfee1e..18f985af6 100644 --- a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala @@ -258,6 +258,23 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala } } } + it should "return 400 for an unrecognized form data request parameter " in { + val bodyParts: Map[String, BodyPart] = Map("incorrectParameter" -> BodyPart(HelloWorld.wdlSource())) + Post(s"/workflows/$version", MultipartFormData(bodyParts)) ~> + submitRoute ~> + check { + assertResult( + s"""{ + | "status": "fail", + | "message": "Unexpected body part name: incorrectParameter" + |}""".stripMargin) { + responseAs[String] + } + assertResult(StatusCodes.BadRequest) { + status + } + } + } it should "succesfully merge and override multiple input files" in { val input1 = Map("wf.a1" -> "hello", "wf.a2" -> "world").toJson.toString @@ -295,6 +312,25 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala } } + it should "return 400 for an submission with no inputs" in { + val bodyParts = Map("wdlSource" -> BodyPart(HelloWorld.wdlSource())) + + Post(s"/workflows/$version/batch", MultipartFormData(bodyParts)) ~> + submitBatchRoute ~> + check { + assertResult( + s"""{ + | "status": "fail", + | "message": "No inputs were provided" + |}""".stripMargin) { + responseAs[String] + } + assertResult(StatusCodes.BadRequest) { + status + } + } + } + // TODO: Test tha batch submission returns expected workflow ids in order // TODO: Also (assuming we still validate on submit) test a batch of mixed inputs that return submitted and failed diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala index 0649da9b7..ec2f2da47 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala @@ -11,9 +11,11 @@ import com.google.cloud.RetryParams import com.google.cloud.storage.StorageOptions import com.google.cloud.storage.contrib.nio.{CloudStorageConfiguration, CloudStorageFileSystem, CloudStoragePath} import com.google.common.base.Preconditions._ +import com.google.common.net.UrlEscapers import cromwell.core.WorkflowOptions import cromwell.core.path.proxy.{PathProxy, RetryableFileSystemProviderProxy} import cromwell.core.path.{CustomRetryParams, PathBuilder} +import cromwell.filesystems.gcs.GcsPathBuilder._ import cromwell.filesystems.gcs.auth.GoogleAuthMode import scala.util.{Failure, Try} @@ -28,19 +30,21 @@ object GcsPathBuilder { checkArgument( uri.getScheme.equalsIgnoreCase(CloudStorageFileSystem.URI_SCHEME), "Cloud Storage URIs must have '%s' scheme: %s", - CloudStorageFileSystem.URI_SCHEME, - uri + CloudStorageFileSystem.URI_SCHEME: Any, + uri: Any ) checkNotNull(uri.getHost, s"%s does not have a host", uri) } def isValidGcsUrl(str: String): Boolean = { - Try(checkValid(URI.create(str))).isSuccess + Try(checkValid(getUri(str))).isSuccess } def isGcsPath(path: Path): Boolean = { path.getFileSystem.provider().getScheme == CloudStorageFileSystem.URI_SCHEME } + + def getUri(string: String) = URI.create(UrlEscapers.urlFragmentEscaper().escape(string)) } class GcsPathBuilder(authMode: GoogleAuthMode, @@ -78,7 +82,7 @@ class GcsPathBuilder(authMode: GoogleAuthMode, def build(string: String): Try[Path] = { Try { - val uri = URI.create(string) + val uri = getUri(string) GcsPathBuilder.checkValid(uri) provider.getPath(uri) } diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala index 8fa93b61d..142485c95 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala @@ -8,10 +8,11 @@ import cats.syntax.validated._ import com.google.api.services.storage.StorageScopes import com.typesafe.config.Config import cromwell.filesystems.gcs.auth._ -import lenthall.config.ConfigValidationException -import lenthall.config.ValidatedConfig._ -import cromwell.core.ErrorOr._ +import lenthall.exception.MessageAggregation +import lenthall.validation.ErrorOr._ +import lenthall.validation.Validation._ import org.slf4j.LoggerFactory +import net.ceedubs.ficus.Ficus._ final case class GoogleConfiguration private (applicationName: String, authsByName: Map[String, GoogleAuthMode]) { @@ -29,31 +30,35 @@ object GoogleConfiguration { import scala.collection.JavaConverters._ private val log = LoggerFactory.getLogger("GoogleConfiguration") + case class GoogleConfigurationException(errorMessages: List[String]) extends MessageAggregation { + override val exceptionContext = "Google configuration" + } + val GoogleScopes = List( StorageScopes.DEVSTORAGE_FULL_CONTROL, StorageScopes.DEVSTORAGE_READ_WRITE, "https://www.googleapis.com/auth/genomics", "https://www.googleapis.com/auth/compute" ).asJava - + def apply(config: Config): GoogleConfiguration = { val googleConfig = config.getConfig("google") - val appName = googleConfig.validateString("application-name") + val appName = validate { googleConfig.as[String]("application-name") } def buildAuth(authConfig: Config): ErrorOr[GoogleAuthMode] = { - def serviceAccountAuth(authConfig: Config, name: String) = authConfig validateAny { - cfg => ServiceAccountMode(name, cfg.getString("service-account-id"), cfg.getString("pem-file"), GoogleScopes) + def serviceAccountAuth(authConfig: Config, name: String): ErrorOr[GoogleAuthMode] = validate { + ServiceAccountMode(name, authConfig.as[String]("service-account-id"), authConfig.as[String]("pem-file"), GoogleScopes) } - def userAccountAuth(authConfig: Config, name: String) = authConfig validateAny { - cfg => UserMode(name, cfg.getString("user"), cfg.getString("secrets-file"), cfg.getString("data-store-dir"), GoogleScopes) + def userAccountAuth(authConfig: Config, name: String): ErrorOr[GoogleAuthMode] = validate { + UserMode(name, authConfig.as[String]("user"), authConfig.as[String]("secrets-file"), authConfig.as[String]("data-store-dir"), GoogleScopes) } - def refreshTokenAuth(authConfig: Config, name: String) = authConfig validateAny { - cfg => RefreshTokenMode(name, cfg.getString("client-id"), cfg.getString("client-secret"), GoogleScopes) + def refreshTokenAuth(authConfig: Config, name: String): ErrorOr[GoogleAuthMode] = validate { + RefreshTokenMode(name, authConfig.as[String]("client-id"), authConfig.as[String]("client-secret"), GoogleScopes) } def applicationDefaultAuth(name: String): ErrorOr[GoogleAuthMode] = ApplicationDefaultMode(name).validNel @@ -69,7 +74,7 @@ object GoogleConfiguration { } } - val listOfErrorOrAuths: List[ErrorOr[GoogleAuthMode]] = googleConfig.getConfigList("auths").asScala.toList map buildAuth + val listOfErrorOrAuths: List[ErrorOr[GoogleAuthMode]] = googleConfig.as[List[Config]]("auths") map buildAuth val errorOrAuthList: ErrorOr[List[GoogleAuthMode]] = listOfErrorOrAuths.sequence[ErrorOr, GoogleAuthMode] def uniqueAuthNames(list: List[GoogleAuthMode]): ErrorOr[Unit] = { @@ -90,7 +95,7 @@ object GoogleConfiguration { case Invalid(f) => val errorMessages = f.toList.mkString(", ") log.error(errorMessages) - throw new ConfigValidationException("Google", errorMessages) + throw new GoogleConfigurationException(f.toList) } } } diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala index cc1c9fd6c..80fd1c137 100644 --- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala +++ b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala @@ -2,8 +2,8 @@ package cromwell.filesystems.gcs import better.files.File import com.typesafe.config.{ConfigException, ConfigFactory} +import cromwell.filesystems.gcs.GoogleConfiguration.GoogleConfigurationException import cromwell.filesystems.gcs.auth.{ApplicationDefaultMode, RefreshTokenMode, ServiceAccountMode, UserMode} -import lenthall.config.ConfigValidationException import org.scalatest.{FlatSpec, Matchers} @@ -90,7 +90,7 @@ class GoogleConfigurationSpec extends FlatSpec with Matchers { |} """.stripMargin - a[ConfigValidationException] shouldBe thrownBy { + a[GoogleConfigurationException] shouldBe thrownBy { GoogleConfiguration(ConfigFactory.parseString(applessGoogleConfig)) } } @@ -110,7 +110,7 @@ class GoogleConfigurationSpec extends FlatSpec with Matchers { |} """.stripMargin - a[ConfigValidationException] shouldBe thrownBy { + a[GoogleConfigurationException] shouldBe thrownBy { GoogleConfiguration(ConfigFactory.parseString(unsupported)) } @@ -167,7 +167,7 @@ class GoogleConfigurationSpec extends FlatSpec with Matchers { |} """.stripMargin - a[ConfigException.Missing] shouldBe thrownBy { + a[GoogleConfigurationException] shouldBe thrownBy { GoogleConfiguration(ConfigFactory.parseString(badKeyInRefreshTokenMode)) } @@ -188,7 +188,7 @@ class GoogleConfigurationSpec extends FlatSpec with Matchers { |} """.stripMargin - a[ConfigException.Missing] shouldBe thrownBy { + a[GoogleConfigurationException] shouldBe thrownBy { GoogleConfiguration(ConfigFactory.parseString(badKeyInUserMode)) } @@ -208,7 +208,7 @@ class GoogleConfigurationSpec extends FlatSpec with Matchers { |} """.stripMargin - a[ConfigException.Missing] shouldBe thrownBy { + a[GoogleConfigurationException] shouldBe thrownBy { GoogleConfiguration(ConfigFactory.parseString(badKeyInServiceAccountMode)) } } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index ee130f095..b06413d9d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,8 +1,8 @@ import sbt._ object Dependencies { - lazy val lenthallV = "0.19" - lazy val wdl4sV = "0.7" + lazy val lenthallV = "0.20" + lazy val wdl4sV = "0.8" lazy val sprayV = "1.3.3" /* spray-json is an independent project from the "spray suite" @@ -12,7 +12,7 @@ object Dependencies { - http://doc.akka.io/docs/akka/2.4/scala/http/common/json-support.html#akka-http-spray-json */ lazy val sprayJsonV = "1.3.2" - lazy val akkaV = "2.4.12" + lazy val akkaV = "2.4.14" lazy val slickV = "3.1.1" lazy val googleClientApiV = "1.22.0" lazy val googleGenomicsServicesApiV = "1.20.0" @@ -115,7 +115,8 @@ object Dependencies { "com.typesafe" % "config" % "1.3.0", "com.typesafe.akka" %% "akka-actor" % akkaV, "com.typesafe.akka" %% "akka-slf4j" % akkaV, - "com.typesafe.akka" %% "akka-testkit" % akkaV % Test + "com.typesafe.akka" %% "akka-testkit" % akkaV % Test, + "com.google.guava" % "guava" % "20.0" ) ++ baseDependencies ++ googleApiClientDependencies ++ // TODO: We're not using the "F" in slf4j. Core only supports logback, specifically the WorkflowLogger. slf4jBindingDependencies diff --git a/project/Settings.scala b/project/Settings.scala index f59d102e8..d2032360f 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -159,7 +159,7 @@ object Settings { val engineSettings = List( name := "cromwell-engine", libraryDependencies ++= engineDependencies - ) ++ commonSettings + ) ++ commonSettings ++ versionConfCompileSettings val rootSettings = List( name := "cromwell", diff --git a/project/Version.scala b/project/Version.scala index 70adedb03..3cd7d5da8 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -4,7 +4,7 @@ import sbt._ object Version { // Upcoming release, or current if we're on a master / hotfix branch - val cromwellVersion = "23" + val cromwellVersion = "24" // Adapted from SbtGit.versionWithGit def cromwellVersionWithGit: Seq[Setting[_]] = @@ -19,6 +19,19 @@ object Version { shellPrompt in ThisBuild := { state => "%s| %s> ".format(GitCommand.prompt.apply(state), cromwellVersion) } ) + val writeVersionConf: Def.Initialize[Task[Seq[File]]] = Def.task { + val file = (resourceManaged in Compile).value / "cromwell-version.conf" + val contents = + s"""|version { + | cromwell: "${version.value}" + |} + |""".stripMargin + IO.write(file, contents) + Seq(file) + } + + val versionConfCompileSettings = List(resourceGenerators in Compile <+= writeVersionConf) + private def makeVersion(versionProperty: String, baseVersion: Option[String], headCommit: Option[String]): String = { diff --git a/services/src/main/scala/cromwell/services/ServiceRegistryActor.scala b/services/src/main/scala/cromwell/services/ServiceRegistryActor.scala index e58cd5c05..c4ff1e476 100644 --- a/services/src/main/scala/cromwell/services/ServiceRegistryActor.scala +++ b/services/src/main/scala/cromwell/services/ServiceRegistryActor.scala @@ -2,6 +2,7 @@ package cromwell.services import akka.actor.SupervisorStrategy.Escalate import akka.actor.{Actor, ActorInitializationException, ActorLogging, ActorRef, OneForOneStrategy, Props} +import cromwell.core.Dispatcher.ServiceDispatcher import com.typesafe.config.{Config, ConfigFactory, ConfigObject} import net.ceedubs.ficus.Ficus._ import scala.collection.JavaConverters._ @@ -13,12 +14,12 @@ object ServiceRegistryActor { def serviceName: String } - def props(config: Config) = Props(new ServiceRegistryActor(serviceNameToPropsMap(config))) + def props(config: Config) = Props(new ServiceRegistryActor(serviceNameToPropsMap(config))).withDispatcher(ServiceDispatcher) // To enable testing, this lets us override a config value with a Props of our choice: def props(config: Config, overrides: Map[String, Props]) = { val fromConfig = serviceNameToPropsMap(config).filterNot { case (name: String, _: Props) => overrides.keys.toList.contains(name) } - Props(new ServiceRegistryActor(fromConfig ++ overrides)) + Props(new ServiceRegistryActor(fromConfig ++ overrides)).withDispatcher(ServiceDispatcher) } def serviceNameToPropsMap(globalConfig: Config): Map[String, Props] = { @@ -34,8 +35,13 @@ object ServiceRegistryActor { val className = serviceStanza.as[Option[String]]("class").getOrElse( throw new IllegalArgumentException(s"Invalid configuration for service $serviceName: missing 'class' definition") ) - - Props.create(Class.forName(className), serviceConfigStanza, globalConfig) + try { + Props.create(Class.forName(className), serviceConfigStanza, globalConfig) + } catch { + case e: ClassNotFoundException => throw new RuntimeException( + s"Class $className for service $serviceName cannot be found in the class path.", e + ) + } } } diff --git a/services/src/main/scala/cromwell/services/keyvalue/impl/SqlKeyValueServiceActor.scala b/services/src/main/scala/cromwell/services/keyvalue/impl/SqlKeyValueServiceActor.scala index a1909d8f2..57452bf21 100644 --- a/services/src/main/scala/cromwell/services/keyvalue/impl/SqlKeyValueServiceActor.scala +++ b/services/src/main/scala/cromwell/services/keyvalue/impl/SqlKeyValueServiceActor.scala @@ -2,6 +2,7 @@ package cromwell.services.keyvalue.impl import akka.actor.Props import com.typesafe.config.Config +import cromwell.core.Dispatcher.ServiceDispatcher import cromwell.services.SingletonServicesStore import cromwell.services.keyvalue.KeyValueServiceActor import cromwell.services.keyvalue.KeyValueServiceActor._ @@ -9,7 +10,7 @@ import cromwell.services.keyvalue.KeyValueServiceActor._ import scala.concurrent.Future object SqlKeyValueServiceActor { - def props(serviceConfig: Config, globalConfig: Config) = Props(SqlKeyValueServiceActor(serviceConfig, globalConfig)) + def props(serviceConfig: Config, globalConfig: Config) = Props(SqlKeyValueServiceActor(serviceConfig, globalConfig)).withDispatcher(ServiceDispatcher) } case class SqlKeyValueServiceActor(override val serviceConfig: Config, override val globalConfig: Config) diff --git a/services/src/main/scala/cromwell/services/metadata/CallMetadataKeys.scala b/services/src/main/scala/cromwell/services/metadata/CallMetadataKeys.scala index bf1df98d9..b22bca0b8 100644 --- a/services/src/main/scala/cromwell/services/metadata/CallMetadataKeys.scala +++ b/services/src/main/scala/cromwell/services/metadata/CallMetadataKeys.scala @@ -15,6 +15,7 @@ object CallMetadataKeys { val Stdout = "stdout" val Stderr = "stderr" val BackendLogsPrefix = "backendLogs" + val BackendStatus = "backendStatus" val JobId = "jobId" val CallRoot = "callRoot" val SubWorkflowId = "subWorkflowId" diff --git a/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala b/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala index 6542c2082..f17118aa7 100644 --- a/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala +++ b/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala @@ -1,11 +1,13 @@ package cromwell.services.metadata +import java.nio.file.Path import java.time.OffsetDateTime import cats.data.NonEmptyList import cromwell.core.WorkflowId +import cromwell.core.path.PathImplicits._ import org.slf4j.LoggerFactory -import wdl4s.values.{WdlBoolean, WdlFloat, WdlInteger, WdlValue} +import wdl4s.values.{WdlBoolean, WdlFloat, WdlInteger, WdlOptionalValue, WdlValue} case class MetadataJobKey(callFqn: String, index: Option[Int], attempt: Int) @@ -35,15 +37,17 @@ case object MetadataNumber extends MetadataType { override val typeName = "numbe case object MetadataBoolean extends MetadataType { override val typeName = "boolean" } object MetadataValue { - def apply(value: Any) = { + def apply(value: Any): MetadataValue = { Option(value).getOrElse("") match { case WdlInteger(i) => new MetadataValue(i.toString, MetadataInt) case WdlFloat(f) => new MetadataValue(f.toString, MetadataNumber) case WdlBoolean(b) => new MetadataValue(b.toString, MetadataBoolean) + case WdlOptionalValue(_, Some(o)) => apply(o) case value: WdlValue => new MetadataValue(value.valueString, MetadataString) case _: Int | Long => new MetadataValue(value.toString, MetadataInt) case _: Double | Float => new MetadataValue(value.toString, MetadataNumber) case _: Boolean => new MetadataValue(value.toString, MetadataBoolean) + case path: Path => new MetadataValue(path.toRealString, MetadataString) case _ => new MetadataValue(value.toString, MetadataString) } } diff --git a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala index e2a62c784..cf589b2a2 100644 --- a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala +++ b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala @@ -6,6 +6,7 @@ import akka.actor.{ActorRef, DeadLetterSuppression} import cats.data.NonEmptyList import cromwell.core.{JobKey, WorkflowId, WorkflowState} import cromwell.services.ServiceRegistryActor.ServiceRegistryMessage +import lenthall.exception.ThrowableAggregation import wdl4s.values._ @@ -109,14 +110,24 @@ object MetadataService { } else { valueMap.toList flatMap { case (key, value) => wdlValueToMetadataEvents(metadataKey.copy(key = metadataKey.key + s":${key.valueString}"), value) } } + case WdlOptionalValue(_, Some(value)) => + wdlValueToMetadataEvents(metadataKey, value) + case WdlPair(left, right) => + wdlValueToMetadataEvents(metadataKey.copy(key = metadataKey.key + ":left"), left) ++ + wdlValueToMetadataEvents(metadataKey.copy(key = metadataKey.key + ":right"), right) case value => List(MetadataEvent(metadataKey, MetadataValue(value))) } def throwableToMetadataEvents(metadataKey: MetadataKey, t: Throwable): List[MetadataEvent] = { - val message = List(MetadataEvent(metadataKey.copy(key = s"${metadataKey.key}:message"), MetadataValue(t.getMessage))) - val cause = Option(t.getCause) map { cause => throwableToMetadataEvents(metadataKey.copy(key = s"${metadataKey.key}:causedBy"), cause) } getOrElse List.empty - message ++ cause + t match { + case aggregation: ThrowableAggregation => + aggregation.errorMessages.toList map { message => MetadataEvent(metadataKey.copy(key = s"${metadataKey.key}:message"), MetadataValue(s"${aggregation.exceptionContext}: $message")) } + case other => + val message = List(MetadataEvent(metadataKey.copy(key = s"${metadataKey.key}:message"), MetadataValue(t.getMessage))) + val cause = Option(t.getCause) map { cause => throwableToMetadataEvents(metadataKey.copy(key = s"${metadataKey.key}:causedBy"), cause) } getOrElse List.empty + message ++ cause + } } } diff --git a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala index 435c14df2..e5e134ca7 100644 --- a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala +++ b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala @@ -6,7 +6,7 @@ import cats.instances.list._ import cats.syntax.traverse._ import cats.syntax.validated._ import cromwell.core.{WorkflowId, WorkflowState} -import cromwell.core.ErrorOr._ +import lenthall.validation.ErrorOr._ import scala.util.{Success, Try} diff --git a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala index f66173657..59618d1c8 100644 --- a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala +++ b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala @@ -7,7 +7,7 @@ import cats.syntax.cartesian._ import cats.syntax.validated._ import cromwell.core.WorkflowId import cromwell.services.metadata.WorkflowQueryKey._ -import cromwell.core.ErrorOr._ +import lenthall.validation.ErrorOr._ case class WorkflowQueryParameters private(statuses: Set[String], diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala index a92b92fbd..2ce7c5b92 100644 --- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala +++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala @@ -4,6 +4,7 @@ import java.util.UUID import akka.actor.{Actor, ActorLogging, ActorRef, Props} import com.typesafe.config.{Config, ConfigFactory} +import cromwell.core.Dispatcher.ServiceDispatcher import cromwell.core.WorkflowId import cromwell.services.SingletonServicesStore import cromwell.services.metadata.MetadataService.{PutMetadataAction, ReadAction, RefreshSummary, ValidateWorkflowIdAndExecute} @@ -20,7 +21,7 @@ object MetadataServiceActor { if (duration.isFinite()) Option(duration.asInstanceOf[FiniteDuration]) else None } - def props(serviceConfig: Config, globalConfig: Config) = Props(MetadataServiceActor(serviceConfig, globalConfig)) + def props(serviceConfig: Config, globalConfig: Config) = Props(MetadataServiceActor(serviceConfig, globalConfig)).withDispatcher(ServiceDispatcher) } case class MetadataServiceActor(serviceConfig: Config, globalConfig: Config) diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala index 007755fa6..ef16bdb89 100644 --- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala +++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala @@ -3,6 +3,7 @@ package cromwell.services.metadata.impl import akka.actor.{ActorRef, LoggingFSM, Props} import com.typesafe.config.ConfigFactory +import cromwell.core.Dispatcher.ServiceDispatcher import cromwell.services.SingletonServicesStore import cromwell.services.metadata.impl.MetadataSummaryRefreshActor._ @@ -21,7 +22,7 @@ object MetadataSummaryRefreshActor { case object MetadataSummarySuccess extends MetadataSummaryActorMessage final case class MetadataSummaryFailure(t: Throwable) extends MetadataSummaryActorMessage - def props() = Props(new MetadataSummaryRefreshActor()) + def props() = Props(new MetadataSummaryRefreshActor()).withDispatcher(ServiceDispatcher) sealed trait SummaryRefreshState case object WaitingForRequest extends SummaryRefreshState diff --git a/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala index 0a2c31f4a..dd45607fd 100644 --- a/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala +++ b/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala @@ -1,13 +1,14 @@ package cromwell.services.metadata.impl import akka.actor.{Actor, ActorLogging, Props} +import cromwell.core.Dispatcher.ServiceDispatcher import cromwell.services.SingletonServicesStore import cromwell.services.metadata.MetadataService.{MetadataPutAcknowledgement, MetadataPutFailed, PutMetadataAction} import scala.util.{Failure, Success} object WriteMetadataActor { - def props() = Props(new WriteMetadataActor()) + def props() = Props(new WriteMetadataActor()).withDispatcher(ServiceDispatcher) } class WriteMetadataActor extends Actor with ActorLogging with MetadataDatabaseAccess with SingletonServicesStore { diff --git a/services/src/test/scala/cromwell/services/ServiceRegistryActorSpec.scala b/services/src/test/scala/cromwell/services/ServiceRegistryActorSpec.scala index bb8658b24..ae51d6283 100644 --- a/services/src/test/scala/cromwell/services/ServiceRegistryActorSpec.scala +++ b/services/src/test/scala/cromwell/services/ServiceRegistryActorSpec.scala @@ -124,8 +124,12 @@ class ServiceRegistryActorSpec extends TestKitSuite("service-registry-actor-spec val probe = buildProbeForInitializationException(ConfigFactory.parseString(missingService)) probe.expectMsgPF(AwaitTimeout) { case e: ActorInitializationException => - e.getCause shouldBe a [ClassNotFoundException] - e.getCause.getMessage shouldBe "cromwell.services.FooWhoServiceActor" + // The class not found exception is wrapped in a Runtime Exception giving the name of the faulty service + val cause = e.getCause + cause shouldBe a [RuntimeException] + val classNotFound = cause.getCause + classNotFound shouldBe a [ClassNotFoundException] + classNotFound.getMessage shouldBe "cromwell.services.FooWhoServiceActor" } } diff --git a/services/src/test/scala/cromwell/services/ServicesSpec.scala b/services/src/test/scala/cromwell/services/ServicesSpec.scala index 629079a5c..b2bf1e49f 100644 --- a/services/src/test/scala/cromwell/services/ServicesSpec.scala +++ b/services/src/test/scala/cromwell/services/ServicesSpec.scala @@ -21,6 +21,7 @@ object ServicesSpec { | debug { | receive = on | } + | guardian-supervisor-strategy = "akka.actor.DefaultSupervisorStrategy" | } | dispatchers { | # A dispatcher for actors performing blocking io operations diff --git a/src/bin/travis/afterSuccess.sh b/src/bin/travis/afterSuccess.sh index e8bd24aae..4fac80d74 100755 --- a/src/bin/travis/afterSuccess.sh +++ b/src/bin/travis/afterSuccess.sh @@ -9,7 +9,12 @@ echo "TRAVIS_PULL_REQUEST='$TRAVIS_PULL_REQUEST'" if [ "$BUILD_TYPE" == "sbt" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then if [ "$TRAVIS_BRANCH" == "develop" ]; then - sbt 'set test in Test := {}' publish + docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" + sbt \ + 'set test in Test := {}' \ + 'set imageNames in docker := Seq(ImageName("broadinstitute/cromwell:'${TRAVIS_BRANCH}'"))' \ + publish \ + dockerBuildAndPush elif [[ "$TRAVIS_BRANCH" =~ ^[0-9\.]+_hotfix$ ]]; then docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" diff --git a/src/bin/travis/resources/centaur.wdl b/src/bin/travis/resources/centaur.wdl index 230b271aa..ddead03bb 100644 --- a/src/bin/travis/resources/centaur.wdl +++ b/src/bin/travis/resources/centaur.wdl @@ -13,7 +13,7 @@ task centaur { cd centaur git checkout ${centaur_branch} cd .. - centaur/test_cromwell.sh -j${cromwell_jar} -c${conf} -r/cromwell_root -t ${secret} -elocaldockertest + centaur/test_cromwell.sh -j${cromwell_jar} -c${conf} -r/cromwell_root -t ${secret} -elocaldockertest -p100 >>> output { diff --git a/src/bin/travis/testCentaurLocal.sh b/src/bin/travis/testCentaurLocal.sh index 337c302bf..3023f0a80 100755 --- a/src/bin/travis/testCentaurLocal.sh +++ b/src/bin/travis/testCentaurLocal.sh @@ -36,4 +36,8 @@ git clone https://github.com/broadinstitute/centaur.git cd centaur git checkout ${CENTAUR_BRANCH} cd .. +# All tests use ubuntu:latest - make sure it's there before starting the tests +# because pulling the image during some of the tests would cause them to fail +# (specifically output_redirection which expects a specific value in stderr) +docker pull ubuntu:latest centaur/test_cromwell.sh -j"${CROMWELL_JAR}" -c${LOCAL_CONF} diff --git a/src/main/scala/cromwell/CromwellCommandLine.scala b/src/main/scala/cromwell/CromwellCommandLine.scala index 230045559..9c2e4f5ae 100644 --- a/src/main/scala/cromwell/CromwellCommandLine.scala +++ b/src/main/scala/cromwell/CromwellCommandLine.scala @@ -9,13 +9,14 @@ import cats.syntax.validated._ import cromwell.core.{WorkflowSourceFilesWithoutImports, WorkflowSourceFilesCollection, WorkflowSourceFilesWithDependenciesZip} import cromwell.util.FileUtil._ import lenthall.exception.MessageAggregation -import cromwell.core.ErrorOr._ +import lenthall.validation.ErrorOr._ import scala.util.{Failure, Success, Try} sealed abstract class CromwellCommandLine case object UsageAndExit extends CromwellCommandLine case object RunServer extends CromwellCommandLine +case object VersionAndExit extends CromwellCommandLine final case class RunSingle(wdlPath: Path, sourceFiles: WorkflowSourceFilesCollection, inputsPath: Option[Path], @@ -27,6 +28,7 @@ object CromwellCommandLine { args.headOption match { case Some("server") if args.size == 1 => RunServer case Some("run") if args.size >= 2 && args.size <= 6 => RunSingle(args.tail) + case Some("-version") if args.size == 1 => VersionAndExit case _ => UsageAndExit } } diff --git a/src/main/scala/cromwell/Main.scala b/src/main/scala/cromwell/Main.scala index c52875572..05fa645e5 100644 --- a/src/main/scala/cromwell/Main.scala +++ b/src/main/scala/cromwell/Main.scala @@ -10,17 +10,27 @@ import org.slf4j.LoggerFactory import scala.collection.JavaConverters._ import scala.concurrent.duration._ import scala.concurrent.{Await, Future} -import scala.util.{Failure, Success} +import scala.language.postfixOps +import scala.util.{Failure, Success, Try} object Main extends App { val CommandLine = CromwellCommandLine(args) initLogging(CommandLine) lazy val Log = LoggerFactory.getLogger("cromwell") - lazy val CromwellSystem = new CromwellSystem {} + lazy val CromwellSystem: CromwellSystem = Try { + new CromwellSystem {} + } recoverWith { + case t: Throwable => + Log.error("Failed to instantiate Cromwell System. Shutting down Cromwell.") + Log.error(t.getMessage) + System.exit(1) + Failure(t) + } get CommandLine match { case UsageAndExit => usageAndExit() + case VersionAndExit => versionAndExit() case RunServer => waitAndExit(CromwellServer.run(CromwellSystem), CromwellSystem) case r: RunSingle => runWorkflow(r) } @@ -118,8 +128,23 @@ object Main extends App { | | Starts a web server on port 8000. See the web server | documentation for more details about the API endpoints. + | + |-version + | + | Returns the version of the Cromwell engine. + | """.stripMargin) System.exit(1) } + + def versionAndExit(): Unit = { + val versionConf = ConfigFactory.load("cromwell-version.conf").getConfig("version") + println( + s""" + |cromwell: ${versionConf.getString("cromwell")} + """.stripMargin + ) + System.exit(1) + } } diff --git a/src/test/scala/cromwell/CromwellCommandLineSpec.scala b/src/test/scala/cromwell/CromwellCommandLineSpec.scala index 66c8ee29c..8fadbcce4 100644 --- a/src/test/scala/cromwell/CromwellCommandLineSpec.scala +++ b/src/test/scala/cromwell/CromwellCommandLineSpec.scala @@ -33,6 +33,10 @@ class CromwellCommandLineSpec extends FlatSpec with Matchers { CromwellCommandLine(List("run", "bork", "bork", "bork", "bork", "bork", "blerg")) } + it should "VersionAndExit when the `-version` flag is passed" in { + CromwellCommandLine(List("-version")) shouldBe VersionAndExit + } + it should "RunSingle when supplying wdl and inputs" in { CromwellCommandLine(List("run", ThreeStepWithoutOptions.wdl, ThreeStepWithoutOptions.inputs)) shouldBe a [RunSingle] } diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala index d005016ca..e36a5f971 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala @@ -15,11 +15,11 @@ import cromwell.core.path.JavaWriterImplicits._ import cromwell.core.path.{DefaultPathBuilder, PathBuilder} import cromwell.services.keyvalue.KeyValueServiceActor._ import cromwell.services.metadata.CallMetadataKeys +import lenthall.util.TryUtil import org.apache.commons.codec.digest.DigestUtils import wdl4s.EvaluatedTaskInputs import wdl4s.parser.MemoryUnit import wdl4s.types.{WdlArrayType, WdlFileType} -import wdl4s.util.TryUtil import wdl4s.values.WdlArray import scala.concurrent.{Future, Promise} diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala index 758d2dbb4..c96fd2f0a 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala @@ -10,7 +10,7 @@ import cromwell.backend.validation.RuntimeAttributesDefault._ import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.backend.validation.RuntimeAttributesValidation._ import cromwell.core._ -import cromwell.core.ErrorOr._ +import lenthall.validation.ErrorOr._ import lenthall.exception.MessageAggregation import wdl4s.types._ import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString, WdlValue} diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala index 659d700fa..9812c44e9 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala @@ -4,12 +4,12 @@ import cromwell.backend.{BackendSpec, MemorySize} import cromwell.backend.validation.ContinueOnReturnCodeSet import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.core.WorkflowOptions +import lenthall.util.TryUtil import org.scalatest.{Matchers, WordSpecLike} import spray.json._ import wdl4s.WdlExpression._ import wdl4s._ import wdl4s.expression.NoFunctions -import wdl4s.util.TryUtil import wdl4s.values.WdlValue class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { @@ -291,7 +291,7 @@ class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { val workflowDescriptor = buildWorkflowDescriptor(wdlSource, runtime = runtimeAttributes) def createLookup(call: Call): ScopedLookupFunction = { - val knownInputs = workflowDescriptor.inputs + val knownInputs = workflowDescriptor.knownValues call.lookupFunction(knownInputs, NoFunctions) } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala index 84da1efbc..447310711 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala @@ -3,32 +3,27 @@ package cromwell.backend.impl.jes import java.net.SocketTimeoutException import java.nio.file.{Path, Paths} -import akka.actor.{Actor, ActorLogging, ActorRef, Props} -import akka.event.LoggingReceive +import akka.actor.{Actor, ActorLogging, ActorRef} import better.files._ -import cats.instances.future._ -import cats.syntax.functor._ import com.google.api.client.googleapis.json.GoogleJsonResponseException import com.google.cloud.storage.contrib.nio.CloudStoragePath -import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse} -import cromwell.backend.BackendLifecycleActor.AbortJobCommand -import cromwell.backend.async.AsyncBackendJobExecutionActor.{ExecutionMode, JobId} -import cromwell.backend.async.{AbortedExecutionHandle, AsyncBackendJobExecutionActor, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle, NonRetryableExecution, SuccessfulExecutionHandle} -import cromwell.backend.impl.jes.JesJobExecutionActor.JesOperationIdKey +import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse +import cromwell.backend._ +import cromwell.backend.async.{AbortedExecutionHandle, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle, PendingExecutionHandle, SuccessfulExecutionHandle} import cromwell.backend.impl.jes.RunStatus.TerminalRunStatus import cromwell.backend.impl.jes.io._ import cromwell.backend.impl.jes.statuspolling.JesPollingActorClient +import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncExecutionActorParams, StandardAsyncJob} +import cromwell.backend.validation.ContinueOnReturnCode import cromwell.backend.wdl.OutputEvaluator -import cromwell.backend.{BackendJobDescriptor, BackendWorkflowDescriptor, PreemptedException} -import cromwell.core.Dispatcher.BackendDispatcher import cromwell.core._ import cromwell.core.logging.JobLogging +import cromwell.core.path.PathFactory._ +import cromwell.core.path.PathImplicits._ import cromwell.core.path.proxy.PathProxy -import cromwell.core.retry.{Retry, SimpleExponentialBackoff} -import cromwell.services.keyvalue.KeyValueServiceActor._ -import cromwell.services.metadata._ +import cromwell.core.retry.SimpleExponentialBackoff import wdl4s._ -import wdl4s.expression.NoFunctions +import wdl4s.expression.{NoFunctions, WdlFunctions} import wdl4s.values._ import scala.concurrent.duration._ @@ -36,58 +31,54 @@ import scala.concurrent.{ExecutionContext, Future, Promise} import scala.language.postfixOps import scala.util.{Failure, Success, Try} -object JesAsyncBackendJobExecutionActor { - - def props(jobDescriptor: BackendJobDescriptor, - completionPromise: Promise[BackendJobExecutionResponse], - jesWorkflowInfo: JesConfiguration, - initializationData: JesBackendInitializationData, - serviceRegistryActor: ActorRef, - jesBackendSingletonActor: ActorRef): Props = { - Props(new JesAsyncBackendJobExecutionActor(jobDescriptor, - completionPromise, - jesWorkflowInfo, - initializationData, - serviceRegistryActor, - jesBackendSingletonActor)).withDispatcher(BackendDispatcher) - } +case class JesAsyncExecutionActorParams +( + override val jobDescriptor: BackendJobDescriptor, + jesConfiguration: JesConfiguration, + jesBackendInitializationData: JesBackendInitializationData, + override val serviceRegistryActor: ActorRef, + jesBackendSingletonActorOption: Option[ActorRef], + override val completionPromise: Promise[BackendJobExecutionResponse] +) extends StandardAsyncExecutionActorParams { + override val jobIdKey: String = JesJobExecutionActor.JesOperationIdKey + override val configurationDescriptor: BackendConfigurationDescriptor = jesConfiguration.configurationDescriptor + override val backendInitializationDataOption: Option[BackendInitializationData] = Option(jesBackendInitializationData) +} +object JesAsyncBackendJobExecutionActor { object WorkflowOptionKeys { val MonitoringScript = "monitoring_script" val GoogleProject = "google_project" val GoogleComputeServiceAccount = "google_compute_service_account" } + type JesPendingExecutionHandle = + PendingExecutionHandle[StandardAsyncJob, Run, RunStatus] private val ExtraConfigParamName = "__extra_config_gcs_path" +} - /** - * Representing a running JES execution, instances of this class are never Done and it is never okay to - * ask them for results. - */ - case class JesPendingExecutionHandle(jobDescriptor: BackendJobDescriptor, - jesOutputs: Set[JesFileOutput], - run: Run, - previousStatus: Option[RunStatus]) extends ExecutionHandle { - override val isDone = false - override val result = NonRetryableExecution(new IllegalStateException("JesPendingExecutionHandle cannot yield a result")) - } +class JesAsyncBackendJobExecutionActor(val jesParams: JesAsyncExecutionActorParams) + extends Actor with ActorLogging with BackendJobLifecycleActor with StandardAsyncExecutionActor + with JesJobCachingActorHelper with JobLogging with JesPollingActorClient { - case class JesJobId(operationId: String) extends JobId -} + import JesAsyncBackendJobExecutionActor._ + override val standardParams: StandardAsyncExecutionActorParams = jesParams + override val jesConfiguration: JesConfiguration = jesParams.jesConfiguration + override val initializationData: JesBackendInitializationData = { + backendInitializationDataAs[JesBackendInitializationData] + } -class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, - override val completionPromise: Promise[BackendJobExecutionResponse], - override val jesConfiguration: JesConfiguration, - override val initializationData: JesBackendInitializationData, - override val serviceRegistryActor: ActorRef, - val jesBackendSingletonActor: ActorRef) - extends Actor with ActorLogging with AsyncBackendJobExecutionActor with JesJobCachingActorHelper with JobLogging with JesPollingActorClient { + val jesBackendSingletonActor: ActorRef = + jesParams.jesBackendSingletonActorOption.getOrElse( + throw new RuntimeException("JES Backend actor cannot exist without the JES backend singleton actor")) - import JesAsyncBackendJobExecutionActor._ + override type StandardAsyncRunInfo = Run - override val pollingActor = jesBackendSingletonActor + override type StandardAsyncRunStatus = RunStatus + + override val pollingActor: ActorRef = jesBackendSingletonActor override lazy val pollBackOff = SimpleExponentialBackoff( initialInterval = 30 seconds, maxInterval = jesAttributes.maxPollingInterval seconds, multiplier = 1.1) @@ -95,51 +86,41 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes override lazy val executeOrRecoverBackOff = SimpleExponentialBackoff( initialInterval = 3 seconds, maxInterval = 20 seconds, multiplier = 1.1) - override lazy val workflowDescriptor = jobDescriptor.workflowDescriptor + override lazy val workflowDescriptor: BackendWorkflowDescriptor = jobDescriptor.workflowDescriptor - private lazy val call = jobDescriptor.key.call + lazy val call: TaskCall = jobDescriptor.key.call - override lazy val retryable = jobDescriptor.key.attempt <= runtimeAttributes.preemptible + override lazy val retryable: Boolean = jobDescriptor.key.attempt <= runtimeAttributes.preemptible private lazy val cmdInput = - JesFileInput(ExecParamName, jesCallPaths.script.toUri.toString, Paths.get(jesCallPaths.scriptFilename), workingDisk) + JesFileInput(ExecParamName, jesCallPaths.script.toRealString, Paths.get(jesCallPaths.scriptFilename), workingDisk) private lazy val jesCommandLine = s"/bin/bash ${cmdInput.containerPath}" - private lazy val rcJesOutput = JesFileOutput(returnCodeFilename, returnCodeGcsPath.toUri.toString, Paths.get(returnCodeFilename), workingDisk) + private lazy val rcJesOutput = JesFileOutput(returnCodeFilename, returnCodeGcsPath.toRealString, Paths.get(returnCodeFilename), workingDisk) private lazy val standardParameters = Seq(rcJesOutput) - private lazy val returnCodeContents = Try(File(returnCodeGcsPath).contentAsString) - private lazy val dockerConfiguration = jesConfiguration.dockerCredentials - private lazy val tag = s"${this.getClass.getSimpleName} [UUID(${workflowId.shortString}):${jobDescriptor.key.tag}]" - private var runId: Option[String] = None + private lazy val dockerConfiguration = jesConfiguration.dockerCredentials - def jesReceiveBehavior: Receive = LoggingReceive { - case AbortJobCommand => - runId foreach { id => - Try(Run(id, initializationData.genomics).abort()) match { - case Success(_) => jobLogger.info("{} Aborted {}", tag: Any, id) - case Failure(ex) => jobLogger.warn("{} Failed to abort {}: {}", tag, id, ex.getMessage) - } - } - context.parent ! AbortedResponse(jobDescriptor.key) - context.stop(self) - case KvPutSuccess(_) => // expected after the KvPut for the operation ID + override def tryAbort(job: StandardAsyncJob): Unit = { + Run(job.jobId, initializationData.genomics).abort() } - override def receive: Receive = pollingActorClientReceive orElse jesReceiveBehavior orElse super.receive + override def requestsAbortAndDiesImmediately: Boolean = true + + override def receive: Receive = pollingActorClientReceive orElse super.receive private def gcsAuthParameter: Option[JesInput] = { if (jesAttributes.auths.gcs.requiresAuthFile || dockerConfiguration.isDefined) - Option(JesLiteralInput(ExtraConfigParamName, jesCallPaths.gcsAuthFilePath.toUri.toString)) + Option(JesLiteralInput(ExtraConfigParamName, jesCallPaths.gcsAuthFilePath.toRealString)) else None } private lazy val callContext = CallContext( callRootPath, - jesStdoutFile.toUri.toString, - jesStderrFile.toUri.toString + jesStdoutFile.toRealString, + jesStderrFile.toRealString ) - private[jes] lazy val callEngineFunctions = new JesExpressionFunctions(List(jesCallPaths.gcsPathBuilder), callContext) + private[jes] lazy val backendEngineFunctions = new JesExpressionFunctions(List(jesCallPaths.gcsPathBuilder), callContext) /** * Takes two arrays of remote and local WDL File paths and generates the necessary JesInputs. @@ -171,7 +152,7 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes private[jes] def generateJesInputs(jobDescriptor: BackendJobDescriptor): Set[JesInput] = { - val writeFunctionFiles = call.task.evaluateFilesFromCommand(jobDescriptor.fullyQualifiedInputs, callEngineFunctions) map { + val writeFunctionFiles = call.task.evaluateFilesFromCommand(jobDescriptor.fullyQualifiedInputs, backendEngineFunctions) map { case (expression, file) => expression.toWdlString.md5SumShort -> Seq(file) } @@ -212,13 +193,6 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes if (referenceName.length <= 127) referenceName else referenceName.md5Sum } - private[jes] def findGlobOutputs(jobDescriptor: BackendJobDescriptor): Set[WdlGlobFile] = { - val globOutputs = (call.task.findOutputFiles(jobDescriptor.fullyQualifiedInputs, NoFunctions) map relativeLocalizationPath) collect { - case glob: WdlGlobFile => glob - } - globOutputs.distinct.toSet - } - private[jes] def generateJesOutputs(jobDescriptor: BackendJobDescriptor): Set[JesFileOutput] = { val wdlFileOutputs = call.task.findOutputFiles(jobDescriptor.fullyQualifiedInputs, NoFunctions) map relativeLocalizationPath @@ -233,17 +207,17 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes } private def generateJesSingleFileOutputs(wdlFile: WdlSingleFile): JesFileOutput = { - val destination = callRootPath.resolve(wdlFile.value.stripPrefix("/")).toUri.toString + val destination = callRootPath.resolve(wdlFile.value.stripPrefix("/")).toRealString val (relpath, disk) = relativePathAndAttachedDisk(wdlFile.value, runtimeAttributes.disks) JesFileOutput(makeSafeJesReferenceName(wdlFile.value), destination, relpath, disk) } private def generateJesGlobFileOutputs(wdlFile: WdlGlobFile): List[JesFileOutput] = { - val globName = callEngineFunctions.globName(wdlFile.value) + val globName = backendEngineFunctions.globName(wdlFile.value) val globDirectory = globName + "/" val globListFile = globName + ".list" - val gcsGlobDirectoryDestinationPath = callRootPath.resolve(globDirectory).toUri.toString - val gcsGlobListFileDestinationPath = callRootPath.resolve(globListFile).toUri.toString + val gcsGlobDirectoryDestinationPath = callRootPath.resolve(globDirectory).toRealString + val gcsGlobListFileDestinationPath = callRootPath.resolve(globListFile).toRealString val (_, globDirectoryDisk) = relativePathAndAttachedDisk(wdlFile.value, runtimeAttributes.disks) @@ -256,12 +230,25 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes ) } - private def instantiateCommand: Try[String] = { - val backendInputs = jobDescriptor.inputDeclarations mapValues gcsPathToLocal - jobDescriptor.call.task.instantiateCommand(backendInputs, callEngineFunctions, valueMapper = gcsPathToLocal) + override lazy val remoteStdErrPath: Path = jesStderrFile + + override lazy val remoteReturnCodePath: Path = returnCodeGcsPath + + override lazy val failOnStdErr: Boolean = runtimeAttributes.failOnStderr + + override lazy val continueOnReturnCode: ContinueOnReturnCode = runtimeAttributes.continueOnReturnCode + + override lazy val commandLineFunctions: WdlFunctions[WdlValue] = backendEngineFunctions + + override lazy val commandLinePreProcessor: (EvaluatedTaskInputs) => Try[EvaluatedTaskInputs] = mapGcsValues + + def mapGcsValues(inputs: EvaluatedTaskInputs): Try[EvaluatedTaskInputs] = { + Try(inputs mapValues gcsPathToLocal) } - private def uploadCommandScript(command: String, withMonitoring: Boolean, globFiles: Set[WdlGlobFile]): Future[Unit] = { + override def commandLineValueMapper: (WdlValue) => WdlValue = gcsPathToLocal + + private def uploadCommandScript(command: String, withMonitoring: Boolean, globFiles: Set[WdlGlobFile]): Unit = { val monitoring = if (withMonitoring) { s"""|touch $JesMonitoringLogFile |chmod u+x $JesMonitoringScript @@ -270,38 +257,42 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes val tmpDir = File(JesWorkingDisk.MountPoint)./("tmp").path val rcPath = File(JesWorkingDisk.MountPoint)./(returnCodeFilename).path + val rcTmpPath = pathPlusSuffix(rcPath, "tmp").path def globManipulation(globFile: WdlGlobFile) = { - val globDir = callEngineFunctions.globName(globFile.value) + val globDir = backendEngineFunctions.globName(globFile.value) val (_, disk) = relativePathAndAttachedDisk(globFile.value, runtimeAttributes.disks) - val globDirectory = Paths.get(s"${disk.mountPoint.toAbsolutePath}/$globDir/") - val globList = Paths.get(s"${disk.mountPoint.toAbsolutePath}/$globDir.list") - - s""" - |mkdir $globDirectory - |ln ${globFile.value} $globDirectory - |ls -1 $globDirectory > $globList - """.stripMargin + val globDirectory = File(disk.mountPoint)./(globDir) + val globList = File(disk.mountPoint)./(s"$globDir.list") + + s"""|mkdir $globDirectory + |( ln -L ${globFile.value} $globDirectory 2> /dev/null ) || ( ln ${globFile.value} $globDirectory ) + |ls -1 $globDirectory > $globList + |""".stripMargin } val globManipulations = globFiles.map(globManipulation).mkString("\n") val fileContent = - s""" - |#!/bin/bash - |export _JAVA_OPTIONS=-Djava.io.tmpdir=$tmpDir - |export TMPDIR=$tmpDir - |$monitoring - |( - |cd ${JesWorkingDisk.MountPoint} - |$command - |$globManipulations - |) - |echo $$? > $rcPath - """.stripMargin.trim - - Future(File(jesCallPaths.script).write(fileContent)) void + s"""|#!/bin/bash + |export _JAVA_OPTIONS=-Djava.io.tmpdir=$tmpDir + |export TMPDIR=$tmpDir + |$monitoring + |( + |cd ${JesWorkingDisk.MountPoint} + |INSTANTIATED_COMMAND + |) + |echo $$? > $rcTmpPath + |( + |cd ${JesWorkingDisk.MountPoint} + |$globManipulations + |) + |mv $rcTmpPath $rcPath + |""".stripMargin.replace("INSTANTIATED_COMMAND", command) + + File(jesCallPaths.script).write(fileContent) + () } private def googleProject(descriptor: BackendWorkflowDescriptor): String = { @@ -312,13 +303,19 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes descriptor.workflowOptions.getOrElse(WorkflowOptionKeys.GoogleComputeServiceAccount, jesAttributes.computeServiceAccount) } - private def createJesRun(jesParameters: Seq[JesParameter], runIdForResumption: Option[String]): Future[Run] = { + override def isTerminal(runStatus: RunStatus): Boolean = { + runStatus match { + case _: TerminalRunStatus => true + case _ => false + } + } - def createRun() = Future(Run( + private def createJesRun(jesParameters: Seq[JesParameter], runIdForResumption: Option[String]): Run = { + Run( runIdForResumption, jobDescriptor = jobDescriptor, runtimeAttributes = runtimeAttributes, - callRootPath = callRootPath.toUri.toString, + callRootPath = callRootPath.toRealString, commandLine = jesCommandLine, logFileName = jesLogFilename, jesParameters, @@ -326,129 +323,61 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes computeServiceAccount(jobDescriptor.workflowDescriptor), retryable, initializationData.genomics - )) - - implicit val system = context.system - Retry.withRetry( - createRun, - isTransient = isTransientJesException, - isFatal = isFatalJesException - ) andThen { - case Success(run) => - // If this execution represents a resumption don't publish the operation ID since clearly it is already persisted. - runId = Option(run.runId) - if (runIdForResumption.isEmpty) { - serviceRegistryActor ! KvPut(KvPair(ScopedKey(jobDescriptor.workflowDescriptor.id, - KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt), - JesOperationIdKey), runId)) - } - } + ) } - protected def runWithJes(command: String, - jesInputs: Set[JesInput], - jesOutputs: Set[JesFileOutput], - runIdForResumption: Option[String], - withMonitoring: Boolean): Future[ExecutionHandle] = { - - tellStartMetadata() - - val jesParameters = standardParameters ++ gcsAuthParameter ++ jesInputs ++ jesOutputs + override def isFatal(throwable: Throwable): Boolean = isFatalJesException(throwable) - val jesJobSetup = for { - _ <- uploadCommandScript(command, withMonitoring, findGlobOutputs(jobDescriptor)) - run <- createJesRun(jesParameters, runIdForResumption) - _ = tellMetadata(Map(CallMetadataKeys.JobId -> run.runId)) - } yield run + override def isTransient(throwable: Throwable): Boolean = isTransientJesException(throwable) - jesJobSetup map { run => JesPendingExecutionHandle(jobDescriptor, jesOutputs, run, previousStatus = None) } + override def execute(): ExecutionHandle = { + runWithJes(None) } - override def executeOrRecover(mode: ExecutionMode)(implicit ec: ExecutionContext): Future[ExecutionHandle] = { + protected def runWithJes(runIdForResumption: Option[String]): ExecutionHandle = { // Force runtimeAttributes to evaluate so we can fail quickly now if we need to: Try(runtimeAttributes) match { - case Success(_) => startExecuting(monitoringOutput, mode) - case Failure(e) => Future.successful(FailedNonRetryableExecutionHandle(e, None)) + case Success(_) => + val command = instantiatedCommand + val jesInputs: Set[JesInput] = generateJesInputs(jobDescriptor) ++ monitoringScript + cmdInput + val jesOutputs: Set[JesFileOutput] = generateJesOutputs(jobDescriptor) ++ monitoringOutput + val withMonitoring = monitoringOutput.isDefined + + val jesParameters = standardParameters ++ gcsAuthParameter ++ jesInputs ++ jesOutputs + + uploadCommandScript(command, withMonitoring, backendEngineFunctions.findGlobOutputs(call, jobDescriptor)) + val run = createJesRun(jesParameters, runIdForResumption) + PendingExecutionHandle(jobDescriptor, StandardAsyncJob(run.runId), Option(run), previousStatus = None) + case Failure(e) => FailedNonRetryableExecutionHandle(e) } } - private def startExecuting(monitoringOutput: Option[JesFileOutput], mode: ExecutionMode): Future[ExecutionHandle] = { - val jesInputs: Set[JesInput] = generateJesInputs(jobDescriptor) ++ monitoringScript + cmdInput - val jesOutputs: Set[JesFileOutput] = generateJesOutputs(jobDescriptor) ++ monitoringOutput - - instantiateCommand match { - case Success(command) => runWithJes(command, jesInputs, jesOutputs, mode.jobId.collectFirst { case j: JesJobId => j.operationId }, monitoringScript.isDefined) - case Failure(ex: SocketTimeoutException) => Future.successful(FailedNonRetryableExecutionHandle(ex)) - case Failure(ex) => Future.successful(FailedNonRetryableExecutionHandle(ex)) - } + override def pollStatusAsync(handle: JesPendingExecutionHandle) + (implicit ec: ExecutionContext): Future[RunStatus] = { + super[JesPollingActorClient].pollStatus(handle.runInfo.get) } - /** - * Update the ExecutionHandle - */ - override def poll(previous: ExecutionHandle)(implicit ec: ExecutionContext): Future[ExecutionHandle] = { - previous match { - case handle: JesPendingExecutionHandle => - jobLogger.debug(s"$tag Polling JES Job ${handle.run.runId}") - pollStatus(handle.run) map updateExecutionHandleSuccess(handle) recover updateExecutionHandleFailure(handle) flatten - case f: FailedNonRetryableExecutionHandle => f.future - case s: SuccessfulExecutionHandle => s.future - case badHandle => Future.failed(new IllegalArgumentException(s"Unexpected execution handle: $badHandle")) - } + + override def customPollStatusFailure: PartialFunction[(ExecutionHandle, Exception), ExecutionHandle] = { + case (oldHandle: JesPendingExecutionHandle@unchecked, e: GoogleJsonResponseException) if e.getStatusCode == 404 => + jobLogger.error(s"$tag JES Job ID ${oldHandle.runInfo.get.runId} has not been found, failing call") + FailedNonRetryableExecutionHandle(e) } - private def updateExecutionHandleFailure(oldHandle: JesPendingExecutionHandle): PartialFunction[Throwable, Future[ExecutionHandle]] = { - case e: GoogleJsonResponseException if e.getStatusCode == 404 => - jobLogger.error(s"$tag JES Job ID ${oldHandle.run.runId} has not been found, failing call") - FailedNonRetryableExecutionHandle(e).future - case e: Exception => - // Log exceptions and return the original handle to try again. - jobLogger.warn(s"Caught exception, retrying", e) - oldHandle.future - case e: Error => Future.failed(e) // JVM-ending calamity. - case throwable => - // Someone has subclassed Throwable directly? - FailedNonRetryableExecutionHandle(throwable).future - } - - private def updateExecutionHandleSuccess(oldHandle: JesPendingExecutionHandle)(status: RunStatus): Future[ExecutionHandle] = { - val previousStatus = oldHandle.previousStatus - if (!(previousStatus contains status)) { - // If this is the first time checking the status, we log the transition as '-' to 'currentStatus'. Otherwise - // just use the state names. - val prevStateName = previousStatus map { _.toString } getOrElse "-" - jobLogger.info(s"$tag Status change from $prevStateName to $status") - tellMetadata(Map("backendStatus" -> status)) - } + override lazy val startMetadataKeyValues: Map[String, Any] = super[JesJobCachingActorHelper].startMetadataKeyValues - status match { - case s: TerminalRunStatus => - val metadata = Map( - JesMetadataKeys.MachineType -> s.machineType.getOrElse("unknown"), - JesMetadataKeys.InstanceName -> s.instanceName.getOrElse("unknown"), - JesMetadataKeys.Zone -> s.zone.getOrElse("unknown") + override def getTerminalMetadata(runStatus: RunStatus): Map[String, Any] = { + runStatus match { + case terminalRunStatus: TerminalRunStatus => + Map( + JesMetadataKeys.MachineType -> terminalRunStatus.machineType.getOrElse("unknown"), + JesMetadataKeys.InstanceName -> terminalRunStatus.instanceName.getOrElse("unknown"), + JesMetadataKeys.Zone -> terminalRunStatus.zone.getOrElse("unknown") ) - - tellMetadata(metadata) - executionResult(s, oldHandle) - case s => oldHandle.copy(previousStatus = Option(s)).future // Copy the current handle with updated previous status. - + case unknown => throw new RuntimeException(s"Attempt to get terminal metadata from non terminal status: $unknown") } } - /** - * Fire and forget start info to the metadata service - */ - private def tellStartMetadata() = tellMetadata(metadataKeyValues) - - /** - * Fire and forget info to the metadata service - */ - def tellMetadata(metadataKeyValues: Map[String, Any]): Unit = { - import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter - serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), metadataKeyValues) - } - private[jes] def wdlValueToGcsPath(jesOutputs: Set[JesFileOutput])(value: WdlValue): WdlValue = { def toGcsPath(wdlFile: WdlFile) = jesOutputs collectFirst { case o if o.name == makeSafeJesReferenceName(wdlFile.valueString) => WdlFile(o.gcs) @@ -469,12 +398,33 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes OutputEvaluator.evaluateOutputs( jobDescriptor, - callEngineFunctions, + backendEngineFunctions, (wdlValueToSuccess _).compose(wdlValueToGcsPath(generateJesOutputs(jobDescriptor))) ) } - private def handleSuccess(outputMappings: Try[CallOutputs], returnCode: Int, jobDetritusFiles: Map[String, Path], executionHandle: ExecutionHandle, events: Seq[ExecutionEvent]): ExecutionHandle = { + override def isSuccess(runStatus: RunStatus): Boolean = { + runStatus match { + case _: RunStatus.Success => true + case _: RunStatus.Failed => false + case unknown => + throw new RuntimeException("isSuccess not called with RunStatus.Success or RunStatus.Failed. " + + s"Instead got $unknown") + } + } + + override def handleExecutionSuccess(runStatus: RunStatus, + handle: StandardAsyncPendingExecutionHandle, + returnCode: Int): ExecutionHandle = { + val success = runStatus match { + case successStatus: RunStatus.Success => successStatus + case unknown => + throw new RuntimeException(s"handleExecutionSuccess not called with RunStatus.Success. Instead got $unknown") + } + val outputMappings = postProcess + val jobDetritusFiles = jesCallPaths.detritusPaths + val executionHandle = handle + val events = success.eventList outputMappings match { case Success(outputs) => SuccessfulExecutionHandle(outputs, returnCode, jobDetritusFiles, events) case Failure(ex: CromwellAggregatedException) if ex.throwables collectFirst { case s: SocketTimeoutException => s } isDefined => @@ -500,14 +450,23 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes } } - private def handleFailure(errorCode: Int, errorMessage: List[String]) = { + override def handleExecutionFailure(runStatus: RunStatus, + handle: StandardAsyncPendingExecutionHandle): ExecutionHandle = { + val failed = runStatus match { + case failedStatus: RunStatus.Failed => failedStatus + case unknown => + throw new RuntimeException(s"handleExecutionFailure not called with RunStatus.Failed. Instead got $unknown") + } + val errorCode = failed.errorCode + val errorMessage = failed.errorMessage + import lenthall.numeric.IntegerUtil._ val taskName = s"${workflowDescriptor.id}:${call.unqualifiedName}" val attempt = jobDescriptor.key.attempt if (errorMessage.exists(_.contains("Operation canceled at"))) { - AbortedExecutionHandle.future + AbortedExecutionHandle } else if (preempted(errorCode, errorMessage)) { val preemptedMsg = s"Task $taskName was preempted for the ${attempt.toOrdinal} time." @@ -516,56 +475,27 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes s"""$preemptedMsg The call will be restarted with another preemptible VM (max preemptible attempts number is $maxPreemption). |Error code $errorCode. Message: $errorMessage""".stripMargin ) - FailedRetryableExecutionHandle(e, None).future + FailedRetryableExecutionHandle(e, None) } else { val e = PreemptedException( s"""$preemptedMsg The maximum number of preemptible attempts ($maxPreemption) has been reached. The call will be restarted with a non-preemptible VM. |Error code $errorCode. Message: $errorMessage)""".stripMargin) - FailedRetryableExecutionHandle(e, None).future + FailedRetryableExecutionHandle(e, None) } } else { val id = workflowDescriptor.id val name = jobDescriptor.call.unqualifiedName val message = if (errorMessage.isEmpty) "null" else errorMessage.mkString(", ") val exception = new RuntimeException(s"Task $id:$name failed: error code $errorCode. Message: $message") - FailedNonRetryableExecutionHandle(exception, None).future + FailedNonRetryableExecutionHandle(exception, None) } } + // TODO: Adapter for left over test code. Not used by main. private[jes] def executionResult(status: TerminalRunStatus, handle: JesPendingExecutionHandle) - (implicit ec: ExecutionContext): Future[ExecutionHandle] = Future { - try { - lazy val stderrLength: Long = File(jesStderrFile).size - lazy val returnCode = returnCodeContents map { _.trim.toInt } - lazy val continueOnReturnCode = runtimeAttributes.continueOnReturnCode - - status match { - case _: RunStatus.Success if runtimeAttributes.failOnStderr && stderrLength.intValue > 0 => - // returnCode will be None if it couldn't be downloaded/parsed, which will yield a null in the DB - FailedNonRetryableExecutionHandle(new RuntimeException( - s"execution failed: stderr has length $stderrLength"), returnCode.toOption).future - case _: RunStatus.Success if returnCodeContents.isFailure => - val exception = returnCode.failed.get - jobLogger.warn(s"could not download return code file, retrying", exception) - // Return handle to try again. - handle.future - case _: RunStatus.Success if returnCode.isFailure => - FailedNonRetryableExecutionHandle(new RuntimeException( - s"execution failed: could not parse return code as integer: ${returnCodeContents.get}")).future - case _: RunStatus.Success if !continueOnReturnCode.continueFor(returnCode.get) => - val badReturnCodeMessage = s"Call ${jobDescriptor.key.tag}: return code was ${returnCode.getOrElse("(none)")}" - FailedNonRetryableExecutionHandle(new RuntimeException(badReturnCodeMessage), returnCode.toOption).future - case success: RunStatus.Success => - handleSuccess(postProcess, returnCode.get, jesCallPaths.detritusPaths, handle, success.eventList).future - case RunStatus.Failed(errorCode, errorMessage, _, _, _, _) => handleFailure(errorCode, errorMessage) - } - } catch { - case e: Exception => - jobLogger.warn("Caught exception trying to download result, retrying", e) - // Return the original handle to try again. - handle.future - } - } flatten + (implicit ec: ExecutionContext): Future[ExecutionHandle] = { + Future.fromTry(Try(handleExecutionResult(status, handle))) + } /** * Takes a path in GCS and comes up with a local path which is unique for the given GCS path. @@ -604,6 +534,4 @@ class JesAsyncBackendJobExecutionActor(override val jobDescriptor: BackendJobDes case _ => wdlValue } } - - protected implicit def ec: ExecutionContext = context.dispatcher } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala index ac2475e64..14ca8040e 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala @@ -1,17 +1,20 @@ package cromwell.backend.impl.jes -import java.net.URL +import java.net.{URI, URL} -import cats.data._ import cats.data.Validated._ import cats.syntax.cartesian._ -import com.typesafe.config.Config +import com.typesafe.config.{Config, ConfigValue} import cromwell.backend.impl.jes.authentication.JesAuths -import cromwell.core.ErrorOr._ import cromwell.filesystems.gcs.GoogleConfiguration -import lenthall.config.ValidatedConfig._ +import lenthall.validation.Validation._ +import lenthall.exception.MessageAggregation +import lenthall.validation.ErrorOr._ import net.ceedubs.ficus.Ficus._ -import wdl4s.ExceptionWithErrors +import net.ceedubs.ficus.readers.{StringReader, ValueReader} +import org.slf4j.LoggerFactory + +import scala.collection.JavaConversions._ case class JesAttributes(project: String, computeServiceAccount: String, @@ -22,33 +25,39 @@ case class JesAttributes(project: String, qps: Int) object JesAttributes { + lazy val Logger = LoggerFactory.getLogger("JesAttributes") + val GenomicsApiDefaultQps = 1000 private val jesKeys = Set( "project", "root", "maximum-polling-interval", - "compute-service-account", + "genomics.compute-service-account", "dockerhub", "genomics", "filesystems", "genomics.auth", "genomics.endpoint-url", - "filesystems.gcs.auth" + "filesystems.gcs.auth", + "genomics-api-queries-per-100-seconds" ) private val context = "Jes" + implicit val urlReader: ValueReader[URL] = StringReader.stringValueReader.map { URI.create(_).toURL } + def apply(googleConfig: GoogleConfiguration, backendConfig: Config): JesAttributes = { - backendConfig.warnNotRecognized(jesKeys, context) + val configKeys = backendConfig.entrySet().toSet map { entry: java.util.Map.Entry[String, ConfigValue] => entry.getKey } + warnNotRecognized(configKeys, jesKeys, context, Logger) - val project: ValidatedNel[String, String] = backendConfig.validateString("project") - val executionBucket: ValidatedNel[String, String] = backendConfig.validateString("root") - val endpointUrl: ErrorOr[URL] = backendConfig.validateURL("genomics.endpoint-url") + val project: ErrorOr[String] = validate { backendConfig.as[String]("project") } + val executionBucket: ErrorOr[String] = validate { backendConfig.as[String]("root") } + val endpointUrl: ErrorOr[URL] = validate { backendConfig.as[URL]("genomics.endpoint-url") } val maxPollingInterval: Int = backendConfig.as[Option[Int]]("maximum-polling-interval").getOrElse(600) val computeServiceAccount: String = backendConfig.as[Option[String]]("genomics.compute-service-account").getOrElse("default") - val genomicsAuthName: ErrorOr[String] = backendConfig.validateString("genomics.auth") - val gcsFilesystemAuthName: ErrorOr[String] = backendConfig.validateString("filesystems.gcs.auth") + val genomicsAuthName: ErrorOr[String] = validate { backendConfig.as[String]("genomics.auth") } + val gcsFilesystemAuthName: ErrorOr[String] = validate { backendConfig.as[String]("filesystems.gcs.auth") } val qps = backendConfig.as[Option[Int]]("genomics-api-queries-per-100-seconds").getOrElse(GenomicsApiDefaultQps) / 100 @@ -61,9 +70,9 @@ object JesAttributes { } match { case Valid(r) => r case Invalid(f) => - throw new IllegalArgumentException with ExceptionWithErrors { - override val message = "Jes Configuration is not valid: Errors" - override val errors = f + throw new IllegalArgumentException with MessageAggregation { + override val exceptionContext = "Jes Configuration is not valid: Errors" + override val errorMessages = f.toList } } } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala index 5c1275ae1..5099cc172 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala @@ -1,6 +1,7 @@ package cromwell.backend.impl.jes import akka.actor.{Actor, ActorLogging, Props} +import cromwell.core.Dispatcher.BackendDispatcher import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll @@ -16,5 +17,5 @@ final case class JesBackendSingletonActor(qps: Int) extends Actor with ActorLogg } object JesBackendSingletonActor { - def props(qps: Int): Props = Props(JesBackendSingletonActor(qps)) + def props(qps: Int): Props = Props(JesBackendSingletonActor(qps)).withDispatcher(BackendDispatcher) } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala index a77dd66c7..b37a9ff31 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala @@ -3,8 +3,9 @@ package cromwell.backend.impl.jes import java.nio.file.Path import akka.actor.{ActorRef, Props} +import cromwell.core.Dispatcher.BackendDispatcher import cromwell.backend.callcaching.CacheHitDuplicating -import cromwell.backend.{BackendCacheHitCopyingActor, BackendJobDescriptor} +import cromwell.backend.{BackendCacheHitCopyingActor, BackendConfigurationDescriptor, BackendJobDescriptor, BackendWorkflowDescriptor} import cromwell.core.path.PathCopier import cromwell.core.logging.JobLogging @@ -13,13 +14,15 @@ case class JesCacheHitCopyingActor(override val jobDescriptor: BackendJobDescrip initializationData: JesBackendInitializationData, serviceRegistryActor: ActorRef) extends BackendCacheHitCopyingActor with CacheHitDuplicating with JesJobCachingActorHelper with JobLogging { - override protected def duplicate(source: Path, destination: Path) = PathCopier.copy(source, destination).get + override protected def duplicate(source: Path, destination: Path): Unit = PathCopier.copy(source, destination).get - override protected def destinationCallRootPath = jesCallPaths.callExecutionRoot + override protected lazy val destinationCallRootPath: Path = jesCallPaths.callExecutionRoot - override protected def destinationJobDetritusPaths = jesCallPaths.detritusPaths + override protected lazy val destinationJobDetritusPaths: Map[String, Path] = jesCallPaths.detritusPaths - override val workflowDescriptor = jobDescriptor.workflowDescriptor + override val workflowDescriptor: BackendWorkflowDescriptor = jobDescriptor.workflowDescriptor + + override lazy val configurationDescriptor: BackendConfigurationDescriptor = jesConfiguration.configurationDescriptor } object JesCacheHitCopyingActor { @@ -28,6 +31,6 @@ object JesCacheHitCopyingActor { jesConfiguration: JesConfiguration, initializationData: JesBackendInitializationData, serviceRegistryActor: ActorRef): Props = { - Props(new JesCacheHitCopyingActor(jobDescriptor, jesConfiguration, initializationData, serviceRegistryActor)) + Props(new JesCacheHitCopyingActor(jobDescriptor, jesConfiguration, initializationData, serviceRegistryActor)).withDispatcher(BackendDispatcher) } } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala index bf29a387f..8e16b08a5 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala @@ -1,34 +1,27 @@ package cromwell.backend.impl.jes -import java.nio.file.{Files, Path} +import java.nio.file.Path +import cromwell.backend.io.GlobFunctions import cromwell.backend.wdl.{ReadLikeFunctions, WriteFunctions} import cromwell.core.CallContext import cromwell.core.path.PathBuilder import cromwell.filesystems.gcs.GcsPathBuilder +import cromwell.core.path.PathImplicits._ import wdl4s.expression.{PureStandardLibraryFunctionsLike, WdlStandardLibraryFunctions} import wdl4s.values._ -import scala.collection.JavaConverters._ import scala.util.{Success, Try} class JesExpressionFunctions(override val pathBuilders: List[PathBuilder], context: CallContext) - extends WdlStandardLibraryFunctions with PureStandardLibraryFunctionsLike with ReadLikeFunctions with WriteFunctions { + extends WdlStandardLibraryFunctions with PureStandardLibraryFunctionsLike with ReadLikeFunctions with WriteFunctions with GlobFunctions { - override def writeTempFile(path: String, prefix: String, suffix: String, content: String): String = super[WriteFunctions].writeTempFile(path, prefix, suffix, content) - private[jes] def globDirectory(glob: String): String = globName(glob) + "/" - private[jes] def globName(glob: String) = s"glob-${glob.md5Sum}" - - override def globPath(glob: String): String = context.root.resolve(globDirectory(glob)).toString + def callContext: CallContext = context - override def glob(path: String, pattern: String): Seq[String] = { - val name = globName(pattern) - val listFile = context.root.resolve(s"$name.list").toRealPath() - Files.readAllLines(listFile).asScala map { fileName => context.root.resolve(s"$name/$fileName").toUri.toString } - } + override def writeTempFile(path: String, prefix: String, suffix: String, content: String): String = super[WriteFunctions].writeTempFile(path, prefix, suffix, content) override def preMapping(str: String): String = if (!GcsPathBuilder.isValidGcsUrl(str)) { - context.root.resolve(str.stripPrefix("/")).toUri.toString + context.root.resolve(str.stripPrefix("/")).toRealString } else str override def stdout(params: Seq[Try[WdlValue]]) = Success(WdlFile(context.stdout)) diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala index fc2d141be..f8dbf0953 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala @@ -6,6 +6,7 @@ import akka.actor.Props import better.files._ import cats.instances.future._ import cats.syntax.functor._ +import cromwell.core.Dispatcher.BackendDispatcher import cromwell.backend.{BackendWorkflowDescriptor, BackendWorkflowFinalizationActor, JobExecutionMap} import cromwell.core.CallOutputs import cromwell.core.Dispatcher.IoDispatcher @@ -18,7 +19,12 @@ import scala.language.postfixOps object JesFinalizationActor { def props(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall], jesConfiguration: JesConfiguration, jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs, initializationData: Option[JesBackendInitializationData]) = { - Props(new JesFinalizationActor(workflowDescriptor, calls, jesConfiguration, jobExecutionMap, workflowOutputs, initializationData)) + Props(new JesFinalizationActor(workflowDescriptor, + calls, + jesConfiguration, + jobExecutionMap, + workflowOutputs, + initializationData)).withDispatcher(BackendDispatcher) } } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala index 03cae0dff..685d2aec3 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala @@ -6,6 +6,7 @@ import akka.actor.{ActorRef, Props} import cats.instances.future._ import cats.syntax.functor._ import com.google.api.services.genomics.Genomics +import cromwell.core.Dispatcher.BackendDispatcher import cromwell.backend.impl.jes.JesInitializationActor._ import cromwell.backend.impl.jes.authentication.{GcsLocalizing, JesAuthInformation} import cromwell.backend.impl.jes.io._ @@ -31,7 +32,7 @@ object JesInitializationActor { calls: Set[TaskCall], jesConfiguration: JesConfiguration, serviceRegistryActor: ActorRef): Props = - Props(new JesInitializationActor(workflowDescriptor, calls, jesConfiguration, serviceRegistryActor: ActorRef)) + Props(new JesInitializationActor(workflowDescriptor, calls, jesConfiguration, serviceRegistryActor: ActorRef)).withDispatcher(BackendDispatcher) } class JesInitializationActor(override val workflowDescriptor: BackendWorkflowDescriptor, diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala index fe39df840..f24551d71 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala @@ -8,6 +8,7 @@ import cromwell.backend.BackendWorkflowDescriptor import cromwell.backend.callcaching.JobCachingActorHelper import cromwell.backend.impl.jes.io.{JesAttachedDisk, JesWorkingDisk} import cromwell.core.logging.JobLogging +import cromwell.core.path.PathImplicits._ import scala.util.Try @@ -17,8 +18,8 @@ trait JesJobCachingActorHelper extends JobCachingActorHelper { val ExecParamName = "exec" val MonitoringParamName = "monitoring" - val JesMonitoringScript = JesWorkingDisk.MountPoint.resolve("monitoring.sh") - val JesMonitoringLogFile = JesWorkingDisk.MountPoint.resolve("monitoring.log") + val JesMonitoringScript: Path = JesWorkingDisk.MountPoint.resolve("monitoring.sh") + val JesMonitoringLogFile: Path = JesWorkingDisk.MountPoint.resolve("monitoring.log") def jesConfiguration: JesConfiguration @@ -30,9 +31,7 @@ trait JesJobCachingActorHelper extends JobCachingActorHelper { def getPath(str: String): Try[Path] = jesCallPaths.getPath(str) - override lazy val configurationDescriptor = jesConfiguration.configurationDescriptor - - lazy val jesCallPaths = { + lazy val jesCallPaths: JesJobPaths = { val workflowPaths = if (workflowDescriptor.breadCrumbs.isEmpty) { initializationData.workflowPaths } else { @@ -44,33 +43,33 @@ trait JesJobCachingActorHelper extends JobCachingActorHelper { lazy val runtimeAttributes = JesRuntimeAttributes(jobDescriptor.runtimeAttributes, jobLogger) - lazy val retryable = jobDescriptor.key.attempt <= runtimeAttributes.preemptible + lazy val retryable: Boolean = jobDescriptor.key.attempt <= runtimeAttributes.preemptible lazy val workingDisk: JesAttachedDisk = runtimeAttributes.disks.find(_.name == JesWorkingDisk.Name).get lazy val callRootPath: Path = jesCallPaths.callExecutionRoot - lazy val returnCodeFilename = jesCallPaths.returnCodeFilename - lazy val returnCodeGcsPath = jesCallPaths.returnCode - lazy val jesStdoutFile = jesCallPaths.stdout - lazy val jesStderrFile = jesCallPaths.stderr - lazy val jesLogFilename = jesCallPaths.jesLogFilename - lazy val defaultMonitoringOutputPath = callRootPath.resolve(JesMonitoringLogFile) - - lazy val maxPreemption = runtimeAttributes.preemptible + lazy val returnCodeFilename: String = jesCallPaths.returnCodeFilename + lazy val returnCodeGcsPath: Path = jesCallPaths.returnCode + lazy val jesStdoutFile: Path = jesCallPaths.stdout + lazy val jesStderrFile: Path = jesCallPaths.stderr + lazy val jesLogFilename: String = jesCallPaths.jesLogFilename + lazy val defaultMonitoringOutputPath: Path = callRootPath.resolve(JesMonitoringLogFile) + + lazy val maxPreemption: Int = runtimeAttributes.preemptible lazy val preemptible: Boolean = jobDescriptor.key.attempt <= maxPreemption - lazy val jesAttributes = jesConfiguration.jesAttributes + lazy val jesAttributes: JesAttributes = jesConfiguration.jesAttributes lazy val monitoringScript: Option[JesInput] = { jesCallPaths.monitoringPath map { path => - JesFileInput(s"$MonitoringParamName-in", path.toUri.toString, + JesFileInput(s"$MonitoringParamName-in", path.toRealString, JesWorkingDisk.MountPoint.resolve(JesMonitoringScript), workingDisk) } } - lazy val monitoringOutput = monitoringScript map { _ => JesFileOutput(s"$MonitoringParamName-out", + lazy val monitoringOutput: Option[JesFileOutput] = monitoringScript map { _ => JesFileOutput(s"$MonitoringParamName-out", defaultMonitoringOutputPath.toString, File(JesMonitoringLogFile).path, workingDisk) } - // Implements CacheHitDuplicating.metadataKeyValues - lazy val metadataKeyValues: Map[String, Any] = { + // Implements CacheHitDuplicating.startMetadataKeyValues + def startMetadataKeyValues: Map[String, Any] = { val runtimeAttributesMetadata: Map[String, Any] = runtimeAttributes.asMap map { case (key, value) => s"runtimeAttributes:$key" -> value } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala index 476ee04aa..44d85cbfd 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala @@ -1,109 +1,61 @@ package cromwell.backend.impl.jes -import akka.actor.SupervisorStrategy.{Decider, Stop} -import akka.actor.{ActorRef, OneForOneStrategy, Props} -import akka.event.LoggingReceive -import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse} -import cromwell.backend.BackendLifecycleActor.AbortJobCommand +import akka.actor.{ActorRef, Props} import cromwell.backend._ -import cromwell.backend.async.AsyncBackendJobExecutionActor.{Execute, Recover} -import cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.JesJobId -import cromwell.backend.impl.jes.JesJobExecutionActor._ -import cromwell.services.keyvalue.KeyValueServiceActor._ -import org.slf4j.LoggerFactory - -import scala.concurrent.{Future, Promise} +import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardSyncExecutionActor, StandardSyncExecutionActorParams} +import cromwell.core.Dispatcher.BackendDispatcher + +/** A default implementation of the sync params. */ +case class JesSyncExecutionActorParams +( + override val jobDescriptor: BackendJobDescriptor, + jesConfiguration: JesConfiguration, + jesBackendInitializationData: JesBackendInitializationData, + override val serviceRegistryActor: ActorRef, + jesBackendSingletonActorOption: Option[ActorRef] +) extends StandardSyncExecutionActorParams { + override val jobIdKey: String = JesJobExecutionActor.JesOperationIdKey + override val asyncJobExecutionActorClass: Class[_ <: StandardAsyncExecutionActor] = classOf[Nothing] + override val configurationDescriptor: BackendConfigurationDescriptor = jesConfiguration.configurationDescriptor + override val backendInitializationDataOption: Option[BackendInitializationData] = Option(jesBackendInitializationData) +} object JesJobExecutionActor { - val logger = LoggerFactory.getLogger("JesBackend") - def props(jobDescriptor: BackendJobDescriptor, jesWorkflowInfo: JesConfiguration, initializationData: JesBackendInitializationData, serviceRegistryActor: ActorRef, jesBackendSingletonActor: Option[ActorRef]): Props = { - Props(new JesJobExecutionActor(jobDescriptor, jesWorkflowInfo, initializationData, serviceRegistryActor, jesBackendSingletonActor)) + val params = JesSyncExecutionActorParams( + jobDescriptor, + jesWorkflowInfo, + initializationData, + serviceRegistryActor, + jesBackendSingletonActor) + Props(new JesJobExecutionActor(params)).withDispatcher(BackendDispatcher) } val JesOperationIdKey = "__jes_operation_id" } -case class JesJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, - jesConfiguration: JesConfiguration, - initializationData: JesBackendInitializationData, - serviceRegistryActor: ActorRef, - jesBackendSingletonActorOption: Option[ActorRef]) - extends BackendJobExecutionActor { - - val jesBackendSingletonActor = jesBackendSingletonActorOption.getOrElse(throw new RuntimeException("JES Backend actor cannot exist without the JES backend singleton actor")) - - private def jesReceiveBehavior: Receive = LoggingReceive { - case AbortJobCommand => - executor.foreach(_ ! AbortJobCommand) - case abortResponse: AbortedResponse => - context.parent ! abortResponse - context.stop(self) - case KvPair(key, id @ Some(operationId)) if key.key == JesOperationIdKey => - // Successful operation ID lookup during recover. - executor foreach { _ ! Recover(JesJobId(operationId))} - case KvKeyLookupFailed(_) => - // Missed operation ID lookup during recover, fall back to execute. - executor foreach { _ ! Execute } - case KvFailure(_, e) => - // Failed operation ID lookup during recover, crash and let the supervisor deal with it. - completionPromise.tryFailure(e) - throw new RuntimeException("Failure attempting to look up JES operation ID for key " + jobDescriptor.key, e) - } - - override def receive = jesReceiveBehavior orElse super.receive - - override val configurationDescriptor = jesConfiguration.configurationDescriptor - - private lazy val completionPromise = Promise[BackendJobExecutionResponse]() - - private var executor: Option[ActorRef] = None - - private[jes] def jabjeaProps = JesAsyncBackendJobExecutionActor.props(jobDescriptor, - completionPromise, - jesConfiguration, - initializationData, - serviceRegistryActor, - jesBackendSingletonActor) - - private def launchExecutor: Future[Unit] = Future { - val executionProps = jabjeaProps - val executorRef = context.actorOf(executionProps, "JesAsyncBackendJobExecutionActor") - executor = Option(executorRef) - () - } - - override def recover: Future[BackendJobExecutionResponse] = { - import JesJobExecutionActor._ - - for { - _ <- launchExecutor - _ = serviceRegistryActor ! KvGet(ScopedKey(jobDescriptor.workflowDescriptor.id, - KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt), - JesOperationIdKey)) - c <- completionPromise.future - } yield c - } - - override def execute: Future[BackendJobExecutionResponse] = { - for { - _ <- launchExecutor - _ = executor foreach { _ ! Execute } - c <- completionPromise.future - } yield c - } - - override def abort(): Unit = {} - - // Supervision strategy: if the JABJEA throws an exception, stop the actor and fail the job. - def jobFailingDecider: Decider = { - case e: Exception => - completionPromise.tryFailure(new RuntimeException("JesAsyncBackendJobExecutionActor failed and didn't catch its exception.", e)) - Stop +case class JesJobExecutionActor(jesParams: JesSyncExecutionActorParams) + extends StandardSyncExecutionActor(jesParams) { + + override def createAsyncRefName(): String = "JesAsyncBackendJobExecutionActor" + + override def createAsyncProps(): Props = jabjeaProps + + private[jes] def jabjeaProps = { + Props( + new JesAsyncBackendJobExecutionActor( + JesAsyncExecutionActorParams( + jesParams.jobDescriptor, + jesParams.jesConfiguration, + jesParams.jesBackendInitializationData, + jesParams.serviceRegistryActor, + jesParams.jesBackendSingletonActorOption, + completionPromise) + ) + ) } - override val supervisorStrategy = OneForOneStrategy()(jobFailingDecider) } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala index 593d1e6df..374ce05b9 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala @@ -9,9 +9,8 @@ import cromwell.backend.validation.RuntimeAttributesDefault._ import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.backend.validation.RuntimeAttributesValidation._ import cromwell.backend.validation._ -import cromwell.core._ import lenthall.exception.MessageAggregation -import cromwell.core.ErrorOr._ +import lenthall.validation.ErrorOr._ import org.slf4j.Logger import wdl4s.types._ import wdl4s.values._ diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala index a7ac5e50a..7feade96d 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala @@ -9,6 +9,7 @@ import cromwell.backend.io.WorkflowPaths import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} import cromwell.core.WorkflowOptions import cromwell.core.path.PathBuilder +import cromwell.core.path.PathImplicits._ import cromwell.filesystems.gcs.{GcsPathBuilderFactory, RetryableGcsPathBuilder} import scala.language.postfixOps @@ -42,7 +43,7 @@ class JesWorkflowPaths(val workflowDescriptor: BackendWorkflowDescriptor, // The default auth file bucket is always at the root of the root workflow val defaultBucket = executionRoot.resolve(workflowDescriptor.rootWorkflow.unqualifiedName).resolve(workflowDescriptor.rootWorkflowId.toString) - val bucket = workflowDescriptor.workflowOptions.get(JesWorkflowPaths.AuthFilePathOptionKey) getOrElse defaultBucket.toUri.toString + val bucket = workflowDescriptor.workflowOptions.get(JesWorkflowPaths.AuthFilePathOptionKey) getOrElse defaultBucket.toRealString val authBucket = GcsPathBuilderFactory(genomicsCredentials).withOptions(workflowOptions).build(bucket) recover { case ex => throw new Exception(s"Invalid gcs auth_bucket path $bucket", ex) } get diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala index c1ce80b4f..4066c1f41 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala @@ -6,8 +6,8 @@ import cats.data.Validated._ import cats.syntax.cartesian._ import cats.syntax.validated._ import com.google.api.services.genomics.model.Disk -import cromwell.core.ErrorOr._ -import wdl4s.ExceptionWithErrors +import lenthall.validation.ErrorOr._ +import lenthall.exception.MessageAggregation import wdl4s.values._ import scala.util.Try @@ -36,9 +36,9 @@ object JesAttachedDisk { Try(validation match { case Valid(localDisk) => localDisk case Invalid(nels) => - throw new UnsupportedOperationException with ExceptionWithErrors { - val message = "" - val errors = nels + throw new UnsupportedOperationException with MessageAggregation { + val exceptionContext = "" + val errorMessages = nels.toList } }) } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala index 12fe055dc..9d7a7226c 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala @@ -4,6 +4,7 @@ import akka.actor.{Actor, ActorLogging, ActorRef, Props, SupervisorStrategy, Ter import cats.data.NonEmptyList import cromwell.backend.impl.jes.Run import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager._ +import cromwell.core.Dispatcher.BackendDispatcher import scala.collection.immutable.Queue @@ -99,7 +100,7 @@ class JesApiQueryManager(val qps: Int) extends Actor with ActorLogging { object JesApiQueryManager { - def props(qps: Int): Props = Props(new JesApiQueryManager(qps)) + def props(qps: Int): Props = Props(new JesApiQueryManager(qps)).withDispatcher(BackendDispatcher) /** * Poll the job represented by the Run. diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala index 31a9d114a..9a0d55182 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala @@ -10,6 +10,7 @@ import com.google.api.services.genomics.model.Operation import cromwell.backend.impl.jes.{JesAttributes, Run} import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.{JesPollingWorkBatch, JesStatusPollQuery, NoWorkToDo} import cromwell.backend.impl.jes.statuspolling.JesPollingActor._ +import cromwell.core.Dispatcher.BackendDispatcher import scala.collection.JavaConversions._ import scala.concurrent.{ExecutionContext, Future, Promise} @@ -127,7 +128,7 @@ class JesPollingActor(val pollingManager: ActorRef, val qps: Int) extends Actor } object JesPollingActor { - def props(pollingManager: ActorRef, qps: Int) = Props(new JesPollingActor(pollingManager, qps)) + def props(pollingManager: ActorRef, qps: Int) = Props(new JesPollingActor(pollingManager, qps)).withDispatcher(BackendDispatcher) // The Batch API limits us to 100 at a time val MaxBatchSize = 100 diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala index 40078cac6..bbd713343 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala @@ -14,9 +14,11 @@ import cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.JesPendingExec import cromwell.backend.impl.jes.RunStatus.Failed import cromwell.backend.impl.jes.io.{DiskType, JesWorkingDisk} import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll +import cromwell.backend.standard.StandardAsyncJob import cromwell.core.logging.JobLogger +import cromwell.core.path.PathImplicits._ import cromwell.core.{WorkflowId, WorkflowOptions, _} -import cromwell.filesystems.gcs.GcsPathBuilderFactory +import cromwell.filesystems.gcs.{GcsPathBuilder, GcsPathBuilderFactory} import cromwell.filesystems.gcs.auth.GoogleAuthMode.NoAuthMode import cromwell.util.SampleWdl import org.scalatest._ @@ -35,13 +37,13 @@ import scala.util.{Success, Try} class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackendJobExecutionActorSpec") with FlatSpecLike with Matchers with ImplicitSender with Mockito with BackendSpec { - val mockPathBuilder = GcsPathBuilderFactory(NoAuthMode).withOptions(mock[WorkflowOptions]) + val mockPathBuilder: GcsPathBuilder = GcsPathBuilderFactory(NoAuthMode).withOptions(mock[WorkflowOptions]) import JesTestConfig._ - implicit val Timeout = 5.seconds.dilated + implicit val Timeout: FiniteDuration = 5.seconds.dilated - val YoSup = + val YoSup: String = s""" |task sup { | String addressee @@ -68,7 +70,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val TestableCallContext = CallContext(mockPathBuilder.build("gs://root").get, "out", "err") - val TestableJesExpressionFunctions = { + val TestableJesExpressionFunctions: JesExpressionFunctions = { new JesExpressionFunctions(List(mockPathBuilder), TestableCallContext) } @@ -77,25 +79,39 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend JesBackendInitializationData(workflowPaths, null) } - class TestableJesJobExecutionActor(jobDescriptor: BackendJobDescriptor, - promise: Promise[BackendJobExecutionResponse], - jesConfiguration: JesConfiguration, - functions: JesExpressionFunctions = TestableJesExpressionFunctions, - jesSingletonActor: ActorRef = emptyActor) - extends JesAsyncBackendJobExecutionActor(jobDescriptor, promise, jesConfiguration, buildInitializationData(jobDescriptor, jesConfiguration), emptyActor, jesSingletonActor) { + class TestableJesJobExecutionActor(jesParams: JesAsyncExecutionActorParams, functions: JesExpressionFunctions) + extends JesAsyncBackendJobExecutionActor(jesParams) { + + def this(jobDescriptor: BackendJobDescriptor, + promise: Promise[BackendJobExecutionResponse], + jesConfiguration: JesConfiguration, + functions: JesExpressionFunctions = TestableJesExpressionFunctions, + jesSingletonActor: ActorRef = emptyActor) = { + this( + JesAsyncExecutionActorParams( + jobDescriptor, + jesConfiguration, + buildInitializationData(jobDescriptor, jesConfiguration), + emptyActor, + Option(jesSingletonActor), + promise + ), + functions + ) + } override lazy val jobLogger = new JobLogger("TestLogger", workflowId, jobTag, akkaLogger = Option(log)) { override def tag: String = s"$name [UUID(${workflowId.shortString})$jobTag]" override val slf4jLoggers: Set[Logger] = Set.empty } - override lazy val callEngineFunctions = functions + override lazy val backendEngineFunctions: JesExpressionFunctions = functions } private val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) private val workingDisk = JesWorkingDisk(DiskType.SSD, 200) - val DockerAndDiskRuntime = + val DockerAndDiskRuntime: String = """ |runtime { | docker: "ubuntu:latest" @@ -124,7 +140,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend // Mock/stub out the bits that would reach out to JES. val run = mock[Run] - val handle = JesPendingExecutionHandle(jobDescriptor, Set.empty, run, None) + val handle = new JesPendingExecutionHandle(jobDescriptor, StandardAsyncJob(run.runId), Option(run), None) class ExecuteOrRecoverActor extends TestableJesJobExecutionActor(jobDescriptor, promise, jesConfiguration, jesSingletonActor = jesSingletonActor) { override def executeOrRecover(mode: ExecutionMode)(implicit ec: ExecutionContext): Future[ExecutionHandle] = Future.successful(handle) @@ -593,13 +609,13 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val jesBackend = testActorRef.underlyingActor jesBackend.jesCallPaths.stdout should be(a[CloudStoragePath]) - jesBackend.jesCallPaths.stdout.toUri.toString shouldBe + jesBackend.jesCallPaths.stdout.toRealString shouldBe "gs://path/to/gcs_root/wf_hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello-stdout.log" jesBackend.jesCallPaths.stderr should be(a[CloudStoragePath]) - jesBackend.jesCallPaths.stderr.toUri.toString shouldBe + jesBackend.jesCallPaths.stderr.toRealString shouldBe "gs://path/to/gcs_root/wf_hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello-stderr.log" jesBackend.jesCallPaths.jesLogPath should be(a[CloudStoragePath]) - jesBackend.jesCallPaths.jesLogPath.toUri.toString shouldBe + jesBackend.jesCallPaths.jesLogPath.toRealString shouldBe "gs://path/to/gcs_root/wf_hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello.log" } @@ -624,13 +640,13 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend val jesBackend = testActorRef.underlyingActor jesBackend.jesCallPaths.stdout should be(a[CloudStoragePath]) - jesBackend.jesCallPaths.stdout.toUri.toString shouldBe + jesBackend.jesCallPaths.stdout.toRealString shouldBe "gs://path/to/gcs_root/w/e6236763-c518-41d0-9688-432549a8bf7d/call-B/shard-2/B-2-stdout.log" jesBackend.jesCallPaths.stderr should be(a[CloudStoragePath]) - jesBackend.jesCallPaths.stderr.toUri.toString shouldBe + jesBackend.jesCallPaths.stderr.toRealString shouldBe "gs://path/to/gcs_root/w/e6236763-c518-41d0-9688-432549a8bf7d/call-B/shard-2/B-2-stderr.log" jesBackend.jesCallPaths.jesLogPath should be(a[CloudStoragePath]) - jesBackend.jesCallPaths.jesLogPath.toUri.toString shouldBe + jesBackend.jesCallPaths.jesLogPath.toRealString shouldBe "gs://path/to/gcs_root/w/e6236763-c518-41d0-9688-432549a8bf7d/call-B/shard-2/B-2.log" } diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala index 0eec23cbe..00eeb24fd 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala @@ -5,8 +5,8 @@ import java.net.URL import com.typesafe.config.ConfigFactory import cromwell.core.Tags._ import cromwell.filesystems.gcs.GoogleConfiguration +import lenthall.exception.MessageAggregation import org.scalatest.{FlatSpec, Matchers} -import wdl4s.ExceptionWithErrors class JesAttributesSpec extends FlatSpec with Matchers { @@ -58,18 +58,18 @@ class JesAttributesSpec extends FlatSpec with Matchers { val googleConfig = GoogleConfiguration(JesGlobalConfig) - val exception = intercept[IllegalArgumentException with ExceptionWithErrors] { + val exception = intercept[IllegalArgumentException with MessageAggregation] { JesAttributes(googleConfig, nakedConfig) } - val errorsList = exception.errors.toList - errorsList should contain("Could not find key: project") - errorsList should contain("Could not find key: root") - errorsList should contain("Could not find key: genomics.auth") - errorsList should contain("Could not find key: filesystems.gcs.auth") - errorsList should contain("no protocol: myEndpoint") + val errorsList = exception.errorMessages.toList + errorsList should contain("No configuration setting found for key 'project'") + errorsList should contain("No configuration setting found for key 'root'") + errorsList should contain("No configuration setting found for key 'genomics.auth'") + errorsList should contain("No configuration setting found for key 'filesystems'") + errorsList should contain("URI is not absolute") } - def configString(preemptible: String = "", genomics: String = "") = + def configString(preemptible: String = "", genomics: String = ""): String = s""" |{ | project = "myProject" diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala index 58841f756..7682ffb7b 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala @@ -2,6 +2,7 @@ package cromwell.backend.impl.jes import cromwell.backend.BackendSpec import cromwell.core.TestKitSuite +import cromwell.core.path.PathImplicits._ import cromwell.util.SampleWdl import org.scalatest.{FlatSpecLike, Matchers} import org.specs2.mock.Mockito @@ -32,13 +33,13 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) val callPaths = JesJobPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration) - callPaths.returnCode.toUri.toString should + callPaths.returnCode.toRealString should be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello-rc.txt") - callPaths.stdout.toUri.toString should + callPaths.stdout.toRealString should be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello-stdout.log") - callPaths.stderr.toUri.toString should + callPaths.stderr.toRealString should be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello-stderr.log") - callPaths.jesLogPath.toUri.toString should + callPaths.jesLogPath.toRealString should be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello.log") } @@ -48,7 +49,7 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) val callPaths = JesJobPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration) - callPaths.callContext.root.toUri.toString should + callPaths.callContext.root.toRealString should be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello") callPaths.callContext.stdout should be("hello-stdout.log") callPaths.callContext.stderr should be("hello-stderr.log") diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala index 2c0853718..b12703c23 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala @@ -31,8 +31,10 @@ class JesJobExecutionActorSpec extends TestKitSuite("JesJobExecutionActorSpec") val parent = TestProbe() val deathwatch = TestProbe() + val params = JesSyncExecutionActorParams(jobDescriptor, jesWorkflowInfo, initializationData, serviceRegistryActor, + jesBackendSingletonActor) val testJJEA = TestActorRef[TestJesJobExecutionActor]( - props = Props(new TestJesJobExecutionActor(jobDescriptor, jesWorkflowInfo, initializationData, serviceRegistryActor, jesBackendSingletonActor, Props(new ConstructorFailingJABJEA))), + props = Props(new TestJesJobExecutionActor(params, Props(new ConstructorFailingJABJEA))), supervisor = parent.ref) deathwatch watch testJJEA @@ -43,8 +45,8 @@ class JesJobExecutionActorSpec extends TestKitSuite("JesJobExecutionActorSpec") testJJEA.tell(msg = ExecuteJobCommand, sender = parent.ref) parent.expectMsgPF(max = TimeoutDuration) { - case JobFailedNonRetryableResponse(jobKey, e, errorCode) => - e.getMessage should be("JesAsyncBackendJobExecutionActor failed and didn't catch its exception.") + case JobFailedNonRetryableResponse(_, throwable, _) => + throwable.getMessage should be("JesAsyncBackendJobExecutionActor failed and didn't catch its exception.") } } @@ -58,8 +60,10 @@ class JesJobExecutionActorSpec extends TestKitSuite("JesJobExecutionActorSpec") val parent = TestProbe() val deathwatch = TestProbe() val jabjeaConstructionPromise = Promise[ActorRef]() + val params = JesSyncExecutionActorParams(jobDescriptor, jesWorkflowInfo, initializationData, serviceRegistryActor, + jesBackendSingletonActor) val testJJEA = TestActorRef[TestJesJobExecutionActor]( - props = Props(new TestJesJobExecutionActor(jobDescriptor, jesWorkflowInfo, initializationData, serviceRegistryActor, jesBackendSingletonActor, Props(new ControllableFailingJabjea(jabjeaConstructionPromise)))), + props = Props(new TestJesJobExecutionActor(params, Props(new ControllableFailingJabjea(jabjeaConstructionPromise)))), supervisor = parent.ref) deathwatch watch testJJEA @@ -75,18 +79,14 @@ class JesJobExecutionActorSpec extends TestKitSuite("JesJobExecutionActorSpec") jabjeaConstructionPromise.future foreach { _ ! JabjeaExplode } parent.expectMsgPF(max = TimeoutDuration) { - case JobFailedNonRetryableResponse(jobKey, e, errorCode) => - e.getMessage should be("JesAsyncBackendJobExecutionActor failed and didn't catch its exception.") + case JobFailedNonRetryableResponse(_, throwable, _) => + throwable.getMessage should be("JesAsyncBackendJobExecutionActor failed and didn't catch its exception.") } } } -class TestJesJobExecutionActor(jobDescriptor: BackendJobDescriptor, - jesWorkflowInfo: JesConfiguration, - initializationData: JesBackendInitializationData, - serviceRegistryActor: ActorRef, - jesBackendSingletonActor: Option[ActorRef], - fakeJabjeaProps: Props) extends JesJobExecutionActor(jobDescriptor, jesWorkflowInfo, initializationData, serviceRegistryActor, jesBackendSingletonActor) { +class TestJesJobExecutionActor(jesParams: JesSyncExecutionActorParams, + fakeJabjeaProps: Props) extends JesJobExecutionActor(jesParams) { override def jabjeaProps: Props = fakeJabjeaProps } @@ -96,12 +96,12 @@ class ConstructorFailingJABJEA extends ControllableFailingJabjea(Promise[ActorRe } class ControllableFailingJabjea(constructionPromise: Promise[ActorRef]) extends Actor { - def explode() = { + def explode(): Unit = { val boom = 1 == 1 if (boom) throw new RuntimeException("Test Exception! Don't panic if this appears during a test run!") } constructionPromise.trySuccess(self) - override def receive = { + override def receive: Receive = { case JabjeaExplode => explode() } } diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala index 48dd3d74c..9826b4ef0 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala @@ -2,6 +2,7 @@ package cromwell.backend.impl.jes import cromwell.backend.BackendSpec import cromwell.core.TestKitSuite +import cromwell.core.path.PathImplicits._ import cromwell.util.SampleWdl import org.scalatest.{FlatSpecLike, Matchers} import org.specs2.mock.Mockito @@ -17,10 +18,10 @@ class JesWorkflowPathsSpec extends TestKitSuite with FlatSpecLike with Matchers val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor) val workflowPaths = JesWorkflowPaths(workflowDescriptor, jesConfiguration)(system) - workflowPaths.executionRoot.toUri.toString should be("gs://my-cromwell-workflows-bucket/") - workflowPaths.workflowRoot.toUri.toString should + workflowPaths.executionRoot.toRealString should be("gs://my-cromwell-workflows-bucket/") + workflowPaths.workflowRoot.toRealString should be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/") - workflowPaths.gcsAuthFilePath.toUri.toString should + workflowPaths.gcsAuthFilePath.toRealString should be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/${workflowDescriptor.id}_auth.json") } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala index a47209c20..12ca089eb 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala @@ -4,8 +4,9 @@ import java.nio.file.Path import better.files._ import cromwell.backend.impl.sfs.config.ConfigConstants._ -import cromwell.backend.sfs.SharedFileSystem._ import cromwell.backend.sfs._ +import cromwell.backend.standard.{StandardAsyncExecutionActorParams, StandardAsyncJob} +import cromwell.core.path.PathFactory._ import wdl4s._ import wdl4s.expression.NoFunctions import wdl4s.values.WdlString @@ -19,10 +20,7 @@ import wdl4s.values.WdlString */ sealed trait ConfigAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecutionActor { - lazy val configInitializationData: ConfigInitializationData = params.backendInitializationDataOption match { - case Some(data: ConfigInitializationData) => data - case other => throw new RuntimeException(s"Unable to get config initialization data from $other") - } + lazy val configInitializationData: ConfigInitializationData = backendInitializationDataAs[ConfigInitializationData] /** * Returns the arguments for submitting the job, either with or without docker. @@ -110,18 +108,18 @@ $command /** * Submits a job and sends it to the background via "&". Saves the unix PID for status or killing later. * - * @param params Params for running a shared file system job. + * @param standardParams Params for running a shared file system job. */ -class BackgroundConfigAsyncJobExecutionActor(override val params: SharedFileSystemAsyncJobExecutionActorParams) +class BackgroundConfigAsyncJobExecutionActor(override val standardParams: StandardAsyncExecutionActorParams) extends ConfigAsyncJobExecutionActor with BackgroundAsyncJobExecutionActor /** * Submits a job and returns relatively quickly. The job-id-regex is then used to read the job id for status or killing * later. * - * @param params Params for running a shared file system job. + * @param standardParams Params for running a shared file system job. */ -class DispatchedConfigAsyncJobExecutionActor(override val params: SharedFileSystemAsyncJobExecutionActorParams) +class DispatchedConfigAsyncJobExecutionActor(override val standardParams: StandardAsyncExecutionActorParams) extends ConfigAsyncJobExecutionActor { /** @@ -132,11 +130,11 @@ class DispatchedConfigAsyncJobExecutionActor(override val params: SharedFileSyst * @param stderr The stderr from dispatching the job. * @return The wrapped job id. */ - override def getJob(exitValue: Int, stdout: Path, stderr: Path): SharedFileSystemJob = { + override def getJob(exitValue: Int, stdout: Path, stderr: Path): StandardAsyncJob = { val jobIdRegex = configurationDescriptor.backendConfig.getString(JobIdRegexConfig).r val output = File(stdout).contentAsString.stripLineEnd output match { - case jobIdRegex(jobId) => SharedFileSystemJob(jobId) + case jobIdRegex(jobId) => StandardAsyncJob(jobId) case _ => throw new RuntimeException("Could not find job ID from stdout file. " + s"Check the stderr file for possible errors: $stderr") @@ -149,7 +147,7 @@ class DispatchedConfigAsyncJobExecutionActor(override val params: SharedFileSyst * @param job The job to check. * @return A command that checks if the job is alive. */ - override def checkAliveArgs(job: SharedFileSystemJob): SharedFileSystemCommand = { + override def checkAliveArgs(job: StandardAsyncJob): SharedFileSystemCommand = { jobScriptArgs(job, "check", CheckAliveTask) } @@ -159,7 +157,7 @@ class DispatchedConfigAsyncJobExecutionActor(override val params: SharedFileSyst * @param job The job id to kill. * @return A command that may be used to kill the job. */ - override def killArgs(job: SharedFileSystemJob): SharedFileSystemCommand = { + override def killArgs(job: StandardAsyncJob): SharedFileSystemCommand = { jobScriptArgs(job, "kill", KillTask) } @@ -171,7 +169,7 @@ class DispatchedConfigAsyncJobExecutionActor(override val params: SharedFileSyst * @param task The config task that defines the command. * @return A runnable command. */ - private def jobScriptArgs(job: SharedFileSystemJob, suffix: String, task: String): SharedFileSystemCommand = { + private def jobScriptArgs(job: StandardAsyncJob, suffix: String, task: String): SharedFileSystemCommand = { val script = pathPlusSuffix(jobPaths.script, suffix) writeTaskScript(script, task, Map(JobIdInput -> WdlString(job.jobId))) SharedFileSystemCommand("/bin/bash", script) diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala index 37a42a5c3..707f1a93f 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala @@ -33,16 +33,19 @@ object DeclarationValidation { new MemoryDeclarationValidation(declaration) // All other declarations must be a Boolean, Float, Integer, or String. case _ => - val validator: PrimitiveRuntimeAttributesValidation[_] = declaration.wdlType match { - case WdlBooleanType => new BooleanRuntimeAttributesValidation(declaration.unqualifiedName) - case WdlFloatType => new FloatRuntimeAttributesValidation(declaration.unqualifiedName) - case WdlIntegerType => new IntRuntimeAttributesValidation(declaration.unqualifiedName) - case WdlStringType => new StringRuntimeAttributesValidation(declaration.unqualifiedName) - case other => throw new RuntimeException(s"Unsupported config runtime attribute $other ${declaration.unqualifiedName}") - } - new DeclarationValidation(declaration, validator) + val validatedRuntimeAttr = validator(declaration.wdlType, declaration.unqualifiedName) + new DeclarationValidation(declaration, validatedRuntimeAttr) } } + + private def validator(wdlType: WdlType, unqualifiedName: String): PrimitiveRuntimeAttributesValidation[_] = wdlType match { + case WdlBooleanType => new BooleanRuntimeAttributesValidation(unqualifiedName) + case WdlFloatType => new FloatRuntimeAttributesValidation(unqualifiedName) + case WdlIntegerType => new IntRuntimeAttributesValidation(unqualifiedName) + case WdlStringType => new StringRuntimeAttributesValidation(unqualifiedName) + case WdlOptionalType(x) => validator(x, unqualifiedName) + case other => throw new RuntimeException(s"Unsupported config runtime attribute $other $unqualifiedName") + } } /** diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala index 8ac1e2c21..8367ffb2a 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala @@ -3,7 +3,8 @@ package cromwell.backend.sfs import java.nio.file.Path import better.files._ -import cromwell.backend.sfs.SharedFileSystem._ +import cromwell.backend.standard.StandardAsyncJob +import cromwell.core.path.PathFactory._ trait BackgroundAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecutionActor { @@ -49,22 +50,22 @@ trait BackgroundAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecution () } - override def getJob(exitValue: Int, stdout: Path, stderr: Path) = { + override def getJob(exitValue: Int, stdout: Path, stderr: Path): StandardAsyncJob = { val pid = File(stdout).contentAsString.stripLineEnd - SharedFileSystemJob(pid) + StandardAsyncJob(pid) } - override def checkAliveArgs(job: SharedFileSystemJob) = { + override def checkAliveArgs(job: StandardAsyncJob): SharedFileSystemCommand = { SharedFileSystemCommand("ps", job.jobId) } - override def killArgs(job: SharedFileSystemJob) = { + override def killArgs(job: StandardAsyncJob): SharedFileSystemCommand = { val killScript = pathPlusSuffix(jobPaths.script, "kill") writeKillScript(killScript, job) SharedFileSystemCommand("/bin/bash", killScript) } - private def writeKillScript(killScript: File, job: SharedFileSystemJob): Unit = { + private def writeKillScript(killScript: File, job: StandardAsyncJob): Unit = { /* Use pgrep to find the children of a process, and recursively kill the children before killing the parent. */ diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala index 188391c45..fc1c4168d 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala @@ -1,5 +1,6 @@ package cromwell.backend.sfs +import java.io.{FileNotFoundException, IOException} import java.nio.file.{Path, Paths} import cats.instances.try_._ @@ -7,9 +8,10 @@ import cats.syntax.functor._ import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import cromwell.backend.io.JobPaths -import cromwell.core._ +import cromwell.core.CromwellFatalExceptionMarker import cromwell.core.path.PathFactory -import cromwell.util.TryUtil +import cromwell.core.path.PathFactory._ +import lenthall.util.TryUtil import wdl4s.EvaluatedTaskInputs import wdl4s.types.{WdlArrayType, WdlMapType} import wdl4s.values._ @@ -22,7 +24,7 @@ object SharedFileSystem extends StrictLogging { import better.files._ final case class AttemptedLookupResult(name: String, value: Try[WdlValue]) { - def toPair = name -> value + def toPair: (String, Try[WdlValue]) = name -> value } object AttemptedLookupResult { @@ -46,7 +48,7 @@ object SharedFileSystem extends StrictLogging { private def localizePathViaCopy(originalPath: File, executionPath: File): Try[Unit] = { val action = Try { executionPath.parent.createDirectories() - val executionTmpPath = pathPlusSuffix(executionPath, ".tmp") + val executionTmpPath = pathPlusSuffix(executionPath, "tmp") originalPath.copyTo(executionTmpPath, overwrite = true).moveTo(executionPath, overwrite = true) }.void logOnFailure(action, "copy") @@ -62,6 +64,7 @@ object SharedFileSystem extends StrictLogging { private def localizePathViaSymbolicLink(originalPath: File, executionPath: File): Try[Unit] = { if (originalPath.isDirectory) Failure(new UnsupportedOperationException("Cannot localize directory with symbolic links")) + else if (!originalPath.exists) Failure(new FileNotFoundException(originalPath.pathAsString)) else { val action = Try { executionPath.parent.createDirectories() @@ -76,15 +79,14 @@ object SharedFileSystem extends StrictLogging { action } - private def duplicate(description: String, source: File, dest: File, strategies: Stream[DuplicationStrategy]) = { + private def duplicate(description: String, source: File, dest: File, strategies: Stream[DuplicationStrategy]): Try[Unit] = { import cromwell.util.FileUtil._ - strategies.map(_ (source.followSymlinks, dest)).find(_.isSuccess) getOrElse { - Failure(new UnsupportedOperationException(s"Could not $description $source -> $dest")) + val attempts: Stream[Try[Unit]] = strategies.map(_ (source.followSymlinks, dest)) + attempts.find(_.isSuccess) getOrElse { + TryUtil.sequence(attempts, s"Could not $description $source -> $dest").void } } - - def pathPlusSuffix(path: File, suffix: String) = path.sibling(s"${path.name}.$suffix") } trait SharedFileSystem extends PathFactory { @@ -95,12 +97,12 @@ trait SharedFileSystem extends PathFactory { lazy val DefaultStrategies = Seq("hard-link", "soft-link", "copy") - lazy val LocalizationStrategies = getConfigStrategies("localization") - lazy val Localizers = createStrategies(LocalizationStrategies, docker = false) - lazy val DockerLocalizers = createStrategies(LocalizationStrategies, docker = true) + lazy val LocalizationStrategies: Seq[String] = getConfigStrategies("localization") + lazy val Localizers: Seq[DuplicationStrategy] = createStrategies(LocalizationStrategies, docker = false) + lazy val DockerLocalizers: Seq[DuplicationStrategy] = createStrategies(LocalizationStrategies, docker = true) - lazy val CachingStrategies = getConfigStrategies("caching.duplication-strategy") - lazy val Cachers = createStrategies(CachingStrategies, docker = false) + lazy val CachingStrategies: Seq[String] = getConfigStrategies("caching.duplication-strategy") + lazy val Cachers: Seq[DuplicationStrategy] = createStrategies(CachingStrategies, docker = false) private def getConfigStrategies(configPath: String): Seq[String] = { if (sharedFileSystemConfig.hasPath(configPath)) { @@ -130,7 +132,7 @@ trait SharedFileSystem extends PathFactory { } private def hostAbsoluteFilePath(callRoot: Path, pathString: String): File = { - val wdlPath = Paths.get(pathString) + val wdlPath = PathFactory.buildPath(pathString, pathBuilders) callRoot.resolve(wdlPath).toAbsolutePath } @@ -194,8 +196,8 @@ trait SharedFileSystem extends PathFactory { case (declaration, value) => localizeFunction(value) map { declaration -> _ } } - TryUtil.sequence(localizedValues, "Failures during localization").map(_.toMap) recover { - case e => throw new CromwellFatalException(e) + TryUtil.sequence(localizedValues, "Failures during localization").map(_.toMap) recoverWith { + case e => Failure(new IOException(e.getMessage) with CromwellFatalExceptionMarker) } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala index ef5ef5e89..f0a0814c3 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala @@ -2,46 +2,29 @@ package cromwell.backend.sfs import java.nio.file.{FileAlreadyExistsException, Path} -import akka.actor.{Actor, ActorLogging, ActorRef} -import akka.event.LoggingReceive +import akka.actor.{Actor, ActorLogging} import better.files._ -import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse -import cromwell.backend.BackendLifecycleActor.AbortJobCommand -import cromwell.backend.async.AsyncBackendJobExecutionActor._ -import cromwell.backend.async.{AbortedExecutionHandle, AsyncBackendJobExecutionActor, ExecutionHandle, FailedNonRetryableExecutionHandle, NonRetryableExecution, SuccessfulExecutionHandle} +import cromwell.backend._ +import cromwell.backend.async.{AsyncBackendJobExecutionActor, ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle, SuccessfulExecutionHandle} import cromwell.backend.io.WorkflowPathsBackendInitializationData -import cromwell.backend.sfs.SharedFileSystem._ +import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncJob} import cromwell.backend.validation._ -import cromwell.backend.wdl.{OutputEvaluator, Command} -import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor} -import cromwell.core.CallOutputs -import cromwell.core.logging.JobLogging -import cromwell.core.path.DefaultPathBuilder +import cromwell.backend.wdl.OutputEvaluator +import cromwell.core.WorkflowId +import cromwell.core.path.PathFactory._ +import cromwell.core.path.{DefaultPathBuilder, PathBuilder} import cromwell.core.retry.SimpleExponentialBackoff -import cromwell.services.keyvalue.KeyValueServiceActor._ -import wdl4s.values.{WdlArray, WdlFile, WdlMap, WdlValue} +import wdl4s.values.{WdlArray, WdlFile, WdlGlobFile, WdlMap, WdlValue} +import wdl4s.{EvaluatedTaskInputs, TaskCall} import scala.concurrent.duration._ -import scala.concurrent.{ExecutionContext, Future, Promise} import scala.util.{Failure, Success, Try} object SharedFileSystemJob { val JobIdKey = "sfs_job_id" } -/** - * A generic job that runs and tracks some string identifier for the job. - */ -case class SharedFileSystemJob(jobId: String) extends JobId - -case class SharedFileSystemAsyncJobExecutionActorParams -( - serviceRegistryActor: ActorRef, - jobDescriptor: BackendJobDescriptor, - configurationDescriptor: BackendConfigurationDescriptor, - completionPromise: Promise[BackendJobExecutionResponse], - backendInitializationDataOption: Option[BackendInitializationData] -) +case class SharedFileSystemRunStatus(returnCodeFileExists: Boolean) /** * Runs a job on a shared backend, with the ability to (abstractly) submit asynchronously, then poll, kill, etc. @@ -68,29 +51,17 @@ case class SharedFileSystemAsyncJobExecutionActorParams * messages. */ trait SharedFileSystemAsyncJobExecutionActor - extends Actor with ActorLogging with AsyncBackendJobExecutionActor with SharedFileSystemJobCachingActorHelper - with JobLogging { - - case class SharedFileSystemPendingExecutionHandle(jobDescriptor: BackendJobDescriptor, - run: SharedFileSystemJob) extends ExecutionHandle { - override val isDone = false - override val result = NonRetryableExecution(new IllegalStateException( - "SharedFileSystemPendingExecutionHandle cannot yield a result")) - } + extends Actor with ActorLogging with BackendJobLifecycleActor with AsyncBackendJobExecutionActor + with StandardAsyncExecutionActor with SharedFileSystemJobCachingActorHelper { - context.become(sharedReceive(None) orElse super.receive) + override type StandardAsyncRunInfo = Any - val SIGTERM = 143 - val SIGINT = 130 + override type StandardAsyncRunStatus = SharedFileSystemRunStatus override lazy val pollBackOff = SimpleExponentialBackoff(1.second, 5.minutes, 1.1) override lazy val executeOrRecoverBackOff = SimpleExponentialBackoff(3.seconds, 30.seconds, 1.1) - override protected implicit def ec = context.dispatcher - - val params: SharedFileSystemAsyncJobExecutionActorParams - /** * Returns the command for running the job. The returned command may or may not run the job asynchronously in the * background. If the command does not run the script asynchronously in the background or on some job scheduler, the @@ -109,7 +80,7 @@ trait SharedFileSystemAsyncJobExecutionActor * @param stderr The stderr of the submit. * @return The job id wrapped in a SharedFileSystemJob. */ - def getJob(exitValue: Int, stdout: Path, stderr: Path): SharedFileSystemJob + def getJob(exitValue: Int, stdout: Path, stderr: Path): StandardAsyncJob /** * Returns the command for checking if a job is alive, returing non-zero if the job cannot be found or has errored. @@ -117,7 +88,7 @@ trait SharedFileSystemAsyncJobExecutionActor * @param job The job to check. * @return The command for checking if a job is alive. */ - def checkAliveArgs(job: SharedFileSystemJob): SharedFileSystemCommand + def checkAliveArgs(job: StandardAsyncJob): SharedFileSystemCommand /** * Returns the command for killing a job. @@ -125,21 +96,11 @@ trait SharedFileSystemAsyncJobExecutionActor * @param job The job to kill. * @return The command for killing a job. */ - def killArgs(job: SharedFileSystemJob): SharedFileSystemCommand - - override lazy val jobDescriptor = params.jobDescriptor - - override lazy val completionPromise = params.completionPromise - - override lazy val serviceRegistryActor = params.serviceRegistryActor - - override lazy val configurationDescriptor = params.configurationDescriptor - - override lazy val backendInitializationDataOption = params.backendInitializationDataOption + def killArgs(job: StandardAsyncJob): SharedFileSystemCommand def toUnixPath(docker: Boolean)(path: WdlValue): WdlValue = { path match { - case file: WdlFile => + case _: WdlFile => val cleanPath = DefaultPathBuilder.build(path.valueString).get WdlFile(if (docker) jobPaths.toDockerPath(cleanPath).toString else cleanPath.toString) case array: WdlArray => WdlArray(array.wdlType, array.value map toUnixPath(docker)) @@ -150,73 +111,33 @@ trait SharedFileSystemAsyncJobExecutionActor def jobName: String = s"cromwell_${jobDescriptor.workflowDescriptor.id.shortString}_${jobDescriptor.call.unqualifiedName}" - override def retryable = false + lazy val workflowDescriptor: BackendWorkflowDescriptor = jobDescriptor.workflowDescriptor + lazy val call: TaskCall = jobDescriptor.key.call + lazy val pathBuilders: List[PathBuilder] = WorkflowPathsBackendInitializationData.pathBuilders(backendInitializationDataOption) + private[sfs] lazy val backendEngineFunctions = SharedFileSystemExpressionFunctions(jobPaths, pathBuilders) + override lazy val workflowId: WorkflowId = jobDescriptor.workflowDescriptor.id + override lazy val jobTag: String = jobDescriptor.key.tag - lazy val workflowDescriptor = jobDescriptor.workflowDescriptor - lazy val call = jobDescriptor.key.call - lazy val pathBuilders = WorkflowPathsBackendInitializationData.pathBuilders(backendInitializationDataOption) - lazy val callEngineFunction = SharedFileSystemExpressionFunctions(jobPaths, pathBuilders) - override lazy val workflowId = jobDescriptor.workflowDescriptor.id - override lazy val jobTag = jobDescriptor.key.tag - - lazy val isDockerRun = RuntimeAttributesValidation.extractOption( + lazy val isDockerRun: Boolean = RuntimeAttributesValidation.extractOption( DockerValidation.instance, validatedRuntimeAttributes).isDefined - def sharedReceive(jobOption: Option[SharedFileSystemJob]): Receive = LoggingReceive { - case AbortJobCommand => - jobOption foreach tryKill - case KvPutSuccess(_) => // expected after the KvPut in tellKvJobId - } + override lazy val commandLineFunctions: SharedFileSystemExpressionFunctions = backendEngineFunctions - def instantiatedScript: String = { - val pathTransformFunction = toUnixPath(isDockerRun) _ - val localizer = sharedFileSystem.localizeInputs(jobPaths.callInputsRoot, isDockerRun) _ - - Command.instantiate( - jobDescriptor, - callEngineFunction, - localizer, - pathTransformFunction - ) match { - case Success(command) => command - case Failure(ex) => throw new RuntimeException("Failed to instantiate command line", ex) - } - } + override lazy val commandLinePreProcessor: (EvaluatedTaskInputs) => Try[EvaluatedTaskInputs] = + sharedFileSystem.localizeInputs(jobPaths.callInputsRoot, isDockerRun) - override def executeOrRecover(mode: ExecutionMode)(implicit ec: ExecutionContext) = { - // Run now in receive, not in yet another Runnable. - Future.fromTry(Try { - mode match { - case Execute => - tellMetadata(metadataKeyValues) - executeScript() - case Recover(recoveryId) => - recoveryId match { - case job: SharedFileSystemJob => recoverScript(job) - case other => throw new RuntimeException(s"Unable to recover $other") - } - } - } recoverWith { - case exception: Exception => - jobLogger.error(s"Error attempting to $mode the script", exception) - Failure(exception) - }) - } + override lazy val commandLineValueMapper: (WdlValue) => WdlValue = toUnixPath(isDockerRun) - /** - * Fire and forget info to the metadata service - */ - def tellMetadata(metadataKeyValues: Map[String, Any]): Unit = { - import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter - serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), metadataKeyValues) + override lazy val startMetadataKeyValues: Map[String, Any] = { + super[SharedFileSystemJobCachingActorHelper].startMetadataKeyValues } - def executeScript(): ExecutionHandle = { - val script = instantiatedScript + override def execute(): ExecutionHandle = { + val script = instantiatedCommand jobLogger.info(s"`$script`") File(jobPaths.callExecutionRoot).createDirectories() val cwd = if (isDockerRun) jobPaths.callExecutionDockerRoot else jobPaths.callExecutionRoot - writeScript(script, cwd) + writeScript(script, cwd, backendEngineFunctions.findGlobOutputs(call, jobDescriptor)) jobLogger.info(s"command: $processArgs") val runner = makeProcessRunner() val exitValue = runner.run() @@ -225,11 +146,7 @@ trait SharedFileSystemAsyncJobExecutionActor s"Check the stderr file for possible errors: ${runner.stderrPath}")) } else { val runningJob = getJob(exitValue, runner.stdoutPath, runner.stderrPath) - context.become(sharedReceive(Option(runningJob)) orElse super.receive) - tellKvJobId(runningJob) - jobLogger.info(s"job id: ${runningJob.jobId}") - tellMetadata(Map("jobId" -> runningJob.jobId)) - SharedFileSystemPendingExecutionHandle(jobDescriptor, runningJob) + PendingExecutionHandle(jobDescriptor, runningJob, None, None) } } @@ -250,48 +167,49 @@ trait SharedFileSystemAsyncJobExecutionActor * Writes the script file containing the user's command from the WDL as well * as some extra shell code for monitoring jobs */ - private def writeScript(instantiatedCommand: String, cwd: Path) = { + private def writeScript(instantiatedCommand: String, cwd: Path, globFiles: Set[WdlGlobFile]) = { val rcPath = if (isDockerRun) jobPaths.toDockerPath(jobPaths.returnCode) else jobPaths.returnCode - val rcTmpPath = s"$rcPath.tmp" + val rcTmpPath = pathPlusSuffix(rcPath, "tmp").path - val scriptBody = s""" + def globManipulation(globFile: WdlGlobFile) = { -#!/bin/sh -( - cd $cwd - $instantiatedCommand -) -echo $$? > $rcTmpPath -mv $rcTmpPath $rcPath + // TODO: Move glob list and directory generation into trait GlobFunctions? There is already a globPath using callContext + val globDir = backendEngineFunctions.globName(globFile.value) + val globDirectory = File(cwd)./(globDir) + val globList = File(cwd)./(s"$globDir.list") -""".trim + "\n" + s"""|mkdir $globDirectory + |( ln -L ${globFile.value} $globDirectory 2> /dev/null ) || ( ln ${globFile.value} $globDirectory ) + |ls -1 $globDirectory > $globList + |""".stripMargin + } - File(jobPaths.script).write(scriptBody) - } + val globManipulations = globFiles.map(globManipulation).mkString("\n") + + val scriptBody = + s"""|#!/bin/sh + |( + |cd $cwd + |INSTANTIATED_COMMAND + |) + |echo $$? > $rcTmpPath + |( + |cd $cwd + |$globManipulations + |) + |mv $rcTmpPath $rcPath + |""".stripMargin.replace("INSTANTIATED_COMMAND", instantiatedCommand) - /** - * Send the job id of the running job to the key value store. - * - * @param runningJob The running job. - */ - private def tellKvJobId(runningJob: SharedFileSystemJob): Unit = { - val kvJobKey = - KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt) - val scopedKey = ScopedKey(jobDescriptor.workflowDescriptor.id, kvJobKey, SharedFileSystemJob.JobIdKey) - val kvValue = Option(runningJob.jobId) - val kvPair = KvPair(scopedKey, kvValue) - val kvPut = KvPut(kvPair) - serviceRegistryActor ! kvPut + File(jobPaths.script).write(scriptBody) } - def recoverScript(job: SharedFileSystemJob): ExecutionHandle = { - context.become(sharedReceive(Option(job)) orElse super.receive) + override def recover(job: StandardAsyncJob): ExecutionHandle = { // To avoid race conditions, check for the rc file after checking if the job is alive. if (isAlive(job) || File(jobPaths.returnCode).exists) { // If we're done, we'll get to the rc during the next poll. // Or if we're still running, return pending also. jobLogger.info(s"Recovering using job id: ${job.jobId}") - SharedFileSystemPendingExecutionHandle(jobDescriptor, job) + PendingExecutionHandle(jobDescriptor, job, None, None) } else { // Could start executeScript(), but for now fail because we shouldn't be in this state. FailedNonRetryableExecutionHandle(new RuntimeException( @@ -299,7 +217,7 @@ mv $rcTmpPath $rcPath } } - def isAlive(job: SharedFileSystemJob): Boolean = { + def isAlive(job: StandardAsyncJob): Boolean = { val argv = checkAliveArgs(job).argv val stdout = pathPlusSuffix(jobPaths.stdout, "check") val stderr = pathPlusSuffix(jobPaths.stderr, "check") @@ -307,7 +225,7 @@ mv $rcTmpPath $rcPath checkAlive.run() == 0 } - def tryKill(job: SharedFileSystemJob): Unit = { + override def tryAbort(job: StandardAsyncJob): Unit = { val returnCodeTmp = pathPlusSuffix(jobPaths.returnCode, "kill") returnCodeTmp.write(s"$SIGTERM\n") try { @@ -325,68 +243,32 @@ mv $rcTmpPath $rcPath () } - def processReturnCode()(implicit ec: ExecutionContext): Future[ExecutionHandle] = { - val returnCodeTry = Try(File(jobPaths.returnCode).contentAsString.stripLineEnd.toInt) - - lazy val badReturnCodeMessage = s"Call ${jobDescriptor.key.tag}: return code was ${returnCodeTry.getOrElse("(none)")}" + override def remoteStdErrPath: Path = jobPaths.stderr - lazy val badReturnCodeResponse = Future.successful( - FailedNonRetryableExecutionHandle(new Exception(badReturnCodeMessage), returnCodeTry.toOption)) + override def remoteReturnCodePath: Path = jobPaths.returnCode - lazy val abortResponse = Future.successful(AbortedExecutionHandle) + override def continueOnReturnCode: ContinueOnReturnCode = RuntimeAttributesValidation.extract( + ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes) - def processSuccess(returnCode: Int) = { - val successfulFuture = for { - outputs <- Future.fromTry(processOutputs()) - } yield SuccessfulExecutionHandle(outputs, returnCode, jobPaths.detritusPaths, Seq.empty) + override def failOnStdErr: Boolean = RuntimeAttributesValidation.extract( + FailOnStderrValidation.instance, validatedRuntimeAttributes) - successfulFuture recover { - case failed: Throwable => - FailedNonRetryableExecutionHandle(failed, Option(returnCode)) - } - } - - def stopFor(returnCode: Int) = { - val continueOnReturnCode = RuntimeAttributesValidation.extract( - ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes) - !continueOnReturnCode.continueFor(returnCode) - } - - def failForStderr = { - val failOnStderr = RuntimeAttributesValidation.extract( - FailOnStderrValidation.instance, validatedRuntimeAttributes) - failOnStderr && File(jobPaths.stderr).size > 0 - } + override def pollStatus(handle: StandardAsyncPendingExecutionHandle): SharedFileSystemRunStatus = { + SharedFileSystemRunStatus(File(jobPaths.returnCode).exists) + } - returnCodeTry match { - case Success(SIGTERM) => abortResponse // Special case to check for SIGTERM exit code - implying abort - case Success(SIGINT) => abortResponse // Special case to check for SIGINT exit code - implying abort - case Success(returnCode) if stopFor(returnCode) => badReturnCodeResponse - case Success(returnCode) if failForStderr => badReturnCodeResponse - case Success(returnCode) => processSuccess(returnCode) - case Failure(e) => badReturnCodeResponse - } + override def isTerminal(runStatus: StandardAsyncRunStatus): Boolean = { + runStatus.returnCodeFileExists } - override def poll(previous: ExecutionHandle)(implicit ec: ExecutionContext) = { - previous match { - case handle: SharedFileSystemPendingExecutionHandle => - val runId = handle.run - jobLogger.debug(s"Polling Job $runId") - File(jobPaths.returnCode).exists match { - case true => - processReturnCode() - case false => - jobLogger.debug(s"'${jobPaths.returnCode}' file does not exist yet") - Future.successful(previous) - } - case failed: FailedNonRetryableExecutionHandle => Future.successful(failed) - case successful: SuccessfulExecutionHandle => Future.successful(successful) - case bad => Future.failed(new IllegalArgumentException(s"Unexpected execution handle: $bad")) + override def handleExecutionSuccess(runStatus: StandardAsyncRunStatus, handle: StandardAsyncPendingExecutionHandle, + returnCode: Int): ExecutionHandle = { + val outputsTry = + OutputEvaluator.evaluateOutputs(jobDescriptor, backendEngineFunctions, sharedFileSystem.outputMapper(jobPaths)) + outputsTry match { + case Success(outputs) => SuccessfulExecutionHandle(outputs, returnCode, jobPaths.detritusPaths, Seq.empty) + case Failure(throwable) => FailedNonRetryableExecutionHandle(throwable, Option(returnCode)) } } - private def processOutputs(): Try[CallOutputs] = { - OutputEvaluator.evaluateOutputs(jobDescriptor, callEngineFunction, sharedFileSystem.outputMapper(jobPaths)) - } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala index 1ac3093dc..43d77dd5d 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala @@ -2,8 +2,8 @@ package cromwell.backend.sfs import akka.actor.{ActorRef, Props} import cats.data.Validated.{Invalid, Valid} -import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse -import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, BackendJobDescriptorKey, BackendLifecycleActorFactory, BackendWorkflowDescriptor} +import cromwell.backend._ +import cromwell.backend.standard.StandardLifecycleActorFactory import cromwell.core.Dispatcher import cromwell.core.Dispatcher._ import cromwell.core.path.{DefaultPathBuilderFactory, PathBuilderFactory} @@ -13,14 +13,14 @@ import net.ceedubs.ficus.Ficus._ import wdl4s.TaskCall import wdl4s.expression.WdlStandardLibraryFunctions -import scala.concurrent.Promise - /** * A factory that can be extended for any shared file system implementation. * * See the SharedFileSystemAsyncJobExecutionActor for more info. */ -trait SharedFileSystemBackendLifecycleActorFactory extends BackendLifecycleActorFactory { +trait SharedFileSystemBackendLifecycleActorFactory extends StandardLifecycleActorFactory { + + override def jobIdKey: String = SharedFileSystemJob.JobIdKey /** * If the backend sets a gcs authentication mode, try to create a PathBuilderFactory with it. @@ -31,6 +31,7 @@ trait SharedFileSystemBackendLifecycleActorFactory extends BackendLifecycleActor case Valid(auth) => GcsPathBuilderFactory(auth) case Invalid(error) => throw new MessageAggregation { override def exceptionContext: String = "Failed to parse gcs auth configuration" + override def errorMessages: Traversable[String] = error.toList } } @@ -40,51 +41,20 @@ trait SharedFileSystemBackendLifecycleActorFactory extends BackendLifecycleActor lazy val pathBuilderFactories: List[PathBuilderFactory] = List(gcsPathBuilderFactory, Option(DefaultPathBuilderFactory)).flatten /** - * Config values for the backend, and a pointer to the global config. - * - * This is the single parameter passed into each factory during creation. - * - * @return The backend configuration. - */ - def configurationDescriptor: BackendConfigurationDescriptor - - /** * Returns the initialization class, or by default uses the `SharedFileSystemInitializationActor`. * * @return the initialization class. */ def initializationActorClass: Class[_ <: SharedFileSystemInitializationActor] = - classOf[SharedFileSystemInitializationActor] - - /** - * Returns the main engine for async execution. - * - * @return the main engine for async execution. - */ - def asyncJobExecutionActorClass: Class[_ <: SharedFileSystemAsyncJobExecutionActor] + classOf[SharedFileSystemInitializationActor] override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall], - serviceRegistryActor: ActorRef) = { + serviceRegistryActor: ActorRef): Option[Props] = { val params = SharedFileSystemInitializationActorParams(serviceRegistryActor, workflowDescriptor, configurationDescriptor, calls, pathBuilderFactories) Option(Props(initializationActorClass, params).withDispatcher(Dispatcher.BackendDispatcher)) } - override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, - initializationDataOption: Option[BackendInitializationData], - serviceRegistryActor: ActorRef, - backendSingletonActor: Option[ActorRef]) = { - def propsCreator(completionPromise: Promise[BackendJobExecutionResponse]): Props = { - val params = SharedFileSystemAsyncJobExecutionActorParams(serviceRegistryActor, jobDescriptor, - configurationDescriptor, completionPromise, initializationDataOption) - Props(asyncJobExecutionActorClass, params).withDispatcher(Dispatcher.BackendDispatcher) - } - - Props(new SharedFileSystemJobExecutionActor( - jobDescriptor, configurationDescriptor, serviceRegistryActor, propsCreator) - ).withDispatcher(Dispatcher.BackendDispatcher) - } - override def cacheHitCopyingActorProps = Option(cacheHitCopyingActorInner _) def cacheHitCopyingActorInner(jobDescriptor: BackendJobDescriptor, diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala index 7dc5172ba..859b98830 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala @@ -2,15 +2,14 @@ package cromwell.backend.sfs import java.nio.file.Path -import cromwell.backend.io.{JobPaths, JobPathsWithDocker, WorkflowPathsBackendInitializationData} +import cromwell.backend.io._ import cromwell.backend.wdl._ -import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptorKey, BackendWorkflowDescriptor} +import cromwell.backend._ import cromwell.core.CallContext import cromwell.core.path.PathBuilder import wdl4s.expression.PureStandardLibraryFunctionsLike -import wdl4s.values.{WdlFile, WdlValue} +import wdl4s.values._ -import scala.language.postfixOps import scala.util.{Success, Try} object SharedFileSystemExpressionFunctions { @@ -57,15 +56,12 @@ object SharedFileSystemExpressionFunctions { class SharedFileSystemExpressionFunctions(override val pathBuilders: List[PathBuilder], context: CallContext - ) extends PureStandardLibraryFunctionsLike with ReadLikeFunctions with WriteFunctions { + ) extends PureStandardLibraryFunctionsLike with ReadLikeFunctions with WriteFunctions with GlobFunctions { import SharedFileSystemExpressionFunctions._ - import better.files._ + + def callContext: CallContext = context override def writeTempFile(path: String, prefix: String, suffix: String, content: String): String = super[WriteFunctions].writeTempFile(path, prefix, suffix, content) - override def globPath(glob: String) = context.root.toString - override def glob(path: String, pattern: String): Seq[String] = { - File(context.root).glob(s"**/$pattern") map { _.pathAsString } toSeq - } override val writeDirectory = context.root diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala index 54453ab75..2c2954cf6 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala @@ -7,6 +7,7 @@ import cromwell.backend.callcaching.JobCachingActorHelper import cromwell.backend.io.{JobPathsWithDocker, WorkflowPathsBackendInitializationData} import cromwell.backend.validation.{RuntimeAttributesValidation, ValidatedRuntimeAttributes} import cromwell.core.logging.JobLogging +import cromwell.core.path.PathBuilder import net.ceedubs.ficus.Ficus._ trait SharedFileSystemJobCachingActorHelper extends JobCachingActorHelper { @@ -19,7 +20,7 @@ trait SharedFileSystemJobCachingActorHelper extends JobCachingActorHelper { lazy val jobPaths = new JobPathsWithDocker(jobDescriptor.key, jobDescriptor.workflowDescriptor, configurationDescriptor.backendConfig) - lazy val initializationData = BackendInitializationData. + lazy val initializationData: SharedFileSystemBackendInitializationData = BackendInitializationData. as[SharedFileSystemBackendInitializationData](backendInitializationDataOption) lazy val validatedRuntimeAttributes: ValidatedRuntimeAttributes = { @@ -27,7 +28,7 @@ trait SharedFileSystemJobCachingActorHelper extends JobCachingActorHelper { builder.build(jobDescriptor.runtimeAttributes, jobLogger) } - lazy val metadataKeyValues: Map[String, Any] = { + def startMetadataKeyValues: Map[String, Any] = { val runtimeAttributesMetadata = RuntimeAttributesValidation.extract(validatedRuntimeAttributes) map { case (key, value) => (s"runtimeAttributes:$key", value) } @@ -37,8 +38,10 @@ trait SharedFileSystemJobCachingActorHelper extends JobCachingActorHelper { } lazy val sharedFileSystem = new SharedFileSystem { - override val pathBuilders = WorkflowPathsBackendInitializationData.pathBuilders(backendInitializationDataOption) - override lazy val sharedFileSystemConfig = { + override val pathBuilders: List[PathBuilder] = { + WorkflowPathsBackendInitializationData.pathBuilders(backendInitializationDataOption) + } + override lazy val sharedFileSystemConfig: Config = { configurationDescriptor.backendConfig.as[Option[Config]]("filesystems.local").getOrElse(ConfigFactory.empty()) } } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActor.scala deleted file mode 100644 index beac805dd..000000000 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActor.scala +++ /dev/null @@ -1,85 +0,0 @@ -package cromwell.backend.sfs - -import akka.actor.{ActorRef, Props} -import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse} -import cromwell.backend.BackendLifecycleActor.AbortJobCommand -import cromwell.backend.async.AsyncBackendJobExecutionActor.{Execute, Recover} -import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobExecutionActor} -import cromwell.services.keyvalue.KeyValueServiceActor._ - -import scala.concurrent.{Future, Promise} - -/** - * Facade to the asynchronous execution actor. - * - * Creates the asynchronous execution actor, then relays messages to that actor. - * - * NOTE: Although some methods return futures due to the (current) contract in BJEA/ABJEA, this actor only executes - * during the receive, and does not launch new runnables/futures from inside "receive". - * - * Thus there are no vars, and the context switches during "receive", once the asynchronous actor has been created. - * - * @param jobDescriptor The job to execute. - * @param configurationDescriptor The configuration. - * @param asyncPropsCreator A function that can create the specific asynchronous backend. - */ -class SharedFileSystemJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, - override val configurationDescriptor: BackendConfigurationDescriptor, - serviceRegistryActor: ActorRef, - asyncPropsCreator: Promise[BackendJobExecutionResponse] => Props) - extends BackendJobExecutionActor { - - context.become(startup orElse super.receive) - - private def startup: Receive = { - case AbortJobCommand => - context.parent ! AbortedResponse(jobDescriptor.key) - context.stop(self) - } - - private def running(executor: ActorRef): Receive = { - case AbortJobCommand => - executor ! AbortJobCommand - case abortResponse: AbortedResponse => - context.parent ! abortResponse - context.stop(self) - case KvPair(key, id@Some(jobId)) if key.key == SharedFileSystemJob.JobIdKey => - // Successful operation ID lookup during recover. - executor ! Recover(SharedFileSystemJob(jobId)) - case KvKeyLookupFailed(_) => - // Missed operation ID lookup during recover, fall back to execute. - executor ! Execute - case KvFailure(_, e) => - // Failed operation ID lookup during recover, crash and let the supervisor deal with it. - completionPromise.tryFailure(e) - throw new RuntimeException(s"Failure attempting to look up job id for key ${jobDescriptor.key}", e) - } - - /** - * This "synchronous" actor isn't finished until this promise finishes over in the asynchronous version. - * - * Still not sure why the AsyncBackendJobExecutionActor doesn't wait for an Akka message instead of using Scala promises. - */ - private lazy val completionPromise = Promise[BackendJobExecutionResponse]() - - override def execute: Future[BackendJobExecutionResponse] = { - val executorRef = context.actorOf(asyncPropsCreator(completionPromise), "SharedFileSystemAsyncJobExecutionActor") - context.become(running(executorRef) orElse super.receive) - executorRef ! Execute - completionPromise.future - } - - override def recover: Future[BackendJobExecutionResponse] = { - val executorRef = context.actorOf(asyncPropsCreator(completionPromise), "SharedFileSystemAsyncJobExecutionActor") - context.become(running(executorRef) orElse super.receive) - val kvJobKey = - KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt) - val kvGet = KvGet(ScopedKey(jobDescriptor.workflowDescriptor.id, kvJobKey, SharedFileSystemJob.JobIdKey)) - serviceRegistryActor ! kvGet - completionPromise.future - } - - override def abort() = { - throw new NotImplementedError("Abort is implemented via a custom receive of the message AbortJobCommand.") - } -} diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala index cd5735766..c5053e416 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala @@ -14,11 +14,11 @@ import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, B import cromwell.core.Tags._ import cromwell.core._ import cromwell.services.keyvalue.KeyValueServiceActor.{KvJobKey, KvPair, ScopedKey} +import lenthall.exception.AggregatedException import org.scalatest.concurrent.PatienceConfiguration.Timeout import org.scalatest.prop.TableDrivenPropertyChecks -import org.scalatest.{FlatSpecLike, OptionValues} +import org.scalatest.{Assertion, FlatSpecLike, OptionValues} import wdl4s.types._ -import wdl4s.util.AggregatedException import wdl4s.values._ import scala.concurrent.duration._ @@ -28,9 +28,10 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst behavior of "SharedFileSystemJobExecutionActor" - lazy val runtimeAttributeDefinitions = SharedFileSystemValidatedRuntimeAttributesBuilder.default.definitions.toSet + lazy val runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition] = + SharedFileSystemValidatedRuntimeAttributesBuilder.default.definitions.toSet - def executeSpec(docker: Boolean) = { + def executeSpec(docker: Boolean): Any = { val expectedOutputs: CallOutputs = Map( "salutation" -> JobOutput(WdlString("Hello you !")) ) @@ -51,13 +52,14 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst } it should "send back an execution failure if the task fails" in { - val expectedResponse = JobFailedNonRetryableResponse(mock[BackendJobDescriptorKey], new Exception(""), Option(1)) + val expectedResponse = + JobFailedNonRetryableResponse(mock[BackendJobDescriptorKey], new RuntimeException(""), Option(1)) val workflow = TestWorkflow(buildWorkflowDescriptor(GoodbyeWorld), emptyBackendConfig, expectedResponse) val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflow.workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions), workflow.config) testWorkflow(workflow, backend) } - def localizationSpec(docker: Boolean) = { + def localizationSpec(docker: Boolean): Assertion = { def templateConf(localizers: String) = BackendConfigurationDescriptor( ConfigFactory.parseString( s"""{ @@ -150,7 +152,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst } } - def recoverSpec(completed: Boolean, writeReturnCode: Boolean = true) = { + def recoverSpec(completed: Boolean, writeReturnCode: Boolean = true): Assertion = { val workflowDescriptor = buildWorkflowDescriptor(HelloWorld) val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions) val backendRef = createBackendRef(jobDescriptor, emptyBackendConfig) @@ -259,7 +261,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst it should "fail post processing if an output file is not found" in { val expectedResponse = JobFailedNonRetryableResponse(mock[BackendJobDescriptorKey], - AggregatedException(Seq.empty, "Could not process output, file not found"), Option(0)) + AggregatedException("Could not process output, file not found:", Seq.empty), Option(0)) val workflow = TestWorkflow(buildWorkflowDescriptor(MissingOutputProcess), emptyBackendConfig, expectedResponse) val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflow.workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions), workflow.config) testWorkflow(workflow, backend) diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala index d7c39924c..228943633 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala @@ -4,8 +4,9 @@ import java.nio.file.Files import better.files._ import com.typesafe.config.{Config, ConfigFactory} -import cromwell.core.path.DefaultPathBuilder import cromwell.backend.BackendSpec +import cromwell.core.CromwellFatalExceptionMarker +import cromwell.core.path.DefaultPathBuilder import org.scalatest.prop.TableDrivenPropertyChecks import org.scalatest.{FlatSpec, Matchers} import org.specs2.mock.Mockito @@ -78,6 +79,20 @@ class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with Tabl it should "localize a file via symbolic link" in { localizationTest(softLinkLocalization, docker = false, symlink = true) } + + it should "throw a fatal exception if localization fails" in { + val callDir = File.newTemporaryDirectory("SharedFileSystem") + val orig = File("/made/up/origin") + + val inputs = fqnMapToDeclarationMap(Map("input" -> WdlFile(orig.pathAsString))) + val sharedFS = new SharedFileSystem { + override val pathBuilders = localPathBuilder + override val sharedFileSystemConfig = defaultLocalization + } + val result = sharedFS.localizeInputs(callDir.path, docker = false)(inputs) + result.isFailure shouldBe true + result.failed.get.isInstanceOf[CromwellFatalExceptionMarker] shouldBe true + } private[this] def countLinks(file: File): Int = Files.getAttribute(file.path, "unix:nlink").asInstanceOf[Int] diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala index 72d2c6e99..39fc21d6b 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala @@ -2,16 +2,14 @@ package cromwell.backend.sfs import akka.actor.{ActorSystem, Props} import akka.testkit.TestActorRef -import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse +import cromwell.backend.standard.{StandardAsyncExecutionActorParams, StandardSyncExecutionActor, DefaultStandardSyncExecutionActorParams} import cromwell.backend.io.WorkflowPathsWithDocker import cromwell.backend.validation.{DockerValidation, RuntimeAttributesValidation} import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor} -import scala.concurrent.Promise - -class TestLocalAsyncJobExecutionActor(override val params: SharedFileSystemAsyncJobExecutionActorParams) +class TestLocalAsyncJobExecutionActor(override val standardParams: StandardAsyncExecutionActorParams) extends BackgroundAsyncJobExecutionActor { - override lazy val processArgs = { + override lazy val processArgs: SharedFileSystemCommand = { val script = jobPaths.script.toString if (isDockerRun) { val docker = RuntimeAttributesValidation.extract(DockerValidation.instance, validatedRuntimeAttributes) @@ -27,24 +25,21 @@ class TestLocalAsyncJobExecutionActor(override val params: SharedFileSystemAsync object TestLocalAsyncJobExecutionActor { def createBackend(jobDescriptor: BackendJobDescriptor, configurationDescriptor: BackendConfigurationDescriptor) - (implicit system: ActorSystem): SharedFileSystemJobExecutionActor = { + (implicit system: ActorSystem): StandardSyncExecutionActor = { createBackendRef(jobDescriptor, configurationDescriptor).underlyingActor } def createBackendRef(jobDescriptor: BackendJobDescriptor, configurationDescriptor: BackendConfigurationDescriptor) - (implicit system: ActorSystem): TestActorRef[SharedFileSystemJobExecutionActor] = { + (implicit system: ActorSystem): TestActorRef[StandardSyncExecutionActor] = { val emptyActor = system.actorOf(Props.empty) val workflowPaths = new WorkflowPathsWithDocker(jobDescriptor.workflowDescriptor, configurationDescriptor.backendConfig) val initializationData = new SharedFileSystemBackendInitializationData(workflowPaths, SharedFileSystemValidatedRuntimeAttributesBuilder.default.withValidation(DockerValidation.optional)) + val asyncClass = classOf[TestLocalAsyncJobExecutionActor] - def propsCreator(completionPromise: Promise[BackendJobExecutionResponse]): Props = { - val params = SharedFileSystemAsyncJobExecutionActorParams(emptyActor, jobDescriptor, - configurationDescriptor, completionPromise, Option(initializationData)) - Props(classOf[TestLocalAsyncJobExecutionActor], params) - } + val params = DefaultStandardSyncExecutionActorParams(SharedFileSystemJob.JobIdKey, emptyActor, jobDescriptor, + configurationDescriptor, Option(initializationData), asyncClass) - TestActorRef(new SharedFileSystemJobExecutionActor( - jobDescriptor, configurationDescriptor, emptyActor, propsCreator)) + TestActorRef(new StandardSyncExecutionActor(params)) } } diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala index 927ac7d45..7deb08d15 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala @@ -11,8 +11,8 @@ import cromwell.backend.wdl.Command import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobExecutionActor} import cromwell.core.path.JavaWriterImplicits._ import cromwell.core.path.{DefaultPathBuilder, TailedWriter, UntailedWriter} +import lenthall.util.TryUtil import wdl4s.parser.MemoryUnit -import wdl4s.util.TryUtil import scala.concurrent.{Future, Promise} import scala.sys.process.ProcessLogger diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkRuntimeAttributes.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkRuntimeAttributes.scala index 778b85a86..8df2b34b4 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkRuntimeAttributes.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkRuntimeAttributes.scala @@ -8,7 +8,7 @@ import cromwell.backend.validation.RuntimeAttributesDefault._ import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.backend.validation.RuntimeAttributesValidation._ import cromwell.core._ -import cromwell.core.ErrorOr._ +import lenthall.validation.ErrorOr._ import lenthall.exception.MessageAggregation import wdl4s.types.{WdlBooleanType, WdlIntegerType, WdlStringType, WdlType} import wdl4s.values.{WdlBoolean, WdlInteger, WdlString, WdlValue} diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala index 6167f0fd7..fafec09e0 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala @@ -3,11 +3,11 @@ package cromwell.backend.impl.spark import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.backend.{BackendWorkflowDescriptor, MemorySize} import cromwell.core.{WorkflowId, WorkflowOptions} +import lenthall.util.TryUtil import org.scalatest.{Matchers, WordSpecLike} import spray.json.{JsBoolean, JsNumber, JsObject, JsValue} import wdl4s.WdlExpression._ import wdl4s.expression.NoFunctions -import wdl4s.util.TryUtil import wdl4s.values.WdlValue import wdl4s.{Call, _} @@ -100,7 +100,7 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers { val workflowDescriptor = buildWorkflowDescriptor(wdlSource, runtime = runtimeAttributes) def createLookup(call: Call): ScopedLookupFunction = { - val knownInputs = workflowDescriptor.inputs + val knownInputs = workflowDescriptor.knownValues call.lookupFunction(knownInputs, NoFunctions) }